From 3fed27b42fc0cbf4934179435e9572c2ec015877 Mon Sep 17 00:00:00 2001 From: Tobias Reisinger Date: Thu, 3 Oct 2024 03:46:55 +0200 Subject: [PATCH] Move services.json into ansible --- .gitignore | 1 + Makefile | 24 ++-- dns/services.json | 89 ------------- inventory/group_vars/all/all_services.yml | 120 ++++++++++++++++++ inventory/serguzim.net.yml | 35 ----- playbooks/create_services_for_dnscontrol.yml | 11 ++ .../filter_plugins/my_service_attributes.py | 17 --- playbooks/filter_plugins/service_filters.py | 38 ++++++ playbooks/serguzim.net.yml | 2 +- roles/backup/vars/main.yml | 2 +- roles/caddy/vars/main.yml | 2 +- 11 files changed, 188 insertions(+), 153 deletions(-) delete mode 100644 dns/services.json create mode 100644 playbooks/create_services_for_dnscontrol.yml delete mode 100644 playbooks/filter_plugins/my_service_attributes.py create mode 100644 playbooks/filter_plugins/service_filters.py diff --git a/.gitignore b/.gitignore index 65dbe42..9e7b38f 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ types-dnscontrol.d.ts dns/hosts.json +dns/services.json secrets.auto.tfvars .terraform diff --git a/Makefile b/Makefile index 58e10e2..0c635e7 100644 --- a/Makefile +++ b/Makefile @@ -3,27 +3,33 @@ SHELL := /bin/bash include .env export -DNS_OUTPUT = "dns/hosts.json" -SERVICES_OUTPUT = "inventory/group_vars/all/opentofu.yaml" +PWD := $(shell pwd) -$(DNS_OUTPUT): +.FORCE: + +./dns/hosts.json: .FORCE tofu output --json \ | jq 'with_entries(.value |= .value).hosts' \ - > $(DNS_OUTPUT) + > ./dns/hosts.json -$(SERVICES_OUTPUT): +./dns/services.json: ./inventory/group_vars/all/all_services.yml + ansible-playbook \ + -e services_json_file=$(PWD)/dns/services.json \ + playbooks/create_services_for_dnscontrol.yml + +./inventory/group_vars/all/opentofu.yaml: .FORCE tofu output --json \ | yq -y '{opentofu: with_entries(.value |= .value)}' \ - > $(SERVICES_OUTPUT) + > ./inventory/group_vars/all/opentofu.yaml -outputs: $(DNS_OUTPUT) $(SERVICES_OUTPUT) +outputs: ./dns/hosts.json ./dns/services.json ./inventory/group_vars/all/opentofu.yaml ./types-dnscontrol.d.ts: dnscontrol write-types -dns: $(DNS_OUTPUT) ./types-dnscontrol.d.ts +dns: ./types-dnscontrol.d.ts ./dns/hosts.json ./dns/services.json dnscontrol push -dns-check: $(DNS_OUTPUT) ./types-dnscontrol.d.ts +dns-check: ./types-dnscontrol.d.ts ./dns/hosts.json ./dns/services.json dnscontrol check-creds ovh diff --git a/dns/services.json b/dns/services.json deleted file mode 100644 index 9e0999c..0000000 --- a/dns/services.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "catch-all-serguzim.me": { - "target": "*", - "domain": "serguzim.me", - "host": "node002" - }, - "acme_dns": { - "target": "acme", - "domain": "serguzim.me", - "host": "node003" - }, - "authentik": { - "target": "auth", - "domain": "serguzim.me", - "host": "node002" - }, - "openfaas": { - "target": "faas", - "domain": "serguzim.me", - "host": "node002" - }, - "mailcow": { - "target": "mail", - "domain": "serguzim.me", - "host": "node003" - }, - "synapse": { - "target": "matrix", - "domain": "serguzim.me", - "host": "node002" - }, - "linkwarden": { - "target": "bookmarks", - "domain": "serguzim.me", - "host": "node003" - }, - "minio": { - "target": "s3", - "domain": "serguzim.me", - "host": "node002" - }, - "minio-console": { - "target": "console.s3", - "domain": "serguzim.me", - "alias": "minio" - }, - "umami": { - "target": "analytics", - "domain": "serguzim.me", - "host": "node003" - }, - "webpage-serguzim.me": { - "target": "@", - "domain": "serguzim.me", - "alias": "openfaas" - }, - "wiki_js": { - "target": "wiki", - "domain": "serguzim.me", - "host": "node003" - }, - - "synapse_msrg.cc": { - "target": "matrix", - "domain": "msrg.cc", - "alias": "synapse" - }, - "shlink": { - "target": "@", - "domain": "msrg.cc", - "host": "node002" - }, - "shlink-msvg.cc": { - "target": "@", - "domain": "msvg.cc", - "alias": "shlink" - }, - - "reitanlage_oranienburg": { - "target": "@", - "domain": "reitanlage-oranienburg.de", - "host": "node002" - }, - "reitanlage_oranienburg-www": { - "target": "www", - "domain": "reitanlage-oranienburg.de", - "alias": "reitanlage_oranienburg" - } -} diff --git a/inventory/group_vars/all/all_services.yml b/inventory/group_vars/all/all_services.yml index 84d3986..85a9ca6 100644 --- a/inventory/group_vars/all/all_services.yml +++ b/inventory/group_vars/all/all_services.yml @@ -1,58 +1,178 @@ all_services: - name: acme_dns + host: node003 + dns: + - domain: serguzim.me + target: acme - name: authentik + host: node002 + dns: + - domain: serguzim.me + target: auth - name: extra_services + host: node002 + dns: + - domain: serguzim.me + target: "*" - name: faas + host: node002 + dns: + - domain: serguzim.me + target: faas + - domain: serguzim.me + target: "@" + name: webpage-serguzim + alias: faas - name: forgejo + host: node002 + dns: + - domain: serguzim.me + target: forgejo volumes_backup: - forgejo_data - name: forgejo_runner + host: node002 - name: healthcheck + host: node002 - name: homebox + host: node002 + dns: + - domain: serguzim.me + target: inventory volumes_backup: - homebox_data - name: immich + host: node002 + dns: + - domain: serguzim.me + target: gallery volumes_backup: - immich_upload - name: influxdb + host: node002 + dns: + - domain: serguzim.me + target: tick volumes_backup: - influxdb_data - name: jellyfin + host: node002 + dns: + - domain: serguzim.me + target: media volumes_backup: - jellyfin_config #- jellyfin_media # TODO - name: linkwarden + host: node003 + dns: + - domain: serguzim.me + target: bookmarks - name: mailcow + host: node003 + dns: + - domain: serguzim.me + target: mail - name: minio + host: node002 + dns: + - domain: serguzim.me + target: s3 + - domain: serguzim.me + target: console.s3 + name: minio-console + alias: minio volumes_backup: - minio_data - name: ntfy + host: node002 + dns: + - domain: serguzim.me + target: push volumes_backup: - ntfy_data - name: reitanlage_oranienburg + host: node002 + dns: + - domain: reitanlage-oranienburg.de + target: "@" + - domain: reitanlage-oranienburg.de + target: www + name: reitanlage_oranienburg-www + alias: reitanlage_oranienburg volumes_backup: - reitanlage-oranienburg_data - name: shlink + host: node002 + dns: + - domain: msrg.cc + target: "@" + - domain: msvg.cc + target: "@" + name: shlink-msvg + alias: shlink - name: synapse + host: node002 + dns: + - domain: serguzim.me + target: matrix + - domain: msrg.cc + target: matrix + name: synapse_msrg + alias: synapse volumes_backup: - synapse_media_store ports: - 8448:8448 - name: tandoor + host: node002 + dns: + - domain: serguzim.me + target: recipes volumes_backup: - tandoor_mediafiles - name: teamspeak_fallback + host: node002 volumes_backup: - teamspeak-fallback-data - name: telegraf + host: node002 - name: tinytinyrss + host: node002 + dns: + - domain: serguzim.me + target: rss - name: umami + host: node003 + dns: + - domain: serguzim.me + target: analytics - name: uptime_kuma + host: node002 + dns: + - domain: serguzim.me + target: status volumes_backup: - uptime-kuma_data - name: vikunja + host: node002 + dns: + - domain: serguzim.me + target: todo volumes_backup: - vikunja_data - name: webhook + host: node002 + dns: + - domain: serguzim.me + target: hook - name: wiki_js + host: node003 + dns: + - domain: serguzim.me + target: wiki - name: woodpecker + host: node002 + dns: + - domain: serguzim.me + target: ci diff --git a/inventory/serguzim.net.yml b/inventory/serguzim.net.yml index 550a34f..563f930 100644 --- a/inventory/serguzim.net.yml +++ b/inventory/serguzim.net.yml @@ -19,8 +19,6 @@ all: host_backup: hc_uid: "{{ vault_node001.backup.hc_uid }}" uptime_kuma_token: "{{ vault_node001.backup.uptime_kuma_token }}" - volumes: - - minecraft-2_data node002: ansible_host: node002.vpn.serguzim.net @@ -33,32 +31,6 @@ all: host_backup: hc_uid: "{{ vault_node002.backup.hc_uid }}" uptime_kuma_token: "{{ vault_node002.backup.uptime_kuma_token }}" - volumes: - host_services: - - authentik - - extra_services - - faas - - forgejo - - forgejo_runner - - healthcheck - - homebox - - immich - - influxdb - - jellyfin - - minio - - ntfy - - reitanlage_oranienburg - - shlink - - synapse - - tandoor - - teamspeak_fallback - - telegraf - - tinytinyrss - - uptime_kuma - - vikunja - - watchtower - - webhook - - woodpecker node003: ansible_host: node003.vpn.serguzim.net @@ -71,10 +43,3 @@ all: host_backup: hc_uid: "{{ vault_node003.backup.hc_uid }}" uptime_kuma_token: "{{ vault_node003.backup.uptime_kuma_token }}" - volumes: [] - host_services: - - acme_dns - - linkwarden - - mailcow - - umami - - wiki_js diff --git a/playbooks/create_services_for_dnscontrol.yml b/playbooks/create_services_for_dnscontrol.yml new file mode 100644 index 0000000..e81198a --- /dev/null +++ b/playbooks/create_services_for_dnscontrol.yml @@ -0,0 +1,11 @@ +--- +- name: Create services for dnscontrol + hosts: localhost + tasks: + - name: Create the services json file + ansible.builtin.template: + src: "json.j2" + dest: "{{ services_json_file }}" + mode: "0644" + vars: + json: "{{ all_services | services_to_dnscontrol() }}" diff --git a/playbooks/filter_plugins/my_service_attributes.py b/playbooks/filter_plugins/my_service_attributes.py deleted file mode 100644 index 140bffe..0000000 --- a/playbooks/filter_plugins/my_service_attributes.py +++ /dev/null @@ -1,17 +0,0 @@ -class FilterModule(object): - def filters(self): - return { - 'my_service_attributes': self.my_service_attributes, - } - - def my_service_attributes(self, services, my_services, attribute="name"): - result = [] - for service in services: - if service["name"] in my_services: - if attribute in service: - if type(service[attribute]) == list: - result.extend(service[attribute]) - else: - result.append(service[attribute]) - - return result diff --git a/playbooks/filter_plugins/service_filters.py b/playbooks/filter_plugins/service_filters.py new file mode 100644 index 0000000..a8b3cab --- /dev/null +++ b/playbooks/filter_plugins/service_filters.py @@ -0,0 +1,38 @@ +class FilterModule(object): + def filters(self): + return { + 'my_service_attributes': self.my_service_attributes, + 'services_to_dnscontrol': self.services_to_dnscontrol, + } + + def my_service_attributes(self, services, host, attribute="name"): + result = [] + for service in services: + if service["host"] != host: + continue + if not attribute in service: + continue + + if type(service[attribute]) == list: + result.extend(service[attribute]) + else: + result.append(service[attribute]) + + return result + + def services_to_dnscontrol(self, services): + result = {} + for service in services: + for dns in service.get("dns", []): + name = dns.get("name", service["name"]) + result[name] = { + "target": dns["target"], + "domain": dns["domain"], + } + + if "alias" in dns: + result[name]["alias"] = dns["alias"] + else: + result[name]["host"] = service["host"] + + return result diff --git a/playbooks/serguzim.net.yml b/playbooks/serguzim.net.yml index 2b90f77..ef3e480 100644 --- a/playbooks/serguzim.net.yml +++ b/playbooks/serguzim.net.yml @@ -23,6 +23,6 @@ apply: tags: "{{ services_item }}" tags: always - loop: "{{ all_services | my_service_attributes(host_services) }}" + loop: "{{ all_services | my_service_attributes(inventory_hostname) }}" loop_control: loop_var: services_item diff --git a/roles/backup/vars/main.yml b/roles/backup/vars/main.yml index 99565f5..1a25cc3 100644 --- a/roles/backup/vars/main.yml +++ b/roles/backup/vars/main.yml @@ -5,7 +5,7 @@ backup_image: "{{ (container_registry.public, 'services/backup') | path_join }}" backup_svc: name: backup -backup_volumes_list: "{{ all_services | my_service_attributes(host_services, 'volumes_backup') }}" +backup_volumes_list: "{{ all_services | my_service_attributes(inventory_hostname, 'volumes_backup') }}" backup_volumes_service: "{{ backup_volumes_list | map_backup_volumes_service }}" backup_env: diff --git a/roles/caddy/vars/main.yml b/roles/caddy/vars/main.yml index 6450f09..7724dca 100644 --- a/roles/caddy/vars/main.yml +++ b/roles/caddy/vars/main.yml @@ -9,7 +9,7 @@ caddy_ports_default: - 443:443 - 443:443/udp - "{{ host_vpn.ip }}:2019:2019" -caddy_ports_extra: "{{ all_services | my_service_attributes(host_services, 'ports') }}" +caddy_ports_extra: "{{ all_services | my_service_attributes(inventory_hostname, 'ports') }}" caddy_ports: "{{ caddy_ports_default | union(caddy_ports_extra) }}" caddy_svc: