From ff92241ddb63a4038ee2a617edfeb889dd102167 Mon Sep 17 00:00:00 2001 From: Tobias Reisinger Date: Mon, 21 Oct 2024 01:29:01 +0200 Subject: [PATCH] Remove special handling of common services and refactor getting service attributes --- inventory/group_vars/all/main.yml | 6 ---- playbooks/filter_plugins/service_filters.py | 28 ++++++--------- playbooks/for-ansible-lint.yml | 2 +- playbooks/roles/backup/vars/main.yml | 4 +-- playbooks/roles/caddy/vars/main.yml | 5 +-- playbooks/roles/lego/vars/main.yml | 2 +- playbooks/serguzim.net.yml | 6 ++-- playbooks/stop-and-backup-unused.yml | 2 +- playbooks/tasks/get-unused.yml | 2 +- playbooks/tasks/reload-caddy.yml | 2 +- services.auto.tfvars | 38 +++++++++++++++++++++ templates/infrastructure.d2.j2 | 9 ++++- visualize.py | 35 +++++++++++++------ 13 files changed, 92 insertions(+), 49 deletions(-) diff --git a/inventory/group_vars/all/main.yml b/inventory/group_vars/all/main.yml index 16c1369..3b11b69 100644 --- a/inventory/group_vars/all/main.yml +++ b/inventory/group_vars/all/main.yml @@ -19,12 +19,6 @@ container_registry: services_path: /opt/services/ -common_services: - - backup - - lego - - caddy - - watchtower - caddy_path: "{{ (services_path, 'caddy') | path_join }}" caddy_config_path: "{{ (caddy_path, 'config', 'conf.d') | path_join }}" diff --git a/playbooks/filter_plugins/service_filters.py b/playbooks/filter_plugins/service_filters.py index a9d4d6a..d3a0f38 100644 --- a/playbooks/filter_plugins/service_filters.py +++ b/playbooks/filter_plugins/service_filters.py @@ -1,29 +1,25 @@ class FilterModule(object): def filters(self): return { - 'my_service_attributes': self.my_service_attributes, + 'services_for_host': self.services_for_host, 'services_to_dnscontrol': self.services_to_dnscontrol, 'services_get_backups': self.services_get_backups, 'service_get_backups': self.service_get_backups, 'service_get_domain': self.service_get_domain, } - def my_service_attributes(self, services, host, attribute="name"): + def services_for_host(self, services, host): result = [] for service in services: - # only compare the host if it is set - if host and service["host"] != host: + if not host: + result.append(service) continue - - attribute_value = service.get(attribute) - if not attribute_value: + if service["host"] == host: + result.append(service) + continue + if service["host"] == "*": + result.append(service) continue - - if type(attribute_value) == list: - result.extend(attribute_value) - else: - result.append(attribute_value) - return result def find_service(self, services, name): @@ -34,10 +30,8 @@ class FilterModule(object): def services_get_backups(self, all_services, wanted_services): result = [] - for service in all_services: - if service.get("name") in wanted_services: - for backup in service.get("backup") or []: - result.append(backup["name"]) + for wanted_service in wanted_services: + result.extend(self.service_get_backups(all_services, wanted_service)) return result def service_get_backups(self, all_services, wanted_service): diff --git a/playbooks/for-ansible-lint.yml b/playbooks/for-ansible-lint.yml index c5043ee..ec44b2b 100644 --- a/playbooks/for-ansible-lint.yml +++ b/playbooks/for-ansible-lint.yml @@ -2,7 +2,7 @@ - name: Run all roles hosts: serguzim_net vars: - host_services: "{{ all_services | my_service_attributes(inventory_hostname) | union(common_services) }}" + host_services: "{{ all_services | services_for_host(inventory_hostname) }}" roles: - acme_dns - always diff --git a/playbooks/roles/backup/vars/main.yml b/playbooks/roles/backup/vars/main.yml index 026b825..89d1d9c 100644 --- a/playbooks/roles/backup/vars/main.yml +++ b/playbooks/roles/backup/vars/main.yml @@ -1,6 +1,6 @@ --- -backup_list: "{{ all_services | my_service_attributes(inventory_hostname, 'backup') }}" -backup_list_all: "{{ all_services | my_service_attributes('', 'backup') }}" +backup_list: "{{ host_services | map(attribute='backup') | flatten }}" +backup_list_all: "{{ all_services | map(attribute='backup') | flatten }}" backup_msg_start: "Backup started" backup_msg_fail: "Backup failed" diff --git a/playbooks/roles/caddy/vars/main.yml b/playbooks/roles/caddy/vars/main.yml index d537d1d..c75c710 100644 --- a/playbooks/roles/caddy/vars/main.yml +++ b/playbooks/roles/caddy/vars/main.yml @@ -5,11 +5,8 @@ caddy_acmedns_subd: "{{ vault_caddy.acmedns.subd }}" caddy_acmedns_url: "https://{{ acme_dns.host }}" caddy_ports_default: - - 80:80 - - 443:443 - - 443:443/udp - "{{ host_vpn.ip }}:2019:2019" -caddy_ports_extra: "{{ all_services | my_service_attributes(inventory_hostname, 'ports') }}" +caddy_ports_extra: "{{ host_services | map(attribute='ports') | flatten }}" caddy_ports: "{{ caddy_ports_default | union(caddy_ports_extra) }}" caddy_env: diff --git a/playbooks/roles/lego/vars/main.yml b/playbooks/roles/lego/vars/main.yml index ce128d5..d9884e5 100644 --- a/playbooks/roles/lego/vars/main.yml +++ b/playbooks/roles/lego/vars/main.yml @@ -1,5 +1,5 @@ --- -lego_host_certificates: "{{ all_services | my_service_attributes(inventory_hostname, 'certificates') }}" +lego_host_certificates: "{{ host_services | map(attribute='certificates') | flatten }}" lego_env: ACME_DNS_API_BASE: https://{{ acme_dns.host }} diff --git a/playbooks/serguzim.net.yml b/playbooks/serguzim.net.yml index 41b1ba5..df28901 100644 --- a/playbooks/serguzim.net.yml +++ b/playbooks/serguzim.net.yml @@ -2,7 +2,7 @@ - name: Run all roles hosts: serguzim_net vars: - host_services: "{{ all_services | my_service_attributes(inventory_hostname) | union(common_services) }}" + host_services: "{{ all_services | services_for_host(inventory_hostname) }}" tasks: - name: Install software ansible.builtin.include_role: @@ -21,9 +21,9 @@ - name: Include service roles ansible.builtin.include_role: - name: "{{ services_item }}" + name: "{{ services_item.name }}" apply: - tags: "{{ services_item }}" + tags: "{{ services_item.name }}" tags: always loop: "{{ host_services }}" loop_control: diff --git a/playbooks/stop-and-backup-unused.yml b/playbooks/stop-and-backup-unused.yml index c126e24..5b6d3d5 100644 --- a/playbooks/stop-and-backup-unused.yml +++ b/playbooks/stop-and-backup-unused.yml @@ -2,7 +2,7 @@ - name: Stop and backup services hosts: serguzim_net vars: - host_services: "{{ all_services | my_service_attributes(inventory_hostname) | union(common_services) }}" + host_services: "{{ all_services | services_for_host(inventory_hostname) }}" tasks: - name: Get unused services ansible.builtin.include_tasks: diff --git a/playbooks/tasks/get-unused.yml b/playbooks/tasks/get-unused.yml index 107fd34..598e222 100644 --- a/playbooks/tasks/get-unused.yml +++ b/playbooks/tasks/get-unused.yml @@ -14,4 +14,4 @@ - name: Set unused services ansible.builtin.set_fact: - unused_services: "{{ docker_compose_projects_result.stdout_lines | difference(host_services) }}" + unused_services: "{{ docker_compose_projects_result.stdout_lines | difference(host_services | map(attribute='name')) }}" diff --git a/playbooks/tasks/reload-caddy.yml b/playbooks/tasks/reload-caddy.yml index bf2ee32..b067ea1 100644 --- a/playbooks/tasks/reload-caddy.yml +++ b/playbooks/tasks/reload-caddy.yml @@ -6,7 +6,7 @@ - name: Map exisiting/wanted caddy site configs ansible.builtin.set_fact: caddy_site_configs_have: "{{ find_result.files | map(attribute='path') }}" - caddy_site_configs_want: "{{ all_services | my_service_attributes(inventory_hostname) | list_prefix_path_suffix(caddy_config_path, '.conf') }}" + caddy_site_configs_want: "{{ host_services | map(attribute='name') | list_prefix_path_suffix(caddy_config_path, '.conf') }}" - name: Remove unwanted caddy site configs ansible.builtin.file: diff --git a/services.auto.tfvars b/services.auto.tfvars index e22ddc7..11d0bf5 100644 --- a/services.auto.tfvars +++ b/services.auto.tfvars @@ -30,6 +30,28 @@ services = { s3 = false }, + "backup" = { + name = "backup" + host = "*" + auth = false + database = false + s3 = false + }, + + "caddy" = { + name = "caddy" + host = "*" + ports = [ + "80:80", + "443:443", + "443:443/udp", + #"2019:2019", + ] + auth = false + database = false + s3 = false + }, + "extra_services" = { name = "extra_services" host = "node001" @@ -211,6 +233,14 @@ services = { s3 = false }, + "lego" = { + name = "lego" + host = "*" + auth = false + database = false + s3 = false + }, + "linkwarden" = { name = "linkwarden" host = "node003" @@ -497,6 +527,14 @@ services = { s3 = false }, + "watchtower" = { + name = "watchtower" + host = "*" + auth = false + database = false + s3 = false + }, + "wiki_js" = { name = "wiki_js" host = "node001" diff --git a/templates/infrastructure.d2.j2 b/templates/infrastructure.d2.j2 index f898a7c..b78bd65 100644 --- a/templates/infrastructure.d2.j2 +++ b/templates/infrastructure.d2.j2 @@ -18,6 +18,13 @@ external: { {% for host in hosts %} {{ host.key }}: { } + +{{ host.key }}.backup -> external.restic { + style: { + stroke: "#0f0" + stroke-dash: 3 + } +} {% endfor %}{# host #} {% for svc in svcs %} @@ -29,7 +36,7 @@ external: { } {% for backup in svc.backup or [] %} -{{ svc.key }} -> external.restic.{{ svc.host }}: {{ backup.name }} { +{{ svc.key }} -> {{ svc.host_key }}.backup: {{ backup.name }} { style: { stroke: "#0f0" stroke-dash: 3 diff --git a/visualize.py b/visualize.py index 9b907b5..b187341 100755 --- a/visualize.py +++ b/visualize.py @@ -8,10 +8,12 @@ import hcl2 icon_overrides = { "acme_dns": "lets-encrypt", + "backup": "restic", "extra_services": None, "faas": None, "forgejo_runner": "forgejo", "healthcheck": "healthchecks", + "lego": "lets-encrypt", "mailcowdockerized": "mailcow", "reitanlage_oranienburg": "grav", "tandoor": "tandoor-recipes", @@ -22,9 +24,11 @@ icon_overrides = { } icon_format = { + "restic": "webp", "linkwarden": "webp", "telegraf": "webp", "tiny-tiny-rss": "webp", + "watchtower": "webp", # TODO revert when icon is fixed } def get_icon(svc): @@ -52,6 +56,20 @@ def parse_hosts(hosts): }) return result +def parse_service(svc, data, hosts): + svc_key = service_key(svc, data, hosts) + + domains = [] + for dns in data.get("dns") or []: + domains.append(f"- {dns['domain']}") + + data['key'] = svc_key + data['host_key'] = host_key(data["host"], hosts) + data['label'] = "\\n".join([svc] + domains) + data['icon'] = get_icon(svc) + + return dict(data) + def parse_services(services, hosts): result = [] @@ -59,17 +77,12 @@ def parse_services(services, hosts): authentik_key = service_key_find("authentik", services, hosts) for svc, data in services.items(): - svc_key = service_key(svc, data, hosts) - - domains = [] - for dns in data.get("dns") or []: - domains.append(f"- {dns['domain']}") - - data['key'] = svc_key - data['label'] = "\\n".join([svc] + domains) - data['icon'] = get_icon(svc) - - result.append(data) + if data["host"] == "*": + for host in hosts.keys(): + data["host"] = host + result.append(parse_service(svc, data, hosts)) + else: + result.append(parse_service(svc, data, hosts)) return result