Remove special handling of common services and refactor getting service attributes

inventory/group_vars/all/main.yml

@@ -19,12 +19,6 @@ container_registry:
 
 services_path: /opt/services/
 
-common_services:
-  - backup
-  - lego
-  - caddy
-  - watchtower
-
 caddy_path: "{{ (services_path, 'caddy') | path_join }}"
 caddy_config_path: "{{ (caddy_path, 'config', 'conf.d') | path_join }}"
 

playbooks/filter_plugins/service_filters.py

@@ -1,29 +1,25 @@
 class FilterModule(object):
     def filters(self):
         return {
-            'my_service_attributes': self.my_service_attributes,
+            'services_for_host': self.services_for_host,
             'services_to_dnscontrol': self.services_to_dnscontrol,
             'services_get_backups': self.services_get_backups,
             'service_get_backups': self.service_get_backups,
             'service_get_domain': self.service_get_domain,
         }
 
-    def my_service_attributes(self, services, host, attribute="name"):
+    def services_for_host(self, services, host):
         result = []
         for service in services:
-            # only compare the host if it is set
+            if not host:
-            if host and service["host"] != host:
+                result.append(service)
                 continue
-
+            if service["host"] == host:
-            attribute_value = service.get(attribute)
+                result.append(service)
-            if not attribute_value:
+                continue
+            if service["host"] == "*":
+                result.append(service)
                 continue
-
-            if type(attribute_value) == list:
-                result.extend(attribute_value)
-            else:
-                result.append(attribute_value)
-
         return result
 
     def find_service(self, services, name):
@@ -34,10 +30,8 @@ class FilterModule(object):
 
     def services_get_backups(self, all_services, wanted_services):
         result = []
-        for service in all_services:
+        for wanted_service in wanted_services:
-            if service.get("name") in wanted_services:
+            result.extend(self.service_get_backups(all_services, wanted_service))
-                for backup in service.get("backup") or []:
-                    result.append(backup["name"])
         return result
 
     def service_get_backups(self, all_services, wanted_service):

playbooks/for-ansible-lint.yml

@@ -2,7 +2,7 @@
 - name: Run all roles
   hosts: serguzim_net
   vars:
-    host_services: "{{ all_services | my_service_attributes(inventory_hostname) | union(common_services) }}"
+    host_services: "{{ all_services | services_for_host(inventory_hostname) }}"
   roles:
     - acme_dns
     - always

playbooks/roles/backup/vars/main.yml

@@ -1,6 +1,6 @@
 ---
-backup_list: "{{ all_services | my_service_attributes(inventory_hostname, 'backup') }}"
+backup_list: "{{ host_services | map(attribute='backup') | flatten }}"
-backup_list_all: "{{ all_services | my_service_attributes('', 'backup') }}"
+backup_list_all: "{{ all_services | map(attribute='backup') | flatten }}"
 
 backup_msg_start: "Backup started"
 backup_msg_fail: "Backup failed"

playbooks/roles/caddy/vars/main.yml

@@ -5,11 +5,8 @@ caddy_acmedns_subd: "{{ vault_caddy.acmedns.subd }}"
 caddy_acmedns_url: "https://{{ acme_dns.host }}"
 
 caddy_ports_default:
-  - 80:80
-  - 443:443
-  - 443:443/udp
   - "{{ host_vpn.ip }}:2019:2019"
-caddy_ports_extra: "{{ all_services | my_service_attributes(inventory_hostname, 'ports') }}"
+caddy_ports_extra: "{{ host_services | map(attribute='ports') | flatten }}"
 caddy_ports: "{{ caddy_ports_default | union(caddy_ports_extra) }}"
 
 caddy_env:

playbooks/roles/lego/vars/main.yml

@@ -1,5 +1,5 @@
 ---
-lego_host_certificates: "{{ all_services | my_service_attributes(inventory_hostname, 'certificates') }}"
+lego_host_certificates: "{{ host_services | map(attribute='certificates') | flatten }}"
 
 lego_env:
   ACME_DNS_API_BASE: https://{{ acme_dns.host }}

playbooks/serguzim.net.yml

@@ -2,7 +2,7 @@
 - name: Run all roles
   hosts: serguzim_net
   vars:
-    host_services: "{{ all_services | my_service_attributes(inventory_hostname) | union(common_services) }}"
+    host_services: "{{ all_services | services_for_host(inventory_hostname) }}"
   tasks:
     - name: Install software
       ansible.builtin.include_role:
@@ -21,9 +21,9 @@
 
     - name: Include service roles
       ansible.builtin.include_role:
-        name: "{{ services_item }}"
+        name: "{{ services_item.name }}"
         apply:
-          tags: "{{ services_item }}"
+          tags: "{{ services_item.name }}"
       tags: always
       loop: "{{ host_services }}"
       loop_control:

playbooks/stop-and-backup-unused.yml

@@ -2,7 +2,7 @@
 - name: Stop and backup services
   hosts: serguzim_net
   vars:
-    host_services: "{{ all_services | my_service_attributes(inventory_hostname) | union(common_services) }}"
+    host_services: "{{ all_services | services_for_host(inventory_hostname) }}"
   tasks:
     - name: Get unused services
       ansible.builtin.include_tasks:

playbooks/tasks/get-unused.yml

@@ -14,4 +14,4 @@
 
 - name: Set unused services
   ansible.builtin.set_fact:
-    unused_services: "{{ docker_compose_projects_result.stdout_lines | difference(host_services) }}"
+    unused_services: "{{ docker_compose_projects_result.stdout_lines | difference(host_services | map(attribute='name')) }}"

playbooks/tasks/reload-caddy.yml

@@ -6,7 +6,7 @@
 - name: Map exisiting/wanted caddy site configs
   ansible.builtin.set_fact:
     caddy_site_configs_have: "{{ find_result.files | map(attribute='path') }}"
-    caddy_site_configs_want: "{{ all_services | my_service_attributes(inventory_hostname) | list_prefix_path_suffix(caddy_config_path, '.conf') }}"
+    caddy_site_configs_want: "{{ host_services | map(attribute='name') | list_prefix_path_suffix(caddy_config_path, '.conf') }}"
 
 - name: Remove unwanted caddy site configs
   ansible.builtin.file:

services.auto.tfvars

@@ -30,6 +30,28 @@ services = {
     s3 = false
   },
 
+  "backup" = {
+    name = "backup"
+    host = "*"
+    auth = false
+    database = false
+    s3 = false
+  },
+
+  "caddy" = {
+    name = "caddy"
+    host = "*"
+    ports = [
+      "80:80",
+      "443:443",
+      "443:443/udp",
+      #"2019:2019",
+    ]
+    auth = false
+    database = false
+    s3 = false
+  },
+
   "extra_services" = {
     name = "extra_services"
     host = "node001"
@@ -211,6 +233,14 @@ services = {
     s3 = false
   },
 
+  "lego" = {
+    name = "lego"
+    host = "*"
+    auth = false
+    database = false
+    s3 = false
+  },
+
   "linkwarden" = {
     name = "linkwarden"
     host = "node003"
@@ -497,6 +527,14 @@ services = {
     s3 = false
   },
 
+  "watchtower" = {
+    name = "watchtower"
+    host = "*"
+    auth = false
+    database = false
+    s3 = false
+  },
+
   "wiki_js" = {
     name = "wiki_js"
     host = "node001"

templates/infrastructure.d2.j2

@@ -18,6 +18,13 @@ external: {
 {% for host in hosts %}
 {{ host.key }}: {
 }
+
+{{ host.key }}.backup -> external.restic {
+    style: {
+        stroke: "#0f0"
+        stroke-dash: 3
+    }
+}
 {% endfor %}{# host #}
 
 {% for svc in svcs %}
@@ -29,7 +36,7 @@ external: {
 }
 
 {% for backup in svc.backup or [] %}
-{{ svc.key }} -> external.restic.{{ svc.host }}: {{ backup.name }} {
+{{ svc.key }} -> {{ svc.host_key }}.backup: {{ backup.name }} {
     style: {
         stroke: "#0f0"
         stroke-dash: 3

visualize.py

@@ -8,10 +8,12 @@ import hcl2
 
 icon_overrides = {
     "acme_dns": "lets-encrypt",
+    "backup": "restic",
     "extra_services": None,
     "faas": None,
     "forgejo_runner": "forgejo",
     "healthcheck": "healthchecks",
+    "lego": "lets-encrypt",
     "mailcowdockerized": "mailcow",
     "reitanlage_oranienburg": "grav",
     "tandoor": "tandoor-recipes",
@@ -22,9 +24,11 @@ icon_overrides = {
 }
 
 icon_format = {
+    "restic": "webp",
     "linkwarden": "webp",
     "telegraf": "webp",
     "tiny-tiny-rss": "webp",
+    "watchtower": "webp", # TODO revert when icon is fixed
 }
 
 def get_icon(svc):
@@ -52,6 +56,20 @@ def parse_hosts(hosts):
         })
     return result
 
+def parse_service(svc, data, hosts):
+    svc_key = service_key(svc, data, hosts)
+
+    domains = []
+    for dns in data.get("dns") or []:
+        domains.append(f"- {dns['domain']}")
+
+    data['key'] = svc_key
+    data['host_key'] = host_key(data["host"], hosts)
+    data['label'] = "\\n".join([svc] + domains)
+    data['icon'] = get_icon(svc)
+
+    return dict(data)
+
 def parse_services(services, hosts):
     result = []
 
@@ -59,17 +77,12 @@ def parse_services(services, hosts):
     authentik_key = service_key_find("authentik", services, hosts)
 
     for svc, data in services.items():
-        svc_key = service_key(svc, data, hosts)
+        if data["host"] == "*":
-
+            for host in hosts.keys():
-        domains = []
+                data["host"] = host
-        for dns in data.get("dns") or []:
+                result.append(parse_service(svc, data, hosts))
-            domains.append(f"- {dns['domain']}")
+        else:
-
+            result.append(parse_service(svc, data, hosts))
-        data['key'] = svc_key
-        data['label'] = "\\n".join([svc] + domains)
-        data['icon'] = get_icon(svc)
-
-        result.append(data)
 
     return result