From 2ad3cce749ecc6e3868b8d3f2d90524e37dc21e9 Mon Sep 17 00:00:00 2001 From: Tobias Reisinger Date: Sun, 6 Oct 2024 19:08:13 +0200 Subject: [PATCH] Move all_services into opentofu --- .gitignore | 3 +- Makefile | 18 +- inventory/group_vars/all/all_services.yml | 232 ------------- modules/infrastructure/variables.tf | 13 +- modules/services/authentik.tf | 2 +- modules/services/variables.tf | 13 +- output.tf | 5 + playbooks/filter_plugins/service_filters.py | 16 +- services.auto.tfvars | 367 +++++++++++++++++++- variables.tf | 13 +- 10 files changed, 415 insertions(+), 267 deletions(-) delete mode 100644 inventory/group_vars/all/all_services.yml diff --git a/.gitignore b/.gitignore index 9e7b38f..2e3cc57 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ secrets.auto.tfvars .terraform inventory/group_vars/all/serguzim.net.yml -inventory/group_vars/all/opentofu.yaml +inventory/group_vars/all/opentofu.yml +inventory/group_vars/all/all_services.yml diff --git a/Makefile b/Makefile index 6491fa4..3c534c9 100644 --- a/Makefile +++ b/Makefile @@ -7,6 +7,16 @@ PWD := $(shell pwd) .FORCE: +./inventory/group_vars/all/all_services.yml: .FORCE + tofu output --json \ + | yq -y '{all_services: with_entries(.value |= .value).services | to_entries | map(.value)}' \ + > ./inventory/group_vars/all/all_services.yml + +./inventory/group_vars/all/opentofu.yml: .FORCE + tofu output --json \ + | yq -y '{opentofu: with_entries(.value |= .value)}' \ + > ./inventory/group_vars/all/opentofu.yml + ./dns/hosts.json: .FORCE tofu output --json \ | jq 'with_entries(.value |= .value).hosts' \ @@ -17,12 +27,8 @@ PWD := $(shell pwd) -e services_json_file=$(PWD)/dns/services.json \ playbooks/create_services_for_dnscontrol.yml -./inventory/group_vars/all/opentofu.yaml: .FORCE - tofu output --json \ - | yq -y '{opentofu: with_entries(.value |= .value)}' \ - > ./inventory/group_vars/all/opentofu.yaml -output: ./dns/hosts.json ./dns/services.json ./inventory/group_vars/all/opentofu.yaml +output: ./dns/hosts.json ./dns/services.json ./inventory/group_vars/all/opentofu.yml ./types-dnscontrol.d.ts: @@ -30,7 +36,7 @@ output: ./dns/hosts.json ./dns/services.json ./inventory/group_vars/all/opentofu tofu: tofu apply - echo "\n=====\n" + @printf "\n=====\n\n" $(MAKE) output dns: ./types-dnscontrol.d.ts ./dns/hosts.json ./dns/services.json diff --git a/inventory/group_vars/all/all_services.yml b/inventory/group_vars/all/all_services.yml deleted file mode 100644 index a1314b2..0000000 --- a/inventory/group_vars/all/all_services.yml +++ /dev/null @@ -1,232 +0,0 @@ -all_services: - - name: acme_dns - host: node003 - dns: - - domain: serguzim.me - target: acme - - - name: authentik - host: node002 - dns: - - domain: serguzim.me - target: auth - - - name: extra_services - host: node002 - dns: - - domain: serguzim.me - target: "*" - - - name: faas - host: node002 - dns: - - domain: serguzim.me - target: faas - - domain: serguzim.me - target: "@" - name: webpage-serguzim - alias: faas - - - name: forgejo - host: node002 - dns: - - domain: serguzim.me - target: forgejo - backup: - - name: forgejo_data - type: docker - - - name: forgejo_runner - host: node002 - - - name: healthcheck - host: node002 - - - name: homebox - host: node002 - dns: - - domain: serguzim.me - target: inventory - backup: - - name: homebox_data - type: docker - - - name: immich - host: node002 - dns: - - domain: serguzim.me - target: gallery - backup: - - name: immich_upload - type: docker - - name: immich_database - type: hook - - - name: influxdb - host: node002 - dns: - - domain: serguzim.me - target: tick - backup: - - name: influxdb_data - type: docker - - - name: jellyfin - host: node002 - dns: - - domain: serguzim.me - target: media - backup: - - name: jellyfin_config - type: docker - #- jellyfin_media # TODO - - - name: linkwarden - host: node003 - dns: - - domain: serguzim.me - target: bookmarks - - - name: mailcow - host: node003 - dns: - - domain: serguzim.me - target: mail - backup: - - name: mailcow - type: hook - - - name: minio - host: node002 - dns: - - domain: serguzim.me - target: s3 - - domain: serguzim.me - target: console.s3 - name: minio-console - alias: minio - backup: - - name: minio_data - type: docker - - - name: ntfy - host: node002 - dns: - - domain: serguzim.me - target: push - backup: - - name: ntfy_data - type: docker - - - name: postgresql - host: node002 - backup: - - name: postgresql - type: hook - - - name: reitanlage_oranienburg - host: node002 - dns: - - domain: reitanlage-oranienburg.de - target: "@" - - domain: reitanlage-oranienburg.de - target: www - name: reitanlage_oranienburg-www - alias: reitanlage_oranienburg - backup: - - name: reitanlage-oranienburg_data - type: docker - - - name: shlink - host: node002 - dns: - - domain: msrg.cc - target: "@" - - domain: msvg.cc - target: "@" - name: shlink-msvg - alias: shlink - - - name: synapse - host: node002 - dns: - - domain: serguzim.me - target: matrix - - domain: msrg.cc - target: matrix - name: synapse_msrg - alias: synapse - backup: - - name: synapse_media_store - type: docker - ports: - - 8448:8448 - - - name: tandoor - host: node002 - dns: - - domain: serguzim.me - target: recipes - backup: - - name: tandoor_mediafiles - type: docker - - - name: teamspeak_fallback - host: node002 - dns: - - domain: serguzim.me - target: ts - backup: - - name: teamspeak-fallback-data - type: docker - - - name: telegraf - host: node002 - - - name: tinytinyrss - host: node002 - dns: - - domain: serguzim.me - target: rss - - - name: umami - host: node003 - dns: - - domain: serguzim.me - target: analytics - - - name: uptime_kuma - host: node002 - dns: - - domain: serguzim.me - target: status - backup: - - name: uptime-kuma_data - type: docker - - - name: vikunja - host: node002 - dns: - - domain: serguzim.me - target: todo - backup: - - name: vikunja_data - type: docker - - - name: webhook - host: node002 - dns: - - domain: serguzim.me - target: hook - - - name: wiki_js - host: node003 - dns: - - domain: serguzim.me - target: wiki - - - name: woodpecker - host: node002 - dns: - - domain: serguzim.me - target: ci diff --git a/modules/infrastructure/variables.tf b/modules/infrastructure/variables.tf index 9cc6dbc..9252827 100644 --- a/modules/infrastructure/variables.tf +++ b/modules/infrastructure/variables.tf @@ -75,7 +75,18 @@ variable "default_ssh_key" { variable "services" { type = map(object({ name = string - subdomain = string + host = string + dns = optional(list(object({ + domain = string + target = string + name = optional(string) + alias = optional(string) + }))) + backup = optional(list(object({ + name = string + type = string + }))) + ports = optional(list(string)) auth = bool auth_redirects = optional(list(string)) s3 = bool diff --git a/modules/services/authentik.tf b/modules/services/authentik.tf index d98cb7d..08058ab 100644 --- a/modules/services/authentik.tf +++ b/modules/services/authentik.tf @@ -28,6 +28,6 @@ resource "authentik_provider_oauth2" "service_providers" { resource "authentik_application" "service_applications" { for_each = local.services_auth name = each.value.name - slug = "${each.value.subdomain}-serguzim-me" + slug = "${each.value.dns[0].target}-serguzim-me" protocol_provider = authentik_provider_oauth2.service_providers[each.key].id } diff --git a/modules/services/variables.tf b/modules/services/variables.tf index eaa8afc..faa5db9 100644 --- a/modules/services/variables.tf +++ b/modules/services/variables.tf @@ -24,7 +24,18 @@ variable "postgresql_password" { variable "services" { type = map(object({ name = string - subdomain = string + host = string + dns = optional(list(object({ + domain = string + target = string + name = optional(string) + alias = optional(string) + }))) + backup = optional(list(object({ + name = string + type = string + }))) + ports = optional(list(string)) auth = bool auth_redirects = optional(list(string)) s3 = bool diff --git a/output.tf b/output.tf index a99a181..cbd6da4 100644 --- a/output.tf +++ b/output.tf @@ -2,6 +2,11 @@ output "hosts" { value = module.infrastructure.hosts } +output "services" { + value = var.services + sensitive = true # Would be too much output +} + output "authentik_data" { value = module.services.authentik_data sensitive = true diff --git a/playbooks/filter_plugins/service_filters.py b/playbooks/filter_plugins/service_filters.py index a8b3cab..8c356c0 100644 --- a/playbooks/filter_plugins/service_filters.py +++ b/playbooks/filter_plugins/service_filters.py @@ -10,27 +10,29 @@ class FilterModule(object): for service in services: if service["host"] != host: continue - if not attribute in service: + + attribute_value = service.get(attribute) + if not attribute_value: continue - if type(service[attribute]) == list: - result.extend(service[attribute]) + if type(attribute_value) == list: + result.extend(attribute_value) else: - result.append(service[attribute]) + result.append(attribute_value) return result def services_to_dnscontrol(self, services): result = {} for service in services: - for dns in service.get("dns", []): - name = dns.get("name", service["name"]) + for dns in service.get("dns") or []: + name = dns.get("name") or service["name"] result[name] = { "target": dns["target"], "domain": dns["domain"], } - if "alias" in dns: + if dns.get("alias"): result[name]["alias"] = dns["alias"] else: result[name]["host"] = service["host"] diff --git a/services.auto.tfvars b/services.auto.tfvars index 4bdd510..bf58dcb 100644 --- a/services.auto.tfvars +++ b/services.auto.tfvars @@ -1,113 +1,446 @@ services = { "acme_dns" = { name = "acme_dns" - subdomain = "acme" + host = "node003" + dns = [{ + domain = "serguzim.me" + target = "acme" + }] auth = false database = true s3 = false }, + "authentik" = { name = "authentik" - subdomain = "auth" + host = "node002" + dns = [{ + domain = "serguzim.me" + target = "auth" + }] auth = false database = true s3 = false }, + + "extra_services" = { + name = "extra_services" + host = "node002" + dns = [{ + domain = "serguzim.me" + target = "*" + }] + auth = false + database = false + s3 = false + }, + + "faas" = { + name = "faas" + host = "node002" + dns = [ + { + domain = "serguzim.me" + target = "faas" + }, + { + domain = "serguzim.me" + target = "@" + name = "webpage-serguzim" + alias = "faas" + } + ] + auth = false + database = false + s3 = false + }, + "forgejo" = { name = "forgejo" - subdomain = "git" + host = "node002" + dns = [{ + domain = "serguzim.me" + target = "git" + }] + backup = [{ + name = "forgejo_data" + type = "docker" + }] auth = true auth_redirects = ["https://git.serguzim.me/user/oauth2/auth.serguzim.me/callback"] database = true s3 = true }, + + "forgejo_runner" = { + name = "forgejo_runner" + host = "node002" + auth = false + database = false + s3 = false + }, + + "healthcheck" = { + name = "healthcheck" + host = "node002" + auth = false + database = false + s3 = false + }, + + "homebox" = { + name = "homebox" + host = "node002" + dns = [{ + domain = "serguzim.me" + target = "inventory" + }] + backup = [{ + name = "homebox_data" + type = "docker" + }] + auth = false + database = false + s3 = false + }, + "immich" = { name = "immich" - subdomain = "gallery" + host = "node002" + dns = [{ + domain = "serguzim.me" + target = "gallery" + }] + backup = [ + { + name = "immich_upload" + type = "docker" + }, + { + name = "immich_database" + type = "hook" + } + ] auth = true auth_redirects = ["https://gallery.serguzim.me/auth/login"] database = false s3 = false }, + + "influxdb" = { + name = "influxdb" + host = "node002" + dns = [{ + domain = "serguzim.me" + target = "tick" + }] + backup = [{ + name = "influxdb_data" + type = "docker" + }] + auth = false + database = false + s3 = false + }, + + "jellyfin" = { + name = "jellyfin" + host = "node002" + dns = [{ + domain = "serguzim.me" + target = "media" + }] + backup = [{ + name = "jellyfin_config" + type = "docker" + }] # TODO add jellyfin_media + auth = false + database = false + s3 = false + }, + "linkwarden" = { name = "linkwarden" - subdomain = "bookmarks" + host = "node003" + dns = [{ + domain = "serguzim.me" + target = "bookmarks" + }] auth = true auth_redirects = ["https://bookmarks.serguzim.me/api/v1/auth/callback/authentik"] database = true s3 = true }, + + mailcow = { + name = "mailcow" + host = "node003" + dns = [{ + domain = "serguzim.me" + target = "mail" + }] + backup = [{ + name = "mailcow" + type = "hook" + }] + auth = false + database = false + s3 = false + }, + + "minio" = { + name = "minio" + host = "node002" + dns = [ + { + domain = "serguzim.me" + target = "s3" + }, + { + domain = "serguzim.me" + target = "console.s3" + name = "minio-console" + alias = "minio" + } + ] + backup = [{ + name = "minio_data" + type = "docker" + }] + auth = false + database = false + s3 = false + }, + + "ntfy" = { + name = "ntfy" + host = "node002" + dns = [{ + domain = "serguzim.me" + target = "push" + }] + backup = [{ + name = "ntfy_data" + type = "docker" + }] + auth = false + database = false + s3 = false + }, + + "postgresql" = { + name = "postgresql" + host = "node002" + backup = [{ + name = "postgresql" + type = "hook" + }] + auth = false + database = false + s3 = false + }, + + "reitanlage_oranienburg" = { + name = "reitanlage_oranienburg" + host = "node002" + dns = [ + { + domain = "reitanlage-oranienburg.de" + target = "@" + }, + { + domain = "reitanlage-oranienburg.de" + target = "www" + name = "reitanlage_oranienburg-www" + alias = "reitanlage_oranienburg" + } + ] + backup = [{ + name = "reitanlage-oranienburg_data" + type = "docker" + }] + auth = false + database = false + s3 = false + }, + "shlink" = { name = "shlink" - subdomain = "redirect" # not really. It's msrg.cc + host = "node002" + dns = [ + { + domain = "msrg.cc" + target = "@" + }, + { + domain = "msvg.cc" + target = "@" + name = "shlink-msvg" + alias = "shlink" + } + ] auth = false database = true s3 = false }, + "synapse" = { name = "synapse" - subdomain = "matrix" + host = "node002" + dns = [ + { + domain = "serguzim.me" + target = "matrix" + }, + { + domain = "msrg.cc" + target = "matrix" + name = "synapse-msrg" + alias = "synapse" + } + ] + backup = [{ + name = "synapse_media_store" + type = "docker" + }] + ports = ["8448:8448"] auth = true auth_redirects = ["https://matrix.serguzim.me/_synapse/client/oidc/callback"] database = true s3 = false }, + "tandoor" = { name = "tandoor" - subdomain = "recipes" + host = "node002" + dns = [{ + domain = "serguzim.me" + target = "recipes" + }] + backup = [{ + name = "tandoor_mediafiles" + type = "docker" + }] auth = false database = true s3 = false }, + + "teamspeak_fallback" = { + name = "teamspeak_fallback" + host = "node002" + dns = [{ + domain = "serguzim.me" + target = "ts" + }] + backup = [{ + name = "teamspeak-fallback-data" + type = "docker" + }] + auth = false + database = false + s3 = false + } + "telegraf" = { name = "telegraf" - subdomain = "telegraf" # actually no subdomain + host = "node002" auth = false database = true s3 = false }, + "tinytinyrss" = { name = "tinytinyrss" - subdomain = "rss" + host = "node002" + dns = [{ + domain = "serguzim.me" + target = "rss" + }] auth = false database = true s3 = false }, + "umami" = { name = "umami" - subdomain = "analytics" + host = "node003" + dns = [{ + domain = "serguzim.me" + target = "analytics" + }] auth = false database = true s3 = false }, + "uptime_kuma" = { name = "uptime_kuma" - subdomain = "status" + host = "node002" + dns = [{ + domain = "serguzim.me" + target = "status" + }] + backup = [{ + name = "uptime-kuma_data" + type = "docker" + }] auth = false database = true s3 = false }, + "vikunja" = { name = "vikunja" - subdomain = "todo" + host = "node002" + dns = [{ + domain = "serguzim.me" + target = "todo" + }] + backup = [{ + name = "vikunja_data" + type = "docker" + }] auth = true auth_redirects = ["https://todo.serguzim.me/auth/openid/authserguzimme"] database = true s3 = false - } + }, + + "webhook" = { + name = "webhook" + host = "node002" + dns = [{ + domain = "serguzim.me" + target = "hook" + }] + auth = false + database = false + s3 = false + }, + "wiki_js" = { name = "wiki_js" - subdomain = "wiki" + host = "node003" + dns = [{ + domain = "serguzim.me" + target = "wiki" + }] auth = true auth_redirects = ["https://wiki.serguzim.me/login/f792bc7d-1a25-4437-944e-55eaf0111102/callback"] database = true s3 = false }, + "woodpecker" = { name = "woodpecker" - subdomain = "ci" + host = "node002" + dns = [{ + domain = "serguzim.me" + target = "ci" + }] auth = false database = true s3 = false - }, + } } diff --git a/variables.tf b/variables.tf index 6e87e35..f40bb29 100644 --- a/variables.tf +++ b/variables.tf @@ -126,7 +126,18 @@ variable "default_ssh_key" { variable "services" { type = map(object({ name = string - subdomain = string + host = string + dns = optional(list(object({ + domain = string + target = string + name = optional(string) + alias = optional(string) + }))) + backup = optional(list(object({ + name = string + type = string + }))) + ports = optional(list(string)) auth = bool auth_redirects = optional(list(string)) s3 = bool