Move _ansible directory into main directory

.gitignore

@@ -1,47 +1 @@
-*.secret.env
-
-.lego/
-
-# services already handled by ansible
-/acme-dns/
-/authentik/
-/coder/
-/forgejo/
-/forgejo-runner/
-/gitea/
-/gitea-runner/
-/harbor/
-/healthcheck/
-/homebox/
-/influxdb/
-/jellyfin/
-/minecraft-2/
-/minio/
-/synapse/
-/tandoor/
-/telegraf/
-/tinytinyrss/
-/umami/
-/uptime-kuma/
-/watchtower/
-/webdis/
-/wiki-js/
-/woodpecker/
-
-/caddy/config/conf.002.d/acme.serguzim.me.conf
-/caddy/config/conf.002.d/auth.serguzim.me.conf
-/caddy/config/conf.002.d/analytics.serguzim.me.conf
-/caddy/config/conf.002.d/ci.serguzim.me.conf
-/caddy/config/conf.002.d/coder.serguzim.me.conf
-/caddy/config/conf.002.d/faas.serguzim.me.conf
-/caddy/config/conf.002.d/git.serguzim.me.conf
-/caddy/config/conf.002.d/inventory.serguzim.me.conf
-/caddy/config/conf.002.d/matrix.msrg.cc.conf
-/caddy/config/conf.002.d/media.serguzim.me.conf
-/caddy/config/conf.002.d/recipes.serguzim.me.conf
-/caddy/config/conf.002.d/registry.serguzim.me.conf
-/caddy/config/conf.002.d/rss.serguzim.me.conf
-/caddy/config/conf.002.d/status.serguzim.me.conf
-/caddy/config/conf.002.d/tick.serguzim.me.conf
-/caddy/config/conf.002.d/webdis.huck.serguzim.me.conf
-/caddy/config/conf.002.d/wiki.serguzim.me.conf
+inventory/group_vars/all/serguzim.net.yml

.pre-commit-config.yaml

@@ -4,4 +4,4 @@ repos:
     hooks:
       - id: ansible-lint
         args:
-          - _ansible/serguzim.net.yml
+          - serguzim.net.yml

_ansible/.gitignore

@@ -1,2 +0,0 @@
-.vault_pass
-inventory/group_vars/all/serguzim.net.yml

_ansible/inventory/host_vars/node002/main.yml

@@ -1,2 +0,0 @@
-ansible_port: "{{ vault_node002.ansible_port }}"
-ansible_user: "{{ vault_node002.ansible_user }}"

_utils/cert-basic.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env sh
 
-LEGO_WORKING_PATH="/opt/services/.lego/"
+export LEGO_WORKING_PATH="/opt/lego/"
 
 set -e
 

backup/.env

@@ -1,8 +0,0 @@
-HC_UID=
-UPTIME_KUMA_TOKEN=
-
-RESTIC_REPOSITORY=
-RESTIC_PASSWORD=
-
-AWS_ACCESS_KEY_ID=
-AWS_SECRET_ACCESS_KEY=

backup/.gitignore

@@ -1,2 +0,0 @@
-/mailcheck.mail
-/msmtprc

backup/backup.service

@@ -1,4 +0,0 @@
-[Service]
-Type=oneshot
-ExecStart=/opt/services/backup/backup.sh
-WorkingDirectory=/opt/services/backup/

backup/backup.sh

@@ -1,65 +0,0 @@
-#!/usr/bin/env sh
-
-set -e
-
-set -a
-. "/opt/services/backup/.env"
-. "/opt/services/backup/.secret.env"
-set +a
-
-hc_url="https://hc-ping.com/$HC_UID"
-
-curl -fsSL --retry 3 "$hc_url/start" >/dev/null
-duration_start=$(date +%s)
-
-BACKUP_LOCATION="/tmp/backup-misc"
-
-rm -rf "$BACKUP_LOCATION"
-mkdir -p "$BACKUP_LOCATION"
-cd "$BACKUP_LOCATION" || exit
-
-_hc_fail () {
-	curl -fsSL --retry 3 "$hc_url/fail"
-	exit 1
-}
-
-_backup_prepare_postgres () {
-	mkdir -p "$BACKUP_LOCATION/postgres"
-	cd "$BACKUP_LOCATION/postgres" || exit
-
-	postgres_tables=$(sudo -u postgres psql -Atc "SELECT datname FROM pg_database WHERE datistemplate = false;")
-
-	for i in $postgres_tables
-	do
-		echo "dumping $i"
-		sudo -u postgres pg_dump "$i" | gzip >"pg_dump_$i.gz"
-		echo "done with $i"
-		echo ""
-	done
-
-	echo "dumping all"
-	sudo -u postgres pg_dumpall | gzip >"pg_dumpall.gz"
-}
-
-_backup_prepare_mailcow () {
-	export MAILCOW_BACKUP_LOCATION="$BACKUP_LOCATION/mailcow"
-	mkdir -p "$MAILCOW_BACKUP_LOCATION"
-	/opt/mailcow-dockerized/helper-scripts/backup_and_restore.sh \
-		backup all --delete-days 3
-}
-
-echo ""
-echo "preparing postgres"
-time _backup_prepare_postgres >/tmp/backup-postgres.log || _hc_fail
-echo ""
-echo "preparing mailcow"
-time _backup_prepare_mailcow >/tmp/backup-mailcow.log || _hc_fail
-
-cd /opt/services/backup/
-docker compose run --rm backup || _hc_fail
-
-duration_end=$(date +%s)
-curl -fsSL --retry 3 "$hc_url"
-
-duration=$((duration_end - duration_start))
-curl -fsSL --retry 3 "https://status.serguzim.me/api/push/$UPTIME_KUMA_TOKEN?status=up&msg=OK&ping=${duration}000"

filter_plugins/map_backup_volumes.py

@@ -0,0 +1,24 @@
+class FilterModule(object):
+    def filters(self):
+        return {
+            'map_backup_volumes': self.map_backup_volumes,
+            'map_backup_volumes_service': self.map_backup_volumes_service
+        }
+
+    def map_backup_volumes(self, volumes):
+        result = {}
+
+        for volume in volumes:
+            result[volume] = {
+                "external": True,
+            }
+
+        return result
+
+    def map_backup_volumes_service(self, volumes):
+        result = []
+
+        for volume in volumes:
+            result.append("{volume_name}:/backup/volumes/{volume_name}".format(volume_name=volume))
+
+        return result

inventory/host_vars/node002/main.yml

@@ -0,0 +1,13 @@
+ansible_port: "{{ vault_node002.ansible_port }}"
+ansible_user: "{{ vault_node002.ansible_user }}"
+host_backup:
+  backup:
+    hc_uid: "{{ vault_node002.backup.hc_uid }}"
+    uptime_kuma_token: "{{ vault_node002.backup.uptime_kuma_token }}"
+    volumes:
+      - gitea_data
+      - influxdb_data
+      - reitanlage-oranienburg_data
+      - synapse_media_store
+      - tandoor_mediafiles
+

local-dev.yml

@@ -1,8 +1,9 @@
 ---
 - name: Run roles for local-dev
   vars:
-    # Remove _ansible and inventory
-    services_path: "{{ inventory_dir.split('/')[0:-2] | join('/') }}"
+    # Remove inventory
+    base_path: "{{ inventory_dir.split('/')[0:-1] | join('/') }}"
+    services_path: "{{ (base_path, '_services') | path_join }}"
     caddy_config_path: "{{ (services_path, 'caddy', 'config', 'conf.d') | path_join }}"
 
   hosts: local-dev

node002.yml

@@ -4,6 +4,8 @@
   roles:
     - role: common
       tags: [always]
+    - role: backup
+      tags: [backup]
     - role: caddy
       tags: [caddy, reverse-proxy, webserver]
       vars:

roles/backup/files/docker-compose.yml

@@ -7,8 +7,7 @@ services:
     image: backup
     restart: never
     env_file:
-      - .env
-      - .secret.env
+      - service.env
     volumes:
       - /tmp/backup-misc:/backup/misc
       - gitea_data:/backup/volumes/gitea_data

roles/backup/files/node001/mailcow.sh

@@ -0,0 +1,3 @@
+export MAILCOW_BACKUP_LOCATION="$BACKUP_LOCATION/mailcow"
+mkdir -p "$MAILCOW_BACKUP_LOCATION"
+/opt/mailcow-dockerized/helper-scripts/backup_and_restore.sh backup all

roles/backup/files/node002/postgres.sh

@@ -0,0 +1,14 @@
+mkdir -p "$BACKUP_LOCATION/postgres"
+cd "$BACKUP_LOCATION/postgres" || exit
+
+postgres_tables=$(sudo -u postgres psql -Atc "SELECT datname FROM pg_database WHERE datistemplate = false;")
+
+for i in $postgres_tables
+do
+	printf "dumping %s ..." "$i"
+	sudo -u postgres pg_dump "$i" | gzip >"pg_dump_$i.gz"
+	echo " done"
+done
+
+echo "dumping all"
+sudo -u postgres pg_dumpall | gzip >"pg_dumpall.gz"

roles/backup/tasks/main.yml

@@ -0,0 +1,84 @@
+---
+- name: Set common facts
+  ansible.builtin.import_tasks: tasks/set-default-facts.yml
+
+- name: Deploy {{ svc.name }}
+  vars:
+    svc: "{{ backup_svc }}"
+    env: "{{ backup_env }}"
+    compose: "{{ backup_compose }}"
+  block:
+    - name: Import prepare tasks for common service
+      ansible.builtin.import_tasks: tasks/prepare-common-service.yml
+
+    - name: Copy the Dockerfile
+      ansible.builtin.copy:
+        src: Dockerfile
+        dest: "{{ (service_path, 'Dockerfile') | path_join }}"
+        mode: "0644"
+      register: cmd_result
+
+    - name: Set the docker rebuild flag
+      ansible.builtin.set_fact:
+        docker_rebuild: true
+      when: cmd_result.changed # noqa: no-handler We need to handle the restart per service. Handlers don't support variables.
+
+    - name: Set backup.d path
+      ansible.builtin.set_fact:
+        backup_d_path: "{{ (service_path, 'backup.d') | path_join }}"
+
+    - name: Create backup.d directory
+      ansible.builtin.file:
+        path: "{{ backup_d_path }}"
+        state: directory
+        mode: "0755"
+
+    - name: Copy the additional backup scripts
+      ansible.builtin.copy:
+        src: "{{ item }}"
+        dest: "{{ backup_d_path }}"
+        mode: "0755"
+      with_fileglob:
+        - "{{ ansible_facts.hostname }}/*"
+
+    - name: Copy the main backup scripts
+      ansible.builtin.template:
+        src: "backup.sh.j2"
+        dest: "{{ (service_path, 'backup.sh') | path_join }}"
+        mode: "0755"
+
+    - name: Copy the system service
+      ansible.builtin.template:
+        src: backup.service.j2
+        dest: /etc/systemd/system/backup.service
+        mode: "0644"
+      become: true
+    - name: Copy the system timer
+      ansible.builtin.copy:
+        src: backup.timer
+        dest: /etc/systemd/system/backup.timer
+        mode: "0644"
+      become: true
+    - name: Enable the system timer
+      ansible.builtin.systemd_service:
+        name: backup.timer
+        state: started
+        enabled: true
+        daemon_reload: true
+      become: true
+
+    - name: Import tasks create a service.env file
+      ansible.builtin.import_tasks: tasks/steps/template-service-env.yml
+
+    - name: Build service
+      ansible.builtin.command:
+        cmd: docker compose build --pull
+        chdir: "{{ service_path }}"
+      register: cmd_result
+      changed_when: true # TODO
+
+    - name: Verify service
+      ansible.builtin.command:
+        cmd: docker compose run --rm app check
+        chdir: "{{ service_path }}"
+      changed_when: false

roles/backup/templates/backup.service.j2

@@ -0,0 +1,4 @@
+[Service]
+Type=oneshot
+ExecStart={{ service_path }}/backup.sh
+WorkingDirectory={{ service_path }}

roles/backup/templates/backup.sh.j2

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+set -e
+
+set -a
+. "{{ service_path }}/service.env"
+set +a
+
+hc_url="https://hc-ping.com/$HC_UID"
+_hc_ping () {
+	curl -fsSL --retry 3 "$hc_url$1" >/dev/null
+}
+
+_hc_fail () {
+	_hc_ping "/fail"
+	exit 1
+}
+
+_hc_ping "/start"
+duration_start=$(date +%s)
+
+BACKUP_LOCATION="$(mktemp -d)"
+export BACKUP_LOCATION
+cd "$BACKUP_LOCATION" || exit
+
+shopt -s nullglob
+for file in "{{ service_path }}/backup.d/"*
+do
+	echo ""
+	echo "running $file"
+	time "$file" || _hc_fail
+done || true
+
+cd "{{ service_path }}"
+docker compose run --rm -v "$BACKUP_LOCATION:/backup/misc" app backup /backup || _hc_fail
+
+duration_end=$(date +%s)
+_hc_ping
+
+duration=$((duration_end - duration_start))
+curl -fsSL --retry 3 "https://status.serguzim.me/api/push/$UPTIME_KUMA_TOKEN?status=up&msg=OK&ping=${duration}000"
+
+rm -rf "$BACKUP_LOCATION"
+
+echo "forgetting old backups for $(hostname)"
+docker compose run --rm app forget --host "$(hostname)" --prune \
+	--keep-daily 14 \
+	--keep-weekly 28 \
+	--keep-monthly 12 \
+	--keep-yearly 2 

roles/backup/vars/main.yml

@@ -0,0 +1,28 @@
+---
+backup_svc:
+  name: backup
+
+backup_volumes_service: "{{ host_backup.backup.volumes | map_backup_volumes_service }}"
+
+backup_env:
+  HC_UID: "{{ host_backup.backup.hc_uid }}"
+  UPTIME_KUMA_TOKEN: "{{ host_backup.backup.uptime_kuma_token }}"
+
+  RESTIC_REPOSITORY: "{{ vault_backup.restic.repository }}"
+  RESTIC_PASSWORD: "{{ vault_backup.restic.password }}"
+
+  AWS_ACCESS_KEY_ID: "{{ vault_backup.aws_secret.key_id }}"
+  AWS_SECRET_ACCESS_KEY: "{{ vault_backup.aws_secret.access_key }}"
+
+backup_compose:
+  watchtower: false
+  image: backup
+  volumes: "{{ backup_volumes_service }}"
+  file:
+    services:
+      app:
+        build:
+          context: .
+        restart: never
+        hostname: "{{ ansible_facts.hostname }}"
+    volumes: "{{ host_backup.backup.volumes | map_backup_volumes }}"

roles/harbor/vars/main.yml

@@ -36,8 +36,8 @@ harbor_yml:
     port: "{{ harbor_port_http }}"
   https:
     port: "{{ harbor_port_https }}"
-    certificate: /opt/services/.lego/certificates/registry.serguzim.me.crt
-    private_key: /opt/services/.lego/certificates/registry.serguzim.me.key
+    certificate: /opt/services/.lego/certificates/registry.serguzim.me.crt # TODO
+    private_key: /opt/services/.lego/certificates/registry.serguzim.me.key # TODO
   external_url: https://registry.serguzim.me
   harbor_admin_password: "{{ vault_harbor.admin_password }}"
   data_volume: "{{ (service_path, 'data') | path_join }}"

roles/healthcheck/files/data/http

@@ -14,7 +14,7 @@ check_url ()
     printf "checking url %s ." "$url"
     dig A "$1" >/dev/null
 	printf "."
-	result=$(curl -LsSfv "$url" 2>&1)
+	result=$(curl -LsSfv --retry 3 "$url" 2>&1)
 	code="$?"
 	printf ".\n"
 	#shellcheck disable=SC2181

roles/healthcheck/files/healthcheck@.service

@@ -1,4 +1,4 @@
 [Service]
 Type=oneshot
 ExecStart=/usr/bin/docker compose run --rm %i
-WorkingDirectory=/opt/services/healthcheck/
+WorkingDirectory={{ service_path }}

roles/healthcheck/tasks/main.yml

@@ -39,6 +39,14 @@
         dest: /etc/systemd/system/healthcheck@.timer
         mode: "0644"
       become: true
+    - name: Enable the system timer
+      ansible.builtin.systemd_service:
+        name: healthcheck@{{ item }}.timer
+        state: started
+        enabled: true
+        daemon_reload: true
+      loop: "{{ healthcheck_svc.checks }}"
+      become: true
 
     - name: Import tasks create a service.env file
       ansible.builtin.import_tasks: tasks/steps/template-service-env.yml

roles/healthcheck/vars/main.yml

@@ -1,6 +1,10 @@
 ---
 healthcheck_svc:
   name: healthcheck
+  checks:
+    - http
+    - mail
+    - matrix
 
 healthcheck_env:
   USER_AGENT: healthcheck-bot for serguzim.net