From ed51a869355c4523fbf9841603171442a81b98f6 Mon Sep 17 00:00:00 2001
From: Tobias Reisinger <tobias@msrg.cc>
Date: Sun, 6 Oct 2024 01:59:46 +0200
Subject: [PATCH] Replace backup script with autorestic
---
inventory/group_vars/all/all_services.yml | 76 +++++++++++------
.../filter_plugins/map_backup_locations.py | 38 +++++++++
.../filter_plugins/map_backup_volumes.py | 24 ------
roles/backup/files/Dockerfile | 3 -
.../immich.sh => hooks/immich_database} | 5 +-
roles/backup/files/hooks/mailcow | 5 ++
.../{node002/postgres.sh => hooks/postgresql} | 5 +-
roles/backup/files/node001/mailcow.sh | 3 -
roles/backup/files/node003/mailcow.sh | 3 -
roles/backup/tasks/backup.d.yml | 16 ----
roles/backup/tasks/docker.yml | 12 ---
roles/backup/tasks/hooks.yml | 23 ++++++
roles/backup/tasks/main.yml | 31 +++----
roles/backup/templates/backup.service.j2 | 4 +-
roles/backup/templates/backup.sh.j2 | 74 ++---------------
roles/backup/vars/main.yml | 82 ++++++++-----------
roles/postgresql/.gitkeep | 0
17 files changed, 180 insertions(+), 224 deletions(-)
create mode 100644 playbooks/filter_plugins/map_backup_locations.py
delete mode 100644 playbooks/filter_plugins/map_backup_volumes.py
delete mode 100644 roles/backup/files/Dockerfile
rename roles/backup/files/{node002/immich.sh => hooks/immich_database} (70%)
create mode 100755 roles/backup/files/hooks/mailcow
rename roles/backup/files/{node002/postgres.sh => hooks/postgresql} (81%)
delete mode 100755 roles/backup/files/node001/mailcow.sh
delete mode 100755 roles/backup/files/node003/mailcow.sh
delete mode 100644 roles/backup/tasks/backup.d.yml
delete mode 100644 roles/backup/tasks/docker.yml
create mode 100644 roles/backup/tasks/hooks.yml
mode change 100755 => 100644 roles/backup/templates/backup.sh.j2
create mode 100644 roles/postgresql/.gitkeep
diff --git a/inventory/group_vars/all/all_services.yml b/inventory/group_vars/all/all_services.yml
index 8b7ca15..a1314b2 100644
--- a/inventory/group_vars/all/all_services.yml
+++ b/inventory/group_vars/all/all_services.yml
@@ -32,8 +32,9 @@ all_services:
dns:
- domain: serguzim.me
target: forgejo
- volumes_backup:
- - forgejo_data
+ backup:
+ - name: forgejo_data
+ type: docker
- name: forgejo_runner
host: node002
@@ -46,32 +47,38 @@ all_services:
dns:
- domain: serguzim.me
target: inventory
- volumes_backup:
- - homebox_data
+ backup:
+ - name: homebox_data
+ type: docker
- name: immich
host: node002
dns:
- domain: serguzim.me
target: gallery
- volumes_backup:
- - immich_upload
+ backup:
+ - name: immich_upload
+ type: docker
+ - name: immich_database
+ type: hook
- name: influxdb
host: node002
dns:
- domain: serguzim.me
target: tick
- volumes_backup:
- - influxdb_data
+ backup:
+ - name: influxdb_data
+ type: docker
- name: jellyfin
host: node002
dns:
- domain: serguzim.me
target: media
- volumes_backup:
- - jellyfin_config
+ backup:
+ - name: jellyfin_config
+ type: docker
#- jellyfin_media # TODO
- name: linkwarden
@@ -85,6 +92,9 @@ all_services:
dns:
- domain: serguzim.me
target: mail
+ backup:
+ - name: mailcow
+ type: hook
- name: minio
host: node002
@@ -95,16 +105,24 @@ all_services:
target: console.s3
name: minio-console
alias: minio
- volumes_backup:
- - minio_data
+ backup:
+ - name: minio_data
+ type: docker
- name: ntfy
host: node002
dns:
- domain: serguzim.me
target: push
- volumes_backup:
- - ntfy_data
+ backup:
+ - name: ntfy_data
+ type: docker
+
+ - name: postgresql
+ host: node002
+ backup:
+ - name: postgresql
+ type: hook
- name: reitanlage_oranienburg
host: node002
@@ -115,8 +133,9 @@ all_services:
target: www
name: reitanlage_oranienburg-www
alias: reitanlage_oranienburg
- volumes_backup:
- - reitanlage-oranienburg_data
+ backup:
+ - name: reitanlage-oranienburg_data
+ type: docker
- name: shlink
host: node002
@@ -137,8 +156,9 @@ all_services:
target: matrix
name: synapse_msrg
alias: synapse
- volumes_backup:
- - synapse_media_store
+ backup:
+ - name: synapse_media_store
+ type: docker
ports:
- 8448:8448
@@ -147,16 +167,18 @@ all_services:
dns:
- domain: serguzim.me
target: recipes
- volumes_backup:
- - tandoor_mediafiles
+ backup:
+ - name: tandoor_mediafiles
+ type: docker
- name: teamspeak_fallback
host: node002
dns:
- domain: serguzim.me
target: ts
- volumes_backup:
- - teamspeak-fallback-data
+ backup:
+ - name: teamspeak-fallback-data
+ type: docker
- name: telegraf
host: node002
@@ -178,16 +200,18 @@ all_services:
dns:
- domain: serguzim.me
target: status
- volumes_backup:
- - uptime-kuma_data
+ backup:
+ - name: uptime-kuma_data
+ type: docker
- name: vikunja
host: node002
dns:
- domain: serguzim.me
target: todo
- volumes_backup:
- - vikunja_data
+ backup:
+ - name: vikunja_data
+ type: docker
- name: webhook
host: node002
diff --git a/playbooks/filter_plugins/map_backup_locations.py b/playbooks/filter_plugins/map_backup_locations.py
new file mode 100644
index 0000000..193d543
--- /dev/null
+++ b/playbooks/filter_plugins/map_backup_locations.py
@@ -0,0 +1,38 @@
+import copy
+
+class FilterModule(object):
+ def filters(self):
+ return {
+ 'map_backup_locations': self.map_backup_locations
+ }
+
+ def map_backup_locations(self, locations, backends, hooks):
+ result = {}
+ backends_list = list(backends.keys())
+
+ for location in locations:
+ name = location["name"]
+
+ new_location = {
+ "to": backends_list,
+ "forget": "yes",
+ "hooks": copy.deepcopy(hooks)
+ }
+
+ if location["type"] == "docker":
+ new_location["from"] = name
+ new_location["type"] = "volume"
+
+ if location["type"] == "hook":
+ backup_dir = f"/opt/services/_backup/{name}"
+ new_location["from"] = backup_dir
+ if not "before" in new_location["hooks"]:
+ new_location["hooks"]["before"] = []
+ new_location["hooks"]["before"].append(f"/opt/services/backup/hooks/{name} '{backup_dir}'")
+
+ if location["type"] == "directory":
+ new_location["from"] = location["path"]
+
+ result[name.lower()] = new_location
+
+ return result
diff --git a/playbooks/filter_plugins/map_backup_volumes.py b/playbooks/filter_plugins/map_backup_volumes.py
deleted file mode 100644
index 77c1fbc..0000000
--- a/playbooks/filter_plugins/map_backup_volumes.py
+++ /dev/null
@@ -1,24 +0,0 @@
-class FilterModule(object):
- def filters(self):
- return {
- 'map_backup_volumes': self.map_backup_volumes,
- 'map_backup_volumes_service': self.map_backup_volumes_service
- }
-
- def map_backup_volumes(self, volumes):
- result = {}
-
- for volume in volumes:
- result[volume] = {
- "external": True,
- }
-
- return result
-
- def map_backup_volumes_service(self, volumes):
- result = []
-
- for volume in volumes:
- result.append("{volume_name}:/backup/volumes/{volume_name}".format(volume_name=volume))
-
- return result
diff --git a/roles/backup/files/Dockerfile b/roles/backup/files/Dockerfile
deleted file mode 100644
index 5cb0994..0000000
--- a/roles/backup/files/Dockerfile
+++ /dev/null
@@ -1,3 +0,0 @@
-FROM restic/restic
-
-RUN apk add curl
diff --git a/roles/backup/files/node002/immich.sh b/roles/backup/files/hooks/immich_database
similarity index 70%
rename from roles/backup/files/node002/immich.sh
rename to roles/backup/files/hooks/immich_database
index c1b4a18..dda7298 100755
--- a/roles/backup/files/node002/immich.sh
+++ b/roles/backup/files/hooks/immich_database
@@ -1,5 +1,6 @@
-backup_path="$BACKUP_LOCATION/immich"
-mkdir -p "$backup_path"
+#!/usr/bin/env bash
+
+backup_path="$1"
cd /opt/services/immich || exit
docker compose exec database sh -c 'pg_dump -U "$DB_USERNAME" "$DB_DATABASE"' | gzip >"$backup_path/immich.sql.gz"
diff --git a/roles/backup/files/hooks/mailcow b/roles/backup/files/hooks/mailcow
new file mode 100755
index 0000000..5d7426f
--- /dev/null
+++ b/roles/backup/files/hooks/mailcow
@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+export MAILCOW_BACKUP_LOCATION="$1"
+
+/opt/mailcow-dockerized/helper-scripts/backup_and_restore.sh backup all
diff --git a/roles/backup/files/node002/postgres.sh b/roles/backup/files/hooks/postgresql
similarity index 81%
rename from roles/backup/files/node002/postgres.sh
rename to roles/backup/files/hooks/postgresql
index b4ddb73..f3db61d 100755
--- a/roles/backup/files/node002/postgres.sh
+++ b/roles/backup/files/hooks/postgresql
@@ -1,5 +1,6 @@
-mkdir -p "$BACKUP_LOCATION/postgres"
-cd "$BACKUP_LOCATION/postgres" || exit
+#!/usr/bin/env bash
+
+cd "$1"
postgres_tables=$(sudo -u postgres psql -Atc "SELECT datname FROM pg_database WHERE datistemplate = false;")
diff --git a/roles/backup/files/node001/mailcow.sh b/roles/backup/files/node001/mailcow.sh
deleted file mode 100755
index 30a110f..0000000
--- a/roles/backup/files/node001/mailcow.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-export MAILCOW_BACKUP_LOCATION="$BACKUP_LOCATION/mailcow"
-mkdir -p "$MAILCOW_BACKUP_LOCATION"
-/opt/mailcow-dockerized/helper-scripts/backup_and_restore.sh backup all
diff --git a/roles/backup/files/node003/mailcow.sh b/roles/backup/files/node003/mailcow.sh
deleted file mode 100755
index 30a110f..0000000
--- a/roles/backup/files/node003/mailcow.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-export MAILCOW_BACKUP_LOCATION="$BACKUP_LOCATION/mailcow"
-mkdir -p "$MAILCOW_BACKUP_LOCATION"
-/opt/mailcow-dockerized/helper-scripts/backup_and_restore.sh backup all
diff --git a/roles/backup/tasks/backup.d.yml b/roles/backup/tasks/backup.d.yml
deleted file mode 100644
index fb28870..0000000
--- a/roles/backup/tasks/backup.d.yml
+++ /dev/null
@@ -1,16 +0,0 @@
----
-- name: Set backup.d path
- ansible.builtin.set_fact:
- backup_d_path: "{{ (service_path, 'backup.d') | path_join }}"
-- name: Create backup.d directory
- ansible.builtin.file:
- path: "{{ backup_d_path }}"
- state: directory
- mode: "0755"
-- name: Copy the additional backup scripts
- ansible.builtin.copy:
- src: "{{ item }}"
- dest: "{{ backup_d_path }}"
- mode: "0755"
- with_fileglob:
- - "{{ ansible_facts.hostname }}/*"
diff --git a/roles/backup/tasks/docker.yml b/roles/backup/tasks/docker.yml
deleted file mode 100644
index f5ae9f2..0000000
--- a/roles/backup/tasks/docker.yml
+++ /dev/null
@@ -1,12 +0,0 @@
----
-- name: Copy the Dockerfile
- ansible.builtin.copy:
- src: Dockerfile
- dest: "{{ (service_path, 'Dockerfile') | path_join }}"
- mode: "0644"
- register: cmd_result
-
-- name: Set the docker rebuild flag
- ansible.builtin.set_fact:
- docker_rebuild: true
- when: cmd_result.changed # noqa: no-handler We need to handle the restart per service. Handlers don't support variables.
diff --git a/roles/backup/tasks/hooks.yml b/roles/backup/tasks/hooks.yml
new file mode 100644
index 0000000..277f906
--- /dev/null
+++ b/roles/backup/tasks/hooks.yml
@@ -0,0 +1,23 @@
+---
+- name: Set hooks path
+ ansible.builtin.set_fact:
+ hooks_path: "{{ (service_path, 'hooks') | path_join }}"
+- name: Create hooks directory
+ ansible.builtin.file:
+ path: "{{ hooks_path }}"
+ state: directory
+ mode: "0755"
+- name: Copy the hooks
+ ansible.builtin.copy:
+ src: "{{ item }}"
+ dest: "{{ hooks_path }}"
+ mode: "0755"
+ with_fileglob:
+ - "hooks/*"
+- name: Create the from directories
+ ansible.builtin.file:
+ path: "{{ ('/opt/services/_backup', item | basename) | path_join }}"
+ state: directory
+ mode: "0755"
+ with_fileglob:
+ - "hooks/*"
diff --git a/roles/backup/tasks/main.yml b/roles/backup/tasks/main.yml
index c165ce5..b62eb90 100644
--- a/roles/backup/tasks/main.yml
+++ b/roles/backup/tasks/main.yml
@@ -4,36 +4,31 @@
- name: Deploy {{ svc.name }}
vars:
- svc: "{{ backup_svc }}"
- env: "{{ backup_env }}"
- compose: "{{ backup_compose }}"
+ yml: "{{ backup_yml }}"
block:
- name: Import prepare tasks for common service
ansible.builtin.import_tasks: tasks/prepare-common-service.yml
- - name: Copy the main backup script
+ - name: Template the main backup script
ansible.builtin.template:
- src: "backup.sh.j2"
+ src: backup.sh.j2
dest: "{{ (service_path, 'backup.sh') | path_join }}"
mode: "0755"
- - name: Import tasks specific to docker
- ansible.builtin.import_tasks: docker.yml
- - name: Import tasks specific to the backup.d scripts
- ansible.builtin.import_tasks: backup.d.yml
+ - name: Template autorestic.yml
+ ansible.builtin.template:
+ src: yml.j2
+ dest: "{{ (service_path, '.autorestic.yml') | path_join }}"
+ mode: "0644"
+
+ - name: Import tasks specific to the hooks scripts
+ ansible.builtin.import_tasks: hooks.yml
- name: Import tasks specific to systemd
ansible.builtin.import_tasks: systemd.yml
- - name: Build service
- ansible.builtin.command:
- cmd: docker compose build --pull
- chdir: "{{ service_path }}"
- register: cmd_result
- when: docker_rebuild
- changed_when: true
-
- name: Verify service
ansible.builtin.command:
- cmd: docker compose run --rm app check
+ cmd: autorestic -v check
chdir: "{{ service_path }}"
changed_when: false
+ become: true
diff --git a/roles/backup/templates/backup.service.j2 b/roles/backup/templates/backup.service.j2
index 131b7d4..49ef460 100644
--- a/roles/backup/templates/backup.service.j2
+++ b/roles/backup/templates/backup.service.j2
@@ -1,11 +1,11 @@
[Unit]
-Description=Autostart several tools and services
+Description=Run the backup script
StartLimitIntervalSec=7200
StartLimitBurst=5
[Service]
Type=oneshot
-ExecStart={{ service_path }}/backup.sh
+ExecStart={{ (service_path, 'backup.sh') | path_join }}
WorkingDirectory={{ service_path }}
Restart=on-failure
RestartSec=15min
diff --git a/roles/backup/templates/backup.sh.j2 b/roles/backup/templates/backup.sh.j2
old mode 100755
new mode 100644
index cdfff87..c3c8a05
--- a/roles/backup/templates/backup.sh.j2
+++ b/roles/backup/templates/backup.sh.j2
@@ -1,68 +1,12 @@
#!/usr/bin/env bash
-set -e
+{{ backup_hc_command_start }}
-set -a
-. "{{ service_path }}/service.env"
-set +a
-
-duration_start=$(date +%s)
-_duration_get () {
- duration_end=$(date +%s)
- echo "$((duration_end - duration_start))"
-}
-
-hc_url="https://hc-ping.com/$HC_UID"
-uptime_kuma_url="https://status.serguzim.me/api/push/$UPTIME_KUMA_TOKEN"
-_hc_ping () {
- curl -fsSL --retry 3 "$hc_url$1" >/dev/null
-}
-_uptime_kuma_ping () {
- duration=$(_duration_get)
- curl -fsSL --retry 3 \
- --url-query "status=$1" \
- --url-query "msg=$2" \
- --url-query "ping=${duration}000" \
- "$uptime_kuma_url" >/dev/null
-}
-
-_fail () {
- _hc_ping "/fail"
- _uptime_kuma_ping "down" "$1"
- rm -rf "$BACKUP_LOCATION"
- exit 1
-}
-_success () {
- _hc_ping
- _uptime_kuma_ping "up" "backup successful"
-}
-
-_hc_ping "/start"
-
-BACKUP_LOCATION="$(mktemp -d --suffix=-backup)"
-export BACKUP_LOCATION
-cd "$BACKUP_LOCATION" || _fail "failed to cd to $BACKUP_LOCATION"
-
-shopt -s nullglob
-for file in "{{ service_path }}/backup.d/"*
-do
- file_name="$(basename "$file")"
- echo ""
- echo "running $file_name"
- time "$file" >"/tmp/$file_name.log" || _fail "error while running $file_name"
-done || true
-
-cd "{{ service_path }}"
-docker compose run --rm -v "$BACKUP_LOCATION:/backup/misc" app backup /backup || _fail "error during restic backup"
-
-_success
-
-rm -rf "$BACKUP_LOCATION"
-
-echo "forgetting old backups for {{ ansible_facts.hostname }}"
-docker compose run --rm app forget --host "{{ ansible_facts.hostname }}" --prune \
- --keep-last 7 \
- --keep-daily 14 \
- --keep-weekly 16 \
- --keep-monthly 12 \
- --keep-yearly 2
+if autorestic backup -av --ci
+then
+ {{ backup_hc_command_success }}
+ {{ backup_uk_command_success }}
+else
+ {{ backup_hc_command_fail }}
+ {{ backup_uk_command_fail }}
+fi
diff --git a/roles/backup/vars/main.yml b/roles/backup/vars/main.yml
index 1a25cc3..aa554d9 100644
--- a/roles/backup/vars/main.yml
+++ b/roles/backup/vars/main.yml
@@ -1,60 +1,46 @@
---
-backup_image: "{{ (container_registry.public, 'services/backup') | path_join }}"
-
backup_svc:
name: backup
-backup_volumes_list: "{{ all_services | my_service_attributes(inventory_hostname, 'volumes_backup') }}"
-backup_volumes_service: "{{ backup_volumes_list | map_backup_volumes_service }}"
+backup_list: "{{ all_services | my_service_attributes(inventory_hostname, 'backup') }}"
-backup_env:
- HC_UID: "{{ host_backup.hc_uid }}"
- UPTIME_KUMA_TOKEN: "{{ host_backup.uptime_kuma_token }}"
+backup_msg_start: "Backup started"
+backup_msg_fail: "Backup failed"
+backup_msg_fail_location: "Backup failed for location: "
+backup_msg_success: "Backup successful"
- RESTIC_REPOSITORY: "{{ vault_backup.restic.s3.repository }}"
- RESTIC_PASSWORD: "{{ vault_backup.restic.s3.password }}"
+backup_curl_base: 'curl -L -m 10 --retry 5'
+backup_hc_curl_base: '{{ backup_curl_base }} -X POST -H "Content-Type: text/plain"'
+backup_uk_curl_base: '{{ backup_curl_base }}'
+backup_hc_url: 'https://hc-ping.com/{{ host_backup.hc_uid }}'
+backup_uk_url: 'https://status.serguzim.me/api/push/{{ host_backup.uptime_kuma_token }}'
- AWS_ACCESS_KEY_ID: "{{ vault_backup.restic.s3.access_key_id }}"
- AWS_SECRET_ACCESS_KEY: "{{ vault_backup.restic.s3.secret_access_key }}"
+backup_hc_command_start: '{{ backup_hc_curl_base }} --data "{{ backup_msg_start }}" {{ backup_hc_url }}/start'
+backup_hc_command_success: '{{ backup_hc_curl_base }} --data "{{ backup_msg_success }}" {{ backup_hc_url }}'
+backup_uk_command_success: '{{ backup_uk_curl_base }} "{{ backup_uk_url }}?status=up&msg={{ backup_msg_success | urlencode }}&ping="'
+backup_hc_command_fail: '{{ backup_hc_curl_base }} --data "{{ backup_msg_fail }}" {{ backup_hc_url }}/fail'
+backup_uk_command_fail: '{{ backup_uk_curl_base }} "{{ backup_uk_url }}?status=down&msg={{ backup_msg_fail | urlencode }}&ping="'
- #RESTIC_S3_REPOSITORY: "{{ vault_backup.restic.s3.repository }}"
- #RESTIC_S3_PASSWORD: "{{ vault_backup.restic.s3.password }}"
- #RESITC_S3_ACCESS_KEY_ID: "{{ vault_backup.restic.s3.access_key_id }}"
- #RESITC_S3_SECRET_ACCESS_KEY: "{{ vault_backup.restic.s3.secret_access_key }}"
+backup_default_hooks:
+ failure:
+ - '{{ backup_hc_curl_base }} --data "{{ backup_msg_fail_location }}${AUTORESTIC_LOCATION}" {{ backup_hc_url }}/fail'
+ - '{{ backup_uk_curl_base }} "{{ backup_uk_url }}?status=down&msg={{ backup_msg_fail_location | urlencode }}${AUTORESTIC_LOCATION}&ping="'
- #RESTIC_BORGBASE: "{{ vault_backup.restic.borgbase }}"
+backup_yml:
+ version: 2
-backup_compose:
- watchtower: false
- image: "{{ backup_image }}"
- volumes: "{{ backup_volumes_service }}"
- file:
- services:
- app:
- build:
- context: .
- entrypoint:
- - /usr/bin/restic
- - --retry-lock=1m
- restart: never
- hostname: "{{ ansible_facts.hostname }}"
- mount:
- build:
- context: .
- image: "{{ backup_image }}"
- restart: never
- hostname: "{{ ansible_facts.hostname }}"
- env_file:
- - service.env
- entrypoint:
- - /usr/bin/restic
- - --retry-lock=1m
- command:
- - mount
- - /mnt
- privileged: true
- devices:
- - /dev/fuse
+ backends: "{{ vault_backup.locations }}"
- volumes: "{{ backup_volumes_list | map_backup_volumes }}"
+ locations: "{{ backup_list | map_backup_locations(vault_backup.locations, backup_default_hooks ) }}"
+
+ global:
+ forget:
+ keep-last: 7
+ keep-daily: 14
+ keep-weekly: 16
+ keep-monthly: 12
+ keep-yearly: 2
+ host: "{{ ansible_facts.hostname }}"
+ backup:
+ host: "{{ ansible_facts.hostname }}"
diff --git a/roles/postgresql/.gitkeep b/roles/postgresql/.gitkeep
new file mode 100644
index 0000000..e69de29