Add healthcheck to ansible

_ansible/node002.yml

@@ -6,6 +6,7 @@
     - coder
     - forgejo
     - forgejo-runner
+    - healthcheck
     - homebox
     - influxdb
     - jellyfin

_ansible/roles/healthcheck/files/Dockerfile

@@ -0,0 +1,7 @@
+FROM ubuntu
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt update -y \
+	&& apt install -y curl dnsutils msmtp gettext-base python3-pip python3-requests \
+	&& pip install matrix-nio

_ansible/roles/healthcheck/files/data/http

@@ -0,0 +1,54 @@
+#!/usr/bin/sh
+
+cd /opt/ || exit
+
+hc_url="https://hc-ping.com/$HTTP_HC_UID"
+services_down=""
+error=""
+
+alias curl_hc='curl -LA "$USER_AGENT" --retry 3'
+
+check_url ()
+{
+	url="https://$1$2"
+    printf "checking url %s ." "$url"
+    dig A "$1" >/dev/null
+	printf "."
+	result=$(curl -LsSfv "$url" 2>&1)
+	code="$?"
+	printf ".\n"
+	#shellcheck disable=SC2181
+	if [ "$code" = "0" ]
+    then
+		echo "... good"
+	else
+		services_down=$(printf "%s\n%s" "$services_down" "$1")
+		error=$(printf "%s\n==========\n%s:\n%s" "$error" "$1" "$result")
+		echo "... bad"
+    fi
+}
+
+check_url "acme.serguzim.me" "/health"
+check_url "analytics.serguzim.me"
+check_url "auth.serguzim.me"
+check_url "ci.serguzim.me"
+#check_url "cloud.serguzim.me" "/login?noredir=1"
+check_url "git.serguzim.me"
+check_url "hook.serguzim.me"
+check_url "mail.serguzim.me"
+#check_url "msrg.cc" # disabled because it keeps creating false alerts
+check_url "registry.serguzim.me" "/account/sign-in"
+check_url "rss.serguzim.me"
+#check_url "serguzim.me" # disabled because it keeps creating false alerts
+check_url "status.serguzim.me" "/status/serguzim-net"
+check_url "tick.serguzim.me"
+check_url "wiki.serguzim.me"
+check_url "www.reitanlage-oranienburg.de"
+
+if [ "$error" = "" ]
+then
+    curl_hc "$hc_url" >/dev/null
+	echo "ALL GOOD"
+else
+	curl_hc --data-raw "$services_down$error" "$hc_url/fail" >/dev/null
+fi

_ansible/roles/healthcheck/files/data/mail

@@ -0,0 +1,17 @@
+#!/usr/bin/sh
+
+cd /opt/ || exit
+
+hc_url="https://hc-ping.com/$MAIL_HC_UID"
+
+alias curl_hc='curl -LA "$USER_AGENT" --retry 3'
+
+envsubst < template.msmtprc > /tmp/msmtprc
+envsubst < mailcheck.template.mail > /tmp/mailcheck.mail
+
+result=$(msmtp -C /tmp/msmtprc -a default "$MAIL_HC_UID@hc-ping.com" < /tmp/mailcheck.mail 2>&1)
+if [ "$?" != "0" ]
+then
+	echo "$result"
+	curl_hc --data-raw "$result" "$hc_url/fail" >/dev/null
+fi

_ansible/roles/healthcheck/files/data/mailcheck.template.mail

@@ -0,0 +1,5 @@
+To: ${MAIL_HC_UID}@hc-ping.com
+From: ${MAIL_USER}
+Subject: Healthcheck
+
+Mailserver alive

_ansible/roles/healthcheck/files/data/matrix

@@ -0,0 +1,54 @@
+#!/usr/bin/python3
+
+import datetime
+import os
+import requests
+import sys
+
+import asyncio
+from nio import AsyncClient, RoomMessageNotice
+
+healthcheck_url = "https://hc-ping.com/" + os.environ['MATRIX_HC_UID']
+
+def send_ping(success, msg=""):
+    url = healthcheck_url
+    if not success:
+        url += "/fail"
+
+    requests.get(url, data=msg, headers={'user-agent': os.environ['USER_AGENT']})
+
+async def main():
+    try:
+        client = AsyncClient(os.environ['MATRIX_SERVER'])
+        client.access_token = os.environ['MATRIX_TOKEN']
+        client.device_id = os.environ['USER_AGENT']
+        await client.room_send(
+            room_id = os.environ['MATRIX_ROOM'],
+            message_type = "m.room.message",
+            content = {
+                "msgtype": "m.text",
+                "body": "!ping"
+            }
+        )
+    except Exception as e:
+        print(e)
+
+        print("exception during login or sending")
+        send_ping(False)
+        sys.exit(1)
+    await client.close()
+
+    url = "https://federationtester.matrix.org/api/report?server_name=" \
+        + os.environ['MATRIX_SERVER_FEDTESTER']
+    resp = requests.get(url)
+    data = resp.json() # Check the JSON Response Content documentation below
+    if data["FederationOK"] != True:
+        send_ping(False)
+        sys.exit(1)
+
+    send_ping(True)
+    sys.exit(0)
+
+
+
+asyncio.new_event_loop().run_until_complete(main())

_ansible/roles/healthcheck/files/data/template.msmtprc

@@ -0,0 +1,13 @@
+defaults
+auth on
+tls on
+tls_trust_file /etc/ssl/certs/ca-certificates.crt
+logfile /tmp/msmtp.log
+
+account default
+host ${MAIL_HOST}
+port ${MAIL_PORT}
+tls_starttls on
+from ${MAIL_USER}
+user ${MAIL_USER}
+password ${MAIL_PASS}

_ansible/roles/healthcheck/files/docker-compose.yml

@@ -0,0 +1,28 @@
+version: "3.7"
+
+x-common-elements:
+  &common-elements
+  build:
+    context: .
+  image: healthcheck
+  restart: never
+  env_file:
+    - service.env
+  volumes:
+    - ./data/:/opt
+
+services:
+  http:
+    <<: *common-elements
+    command: "/opt/http"
+  matrix:
+    <<: *common-elements
+    command: "/opt/matrix"
+  mail:
+    <<: *common-elements
+    command: "/opt/mail"
+
+networks:
+  default:
+    name: healthcheck
+    external: true

_ansible/roles/healthcheck/files/healthcheck@.service

@@ -0,0 +1,4 @@
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/docker compose run --rm %i
+WorkingDirectory=/opt/services/healthcheck/

_ansible/roles/healthcheck/files/healthcheck@.timer

@@ -0,0 +1,4 @@
+[Timer]
+OnCalendar=*:0/5
+[Install]
+WantedBy=timers.target

_ansible/roles/healthcheck/tasks/main.yml

@@ -0,0 +1,46 @@
+---
+- name: Deploy {{ svc.name }}
+  tags:
+    - healthcheck
+  block:
+    - import_tasks: steps/create-service-directory.yml
+
+    - name: Copy the docker-compose file
+      ansible.builtin.copy:
+        src: docker-compose.yml
+        dest: "{{ (service_path, 'docker-compose.yml') | path_join }}"
+        mode: '0644'
+    - name: Copy the Dockerfile
+      ansible.builtin.copy:
+        src: Dockerfile
+        dest: "{{ (service_path, 'Dockerfile') | path_join }}"
+        mode: '0644'
+    - name: Copy the data files
+      ansible.builtin.copy:
+        src: data
+        dest: "{{ service_path }}"
+        mode: '0755'
+
+    - name: Copy the system service
+      ansible.builtin.copy:
+        src: healthcheck@.service
+        dest: "/etc/systemd/system/healthcheck@.service"
+        mode: '0644'
+      become: true
+    - name: Copy the system timer
+      ansible.builtin.copy:
+        src: healthcheck@.timer
+        dest: "/etc/systemd/system/healthcheck@.timer"
+        mode: '0644'
+      become: true
+
+    - import_tasks: steps/template-service-env.yml
+
+    - name: Build service
+      ansible.builtin.command:
+        cmd: docker compose build --pull
+        chdir: "{{ service_path }}"
+      when:
+        - "'local-dev' != inventory_hostname"
+      register: cmd_result
+      changed_when: True

_ansible/roles/healthcheck/vars/main.yml

@@ -0,0 +1,19 @@
+svc:
+  name: healthcheck
+
+svc_env:
+  USER_AGENT: "healthcheck-bot for serguzim.net"
+
+  HTTP_HC_UID: "{{ vault_healthcheck.hc_uid.http }}"
+
+  MATRIX_SERVER: "https://matrix.msrg.cc"
+  MATRIX_SERVER_FEDTESTER: "msrg.cc"
+  MATRIX_HC_UID: "{{ vault_healthcheck.hc_uid.matrix }}"
+  MATRIX_TOKEN: "{{ vault_healthcheck.matrix.token }}"
+  MATRIX_ROOM: "{{ vault_healthcheck.matrix.room }}"
+
+  MAIL_HC_UID: "{{ vault_healthcheck.hc_uid.mail }}"
+  MAIL_HOST: "{{ mailer.host }}"
+  MAIL_PORT: "{{ mailer.port }}"
+  MAIL_USER: "{{ vault_healthcheck.mailer.user }}"
+  MAIL_PASS: "{{ vault_healthcheck.mailer.pass }}"

_ansible/roles/tandoor/vars/main.yml

@@ -15,7 +15,7 @@ svc_env:
 
   ALLOWED_HOSTS: recipes.serguzim.me
   SECRET_KEY: "{{ vault_tandoor.secret_key }}"
-  TIMEZONE: "{{ timezone }}"
+  TZ: "{{ timezone }}"
 
   DB_ENGINE: django.db.backends.postgresql
   DB_OPTIONS: "{\"sslmode\": \"require\"}"