Replace uptime kuma with gatus

Makefile

@@ -44,3 +44,9 @@ dns: ./types-dnscontrol.d.ts ./dns/hosts.json ./dns/services.json
 
 dns-check: ./types-dnscontrol.d.ts ./dns/hosts.json ./dns/services.json
 	dnscontrol check-creds ovh
+
+all:
+	$(MAKE) tofu
+	$(MAKE) dns
+	@printf "\n=====\n\n"
+	ansible-playbook ./playbooks/serguzim.net.yml

playbooks/filter_plugins/gatus.py

@@ -0,0 +1,86 @@
+class FilterModule(object):
+    def filters(self):
+        return {
+            'hosts_to_gatus': self.hosts_to_gatus,
+            'vault_hosts_backup_to_gatus': self.vault_hosts_backup_to_gatus,
+            'services_to_gatus': self.services_to_gatus,
+        }
+
+    default_alerts = [
+        {
+            "type": "ntfy",
+            "send-on-resolved": True,
+        },
+        {
+            "type": "email",
+            "send-on-resolved": True,
+        },
+    ]
+
+    def hosts_to_gatus(self, hosts):
+        result = []
+        for host in hosts.values():
+            result.append({
+                "name": host["hostname"],
+                "url": f"icmp://{host['fqdn']}",
+                "group": "1-hosts",
+                "conditions": [
+                    "[CONNECTED] == true",
+                ],
+                "alerts": self.default_alerts,
+            })
+        return result
+
+    def vault_hosts_backup_to_gatus(self, hosts):
+        result = []
+        for name, host_data in hosts.items():
+            result.append({
+                "name": f"backup@{name}",
+                "group": "8-backups",
+                "token": host_data["backup"]["gatus_token"],
+                "alerts": self.default_alerts,
+            })
+        return result
+
+    def services_to_gatus(self, services):
+        result = []
+
+        default_conditions = [
+            "[STATUS] == any(200, 204)",
+        ]
+
+        for service in services:
+            if mon := service.get("monitoring"):
+                if service.get("dns"):
+                    dns = service["dns"][0]
+                    url = "https://"
+                    if dns.get("target") != "@":
+                        url += f"{dns["target"]}."
+                    url += dns['domain']
+
+                if mon_url := mon.get("url"):
+                    if mon_url.startswith("/"):
+                        url += mon_url
+                    else:
+                        url = mon_url
+
+                if conditions := mon.get("conditions"):
+                    if conditions[0] == "DEFAULT":
+                        conditions.pop(0)
+                        conditions[:0] = default_conditions
+                    else:
+                        conditions = conditions
+                else:
+                    conditions = default_conditions
+
+                new_endpoint = {
+                    "name": service["name"],
+                    "group": mon.get("group"),
+                    "url": url,
+                    "conditions": conditions,
+                    "alerts": self.default_alerts,
+                }
+
+                result.append(new_endpoint)
+
+        return result

roles/gatus/tasks/main.yml

@@ -0,0 +1,27 @@
+---
+- name: Set common facts
+  ansible.builtin.import_tasks: tasks/set-default-facts.yml
+
+- name: Deploy {{ svc.name }}
+  vars:
+    svc: "{{ gatus_svc }}"
+    yml: "{{ gatus_yml }}"
+    compose: "{{ gatus_compose }}"
+  block:
+    - name: Import prepare tasks for common service
+      ansible.builtin.import_tasks: tasks/prepare-common-service.yml
+
+    - name: Template config
+      ansible.builtin.template:
+        src: yml.j2
+        dest: "{{ (service_path, 'config.yaml') | path_join }}"
+        mode: "0644"
+      register: cmd_result
+
+    - name: Set the docker force-recreate flag
+      ansible.builtin.set_fact:
+        docker_force_recreate: --force-recreate
+      when: cmd_result.changed # noqa: no-handler We need to handle the restart per service. Handlers don't support variables.
+
+    - name: Import start tasks for common service
+      ansible.builtin.import_tasks: tasks/start-common-service.yml

roles/gatus/vars/main.yml

@@ -0,0 +1,69 @@
+---
+gatus_svc:
+  domain: status.serguzim.me
+  name: gatus
+  port: 8080
+
+gatus_external_endpoints_backups: "{{ vault_hosts | vault_hosts_backup_to_gatus() }}"
+
+gatus_endpoints_hosts: "{{ opentofu.hosts | hosts_to_gatus() }}"
+gatus_endpoints_services: "{{ all_services | services_to_gatus() }}"
+
+gatus_endpoints_other:
+  - name: matrix-federation
+    url: https://federationtester.matrix.org/api/report?server_name=msrg.cc
+    group: 9-external
+    interval: 5m
+    conditions:
+      - '[STATUS] == 200'
+      - '[BODY].FederationOK == true'
+    ui:
+      hide-url: true
+    alerts:
+      - type: ntfy
+        send-on-resolved: true
+      - type: email
+        send-on-resolved: true
+  - name: healthchecks-io
+    url: "{{ opentofu.healthchecksio.status.ping_url }}"
+    group: 9-external
+    interval: 5m
+    conditions:
+      - '[STATUS] == 200'
+    ui:
+      hide-url: true
+
+gatus_yml:
+  storage:
+    type: sqlite
+    path: /data/data.db
+
+  connectivity:
+    checker:
+      target: 1.1.1.1:53
+      interval: 60s
+
+  security:
+    oidc:
+      issuer-url: "{{ opentofu.authentik_data.gatus.base_url }}"
+      redirect-url: "https://{{ gatus_svc.domain }}/authorization-code/callback"
+      client-id: "{{ opentofu.authentik_data.gatus.client_id }}"
+      client-secret: "{{ opentofu.authentik_data.gatus.client_secret }}"
+      scopes: ["openid"]
+
+  alerting:
+    email: "{{ vault_gatus.alerting.email }}"
+    ntfy: "{{ vault_gatus.alerting.ntfy }}"
+
+  external-endpoints: "{{ gatus_external_endpoints_backups }}"
+  endpoints: "{{ gatus_endpoints_hosts | union(gatus_endpoints_services) | union(gatus_endpoints_other) }}"
+
+gatus_compose:
+  watchtower: true
+  image: twinproduction/gatus
+  volumes:
+    - ./config.yaml:/config/config.yaml
+    - data:/data
+  file:
+    volumes:
+      data:

roles/healthcheck/files/data/http

@@ -38,7 +38,7 @@ check_url "mail.serguzim.me"
 #check_url "msrg.cc" # disabled because it keeps creating false alerts
 check_url "rss.serguzim.me"
 #check_url "serguzim.me" # disabled because it keeps creating false alerts
-check_url "status.serguzim.me" "/status/serguzim-net"
+#check_url "status.serguzim.me" "/status/serguzim-net"
 check_url "tick.serguzim.me"
 check_url "wiki.serguzim.me"
 check_url "www.reitanlage-oranienburg.de"

roles/uptime_kuma/tasks/main.yml

@@ -1,12 +0,0 @@
----
-- name: Set common facts
-  ansible.builtin.import_tasks: tasks/set-default-facts.yml
-
-- name: Deploy {{ svc.name }}
-  vars:
-    svc: "{{ uptime_kuma_svc }}"
-    env: "{{ uptime_kuma_env }}"
-    compose: "{{ uptime_kuma_compose }}"
-  block:
-    - name: Import tasks to deploy common service
-      ansible.builtin.import_tasks: tasks/deploy-common-service.yml

roles/uptime_kuma/vars/main.yml

@@ -1,16 +0,0 @@
----
-uptime_kuma_svc:
-  domain: status.serguzim.me
-  additional_domains:
-    - status.serguzim.net
-  name: uptime-kuma
-  port: 3001
-
-uptime_kuma_compose:
-  watchtower: true
-  image: louislam/uptime-kuma:1
-  volumes:
-    - data:/app/data
-  file:
-    volumes:
-      data:

services.auto.tfvars

@@ -6,6 +6,10 @@ services = {
       domain = "serguzim.me"
       target = "acme"
     }]
+    monitoring = {
+      url = "/health"
+      group = "7-support"
+    }
     auth = false
     database = true
     s3 = false
@@ -18,6 +22,10 @@ services = {
       domain = "serguzim.me"
       target = "auth"
     }]
+    monitoring = {
+      url = "/-/health/live/"
+      group = "4-services"
+    }
     auth = false
     database = true
     s3 = false
@@ -50,6 +58,10 @@ services = {
         alias = "faas"
       }
     ]
+    monitoring = {
+      url = "/healthz"
+      group = "7-support"
+    }
     auth = false
     database = false
     s3 = false
@@ -66,6 +78,10 @@ services = {
       name = "forgejo_data"
       type = "docker"
     }]
+    monitoring = {
+      url = "/api/v1/version"
+      group = "4-services"
+    }
     auth = true
     auth_redirects = ["https://git.serguzim.me/user/oauth2/auth.serguzim.me/callback"]
     database = true
@@ -88,6 +104,23 @@ services = {
     s3 = false
   },
 
+  "gatus" = {
+    name = "gatus"
+    host = "node003"
+    dns = [{
+      domain = "serguzim.me"
+      target = "status"
+    }]
+    backup = [{
+      name = "gatus_data"
+      type = "docker"
+    }]
+    auth = true
+    auth_redirects = ["https://status.serguzim.me/authorization-code/callback"]
+    database = false
+    s3 = false
+  },
+
   "homebox" = {
     name = "homebox"
     host = "node002"
@@ -99,6 +132,14 @@ services = {
       name = "homebox_data"
       type = "docker"
     }]
+    monitoring = {
+      url = "/api/v1/status"
+      group = "4-services"
+      conditions = [
+        "DEFAULT",
+        "[BODY].health == true"
+      ]
+    }
     auth = false
     database = false
     s3 = false
@@ -121,6 +162,9 @@ services = {
         type = "hook"
       }
     ]
+    monitoring = {
+      group = "4-services"
+    }
     auth = true
     auth_redirects = ["https://gallery.serguzim.me/auth/login"]
     database = false
@@ -138,6 +182,14 @@ services = {
       name = "influxdb_data"
       type = "docker"
     }]
+    monitoring = {
+      url = "/health"
+      group = "4-services"
+      conditions = [
+        "DEFAULT",
+        "[BODY].status == pass"
+      ]
+    }
     auth = false
     database = false
     s3 = false
@@ -154,6 +206,14 @@ services = {
       name = "jellyfin_config"
       type = "docker"
     }] # TODO add jellyfin_media
+    monitoring = {
+      url = "/health"
+      group = "4-services"
+      conditions = [
+        "DEFAULT",
+        "[BODY] == Healthy"
+      ]
+    }
     auth = false
     database = false
     s3 = false
@@ -166,6 +226,10 @@ services = {
       domain = "serguzim.me"
       target = "bookmarks"
     }]
+    monitoring = {
+      url = "/api/v1/logins"
+      group = "4-services"
+    }
     auth = true
     auth_redirects = ["https://bookmarks.serguzim.me/api/v1/auth/callback/authentik"]
     database = true
@@ -183,6 +247,9 @@ services = {
       name = "mailcow"
       type = "hook"
     }]
+    monitoring = {
+      group = "4-services"
+    }
     auth = false
     database = false
     s3 = false
@@ -207,6 +274,10 @@ services = {
       name = "minio_data"
       type = "docker"
     }]
+    monitoring = {
+      url = "/minio/health/live"
+      group = "7-support"
+    }
     auth = false
     database = false
     s3 = false
@@ -223,6 +294,14 @@ services = {
       name = "ntfy_data"
       type = "docker"
     }]
+    monitoring = {
+      url = "/v1/health"
+      group = "4-services"
+      conditions = [
+        "DEFAULT",
+        "[BODY].healthy == true"
+      ]
+    }
     auth = false
     database = false
     s3 = false
@@ -235,6 +314,7 @@ services = {
       name = "postgresql"
       type = "hook"
     }]
+    # TODO add monitoring
     auth = false
     database = false
     s3 = false
@@ -259,6 +339,9 @@ services = {
       name = "reitanlage-oranienburg_data"
       type = "docker"
     }]
+    monitoring = {
+      group = "5-websites"
+    }
     auth = false
     database = false
     s3 = false
@@ -279,6 +362,14 @@ services = {
         alias = "shlink"
       }
     ]
+    monitoring = {
+      url = "/rest/health"
+      group = "4-services"
+      conditions = [
+        "DEFAULT",
+        "[BODY].status == pass"
+      ]
+    }
     auth = false
     database = true
     s3 = false
@@ -303,6 +394,10 @@ services = {
       name = "synapse_media_store"
       type = "docker"
     }]
+    monitoring = {
+      url = "/_matrix/client/versions"
+      group = "4-services"
+    }
     ports = ["8448:8448"]
     auth = true
     auth_redirects = ["https://matrix.serguzim.me/_synapse/client/oidc/callback"]
@@ -321,6 +416,10 @@ services = {
       name = "tandoor_mediafiles"
       type = "docker"
     }]
+    monitoring = {
+      url = "/accounts/login/"
+      group = "4-services"
+    }
     auth = false
     database = true
     s3 = false
@@ -357,6 +456,10 @@ services = {
       domain = "serguzim.me"
       target = "rss"
     }]
+    monitoring = {
+      url = "/tt-rss/"
+      group = "4-services"
+    }
     auth = false
     database = true
     s3 = false
@@ -369,22 +472,10 @@ services = {
       domain = "serguzim.me"
       target = "analytics"
     }]
-    auth = false
+    monitoring = {
-    database = true
+      url = "/api/heartbeat"
-    s3 = false
+      group = "4-services"
-  },
+    }
-
-  "uptime_kuma" = {
-    name = "uptime_kuma"
-    host = "node002"
-    dns = [{
-      domain = "serguzim.me"
-      target = "status"
-    }]
-    backup = [{
-      name = "uptime-kuma_data"
-      type = "docker"
-    }]
     auth = false
     database = true
     s3 = false
@@ -401,6 +492,10 @@ services = {
       name = "vikunja_data"
       type = "docker"
     }]
+    monitoring = {
+      url = "/api/v1/info"
+      group = "4-services"
+    }
     auth = true
     auth_redirects = ["https://todo.serguzim.me/auth/openid/authserguzimme"]
     database = true
@@ -414,6 +509,9 @@ services = {
       domain = "serguzim.me"
       target = "hook"
     }]
+    monitoring = {
+      group = "7-support"
+    }
     auth = false
     database = false
     s3 = false
@@ -426,6 +524,9 @@ services = {
       domain = "serguzim.me"
       target = "wiki"
     }]
+    monitoring = {
+      group = "4-services"
+    }
     auth = true
     auth_redirects = ["https://wiki.serguzim.me/login/f792bc7d-1a25-4437-944e-55eaf0111102/callback"]
     database = true
@@ -439,6 +540,10 @@ services = {
       domain = "serguzim.me"
       target = "ci"
     }]
+    monitoring = {
+      url = "/healthz"
+      group = "4-services"
+    }
     auth = false
     database = true
     s3 = false

variables.tf

@@ -137,6 +137,11 @@ variable "services" {
       name = string
       type = string
     })))
+    monitoring = optional(object({
+      url   = optional(string)
+      group = optional(string)
+      conditions = optional(list(string))
+    }))
     ports = optional(list(string))
     auth = bool
     auth_redirects = optional(list(string))