Add healthcheck to ansible
This commit is contained in:
parent
68463c03f1
commit
1cb92e3825
13 changed files with 68 additions and 6 deletions
|
|
@ -6,6 +6,7 @@
|
||||||
- coder
|
- coder
|
||||||
- forgejo
|
- forgejo
|
||||||
- forgejo-runner
|
- forgejo-runner
|
||||||
|
- healthcheck
|
||||||
- homebox
|
- homebox
|
||||||
- influxdb
|
- influxdb
|
||||||
- jellyfin
|
- jellyfin
|
||||||
|
|
|
||||||
7
_ansible/roles/healthcheck/files/Dockerfile
Normal file
7
_ansible/roles/healthcheck/files/Dockerfile
Normal file
|
|
@ -0,0 +1,7 @@
|
||||||
|
FROM ubuntu
|
||||||
|
|
||||||
|
ENV DEBIAN_FRONTEND=noninteractive
|
||||||
|
|
||||||
|
RUN apt update -y \
|
||||||
|
&& apt install -y curl dnsutils msmtp gettext-base python3-pip python3-requests \
|
||||||
|
&& pip install matrix-nio
|
||||||
54
_ansible/roles/healthcheck/files/data/http
Executable file
54
_ansible/roles/healthcheck/files/data/http
Executable file
|
|
@ -0,0 +1,54 @@
|
||||||
|
#!/usr/bin/sh
|
||||||
|
|
||||||
|
cd /opt/ || exit
|
||||||
|
|
||||||
|
hc_url="https://hc-ping.com/$HTTP_HC_UID"
|
||||||
|
services_down=""
|
||||||
|
error=""
|
||||||
|
|
||||||
|
alias curl_hc='curl -LA "$USER_AGENT" --retry 3'
|
||||||
|
|
||||||
|
check_url ()
|
||||||
|
{
|
||||||
|
url="https://$1$2"
|
||||||
|
printf "checking url %s ." "$url"
|
||||||
|
dig A "$1" >/dev/null
|
||||||
|
printf "."
|
||||||
|
result=$(curl -LsSfv "$url" 2>&1)
|
||||||
|
code="$?"
|
||||||
|
printf ".\n"
|
||||||
|
#shellcheck disable=SC2181
|
||||||
|
if [ "$code" = "0" ]
|
||||||
|
then
|
||||||
|
echo "... good"
|
||||||
|
else
|
||||||
|
services_down=$(printf "%s\n%s" "$services_down" "$1")
|
||||||
|
error=$(printf "%s\n==========\n%s:\n%s" "$error" "$1" "$result")
|
||||||
|
echo "... bad"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
check_url "acme.serguzim.me" "/health"
|
||||||
|
check_url "analytics.serguzim.me"
|
||||||
|
check_url "auth.serguzim.me"
|
||||||
|
check_url "ci.serguzim.me"
|
||||||
|
#check_url "cloud.serguzim.me" "/login?noredir=1"
|
||||||
|
check_url "git.serguzim.me"
|
||||||
|
check_url "hook.serguzim.me"
|
||||||
|
check_url "mail.serguzim.me"
|
||||||
|
#check_url "msrg.cc" # disabled because it keeps creating false alerts
|
||||||
|
check_url "registry.serguzim.me" "/account/sign-in"
|
||||||
|
check_url "rss.serguzim.me"
|
||||||
|
#check_url "serguzim.me" # disabled because it keeps creating false alerts
|
||||||
|
check_url "status.serguzim.me" "/status/serguzim-net"
|
||||||
|
check_url "tick.serguzim.me"
|
||||||
|
check_url "wiki.serguzim.me"
|
||||||
|
check_url "www.reitanlage-oranienburg.de"
|
||||||
|
|
||||||
|
if [ "$error" = "" ]
|
||||||
|
then
|
||||||
|
curl_hc "$hc_url" >/dev/null
|
||||||
|
echo "ALL GOOD"
|
||||||
|
else
|
||||||
|
curl_hc --data-raw "$services_down$error" "$hc_url/fail" >/dev/null
|
||||||
|
fi
|
||||||
17
_ansible/roles/healthcheck/files/data/mail
Executable file
17
_ansible/roles/healthcheck/files/data/mail
Executable file
|
|
@ -0,0 +1,17 @@
|
||||||
|
#!/usr/bin/sh
|
||||||
|
|
||||||
|
cd /opt/ || exit
|
||||||
|
|
||||||
|
hc_url="https://hc-ping.com/$MAIL_HC_UID"
|
||||||
|
|
||||||
|
alias curl_hc='curl -LA "$USER_AGENT" --retry 3'
|
||||||
|
|
||||||
|
envsubst < template.msmtprc > /tmp/msmtprc
|
||||||
|
envsubst < mailcheck.template.mail > /tmp/mailcheck.mail
|
||||||
|
|
||||||
|
result=$(msmtp -C /tmp/msmtprc -a default "$MAIL_HC_UID@hc-ping.com" < /tmp/mailcheck.mail 2>&1)
|
||||||
|
if [ "$?" != "0" ]
|
||||||
|
then
|
||||||
|
echo "$result"
|
||||||
|
curl_hc --data-raw "$result" "$hc_url/fail" >/dev/null
|
||||||
|
fi
|
||||||
|
|
@ -0,0 +1,5 @@
|
||||||
|
To: ${MAIL_HC_UID}@hc-ping.com
|
||||||
|
From: ${MAIL_USER}
|
||||||
|
Subject: Healthcheck
|
||||||
|
|
||||||
|
Mailserver alive
|
||||||
54
_ansible/roles/healthcheck/files/data/matrix
Executable file
54
_ansible/roles/healthcheck/files/data/matrix
Executable file
|
|
@ -0,0 +1,54 @@
|
||||||
|
#!/usr/bin/python3
|
||||||
|
|
||||||
|
import datetime
|
||||||
|
import os
|
||||||
|
import requests
|
||||||
|
import sys
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
from nio import AsyncClient, RoomMessageNotice
|
||||||
|
|
||||||
|
healthcheck_url = "https://hc-ping.com/" + os.environ['MATRIX_HC_UID']
|
||||||
|
|
||||||
|
def send_ping(success, msg=""):
|
||||||
|
url = healthcheck_url
|
||||||
|
if not success:
|
||||||
|
url += "/fail"
|
||||||
|
|
||||||
|
requests.get(url, data=msg, headers={'user-agent': os.environ['USER_AGENT']})
|
||||||
|
|
||||||
|
async def main():
|
||||||
|
try:
|
||||||
|
client = AsyncClient(os.environ['MATRIX_SERVER'])
|
||||||
|
client.access_token = os.environ['MATRIX_TOKEN']
|
||||||
|
client.device_id = os.environ['USER_AGENT']
|
||||||
|
await client.room_send(
|
||||||
|
room_id = os.environ['MATRIX_ROOM'],
|
||||||
|
message_type = "m.room.message",
|
||||||
|
content = {
|
||||||
|
"msgtype": "m.text",
|
||||||
|
"body": "!ping"
|
||||||
|
}
|
||||||
|
)
|
||||||
|
except Exception as e:
|
||||||
|
print(e)
|
||||||
|
|
||||||
|
print("exception during login or sending")
|
||||||
|
send_ping(False)
|
||||||
|
sys.exit(1)
|
||||||
|
await client.close()
|
||||||
|
|
||||||
|
url = "https://federationtester.matrix.org/api/report?server_name=" \
|
||||||
|
+ os.environ['MATRIX_SERVER_FEDTESTER']
|
||||||
|
resp = requests.get(url)
|
||||||
|
data = resp.json() # Check the JSON Response Content documentation below
|
||||||
|
if data["FederationOK"] != True:
|
||||||
|
send_ping(False)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
send_ping(True)
|
||||||
|
sys.exit(0)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
asyncio.new_event_loop().run_until_complete(main())
|
||||||
13
_ansible/roles/healthcheck/files/data/template.msmtprc
Normal file
13
_ansible/roles/healthcheck/files/data/template.msmtprc
Normal file
|
|
@ -0,0 +1,13 @@
|
||||||
|
defaults
|
||||||
|
auth on
|
||||||
|
tls on
|
||||||
|
tls_trust_file /etc/ssl/certs/ca-certificates.crt
|
||||||
|
logfile /tmp/msmtp.log
|
||||||
|
|
||||||
|
account default
|
||||||
|
host ${MAIL_HOST}
|
||||||
|
port ${MAIL_PORT}
|
||||||
|
tls_starttls on
|
||||||
|
from ${MAIL_USER}
|
||||||
|
user ${MAIL_USER}
|
||||||
|
password ${MAIL_PASS}
|
||||||
28
_ansible/roles/healthcheck/files/docker-compose.yml
Normal file
28
_ansible/roles/healthcheck/files/docker-compose.yml
Normal file
|
|
@ -0,0 +1,28 @@
|
||||||
|
version: "3.7"
|
||||||
|
|
||||||
|
x-common-elements:
|
||||||
|
&common-elements
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
image: healthcheck
|
||||||
|
restart: never
|
||||||
|
env_file:
|
||||||
|
- service.env
|
||||||
|
volumes:
|
||||||
|
- ./data/:/opt
|
||||||
|
|
||||||
|
services:
|
||||||
|
http:
|
||||||
|
<<: *common-elements
|
||||||
|
command: "/opt/http"
|
||||||
|
matrix:
|
||||||
|
<<: *common-elements
|
||||||
|
command: "/opt/matrix"
|
||||||
|
mail:
|
||||||
|
<<: *common-elements
|
||||||
|
command: "/opt/mail"
|
||||||
|
|
||||||
|
networks:
|
||||||
|
default:
|
||||||
|
name: healthcheck
|
||||||
|
external: true
|
||||||
4
_ansible/roles/healthcheck/files/healthcheck@.service
Normal file
4
_ansible/roles/healthcheck/files/healthcheck@.service
Normal file
|
|
@ -0,0 +1,4 @@
|
||||||
|
[Service]
|
||||||
|
Type=oneshot
|
||||||
|
ExecStart=/usr/bin/docker compose run --rm %i
|
||||||
|
WorkingDirectory=/opt/services/healthcheck/
|
||||||
4
_ansible/roles/healthcheck/files/healthcheck@.timer
Normal file
4
_ansible/roles/healthcheck/files/healthcheck@.timer
Normal file
|
|
@ -0,0 +1,4 @@
|
||||||
|
[Timer]
|
||||||
|
OnCalendar=*:0/5
|
||||||
|
[Install]
|
||||||
|
WantedBy=timers.target
|
||||||
46
_ansible/roles/healthcheck/tasks/main.yml
Normal file
46
_ansible/roles/healthcheck/tasks/main.yml
Normal file
|
|
@ -0,0 +1,46 @@
|
||||||
|
---
|
||||||
|
- name: Deploy {{ svc.name }}
|
||||||
|
tags:
|
||||||
|
- healthcheck
|
||||||
|
block:
|
||||||
|
- import_tasks: steps/create-service-directory.yml
|
||||||
|
|
||||||
|
- name: Copy the docker-compose file
|
||||||
|
ansible.builtin.copy:
|
||||||
|
src: docker-compose.yml
|
||||||
|
dest: "{{ (service_path, 'docker-compose.yml') | path_join }}"
|
||||||
|
mode: '0644'
|
||||||
|
- name: Copy the Dockerfile
|
||||||
|
ansible.builtin.copy:
|
||||||
|
src: Dockerfile
|
||||||
|
dest: "{{ (service_path, 'Dockerfile') | path_join }}"
|
||||||
|
mode: '0644'
|
||||||
|
- name: Copy the data files
|
||||||
|
ansible.builtin.copy:
|
||||||
|
src: data
|
||||||
|
dest: "{{ service_path }}"
|
||||||
|
mode: '0755'
|
||||||
|
|
||||||
|
- name: Copy the system service
|
||||||
|
ansible.builtin.copy:
|
||||||
|
src: healthcheck@.service
|
||||||
|
dest: "/etc/systemd/system/healthcheck@.service"
|
||||||
|
mode: '0644'
|
||||||
|
become: true
|
||||||
|
- name: Copy the system timer
|
||||||
|
ansible.builtin.copy:
|
||||||
|
src: healthcheck@.timer
|
||||||
|
dest: "/etc/systemd/system/healthcheck@.timer"
|
||||||
|
mode: '0644'
|
||||||
|
become: true
|
||||||
|
|
||||||
|
- import_tasks: steps/template-service-env.yml
|
||||||
|
|
||||||
|
- name: Build service
|
||||||
|
ansible.builtin.command:
|
||||||
|
cmd: docker compose build --pull
|
||||||
|
chdir: "{{ service_path }}"
|
||||||
|
when:
|
||||||
|
- "'local-dev' != inventory_hostname"
|
||||||
|
register: cmd_result
|
||||||
|
changed_when: True
|
||||||
19
_ansible/roles/healthcheck/vars/main.yml
Normal file
19
_ansible/roles/healthcheck/vars/main.yml
Normal file
|
|
@ -0,0 +1,19 @@
|
||||||
|
svc:
|
||||||
|
name: healthcheck
|
||||||
|
|
||||||
|
svc_env:
|
||||||
|
USER_AGENT: "healthcheck-bot for serguzim.net"
|
||||||
|
|
||||||
|
HTTP_HC_UID: "{{ vault_healthcheck.hc_uid.http }}"
|
||||||
|
|
||||||
|
MATRIX_SERVER: "https://matrix.msrg.cc"
|
||||||
|
MATRIX_SERVER_FEDTESTER: "msrg.cc"
|
||||||
|
MATRIX_HC_UID: "{{ vault_healthcheck.hc_uid.matrix }}"
|
||||||
|
MATRIX_TOKEN: "{{ vault_healthcheck.matrix.token }}"
|
||||||
|
MATRIX_ROOM: "{{ vault_healthcheck.matrix.room }}"
|
||||||
|
|
||||||
|
MAIL_HC_UID: "{{ vault_healthcheck.hc_uid.mail }}"
|
||||||
|
MAIL_HOST: "{{ mailer.host }}"
|
||||||
|
MAIL_PORT: "{{ mailer.port }}"
|
||||||
|
MAIL_USER: "{{ vault_healthcheck.mailer.user }}"
|
||||||
|
MAIL_PASS: "{{ vault_healthcheck.mailer.pass }}"
|
||||||
|
|
@ -15,7 +15,7 @@ svc_env:
|
||||||
|
|
||||||
ALLOWED_HOSTS: recipes.serguzim.me
|
ALLOWED_HOSTS: recipes.serguzim.me
|
||||||
SECRET_KEY: "{{ vault_tandoor.secret_key }}"
|
SECRET_KEY: "{{ vault_tandoor.secret_key }}"
|
||||||
TIMEZONE: "{{ timezone }}"
|
TZ: "{{ timezone }}"
|
||||||
|
|
||||||
DB_ENGINE: django.db.backends.postgresql
|
DB_ENGINE: django.db.backends.postgresql
|
||||||
DB_OPTIONS: "{\"sslmode\": \"require\"}"
|
DB_OPTIONS: "{\"sslmode\": \"require\"}"
|
||||||
|
|
|
||||||
Reference in a new issue