From 0359a4ffa70218d82f089d73bd42dbcc13629c1a Mon Sep 17 00:00:00 2001
From: Tobias Reisinger <tobias@msrg.cc>
Date: Thu, 1 May 2025 01:28:24 +0200
Subject: [PATCH] Add basic lgtm-stack (WIP)
---
modules/services/authentik.tf | 17 ++
playbooks/roles/lgtm_stack/tasks/grafana.yml | 17 ++
playbooks/roles/lgtm_stack/tasks/main.yml | 32 ++++
.../lgtm_stack/templates/config.alloy.j2 | 24 +++
playbooks/roles/lgtm_stack/vars/main.yml | 173 ++++++++++++++++++
scripts/visualize.py | 1 +
services.auto.tfvars | 25 +++
7 files changed, 289 insertions(+)
create mode 100644 playbooks/roles/lgtm_stack/tasks/grafana.yml
create mode 100644 playbooks/roles/lgtm_stack/tasks/main.yml
create mode 100644 playbooks/roles/lgtm_stack/templates/config.alloy.j2
create mode 100644 playbooks/roles/lgtm_stack/vars/main.yml
diff --git a/modules/services/authentik.tf b/modules/services/authentik.tf
index 94644e9..323ec21 100644
--- a/modules/services/authentik.tf
+++ b/modules/services/authentik.tf
@@ -54,6 +54,23 @@ resource "authentik_group" "minio_users" {
users = []
}
+resource "authentik_group" "grafana_grafana_admins" {
+ name = "Grafana GrafanaAdmins"
+ users = [authentik_user.default.id]
+}
+
+resource "authentik_group" "grafana_admins" {
+ name = "Grafana Admins"
+}
+
+resource "authentik_group" "grafana_editors" {
+ name = "Grafana Editors"
+}
+
+resource "authentik_group" "grafana_viewers" {
+ name = "Grafana Viewers"
+}
+
resource "authentik_provider_oauth2" "service_providers" {
for_each = local.services_auth
diff --git a/playbooks/roles/lgtm_stack/tasks/grafana.yml b/playbooks/roles/lgtm_stack/tasks/grafana.yml
new file mode 100644
index 0000000..1417eb4
--- /dev/null
+++ b/playbooks/roles/lgtm_stack/tasks/grafana.yml
@@ -0,0 +1,17 @@
+- name: Set grafana datasources path
+ ansible.builtin.set_fact:
+ datasources_path: "{{ (service_path, 'datasources') | path_join }}"
+
+- name: Create datasources directory
+ ansible.builtin.file:
+ path: "{{ datasources_path }}"
+ state: directory
+ mode: "0755"
+
+- name: Template default datasources
+ ansible.builtin.template:
+ src: yml.j2
+ dest: "{{ (datasources_path, 'default.yaml') | path_join }}"
+ mode: "0644"
+ vars:
+ yml: "{{ lgtm_stack_grafana_datasources }}"
diff --git a/playbooks/roles/lgtm_stack/tasks/main.yml b/playbooks/roles/lgtm_stack/tasks/main.yml
new file mode 100644
index 0000000..3e4a70f
--- /dev/null
+++ b/playbooks/roles/lgtm_stack/tasks/main.yml
@@ -0,0 +1,32 @@
+---
+- name: Set common facts
+ ansible.builtin.import_tasks: tasks/set-default-facts.yml
+
+- name: Deploy {{ role_name }}
+ vars:
+ svc: "{{ lgtm_stack_svc }}"
+ env: "{{ lgtm_stack_env }}"
+ compose: "{{ lgtm_stack_compose }}"
+ block:
+ - name: Import prepare tasks for common service
+ ansible.builtin.import_tasks: tasks/prepare-common-service.yml
+
+ - name: Run grafana specific tasks
+ ansible.builtin.import_tasks: grafana.yml
+
+ - name: Template alloy config file
+ ansible.builtin.template:
+ src: config.alloy.j2
+ dest: "{{ (service_path, 'config.alloy') | path_join }}"
+ mode: "0644"
+
+ - name: Template mimir config file
+ ansible.builtin.template:
+ src: yml.j2
+ dest: "{{ (service_path, 'mimir.yaml') | path_join }}"
+ mode: "0644"
+ vars:
+ yml: "{{ lgtm_stack_mimir_yml }}"
+
+ - name: Import start tasks for common service
+ ansible.builtin.import_tasks: tasks/start-common-service.yml
diff --git a/playbooks/roles/lgtm_stack/templates/config.alloy.j2 b/playbooks/roles/lgtm_stack/templates/config.alloy.j2
new file mode 100644
index 0000000..75d869c
--- /dev/null
+++ b/playbooks/roles/lgtm_stack/templates/config.alloy.j2
@@ -0,0 +1,24 @@
+logging {
+ level = "info"
+ format = "logfmt"
+}
+
+prometheus.exporter.self "alloy" {}
+prometheus.scrape "alloy" {
+ targets = prometheus.exporter.self.alloy.targets
+ forward_to = [prometheus.remote_write.mimir.receiver]
+}
+
+// prometheus.exporter.node_exporter "node_exporter" {}
+prometheus.scrape "node_exporter" {
+ targets = [
+ {"__address__" = "node_exporter:9100", "job" = "node_exporter"},
+ ]
+ forward_to = [prometheus.remote_write.mimir.receiver]
+}
+
+prometheus.remote_write "mimir" {
+ endpoint {
+ url = "https://{{ lgtm_stack_mimir_domain }}/api/v1/push"
+ }
+}
diff --git a/playbooks/roles/lgtm_stack/vars/main.yml b/playbooks/roles/lgtm_stack/vars/main.yml
new file mode 100644
index 0000000..2e3a80a
--- /dev/null
+++ b/playbooks/roles/lgtm_stack/vars/main.yml
@@ -0,0 +1,173 @@
+---
+lgtm_stack_domain: "{{ all_services | service_get_domain(role_name) }}"
+lgtm_stack_mimir_domain: mimir.serguzim.me
+lgtm_stack_alloy_domain: alloy.serguzim.me
+
+lgtm_stack_svc:
+ domain: "{{ lgtm_stack_domain }}"
+ port: 3000
+ extra_svcs:
+ - domain: "{{ lgtm_stack_alloy_domain }}"
+ docker_host: lgtm_stack_alloy
+ port: 12345
+ caddy_extra: import vpn_only
+ - domain: "{{ lgtm_stack_mimir_domain }}"
+ docker_host: lgtm_stack_mimir
+ port: 9009
+ caddy_extra: import vpn_only
+
+lgtm_stack_env:
+
+ GF_DEFAULT_INSTANCE_NAME: "{{ lgtm_stack_domain }}"
+ GF_SERVER_PROTOCOL: "http"
+ GF_SERVER_DOMAIN: "{{ lgtm_stack_domain }}"
+ GF_SERVER_ROOT_URL: "https://{{ lgtm_stack_domain }}/"
+
+ GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION: true
+ GF_SECURITY_ADMIN_USER: "{{ admin_email }}"
+ GF_SECURITY_SECRET_KEY: "{{ vault_lgtm_stack.grafana.secret_key }}"
+ GF_SECURITY_COOKIE_SECURE: true
+ GF_SECURITY_COOKIE_SAMESITE: "strict"
+
+ GF_PLUGINS_PLUGIN_ADMIN_ENABLED: true
+
+ GF_DATABASE_TYPE: "postgres"
+ GF_DATABASE_HOST: "{{ postgres.host }}"
+ GF_DATABASE_NAME: "{{ opentofu.postgresql_data.lgtm_stack.database }}"
+ GF_DATABASE_USER: "{{ opentofu.postgresql_data.lgtm_stack.user }}"
+ GF_DATABASE_PASSWORD: "{{ opentofu.postgresql_data.lgtm_stack.pass }}"
+ GF_DATABASE_SSL_MODE: "verify-full"
+
+ GF_USERS_ALLOW_SIGN_UP: false
+ GF_AUTH_DISABLE_LOGIN_FORM: true
+ GF_SIGNOUT_REDIRECT_URL: "https://{{ lgtm_stack_domain }}/"
+ GF_OAUTH_AUTO_LOGIN: true
+ GF_AUTH_ANONYMOUS_ENABLED: true
+ GF_AUTH_ANONYMOUS_ORG_NAME: "Main Org."
+ GF_AUTH_ANONYMOUS_ORG_ROLE: "Viewer"
+ GF_AUTH_GENERIC_OAUTH_ENABLED: true
+ GF_AUTH_GENERIC_OAUTH_NAME: "auth.serguzim.me"
+ GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP: true
+ GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH: "contains(groups, 'Grafana GrafanaAdmins') && 'GrafanaAdmin' || contains(groups, 'Grafana Admins') && 'Admin' || contains(groups, 'Grafana Editors') && 'Editor' || 'Viewer'"
+ GF_AUTH_GENERIC_OAUTH_ALLOW_ASSIGN_GRAFANA_ADMIN: true
+ GF_AUTH_GENERIC_OAUTH_CLIENT_ID: "{{ opentofu.authentik_data.lgtm_stack.client_id }}"
+ GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: "{{ opentofu.authentik_data.lgtm_stack.client_secret }}"
+ GF_AUTH_GENERIC_OAUTH_SCOPES: "openid profile email"
+ GF_AUTH_GENERIC_OAUTH_AUTH_URL: "https://auth.serguzim.me/application/o/authorize/"
+ GF_AUTH_GENERIC_OAUTH_TOKEN_URL: "https://auth.serguzim.me/application/o/token/"
+ GF_AUTH_GENERIC_OAUTH_API_URL: "https://auth.serguzim.me/application/o/userinfo/"
+ GF_AUTH_SIGNOUT_REDIRECT_URL: "{{ (opentofu.authentik_data.lgtm_stack.base_url, 'end-session') | path_join }}/"
+ GF_AUTH_OAUTH_AUTO_LOGIN: true
+
+ GF_SMTP_ENABLED: true
+ GF_SMTP_HOST: "{{ mailer.host }}:{{ mailer.port }}"
+ GF_SMTP_USER: "{{ opentofu.mailcow_data.lgtm_stack.address }}"
+ GF_SMTP_PASSWORD: "{{ opentofu.mailcow_data.lgtm_stack.password }}"
+ GF_SMTP_FROM_ADDRESS: "{{ opentofu.mailcow_data.lgtm_stack.address }}"
+ GF_SMTP_FROM_NAME: "Monitoring"
+
+lgtm_stack_grafana_datasources:
+ apiVersion: 1
+
+ deleteDatasources:
+ - name: Mimir
+
+ datasources:
+ - name: Mimir
+ type: prometheus
+ access: proxy
+ orgId: 1
+ url: "https://{{ lgtm_stack_mimir_domain }}/prometheus"
+ version: 1
+ editable: true
+ jsonData:
+ timeInterval: 60s
+ prometheusType: Mimir
+
+lgtm_stack_mimir_yml:
+ multitenancy_enabled: false
+ target: all
+
+ common:
+ storage:
+ backend: s3
+ s3:
+ endpoint: "{{ opentofu.scaleway_data.mimir_blocks.api_endpoint | regex_replace('^https://', '') }}"
+ region: "{{ opentofu.scaleway_data.mimir_blocks.region }}"
+ access_key_id: "{{ opentofu.scaleway_data.mimir_blocks.access_key }}"
+ secret_access_key: "{{ opentofu.scaleway_data.mimir_blocks.secret_key }}"
+ blocks_storage:
+ s3:
+ bucket_name: "{{ opentofu.scaleway_data.mimir_blocks.name }}"
+ alertmanager_storage:
+ s3:
+ bucket_name: "{{ opentofu.scaleway_data.mimir_alertmanager.name }}"
+ ruler_storage:
+ s3:
+ bucket_name: "{{ opentofu.scaleway_data.mimir_ruler.name }}"
+
+ server:
+ http_listen_port: 9009
+
+ # Configure the server to allow messages up to 100MB.
+ grpc_server_max_recv_msg_size: 104857600
+ grpc_server_max_send_msg_size: 104857600
+ grpc_server_max_concurrent_streams: 1000
+
+ ingester:
+ ring:
+ replication_factor: 1
+
+lgtm_stack_compose:
+ watchtower: update
+ image: grafana/grafana-oss
+ volumes:
+ - ./datasources:/etc/grafana/provisioning/datasources
+ - grafana-data:/var/lib/grafana
+ file:
+ services:
+ alloy:
+ image: grafana/alloy:latest
+ restart: always
+ volumes:
+ - ./config.alloy:/etc/alloy/config.alloy:ro
+ command:
+ - run
+ - /etc/alloy/config.alloy
+ - --storage.path=/var/lib/alloy/data
+ - --server.http.listen-addr=0.0.0.0:12345
+ - --stability.level=experimental
+ networks:
+ apps:
+ aliases:
+ - lgtm_stack_alloy
+ default:
+ node_exporter:
+ image: prom/node-exporter
+ hostname: "{{ inventory_hostname }}"
+ restart: always
+ volumes:
+ - /proc:/host/proc:ro
+ - /sys:/host/sys:ro
+ command:
+ - '--path.procfs=/host/proc'
+ - '--path.sysfs=/host/sys'
+ - '--collector.filesystem.ignored-mount-points'
+ - '^/(sys|proc|dev|host|etc|rootfs/var/lib/docker/containers|rootfs/var/lib/docker/overlay2|rootfs/run/docker/netns|rootfs/var/lib/docker/aufs)($$|/)'
+ networks:
+ default:
+
+ mimir:
+ image: grafana/mimir:latest
+ restart: always
+ command:
+ - -config.file=/etc/mimir-config/mimir.yaml
+ volumes:
+ - ./mimir.yaml:/etc/mimir-config/mimir.yaml:ro
+ networks:
+ default:
+ apps:
+ aliases:
+ - lgtm_stack_mimir
+ volumes:
+ grafana-data:
diff --git a/scripts/visualize.py b/scripts/visualize.py
index d288d15..7cbb808 100755
--- a/scripts/visualize.py
+++ b/scripts/visualize.py
@@ -13,6 +13,7 @@ icon_overrides = {
"forgejo_runner": "forgejo",
"healthcheck": "healthchecks",
"lego": "lets-encrypt",
+ "lgtm_stack": "grafana",
"mailcowdockerized": "mailcow",
"minecraft_3": "minecraft",
"reitanlage_oranienburg": "grav",
diff --git a/services.auto.tfvars b/services.auto.tfvars
index 869ae8b..cf82f07 100644
--- a/services.auto.tfvars
+++ b/services.auto.tfvars
@@ -403,6 +403,31 @@ services = {
s3 = false
},
+ "lgtm_stack" = {
+ host = "node001"
+ dns = [
+ {
+ domain = "monitoring.serguzim.me"
+ },
+ {
+ domain = "alloy.serguzim.me"
+ name = "alloy"
+ vpn = true
+ },
+ {
+ domain = "mimir.serguzim.me"
+ name = "mimir"
+ vpn = true
+ }
+ ]
+ auth = true
+ auth_redirects = ["https://monitoring.serguzim.me/login/generic_oauth"]
+ database = true
+ s3 = true
+ s3_buckets = ["mimir_blocks", "mimir_alertmanager", "mimir_ruler"]
+ mail = "monitoring"
+ }
+
"minecraft_3" = {
host = ""
dns = [