Add basic lgtm-stack (WIP)

modules/services/authentik.tf

@@ -54,6 +54,23 @@ resource "authentik_group" "minio_users" {
   users        = []
 }
 
+resource "authentik_group" "grafana_grafana_admins" {
+  name    = "Grafana GrafanaAdmins"
+  users   = [authentik_user.default.id]
+}
+
+resource "authentik_group" "grafana_admins" {
+  name    = "Grafana Admins"
+}
+
+resource "authentik_group" "grafana_editors" {
+  name    = "Grafana Editors"
+}
+
+resource "authentik_group" "grafana_viewers" {
+  name    = "Grafana Viewers"
+}
+
 
 resource "authentik_provider_oauth2" "service_providers" {
   for_each              = local.services_auth

playbooks/roles/lgtm_stack/tasks/grafana.yml

@@ -0,0 +1,17 @@
+- name: Set grafana datasources path
+  ansible.builtin.set_fact:
+    datasources_path: "{{ (service_path, 'datasources') | path_join }}"
+
+- name: Create datasources directory
+  ansible.builtin.file:
+    path: "{{ datasources_path }}"
+    state: directory
+    mode: "0755"
+
+- name: Template default datasources
+  ansible.builtin.template:
+    src: yml.j2
+    dest: "{{ (datasources_path, 'default.yaml') | path_join }}"
+    mode: "0644"
+  vars:
+    yml: "{{ lgtm_stack_grafana_datasources }}"

playbooks/roles/lgtm_stack/tasks/main.yml

@@ -0,0 +1,32 @@
+---
+- name: Set common facts
+  ansible.builtin.import_tasks: tasks/set-default-facts.yml
+
+- name: Deploy {{ role_name }}
+  vars:
+    svc: "{{ lgtm_stack_svc }}"
+    env: "{{ lgtm_stack_env }}"
+    compose: "{{ lgtm_stack_compose }}"
+  block:
+    - name: Import prepare tasks for common service
+      ansible.builtin.import_tasks: tasks/prepare-common-service.yml
+
+    - name: Run grafana specific tasks
+      ansible.builtin.import_tasks: grafana.yml
+
+    - name: Template alloy config file
+      ansible.builtin.template:
+        src: config.alloy.j2
+        dest: "{{ (service_path, 'config.alloy') | path_join }}"
+        mode: "0644"
+
+    - name: Template mimir config file
+      ansible.builtin.template:
+        src: yml.j2
+        dest: "{{ (service_path, 'mimir.yaml') | path_join }}"
+        mode: "0644"
+      vars:
+        yml: "{{ lgtm_stack_mimir_yml }}"
+
+    - name: Import start tasks for common service
+      ansible.builtin.import_tasks: tasks/start-common-service.yml

playbooks/roles/lgtm_stack/templates/config.alloy.j2

@@ -0,0 +1,24 @@
+logging {
+  level  = "info"
+  format = "logfmt"
+}
+
+prometheus.exporter.self "alloy" {}
+prometheus.scrape "alloy" {
+	targets    = prometheus.exporter.self.alloy.targets
+	forward_to = [prometheus.remote_write.mimir.receiver]
+}
+
+// prometheus.exporter.node_exporter "node_exporter" {}
+prometheus.scrape "node_exporter" {
+  targets = [
+    {"__address__" = "node_exporter:9100", "job" = "node_exporter"},
+  ]
+  forward_to = [prometheus.remote_write.mimir.receiver]
+}
+
+prometheus.remote_write "mimir" {
+	endpoint {
+		url = "https://{{ lgtm_stack_mimir_domain }}/api/v1/push"
+	}
+}

playbooks/roles/lgtm_stack/vars/main.yml

@@ -0,0 +1,173 @@
+---
+lgtm_stack_domain: "{{ all_services | service_get_domain(role_name) }}"
+lgtm_stack_mimir_domain: mimir.serguzim.me
+lgtm_stack_alloy_domain: alloy.serguzim.me
+
+lgtm_stack_svc:
+  domain: "{{ lgtm_stack_domain }}"
+  port: 3000
+  extra_svcs:
+    - domain: "{{ lgtm_stack_alloy_domain }}"
+      docker_host: lgtm_stack_alloy
+      port: 12345
+      caddy_extra: import vpn_only
+    - domain: "{{ lgtm_stack_mimir_domain }}"
+      docker_host: lgtm_stack_mimir
+      port: 9009
+      caddy_extra: import vpn_only
+
+lgtm_stack_env:
+
+  GF_DEFAULT_INSTANCE_NAME: "{{ lgtm_stack_domain }}"
+  GF_SERVER_PROTOCOL: "http"
+  GF_SERVER_DOMAIN: "{{ lgtm_stack_domain }}"
+  GF_SERVER_ROOT_URL: "https://{{ lgtm_stack_domain }}/"
+
+  GF_SECURITY_DISABLE_INITIAL_ADMIN_CREATION: true
+  GF_SECURITY_ADMIN_USER: "{{ admin_email }}"
+  GF_SECURITY_SECRET_KEY: "{{ vault_lgtm_stack.grafana.secret_key }}"
+  GF_SECURITY_COOKIE_SECURE: true
+  GF_SECURITY_COOKIE_SAMESITE: "strict"
+
+  GF_PLUGINS_PLUGIN_ADMIN_ENABLED: true
+
+  GF_DATABASE_TYPE: "postgres"
+  GF_DATABASE_HOST: "{{ postgres.host }}"
+  GF_DATABASE_NAME: "{{ opentofu.postgresql_data.lgtm_stack.database }}"
+  GF_DATABASE_USER: "{{ opentofu.postgresql_data.lgtm_stack.user }}"
+  GF_DATABASE_PASSWORD: "{{ opentofu.postgresql_data.lgtm_stack.pass }}"
+  GF_DATABASE_SSL_MODE: "verify-full"
+
+  GF_USERS_ALLOW_SIGN_UP: false
+  GF_AUTH_DISABLE_LOGIN_FORM: true
+  GF_SIGNOUT_REDIRECT_URL: "https://{{ lgtm_stack_domain }}/"
+  GF_OAUTH_AUTO_LOGIN: true
+  GF_AUTH_ANONYMOUS_ENABLED: true
+  GF_AUTH_ANONYMOUS_ORG_NAME: "Main Org."
+  GF_AUTH_ANONYMOUS_ORG_ROLE: "Viewer"
+  GF_AUTH_GENERIC_OAUTH_ENABLED: true
+  GF_AUTH_GENERIC_OAUTH_NAME: "auth.serguzim.me"
+  GF_AUTH_GENERIC_OAUTH_ALLOW_SIGN_UP: true
+  GF_AUTH_GENERIC_OAUTH_ROLE_ATTRIBUTE_PATH: "contains(groups, 'Grafana GrafanaAdmins') && 'GrafanaAdmin' || contains(groups, 'Grafana Admins') && 'Admin' || contains(groups, 'Grafana Editors') && 'Editor' || 'Viewer'"
+  GF_AUTH_GENERIC_OAUTH_ALLOW_ASSIGN_GRAFANA_ADMIN: true
+  GF_AUTH_GENERIC_OAUTH_CLIENT_ID: "{{ opentofu.authentik_data.lgtm_stack.client_id }}"
+  GF_AUTH_GENERIC_OAUTH_CLIENT_SECRET: "{{ opentofu.authentik_data.lgtm_stack.client_secret }}"
+  GF_AUTH_GENERIC_OAUTH_SCOPES: "openid profile email"
+  GF_AUTH_GENERIC_OAUTH_AUTH_URL: "https://auth.serguzim.me/application/o/authorize/"
+  GF_AUTH_GENERIC_OAUTH_TOKEN_URL: "https://auth.serguzim.me/application/o/token/"
+  GF_AUTH_GENERIC_OAUTH_API_URL: "https://auth.serguzim.me/application/o/userinfo/"
+  GF_AUTH_SIGNOUT_REDIRECT_URL: "{{ (opentofu.authentik_data.lgtm_stack.base_url, 'end-session') | path_join }}/"
+  GF_AUTH_OAUTH_AUTO_LOGIN: true
+
+  GF_SMTP_ENABLED: true
+  GF_SMTP_HOST: "{{ mailer.host }}:{{ mailer.port }}"
+  GF_SMTP_USER: "{{ opentofu.mailcow_data.lgtm_stack.address }}"
+  GF_SMTP_PASSWORD: "{{ opentofu.mailcow_data.lgtm_stack.password }}"
+  GF_SMTP_FROM_ADDRESS: "{{ opentofu.mailcow_data.lgtm_stack.address }}"
+  GF_SMTP_FROM_NAME: "Monitoring"
+
+lgtm_stack_grafana_datasources:
+  apiVersion: 1
+
+  deleteDatasources:
+    - name: Mimir
+
+  datasources:
+    - name: Mimir
+      type: prometheus
+      access: proxy
+      orgId: 1
+      url: "https://{{ lgtm_stack_mimir_domain }}/prometheus"
+      version: 1
+      editable: true
+      jsonData:
+        timeInterval: 60s
+        prometheusType: Mimir
+
+lgtm_stack_mimir_yml:
+  multitenancy_enabled: false
+  target: all
+
+  common:
+    storage:
+      backend: s3
+      s3:
+        endpoint: "{{ opentofu.scaleway_data.mimir_blocks.api_endpoint |  regex_replace('^https://', '') }}"
+        region: "{{ opentofu.scaleway_data.mimir_blocks.region }}"
+        access_key_id: "{{ opentofu.scaleway_data.mimir_blocks.access_key }}"
+        secret_access_key: "{{ opentofu.scaleway_data.mimir_blocks.secret_key }}"
+  blocks_storage:
+    s3:
+      bucket_name: "{{ opentofu.scaleway_data.mimir_blocks.name }}"
+  alertmanager_storage:
+    s3:
+      bucket_name: "{{ opentofu.scaleway_data.mimir_alertmanager.name }}"
+  ruler_storage:
+    s3:
+      bucket_name: "{{ opentofu.scaleway_data.mimir_ruler.name }}"
+
+  server:
+    http_listen_port: 9009
+
+    # Configure the server to allow messages up to 100MB.
+    grpc_server_max_recv_msg_size: 104857600
+    grpc_server_max_send_msg_size: 104857600
+    grpc_server_max_concurrent_streams: 1000
+
+  ingester:
+    ring:
+      replication_factor: 1
+
+lgtm_stack_compose:
+  watchtower: update
+  image: grafana/grafana-oss
+  volumes:
+    - ./datasources:/etc/grafana/provisioning/datasources
+    - grafana-data:/var/lib/grafana
+  file:
+    services:
+      alloy:
+        image: grafana/alloy:latest
+        restart: always
+        volumes:
+          - ./config.alloy:/etc/alloy/config.alloy:ro
+        command:
+          - run
+          - /etc/alloy/config.alloy
+          - --storage.path=/var/lib/alloy/data
+          - --server.http.listen-addr=0.0.0.0:12345
+          - --stability.level=experimental
+        networks:
+          apps:
+            aliases:
+            - lgtm_stack_alloy
+          default:
+      node_exporter:
+        image: prom/node-exporter
+        hostname: "{{ inventory_hostname }}"
+        restart: always
+        volumes:
+          - /proc:/host/proc:ro
+          - /sys:/host/sys:ro
+        command:
+          - '--path.procfs=/host/proc'
+          - '--path.sysfs=/host/sys'
+          - '--collector.filesystem.ignored-mount-points'
+          - '^/(sys|proc|dev|host|etc|rootfs/var/lib/docker/containers|rootfs/var/lib/docker/overlay2|rootfs/run/docker/netns|rootfs/var/lib/docker/aufs)($$|/)'
+        networks:
+          default:
+
+      mimir:
+        image: grafana/mimir:latest
+        restart: always
+        command:
+          - -config.file=/etc/mimir-config/mimir.yaml
+        volumes:
+          - ./mimir.yaml:/etc/mimir-config/mimir.yaml:ro
+        networks:
+          default:
+          apps:
+            aliases:
+            - lgtm_stack_mimir
+    volumes:
+      grafana-data:

scripts/visualize.py

@@ -13,6 +13,7 @@ icon_overrides = {
     "forgejo_runner": "forgejo",
     "healthcheck": "healthchecks",
     "lego": "lets-encrypt",
+    "lgtm_stack": "grafana",
     "mailcowdockerized": "mailcow",
     "minecraft_3": "minecraft",
     "reitanlage_oranienburg": "grav",

services.auto.tfvars

@@ -403,6 +403,31 @@ services = {
     s3 = false
   },
 
+  "lgtm_stack" = {
+    host = "node001"
+    dns = [
+      {
+        domain = "monitoring.serguzim.me"
+      },
+      {
+        domain = "alloy.serguzim.me"
+        name = "alloy"
+        vpn = true
+      },
+      {
+        domain = "mimir.serguzim.me"
+        name = "mimir"
+        vpn = true
+      }
+    ]
+    auth = true
+    auth_redirects = ["https://monitoring.serguzim.me/login/generic_oauth"]
+    database = true
+    s3 = true
+    s3_buckets = ["mimir_blocks", "mimir_alertmanager", "mimir_ruler"]
+    mail = "monitoring"
+  }
+
   "minecraft_3" = {
     host = ""
     dns = [