Refactor node-exporter to every node

playbooks/roles/lgtm_stack/tasks/main.yml

@@ -19,6 +19,12 @@
         src: config.alloy.j2
         dest: "{{ (service_path, 'config.alloy') | path_join }}"
         mode: "0644"
+      register: cmd_result
+
+    - name: Set the docker force-recreate flag
+      ansible.builtin.set_fact:
+        docker_force_recreate: --force-recreate
+      when: cmd_result.changed # noqa: no-handler We need to handle the restart per service. Handlers don't support variables.
 
     - name: Template mimir config file
       ansible.builtin.template:
@@ -27,6 +33,12 @@
         mode: "0644"
       vars:
         yml: "{{ lgtm_stack_mimir_yml }}"
+      register: cmd_result
+
+    - name: Set the docker force-recreate flag
+      ansible.builtin.set_fact:
+        docker_force_recreate: --force-recreate
+      when: cmd_result.changed # noqa: no-handler We need to handle the restart per service. Handlers don't support variables.
 
     - name: Import start tasks for common service
       ansible.builtin.import_tasks: tasks/start-common-service.yml

playbooks/roles/lgtm_stack/templates/config.alloy.j2

@@ -9,10 +9,11 @@ prometheus.scrape "alloy" {
 	forward_to = [prometheus.remote_write.mimir.receiver]
 }
 
-// prometheus.exporter.node_exporter "node_exporter" {}
 prometheus.scrape "node_exporter" {
   targets = [
-    {"__address__" = "node_exporter:9100", "job" = "node_exporter"},
+{% for host_data in opentofu.hosts.values() %}
+    {"__address__" = "{{ host_data.fqdn_vpn }}:9100", "job" = "node_exporter"},
+{% endfor %}
   ]
   forward_to = [prometheus.remote_write.mimir.receiver]
 }

playbooks/roles/lgtm_stack/vars/main.yml

@@ -42,7 +42,7 @@ lgtm_stack_env:
   GF_AUTH_DISABLE_LOGIN_FORM: true
   GF_SIGNOUT_REDIRECT_URL: "https://{{ lgtm_stack_domain }}/"
   GF_OAUTH_AUTO_LOGIN: true
-  GF_AUTH_ANONYMOUS_ENABLED: true
+  GF_AUTH_ANONYMOUS_ENABLED: false
   GF_AUTH_ANONYMOUS_ORG_NAME: "Main Org."
   GF_AUTH_ANONYMOUS_ORG_ROLE: "Viewer"
   GF_AUTH_GENERIC_OAUTH_ENABLED: true
@@ -142,20 +142,6 @@ lgtm_stack_compose:
             aliases:
             - lgtm_stack_alloy
           default:
-      node_exporter:
-        image: prom/node-exporter
-        hostname: "{{ inventory_hostname }}"
-        restart: always
-        volumes:
-          - /proc:/host/proc:ro
-          - /sys:/host/sys:ro
-        command:
-          - '--path.procfs=/host/proc'
-          - '--path.sysfs=/host/sys'
-          - '--collector.filesystem.ignored-mount-points'
-          - '^/(sys|proc|dev|host|etc|rootfs/var/lib/docker/containers|rootfs/var/lib/docker/overlay2|rootfs/run/docker/netns|rootfs/var/lib/docker/aufs)($$|/)'
-        networks:
-          default:
 
       mimir:
         image: grafana/mimir:latest

playbooks/roles/node_exporter/tasks/main.yml

@@ -0,0 +1,4 @@
+---
+- name: Deploy {{ role_name }}
+  ansible.builtin.import_role:
+    name: prometheus.prometheus.node_exporter

playbooks/roles/node_exporter/vars/main.yml

@@ -0,0 +1,6 @@
+---
+node_exporter_disabled_collectors:
+  - "textfile"
+
+node_exporter_enabled_collectors:
+  - "systemd"

requirements.yml

@@ -0,0 +1,2 @@
+collections:
+  - name: prometheus.prometheus

scripts/visualize.py

@@ -16,6 +16,7 @@ icon_overrides = {
     "lgtm_stack": "grafana",
     "mailcowdockerized": "mailcow",
     "minecraft_3": "minecraft",
+    "node_exporter": "prometheus",
     "reitanlage_oranienburg": "grav",
     "tandoor": "tandoor-recipes",
     "tinytinyrss": "tiny-tiny-rss",

services.auto.tfvars

@@ -485,6 +485,13 @@ services = {
     s3 = false
   },
 
+  "node_exporter" = {
+    host = "*"
+    auth = false
+    database = false
+    s3 = false
+  },
+
   "ntfy" = {
     host = "node001"
     dns = [{