Add prometheus metrics to alloy

playbooks/roles/lgtm_stack/templates/config.alloy.j2

@@ -3,23 +3,66 @@ logging {
   format = "logfmt"
 }
 
+prometheus.remote_write "mimir" {
+    endpoint {
+        url = "https://{{ lgtm_stack_mimir_domain }}/api/v1/push"
+    }
+}
+
 prometheus.exporter.self "alloy" {}
 prometheus.scrape "alloy" {
 	targets    = prometheus.exporter.self.alloy.targets
 	forward_to = [prometheus.remote_write.mimir.receiver]
 }
 
-prometheus.scrape "node_exporter" {
-  targets = [
-{% for host_data in opentofu.hosts.values() %}
-    {"__address__" = "{{ host_data.fqdn_vpn }}:9100", "job" = "node_exporter"},
-{% endfor %}
-  ]
-  forward_to = [prometheus.remote_write.mimir.receiver]
+prometheus.exporter.postgres "default" {
+    data_source_names = ["postgresql://{{ svc.postgresql_collector.user }}:{{ svc.postgresql_collector.pass }}@{{ svc.postgresql_collector.host }}:{{ svc.postgresql_collector.port }}/{{ svc.postgresql_collector.database }}?sslmode=verify-full"]
+
+    autodiscovery {
+        enabled = true
+    }
+}
+prometheus.scrape "postgres" {
+    targets    = prometheus.exporter.postgres.default.targets
+    forward_to = [prometheus.remote_write.mimir.receiver]
 }
 
-prometheus.remote_write "mimir" {
-	endpoint {
-		url = "https://{{ lgtm_stack_mimir_domain }}/api/v1/push"
-	}
+prometheus.scrape "node_exporter" {
+    targets = [
+{% for host_data in opentofu.hosts.values() %}
+        {"__address__" = "{{ host_data.fqdn_vpn }}:9100", "instance" = "{{ host_data.hostname }}"},
+{% endfor %}
+    ]
+    forward_to = [prometheus.remote_write.mimir.receiver]
 }
+
+prometheus.scrape "caddy" {
+    targets = [
+{% for host_data in opentofu.hosts.values() %}
+        {"__address__" = "{{ host_data.fqdn_vpn }}:2019", "instance" = "{{ host_data.hostname }}"},
+{% endfor %}
+    ]
+    forward_to = [prometheus.remote_write.mimir.receiver]
+}
+
+
+{% for job in lgtm_stack_alloy_jobs %}
+
+prometheus.scrape "{{ job.name }}" {
+    targets = [
+{% for target in job.targets %}
+        {
+            "__address__" = "{{ target.address }}",
+            "__metrics_path__" = "{{ target.path }}",
+            "__scheme__" = "https",
+            {% if 'job' in target %}"job" = "{{ target.job }}",{% endif %}
+            {% if 'instance' in target %}"instance" = "{{ target.instance }}",{% endif %}
+        },
+{% endfor %}
+    ]
+
+    scrape_interval = "{{ job.scrape_interval }}"
+    forward_to = [prometheus.remote_write.mimir.receiver]
+}
+
+{% endfor %}

playbooks/roles/lgtm_stack/vars/main.yml

@@ -3,6 +3,9 @@ lgtm_stack_domain: "{{ all_services | service_get_domain(role_name) }}"
 lgtm_stack_mimir_domain: mimir.serguzim.me
 lgtm_stack_alloy_domain: alloy.serguzim.me
 
+lgtm_stack_alloy_jobs: "{{ all_services | services_to_alloy() }}"
+
+
 lgtm_stack_svc:
   domain: "{{ lgtm_stack_domain }}"
   port: 3000
@@ -15,6 +18,12 @@ lgtm_stack_svc:
       docker_host: lgtm_stack_mimir
       port: 9009
       caddy_extra: import vpn_only
+  postgresql_collector:
+    host: "{{ postgres.host }}"
+    port: "{{ postgres.port }}"
+    user: "{{ opentofu.postgresql_metrics_collector.user }}"
+    pass: "{{ opentofu.postgresql_metrics_collector.pass }}"
+    database: "{{ opentofu.postgresql_metrics_collector.database }}"
 
 lgtm_stack_env: