diff --git a/modules/services/output.tf b/modules/services/output.tf
index 53816ba..c63d091 100644
--- a/modules/services/output.tf
+++ b/modules/services/output.tf
@@ -1,9 +1,9 @@
output "authentik_data" {
value = {
for key in keys(authentik_application.service_applications) : key => {
- "base_url" = "${var.authentik_url}/application/o/${authentik_application.service_applications[key].slug}"
- "client_id" = authentik_provider_oauth2.service_providers[key].client_id
- "client_secret" = authentik_provider_oauth2.service_providers[key].client_secret
+ base_url = "${var.authentik_url}/application/o/${authentik_application.service_applications[key].slug}"
+ client_id = authentik_provider_oauth2.service_providers[key].client_id
+ client_secret = authentik_provider_oauth2.service_providers[key].client_secret
}
}
sensitive = true
@@ -12,19 +12,28 @@ output "authentik_data" {
output "postgresql_data" {
value = {
for key in keys(postgresql_database.service_databases) : key => {
- "user" = postgresql_role.service_roles[key].name
- "pass" = postgresql_role.service_roles[key].password
- "database" = postgresql_database.service_databases[key].name
+ user = postgresql_role.service_roles[key].name
+ pass = postgresql_role.service_roles[key].password
+ database = postgresql_database.service_databases[key].name
}
}
sensitive = true
}
+output "postgresql_metrics_collector" {
+ value = {
+ user = postgresql_role.metrics_collector_role.name
+ pass = postgresql_role.metrics_collector_role.password
+ database = postgresql_database.metrics_collector_database.name
+ }
+ sensitive = true
+}
+
output "mailcow_data" {
value = {
for key in keys(mailcow_mailbox.services) : key => {
- "address" = mailcow_mailbox.services[key].address
- "password" = mailcow_mailbox.services[key].password
+ address = mailcow_mailbox.services[key].address
+ password = mailcow_mailbox.services[key].password
}
}
sensitive = true
diff --git a/modules/services/postgresql.tf b/modules/services/postgresql.tf
index ca1ab84..ee3040a 100644
--- a/modules/services/postgresql.tf
+++ b/modules/services/postgresql.tf
@@ -16,3 +16,21 @@ resource "postgresql_database" "service_databases" {
name = each.key
owner = postgresql_role.service_roles[each.key].name
}
+
+resource "random_password" "postgresql_metrics_collector_password" {
+ length = 32
+ special = false
+}
+
+resource "postgresql_role" "metrics_collector_role" {
+ name = "metrics_collector"
+ login = true
+ password = random_password.postgresql_metrics_collector_password.result
+ search_path = ["postgres_exporter", "pg_catalog"]
+ roles = ["pg_monitor", "pg_read_all_stats"]
+}
+
+resource "postgresql_database" "metrics_collector_database" {
+ name = "metrics_collector"
+ owner = postgresql_role.metrics_collector_role.name
+}
diff --git a/output.tf b/output.tf
index 5b4e5ca..61c3440 100644
--- a/output.tf
+++ b/output.tf
@@ -27,6 +27,11 @@ output "postgresql_data" {
sensitive = true
}
+output "postgresql_metrics_collector" {
+ value = module.services.postgresql_metrics_collector
+ sensitive = true
+}
+
output "postgresql" {
value = {
"host" = var.postgresql_host
diff --git a/playbooks/filter_plugins/alloy.py b/playbooks/filter_plugins/alloy.py
new file mode 100644
index 0000000..72ac4c0
--- /dev/null
+++ b/playbooks/filter_plugins/alloy.py
@@ -0,0 +1,44 @@
+def transfer_optional_param(source, target, name, target_name=None):
+ if param := source.get(name):
+ target[target_name or name] = param
+
+class FilterModule(object):
+ def filters(self):
+ return {
+ 'services_to_alloy': self.services_to_alloy,
+ }
+
+ def services_to_alloy(self, services):
+ result = []
+
+ for name, service in services.items():
+ if not bool(service.get("host")):
+ continue
+
+ if targets := service.get("metrics") or []:
+ job = {
+ "name": name,
+ "targets": [],
+ "scrape_interval": "60s",
+ }
+
+ for target in targets:
+
+ address = target.get("address") or service["dns"][0]['domain']
+
+ transfer_optional_param(target, job, "interval", "scrape_interval")
+
+ new_target = {
+ "address": address,
+ "path": target["path"],
+ "instance": name
+ }
+
+ transfer_optional_param(target, new_target, "instance")
+ transfer_optional_param(target, new_target, "job")
+
+ job["targets"].append(new_target)
+
+ result.append(job)
+
+ return result
diff --git a/playbooks/roles/lgtm_stack/templates/config.alloy.j2 b/playbooks/roles/lgtm_stack/templates/config.alloy.j2
index 8fc772a..950059e 100644
--- a/playbooks/roles/lgtm_stack/templates/config.alloy.j2
+++ b/playbooks/roles/lgtm_stack/templates/config.alloy.j2
@@ -3,23 +3,66 @@ logging {
format = "logfmt"
}
+prometheus.remote_write "mimir" {
+ endpoint {
+ url = "https://{{ lgtm_stack_mimir_domain }}/api/v1/push"
+ }
+}
+
prometheus.exporter.self "alloy" {}
prometheus.scrape "alloy" {
targets = prometheus.exporter.self.alloy.targets
forward_to = [prometheus.remote_write.mimir.receiver]
}
-prometheus.scrape "node_exporter" {
- targets = [
-{% for host_data in opentofu.hosts.values() %}
- {"__address__" = "{{ host_data.fqdn_vpn }}:9100", "job" = "node_exporter"},
-{% endfor %}
- ]
- forward_to = [prometheus.remote_write.mimir.receiver]
+prometheus.exporter.postgres "default" {
+ data_source_names = ["postgresql://{{ svc.postgresql_collector.user }}:{{ svc.postgresql_collector.pass }}@{{ svc.postgresql_collector.host }}:{{ svc.postgresql_collector.port }}/{{ svc.postgresql_collector.database }}?sslmode=verify-full"]
+
+ autodiscovery {
+ enabled = true
+ }
+}
+prometheus.scrape "postgres" {
+ targets = prometheus.exporter.postgres.default.targets
+ forward_to = [prometheus.remote_write.mimir.receiver]
}
-prometheus.remote_write "mimir" {
- endpoint {
- url = "https://{{ lgtm_stack_mimir_domain }}/api/v1/push"
- }
+prometheus.scrape "node_exporter" {
+ targets = [
+{% for host_data in opentofu.hosts.values() %}
+ {"__address__" = "{{ host_data.fqdn_vpn }}:9100", "instance" = "{{ host_data.hostname }}"},
+{% endfor %}
+ ]
+ forward_to = [prometheus.remote_write.mimir.receiver]
}
+
+prometheus.scrape "caddy" {
+ targets = [
+{% for host_data in opentofu.hosts.values() %}
+ {"__address__" = "{{ host_data.fqdn_vpn }}:2019", "instance" = "{{ host_data.hostname }}"},
+{% endfor %}
+ ]
+ forward_to = [prometheus.remote_write.mimir.receiver]
+}
+
+
+{% for job in lgtm_stack_alloy_jobs %}
+
+prometheus.scrape "{{ job.name }}" {
+ targets = [
+{% for target in job.targets %}
+ {
+ "__address__" = "{{ target.address }}",
+ "__metrics_path__" = "{{ target.path }}",
+ "__scheme__" = "https",
+ {% if 'job' in target %}"job" = "{{ target.job }}",{% endif %}
+ {% if 'instance' in target %}"instance" = "{{ target.instance }}",{% endif %}
+ },
+{% endfor %}
+ ]
+
+ scrape_interval = "{{ job.scrape_interval }}"
+ forward_to = [prometheus.remote_write.mimir.receiver]
+}
+
+{% endfor %}
\ No newline at end of file
diff --git a/playbooks/roles/lgtm_stack/vars/main.yml b/playbooks/roles/lgtm_stack/vars/main.yml
index b8b1d48..2603252 100644
--- a/playbooks/roles/lgtm_stack/vars/main.yml
+++ b/playbooks/roles/lgtm_stack/vars/main.yml
@@ -3,6 +3,9 @@ lgtm_stack_domain: "{{ all_services | service_get_domain(role_name) }}"
lgtm_stack_mimir_domain: mimir.serguzim.me
lgtm_stack_alloy_domain: alloy.serguzim.me
+lgtm_stack_alloy_jobs: "{{ all_services | services_to_alloy() }}"
+
+
lgtm_stack_svc:
domain: "{{ lgtm_stack_domain }}"
port: 3000
@@ -15,6 +18,12 @@ lgtm_stack_svc:
docker_host: lgtm_stack_mimir
port: 9009
caddy_extra: import vpn_only
+ postgresql_collector:
+ host: "{{ postgres.host }}"
+ port: "{{ postgres.port }}"
+ user: "{{ opentofu.postgresql_metrics_collector.user }}"
+ pass: "{{ opentofu.postgresql_metrics_collector.pass }}"
+ database: "{{ opentofu.postgresql_metrics_collector.database }}"
lgtm_stack_env:
diff --git a/services.auto.tfvars b/services.auto.tfvars
index dcec17e..678c781 100644
--- a/services.auto.tfvars
+++ b/services.auto.tfvars
@@ -196,6 +196,10 @@ services = {
name = "forgejo_data"
type = "docker"
}]
+ # TODO: add auth stuff to alloy
+ #metrics = [{
+ # path = "/metrics"
+ #}]
monitoring = {
url = "/api/v1/version"
group = "3-services"
@@ -300,6 +304,9 @@ services = {
name = "influxdb_data"
type = "docker"
}]
+ metrics = [{
+ path = "/metrics"
+ }]
monitoring = {
url = "/health"
group = "3-services"
@@ -501,6 +508,9 @@ services = {
name = "ntfy_data"
type = "docker"
}]
+ metrics = [{
+ path = "/metrics"
+ }]
monitoring = {
url = "/v1/health"
group = "3-services"
@@ -607,6 +617,9 @@ services = {
name = "synapse_media_store"
type = "docker"
}]
+ metrics = [{
+ path = "/_synapse/metrics"
+ }]
monitoring = {
url = "/_matrix/client/versions"
group = "3-services"
@@ -732,6 +745,9 @@ services = {
name = "vikunja_data"
type = "docker"
}]
+ metrics = [{
+ path = "/api/v1/metrics"
+ }]
monitoring = {
url = "/api/v1/info"
group = "3-services"
@@ -792,6 +808,10 @@ services = {
alias = "woodpecker"
}
]
+ # TODO: add auth stuff to alloy
+ #metrics = [{
+ # path = "/metrics"
+ #}]
monitoring = {
url = "/healthz"
group = "3-services"
diff --git a/variables.tf b/variables.tf
index cf4ebf8..271854e 100644
--- a/variables.tf
+++ b/variables.tf
@@ -139,6 +139,13 @@ variable "services" {
interval = optional(string)
conditions = optional(list(string))
}))
+ metrics = optional(list(object({
+ path = string
+ address = optional(string)
+ instance = optional(string)
+ job = optional(string)
+ interval = optional(string)
+ })))
ports = optional(list(object({
description = string
port = string