Add loki service

inventory/group_vars/all/compose_defaults.yml

@@ -4,6 +4,7 @@ compose_file_main:
       image: "{{ compose.image }}"
       restart: always
       labels:
+        net.serguzim.logs.collect: "{{ compose.collect_logs | default(False) }}"
         com.centurylinklabs.watchtower.enable: "{{ compose.watchtower | default('') == 'update' }}"
         com.centurylinklabs.watchtower.monitor-only: "{{ compose.watchtower | default('') == 'monitor' }}"
 

playbooks/roles/forgejo/vars/main.yml

@@ -64,6 +64,8 @@ forgejo_env:
 
   FORGEJO__oauth2__JWT_SECRET: "{{ vault_forgejo.oauth2_jwt_secret }}"
 
+  FORGEJO__log.console__FLAGS: "level,medfile,shortfuncname"
+
   FORGEJO__metrics__ENABLED: true
   FORGEJO__metrics__TOKEN: "{{ vault_metrics_token }}"
 
@@ -82,6 +84,7 @@ forgejo_env:
 
 forgejo_compose:
   watchtower: update
+  collect_logs: true
   image: codeberg.org/forgejo/forgejo:11
   volumes:
     - data:/data

playbooks/roles/forgejo_runner/vars/main.yml

@@ -5,6 +5,7 @@ forgejo_runner_env:
   DOCKER_HOST: tcp://docker-in-docker:2375
 
 forgejo_runner_compose:
+  collect_logs: true
   watchtower: update
   image: code.forgejo.org/forgejo/runner:3.3.0
   volumes:

playbooks/roles/homebox/vars/main.yml

@@ -13,6 +13,7 @@ homebox_env:
   HBOX_SWAGGER_SCHEMA: https
 
 homebox_compose:
+  collect_logs: true
   watchtower: update
   image: ghcr.io/sysadminsmedia/homebox:0-rootless
   volumes:

playbooks/roles/immich/vars/main.yml

@@ -32,6 +32,7 @@ immich_env:
   REDIS_HOSTNAME: redis
 
 immich_compose:
+  collect_logs: true
   watchtower: monitor
   image: ghcr.io/immich-app/immich-server:{{ immich_docker_tag }}
   volumes:
@@ -45,6 +46,8 @@ immich_compose:
 
       machine-learning:
         image: ghcr.io/immich-app/immich-machine-learning:{{ immich_docker_tag }}
+        labels:
+          net.serguzim.logs.collect: true
         volumes:
           - model-cache:/cache
         env_file:
@@ -61,6 +64,8 @@ immich_compose:
 
       database:
         image: ghcr.io/immich-app/postgres:16-vectorchord0.3.0-pgvectors0.3.0
+        labels:
+          net.serguzim.logs.collect: true
         env_file:
           - service.env
         volumes:

playbooks/roles/lgtm_stack/templates/config.alloy.j2

@@ -1,6 +1,6 @@
 logging {
-  level  = "info"
+    level  = "info"
-  format = "logfmt"
+    write_to = [loki.relabel.alloy_logs.receiver]
 }
 
 prometheus.remote_write "mimir" {
@@ -65,4 +65,60 @@ prometheus.scrape "{{ job.name }}" {
     forward_to = [prometheus.remote_write.mimir.receiver]
 }
 
-{% endfor %}
+{% endfor %}
+
+loki.write "loki" {
+    endpoint {
+        url = "https://{{ lgtm_stack_loki_domain }}/loki/api/v1/push"
+    }
+}
+
+loki.relabel "alloy_logs" {
+    rule {
+        target_label = "instance"
+        replacement = "{{ inventory_hostname }}"
+    }
+
+    rule {
+        target_label = "job"
+        replacement = "integrations/self"
+    }
+
+    forward_to = [loki.write.loki.receiver]
+}
+
+discovery.docker "linux" {
+    host = "unix:///var/run/docker.sock"
+
+    filter {
+        name = "label"
+        values = ["net.serguzim.logs.collect=true"]
+    }
+}
+
+loki.source.docker "default" {
+    host       = "unix:///var/run/docker.sock"
+    targets    = discovery.docker.linux.targets
+    labels     = {"app" = "docker"}
+    relabel_rules = discovery.relabel.logs_docker_containers.rules
+    forward_to = [loki.write.loki.receiver]
+}
+
+discovery.relabel "logs_docker_containers" {
+    targets = []
+    rule {
+        source_labels = ["__meta_docker_container_label_com_docker_compose_project"]
+        target_label  = "compose_project"
+    }
+
+    rule {
+        source_labels = ["__meta_docker_container_label_com_docker_compose_service"]
+        target_label  = "compose_service"
+    }
+
+    rule {
+        source_labels = ["__meta_docker_container_name"]
+        target_label  = "container_name"
+    }
+}
+

playbooks/roles/lgtm_stack/vars/main.yml

@@ -2,6 +2,7 @@
 lgtm_stack_domain: "{{ all_services | service_get_domain(role_name) }}"
 lgtm_stack_mimir_domain: mimir.serguzim.me
 lgtm_stack_alloy_domain: alloy.serguzim.me
+lgtm_stack_loki_domain: "{{ all_services | service_get_domain('loki') }}"
 
 lgtm_stack_alloy_jobs: "{{ all_services | services_to_alloy() }}"
 
@@ -142,6 +143,7 @@ lgtm_stack_compose:
         image: grafana/alloy:latest
         restart: always
         volumes:
+          - /var/run/docker.sock:/var/run/docker.sock
           - ./config.alloy:/etc/alloy/config.alloy:ro
         command:
           - run

playbooks/roles/loki/handlers/main.yml

@@ -0,0 +1,3 @@
+---
+- name: Restart service {{ role_name }}
+  ansible.builtin.include_tasks: tasks/restart-service.yml

playbooks/roles/loki/tasks/main.yml

@@ -0,0 +1,22 @@
+---
+- name: Set common facts
+  ansible.builtin.import_tasks: tasks/set-default-facts.yml
+
+- name: Deploy {{ role_name }}
+  vars:
+    svc: "{{ loki_svc }}"
+    yml: "{{ loki_yml }}"
+    compose: "{{ loki_compose }}"
+  block:
+    - name: Import prepare tasks for common service
+      ansible.builtin.import_tasks: tasks/prepare-common-service.yml
+
+    - name: Template loki config file
+      ansible.builtin.template:
+        src: yml.j2
+        dest: "{{ (service_path, 'loki.yaml') | path_join }}"
+        mode: "0644"
+      notify: Restart service {{ role_name }}
+
+    - name: Import start tasks for common service
+      ansible.builtin.import_tasks: tasks/start-common-service.yml

playbooks/roles/loki/vars/main.yml

@@ -0,0 +1,59 @@
+---
+loki_svc:
+  domain: "{{ all_services | service_get_domain(role_name) }}"
+  port: 3100
+  caddy_extra: import vpn_only
+
+loki_yml: # https://grafana.com/docs/loki/latest/configure/examples/configuration-examples/#2-s3-cluster-exampleyaml
+  auth_enabled: false
+
+  common:
+    storage:
+      s3:
+        s3forcepathstyle: true
+        bucketnames: "{{ opentofu.minio_data.loki.name }}"
+        endpoint: "{{ opentofu.minio_data.loki.api_endpoint }}"
+        region: "{{ opentofu.minio_data.loki.region }}"
+        access_key_id: "{{ opentofu.minio_data.loki.access_key }}"
+        secret_access_key: "{{ opentofu.minio_data.loki.secret_key }}"
+    ring:
+      instance_addr: 127.0.0.1
+      kvstore:
+        store: inmemory
+    replication_factor: 1
+    path_prefix: /loki
+
+  ui:
+    enabled: true
+
+  storage_config:
+    tsdb_shipper:
+      active_index_directory: /loki/index
+      cache_location: /loki/index_cache
+
+  schema_config:
+    configs:
+      - from: "2020-05-15"
+        store: tsdb
+        object_store: s3
+        schema: v13
+        index:
+          prefix: index_
+          period: 24h
+
+  compactor:
+    working_directory: /loki/compactor
+
+
+loki_compose:
+  watchtower: update
+  image: grafana/loki:latest
+  volumes:
+    - ./loki.yaml:/etc/loki-config/loki.yaml:ro
+  file:
+    services:
+      app:
+        command:
+          - -config.file=/etc/loki-config/loki.yaml
+    volumes:
+      data:

services.auto.tfvars

@@ -447,6 +447,20 @@ services = {
     mail = "monitoring"
   }
 
+  "loki" = {
+    host = "node001"
+    dns = [
+      {
+        domain = "loki.serguzim.me"
+        name = "loki"
+        vpn = true
+      }
+    ]
+    auth = false
+    database = false
+    s3 = "internal"
+  }
+
   "minecraft_3" = {
     host = ""
     dns = [