diff --git a/inventory/group_vars/all/compose_defaults.yml b/inventory/group_vars/all/compose_defaults.yml index 536e2ca..e48a349 100644 --- a/inventory/group_vars/all/compose_defaults.yml +++ b/inventory/group_vars/all/compose_defaults.yml @@ -4,6 +4,7 @@ compose_file_main: image: "{{ compose.image }}" restart: always labels: + net.serguzim.logs.collect: "{{ compose.collect_logs | default(False) }}" com.centurylinklabs.watchtower.enable: "{{ compose.watchtower | default('') == 'update' }}" com.centurylinklabs.watchtower.monitor-only: "{{ compose.watchtower | default('') == 'monitor' }}" diff --git a/playbooks/roles/forgejo/vars/main.yml b/playbooks/roles/forgejo/vars/main.yml index a3e401f..39eff82 100644 --- a/playbooks/roles/forgejo/vars/main.yml +++ b/playbooks/roles/forgejo/vars/main.yml @@ -64,6 +64,8 @@ forgejo_env: FORGEJO__oauth2__JWT_SECRET: "{{ vault_forgejo.oauth2_jwt_secret }}" + FORGEJO__log.console__FLAGS: "level,medfile,shortfuncname" + FORGEJO__metrics__ENABLED: true FORGEJO__metrics__TOKEN: "{{ vault_metrics_token }}" @@ -82,6 +84,7 @@ forgejo_env: forgejo_compose: watchtower: update + collect_logs: true image: codeberg.org/forgejo/forgejo:11 volumes: - data:/data diff --git a/playbooks/roles/forgejo_runner/vars/main.yml b/playbooks/roles/forgejo_runner/vars/main.yml index 6fd207d..cc25721 100644 --- a/playbooks/roles/forgejo_runner/vars/main.yml +++ b/playbooks/roles/forgejo_runner/vars/main.yml @@ -5,6 +5,7 @@ forgejo_runner_env: DOCKER_HOST: tcp://docker-in-docker:2375 forgejo_runner_compose: + collect_logs: true watchtower: update image: code.forgejo.org/forgejo/runner:3.3.0 volumes: diff --git a/playbooks/roles/homebox/vars/main.yml b/playbooks/roles/homebox/vars/main.yml index 34895b9..93a5104 100644 --- a/playbooks/roles/homebox/vars/main.yml +++ b/playbooks/roles/homebox/vars/main.yml @@ -13,6 +13,7 @@ homebox_env: HBOX_SWAGGER_SCHEMA: https homebox_compose: + collect_logs: true watchtower: update image: ghcr.io/sysadminsmedia/homebox:0-rootless volumes: diff --git a/playbooks/roles/immich/vars/main.yml b/playbooks/roles/immich/vars/main.yml index ecad1c1..b6d6ab5 100644 --- a/playbooks/roles/immich/vars/main.yml +++ b/playbooks/roles/immich/vars/main.yml @@ -32,6 +32,7 @@ immich_env: REDIS_HOSTNAME: redis immich_compose: + collect_logs: true watchtower: monitor image: ghcr.io/immich-app/immich-server:{{ immich_docker_tag }} volumes: @@ -45,6 +46,8 @@ immich_compose: machine-learning: image: ghcr.io/immich-app/immich-machine-learning:{{ immich_docker_tag }} + labels: + net.serguzim.logs.collect: true volumes: - model-cache:/cache env_file: @@ -61,6 +64,8 @@ immich_compose: database: image: ghcr.io/immich-app/postgres:16-vectorchord0.3.0-pgvectors0.3.0 + labels: + net.serguzim.logs.collect: true env_file: - service.env volumes: diff --git a/playbooks/roles/lgtm_stack/templates/config.alloy.j2 b/playbooks/roles/lgtm_stack/templates/config.alloy.j2 index d82a415..8f2cc96 100644 --- a/playbooks/roles/lgtm_stack/templates/config.alloy.j2 +++ b/playbooks/roles/lgtm_stack/templates/config.alloy.j2 @@ -1,6 +1,6 @@ logging { - level = "info" - format = "logfmt" + level = "info" + write_to = [loki.relabel.alloy_logs.receiver] } prometheus.remote_write "mimir" { @@ -65,4 +65,60 @@ prometheus.scrape "{{ job.name }}" { forward_to = [prometheus.remote_write.mimir.receiver] } -{% endfor %} \ No newline at end of file +{% endfor %} + +loki.write "loki" { + endpoint { + url = "https://{{ lgtm_stack_loki_domain }}/loki/api/v1/push" + } +} + +loki.relabel "alloy_logs" { + rule { + target_label = "instance" + replacement = "{{ inventory_hostname }}" + } + + rule { + target_label = "job" + replacement = "integrations/self" + } + + forward_to = [loki.write.loki.receiver] +} + +discovery.docker "linux" { + host = "unix:///var/run/docker.sock" + + filter { + name = "label" + values = ["net.serguzim.logs.collect=true"] + } +} + +loki.source.docker "default" { + host = "unix:///var/run/docker.sock" + targets = discovery.docker.linux.targets + labels = {"app" = "docker"} + relabel_rules = discovery.relabel.logs_docker_containers.rules + forward_to = [loki.write.loki.receiver] +} + +discovery.relabel "logs_docker_containers" { + targets = [] + rule { + source_labels = ["__meta_docker_container_label_com_docker_compose_project"] + target_label = "compose_project" + } + + rule { + source_labels = ["__meta_docker_container_label_com_docker_compose_service"] + target_label = "compose_service" + } + + rule { + source_labels = ["__meta_docker_container_name"] + target_label = "container_name" + } +} + diff --git a/playbooks/roles/lgtm_stack/vars/main.yml b/playbooks/roles/lgtm_stack/vars/main.yml index c416844..cfdc4f8 100644 --- a/playbooks/roles/lgtm_stack/vars/main.yml +++ b/playbooks/roles/lgtm_stack/vars/main.yml @@ -2,6 +2,7 @@ lgtm_stack_domain: "{{ all_services | service_get_domain(role_name) }}" lgtm_stack_mimir_domain: mimir.serguzim.me lgtm_stack_alloy_domain: alloy.serguzim.me +lgtm_stack_loki_domain: "{{ all_services | service_get_domain('loki') }}" lgtm_stack_alloy_jobs: "{{ all_services | services_to_alloy() }}" @@ -142,6 +143,7 @@ lgtm_stack_compose: image: grafana/alloy:latest restart: always volumes: + - /var/run/docker.sock:/var/run/docker.sock - ./config.alloy:/etc/alloy/config.alloy:ro command: - run diff --git a/playbooks/roles/loki/handlers/main.yml b/playbooks/roles/loki/handlers/main.yml new file mode 100644 index 0000000..4ff0834 --- /dev/null +++ b/playbooks/roles/loki/handlers/main.yml @@ -0,0 +1,3 @@ +--- +- name: Restart service {{ role_name }} + ansible.builtin.include_tasks: tasks/restart-service.yml diff --git a/playbooks/roles/loki/tasks/main.yml b/playbooks/roles/loki/tasks/main.yml new file mode 100644 index 0000000..9c799b6 --- /dev/null +++ b/playbooks/roles/loki/tasks/main.yml @@ -0,0 +1,22 @@ +--- +- name: Set common facts + ansible.builtin.import_tasks: tasks/set-default-facts.yml + +- name: Deploy {{ role_name }} + vars: + svc: "{{ loki_svc }}" + yml: "{{ loki_yml }}" + compose: "{{ loki_compose }}" + block: + - name: Import prepare tasks for common service + ansible.builtin.import_tasks: tasks/prepare-common-service.yml + + - name: Template loki config file + ansible.builtin.template: + src: yml.j2 + dest: "{{ (service_path, 'loki.yaml') | path_join }}" + mode: "0644" + notify: Restart service {{ role_name }} + + - name: Import start tasks for common service + ansible.builtin.import_tasks: tasks/start-common-service.yml diff --git a/playbooks/roles/loki/vars/main.yml b/playbooks/roles/loki/vars/main.yml new file mode 100644 index 0000000..ab787c0 --- /dev/null +++ b/playbooks/roles/loki/vars/main.yml @@ -0,0 +1,59 @@ +--- +loki_svc: + domain: "{{ all_services | service_get_domain(role_name) }}" + port: 3100 + caddy_extra: import vpn_only + +loki_yml: # https://grafana.com/docs/loki/latest/configure/examples/configuration-examples/#2-s3-cluster-exampleyaml + auth_enabled: false + + common: + storage: + s3: + s3forcepathstyle: true + bucketnames: "{{ opentofu.minio_data.loki.name }}" + endpoint: "{{ opentofu.minio_data.loki.api_endpoint }}" + region: "{{ opentofu.minio_data.loki.region }}" + access_key_id: "{{ opentofu.minio_data.loki.access_key }}" + secret_access_key: "{{ opentofu.minio_data.loki.secret_key }}" + ring: + instance_addr: 127.0.0.1 + kvstore: + store: inmemory + replication_factor: 1 + path_prefix: /loki + + ui: + enabled: true + + storage_config: + tsdb_shipper: + active_index_directory: /loki/index + cache_location: /loki/index_cache + + schema_config: + configs: + - from: "2020-05-15" + store: tsdb + object_store: s3 + schema: v13 + index: + prefix: index_ + period: 24h + + compactor: + working_directory: /loki/compactor + + +loki_compose: + watchtower: update + image: grafana/loki:latest + volumes: + - ./loki.yaml:/etc/loki-config/loki.yaml:ro + file: + services: + app: + command: + - -config.file=/etc/loki-config/loki.yaml + volumes: + data: diff --git a/services.auto.tfvars b/services.auto.tfvars index 31914b3..0b5ea72 100644 --- a/services.auto.tfvars +++ b/services.auto.tfvars @@ -447,6 +447,20 @@ services = { mail = "monitoring" } + "loki" = { + host = "node001" + dns = [ + { + domain = "loki.serguzim.me" + name = "loki" + vpn = true + } + ] + auth = false + database = false + s3 = "internal" + } + "minecraft_3" = { host = "" dns = [