From 6511e513a3cd9eef4ff3139cf9b75ae2f7baf1b7 Mon Sep 17 00:00:00 2001 From: Chris Kruining Date: Tue, 19 Aug 2025 15:01:22 +0200 Subject: [PATCH] initial observability setup --- .../services/development/forgejo/default.nix | 3 +- .../grafana/dashboards/default.json | 7 ++ .../observability/grafana/default.nix | 100 ++++++++++++++++++ .../services/observability/loki/default.nix | 49 +++++++++ .../observability/prometheus/default.nix | 32 ++++++ .../observability/promtail/default.nix | 56 ++++++++++ systems/x86_64-linux/ulmo/default.nix | 9 +- 7 files changed, 253 insertions(+), 3 deletions(-) create mode 100644 modules/nixos/services/observability/grafana/dashboards/default.json create mode 100644 modules/nixos/services/observability/grafana/default.nix create mode 100644 modules/nixos/services/observability/loki/default.nix create mode 100644 modules/nixos/services/observability/prometheus/default.nix create mode 100644 modules/nixos/services/observability/promtail/default.nix diff --git a/modules/nixos/services/development/forgejo/default.nix b/modules/nixos/services/development/forgejo/default.nix index 9945691..22c3123 100644 --- a/modules/nixos/services/development/forgejo/default.nix +++ b/modules/nixos/services/development/forgejo/default.nix @@ -113,10 +113,9 @@ in SMTP_ADDR = "smpts://smtp.black-mail.nl"; FROM = "noreply@kruining.eu"; USER = "noreply@kruining.eu"; + PASSWD = "/var/lib/forgejo/custom/mail_password"; }; }; - - mailerPasswordFile = "/var/lib/forgejo/custom/mail_password"; }; openssh.settings.AllowUsers = [ "forgejo" ]; diff --git a/modules/nixos/services/observability/grafana/dashboards/default.json b/modules/nixos/services/observability/grafana/dashboards/default.json new file mode 100644 index 0000000..f8ea8dc --- /dev/null +++ b/modules/nixos/services/observability/grafana/dashboards/default.json @@ -0,0 +1,7 @@ +{ + "title": "Default Dash", + "description": "The default dashboard", + "timezone": "browser", + "editable": false, + "panels": [] +} diff --git a/modules/nixos/services/observability/grafana/default.nix b/modules/nixos/services/observability/grafana/default.nix new file mode 100644 index 0000000..1747330 --- /dev/null +++ b/modules/nixos/services/observability/grafana/default.nix @@ -0,0 +1,100 @@ +{ pkgs, config, lib, namespace, ... }: +let + inherit (lib.modules) mkIf; + inherit (lib.options) mkEnableOption; + + cfg = config.${namespace}.services.observability.grafana; + + db_user = "grafana"; + db_name = "grafana"; +in +{ + options.${namespace}.services.observability.grafana = { + enable = mkEnableOption "enable Grafana"; + }; + + config = mkIf cfg.enable { + services.grafana = { + enable = true; + openFirewall = true; + + settings = { + server = { + http_port = 9001; + http_addr = "0.0.0.0"; + }; + database = { + type = "postgres"; + host = "/var/run/postgresql:5432"; + name = db_name; + user = db_user; + ssl_mode = "disable"; + }; + + users = { + allow_sign_up = false; + allow_org_create = false; + viewers_can_edit = false; + + default_theme = "system"; + }; + + analytics = { + reporting_enabled = false; + check_for_updates = false; + check_for_plugin_updates = false; + feedback_links_enabled = false; + }; + }; + + provision = { + enable = true; + + dashboards.settings = { + apiVersion = 1; + providers = [ + { + name = "Default Dashboard"; + disableDeletion = true; + allowUiUpdates = false; + options = { + path = "/etc/grafana/dashboards"; + foldersFromFilesStructure = true; + }; + } + ]; + }; + + datasources.settings.datasources = [ + { + name = "Prometheus"; + type = "prometheus"; + url = "http://localhost:9002"; + isDefault = true; + editable = false; + } + + { + name = "Loki"; + type = "loki"; + url = "http://localhost:9003"; + editable = false; + } + ]; + }; + }; + + services.postgresql = { + enable = true; + ensureDatabases = [ db_name ]; + ensureUsers = [ + { + name = db_user; + ensureDBOwnership = true; + } + ]; + }; + + environment.etc."/grafana/dashboards/default.json".source = ./dashboards/default.json; + }; +} diff --git a/modules/nixos/services/observability/loki/default.nix b/modules/nixos/services/observability/loki/default.nix new file mode 100644 index 0000000..8f6e0e3 --- /dev/null +++ b/modules/nixos/services/observability/loki/default.nix @@ -0,0 +1,49 @@ +{ pkgs, config, lib, namespace, ... }: +let + inherit (lib.modules) mkIf; + inherit (lib.options) mkEnableOption; + + cfg = config.${namespace}.services.observability.loki; +in +{ + options.${namespace}.services.observability.loki = { + enable = mkEnableOption "enable Grafana Loki"; + }; + + config = mkIf cfg.enable { + services.loki = { + enable = true; + configuration = { + auth_enabled = false; + + server = { + http_listen_port = 9003; + }; + + common = { + ring = { + instance_addr = "127.0.0.1"; + kvstore.store = "inmmemory"; + }; + replication_factor = 1; + path_prefix = "/tmp/loki"; + }; + + schema_config.configs = [ + { + from = "2025-01-01"; + store = "tsdb"; + object_store = "filesystem"; + schema = "v13"; + index = { + prefix = "index_"; + period = "24h"; + }; + } + ]; + }; + }; + + networking.firewall.allowedTCPPorts = [ 9003 ]; + }; +} diff --git a/modules/nixos/services/observability/prometheus/default.nix b/modules/nixos/services/observability/prometheus/default.nix new file mode 100644 index 0000000..666a356 --- /dev/null +++ b/modules/nixos/services/observability/prometheus/default.nix @@ -0,0 +1,32 @@ +{ pkgs, config, lib, namespace, ... }: +let + inherit (lib.modules) mkIf; + inherit (lib.options) mkEnableOption; + + cfg = config.${namespace}.services.observability.prometheus; +in +{ + options.${namespace}.services.observability.prometheus = { + enable = mkEnableOption "enable Prometheus"; + }; + + config = mkIf cfg.enable { + services.prometheus = { + enable = true; + port = 9002; + + globalConfig.scrape_interval = "15s"; + + scrapeConfigs = [ + { + job_name = "prometheus"; + static_configs = [ + { targets = [ "localhost:9002" ]; } + ]; + } + ]; + }; + + networking.firewall.allowedTCPPorts = [ 9002 ]; + }; +} diff --git a/modules/nixos/services/observability/promtail/default.nix b/modules/nixos/services/observability/promtail/default.nix new file mode 100644 index 0000000..1f32adc --- /dev/null +++ b/modules/nixos/services/observability/promtail/default.nix @@ -0,0 +1,56 @@ +{ pkgs, config, lib, namespace, ... }: +let + inherit (lib.modules) mkIf; + inherit (lib.options) mkEnableOption; + + cfg = config.${namespace}.services.observability.promtail; +in +{ + options.${namespace}.services.observability.promtail = { + enable = mkEnableOption "enable Grafana Promtail"; + }; + + config = mkIf cfg.enable { + services.promtail = { + enable = true; + + # Ensures proper permissions + extraFlags = [ + "-config.expand-env=true" + ]; + + configuration = { + server = { + http_listen_port = 9004; + grpc_listen_port = 0; + }; + + positions = { + filename = "filename"; + }; + + clients = { + url = "http://127.0.0.1:3100/loki/api/v1/push"; + }; + + scrape_configs = [ + { + job_name = "journal"; + journal = { + max_age = "12h"; + labels = { + job = "systemd-journal"; + host = "ulmo"; + }; + }; + relabel_configs = [ + { source_labels = [ "__journal__systemd_unit" ]; target_label = "unit"; } + ]; + } + ]; + }; + }; + + networking.firewall.allowedTCPPorts = [ 9004 ]; + }; +} diff --git a/systems/x86_64-linux/ulmo/default.nix b/systems/x86_64-linux/ulmo/default.nix index f47c580..e191367 100644 --- a/systems/x86_64-linux/ulmo/default.nix +++ b/systems/x86_64-linux/ulmo/default.nix @@ -10,12 +10,19 @@ authentication.authelia.enable = true; authentication.zitadel.enable = true; + development.forgejo.enable = true; + networking.ssh.enable = true; media.enable = true; media.nfs.enable = true; - development.forgejo.enable = true; + observability = { + grafana.enable = true; + prometheus.enable = true; + loki.enable = true; + promtail.enable = true; + }; }; editor = {