initial observability setup

modules/nixos/services/development/forgejo/default.nix

@@ -113,10 +113,9 @@ in
             SMTP_ADDR = "smpts://smtp.black-mail.nl";
             FROM = "noreply@kruining.eu";
             USER = "noreply@kruining.eu";
+            PASSWD = "/var/lib/forgejo/custom/mail_password";
           };
         };
-
-        mailerPasswordFile = "/var/lib/forgejo/custom/mail_password";
       };
 
       openssh.settings.AllowUsers = [ "forgejo" ];

modules/nixos/services/observability/grafana/dashboards/default.json

@@ -0,0 +1,7 @@
+{
+  "title": "Default Dash",
+  "description": "The default dashboard",
+  "timezone": "browser",
+  "editable": false,
+  "panels": []
+}

modules/nixos/services/observability/grafana/default.nix

@@ -0,0 +1,100 @@
+{ pkgs, config, lib, namespace, ... }:
+let
+  inherit (lib.modules) mkIf;
+  inherit (lib.options) mkEnableOption;
+
+  cfg = config.${namespace}.services.observability.grafana;
+
+  db_user = "grafana";
+  db_name = "grafana";
+in
+{
+  options.${namespace}.services.observability.grafana = {
+    enable = mkEnableOption "enable Grafana";
+  };
+
+  config = mkIf cfg.enable {
+    services.grafana = {
+      enable = true;
+      openFirewall = true;
+
+      settings = {
+        server = {
+          http_port = 9001;
+          http_addr = "0.0.0.0";
+        };
+        database = {
+          type = "postgres";
+          host = "/var/run/postgresql:5432";
+          name = db_name;
+          user = db_user;
+          ssl_mode = "disable";
+        };
+
+        users = {
+          allow_sign_up = false;
+          allow_org_create = false;
+          viewers_can_edit = false;
+          
+          default_theme = "system";
+        };
+
+        analytics = {
+          reporting_enabled = false;
+          check_for_updates = false;
+          check_for_plugin_updates = false;
+          feedback_links_enabled = false;
+        };
+      };
+
+      provision = {
+        enable = true;
+
+        dashboards.settings = {
+          apiVersion = 1;
+          providers = [
+            {
+              name = "Default Dashboard";
+              disableDeletion = true;
+              allowUiUpdates = false;
+              options = {
+                path = "/etc/grafana/dashboards";
+                foldersFromFilesStructure = true;
+              };
+            }
+          ];
+        };
+
+        datasources.settings.datasources = [
+          {
+            name = "Prometheus";
+            type = "prometheus";
+            url = "http://localhost:9002";
+            isDefault = true;
+            editable = false;
+          }
+
+          {
+            name = "Loki";
+            type = "loki";
+            url = "http://localhost:9003";
+            editable = false;
+          }
+        ];
+      };
+    };
+
+    services.postgresql = {
+      enable = true;
+      ensureDatabases = [ db_name ];
+      ensureUsers = [
+        {
+          name = db_user;
+          ensureDBOwnership = true;
+        }
+      ];
+    };
+
+    environment.etc."/grafana/dashboards/default.json".source = ./dashboards/default.json;
+  };
+}

modules/nixos/services/observability/loki/default.nix

@@ -0,0 +1,49 @@
+{ pkgs, config, lib, namespace, ... }:
+let
+  inherit (lib.modules) mkIf;
+  inherit (lib.options) mkEnableOption;
+
+  cfg = config.${namespace}.services.observability.loki;
+in
+{
+  options.${namespace}.services.observability.loki = {
+    enable = mkEnableOption "enable Grafana Loki";
+  };
+
+  config = mkIf cfg.enable {
+    services.loki = {
+      enable = true;
+      configuration = {
+        auth_enabled = false;
+
+        server = {
+          http_listen_port = 9003;
+        };
+
+        common = {
+          ring = {
+            instance_addr = "127.0.0.1";
+            kvstore.store = "inmmemory";
+          };
+          replication_factor = 1;
+          path_prefix = "/tmp/loki";
+        };
+
+        schema_config.configs = [
+          {
+            from = "2025-01-01";
+            store = "tsdb";
+            object_store = "filesystem";
+            schema = "v13";
+            index = {
+              prefix = "index_";
+              period = "24h";
+            };
+          }
+        ];
+      };
+    };
+
+    networking.firewall.allowedTCPPorts = [ 9003 ];
+  };
+}

modules/nixos/services/observability/prometheus/default.nix

@@ -0,0 +1,32 @@
+{ pkgs, config, lib, namespace, ... }:
+let
+  inherit (lib.modules) mkIf;
+  inherit (lib.options) mkEnableOption;
+
+  cfg = config.${namespace}.services.observability.prometheus;
+in
+{
+  options.${namespace}.services.observability.prometheus = {
+    enable = mkEnableOption "enable Prometheus";
+  };
+
+  config = mkIf cfg.enable {
+    services.prometheus = {
+      enable = true;
+      port = 9002;
+
+      globalConfig.scrape_interval = "15s";
+
+      scrapeConfigs = [
+        {
+          job_name = "prometheus";
+          static_configs = [
+            { targets = [ "localhost:9002" ]; }
+          ];
+        }
+      ];
+    };
+
+    networking.firewall.allowedTCPPorts = [ 9002 ];
+  };
+}

modules/nixos/services/observability/promtail/default.nix

@@ -0,0 +1,56 @@
+{ pkgs, config, lib, namespace, ... }:
+let
+  inherit (lib.modules) mkIf;
+  inherit (lib.options) mkEnableOption;
+
+  cfg = config.${namespace}.services.observability.promtail;
+in
+{
+  options.${namespace}.services.observability.promtail = {
+    enable = mkEnableOption "enable Grafana Promtail";
+  };
+
+  config = mkIf cfg.enable {
+    services.promtail = {
+      enable = true;
+
+      # Ensures proper permissions
+      extraFlags = [
+        "-config.expand-env=true"
+      ];
+
+      configuration = {
+        server = {
+          http_listen_port = 9004;
+          grpc_listen_port = 0;
+        };
+
+        positions = {
+          filename = "filename";
+        };
+
+        clients = {
+          url = "http://127.0.0.1:3100/loki/api/v1/push";
+        };
+
+        scrape_configs = [
+          {
+            job_name = "journal";
+            journal = {
+              max_age = "12h";
+              labels = {
+                job = "systemd-journal";
+                host = "ulmo";
+              };
+            };
+            relabel_configs = [
+              { source_labels = [ "__journal__systemd_unit" ]; target_label = "unit"; }
+            ];
+          }
+        ];
+      };
+    };
+    
+    networking.firewall.allowedTCPPorts = [ 9004 ];
+  };
+}

systems/x86_64-linux/ulmo/default.nix

@@ -10,12 +10,19 @@
       authentication.authelia.enable = true;
       authentication.zitadel.enable = true;
 
+      development.forgejo.enable = true;
+
       networking.ssh.enable = true;
 
       media.enable = true;
       media.nfs.enable = true;
 
-      development.forgejo.enable = true;
+      observability = {
+        grafana.enable = true;
+        prometheus.enable = true;
+        loki.enable = true;
+        promtail.enable = true;
+      };
     };
 
     editor = {