Add observability stack: Alloy, Tempo, and OTEL support

- Add NixOS modules for Alloy and Tempo with default configs
- Update Grafana datasource config for Prometheus, Loki, Tempo
- Add Prometheus remote_write for Alloy
- Implement OTEL metrics/tracing/logging in arrtrix (Go)
- Enable Alloy and Tempo in ulmo system config

modules/nixos/services/observability/alloy/default.nix

@@ -0,0 +1,80 @@
+{ config, lib, namespace, ... }:
+let
+  inherit (builtins) toString;
+  inherit (lib) mkEnableOption mkIf;
+
+  cfg = config.${namespace}.services.observability.alloy;
+
+  httpPort = 9007;
+  otlpGrpcPort = 9010;
+  otlpHttpPort = 9011;
+  tempoOtlpGrpcPort = 9009;
+in
+{
+  options.${namespace}.services.observability.alloy = {
+    enable = mkEnableOption "enable Grafana Alloy";
+  };
+
+  config = mkIf cfg.enable {
+    services.alloy = {
+      enable = true;
+      configPath = "/etc/alloy";
+      extraFlags = [
+        "--disable-reporting"
+        "--server.http.listen-addr=0.0.0.0:${toString httpPort}"
+        "--storage.path=/var/lib/alloy"
+      ];
+    };
+
+    environment.etc."alloy/config.alloy".text = ''
+      otelcol.receiver.otlp "default" {
+        grpc {
+          endpoint = "127.0.0.1:${toString otlpGrpcPort}"
+        }
+
+        http {
+          endpoint = "127.0.0.1:${toString otlpHttpPort}"
+        }
+
+        output {
+          metrics = [otelcol.processor.batch.metrics.input]
+          traces  = [otelcol.processor.batch.traces.input]
+        }
+      }
+
+      otelcol.processor.batch "metrics" {
+        output {
+          metrics = [otelcol.exporter.prometheus.default.input]
+        }
+      }
+
+      otelcol.processor.batch "traces" {
+        output {
+          traces = [otelcol.exporter.otlp.tempo.input]
+        }
+      }
+
+      otelcol.exporter.prometheus "default" {
+        forward_to = [prometheus.remote_write.local.receiver]
+      }
+
+      prometheus.remote_write "local" {
+        endpoint {
+          url = "http://127.0.0.1:${toString config.services.prometheus.port}/api/v1/write"
+        }
+      }
+
+      otelcol.exporter.otlp "tempo" {
+        client {
+          endpoint = "127.0.0.1:${toString tempoOtlpGrpcPort}"
+
+          tls {
+            insecure = true
+          }
+        }
+      }
+    '';
+
+    networking.firewall.allowedTCPPorts = [ httpPort ];
+  };
+}

modules/nixos/services/observability/grafana/default.nix

@@ -102,23 +102,43 @@ in {
           };
 
           datasources.settings.datasources = [
-            {
-              name = "Prometheus";
-              type = "prometheus";
-              url = "http://localhost:9005";
-              isDefault = true;
-              editable = false;
-            }
+             {
+               name = "Prometheus";
+               uid = "prometheus";
+               type = "prometheus";
+               url = "http://localhost:9002";
+               isDefault = true;
+               editable = false;
+             }
 
-            {
-              name = "Loki";
-              type = "loki";
-              url = "http://localhost:9003";
-              editable = false;
-            }
-          ];
-        };
-      };
+             {
+               name = "Loki";
+               uid = "loki";
+               type = "loki";
+               url = "http://localhost:9003";
+               editable = false;
+             }
+
+             {
+               name = "Tempo";
+               uid = "tempo";
+               type = "tempo";
+               url = "http://localhost:9006";
+               editable = false;
+               jsonData = {
+                 nodeGraph.enabled = true;
+                 serviceMap.datasourceUid = "prometheus";
+                 tracesToLogsV2 = {
+                   datasourceUid = "loki";
+                   filterByTraceID = true;
+                   spanStartTimeShift = "-1h";
+                   spanEndTimeShift = "1h";
+                 };
+               };
+             }
+           ];
+         };
+       };
 
       postgresql = {
         enable = true;

modules/nixos/services/observability/prometheus/default.nix

@@ -1,7 +1,7 @@
 { pkgs, config, lib, namespace, ... }:
 let
   inherit (builtins) toString;
-  inherit (lib) mkIf mkEnableOption;
+  inherit (lib) mkEnableOption mkIf optionals;
 
   cfg = config.${namespace}.services.observability.prometheus;
 in
@@ -14,6 +14,9 @@ in
     services.prometheus = {
       enable = true;
       port = 9002;
+      extraFlags = optionals config.${namespace}.services.observability.alloy.enable [
+        "--web.enable-remote-write-receiver"
+      ];
 
       globalConfig.scrape_interval = "15s";
 
@@ -31,6 +34,22 @@ in
             { targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ]; }
           ];
         }
+      ]
+      ++ optionals config.${namespace}.services.observability.alloy.enable [
+        {
+          job_name = "alloy";
+          static_configs = [
+            { targets = [ "localhost:9007" ]; }
+          ];
+        }
+      ]
+      ++ optionals config.${namespace}.services.observability.tempo.enable [
+        {
+          job_name = "tempo";
+          static_configs = [
+            { targets = [ "localhost:9006" ]; }
+          ];
+        }
       ];
 
       exporters = {

modules/nixos/services/observability/tempo/default.nix

@@ -0,0 +1,48 @@
+{ config, lib, namespace, ... }:
+let
+  inherit (lib) mkEnableOption mkIf;
+
+  cfg = config.${namespace}.services.observability.tempo;
+
+  httpPort = 9006;
+  grpcPort = 9008;
+  otlpGrpcPort = 9009;
+  otlpHttpPort = 9012;
+in
+{
+  options.${namespace}.services.observability.tempo = {
+    enable = mkEnableOption "enable Grafana Tempo";
+  };
+
+  config = mkIf cfg.enable {
+    services.tempo = {
+      enable = true;
+      settings = {
+        auth_enabled = false;
+        search_enabled = true;
+
+        server = {
+          http_listen_address = "0.0.0.0";
+          http_listen_port = httpPort;
+          grpc_listen_address = "127.0.0.1";
+          grpc_listen_port = grpcPort;
+        };
+
+        distributor.receivers.otlp.protocols = {
+          grpc.endpoint = "127.0.0.1:${builtins.toString otlpGrpcPort}";
+          http.endpoint = "127.0.0.1:${builtins.toString otlpHttpPort}";
+        };
+
+        storage.trace = {
+          backend = "local";
+          wal.path = "/var/lib/tempo/wal";
+          local.path = "/var/lib/tempo/traces";
+        };
+
+        compactor.compaction.block_retention = "168h";
+      };
+    };
+
+    networking.firewall.allowedTCPPorts = [ httpPort ];
+  };
+}