Add observability stack: Alloy, Tempo, and OTEL support

- Add NixOS modules for Alloy and Tempo with default configs
- Update Grafana datasource config for Prometheus, Loki, Tempo
- Add Prometheus remote_write for Alloy
- Implement OTEL metrics/tracing/logging in arrtrix (Go)
- Enable Alloy and Tempo in ulmo system config

modules/nixos/services/observability/tempo/default.nix

@@ -0,0 +1,48 @@
+{ config, lib, namespace, ... }:
+let
+  inherit (lib) mkEnableOption mkIf;
+
+  cfg = config.${namespace}.services.observability.tempo;
+
+  httpPort = 9006;
+  grpcPort = 9008;
+  otlpGrpcPort = 9009;
+  otlpHttpPort = 9012;
+in
+{
+  options.${namespace}.services.observability.tempo = {
+    enable = mkEnableOption "enable Grafana Tempo";
+  };
+
+  config = mkIf cfg.enable {
+    services.tempo = {
+      enable = true;
+      settings = {
+        auth_enabled = false;
+        search_enabled = true;
+
+        server = {
+          http_listen_address = "0.0.0.0";
+          http_listen_port = httpPort;
+          grpc_listen_address = "127.0.0.1";
+          grpc_listen_port = grpcPort;
+        };
+
+        distributor.receivers.otlp.protocols = {
+          grpc.endpoint = "127.0.0.1:${builtins.toString otlpGrpcPort}";
+          http.endpoint = "127.0.0.1:${builtins.toString otlpHttpPort}";
+        };
+
+        storage.trace = {
+          backend = "local";
+          wal.path = "/var/lib/tempo/wal";
+          local.path = "/var/lib/tempo/traces";
+        };
+
+        compactor.compaction.block_retention = "168h";
+      };
+    };
+
+    networking.firewall.allowedTCPPorts = [ httpPort ];
+  };
+}