feat(infra): Tempo + Prometheus + Grafana observability backplane in compose (refs #122)

Three built images on the cg network with config baked in: Tempo (OTLP ingest,
ports 4317/4318), Prometheus, and Grafana with both datasources auto-provisioned.
Config lives in infra/observability/. The verify-observability step runs early in
CI verify-stack; the three containers are added to the failure log-dump list.
Tempo's Grafana plugin has no datasource /health method, so the check proves
Tempo reachability through Grafana's datasource proxy instead.

refs #122

.gitea/workflows/ci.yaml

@@ -142,6 +142,8 @@ jobs:
       # reaches green health" smoke (it replaces the old compose-smoke job).
       - name: Bring up the full stack & wait for health
         run: make verify-up
+      - name: Observability backplane (Grafana + Tempo + Prometheus datasources)
+        run: OBS_TIMEOUT=180 make verify-observability
       - name: ACL ↔ OpenZaak integration tests
         run: make verify-acl
       - name: OpenZaak → NRC notification delivery
@@ -157,7 +159,7 @@ jobs:
       # Log dump must precede teardown (which removes the containers).
       - name: Dump container logs on failure
         if: failure()
-        run: docker compose -f infra/docker-compose.yml logs --no-color --tail=100 oz-init openzaak nrc-init nrc-web nrc-celery nrc-beat flowable-db flowable-rest flowable-init keycloak acl bff domain projection-db event-subscriber projection-api self-service openbaar behandel 2>&1 || true
+        run: docker compose -f infra/docker-compose.yml logs --no-color --tail=100 oz-init openzaak nrc-init nrc-web nrc-celery nrc-beat flowable-db flowable-rest flowable-init keycloak acl bff domain projection-db event-subscriber projection-api self-service openbaar behandel tempo prometheus grafana 2>&1 || true
       - name: Tear down
         if: always()
         run: make down

infra/docker-compose.yml

@@ -524,6 +524,45 @@ services:
         condition: service_started
     networks: [cg]
 
+  # ── Observability backplane (S-16a, ADR-0023) ──────────────────────────────
+  # Grafana-native stack: Tempo ingests OTLP traces (the .NET services export
+  # straight to it — no collector hop, S-16b), Prometheus scrapes service
+  # /metrics (S-16c), and Grafana reads both with datasources auto-provisioned.
+  # Config is baked into small built images (COPY) rather than streamed into
+  # external config volumes like the upstream CG modules — these aren't verbatim
+  # peer images, so a built image is the simpler path that still reaches sibling
+  # containers on the CI runner. Not in WAIT_SVCS: run-observability-check.sh
+  # polls Grafana itself, so no in-image healthcheck tool is needed.
+  tempo:
+    build:
+      context: ./observability/tempo
+    image: register-referentie/tempo:dev
+    command: ["-config.file=/etc/tempo.yaml"]
+    networks: [cg]
+
+  prometheus:
+    build:
+      context: ./observability/prometheus
+    image: register-referentie/prometheus:dev
+    ports:
+      - "9090:9090"
+    networks: [cg]
+
+  grafana:
+    build:
+      context: ./observability/grafana
+    image: register-referentie/grafana:dev
+    environment:
+      GF_SECURITY_ADMIN_USER: admin
+      GF_SECURITY_ADMIN_PASSWORD: admin
+      GF_AUTH_ANONYMOUS_ENABLED: "true"
+    ports:
+      - "3000:3000"
+    depends_on:
+      - tempo
+      - prometheus
+    networks: [cg]
+
 volumes:
   oz-db:
   nrc-db:

infra/observability/grafana/Dockerfile

@@ -0,0 +1,4 @@
+# Grafana with datasources baked in via provisioning (S-16a, ADR-0023).
+# Dashboards (S-16c, #124) are added under provisioning/dashboards later.
+FROM grafana/grafana:11.3.0
+COPY provisioning/ /etc/grafana/provisioning/

infra/observability/grafana/provisioning/datasources/datasources.yaml

@@ -0,0 +1,17 @@
+# Auto-provisioned datasources (S-16a, ADR-0023). Fixed uids so dashboards (S-16c)
+# and the verify-observability check can reference them by a stable id.
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    uid: prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+
+  - name: Tempo
+    uid: tempo
+    type: tempo
+    access: proxy
+    url: http://tempo:3200

infra/observability/prometheus/Dockerfile

@@ -0,0 +1,2 @@
+FROM prom/prometheus:v2.55.1
+COPY prometheus.yml /etc/prometheus/prometheus.yml

infra/observability/prometheus/prometheus.yml

@@ -0,0 +1,10 @@
+# Prometheus scrape config (S-16a, ADR-0023). For the backplane slice it scrapes
+# only itself; the .NET services' /metrics scrape targets are added in S-16c
+# (#124) when the services expose metrics.
+global:
+  scrape_interval: 15s
+
+scrape_configs:
+  - job_name: prometheus
+    static_configs:
+      - targets: ['localhost:9090']

infra/observability/tempo/Dockerfile

@@ -0,0 +1,4 @@
+# Tempo with our config baked in — so it reaches sibling containers on the CI
+# runner without the external-config-volume dance the upstream CG images need.
+FROM grafana/tempo:2.6.1
+COPY tempo.yaml /etc/tempo.yaml

infra/observability/tempo/tempo.yaml

@@ -0,0 +1,27 @@
+# Grafana Tempo — single-binary, all-in-one, local storage (S-16a, ADR-0023).
+# Ingests OTLP directly (services export straight to Tempo; no collector hop).
+# Storage is ephemeral container fs — this is a local/CI demo backplane, not a
+# retention target. ponytail: local backend, swap for object storage if traces
+# must outlive the stack.
+server:
+  http_listen_port: 3200
+
+distributor:
+  receivers:
+    otlp:
+      protocols:
+        grpc:
+          endpoint: 0.0.0.0:4317
+        http:
+          endpoint: 0.0.0.0:4318
+
+ingester:
+  max_block_duration: 5m
+
+storage:
+  trace:
+    backend: local
+    local:
+      path: /var/tempo/blocks
+    wal:
+      path: /var/tempo/wal

infra/run-observability-check.sh

@@ -12,16 +12,19 @@ GRAFANA="${GRAFANA_URL:-http://localhost:3000}"
 AUTH="${GRAFANA_AUTH:-admin:admin}"
 TIMEOUT="${OBS_TIMEOUT:-60}"
 
-# poll <description> <url> <json-path-expr> <expected>
+# poll <description> <url> <python-bool-expr over `d` (parsed JSON)>
+# Passes when the expression prints True within TIMEOUT. Tempo's Grafana plugin
+# doesn't implement the datasource /health method, so instead of the Prometheus-
+# style health probe we prove reachability through Grafana's datasource proxy.
 poll() {
-  local desc="$1" url="$2" path="$3" want="$4" got deadline
+  local desc="$1" url="$2" expr="$3" got deadline
   deadline=$(( $(date +%s) + TIMEOUT ))
   while :; do
     got="$(curl -fsS -u "$AUTH" "$url" 2>/dev/null \
-      | python3 -c "import sys,json;print(json.load(sys.stdin)$path)" 2>/dev/null || true)"
+      | python3 -c "import sys,json; d=json.load(sys.stdin); print($expr)" 2>/dev/null || true)"
-    [ "$got" = "$want" ] && { echo "  ✓ $desc"; return 0; }
+    [ "$got" = "True" ] && { echo "  ✓ $desc"; return 0; }
     if [ "$(date +%s)" -ge "$deadline" ]; then
-      echo "  ✗ $desc — got '$got', want '$want' ($url)" >&2
+      echo "  ✗ $desc — check failed ($url)" >&2
       return 1
     fi
     sleep 2
@@ -29,7 +32,10 @@ poll() {
 }
 
 echo "Checking observability backplane at $GRAFANA ..."
-poll "Grafana is healthy"               "$GRAFANA/api/health"                             "['database']" "ok"
+poll "Grafana is healthy" \
-poll "Prometheus datasource reachable"  "$GRAFANA/api/datasources/uid/prometheus/health"  "['status']"   "OK"
+  "$GRAFANA/api/health" "d['database'] == 'ok'"
-poll "Tempo datasource reachable"       "$GRAFANA/api/datasources/uid/tempo/health"       "['status']"   "OK"
+poll "Prometheus datasource reachable" \
+  "$GRAFANA/api/datasources/uid/prometheus/health" "d['status'] == 'OK'"
+poll "Tempo datasource reachable (via Grafana proxy)" \
+  "$GRAFANA/api/datasources/proxy/uid/tempo/api/status/buildinfo" "bool(d.get('version'))"
 echo "Observability backplane OK."