From f29a8598fd02acdeca83cfbb33b5935139efb3f1 Mon Sep 17 00:00:00 2001 From: Niek Otten Date: Thu, 23 Jul 2026 12:46:54 +0200 Subject: [PATCH] feat(infra): Tempo + Prometheus + Grafana observability backplane in compose (refs #122) Three built images on the cg network with config baked in: Tempo (OTLP ingest, ports 4317/4318), Prometheus, and Grafana with both datasources auto-provisioned. Config lives in infra/observability/. The verify-observability step runs early in CI verify-stack; the three containers are added to the failure log-dump list. Tempo's Grafana plugin has no datasource /health method, so the check proves Tempo reachability through Grafana's datasource proxy instead. refs #122 --- .gitea/workflows/ci.yaml | 4 +- infra/docker-compose.yml | 39 +++++++++++++++++++ infra/observability/grafana/Dockerfile | 4 ++ .../provisioning/datasources/datasources.yaml | 17 ++++++++ infra/observability/prometheus/Dockerfile | 2 + infra/observability/prometheus/prometheus.yml | 10 +++++ infra/observability/tempo/Dockerfile | 4 ++ infra/observability/tempo/tempo.yaml | 27 +++++++++++++ infra/run-observability-check.sh | 22 +++++++---- 9 files changed, 120 insertions(+), 9 deletions(-) create mode 100644 infra/observability/grafana/Dockerfile create mode 100644 infra/observability/grafana/provisioning/datasources/datasources.yaml create mode 100644 infra/observability/prometheus/Dockerfile create mode 100644 infra/observability/prometheus/prometheus.yml create mode 100644 infra/observability/tempo/Dockerfile create mode 100644 infra/observability/tempo/tempo.yaml diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index d9c6049..bf36a09 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -142,6 +142,8 @@ jobs: # reaches green health" smoke (it replaces the old compose-smoke job). - name: Bring up the full stack & wait for health run: make verify-up + - name: Observability backplane (Grafana + Tempo + Prometheus datasources) + run: OBS_TIMEOUT=180 make verify-observability - name: ACL ↔ OpenZaak integration tests run: make verify-acl - name: OpenZaak → NRC notification delivery @@ -157,7 +159,7 @@ jobs: # Log dump must precede teardown (which removes the containers). - name: Dump container logs on failure if: failure() - run: docker compose -f infra/docker-compose.yml logs --no-color --tail=100 oz-init openzaak nrc-init nrc-web nrc-celery nrc-beat flowable-db flowable-rest flowable-init keycloak acl bff domain projection-db event-subscriber projection-api self-service openbaar behandel 2>&1 || true + run: docker compose -f infra/docker-compose.yml logs --no-color --tail=100 oz-init openzaak nrc-init nrc-web nrc-celery nrc-beat flowable-db flowable-rest flowable-init keycloak acl bff domain projection-db event-subscriber projection-api self-service openbaar behandel tempo prometheus grafana 2>&1 || true - name: Tear down if: always() run: make down diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml index 99f18fe..fddfeef 100644 --- a/infra/docker-compose.yml +++ b/infra/docker-compose.yml @@ -524,6 +524,45 @@ services: condition: service_started networks: [cg] + # ── Observability backplane (S-16a, ADR-0023) ────────────────────────────── + # Grafana-native stack: Tempo ingests OTLP traces (the .NET services export + # straight to it — no collector hop, S-16b), Prometheus scrapes service + # /metrics (S-16c), and Grafana reads both with datasources auto-provisioned. + # Config is baked into small built images (COPY) rather than streamed into + # external config volumes like the upstream CG modules — these aren't verbatim + # peer images, so a built image is the simpler path that still reaches sibling + # containers on the CI runner. Not in WAIT_SVCS: run-observability-check.sh + # polls Grafana itself, so no in-image healthcheck tool is needed. + tempo: + build: + context: ./observability/tempo + image: register-referentie/tempo:dev + command: ["-config.file=/etc/tempo.yaml"] + networks: [cg] + + prometheus: + build: + context: ./observability/prometheus + image: register-referentie/prometheus:dev + ports: + - "9090:9090" + networks: [cg] + + grafana: + build: + context: ./observability/grafana + image: register-referentie/grafana:dev + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: admin + GF_AUTH_ANONYMOUS_ENABLED: "true" + ports: + - "3000:3000" + depends_on: + - tempo + - prometheus + networks: [cg] + volumes: oz-db: nrc-db: diff --git a/infra/observability/grafana/Dockerfile b/infra/observability/grafana/Dockerfile new file mode 100644 index 0000000..aefc6f7 --- /dev/null +++ b/infra/observability/grafana/Dockerfile @@ -0,0 +1,4 @@ +# Grafana with datasources baked in via provisioning (S-16a, ADR-0023). +# Dashboards (S-16c, #124) are added under provisioning/dashboards later. +FROM grafana/grafana:11.3.0 +COPY provisioning/ /etc/grafana/provisioning/ diff --git a/infra/observability/grafana/provisioning/datasources/datasources.yaml b/infra/observability/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..38d835f --- /dev/null +++ b/infra/observability/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,17 @@ +# Auto-provisioned datasources (S-16a, ADR-0023). Fixed uids so dashboards (S-16c) +# and the verify-observability check can reference them by a stable id. +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 diff --git a/infra/observability/prometheus/Dockerfile b/infra/observability/prometheus/Dockerfile new file mode 100644 index 0000000..4845915 --- /dev/null +++ b/infra/observability/prometheus/Dockerfile @@ -0,0 +1,2 @@ +FROM prom/prometheus:v2.55.1 +COPY prometheus.yml /etc/prometheus/prometheus.yml diff --git a/infra/observability/prometheus/prometheus.yml b/infra/observability/prometheus/prometheus.yml new file mode 100644 index 0000000..4e0f800 --- /dev/null +++ b/infra/observability/prometheus/prometheus.yml @@ -0,0 +1,10 @@ +# Prometheus scrape config (S-16a, ADR-0023). For the backplane slice it scrapes +# only itself; the .NET services' /metrics scrape targets are added in S-16c +# (#124) when the services expose metrics. +global: + scrape_interval: 15s + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ['localhost:9090'] diff --git a/infra/observability/tempo/Dockerfile b/infra/observability/tempo/Dockerfile new file mode 100644 index 0000000..f9f600d --- /dev/null +++ b/infra/observability/tempo/Dockerfile @@ -0,0 +1,4 @@ +# Tempo with our config baked in — so it reaches sibling containers on the CI +# runner without the external-config-volume dance the upstream CG images need. +FROM grafana/tempo:2.6.1 +COPY tempo.yaml /etc/tempo.yaml diff --git a/infra/observability/tempo/tempo.yaml b/infra/observability/tempo/tempo.yaml new file mode 100644 index 0000000..d5a4dbb --- /dev/null +++ b/infra/observability/tempo/tempo.yaml @@ -0,0 +1,27 @@ +# Grafana Tempo — single-binary, all-in-one, local storage (S-16a, ADR-0023). +# Ingests OTLP directly (services export straight to Tempo; no collector hop). +# Storage is ephemeral container fs — this is a local/CI demo backplane, not a +# retention target. ponytail: local backend, swap for object storage if traces +# must outlive the stack. +server: + http_listen_port: 3200 + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +ingester: + max_block_duration: 5m + +storage: + trace: + backend: local + local: + path: /var/tempo/blocks + wal: + path: /var/tempo/wal diff --git a/infra/run-observability-check.sh b/infra/run-observability-check.sh index 69b7be5..a8d01fa 100755 --- a/infra/run-observability-check.sh +++ b/infra/run-observability-check.sh @@ -12,16 +12,19 @@ GRAFANA="${GRAFANA_URL:-http://localhost:3000}" AUTH="${GRAFANA_AUTH:-admin:admin}" TIMEOUT="${OBS_TIMEOUT:-60}" -# poll +# poll +# Passes when the expression prints True within TIMEOUT. Tempo's Grafana plugin +# doesn't implement the datasource /health method, so instead of the Prometheus- +# style health probe we prove reachability through Grafana's datasource proxy. poll() { - local desc="$1" url="$2" path="$3" want="$4" got deadline + local desc="$1" url="$2" expr="$3" got deadline deadline=$(( $(date +%s) + TIMEOUT )) while :; do got="$(curl -fsS -u "$AUTH" "$url" 2>/dev/null \ - | python3 -c "import sys,json;print(json.load(sys.stdin)$path)" 2>/dev/null || true)" - [ "$got" = "$want" ] && { echo " ✓ $desc"; return 0; } + | python3 -c "import sys,json; d=json.load(sys.stdin); print($expr)" 2>/dev/null || true)" + [ "$got" = "True" ] && { echo " ✓ $desc"; return 0; } if [ "$(date +%s)" -ge "$deadline" ]; then - echo " ✗ $desc — got '$got', want '$want' ($url)" >&2 + echo " ✗ $desc — check failed ($url)" >&2 return 1 fi sleep 2 @@ -29,7 +32,10 @@ poll() { } echo "Checking observability backplane at $GRAFANA ..." -poll "Grafana is healthy" "$GRAFANA/api/health" "['database']" "ok" -poll "Prometheus datasource reachable" "$GRAFANA/api/datasources/uid/prometheus/health" "['status']" "OK" -poll "Tempo datasource reachable" "$GRAFANA/api/datasources/uid/tempo/health" "['status']" "OK" +poll "Grafana is healthy" \ + "$GRAFANA/api/health" "d['database'] == 'ok'" +poll "Prometheus datasource reachable" \ + "$GRAFANA/api/datasources/uid/prometheus/health" "d['status'] == 'OK'" +poll "Tempo datasource reachable (via Grafana proxy)" \ + "$GRAFANA/api/datasources/proxy/uid/tempo/api/status/buildinfo" "bool(d.get('version'))" echo "Observability backplane OK."