diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index d9c6049..bf36a09 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -142,6 +142,8 @@ jobs: # reaches green health" smoke (it replaces the old compose-smoke job). - name: Bring up the full stack & wait for health run: make verify-up + - name: Observability backplane (Grafana + Tempo + Prometheus datasources) + run: OBS_TIMEOUT=180 make verify-observability - name: ACL ↔ OpenZaak integration tests run: make verify-acl - name: OpenZaak → NRC notification delivery @@ -157,7 +159,7 @@ jobs: # Log dump must precede teardown (which removes the containers). - name: Dump container logs on failure if: failure() - run: docker compose -f infra/docker-compose.yml logs --no-color --tail=100 oz-init openzaak nrc-init nrc-web nrc-celery nrc-beat flowable-db flowable-rest flowable-init keycloak acl bff domain projection-db event-subscriber projection-api self-service openbaar behandel 2>&1 || true + run: docker compose -f infra/docker-compose.yml logs --no-color --tail=100 oz-init openzaak nrc-init nrc-web nrc-celery nrc-beat flowable-db flowable-rest flowable-init keycloak acl bff domain projection-db event-subscriber projection-api self-service openbaar behandel tempo prometheus grafana 2>&1 || true - name: Tear down if: always() run: make down diff --git a/BACKLOG.md b/BACKLOG.md index 5857963..b87a48b 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -253,10 +253,16 @@ Split (issue #11 closed) into two independently-demoable slices per §13 — the **Outcome:** Beheer portal lets an admin view ZTC catalogi (read-only first), and manage the ACL's default-fill configuration via a CRUD UI. MFA on the medewerker realm enforced. -### S-16 · OpenTelemetry traces + Grafana dashboard +### S-16 · OpenTelemetry traces + Grafana dashboard *(split — #17 closed)* **Outcome:** Traces span portal → BFF → Domain → ACL → OpenZaak and portal → BFF → Domain → Flowable. Grafana dashboards pre-built for golden signals. +Split into independently deployable sub-slices (CLAUDE.md §13): + +- **S-16a** (#122) · Observability backplane — Grafana Tempo + Prometheus + Grafana in compose, datasources auto-provisioned (ADR-0023). No collector; config baked into built images. +- **S-16b** (#123) · Distributed traces across the five .NET services (OTLP → Tempo; nginx `traceparent` passthrough). Depends on S-16a. +- **S-16c** (#124) · Prometheus metrics + golden-signal Grafana dashboards. Depends on S-16a. + ### S-17 · Quartz.NET scheduler — herregistratie reminder sweep ✅ **Outcome:** Daily Quartz.NET cron job finds inscriptions within 90 days of their herregistratie deadline and reminds each (flag on the aggregate + log). No outbound notification and no domain event in v1 — the reminder is the persisted flag, surfaced on the read model (ADR-0022, #120). Quartz fires time-triggered sweeps; the existing pumps stay as queue-drainers. diff --git a/Makefile b/Makefile index a0c22bc..87e87c4 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ export DOCKER_HOST := unix://$(PODMAN_SOCK) endif endif -.PHONY: ci lint build unit mutation frontend integration verify verify-up verify-acl verify-nrc verify-projection verify-bff verify-domain verify-notifications smoke up down local verify-local local-down changelog openzaak-up openzaak-smoke openzaak-seed openzaak-down stack-up stack-smoke stack-down keycloak-up keycloak-smoke keycloak-down flowable-up flowable-smoke flowable-down help +.PHONY: ci lint build unit mutation frontend integration verify verify-up verify-acl verify-nrc verify-projection verify-bff verify-domain verify-observability verify-notifications smoke up down local verify-local local-down changelog openzaak-up openzaak-smoke openzaak-seed openzaak-down stack-up stack-smoke stack-down keycloak-up keycloak-smoke keycloak-down flowable-up flowable-smoke flowable-down help ## ci: run the full pipeline — lint, build, unit, mutation, frontend, verify (mirrors Gitea Actions) ## `verify` is the live-stack stage (full stack up once → ACL + notification checks). @@ -170,6 +170,11 @@ verify-bff: verify-e2e: bash infra/run-e2e-check.sh +## verify-observability: assert the observability backplane (Grafana + provisioned Tempo & +## Prometheus datasources) is live, against the already-running stack (S-16a). +verify-observability: + bash infra/run-observability-check.sh + ## verify: local mirror of the CI verify-stack job — full stack up once, all checks, ## tear down (always). For fast single-concern local iteration use `integration` ## (oz-only) or `verify-notifications` (oz+nrc) instead. diff --git a/docs/architecture/adr-0023-observability-stack.md b/docs/architecture/adr-0023-observability-stack.md new file mode 100644 index 0000000..7adb034 --- /dev/null +++ b/docs/architecture/adr-0023-observability-stack.md @@ -0,0 +1,74 @@ +# ADR-0023: Grafana-native observability stack (Tempo + Prometheus + Grafana) + +- **Status:** Accepted +- **Date:** 2026-07-23 +- **Deciders:** Respellion engineering +- **Slice:** S-16a (#122), first of the S-16 (#17) split + +## Context + +The PRD calls for "OpenTelemetry traces, Prometheus metrics; a local Grafana with +pre-built dashboards" (§80). S-16 was split (CLAUDE.md §13) into a backplane slice +(this one), distributed tracing (#123), and metrics + dashboards (#124). The +backplane must stand up first: a local, CI-friendly place for traces and metrics to +land, viewable in one UI, reaching green health within the 3-minute compose budget. + +Two shape decisions are non-obvious enough to record. + +## Decision + +**Run a Grafana-native stack — Grafana Tempo (traces) + Prometheus (metrics) + +Grafana (UI) — with the services exporting OTLP straight to Tempo (no collector), +and ship the config baked into small built images.** + +### Trace backend: Tempo (not Jaeger) + +Tempo keeps everything under one Grafana pane alongside metrics (and later logs), +which is exactly the "local Grafana with dashboards" the PRD asks for. Jaeger would +add a second UI and a second mental model for no benefit at this scale. + +### No OTLP collector + +Tempo ingests OTLP directly (gRPC 4317 / HTTP 4318) and Prometheus scrapes each +service's `/metrics`, so a collector would be a hop that processes nothing. Skipped. +If we later need fan-out, tail sampling, or log processing, a collector is an +additive change — the services already speak OTLP. + +### Config baked into built images, not config volumes + +The upstream Common Ground modules (OpenZaak, NRC, Keycloak, Flowable) run as +**verbatim** images and get their config streamed into external named volumes by +`infra/seed-config.sh`, because bind mounts don't reach sibling containers on the +CI runner (see `docs/runbooks/gitea-actions-gotchas.md`). The observability tools +are **not** peer modules we must run verbatim, so we take the simpler path: a +three-line `Dockerfile` per tool that `COPY`s its config in. This reaches sibling +containers everywhere (docker, podman, CI) with no seed step, no `CFG_VOLS` entry, +and no Makefile sprawl. + +### Verified, not assumed + +`infra/run-observability-check.sh` (the `verify-observability` step, run early in CI +`verify-stack`) asks Grafana to reach both datasources — Prometheus via its health +method, Tempo via the datasource proxy (Tempo's Grafana plugin implements no health +method) — so the check proves the datasources are actually wired, not merely that +containers started. The containers are not in `WAIT_SVCS`; the check polls Grafana +itself, so no in-image healthcheck tool is required. + +## Consequences + +**Positive** + +- One UI for traces + metrics + (future) logs. Config is versioned in + `infra/observability/` and self-contained in the images. +- Backplane is independent of app instrumentation — #123 and #124 build on it. + +**Negative / costs** + +- Three more images built each CI run (kept small; not on the health-gate list). +- Storage is ephemeral container fs — a demo backplane, not a retention target. + Object storage for Tempo / remote-write for Prometheus is a later concern. + +## Coupling rules touched (CLAUDE.md §8) + +None. The stack is passive infrastructure: services *push* OTLP and *expose* +`/metrics`; nothing in the stack calls into a service or a peer module. diff --git a/docs/demo-script.md b/docs/demo-script.md index eeebfbb..c4b7492 100644 --- a/docs/demo-script.md +++ b/docs/demo-script.md @@ -5,6 +5,31 @@ copy-pasteable walkthrough against a local `make up` stack. --- +## S-16a — observability backplane: Tempo + Prometheus + Grafana (#122, ADR-0023) + +**Outcome:** the compose stack now includes a Grafana-native observability backplane — **Tempo** (OTLP +trace ingest on 4317/4318), **Prometheus**, and **Grafana** with both datasources auto-provisioned. +Nothing is instrumented yet (traces land in S-16b, metrics + dashboards in S-16c); this slice stands the +backplane up and proves Grafana can reach both datasources. Config is baked into small built images +(`infra/observability/`) — no collector, no config-volume seeding. + +```bash +# 1. Bring the stack up, then assert the backplane is live (Grafana healthy + Tempo/Prometheus +# datasources reachable through Grafana). This is a CI verify-stack step. +make up +make verify-observability # → ✓ Grafana healthy ✓ Prometheus reachable ✓ Tempo reachable + +# 2. Or just the backplane, no full stack needed (no external egress): +docker compose -f infra/docker-compose.yml up -d --build tempo prometheus grafana +open http://localhost:3000 # Grafana (admin/admin) → Connections → Data sources: Prometheus + Tempo +open http://localhost:9090 # Prometheus +``` + +**The path:** services will export OTLP → **Tempo:4317** and expose `/metrics` ← **Prometheus** scrapes; +**Grafana** (:3000) reads both via provisioned datasources with fixed uids `tempo` / `prometheus`. + +--- + ## S-17 — herregistratie reminder sweep on a Quartz cron (#18, ADR-0022) **Outcome:** an inscription (INGESCHREVEN) now carries the moment it was entered in the register, from diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml index 99f18fe..fddfeef 100644 --- a/infra/docker-compose.yml +++ b/infra/docker-compose.yml @@ -524,6 +524,45 @@ services: condition: service_started networks: [cg] + # ── Observability backplane (S-16a, ADR-0023) ────────────────────────────── + # Grafana-native stack: Tempo ingests OTLP traces (the .NET services export + # straight to it — no collector hop, S-16b), Prometheus scrapes service + # /metrics (S-16c), and Grafana reads both with datasources auto-provisioned. + # Config is baked into small built images (COPY) rather than streamed into + # external config volumes like the upstream CG modules — these aren't verbatim + # peer images, so a built image is the simpler path that still reaches sibling + # containers on the CI runner. Not in WAIT_SVCS: run-observability-check.sh + # polls Grafana itself, so no in-image healthcheck tool is needed. + tempo: + build: + context: ./observability/tempo + image: register-referentie/tempo:dev + command: ["-config.file=/etc/tempo.yaml"] + networks: [cg] + + prometheus: + build: + context: ./observability/prometheus + image: register-referentie/prometheus:dev + ports: + - "9090:9090" + networks: [cg] + + grafana: + build: + context: ./observability/grafana + image: register-referentie/grafana:dev + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: admin + GF_AUTH_ANONYMOUS_ENABLED: "true" + ports: + - "3000:3000" + depends_on: + - tempo + - prometheus + networks: [cg] + volumes: oz-db: nrc-db: diff --git a/infra/observability/grafana/Dockerfile b/infra/observability/grafana/Dockerfile new file mode 100644 index 0000000..aefc6f7 --- /dev/null +++ b/infra/observability/grafana/Dockerfile @@ -0,0 +1,4 @@ +# Grafana with datasources baked in via provisioning (S-16a, ADR-0023). +# Dashboards (S-16c, #124) are added under provisioning/dashboards later. +FROM grafana/grafana:11.3.0 +COPY provisioning/ /etc/grafana/provisioning/ diff --git a/infra/observability/grafana/provisioning/datasources/datasources.yaml b/infra/observability/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..38d835f --- /dev/null +++ b/infra/observability/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,17 @@ +# Auto-provisioned datasources (S-16a, ADR-0023). Fixed uids so dashboards (S-16c) +# and the verify-observability check can reference them by a stable id. +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 diff --git a/infra/observability/prometheus/Dockerfile b/infra/observability/prometheus/Dockerfile new file mode 100644 index 0000000..4845915 --- /dev/null +++ b/infra/observability/prometheus/Dockerfile @@ -0,0 +1,2 @@ +FROM prom/prometheus:v2.55.1 +COPY prometheus.yml /etc/prometheus/prometheus.yml diff --git a/infra/observability/prometheus/prometheus.yml b/infra/observability/prometheus/prometheus.yml new file mode 100644 index 0000000..4e0f800 --- /dev/null +++ b/infra/observability/prometheus/prometheus.yml @@ -0,0 +1,10 @@ +# Prometheus scrape config (S-16a, ADR-0023). For the backplane slice it scrapes +# only itself; the .NET services' /metrics scrape targets are added in S-16c +# (#124) when the services expose metrics. +global: + scrape_interval: 15s + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ['localhost:9090'] diff --git a/infra/observability/tempo/Dockerfile b/infra/observability/tempo/Dockerfile new file mode 100644 index 0000000..f9f600d --- /dev/null +++ b/infra/observability/tempo/Dockerfile @@ -0,0 +1,4 @@ +# Tempo with our config baked in — so it reaches sibling containers on the CI +# runner without the external-config-volume dance the upstream CG images need. +FROM grafana/tempo:2.6.1 +COPY tempo.yaml /etc/tempo.yaml diff --git a/infra/observability/tempo/tempo.yaml b/infra/observability/tempo/tempo.yaml new file mode 100644 index 0000000..d5a4dbb --- /dev/null +++ b/infra/observability/tempo/tempo.yaml @@ -0,0 +1,27 @@ +# Grafana Tempo — single-binary, all-in-one, local storage (S-16a, ADR-0023). +# Ingests OTLP directly (services export straight to Tempo; no collector hop). +# Storage is ephemeral container fs — this is a local/CI demo backplane, not a +# retention target. ponytail: local backend, swap for object storage if traces +# must outlive the stack. +server: + http_listen_port: 3200 + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +ingester: + max_block_duration: 5m + +storage: + trace: + backend: local + local: + path: /var/tempo/blocks + wal: + path: /var/tempo/wal diff --git a/infra/run-observability-check.sh b/infra/run-observability-check.sh new file mode 100755 index 0000000..b340f64 --- /dev/null +++ b/infra/run-observability-check.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# +# S-16a (#122): assert the observability backplane is live against an ALREADY-RUNNING +# stack. Runs curl INSIDE the compose network (like the other verify checks) because +# the stack's published ports aren't on the CI runner's localhost — the stack is a set +# of sibling containers on the host daemon. It asks Grafana to reach its provisioned +# datasources — Prometheus via its health method, Tempo via the datasource proxy (Tempo's +# Grafana plugin implements no health method) — so it proves the datasources are wired, +# not merely that the containers started. Polls, so it tolerates a cold Grafana. +# +# Does NOT manage the stack lifecycle (the caller owns bring-up + teardown). +set -euo pipefail + +TIMEOUT="${OBS_TIMEOUT:-60}" +AUTH="${GRAFANA_AUTH:-admin:admin}" + +gf="$(docker ps -q --filter 'name=[-_]grafana[-_]' | head -1)" +[ -n "$gf" ] || { echo "ERROR: no running grafana container — bring the stack up first" >&2; exit 1; } +net="$(docker inspect -f '{{range $k,$_ := .NetworkSettings.Networks}}{{$k}}{{"\n"}}{{end}}' "$gf" | head -1)" +gf_ip="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$gf")" +base="http://$gf_ip:3000" +echo ">> grafana=$gf_ip network=$net" + +# Run curl inside a throwaway container on the stack network (reaches services by IP). +net_curl() { docker run --rm --network "$net" curlimages/curl:latest "$@"; } + +# poll +poll() { + local desc="$1" pat="$2"; shift 2 + local deadline=$(( $(date +%s) + TIMEOUT )) + while :; do + if net_curl -fsS "$@" 2>/dev/null | grep -Eq "$pat"; then echo " ✓ $desc"; return 0; fi + if [ "$(date +%s)" -ge "$deadline" ]; then echo " ✗ $desc ($*)" >&2; return 1; fi + sleep 3 + done +} + +echo "Checking observability backplane at $base ..." +poll "Grafana is healthy" \ + '"database":[[:space:]]*"ok"' "$base/api/health" +poll "Prometheus datasource reachable" \ + '"status":[[:space:]]*"OK"' -u "$AUTH" "$base/api/datasources/uid/prometheus/health" +poll "Tempo datasource reachable (via Grafana proxy)" \ + '"version"' -u "$AUTH" "$base/api/datasources/proxy/uid/tempo/api/status/buildinfo" +echo "Observability backplane OK."