From f12e9ce746441bce988a99d3d7f8a0ee8878361f Mon Sep 17 00:00:00 2001 From: Niek Otten Date: Thu, 23 Jul 2026 12:41:58 +0200 Subject: [PATCH 1/4] test(infra): verify-observability asserts Grafana + Tempo/Prometheus datasources are live (refs #122) Polls Grafana's datasource-health endpoints so it proves Grafana can actually reach Tempo and Prometheus over the cg network, not merely that the containers started. Fails until the backplane is wired (next commit). refs #122 --- Makefile | 7 ++++++- infra/run-observability-check.sh | 35 ++++++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) create mode 100755 infra/run-observability-check.sh diff --git a/Makefile b/Makefile index a0c22bc..87e87c4 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ export DOCKER_HOST := unix://$(PODMAN_SOCK) endif endif -.PHONY: ci lint build unit mutation frontend integration verify verify-up verify-acl verify-nrc verify-projection verify-bff verify-domain verify-notifications smoke up down local verify-local local-down changelog openzaak-up openzaak-smoke openzaak-seed openzaak-down stack-up stack-smoke stack-down keycloak-up keycloak-smoke keycloak-down flowable-up flowable-smoke flowable-down help +.PHONY: ci lint build unit mutation frontend integration verify verify-up verify-acl verify-nrc verify-projection verify-bff verify-domain verify-observability verify-notifications smoke up down local verify-local local-down changelog openzaak-up openzaak-smoke openzaak-seed openzaak-down stack-up stack-smoke stack-down keycloak-up keycloak-smoke keycloak-down flowable-up flowable-smoke flowable-down help ## ci: run the full pipeline — lint, build, unit, mutation, frontend, verify (mirrors Gitea Actions) ## `verify` is the live-stack stage (full stack up once → ACL + notification checks). @@ -170,6 +170,11 @@ verify-bff: verify-e2e: bash infra/run-e2e-check.sh +## verify-observability: assert the observability backplane (Grafana + provisioned Tempo & +## Prometheus datasources) is live, against the already-running stack (S-16a). +verify-observability: + bash infra/run-observability-check.sh + ## verify: local mirror of the CI verify-stack job — full stack up once, all checks, ## tear down (always). For fast single-concern local iteration use `integration` ## (oz-only) or `verify-notifications` (oz+nrc) instead. diff --git a/infra/run-observability-check.sh b/infra/run-observability-check.sh new file mode 100755 index 0000000..69b7be5 --- /dev/null +++ b/infra/run-observability-check.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# +# S-16a (#122): assert the observability backplane is live against the running stack. +# Not just "the containers started" — it asks Grafana to health-check its provisioned +# Tempo and Prometheus datasources, which proves Grafana can actually reach both over +# the `cg` network and that provisioning landed. Polls, so it tolerates a cold Grafana. +# +# Usage: run-observability-check.sh (override GRAFANA_URL / GRAFANA_AUTH / OBS_TIMEOUT) +set -euo pipefail + +GRAFANA="${GRAFANA_URL:-http://localhost:3000}" +AUTH="${GRAFANA_AUTH:-admin:admin}" +TIMEOUT="${OBS_TIMEOUT:-60}" + +# poll +poll() { + local desc="$1" url="$2" path="$3" want="$4" got deadline + deadline=$(( $(date +%s) + TIMEOUT )) + while :; do + got="$(curl -fsS -u "$AUTH" "$url" 2>/dev/null \ + | python3 -c "import sys,json;print(json.load(sys.stdin)$path)" 2>/dev/null || true)" + [ "$got" = "$want" ] && { echo " ✓ $desc"; return 0; } + if [ "$(date +%s)" -ge "$deadline" ]; then + echo " ✗ $desc — got '$got', want '$want' ($url)" >&2 + return 1 + fi + sleep 2 + done +} + +echo "Checking observability backplane at $GRAFANA ..." +poll "Grafana is healthy" "$GRAFANA/api/health" "['database']" "ok" +poll "Prometheus datasource reachable" "$GRAFANA/api/datasources/uid/prometheus/health" "['status']" "OK" +poll "Tempo datasource reachable" "$GRAFANA/api/datasources/uid/tempo/health" "['status']" "OK" +echo "Observability backplane OK." -- 2.54.0 From f29a8598fd02acdeca83cfbb33b5935139efb3f1 Mon Sep 17 00:00:00 2001 From: Niek Otten Date: Thu, 23 Jul 2026 12:46:54 +0200 Subject: [PATCH 2/4] feat(infra): Tempo + Prometheus + Grafana observability backplane in compose (refs #122) Three built images on the cg network with config baked in: Tempo (OTLP ingest, ports 4317/4318), Prometheus, and Grafana with both datasources auto-provisioned. Config lives in infra/observability/. The verify-observability step runs early in CI verify-stack; the three containers are added to the failure log-dump list. Tempo's Grafana plugin has no datasource /health method, so the check proves Tempo reachability through Grafana's datasource proxy instead. refs #122 --- .gitea/workflows/ci.yaml | 4 +- infra/docker-compose.yml | 39 +++++++++++++++++++ infra/observability/grafana/Dockerfile | 4 ++ .../provisioning/datasources/datasources.yaml | 17 ++++++++ infra/observability/prometheus/Dockerfile | 2 + infra/observability/prometheus/prometheus.yml | 10 +++++ infra/observability/tempo/Dockerfile | 4 ++ infra/observability/tempo/tempo.yaml | 27 +++++++++++++ infra/run-observability-check.sh | 22 +++++++---- 9 files changed, 120 insertions(+), 9 deletions(-) create mode 100644 infra/observability/grafana/Dockerfile create mode 100644 infra/observability/grafana/provisioning/datasources/datasources.yaml create mode 100644 infra/observability/prometheus/Dockerfile create mode 100644 infra/observability/prometheus/prometheus.yml create mode 100644 infra/observability/tempo/Dockerfile create mode 100644 infra/observability/tempo/tempo.yaml diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index d9c6049..bf36a09 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -142,6 +142,8 @@ jobs: # reaches green health" smoke (it replaces the old compose-smoke job). - name: Bring up the full stack & wait for health run: make verify-up + - name: Observability backplane (Grafana + Tempo + Prometheus datasources) + run: OBS_TIMEOUT=180 make verify-observability - name: ACL ↔ OpenZaak integration tests run: make verify-acl - name: OpenZaak → NRC notification delivery @@ -157,7 +159,7 @@ jobs: # Log dump must precede teardown (which removes the containers). - name: Dump container logs on failure if: failure() - run: docker compose -f infra/docker-compose.yml logs --no-color --tail=100 oz-init openzaak nrc-init nrc-web nrc-celery nrc-beat flowable-db flowable-rest flowable-init keycloak acl bff domain projection-db event-subscriber projection-api self-service openbaar behandel 2>&1 || true + run: docker compose -f infra/docker-compose.yml logs --no-color --tail=100 oz-init openzaak nrc-init nrc-web nrc-celery nrc-beat flowable-db flowable-rest flowable-init keycloak acl bff domain projection-db event-subscriber projection-api self-service openbaar behandel tempo prometheus grafana 2>&1 || true - name: Tear down if: always() run: make down diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml index 99f18fe..fddfeef 100644 --- a/infra/docker-compose.yml +++ b/infra/docker-compose.yml @@ -524,6 +524,45 @@ services: condition: service_started networks: [cg] + # ── Observability backplane (S-16a, ADR-0023) ────────────────────────────── + # Grafana-native stack: Tempo ingests OTLP traces (the .NET services export + # straight to it — no collector hop, S-16b), Prometheus scrapes service + # /metrics (S-16c), and Grafana reads both with datasources auto-provisioned. + # Config is baked into small built images (COPY) rather than streamed into + # external config volumes like the upstream CG modules — these aren't verbatim + # peer images, so a built image is the simpler path that still reaches sibling + # containers on the CI runner. Not in WAIT_SVCS: run-observability-check.sh + # polls Grafana itself, so no in-image healthcheck tool is needed. + tempo: + build: + context: ./observability/tempo + image: register-referentie/tempo:dev + command: ["-config.file=/etc/tempo.yaml"] + networks: [cg] + + prometheus: + build: + context: ./observability/prometheus + image: register-referentie/prometheus:dev + ports: + - "9090:9090" + networks: [cg] + + grafana: + build: + context: ./observability/grafana + image: register-referentie/grafana:dev + environment: + GF_SECURITY_ADMIN_USER: admin + GF_SECURITY_ADMIN_PASSWORD: admin + GF_AUTH_ANONYMOUS_ENABLED: "true" + ports: + - "3000:3000" + depends_on: + - tempo + - prometheus + networks: [cg] + volumes: oz-db: nrc-db: diff --git a/infra/observability/grafana/Dockerfile b/infra/observability/grafana/Dockerfile new file mode 100644 index 0000000..aefc6f7 --- /dev/null +++ b/infra/observability/grafana/Dockerfile @@ -0,0 +1,4 @@ +# Grafana with datasources baked in via provisioning (S-16a, ADR-0023). +# Dashboards (S-16c, #124) are added under provisioning/dashboards later. +FROM grafana/grafana:11.3.0 +COPY provisioning/ /etc/grafana/provisioning/ diff --git a/infra/observability/grafana/provisioning/datasources/datasources.yaml b/infra/observability/grafana/provisioning/datasources/datasources.yaml new file mode 100644 index 0000000..38d835f --- /dev/null +++ b/infra/observability/grafana/provisioning/datasources/datasources.yaml @@ -0,0 +1,17 @@ +# Auto-provisioned datasources (S-16a, ADR-0023). Fixed uids so dashboards (S-16c) +# and the verify-observability check can reference them by a stable id. +apiVersion: 1 + +datasources: + - name: Prometheus + uid: prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + + - name: Tempo + uid: tempo + type: tempo + access: proxy + url: http://tempo:3200 diff --git a/infra/observability/prometheus/Dockerfile b/infra/observability/prometheus/Dockerfile new file mode 100644 index 0000000..4845915 --- /dev/null +++ b/infra/observability/prometheus/Dockerfile @@ -0,0 +1,2 @@ +FROM prom/prometheus:v2.55.1 +COPY prometheus.yml /etc/prometheus/prometheus.yml diff --git a/infra/observability/prometheus/prometheus.yml b/infra/observability/prometheus/prometheus.yml new file mode 100644 index 0000000..4e0f800 --- /dev/null +++ b/infra/observability/prometheus/prometheus.yml @@ -0,0 +1,10 @@ +# Prometheus scrape config (S-16a, ADR-0023). For the backplane slice it scrapes +# only itself; the .NET services' /metrics scrape targets are added in S-16c +# (#124) when the services expose metrics. +global: + scrape_interval: 15s + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ['localhost:9090'] diff --git a/infra/observability/tempo/Dockerfile b/infra/observability/tempo/Dockerfile new file mode 100644 index 0000000..f9f600d --- /dev/null +++ b/infra/observability/tempo/Dockerfile @@ -0,0 +1,4 @@ +# Tempo with our config baked in — so it reaches sibling containers on the CI +# runner without the external-config-volume dance the upstream CG images need. +FROM grafana/tempo:2.6.1 +COPY tempo.yaml /etc/tempo.yaml diff --git a/infra/observability/tempo/tempo.yaml b/infra/observability/tempo/tempo.yaml new file mode 100644 index 0000000..d5a4dbb --- /dev/null +++ b/infra/observability/tempo/tempo.yaml @@ -0,0 +1,27 @@ +# Grafana Tempo — single-binary, all-in-one, local storage (S-16a, ADR-0023). +# Ingests OTLP directly (services export straight to Tempo; no collector hop). +# Storage is ephemeral container fs — this is a local/CI demo backplane, not a +# retention target. ponytail: local backend, swap for object storage if traces +# must outlive the stack. +server: + http_listen_port: 3200 + +distributor: + receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +ingester: + max_block_duration: 5m + +storage: + trace: + backend: local + local: + path: /var/tempo/blocks + wal: + path: /var/tempo/wal diff --git a/infra/run-observability-check.sh b/infra/run-observability-check.sh index 69b7be5..a8d01fa 100755 --- a/infra/run-observability-check.sh +++ b/infra/run-observability-check.sh @@ -12,16 +12,19 @@ GRAFANA="${GRAFANA_URL:-http://localhost:3000}" AUTH="${GRAFANA_AUTH:-admin:admin}" TIMEOUT="${OBS_TIMEOUT:-60}" -# poll +# poll +# Passes when the expression prints True within TIMEOUT. Tempo's Grafana plugin +# doesn't implement the datasource /health method, so instead of the Prometheus- +# style health probe we prove reachability through Grafana's datasource proxy. poll() { - local desc="$1" url="$2" path="$3" want="$4" got deadline + local desc="$1" url="$2" expr="$3" got deadline deadline=$(( $(date +%s) + TIMEOUT )) while :; do got="$(curl -fsS -u "$AUTH" "$url" 2>/dev/null \ - | python3 -c "import sys,json;print(json.load(sys.stdin)$path)" 2>/dev/null || true)" - [ "$got" = "$want" ] && { echo " ✓ $desc"; return 0; } + | python3 -c "import sys,json; d=json.load(sys.stdin); print($expr)" 2>/dev/null || true)" + [ "$got" = "True" ] && { echo " ✓ $desc"; return 0; } if [ "$(date +%s)" -ge "$deadline" ]; then - echo " ✗ $desc — got '$got', want '$want' ($url)" >&2 + echo " ✗ $desc — check failed ($url)" >&2 return 1 fi sleep 2 @@ -29,7 +32,10 @@ poll() { } echo "Checking observability backplane at $GRAFANA ..." -poll "Grafana is healthy" "$GRAFANA/api/health" "['database']" "ok" -poll "Prometheus datasource reachable" "$GRAFANA/api/datasources/uid/prometheus/health" "['status']" "OK" -poll "Tempo datasource reachable" "$GRAFANA/api/datasources/uid/tempo/health" "['status']" "OK" +poll "Grafana is healthy" \ + "$GRAFANA/api/health" "d['database'] == 'ok'" +poll "Prometheus datasource reachable" \ + "$GRAFANA/api/datasources/uid/prometheus/health" "d['status'] == 'OK'" +poll "Tempo datasource reachable (via Grafana proxy)" \ + "$GRAFANA/api/datasources/proxy/uid/tempo/api/status/buildinfo" "bool(d.get('version'))" echo "Observability backplane OK." -- 2.54.0 From 43c6c8e0d202df67067a535b346ffb48b236f071 Mon Sep 17 00:00:00 2001 From: Niek Otten Date: Thu, 23 Jul 2026 12:48:02 +0200 Subject: [PATCH 3/4] docs: ADR-0023 + demo note + backlog sync for the observability backplane (refs #122) ADR-0023 records the Grafana-native stack (Tempo, no collector, config baked into images); demo-script and BACKLOG describe S-16a and the S-16 split. refs #122 --- BACKLOG.md | 8 +- .../adr-0023-observability-stack.md | 74 +++++++++++++++++++ docs/demo-script.md | 25 +++++++ 3 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 docs/architecture/adr-0023-observability-stack.md diff --git a/BACKLOG.md b/BACKLOG.md index 5857963..b87a48b 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -253,10 +253,16 @@ Split (issue #11 closed) into two independently-demoable slices per §13 — the **Outcome:** Beheer portal lets an admin view ZTC catalogi (read-only first), and manage the ACL's default-fill configuration via a CRUD UI. MFA on the medewerker realm enforced. -### S-16 · OpenTelemetry traces + Grafana dashboard +### S-16 · OpenTelemetry traces + Grafana dashboard *(split — #17 closed)* **Outcome:** Traces span portal → BFF → Domain → ACL → OpenZaak and portal → BFF → Domain → Flowable. Grafana dashboards pre-built for golden signals. +Split into independently deployable sub-slices (CLAUDE.md §13): + +- **S-16a** (#122) · Observability backplane — Grafana Tempo + Prometheus + Grafana in compose, datasources auto-provisioned (ADR-0023). No collector; config baked into built images. +- **S-16b** (#123) · Distributed traces across the five .NET services (OTLP → Tempo; nginx `traceparent` passthrough). Depends on S-16a. +- **S-16c** (#124) · Prometheus metrics + golden-signal Grafana dashboards. Depends on S-16a. + ### S-17 · Quartz.NET scheduler — herregistratie reminder sweep ✅ **Outcome:** Daily Quartz.NET cron job finds inscriptions within 90 days of their herregistratie deadline and reminds each (flag on the aggregate + log). No outbound notification and no domain event in v1 — the reminder is the persisted flag, surfaced on the read model (ADR-0022, #120). Quartz fires time-triggered sweeps; the existing pumps stay as queue-drainers. diff --git a/docs/architecture/adr-0023-observability-stack.md b/docs/architecture/adr-0023-observability-stack.md new file mode 100644 index 0000000..7adb034 --- /dev/null +++ b/docs/architecture/adr-0023-observability-stack.md @@ -0,0 +1,74 @@ +# ADR-0023: Grafana-native observability stack (Tempo + Prometheus + Grafana) + +- **Status:** Accepted +- **Date:** 2026-07-23 +- **Deciders:** Respellion engineering +- **Slice:** S-16a (#122), first of the S-16 (#17) split + +## Context + +The PRD calls for "OpenTelemetry traces, Prometheus metrics; a local Grafana with +pre-built dashboards" (§80). S-16 was split (CLAUDE.md §13) into a backplane slice +(this one), distributed tracing (#123), and metrics + dashboards (#124). The +backplane must stand up first: a local, CI-friendly place for traces and metrics to +land, viewable in one UI, reaching green health within the 3-minute compose budget. + +Two shape decisions are non-obvious enough to record. + +## Decision + +**Run a Grafana-native stack — Grafana Tempo (traces) + Prometheus (metrics) + +Grafana (UI) — with the services exporting OTLP straight to Tempo (no collector), +and ship the config baked into small built images.** + +### Trace backend: Tempo (not Jaeger) + +Tempo keeps everything under one Grafana pane alongside metrics (and later logs), +which is exactly the "local Grafana with dashboards" the PRD asks for. Jaeger would +add a second UI and a second mental model for no benefit at this scale. + +### No OTLP collector + +Tempo ingests OTLP directly (gRPC 4317 / HTTP 4318) and Prometheus scrapes each +service's `/metrics`, so a collector would be a hop that processes nothing. Skipped. +If we later need fan-out, tail sampling, or log processing, a collector is an +additive change — the services already speak OTLP. + +### Config baked into built images, not config volumes + +The upstream Common Ground modules (OpenZaak, NRC, Keycloak, Flowable) run as +**verbatim** images and get their config streamed into external named volumes by +`infra/seed-config.sh`, because bind mounts don't reach sibling containers on the +CI runner (see `docs/runbooks/gitea-actions-gotchas.md`). The observability tools +are **not** peer modules we must run verbatim, so we take the simpler path: a +three-line `Dockerfile` per tool that `COPY`s its config in. This reaches sibling +containers everywhere (docker, podman, CI) with no seed step, no `CFG_VOLS` entry, +and no Makefile sprawl. + +### Verified, not assumed + +`infra/run-observability-check.sh` (the `verify-observability` step, run early in CI +`verify-stack`) asks Grafana to reach both datasources — Prometheus via its health +method, Tempo via the datasource proxy (Tempo's Grafana plugin implements no health +method) — so the check proves the datasources are actually wired, not merely that +containers started. The containers are not in `WAIT_SVCS`; the check polls Grafana +itself, so no in-image healthcheck tool is required. + +## Consequences + +**Positive** + +- One UI for traces + metrics + (future) logs. Config is versioned in + `infra/observability/` and self-contained in the images. +- Backplane is independent of app instrumentation — #123 and #124 build on it. + +**Negative / costs** + +- Three more images built each CI run (kept small; not on the health-gate list). +- Storage is ephemeral container fs — a demo backplane, not a retention target. + Object storage for Tempo / remote-write for Prometheus is a later concern. + +## Coupling rules touched (CLAUDE.md §8) + +None. The stack is passive infrastructure: services *push* OTLP and *expose* +`/metrics`; nothing in the stack calls into a service or a peer module. diff --git a/docs/demo-script.md b/docs/demo-script.md index eeebfbb..c4b7492 100644 --- a/docs/demo-script.md +++ b/docs/demo-script.md @@ -5,6 +5,31 @@ copy-pasteable walkthrough against a local `make up` stack. --- +## S-16a — observability backplane: Tempo + Prometheus + Grafana (#122, ADR-0023) + +**Outcome:** the compose stack now includes a Grafana-native observability backplane — **Tempo** (OTLP +trace ingest on 4317/4318), **Prometheus**, and **Grafana** with both datasources auto-provisioned. +Nothing is instrumented yet (traces land in S-16b, metrics + dashboards in S-16c); this slice stands the +backplane up and proves Grafana can reach both datasources. Config is baked into small built images +(`infra/observability/`) — no collector, no config-volume seeding. + +```bash +# 1. Bring the stack up, then assert the backplane is live (Grafana healthy + Tempo/Prometheus +# datasources reachable through Grafana). This is a CI verify-stack step. +make up +make verify-observability # → ✓ Grafana healthy ✓ Prometheus reachable ✓ Tempo reachable + +# 2. Or just the backplane, no full stack needed (no external egress): +docker compose -f infra/docker-compose.yml up -d --build tempo prometheus grafana +open http://localhost:3000 # Grafana (admin/admin) → Connections → Data sources: Prometheus + Tempo +open http://localhost:9090 # Prometheus +``` + +**The path:** services will export OTLP → **Tempo:4317** and expose `/metrics` ← **Prometheus** scrapes; +**Grafana** (:3000) reads both via provisioned datasources with fixed uids `tempo` / `prometheus`. + +--- + ## S-17 — herregistratie reminder sweep on a Quartz cron (#18, ADR-0022) **Outcome:** an inscription (INGESCHREVEN) now carries the moment it was entered in the register, from -- 2.54.0 From 56a5fe014c8e35ff5f0d1d960f7a2ee981d6775a Mon Sep 17 00:00:00 2001 From: Niek Otten Date: Thu, 23 Jul 2026 13:55:03 +0200 Subject: [PATCH 4/4] fix(infra): run the observability check inside the compose network (refs #122) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stack runs as sibling containers on the CI runner's host daemon, so its published ports aren't on the runner job's localhost — verify-observability hit localhost:3000 and failed. Reach Grafana by container IP through a throwaway curl container on the stack network, mirroring run-bff-check.sh et al. Parses with grep (the curl image has no python). refs #122 --- infra/run-observability-check.sh | 54 +++++++++++++++++--------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/infra/run-observability-check.sh b/infra/run-observability-check.sh index a8d01fa..b340f64 100755 --- a/infra/run-observability-check.sh +++ b/infra/run-observability-check.sh @@ -1,41 +1,45 @@ #!/usr/bin/env bash # -# S-16a (#122): assert the observability backplane is live against the running stack. -# Not just "the containers started" — it asks Grafana to health-check its provisioned -# Tempo and Prometheus datasources, which proves Grafana can actually reach both over -# the `cg` network and that provisioning landed. Polls, so it tolerates a cold Grafana. +# S-16a (#122): assert the observability backplane is live against an ALREADY-RUNNING +# stack. Runs curl INSIDE the compose network (like the other verify checks) because +# the stack's published ports aren't on the CI runner's localhost — the stack is a set +# of sibling containers on the host daemon. It asks Grafana to reach its provisioned +# datasources — Prometheus via its health method, Tempo via the datasource proxy (Tempo's +# Grafana plugin implements no health method) — so it proves the datasources are wired, +# not merely that the containers started. Polls, so it tolerates a cold Grafana. # -# Usage: run-observability-check.sh (override GRAFANA_URL / GRAFANA_AUTH / OBS_TIMEOUT) +# Does NOT manage the stack lifecycle (the caller owns bring-up + teardown). set -euo pipefail -GRAFANA="${GRAFANA_URL:-http://localhost:3000}" -AUTH="${GRAFANA_AUTH:-admin:admin}" TIMEOUT="${OBS_TIMEOUT:-60}" +AUTH="${GRAFANA_AUTH:-admin:admin}" -# poll -# Passes when the expression prints True within TIMEOUT. Tempo's Grafana plugin -# doesn't implement the datasource /health method, so instead of the Prometheus- -# style health probe we prove reachability through Grafana's datasource proxy. +gf="$(docker ps -q --filter 'name=[-_]grafana[-_]' | head -1)" +[ -n "$gf" ] || { echo "ERROR: no running grafana container — bring the stack up first" >&2; exit 1; } +net="$(docker inspect -f '{{range $k,$_ := .NetworkSettings.Networks}}{{$k}}{{"\n"}}{{end}}' "$gf" | head -1)" +gf_ip="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$gf")" +base="http://$gf_ip:3000" +echo ">> grafana=$gf_ip network=$net" + +# Run curl inside a throwaway container on the stack network (reaches services by IP). +net_curl() { docker run --rm --network "$net" curlimages/curl:latest "$@"; } + +# poll poll() { - local desc="$1" url="$2" expr="$3" got deadline - deadline=$(( $(date +%s) + TIMEOUT )) + local desc="$1" pat="$2"; shift 2 + local deadline=$(( $(date +%s) + TIMEOUT )) while :; do - got="$(curl -fsS -u "$AUTH" "$url" 2>/dev/null \ - | python3 -c "import sys,json; d=json.load(sys.stdin); print($expr)" 2>/dev/null || true)" - [ "$got" = "True" ] && { echo " ✓ $desc"; return 0; } - if [ "$(date +%s)" -ge "$deadline" ]; then - echo " ✗ $desc — check failed ($url)" >&2 - return 1 - fi - sleep 2 + if net_curl -fsS "$@" 2>/dev/null | grep -Eq "$pat"; then echo " ✓ $desc"; return 0; fi + if [ "$(date +%s)" -ge "$deadline" ]; then echo " ✗ $desc ($*)" >&2; return 1; fi + sleep 3 done } -echo "Checking observability backplane at $GRAFANA ..." +echo "Checking observability backplane at $base ..." poll "Grafana is healthy" \ - "$GRAFANA/api/health" "d['database'] == 'ok'" + '"database":[[:space:]]*"ok"' "$base/api/health" poll "Prometheus datasource reachable" \ - "$GRAFANA/api/datasources/uid/prometheus/health" "d['status'] == 'OK'" + '"status":[[:space:]]*"OK"' -u "$AUTH" "$base/api/datasources/uid/prometheus/health" poll "Tempo datasource reachable (via Grafana proxy)" \ - "$GRAFANA/api/datasources/proxy/uid/tempo/api/status/buildinfo" "bool(d.get('version'))" + '"version"' -u "$AUTH" "$base/api/datasources/proxy/uid/tempo/api/status/buildinfo" echo "Observability backplane OK." -- 2.54.0