test(infra): verify-observability asserts Grafana + Tempo/Prometheus datasources are live (refs #122)

Polls Grafana's datasource-health endpoints so it proves Grafana can actually
reach Tempo and Prometheus over the cg network, not merely that the containers
started. Fails until the backplane is wired (next commit).

refs #122

Makefile

@@ -43,7 +43,7 @@ export DOCKER_HOST := unix://$(PODMAN_SOCK)
 endif
 endif
 
-.PHONY: ci lint build unit mutation frontend integration verify verify-up verify-acl verify-nrc verify-projection verify-bff verify-domain verify-notifications smoke up down local verify-local local-down changelog openzaak-up openzaak-smoke openzaak-seed openzaak-down stack-up stack-smoke stack-down keycloak-up keycloak-smoke keycloak-down flowable-up flowable-smoke flowable-down help
+.PHONY: ci lint build unit mutation frontend integration verify verify-up verify-acl verify-nrc verify-projection verify-bff verify-domain verify-observability verify-notifications smoke up down local verify-local local-down changelog openzaak-up openzaak-smoke openzaak-seed openzaak-down stack-up stack-smoke stack-down keycloak-up keycloak-smoke keycloak-down flowable-up flowable-smoke flowable-down help
 
 ## ci: run the full pipeline — lint, build, unit, mutation, frontend, verify (mirrors Gitea Actions)
 ## `verify` is the live-stack stage (full stack up once → ACL + notification checks).
@@ -170,6 +170,11 @@ verify-bff:
 verify-e2e:
 	bash infra/run-e2e-check.sh
 
+## verify-observability: assert the observability backplane (Grafana + provisioned Tempo &
+## Prometheus datasources) is live, against the already-running stack (S-16a).
+verify-observability:
+	bash infra/run-observability-check.sh
+
 ## verify: local mirror of the CI verify-stack job — full stack up once, all checks,
 ## tear down (always). For fast single-concern local iteration use `integration`
 ## (oz-only) or `verify-notifications` (oz+nrc) instead.

infra/run-observability-check.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+#
+# S-16a (#122): assert the observability backplane is live against the running stack.
+# Not just "the containers started" — it asks Grafana to health-check its provisioned
+# Tempo and Prometheus datasources, which proves Grafana can actually reach both over
+# the `cg` network and that provisioning landed. Polls, so it tolerates a cold Grafana.
+#
+# Usage: run-observability-check.sh   (override GRAFANA_URL / GRAFANA_AUTH / OBS_TIMEOUT)
+set -euo pipefail
+
+GRAFANA="${GRAFANA_URL:-http://localhost:3000}"
+AUTH="${GRAFANA_AUTH:-admin:admin}"
+TIMEOUT="${OBS_TIMEOUT:-60}"
+
+# poll <description> <url> <json-path-expr> <expected>
+poll() {
+  local desc="$1" url="$2" path="$3" want="$4" got deadline
+  deadline=$(( $(date +%s) + TIMEOUT ))
+  while :; do
+    got="$(curl -fsS -u "$AUTH" "$url" 2>/dev/null \
+      | python3 -c "import sys,json;print(json.load(sys.stdin)$path)" 2>/dev/null || true)"
+    [ "$got" = "$want" ] && { echo "  ✓ $desc"; return 0; }
+    if [ "$(date +%s)" -ge "$deadline" ]; then
+      echo "  ✗ $desc — got '$got', want '$want' ($url)" >&2
+      return 1
+    fi
+    sleep 2
+  done
+}
+
+echo "Checking observability backplane at $GRAFANA ..."
+poll "Grafana is healthy"               "$GRAFANA/api/health"                             "['database']" "ok"
+poll "Prometheus datasource reachable"  "$GRAFANA/api/datasources/uid/prometheus/health"  "['status']"   "OK"
+poll "Tempo datasource reachable"       "$GRAFANA/api/datasources/uid/tempo/health"       "['status']"   "OK"
+echo "Observability backplane OK."