fix(infra): run the observability check inside the compose network (refs #122)
CI / lint (pull_request) Successful in 1m25s
CI / build (pull_request) Successful in 1m9s
CI / unit (pull_request) Successful in 1m19s
CI / frontend (pull_request) Successful in 2m47s
CI / mutation (pull_request) Successful in 6m9s
CI / verify-stack (pull_request) Successful in 9m20s

The stack runs as sibling containers on the CI runner's host daemon, so its
published ports aren't on the runner job's localhost — verify-observability hit
localhost:3000 and failed. Reach Grafana by container IP through a throwaway curl
container on the stack network, mirroring run-bff-check.sh et al. Parses with grep
(the curl image has no python).

refs #122
This commit is contained in:
not
2026-07-23 13:55:03 +02:00
parent 43c6c8e0d2
commit 56a5fe014c
+29 -25
View File
@@ -1,41 +1,45 @@
#!/usr/bin/env bash #!/usr/bin/env bash
# #
# S-16a (#122): assert the observability backplane is live against the running stack. # S-16a (#122): assert the observability backplane is live against an ALREADY-RUNNING
# Not just "the containers started" — it asks Grafana to health-check its provisioned # stack. Runs curl INSIDE the compose network (like the other verify checks) because
# Tempo and Prometheus datasources, which proves Grafana can actually reach both over # the stack's published ports aren't on the CI runner's localhost — the stack is a set
# the `cg` network and that provisioning landed. Polls, so it tolerates a cold Grafana. # of sibling containers on the host daemon. It asks Grafana to reach its provisioned
# datasources — Prometheus via its health method, Tempo via the datasource proxy (Tempo's
# Grafana plugin implements no health method) — so it proves the datasources are wired,
# not merely that the containers started. Polls, so it tolerates a cold Grafana.
# #
# Usage: run-observability-check.sh (override GRAFANA_URL / GRAFANA_AUTH / OBS_TIMEOUT) # Does NOT manage the stack lifecycle (the caller owns bring-up + teardown).
set -euo pipefail set -euo pipefail
GRAFANA="${GRAFANA_URL:-http://localhost:3000}"
AUTH="${GRAFANA_AUTH:-admin:admin}"
TIMEOUT="${OBS_TIMEOUT:-60}" TIMEOUT="${OBS_TIMEOUT:-60}"
AUTH="${GRAFANA_AUTH:-admin:admin}"
# poll <description> <url> <python-bool-expr over `d` (parsed JSON)> gf="$(docker ps -q --filter 'name=[-_]grafana[-_]' | head -1)"
# Passes when the expression prints True within TIMEOUT. Tempo's Grafana plugin [ -n "$gf" ] || { echo "ERROR: no running grafana container — bring the stack up first" >&2; exit 1; }
# doesn't implement the datasource /health method, so instead of the Prometheus- net="$(docker inspect -f '{{range $k,$_ := .NetworkSettings.Networks}}{{$k}}{{"\n"}}{{end}}' "$gf" | head -1)"
# style health probe we prove reachability through Grafana's datasource proxy. gf_ip="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$gf")"
base="http://$gf_ip:3000"
echo ">> grafana=$gf_ip network=$net"
# Run curl inside a throwaway container on the stack network (reaches services by IP).
net_curl() { docker run --rm --network "$net" curlimages/curl:latest "$@"; }
# poll <description> <grep -E pattern> <curl args...>
poll() { poll() {
local desc="$1" url="$2" expr="$3" got deadline local desc="$1" pat="$2"; shift 2
deadline=$(( $(date +%s) + TIMEOUT )) local deadline=$(( $(date +%s) + TIMEOUT ))
while :; do while :; do
got="$(curl -fsS -u "$AUTH" "$url" 2>/dev/null \ if net_curl -fsS "$@" 2>/dev/null | grep -Eq "$pat"; then echo "$desc"; return 0; fi
| python3 -c "import sys,json; d=json.load(sys.stdin); print($expr)" 2>/dev/null || true)" if [ "$(date +%s)" -ge "$deadline" ]; then echo "$desc ($*)" >&2; return 1; fi
[ "$got" = "True" ] && { echo "$desc"; return 0; } sleep 3
if [ "$(date +%s)" -ge "$deadline" ]; then
echo "$desc — check failed ($url)" >&2
return 1
fi
sleep 2
done done
} }
echo "Checking observability backplane at $GRAFANA ..." echo "Checking observability backplane at $base ..."
poll "Grafana is healthy" \ poll "Grafana is healthy" \
"$GRAFANA/api/health" "d['database'] == 'ok'" '"database":[[:space:]]*"ok"' "$base/api/health"
poll "Prometheus datasource reachable" \ poll "Prometheus datasource reachable" \
"$GRAFANA/api/datasources/uid/prometheus/health" "d['status'] == 'OK'" '"status":[[:space:]]*"OK"' -u "$AUTH" "$base/api/datasources/uid/prometheus/health"
poll "Tempo datasource reachable (via Grafana proxy)" \ poll "Tempo datasource reachable (via Grafana proxy)" \
"$GRAFANA/api/datasources/proxy/uid/tempo/api/status/buildinfo" "bool(d.get('version'))" '"version"' -u "$AUTH" "$base/api/datasources/proxy/uid/tempo/api/status/buildinfo"
echo "Observability backplane OK." echo "Observability backplane OK."