fix(infra): run the observability check inside the compose network (refs #122)
CI / lint (pull_request) Successful in 1m25s
CI / build (pull_request) Successful in 1m9s
CI / unit (pull_request) Successful in 1m19s
CI / frontend (pull_request) Successful in 2m47s
CI / mutation (pull_request) Successful in 6m9s
CI / verify-stack (pull_request) Successful in 9m20s
CI / lint (pull_request) Successful in 1m25s
CI / build (pull_request) Successful in 1m9s
CI / unit (pull_request) Successful in 1m19s
CI / frontend (pull_request) Successful in 2m47s
CI / mutation (pull_request) Successful in 6m9s
CI / verify-stack (pull_request) Successful in 9m20s
The stack runs as sibling containers on the CI runner's host daemon, so its published ports aren't on the runner job's localhost — verify-observability hit localhost:3000 and failed. Reach Grafana by container IP through a throwaway curl container on the stack network, mirroring run-bff-check.sh et al. Parses with grep (the curl image has no python). refs #122
This commit is contained in:
@@ -1,41 +1,45 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# S-16a (#122): assert the observability backplane is live against the running stack.
|
||||
# Not just "the containers started" — it asks Grafana to health-check its provisioned
|
||||
# Tempo and Prometheus datasources, which proves Grafana can actually reach both over
|
||||
# the `cg` network and that provisioning landed. Polls, so it tolerates a cold Grafana.
|
||||
# S-16a (#122): assert the observability backplane is live against an ALREADY-RUNNING
|
||||
# stack. Runs curl INSIDE the compose network (like the other verify checks) because
|
||||
# the stack's published ports aren't on the CI runner's localhost — the stack is a set
|
||||
# of sibling containers on the host daemon. It asks Grafana to reach its provisioned
|
||||
# datasources — Prometheus via its health method, Tempo via the datasource proxy (Tempo's
|
||||
# Grafana plugin implements no health method) — so it proves the datasources are wired,
|
||||
# not merely that the containers started. Polls, so it tolerates a cold Grafana.
|
||||
#
|
||||
# Usage: run-observability-check.sh (override GRAFANA_URL / GRAFANA_AUTH / OBS_TIMEOUT)
|
||||
# Does NOT manage the stack lifecycle (the caller owns bring-up + teardown).
|
||||
set -euo pipefail
|
||||
|
||||
GRAFANA="${GRAFANA_URL:-http://localhost:3000}"
|
||||
AUTH="${GRAFANA_AUTH:-admin:admin}"
|
||||
TIMEOUT="${OBS_TIMEOUT:-60}"
|
||||
AUTH="${GRAFANA_AUTH:-admin:admin}"
|
||||
|
||||
# poll <description> <url> <python-bool-expr over `d` (parsed JSON)>
|
||||
# Passes when the expression prints True within TIMEOUT. Tempo's Grafana plugin
|
||||
# doesn't implement the datasource /health method, so instead of the Prometheus-
|
||||
# style health probe we prove reachability through Grafana's datasource proxy.
|
||||
gf="$(docker ps -q --filter 'name=[-_]grafana[-_]' | head -1)"
|
||||
[ -n "$gf" ] || { echo "ERROR: no running grafana container — bring the stack up first" >&2; exit 1; }
|
||||
net="$(docker inspect -f '{{range $k,$_ := .NetworkSettings.Networks}}{{$k}}{{"\n"}}{{end}}' "$gf" | head -1)"
|
||||
gf_ip="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$gf")"
|
||||
base="http://$gf_ip:3000"
|
||||
echo ">> grafana=$gf_ip network=$net"
|
||||
|
||||
# Run curl inside a throwaway container on the stack network (reaches services by IP).
|
||||
net_curl() { docker run --rm --network "$net" curlimages/curl:latest "$@"; }
|
||||
|
||||
# poll <description> <grep -E pattern> <curl args...>
|
||||
poll() {
|
||||
local desc="$1" url="$2" expr="$3" got deadline
|
||||
deadline=$(( $(date +%s) + TIMEOUT ))
|
||||
local desc="$1" pat="$2"; shift 2
|
||||
local deadline=$(( $(date +%s) + TIMEOUT ))
|
||||
while :; do
|
||||
got="$(curl -fsS -u "$AUTH" "$url" 2>/dev/null \
|
||||
| python3 -c "import sys,json; d=json.load(sys.stdin); print($expr)" 2>/dev/null || true)"
|
||||
[ "$got" = "True" ] && { echo " ✓ $desc"; return 0; }
|
||||
if [ "$(date +%s)" -ge "$deadline" ]; then
|
||||
echo " ✗ $desc — check failed ($url)" >&2
|
||||
return 1
|
||||
fi
|
||||
sleep 2
|
||||
if net_curl -fsS "$@" 2>/dev/null | grep -Eq "$pat"; then echo " ✓ $desc"; return 0; fi
|
||||
if [ "$(date +%s)" -ge "$deadline" ]; then echo " ✗ $desc ($*)" >&2; return 1; fi
|
||||
sleep 3
|
||||
done
|
||||
}
|
||||
|
||||
echo "Checking observability backplane at $GRAFANA ..."
|
||||
echo "Checking observability backplane at $base ..."
|
||||
poll "Grafana is healthy" \
|
||||
"$GRAFANA/api/health" "d['database'] == 'ok'"
|
||||
'"database":[[:space:]]*"ok"' "$base/api/health"
|
||||
poll "Prometheus datasource reachable" \
|
||||
"$GRAFANA/api/datasources/uid/prometheus/health" "d['status'] == 'OK'"
|
||||
'"status":[[:space:]]*"OK"' -u "$AUTH" "$base/api/datasources/uid/prometheus/health"
|
||||
poll "Tempo datasource reachable (via Grafana proxy)" \
|
||||
"$GRAFANA/api/datasources/proxy/uid/tempo/api/status/buildinfo" "bool(d.get('version'))"
|
||||
'"version"' -u "$AUTH" "$base/api/datasources/proxy/uid/tempo/api/status/buildinfo"
|
||||
echo "Observability backplane OK."
|
||||
|
||||
Reference in New Issue
Block a user