fix(infra): run the observability check inside the compose network (refs #122)
CI / lint (pull_request) Successful in 1m25s
CI / build (pull_request) Successful in 1m9s
CI / unit (pull_request) Successful in 1m19s
CI / frontend (pull_request) Successful in 2m47s
CI / mutation (pull_request) Successful in 6m9s
CI / verify-stack (pull_request) Successful in 9m20s
CI / lint (pull_request) Successful in 1m25s
CI / build (pull_request) Successful in 1m9s
CI / unit (pull_request) Successful in 1m19s
CI / frontend (pull_request) Successful in 2m47s
CI / mutation (pull_request) Successful in 6m9s
CI / verify-stack (pull_request) Successful in 9m20s
The stack runs as sibling containers on the CI runner's host daemon, so its published ports aren't on the runner job's localhost — verify-observability hit localhost:3000 and failed. Reach Grafana by container IP through a throwaway curl container on the stack network, mirroring run-bff-check.sh et al. Parses with grep (the curl image has no python). refs #122
This commit is contained in:
@@ -1,41 +1,45 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
#
|
#
|
||||||
# S-16a (#122): assert the observability backplane is live against the running stack.
|
# S-16a (#122): assert the observability backplane is live against an ALREADY-RUNNING
|
||||||
# Not just "the containers started" — it asks Grafana to health-check its provisioned
|
# stack. Runs curl INSIDE the compose network (like the other verify checks) because
|
||||||
# Tempo and Prometheus datasources, which proves Grafana can actually reach both over
|
# the stack's published ports aren't on the CI runner's localhost — the stack is a set
|
||||||
# the `cg` network and that provisioning landed. Polls, so it tolerates a cold Grafana.
|
# of sibling containers on the host daemon. It asks Grafana to reach its provisioned
|
||||||
|
# datasources — Prometheus via its health method, Tempo via the datasource proxy (Tempo's
|
||||||
|
# Grafana plugin implements no health method) — so it proves the datasources are wired,
|
||||||
|
# not merely that the containers started. Polls, so it tolerates a cold Grafana.
|
||||||
#
|
#
|
||||||
# Usage: run-observability-check.sh (override GRAFANA_URL / GRAFANA_AUTH / OBS_TIMEOUT)
|
# Does NOT manage the stack lifecycle (the caller owns bring-up + teardown).
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
GRAFANA="${GRAFANA_URL:-http://localhost:3000}"
|
|
||||||
AUTH="${GRAFANA_AUTH:-admin:admin}"
|
|
||||||
TIMEOUT="${OBS_TIMEOUT:-60}"
|
TIMEOUT="${OBS_TIMEOUT:-60}"
|
||||||
|
AUTH="${GRAFANA_AUTH:-admin:admin}"
|
||||||
|
|
||||||
# poll <description> <url> <python-bool-expr over `d` (parsed JSON)>
|
gf="$(docker ps -q --filter 'name=[-_]grafana[-_]' | head -1)"
|
||||||
# Passes when the expression prints True within TIMEOUT. Tempo's Grafana plugin
|
[ -n "$gf" ] || { echo "ERROR: no running grafana container — bring the stack up first" >&2; exit 1; }
|
||||||
# doesn't implement the datasource /health method, so instead of the Prometheus-
|
net="$(docker inspect -f '{{range $k,$_ := .NetworkSettings.Networks}}{{$k}}{{"\n"}}{{end}}' "$gf" | head -1)"
|
||||||
# style health probe we prove reachability through Grafana's datasource proxy.
|
gf_ip="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$gf")"
|
||||||
|
base="http://$gf_ip:3000"
|
||||||
|
echo ">> grafana=$gf_ip network=$net"
|
||||||
|
|
||||||
|
# Run curl inside a throwaway container on the stack network (reaches services by IP).
|
||||||
|
net_curl() { docker run --rm --network "$net" curlimages/curl:latest "$@"; }
|
||||||
|
|
||||||
|
# poll <description> <grep -E pattern> <curl args...>
|
||||||
poll() {
|
poll() {
|
||||||
local desc="$1" url="$2" expr="$3" got deadline
|
local desc="$1" pat="$2"; shift 2
|
||||||
deadline=$(( $(date +%s) + TIMEOUT ))
|
local deadline=$(( $(date +%s) + TIMEOUT ))
|
||||||
while :; do
|
while :; do
|
||||||
got="$(curl -fsS -u "$AUTH" "$url" 2>/dev/null \
|
if net_curl -fsS "$@" 2>/dev/null | grep -Eq "$pat"; then echo " ✓ $desc"; return 0; fi
|
||||||
| python3 -c "import sys,json; d=json.load(sys.stdin); print($expr)" 2>/dev/null || true)"
|
if [ "$(date +%s)" -ge "$deadline" ]; then echo " ✗ $desc ($*)" >&2; return 1; fi
|
||||||
[ "$got" = "True" ] && { echo " ✓ $desc"; return 0; }
|
sleep 3
|
||||||
if [ "$(date +%s)" -ge "$deadline" ]; then
|
|
||||||
echo " ✗ $desc — check failed ($url)" >&2
|
|
||||||
return 1
|
|
||||||
fi
|
|
||||||
sleep 2
|
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
echo "Checking observability backplane at $GRAFANA ..."
|
echo "Checking observability backplane at $base ..."
|
||||||
poll "Grafana is healthy" \
|
poll "Grafana is healthy" \
|
||||||
"$GRAFANA/api/health" "d['database'] == 'ok'"
|
'"database":[[:space:]]*"ok"' "$base/api/health"
|
||||||
poll "Prometheus datasource reachable" \
|
poll "Prometheus datasource reachable" \
|
||||||
"$GRAFANA/api/datasources/uid/prometheus/health" "d['status'] == 'OK'"
|
'"status":[[:space:]]*"OK"' -u "$AUTH" "$base/api/datasources/uid/prometheus/health"
|
||||||
poll "Tempo datasource reachable (via Grafana proxy)" \
|
poll "Tempo datasource reachable (via Grafana proxy)" \
|
||||||
"$GRAFANA/api/datasources/proxy/uid/tempo/api/status/buildinfo" "bool(d.get('version'))"
|
'"version"' -u "$AUTH" "$base/api/datasources/proxy/uid/tempo/api/status/buildinfo"
|
||||||
echo "Observability backplane OK."
|
echo "Observability backplane OK."
|
||||||
|
|||||||
Reference in New Issue
Block a user