not
f29a8598fd
Three built images on the cg network with config baked in: Tempo (OTLP ingest, ports 4317/4318), Prometheus, and Grafana with both datasources auto-provisioned. Config lives in infra/observability/. The verify-observability step runs early in CI verify-stack; the three containers are added to the failure log-dump list. Tempo's Grafana plugin has no datasource /health method, so the check proves Tempo reachability through Grafana's datasource proxy instead. refs #122
42 lines
1.8 KiB
Bash
Executable File
42 lines
1.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# S-16a (#122): assert the observability backplane is live against the running stack.
|
|
# Not just "the containers started" — it asks Grafana to health-check its provisioned
|
|
# Tempo and Prometheus datasources, which proves Grafana can actually reach both over
|
|
# the `cg` network and that provisioning landed. Polls, so it tolerates a cold Grafana.
|
|
#
|
|
# Usage: run-observability-check.sh (override GRAFANA_URL / GRAFANA_AUTH / OBS_TIMEOUT)
|
|
set -euo pipefail
|
|
|
|
GRAFANA="${GRAFANA_URL:-http://localhost:3000}"
|
|
AUTH="${GRAFANA_AUTH:-admin:admin}"
|
|
TIMEOUT="${OBS_TIMEOUT:-60}"
|
|
|
|
# poll <description> <url> <python-bool-expr over `d` (parsed JSON)>
|
|
# Passes when the expression prints True within TIMEOUT. Tempo's Grafana plugin
|
|
# doesn't implement the datasource /health method, so instead of the Prometheus-
|
|
# style health probe we prove reachability through Grafana's datasource proxy.
|
|
poll() {
|
|
local desc="$1" url="$2" expr="$3" got deadline
|
|
deadline=$(( $(date +%s) + TIMEOUT ))
|
|
while :; do
|
|
got="$(curl -fsS -u "$AUTH" "$url" 2>/dev/null \
|
|
| python3 -c "import sys,json; d=json.load(sys.stdin); print($expr)" 2>/dev/null || true)"
|
|
[ "$got" = "True" ] && { echo " ✓ $desc"; return 0; }
|
|
if [ "$(date +%s)" -ge "$deadline" ]; then
|
|
echo " ✗ $desc — check failed ($url)" >&2
|
|
return 1
|
|
fi
|
|
sleep 2
|
|
done
|
|
}
|
|
|
|
echo "Checking observability backplane at $GRAFANA ..."
|
|
poll "Grafana is healthy" \
|
|
"$GRAFANA/api/health" "d['database'] == 'ok'"
|
|
poll "Prometheus datasource reachable" \
|
|
"$GRAFANA/api/datasources/uid/prometheus/health" "d['status'] == 'OK'"
|
|
poll "Tempo datasource reachable (via Grafana proxy)" \
|
|
"$GRAFANA/api/datasources/proxy/uid/tempo/api/status/buildinfo" "bool(d.get('version'))"
|
|
echo "Observability backplane OK."
|