feat(infra): Tempo + Prometheus + Grafana observability backplane in compose (refs #122)
Three built images on the cg network with config baked in: Tempo (OTLP ingest, ports 4317/4318), Prometheus, and Grafana with both datasources auto-provisioned. Config lives in infra/observability/. The verify-observability step runs early in CI verify-stack; the three containers are added to the failure log-dump list. Tempo's Grafana plugin has no datasource /health method, so the check proves Tempo reachability through Grafana's datasource proxy instead. refs #122
This commit is contained in:
@@ -142,6 +142,8 @@ jobs:
|
||||
# reaches green health" smoke (it replaces the old compose-smoke job).
|
||||
- name: Bring up the full stack & wait for health
|
||||
run: make verify-up
|
||||
- name: Observability backplane (Grafana + Tempo + Prometheus datasources)
|
||||
run: OBS_TIMEOUT=180 make verify-observability
|
||||
- name: ACL ↔ OpenZaak integration tests
|
||||
run: make verify-acl
|
||||
- name: OpenZaak → NRC notification delivery
|
||||
@@ -157,7 +159,7 @@ jobs:
|
||||
# Log dump must precede teardown (which removes the containers).
|
||||
- name: Dump container logs on failure
|
||||
if: failure()
|
||||
run: docker compose -f infra/docker-compose.yml logs --no-color --tail=100 oz-init openzaak nrc-init nrc-web nrc-celery nrc-beat flowable-db flowable-rest flowable-init keycloak acl bff domain projection-db event-subscriber projection-api self-service openbaar behandel 2>&1 || true
|
||||
run: docker compose -f infra/docker-compose.yml logs --no-color --tail=100 oz-init openzaak nrc-init nrc-web nrc-celery nrc-beat flowable-db flowable-rest flowable-init keycloak acl bff domain projection-db event-subscriber projection-api self-service openbaar behandel tempo prometheus grafana 2>&1 || true
|
||||
- name: Tear down
|
||||
if: always()
|
||||
run: make down
|
||||
|
||||
@@ -524,6 +524,45 @@ services:
|
||||
condition: service_started
|
||||
networks: [cg]
|
||||
|
||||
# ── Observability backplane (S-16a, ADR-0023) ──────────────────────────────
|
||||
# Grafana-native stack: Tempo ingests OTLP traces (the .NET services export
|
||||
# straight to it — no collector hop, S-16b), Prometheus scrapes service
|
||||
# /metrics (S-16c), and Grafana reads both with datasources auto-provisioned.
|
||||
# Config is baked into small built images (COPY) rather than streamed into
|
||||
# external config volumes like the upstream CG modules — these aren't verbatim
|
||||
# peer images, so a built image is the simpler path that still reaches sibling
|
||||
# containers on the CI runner. Not in WAIT_SVCS: run-observability-check.sh
|
||||
# polls Grafana itself, so no in-image healthcheck tool is needed.
|
||||
tempo:
|
||||
build:
|
||||
context: ./observability/tempo
|
||||
image: register-referentie/tempo:dev
|
||||
command: ["-config.file=/etc/tempo.yaml"]
|
||||
networks: [cg]
|
||||
|
||||
prometheus:
|
||||
build:
|
||||
context: ./observability/prometheus
|
||||
image: register-referentie/prometheus:dev
|
||||
ports:
|
||||
- "9090:9090"
|
||||
networks: [cg]
|
||||
|
||||
grafana:
|
||||
build:
|
||||
context: ./observability/grafana
|
||||
image: register-referentie/grafana:dev
|
||||
environment:
|
||||
GF_SECURITY_ADMIN_USER: admin
|
||||
GF_SECURITY_ADMIN_PASSWORD: admin
|
||||
GF_AUTH_ANONYMOUS_ENABLED: "true"
|
||||
ports:
|
||||
- "3000:3000"
|
||||
depends_on:
|
||||
- tempo
|
||||
- prometheus
|
||||
networks: [cg]
|
||||
|
||||
volumes:
|
||||
oz-db:
|
||||
nrc-db:
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
# Grafana with datasources baked in via provisioning (S-16a, ADR-0023).
|
||||
# Dashboards (S-16c, #124) are added under provisioning/dashboards later.
|
||||
FROM grafana/grafana:11.3.0
|
||||
COPY provisioning/ /etc/grafana/provisioning/
|
||||
@@ -0,0 +1,17 @@
|
||||
# Auto-provisioned datasources (S-16a, ADR-0023). Fixed uids so dashboards (S-16c)
|
||||
# and the verify-observability check can reference them by a stable id.
|
||||
apiVersion: 1
|
||||
|
||||
datasources:
|
||||
- name: Prometheus
|
||||
uid: prometheus
|
||||
type: prometheus
|
||||
access: proxy
|
||||
url: http://prometheus:9090
|
||||
isDefault: true
|
||||
|
||||
- name: Tempo
|
||||
uid: tempo
|
||||
type: tempo
|
||||
access: proxy
|
||||
url: http://tempo:3200
|
||||
@@ -0,0 +1,2 @@
|
||||
FROM prom/prometheus:v2.55.1
|
||||
COPY prometheus.yml /etc/prometheus/prometheus.yml
|
||||
@@ -0,0 +1,10 @@
|
||||
# Prometheus scrape config (S-16a, ADR-0023). For the backplane slice it scrapes
|
||||
# only itself; the .NET services' /metrics scrape targets are added in S-16c
|
||||
# (#124) when the services expose metrics.
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
|
||||
scrape_configs:
|
||||
- job_name: prometheus
|
||||
static_configs:
|
||||
- targets: ['localhost:9090']
|
||||
@@ -0,0 +1,4 @@
|
||||
# Tempo with our config baked in — so it reaches sibling containers on the CI
|
||||
# runner without the external-config-volume dance the upstream CG images need.
|
||||
FROM grafana/tempo:2.6.1
|
||||
COPY tempo.yaml /etc/tempo.yaml
|
||||
@@ -0,0 +1,27 @@
|
||||
# Grafana Tempo — single-binary, all-in-one, local storage (S-16a, ADR-0023).
|
||||
# Ingests OTLP directly (services export straight to Tempo; no collector hop).
|
||||
# Storage is ephemeral container fs — this is a local/CI demo backplane, not a
|
||||
# retention target. ponytail: local backend, swap for object storage if traces
|
||||
# must outlive the stack.
|
||||
server:
|
||||
http_listen_port: 3200
|
||||
|
||||
distributor:
|
||||
receivers:
|
||||
otlp:
|
||||
protocols:
|
||||
grpc:
|
||||
endpoint: 0.0.0.0:4317
|
||||
http:
|
||||
endpoint: 0.0.0.0:4318
|
||||
|
||||
ingester:
|
||||
max_block_duration: 5m
|
||||
|
||||
storage:
|
||||
trace:
|
||||
backend: local
|
||||
local:
|
||||
path: /var/tempo/blocks
|
||||
wal:
|
||||
path: /var/tempo/wal
|
||||
@@ -12,16 +12,19 @@ GRAFANA="${GRAFANA_URL:-http://localhost:3000}"
|
||||
AUTH="${GRAFANA_AUTH:-admin:admin}"
|
||||
TIMEOUT="${OBS_TIMEOUT:-60}"
|
||||
|
||||
# poll <description> <url> <json-path-expr> <expected>
|
||||
# poll <description> <url> <python-bool-expr over `d` (parsed JSON)>
|
||||
# Passes when the expression prints True within TIMEOUT. Tempo's Grafana plugin
|
||||
# doesn't implement the datasource /health method, so instead of the Prometheus-
|
||||
# style health probe we prove reachability through Grafana's datasource proxy.
|
||||
poll() {
|
||||
local desc="$1" url="$2" path="$3" want="$4" got deadline
|
||||
local desc="$1" url="$2" expr="$3" got deadline
|
||||
deadline=$(( $(date +%s) + TIMEOUT ))
|
||||
while :; do
|
||||
got="$(curl -fsS -u "$AUTH" "$url" 2>/dev/null \
|
||||
| python3 -c "import sys,json;print(json.load(sys.stdin)$path)" 2>/dev/null || true)"
|
||||
[ "$got" = "$want" ] && { echo " ✓ $desc"; return 0; }
|
||||
| python3 -c "import sys,json; d=json.load(sys.stdin); print($expr)" 2>/dev/null || true)"
|
||||
[ "$got" = "True" ] && { echo " ✓ $desc"; return 0; }
|
||||
if [ "$(date +%s)" -ge "$deadline" ]; then
|
||||
echo " ✗ $desc — got '$got', want '$want' ($url)" >&2
|
||||
echo " ✗ $desc — check failed ($url)" >&2
|
||||
return 1
|
||||
fi
|
||||
sleep 2
|
||||
@@ -29,7 +32,10 @@ poll() {
|
||||
}
|
||||
|
||||
echo "Checking observability backplane at $GRAFANA ..."
|
||||
poll "Grafana is healthy" "$GRAFANA/api/health" "['database']" "ok"
|
||||
poll "Prometheus datasource reachable" "$GRAFANA/api/datasources/uid/prometheus/health" "['status']" "OK"
|
||||
poll "Tempo datasource reachable" "$GRAFANA/api/datasources/uid/tempo/health" "['status']" "OK"
|
||||
poll "Grafana is healthy" \
|
||||
"$GRAFANA/api/health" "d['database'] == 'ok'"
|
||||
poll "Prometheus datasource reachable" \
|
||||
"$GRAFANA/api/datasources/uid/prometheus/health" "d['status'] == 'OK'"
|
||||
poll "Tempo datasource reachable (via Grafana proxy)" \
|
||||
"$GRAFANA/api/datasources/proxy/uid/tempo/api/status/buildinfo" "bool(d.get('version'))"
|
||||
echo "Observability backplane OK."
|
||||
|
||||
Reference in New Issue
Block a user