feat(obs): distributed traces across the .NET services (S-16b, closes #123) #126
@@ -154,6 +154,8 @@ jobs:
|
||||
run: make verify-domain
|
||||
- name: BFF → Keycloak + domain + projection
|
||||
run: make verify-bff
|
||||
- name: Distributed traces reach Tempo (one connected trace across services)
|
||||
run: TRACING_TIMEOUT=120 make verify-tracing
|
||||
- name: Self-service e2e (Playwright, login → submit → success)
|
||||
run: make verify-e2e
|
||||
# Log dump must precede teardown (which removes the containers).
|
||||
|
||||
@@ -43,7 +43,7 @@ export DOCKER_HOST := unix://$(PODMAN_SOCK)
|
||||
endif
|
||||
endif
|
||||
|
||||
.PHONY: ci lint build unit mutation frontend integration verify verify-up verify-acl verify-nrc verify-projection verify-bff verify-domain verify-observability verify-notifications smoke up down local verify-local local-down changelog openzaak-up openzaak-smoke openzaak-seed openzaak-down stack-up stack-smoke stack-down keycloak-up keycloak-smoke keycloak-down flowable-up flowable-smoke flowable-down help
|
||||
.PHONY: ci lint build unit mutation frontend integration verify verify-up verify-acl verify-nrc verify-projection verify-bff verify-domain verify-observability verify-tracing verify-notifications smoke up down local verify-local local-down changelog openzaak-up openzaak-smoke openzaak-seed openzaak-down stack-up stack-smoke stack-down keycloak-up keycloak-smoke keycloak-down flowable-up flowable-smoke flowable-down help
|
||||
|
||||
## ci: run the full pipeline — lint, build, unit, mutation, frontend, verify (mirrors Gitea Actions)
|
||||
## `verify` is the live-stack stage (full stack up once → ACL + notification checks).
|
||||
@@ -175,6 +175,11 @@ verify-e2e:
|
||||
verify-observability:
|
||||
bash infra/run-observability-check.sh
|
||||
|
||||
## verify-tracing: assert one connected distributed trace spans the .NET services in Tempo
|
||||
## (S-16b), against the already-running stack.
|
||||
verify-tracing:
|
||||
bash infra/run-tracing-check.sh
|
||||
|
||||
## verify: local mirror of the CI verify-stack job — full stack up once, all checks,
|
||||
## tear down (always). For fast single-concern local iteration use `integration`
|
||||
## (oz-only) or `verify-notifications` (oz+nrc) instead.
|
||||
|
||||
Executable
+27
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# S-16b (#123): assert one connected distributed trace spans the .NET services in Tempo,
|
||||
# against an ALREADY-RUNNING full stack. Runs the driver in a python:3-slim container on the
|
||||
# stack network (services reached by container IP; the runner can't reach published ports —
|
||||
# gitea-actions-gotchas.md §5/§6). Does NOT manage the stack lifecycle.
|
||||
set -euo pipefail
|
||||
|
||||
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
ip() { docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$1"; }
|
||||
|
||||
bff="$(docker ps -q --filter 'name=[-_]bff[-_]' | head -1)"
|
||||
tempo="$(docker ps -q --filter 'name=[-_]tempo[-_]' | head -1)"
|
||||
[ -n "$bff" ] && [ -n "$tempo" ] || { echo "ERROR: bff and/or tempo not running — bring the stack up first" >&2; exit 1; }
|
||||
net="$(docker inspect -f '{{range $k,$_ := .NetworkSettings.Networks}}{{$k}}{{"\n"}}{{end}}' "$bff" | head -1)"
|
||||
bff_ip="$(ip "$bff")"; tempo_ip="$(ip "$tempo")"
|
||||
echo ">> network=$net bff=$bff_ip tempo=$tempo_ip"
|
||||
|
||||
cid="$(docker create --network "$net" \
|
||||
-e "BFF=http://$bff_ip:8080" -e "TEMPO=http://$tempo_ip:3200" \
|
||||
-e "TRACING_TIMEOUT=${TRACING_TIMEOUT:-90}" \
|
||||
python:3-slim python /tracing-check.py)"
|
||||
docker cp "$here/tracing-check.py" "$cid:/tracing-check.py" >/dev/null
|
||||
rc=0; docker start -a "$cid" || rc=$?
|
||||
docker rm -f "$cid" >/dev/null
|
||||
exit $rc
|
||||
Executable
+81
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env python3
|
||||
"""S-16b (#123): prove distributed tracing works end to end.
|
||||
|
||||
Generate anonymous BFF traffic (GET /openbaar/register, which the BFF serves by
|
||||
calling projection-api — no auth, no OpenZaak egress), then query Tempo and assert
|
||||
that ONE trace contains spans from both `bff` and `projection-api`. That proves the
|
||||
services export OTLP to Tempo AND that the W3C traceparent propagates across the
|
||||
HttpClient hop, stitching the request into a single connected trace.
|
||||
|
||||
Stdlib only (urllib/json) so it runs in a bare python:3-slim container in-network.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
BFF = os.environ["BFF"] # http://<bff-ip>:8080
|
||||
TEMPO = os.environ["TEMPO"] # http://<tempo-ip>:3200
|
||||
TIMEOUT = int(os.environ.get("TRACING_TIMEOUT", "90"))
|
||||
WANT = {"bff", "projection-api"} # the two services that must share one trace
|
||||
|
||||
|
||||
def _get(url):
|
||||
with urllib.request.urlopen(url, timeout=10) as r:
|
||||
return r.read()
|
||||
|
||||
|
||||
def generate_traffic():
|
||||
# A non-2xx still produces spans; only total unreachability of the BFF is fatal.
|
||||
for _ in range(3):
|
||||
try:
|
||||
_get(f"{BFF}/openbaar/register")
|
||||
except urllib.error.HTTPError:
|
||||
pass
|
||||
|
||||
|
||||
def search_trace_ids():
|
||||
q = urllib.parse.quote('{ resource.service.name = "bff" }')
|
||||
try:
|
||||
data = json.loads(_get(f"{TEMPO}/api/search?q={q}&limit=50"))
|
||||
except Exception:
|
||||
return []
|
||||
return [t["traceID"] for t in data.get("traces", [])]
|
||||
|
||||
|
||||
def services_in_trace(trace_id):
|
||||
try:
|
||||
data = json.loads(_get(f"{TEMPO}/api/traces/{trace_id}"))
|
||||
except Exception:
|
||||
return set()
|
||||
names = set()
|
||||
for batch in data.get("batches", []):
|
||||
for attr in batch.get("resource", {}).get("attributes", []):
|
||||
if attr.get("key") == "service.name":
|
||||
names.add(attr.get("value", {}).get("stringValue"))
|
||||
return names
|
||||
|
||||
|
||||
def main():
|
||||
deadline = time.time() + TIMEOUT
|
||||
generate_traffic()
|
||||
seen = set()
|
||||
while time.time() < deadline:
|
||||
for tid in search_trace_ids():
|
||||
names = services_in_trace(tid)
|
||||
seen |= names
|
||||
if WANT.issubset(names):
|
||||
print(f"OK — trace {tid} spans {sorted(names)}")
|
||||
return 0
|
||||
time.sleep(3)
|
||||
generate_traffic()
|
||||
print(f"FAIL — no single trace spanned {sorted(WANT)}; services seen: {sorted(seen)}",
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user