test(infra): verify-tracing asserts one connected trace across services in Tempo (refs #123)

Generates anonymous BFF→projection-api traffic and asserts Tempo holds a single
trace containing both service.names — proving OTLP export plus traceparent
propagation across the HttpClient hop. Fails until the services are instrumented
(next commit). Runs in-network like the other verify checks.

refs #123

.gitea/workflows/ci.yaml

@@ -154,6 +154,8 @@ jobs:
         run: make verify-domain
       - name: BFF → Keycloak + domain + projection
         run: make verify-bff
+      - name: Distributed traces reach Tempo (one connected trace across services)
+        run: TRACING_TIMEOUT=120 make verify-tracing
       - name: Self-service e2e (Playwright, login → submit → success)
         run: make verify-e2e
       # Log dump must precede teardown (which removes the containers).

Makefile

@@ -43,7 +43,7 @@ export DOCKER_HOST := unix://$(PODMAN_SOCK)
 endif
 endif
 
-.PHONY: ci lint build unit mutation frontend integration verify verify-up verify-acl verify-nrc verify-projection verify-bff verify-domain verify-observability verify-notifications smoke up down local verify-local local-down changelog openzaak-up openzaak-smoke openzaak-seed openzaak-down stack-up stack-smoke stack-down keycloak-up keycloak-smoke keycloak-down flowable-up flowable-smoke flowable-down help
+.PHONY: ci lint build unit mutation frontend integration verify verify-up verify-acl verify-nrc verify-projection verify-bff verify-domain verify-observability verify-tracing verify-notifications smoke up down local verify-local local-down changelog openzaak-up openzaak-smoke openzaak-seed openzaak-down stack-up stack-smoke stack-down keycloak-up keycloak-smoke keycloak-down flowable-up flowable-smoke flowable-down help
 
 ## ci: run the full pipeline — lint, build, unit, mutation, frontend, verify (mirrors Gitea Actions)
 ## `verify` is the live-stack stage (full stack up once → ACL + notification checks).
@@ -175,6 +175,11 @@ verify-e2e:
 verify-observability:
 	bash infra/run-observability-check.sh
 
+## verify-tracing: assert one connected distributed trace spans the .NET services in Tempo
+## (S-16b), against the already-running stack.
+verify-tracing:
+	bash infra/run-tracing-check.sh
+
 ## verify: local mirror of the CI verify-stack job — full stack up once, all checks,
 ## tear down (always). For fast single-concern local iteration use `integration`
 ## (oz-only) or `verify-notifications` (oz+nrc) instead.

infra/run-tracing-check.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+#
+# S-16b (#123): assert one connected distributed trace spans the .NET services in Tempo,
+# against an ALREADY-RUNNING full stack. Runs the driver in a python:3-slim container on the
+# stack network (services reached by container IP; the runner can't reach published ports —
+# gitea-actions-gotchas.md §5/§6). Does NOT manage the stack lifecycle.
+set -euo pipefail
+
+here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+ip() { docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$1"; }
+
+bff="$(docker ps -q --filter 'name=[-_]bff[-_]' | head -1)"
+tempo="$(docker ps -q --filter 'name=[-_]tempo[-_]' | head -1)"
+[ -n "$bff" ] && [ -n "$tempo" ] || { echo "ERROR: bff and/or tempo not running — bring the stack up first" >&2; exit 1; }
+net="$(docker inspect -f '{{range $k,$_ := .NetworkSettings.Networks}}{{$k}}{{"\n"}}{{end}}' "$bff" | head -1)"
+bff_ip="$(ip "$bff")"; tempo_ip="$(ip "$tempo")"
+echo ">> network=$net bff=$bff_ip tempo=$tempo_ip"
+
+cid="$(docker create --network "$net" \
+  -e "BFF=http://$bff_ip:8080" -e "TEMPO=http://$tempo_ip:3200" \
+  -e "TRACING_TIMEOUT=${TRACING_TIMEOUT:-90}" \
+  python:3-slim python /tracing-check.py)"
+docker cp "$here/tracing-check.py" "$cid:/tracing-check.py" >/dev/null
+rc=0; docker start -a "$cid" || rc=$?
+docker rm -f "$cid" >/dev/null
+exit $rc

infra/tracing-check.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+"""S-16b (#123): prove distributed tracing works end to end.
+
+Generate anonymous BFF traffic (GET /openbaar/register, which the BFF serves by
+calling projection-api — no auth, no OpenZaak egress), then query Tempo and assert
+that ONE trace contains spans from both `bff` and `projection-api`. That proves the
+services export OTLP to Tempo AND that the W3C traceparent propagates across the
+HttpClient hop, stitching the request into a single connected trace.
+
+Stdlib only (urllib/json) so it runs in a bare python:3-slim container in-network.
+"""
+import json
+import os
+import sys
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+
+BFF = os.environ["BFF"]        # http://<bff-ip>:8080
+TEMPO = os.environ["TEMPO"]    # http://<tempo-ip>:3200
+TIMEOUT = int(os.environ.get("TRACING_TIMEOUT", "90"))
+WANT = {"bff", "projection-api"}  # the two services that must share one trace
+
+
+def _get(url):
+    with urllib.request.urlopen(url, timeout=10) as r:
+        return r.read()
+
+
+def generate_traffic():
+    # A non-2xx still produces spans; only total unreachability of the BFF is fatal.
+    for _ in range(3):
+        try:
+            _get(f"{BFF}/openbaar/register")
+        except urllib.error.HTTPError:
+            pass
+
+
+def search_trace_ids():
+    q = urllib.parse.quote('{ resource.service.name = "bff" }')
+    try:
+        data = json.loads(_get(f"{TEMPO}/api/search?q={q}&limit=50"))
+    except Exception:
+        return []
+    return [t["traceID"] for t in data.get("traces", [])]
+
+
+def services_in_trace(trace_id):
+    try:
+        data = json.loads(_get(f"{TEMPO}/api/traces/{trace_id}"))
+    except Exception:
+        return set()
+    names = set()
+    for batch in data.get("batches", []):
+        for attr in batch.get("resource", {}).get("attributes", []):
+            if attr.get("key") == "service.name":
+                names.add(attr.get("value", {}).get("stringValue"))
+    return names
+
+
+def main():
+    deadline = time.time() + TIMEOUT
+    generate_traffic()
+    seen = set()
+    while time.time() < deadline:
+        for tid in search_trace_ids():
+            names = services_in_trace(tid)
+            seen |= names
+            if WANT.issubset(names):
+                print(f"OK — trace {tid} spans {sorted(names)}")
+                return 0
+        time.sleep(3)
+        generate_traffic()
+    print(f"FAIL — no single trace spanned {sorted(WANT)}; services seen: {sorted(seen)}",
+          file=sys.stderr)
+    return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())