diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index bf36a09..4ec07a3 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -154,6 +154,8 @@ jobs: run: make verify-domain - name: BFF → Keycloak + domain + projection run: make verify-bff + - name: Distributed traces reach Tempo (one connected trace across services) + run: TRACING_TIMEOUT=120 make verify-tracing - name: Self-service e2e (Playwright, login → submit → success) run: make verify-e2e # Log dump must precede teardown (which removes the containers). diff --git a/BACKLOG.md b/BACKLOG.md index b87a48b..627a2ef 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -260,7 +260,7 @@ Split (issue #11 closed) into two independently-demoable slices per §13 — the Split into independently deployable sub-slices (CLAUDE.md §13): - **S-16a** (#122) · Observability backplane — Grafana Tempo + Prometheus + Grafana in compose, datasources auto-provisioned (ADR-0023). No collector; config baked into built images. -- **S-16b** (#123) · Distributed traces across the five .NET services (OTLP → Tempo; nginx `traceparent` passthrough). Depends on S-16a. +- **S-16b** (#123) · Distributed traces across the five .NET services (OTLP → Tempo; traceparent propagates via the typed HttpClients). Depends on S-16a. ✅ - **S-16c** (#124) · Prometheus metrics + golden-signal Grafana dashboards. Depends on S-16a. ### S-17 · Quartz.NET scheduler — herregistratie reminder sweep ✅ diff --git a/Makefile b/Makefile index 87e87c4..b5a17c8 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ export DOCKER_HOST := unix://$(PODMAN_SOCK) endif endif -.PHONY: ci lint build unit mutation frontend integration verify verify-up verify-acl verify-nrc verify-projection verify-bff verify-domain verify-observability verify-notifications smoke up down local verify-local local-down changelog openzaak-up openzaak-smoke openzaak-seed openzaak-down stack-up stack-smoke stack-down keycloak-up keycloak-smoke keycloak-down flowable-up flowable-smoke flowable-down help +.PHONY: ci lint build unit mutation frontend integration verify verify-up verify-acl verify-nrc verify-projection verify-bff verify-domain verify-observability verify-tracing verify-notifications smoke up down local verify-local local-down changelog openzaak-up openzaak-smoke openzaak-seed openzaak-down stack-up stack-smoke stack-down keycloak-up keycloak-smoke keycloak-down flowable-up flowable-smoke flowable-down help ## ci: run the full pipeline — lint, build, unit, mutation, frontend, verify (mirrors Gitea Actions) ## `verify` is the live-stack stage (full stack up once → ACL + notification checks). @@ -175,6 +175,11 @@ verify-e2e: verify-observability: bash infra/run-observability-check.sh +## verify-tracing: assert one connected distributed trace spans the .NET services in Tempo +## (S-16b), against the already-running stack. +verify-tracing: + bash infra/run-tracing-check.sh + ## verify: local mirror of the CI verify-stack job — full stack up once, all checks, ## tear down (always). For fast single-concern local iteration use `integration` ## (oz-only) or `verify-notifications` (oz+nrc) instead. diff --git a/docs/demo-script.md b/docs/demo-script.md index c4b7492..02aa957 100644 --- a/docs/demo-script.md +++ b/docs/demo-script.md @@ -5,6 +5,33 @@ copy-pasteable walkthrough against a local `make up` stack. --- +## S-16b — distributed traces across the .NET services (#123, ADR-0023) + +**Outcome:** the five .NET services (BFF, Domain, ACL, projection-api, event-subscriber) now emit +OpenTelemetry traces — ASP.NET Core + `HttpClient` auto-instrumentation, exported over OTLP to Tempo. +Because every cross-service call goes through a typed `HttpClient`, the W3C `traceparent` propagates for +free, so a request is **one connected trace** across the services (bff → domain → acl → openzaak; +bff → projection-api). `/health` is filtered out. No browser-side instrumentation yet, so the trace +begins at the BFF; the async Flowable-poll boundary is a separate trace (ADR-0023). + +```bash +# 1. Automated (a CI verify-stack step): generate BFF traffic and assert Tempo holds one trace +# spanning multiple services. +make verify-tracing # → OK — trace spans ['bff', 'projection-api'] + +# 2. By hand: drive the stack, then explore traces in Grafana. +make up +curl -s localhost:8080/openbaar/register >/dev/null # BFF → projection-api +open http://localhost:3000 # Grafana → Explore → Tempo → Search → service.name = bff → open a trace +``` + +**The path:** each host wires `AddOpenTelemetry().WithTracing(AddAspNetCoreInstrumentation + +AddHttpClientInstrumentation + AddOtlpExporter)`; `OTEL_SERVICE_NAME` / `OTEL_EXPORTER_OTLP_ENDPOINT` +come from compose; spans export to **tempo:4317** and render in Grafana against the provisioned Tempo +datasource. + +--- + ## S-16a — observability backplane: Tempo + Prometheus + Grafana (#122, ADR-0023) **Outcome:** the compose stack now includes a Grafana-native observability backplane — **Tempo** (OTLP diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml index fddfeef..045d24f 100644 --- a/infra/docker-compose.yml +++ b/infra/docker-compose.yml @@ -296,6 +296,10 @@ services: dockerfile: Dockerfile image: register-referentie/acl:dev environment: + # OpenTelemetry traces → Tempo (S-16b, ADR-0023). + OTEL_EXPORTER_OTLP_ENDPOINT: http://tempo:4317 + OTEL_EXPORTER_OTLP_PROTOCOL: grpc + OTEL_SERVICE_NAME: acl # Overridable so verify-domain can point the ACL at the same OpenZaak host that # owns the seeded zaaktype URL (host-consistent zaak creation, ADR-0009). Acl__OpenZaak__BaseUrl: ${ACL_OPENZAAK_BASEURL:-http://openzaak:8000/} @@ -334,6 +338,10 @@ services: dockerfile: Dockerfile image: register-referentie/domain:dev environment: + # OpenTelemetry traces → Tempo (S-16b, ADR-0023). + OTEL_EXPORTER_OTLP_ENDPOINT: http://tempo:4317 + OTEL_EXPORTER_OTLP_PROTOCOL: grpc + OTEL_SERVICE_NAME: domain Flowable__BaseUrl: http://flowable-rest:8080/flowable-rest/ Flowable__Username: rest-admin Flowable__Password: test @@ -360,6 +368,10 @@ services: dockerfile: Dockerfile image: register-referentie/bff:dev environment: + # OpenTelemetry traces → Tempo (S-16b, ADR-0023). + OTEL_EXPORTER_OTLP_ENDPOINT: http://tempo:4317 + OTEL_EXPORTER_OTLP_PROTOCOL: grpc + OTEL_SERVICE_NAME: bff # The BFF is the portals' only backend; it validates digid tokens and fans out (ADR-0010). # Keycloak (start-dev) derives the issuer from the request host, so the BFF authority and the # verify token request both use keycloak:8080 to keep the issuer consistent. @@ -412,6 +424,10 @@ services: dockerfile: services/event-subscriber/Dockerfile image: register-referentie/event-subscriber:dev environment: + # OpenTelemetry traces → Tempo (S-16b, ADR-0023). + OTEL_EXPORTER_OTLP_ENDPOINT: http://tempo:4317 + OTEL_EXPORTER_OTLP_PROTOCOL: grpc + OTEL_SERVICE_NAME: event-subscriber ConnectionStrings__Projection: Host=projection-db;Database=projection;Username=projection;Password=projection # The subscriber enriches the projection with each zaak's reference (identificatie) by asking # the ACL — the only code allowed to read ZGW (§8.1, #78). @@ -441,6 +457,10 @@ services: dockerfile: services/projection-api/Dockerfile image: register-referentie/projection-api:dev environment: + # OpenTelemetry traces → Tempo (S-16b, ADR-0023). + OTEL_EXPORTER_OTLP_ENDPOINT: http://tempo:4317 + OTEL_EXPORTER_OTLP_PROTOCOL: grpc + OTEL_SERVICE_NAME: projection-api ConnectionStrings__Projection: Host=projection-db;Database=projection;Username=projection;Password=projection ports: - "8120:8080" @@ -538,12 +558,16 @@ services: context: ./observability/tempo image: register-referentie/tempo:dev command: ["-config.file=/etc/tempo.yaml"] + # Cap the backplane's footprint so it can't starve the app stack + the Playwright browser on the + # memory-tight CI runner (verify-e2e OOM history, commit d5e5fa2). Generous vs idle (~150M). + mem_limit: 400m networks: [cg] prometheus: build: context: ./observability/prometheus image: register-referentie/prometheus:dev + mem_limit: 400m ports: - "9090:9090" networks: [cg] @@ -552,6 +576,7 @@ services: build: context: ./observability/grafana image: register-referentie/grafana:dev + mem_limit: 512m environment: GF_SECURITY_ADMIN_USER: admin GF_SECURITY_ADMIN_PASSWORD: admin diff --git a/infra/run-tracing-check.sh b/infra/run-tracing-check.sh new file mode 100755 index 0000000..0da07c8 --- /dev/null +++ b/infra/run-tracing-check.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# +# S-16b (#123): assert one connected distributed trace spans the .NET services in Tempo, +# against an ALREADY-RUNNING full stack. Runs the driver in a python:3-slim container on the +# stack network (services reached by container IP; the runner can't reach published ports — +# gitea-actions-gotchas.md §5/§6). Does NOT manage the stack lifecycle. +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +ip() { docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$1"; } + +bff="$(docker ps -q --filter 'name=[-_]bff[-_]' | head -1)" +tempo="$(docker ps -q --filter 'name=[-_]tempo[-_]' | head -1)" +[ -n "$bff" ] && [ -n "$tempo" ] || { echo "ERROR: bff and/or tempo not running — bring the stack up first" >&2; exit 1; } +net="$(docker inspect -f '{{range $k,$_ := .NetworkSettings.Networks}}{{$k}}{{"\n"}}{{end}}' "$bff" | head -1)" +bff_ip="$(ip "$bff")"; tempo_ip="$(ip "$tempo")" +echo ">> network=$net bff=$bff_ip tempo=$tempo_ip" + +cid="$(docker create --network "$net" \ + -e "BFF=http://$bff_ip:8080" -e "TEMPO=http://$tempo_ip:3200" \ + -e "TRACING_TIMEOUT=${TRACING_TIMEOUT:-90}" \ + python:3-slim python /tracing-check.py)" +docker cp "$here/tracing-check.py" "$cid:/tracing-check.py" >/dev/null +rc=0; docker start -a "$cid" || rc=$? +docker rm -f "$cid" >/dev/null +exit $rc diff --git a/infra/tracing-check.py b/infra/tracing-check.py new file mode 100755 index 0000000..8fa3420 --- /dev/null +++ b/infra/tracing-check.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""S-16b (#123): prove distributed tracing works end to end. + +Generate anonymous BFF traffic (GET /openbaar/register, which the BFF serves by +calling projection-api — no auth, no OpenZaak egress), then query Tempo and assert +that ONE trace contains spans from both `bff` and `projection-api`. That proves the +services export OTLP to Tempo AND that the W3C traceparent propagates across the +HttpClient hop, stitching the request into a single connected trace. + +Stdlib only (urllib/json) so it runs in a bare python:3-slim container in-network. +""" +import json +import os +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + +BFF = os.environ["BFF"] # http://:8080 +TEMPO = os.environ["TEMPO"] # http://:3200 +TIMEOUT = int(os.environ.get("TRACING_TIMEOUT", "90")) +WANT = {"bff", "projection-api"} # the two services that must share one trace + + +def _get(url): + with urllib.request.urlopen(url, timeout=10) as r: + return r.read() + + +def generate_traffic(): + # A non-2xx still produces spans; only total unreachability of the BFF is fatal. + for _ in range(3): + try: + _get(f"{BFF}/openbaar/register") + except urllib.error.HTTPError: + pass + + +def search_trace_ids(): + q = urllib.parse.quote('{ resource.service.name = "bff" }') + try: + data = json.loads(_get(f"{TEMPO}/api/search?q={q}&limit=50")) + except Exception: + return [] + return [t["traceID"] for t in data.get("traces", [])] + + +def services_in_trace(trace_id): + try: + data = json.loads(_get(f"{TEMPO}/api/traces/{trace_id}")) + except Exception: + return set() + names = set() + for batch in data.get("batches", []): + for attr in batch.get("resource", {}).get("attributes", []): + if attr.get("key") == "service.name": + names.add(attr.get("value", {}).get("stringValue")) + return names + + +def main(): + deadline = time.time() + TIMEOUT + generate_traffic() + seen = set() + while time.time() < deadline: + for tid in search_trace_ids(): + names = services_in_trace(tid) + seen |= names + if WANT.issubset(names): + print(f"OK — trace {tid} spans {sorted(names)}") + return 0 + time.sleep(3) + generate_traffic() + print(f"FAIL — no single trace spanned {sorted(WANT)}; services seen: {sorted(seen)}", + file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/services/acl/Acl.Api/Acl.Api.csproj b/services/acl/Acl.Api/Acl.Api.csproj index 0a0116c..316816e 100644 --- a/services/acl/Acl.Api/Acl.Api.csproj +++ b/services/acl/Acl.Api/Acl.Api.csproj @@ -5,6 +5,13 @@ + + + + + + + net10.0 enable diff --git a/services/acl/Acl.Api/Program.cs b/services/acl/Acl.Api/Program.cs index 22cceee..052e15d 100644 --- a/services/acl/Acl.Api/Program.cs +++ b/services/acl/Acl.Api/Program.cs @@ -1,8 +1,21 @@ using Acl.Application; using Acl.Infrastructure; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; var builder = WebApplication.CreateBuilder(args); +// OpenTelemetry tracing (S-16b, ADR-0023): auto-instrument incoming ASP.NET Core requests and +// outgoing HttpClient calls (the ACL → OpenZaak hop), exported over OTLP to Tempo. Service name + +// OTLP endpoint come from OTEL_* env (compose); the exporter no-ops when Tempo is unreachable. +builder.Services.AddOpenTelemetry() + .ConfigureResource(r => r.AddService( + builder.Configuration["OTEL_SERVICE_NAME"] ?? builder.Environment.ApplicationName)) + .WithTracing(tracing => tracing + .AddAspNetCoreInstrumentation(o => o.Filter = ctx => ctx.Request.Path != "/health") + .AddHttpClientInstrumentation() + .AddOtlpExporter()); + builder.Services.AddSingleton(); builder.Services.AddSingleton(sp => sp.GetRequiredService() .GetSection("Acl:Defaults").Get() diff --git a/services/bff/Bff.Api/Bff.Api.csproj b/services/bff/Bff.Api/Bff.Api.csproj index b97a2fc..cb4f665 100644 --- a/services/bff/Bff.Api/Bff.Api.csproj +++ b/services/bff/Bff.Api/Bff.Api.csproj @@ -10,6 +10,10 @@ + + + + diff --git a/services/bff/Bff.Api/Program.cs b/services/bff/Bff.Api/Program.cs index 8f7df13..dcadd14 100644 --- a/services/bff/Bff.Api/Program.cs +++ b/services/bff/Bff.Api/Program.cs @@ -3,9 +3,23 @@ using System.Text.Json; using System.Text.Json.Serialization; using Bff.Api; using Microsoft.AspNetCore.Authentication.JwtBearer; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; var builder = WebApplication.CreateBuilder(args); +// OpenTelemetry tracing (S-16b, ADR-0023): auto-instrument incoming ASP.NET Core requests and +// outgoing HttpClient calls (BFF → Domain, BFF → projection-api), exported over OTLP to Tempo, so a +// portal request is one connected trace across the services. Service name + OTLP endpoint come from +// OTEL_* env (compose); the exporter no-ops when Tempo is unreachable. /health is filtered out. +builder.Services.AddOpenTelemetry() + .ConfigureResource(r => r.AddService( + builder.Configuration["OTEL_SERVICE_NAME"] ?? builder.Environment.ApplicationName)) + .WithTracing(tracing => tracing + .AddAspNetCoreInstrumentation(o => o.Filter = ctx => ctx.Request.Path != "/health") + .AddHttpClientInstrumentation() + .AddOtlpExporter()); + var keycloakAuthority = builder.Configuration["Keycloak:Authority"] ?? throw new InvalidOperationException("Missing configuration 'Keycloak:Authority'"); // Behandelaars authenticate against a *different* Keycloak realm (medewerker) than citizens (digid), diff --git a/services/domain/Big.Api/Big.Api.csproj b/services/domain/Big.Api/Big.Api.csproj index 673337b..eed138a 100644 --- a/services/domain/Big.Api/Big.Api.csproj +++ b/services/domain/Big.Api/Big.Api.csproj @@ -6,6 +6,10 @@ + + + + diff --git a/services/domain/Big.Api/Program.cs b/services/domain/Big.Api/Program.cs index 2a2827e..e4167e0 100644 --- a/services/domain/Big.Api/Program.cs +++ b/services/domain/Big.Api/Program.cs @@ -1,10 +1,25 @@ using Big.Application; using Big.Domain; using Big.Infrastructure; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; using Quartz; var builder = WebApplication.CreateBuilder(args); +// OpenTelemetry tracing (S-16b, ADR-0023): auto-instrument incoming ASP.NET Core requests and +// outgoing HttpClient calls, exported over OTLP to Tempo, so a request is one connected trace across +// the services. Service name + OTLP endpoint come from OTEL_* env (compose); the exporter no-ops +// harmlessly when Tempo is unreachable (e.g. a service run standalone). /health is filtered out so +// liveness polls don't flood the traces. +builder.Services.AddOpenTelemetry() + .ConfigureResource(r => r.AddService( + builder.Configuration["OTEL_SERVICE_NAME"] ?? builder.Environment.ApplicationName)) + .WithTracing(tracing => tracing + .AddAspNetCoreInstrumentation(o => o.Filter = ctx => ctx.Request.Path != "/health") + .AddHttpClientInstrumentation() + .AddOtlpExporter()); + // Options bound from configuration (compose sets Flowable__* and Acl__* env vars). builder.Services.AddSingleton(sp => sp.GetRequiredService() .GetSection("Flowable").Get() diff --git a/services/event-subscriber/EventSubscriber.Api/EventSubscriber.Api.csproj b/services/event-subscriber/EventSubscriber.Api/EventSubscriber.Api.csproj index 168c420..925003a 100644 --- a/services/event-subscriber/EventSubscriber.Api/EventSubscriber.Api.csproj +++ b/services/event-subscriber/EventSubscriber.Api/EventSubscriber.Api.csproj @@ -5,6 +5,13 @@ + + + + + + + net10.0 enable diff --git a/services/event-subscriber/EventSubscriber.Api/Program.cs b/services/event-subscriber/EventSubscriber.Api/Program.cs index d71b7eb..f21c8ca 100644 --- a/services/event-subscriber/EventSubscriber.Api/Program.cs +++ b/services/event-subscriber/EventSubscriber.Api/Program.cs @@ -1,9 +1,22 @@ using System.Text.Json; using EventSubscriber.Application; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; using Projection.ReadModel; var builder = WebApplication.CreateBuilder(args); +// OpenTelemetry tracing (S-16b, ADR-0023): auto-instrument the incoming NRC notification callback and +// the outgoing ACL enrichment call, exported over OTLP to Tempo. Service name + OTLP endpoint come +// from OTEL_* env (compose); the exporter no-ops when Tempo is unreachable. +builder.Services.AddOpenTelemetry() + .ConfigureResource(r => r.AddService( + builder.Configuration["OTEL_SERVICE_NAME"] ?? builder.Environment.ApplicationName)) + .WithTracing(tracing => tracing + .AddAspNetCoreInstrumentation(o => o.Filter = ctx => ctx.Request.Path != "/health") + .AddHttpClientInstrumentation() + .AddOtlpExporter()); + var connectionString = builder.Configuration.GetConnectionString("Projection") ?? throw new InvalidOperationException("Missing connection string 'ConnectionStrings:Projection'"); // The exact Authorization header value Open Notificaties sends on each abonnement callback. diff --git a/services/projection-api/ProjectionApi.Api/Program.cs b/services/projection-api/ProjectionApi.Api/Program.cs index a2b88aa..8a14c28 100644 --- a/services/projection-api/ProjectionApi.Api/Program.cs +++ b/services/projection-api/ProjectionApi.Api/Program.cs @@ -1,8 +1,21 @@ using Microsoft.EntityFrameworkCore; +using OpenTelemetry.Resources; +using OpenTelemetry.Trace; using Projection.ReadModel; var builder = WebApplication.CreateBuilder(args); +// OpenTelemetry tracing (S-16b, ADR-0023): auto-instrument incoming ASP.NET Core requests, exported +// over OTLP to Tempo, so a BFF → projection-api read is one connected trace. Service name + OTLP +// endpoint come from OTEL_* env (compose); the exporter no-ops when Tempo is unreachable. +builder.Services.AddOpenTelemetry() + .ConfigureResource(r => r.AddService( + builder.Configuration["OTEL_SERVICE_NAME"] ?? builder.Environment.ApplicationName)) + .WithTracing(tracing => tracing + .AddAspNetCoreInstrumentation(o => o.Filter = ctx => ctx.Request.Path != "/health") + .AddHttpClientInstrumentation() + .AddOtlpExporter()); + var connectionString = builder.Configuration.GetConnectionString("Projection") ?? throw new InvalidOperationException("Missing connection string 'ConnectionStrings:Projection'"); diff --git a/services/projection-api/ProjectionApi.Api/ProjectionApi.Api.csproj b/services/projection-api/ProjectionApi.Api/ProjectionApi.Api.csproj index b91c803..e9b681a 100644 --- a/services/projection-api/ProjectionApi.Api/ProjectionApi.Api.csproj +++ b/services/projection-api/ProjectionApi.Api/ProjectionApi.Api.csproj @@ -4,6 +4,13 @@ + + + + + + + net10.0 enable