feat(obs): OpenTelemetry distributed tracing across the five .NET services (refs #123)

Each host adds AddOpenTelemetry().WithTracing with ASP.NET Core + HttpClient
auto-instrumentation and an OTLP exporter to Tempo; service name + endpoint from
OTEL_* env set per app service in compose. Since every cross-service call goes
through a typed HttpClient, traceparent propagates for free, so a request is one
connected trace (bff → domain/projection → acl → openzaak). /health is filtered
out. The exporter no-ops harmlessly when Tempo is unreachable (verified: domain
boots healthy with no collector).

refs #123

infra/docker-compose.yml

@@ -296,6 +296,10 @@ services:
       dockerfile: Dockerfile
     image: register-referentie/acl:dev
     environment:
+      # OpenTelemetry traces → Tempo (S-16b, ADR-0023).
+      OTEL_EXPORTER_OTLP_ENDPOINT: http://tempo:4317
+      OTEL_EXPORTER_OTLP_PROTOCOL: grpc
+      OTEL_SERVICE_NAME: acl
       # Overridable so verify-domain can point the ACL at the same OpenZaak host that
       # owns the seeded zaaktype URL (host-consistent zaak creation, ADR-0009).
       Acl__OpenZaak__BaseUrl: ${ACL_OPENZAAK_BASEURL:-http://openzaak:8000/}
@@ -334,6 +338,10 @@ services:
       dockerfile: Dockerfile
     image: register-referentie/domain:dev
     environment:
+      # OpenTelemetry traces → Tempo (S-16b, ADR-0023).
+      OTEL_EXPORTER_OTLP_ENDPOINT: http://tempo:4317
+      OTEL_EXPORTER_OTLP_PROTOCOL: grpc
+      OTEL_SERVICE_NAME: domain
       Flowable__BaseUrl: http://flowable-rest:8080/flowable-rest/
       Flowable__Username: rest-admin
       Flowable__Password: test
@@ -360,6 +368,10 @@ services:
       dockerfile: Dockerfile
     image: register-referentie/bff:dev
     environment:
+      # OpenTelemetry traces → Tempo (S-16b, ADR-0023).
+      OTEL_EXPORTER_OTLP_ENDPOINT: http://tempo:4317
+      OTEL_EXPORTER_OTLP_PROTOCOL: grpc
+      OTEL_SERVICE_NAME: bff
       # The BFF is the portals' only backend; it validates digid tokens and fans out (ADR-0010).
       # Keycloak (start-dev) derives the issuer from the request host, so the BFF authority and the
       # verify token request both use keycloak:8080 to keep the issuer consistent.
@@ -412,6 +424,10 @@ services:
       dockerfile: services/event-subscriber/Dockerfile
     image: register-referentie/event-subscriber:dev
     environment:
+      # OpenTelemetry traces → Tempo (S-16b, ADR-0023).
+      OTEL_EXPORTER_OTLP_ENDPOINT: http://tempo:4317
+      OTEL_EXPORTER_OTLP_PROTOCOL: grpc
+      OTEL_SERVICE_NAME: event-subscriber
       ConnectionStrings__Projection: Host=projection-db;Database=projection;Username=projection;Password=projection
       # The subscriber enriches the projection with each zaak's reference (identificatie) by asking
       # the ACL — the only code allowed to read ZGW (§8.1, #78).
@@ -441,6 +457,10 @@ services:
       dockerfile: services/projection-api/Dockerfile
     image: register-referentie/projection-api:dev
     environment:
+      # OpenTelemetry traces → Tempo (S-16b, ADR-0023).
+      OTEL_EXPORTER_OTLP_ENDPOINT: http://tempo:4317
+      OTEL_EXPORTER_OTLP_PROTOCOL: grpc
+      OTEL_SERVICE_NAME: projection-api
       ConnectionStrings__Projection: Host=projection-db;Database=projection;Username=projection;Password=projection
     ports:
       - "8120:8080"

services/acl/Acl.Api/Acl.Api.csproj

@@ -5,6 +5,13 @@
     <ProjectReference Include="..\Acl.Infrastructure\Acl.Infrastructure.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.17.0" />
+    <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.17.0" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.17.0" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.17.0" />
+  </ItemGroup>
+
   <PropertyGroup>
     <TargetFramework>net10.0</TargetFramework>
     <Nullable>enable</Nullable>

services/acl/Acl.Api/Program.cs

@@ -1,8 +1,21 @@
 using Acl.Application;
 using Acl.Infrastructure;
+using OpenTelemetry.Resources;
+using OpenTelemetry.Trace;
 
 var builder = WebApplication.CreateBuilder(args);
 
+// OpenTelemetry tracing (S-16b, ADR-0023): auto-instrument incoming ASP.NET Core requests and
+// outgoing HttpClient calls (the ACL → OpenZaak hop), exported over OTLP to Tempo. Service name +
+// OTLP endpoint come from OTEL_* env (compose); the exporter no-ops when Tempo is unreachable.
+builder.Services.AddOpenTelemetry()
+    .ConfigureResource(r => r.AddService(
+        builder.Configuration["OTEL_SERVICE_NAME"] ?? builder.Environment.ApplicationName))
+    .WithTracing(tracing => tracing
+        .AddAspNetCoreInstrumentation(o => o.Filter = ctx => ctx.Request.Path != "/health")
+        .AddHttpClientInstrumentation()
+        .AddOtlpExporter());
+
 builder.Services.AddSingleton<IClock, SystemClock>();
 builder.Services.AddSingleton(sp => sp.GetRequiredService<IConfiguration>()
     .GetSection("Acl:Defaults").Get<AclDefaults>()

services/bff/Bff.Api/Bff.Api.csproj

@@ -10,6 +10,10 @@
     <!-- OIDC/JWT validation of Keycloak-issued tokens (ADR-0010) and OpenAPI generation. -->
     <PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="10.0.8" />
     <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="10.0.8" />
+    <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.17.0" />
+    <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.17.0" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.17.0" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.17.0" />
   </ItemGroup>
 
 </Project>

services/bff/Bff.Api/Program.cs

@@ -3,9 +3,23 @@ using System.Text.Json;
 using System.Text.Json.Serialization;
 using Bff.Api;
 using Microsoft.AspNetCore.Authentication.JwtBearer;
+using OpenTelemetry.Resources;
+using OpenTelemetry.Trace;
 
 var builder = WebApplication.CreateBuilder(args);
 
+// OpenTelemetry tracing (S-16b, ADR-0023): auto-instrument incoming ASP.NET Core requests and
+// outgoing HttpClient calls (BFF → Domain, BFF → projection-api), exported over OTLP to Tempo, so a
+// portal request is one connected trace across the services. Service name + OTLP endpoint come from
+// OTEL_* env (compose); the exporter no-ops when Tempo is unreachable. /health is filtered out.
+builder.Services.AddOpenTelemetry()
+    .ConfigureResource(r => r.AddService(
+        builder.Configuration["OTEL_SERVICE_NAME"] ?? builder.Environment.ApplicationName))
+    .WithTracing(tracing => tracing
+        .AddAspNetCoreInstrumentation(o => o.Filter = ctx => ctx.Request.Path != "/health")
+        .AddHttpClientInstrumentation()
+        .AddOtlpExporter());
+
 var keycloakAuthority = builder.Configuration["Keycloak:Authority"]
     ?? throw new InvalidOperationException("Missing configuration 'Keycloak:Authority'");
 // Behandelaars authenticate against a *different* Keycloak realm (medewerker) than citizens (digid),

services/domain/Big.Api/Big.Api.csproj

@@ -6,6 +6,10 @@
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.17.0" />
+    <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.17.0" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.17.0" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.17.0" />
     <PackageReference Include="Quartz.Extensions.Hosting" Version="3.18.2" />
   </ItemGroup>
 

services/domain/Big.Api/Program.cs

@@ -1,10 +1,25 @@
 using Big.Application;
 using Big.Domain;
 using Big.Infrastructure;
+using OpenTelemetry.Resources;
+using OpenTelemetry.Trace;
 using Quartz;
 
 var builder = WebApplication.CreateBuilder(args);
 
+// OpenTelemetry tracing (S-16b, ADR-0023): auto-instrument incoming ASP.NET Core requests and
+// outgoing HttpClient calls, exported over OTLP to Tempo, so a request is one connected trace across
+// the services. Service name + OTLP endpoint come from OTEL_* env (compose); the exporter no-ops
+// harmlessly when Tempo is unreachable (e.g. a service run standalone). /health is filtered out so
+// liveness polls don't flood the traces.
+builder.Services.AddOpenTelemetry()
+    .ConfigureResource(r => r.AddService(
+        builder.Configuration["OTEL_SERVICE_NAME"] ?? builder.Environment.ApplicationName))
+    .WithTracing(tracing => tracing
+        .AddAspNetCoreInstrumentation(o => o.Filter = ctx => ctx.Request.Path != "/health")
+        .AddHttpClientInstrumentation()
+        .AddOtlpExporter());
+
 // Options bound from configuration (compose sets Flowable__* and Acl__* env vars).
 builder.Services.AddSingleton(sp => sp.GetRequiredService<IConfiguration>()
     .GetSection("Flowable").Get<FlowableOptions>()

services/event-subscriber/EventSubscriber.Api/EventSubscriber.Api.csproj

@@ -5,6 +5,13 @@
     <ProjectReference Include="..\..\projection-api\Projection.ReadModel\Projection.ReadModel.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.17.0" />
+    <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.17.0" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.17.0" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.17.0" />
+  </ItemGroup>
+
   <PropertyGroup>
     <TargetFramework>net10.0</TargetFramework>
     <Nullable>enable</Nullable>

services/event-subscriber/EventSubscriber.Api/Program.cs

@@ -1,9 +1,22 @@
 using System.Text.Json;
 using EventSubscriber.Application;
+using OpenTelemetry.Resources;
+using OpenTelemetry.Trace;
 using Projection.ReadModel;
 
 var builder = WebApplication.CreateBuilder(args);
 
+// OpenTelemetry tracing (S-16b, ADR-0023): auto-instrument the incoming NRC notification callback and
+// the outgoing ACL enrichment call, exported over OTLP to Tempo. Service name + OTLP endpoint come
+// from OTEL_* env (compose); the exporter no-ops when Tempo is unreachable.
+builder.Services.AddOpenTelemetry()
+    .ConfigureResource(r => r.AddService(
+        builder.Configuration["OTEL_SERVICE_NAME"] ?? builder.Environment.ApplicationName))
+    .WithTracing(tracing => tracing
+        .AddAspNetCoreInstrumentation(o => o.Filter = ctx => ctx.Request.Path != "/health")
+        .AddHttpClientInstrumentation()
+        .AddOtlpExporter());
+
 var connectionString = builder.Configuration.GetConnectionString("Projection")
     ?? throw new InvalidOperationException("Missing connection string 'ConnectionStrings:Projection'");
 // The exact Authorization header value Open Notificaties sends on each abonnement callback.

services/projection-api/ProjectionApi.Api/Program.cs

@@ -1,8 +1,21 @@
 using Microsoft.EntityFrameworkCore;
+using OpenTelemetry.Resources;
+using OpenTelemetry.Trace;
 using Projection.ReadModel;
 
 var builder = WebApplication.CreateBuilder(args);
 
+// OpenTelemetry tracing (S-16b, ADR-0023): auto-instrument incoming ASP.NET Core requests, exported
+// over OTLP to Tempo, so a BFF → projection-api read is one connected trace. Service name + OTLP
+// endpoint come from OTEL_* env (compose); the exporter no-ops when Tempo is unreachable.
+builder.Services.AddOpenTelemetry()
+    .ConfigureResource(r => r.AddService(
+        builder.Configuration["OTEL_SERVICE_NAME"] ?? builder.Environment.ApplicationName))
+    .WithTracing(tracing => tracing
+        .AddAspNetCoreInstrumentation(o => o.Filter = ctx => ctx.Request.Path != "/health")
+        .AddHttpClientInstrumentation()
+        .AddOtlpExporter());
+
 var connectionString = builder.Configuration.GetConnectionString("Projection")
     ?? throw new InvalidOperationException("Missing connection string 'ConnectionStrings:Projection'");
 

services/projection-api/ProjectionApi.Api/ProjectionApi.Api.csproj

@@ -4,6 +4,13 @@
     <ProjectReference Include="..\Projection.ReadModel\Projection.ReadModel.csproj" />
   </ItemGroup>
 
+  <ItemGroup>
+    <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.17.0" />
+    <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.17.0" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.17.0" />
+    <PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.17.0" />
+  </ItemGroup>
+
   <PropertyGroup>
     <TargetFramework>net10.0</TargetFramework>
     <Nullable>enable</Nullable>