From 61805d5ce771db70d1b8d1c3908e783a668b52be Mon Sep 17 00:00:00 2001 From: Niek Otten Date: Fri, 24 Jul 2026 09:55:12 +0200 Subject: [PATCH 1/3] test(bff): /metrics exposes http-server request duration (refs #124) --- .../bff/Bff.Tests/MetricsEndpointTests.cs | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 services/bff/Bff.Tests/MetricsEndpointTests.cs diff --git a/services/bff/Bff.Tests/MetricsEndpointTests.cs b/services/bff/Bff.Tests/MetricsEndpointTests.cs new file mode 100644 index 0000000..87d317f --- /dev/null +++ b/services/bff/Bff.Tests/MetricsEndpointTests.cs @@ -0,0 +1,28 @@ +using System.Net; +using Microsoft.AspNetCore.Mvc.Testing; + +namespace Bff.Tests; + +/// +/// S-16c (#124): the service exposes OTel HTTP-server metrics in Prometheus text format at /metrics, +/// so Prometheus can scrape the golden signals (traffic, errors, latency) for the request path. +/// +public class MetricsEndpointTests(WebApplicationFactory factory) + : IClassFixture> +{ + [Fact] + public async Task Metrics_endpoint_exposes_http_server_request_duration_after_traffic() + { + var client = factory.CreateClient(); + + // One request produces an http.server.request.duration measurement... + await client.GetAsync("/health"); + + // ...which the /metrics scrape endpoint then exposes in Prometheus text format. + var response = await client.GetAsync("/metrics"); + + Assert.Equal(HttpStatusCode.OK, response.StatusCode); + var body = await response.Content.ReadAsStringAsync(); + Assert.Contains("http_server_request_duration", body); + } +} -- 2.54.0 From 965782dd951c504be67047e86697bc79d8f11fc3 Mon Sep 17 00:00:00 2001 From: Niek Otten Date: Fri, 24 Jul 2026 09:56:17 +0200 Subject: [PATCH 2/3] feat(bff): expose OTel golden-signal metrics on /metrics (refs #124) --- services/bff/Bff.Api/Bff.Api.csproj | 1 + services/bff/Bff.Api/Program.cs | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/services/bff/Bff.Api/Bff.Api.csproj b/services/bff/Bff.Api/Bff.Api.csproj index cb4f665..1187014 100644 --- a/services/bff/Bff.Api/Bff.Api.csproj +++ b/services/bff/Bff.Api/Bff.Api.csproj @@ -11,6 +11,7 @@ + diff --git a/services/bff/Bff.Api/Program.cs b/services/bff/Bff.Api/Program.cs index dcadd14..45e6d13 100644 --- a/services/bff/Bff.Api/Program.cs +++ b/services/bff/Bff.Api/Program.cs @@ -3,6 +3,7 @@ using System.Text.Json; using System.Text.Json.Serialization; using Bff.Api; using Microsoft.AspNetCore.Authentication.JwtBearer; +using OpenTelemetry.Metrics; using OpenTelemetry.Resources; using OpenTelemetry.Trace; @@ -18,7 +19,16 @@ builder.Services.AddOpenTelemetry() .WithTracing(tracing => tracing .AddAspNetCoreInstrumentation(o => o.Filter = ctx => ctx.Request.Path != "/health") .AddHttpClientInstrumentation() - .AddOtlpExporter()); + .AddOtlpExporter()) + // OpenTelemetry metrics (S-16c, ADR-0023): the golden signals for the request path — + // http.server.request.duration (traffic/errors/latency) + http.client.* for the downstream hops, + // plus the built-in System.Runtime meter for saturation (GC, CPU, thread pool). Prometheus scrapes + // these from /metrics (mapped below); no OTLP push for metrics, so no collector hop (ADR-0023). + .WithMetrics(metrics => metrics + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddMeter("System.Runtime") + .AddPrometheusExporter()); var keycloakAuthority = builder.Configuration["Keycloak:Authority"] ?? throw new InvalidOperationException("Missing configuration 'Keycloak:Authority'"); @@ -82,6 +92,9 @@ app.UseAuthentication(); app.UseAuthorization(); app.MapHealthChecks("/health"); + +// Prometheus scrape endpoint (S-16c): exposes the OTel metrics above in Prometheus text format. +app.MapPrometheusScrapingEndpoint(); app.MapOpenApi(); // Self-service submit: requires a valid digid token; the bsn comes from the token, not the body, -- 2.54.0 From 4ac2c3ff6c30e9026bab18c00fd8c01d2e2d5791 Mon Sep 17 00:00:00 2001 From: Niek Otten Date: Fri, 24 Jul 2026 10:06:34 +0200 Subject: [PATCH 3/3] feat(obs): golden-signal metrics on /metrics + Prometheus scrape + Grafana dashboard (S-16c, refs #124) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire OTel metrics into the four remaining .NET services (acl, domain, event-subscriber, projection-api) exactly as the BFF: ASP.NET Core + HttpClient instrumentation + the built-in System.Runtime meter, exposed at /metrics via the Prometheus AspNetCore exporter (ADR-0024). Prometheus scrapes one job per service; Grafana ships a pre-built 'Request path — golden signals' dashboard (traffic/errors/latency/saturation). A verify-metrics CI step proves the endpoints are scraped end to end. --- .gitea/workflows/ci.yaml | 2 + .gitignore | 1 + Makefile | 7 +- ...adr-0024-prometheus-aspnetcore-exporter.md | 53 +++++++++++ docs/demo-script.md | 28 ++++++ infra/metrics-check.py | 75 ++++++++++++++++ infra/observability/grafana/Dockerfile | 4 +- .../provisioning/dashboards/dashboards.yaml | 13 +++ .../dashboards/golden-signals.json | 87 +++++++++++++++++++ infra/observability/prometheus/prometheus.yml | 23 ++++- infra/run-metrics-check.sh | 28 ++++++ services/acl/Acl.Api/Acl.Api.csproj | 1 + services/acl/Acl.Api/Program.cs | 15 +++- services/domain/Big.Api/Big.Api.csproj | 1 + services/domain/Big.Api/Program.cs | 15 +++- .../EventSubscriber.Api.csproj | 1 + .../EventSubscriber.Api/Program.cs | 15 +++- .../ProjectionApi.Api/Program.cs | 15 +++- .../ProjectionApi.Api.csproj | 1 + 19 files changed, 375 insertions(+), 10 deletions(-) create mode 100644 docs/architecture/adr-0024-prometheus-aspnetcore-exporter.md create mode 100755 infra/metrics-check.py create mode 100644 infra/observability/grafana/provisioning/dashboards/dashboards.yaml create mode 100644 infra/observability/grafana/provisioning/dashboards/golden-signals.json create mode 100755 infra/run-metrics-check.sh diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 84e9f79..3b376cb 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -170,6 +170,8 @@ jobs: run: make verify-bff - name: Distributed traces reach Tempo (one connected trace across services) run: TRACING_TIMEOUT=120 make verify-tracing + - name: Golden-signal metrics scraped by Prometheus (/metrics on every service) + run: METRICS_TIMEOUT=120 make verify-metrics - name: Self-service e2e (Playwright, login → submit → success) run: make verify-e2e # Log dump must precede teardown (which removes the containers). diff --git a/.gitignore b/.gitignore index 30b841d..ae54b38 100644 --- a/.gitignore +++ b/.gitignore @@ -57,3 +57,4 @@ vitest.config.*.timestamp* tests/e2e/node_modules/ tests/e2e/test-results/ tests/e2e/playwright-report/ +__pycache__/ diff --git a/Makefile b/Makefile index b5a17c8..4e12315 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ export DOCKER_HOST := unix://$(PODMAN_SOCK) endif endif -.PHONY: ci lint build unit mutation frontend integration verify verify-up verify-acl verify-nrc verify-projection verify-bff verify-domain verify-observability verify-tracing verify-notifications smoke up down local verify-local local-down changelog openzaak-up openzaak-smoke openzaak-seed openzaak-down stack-up stack-smoke stack-down keycloak-up keycloak-smoke keycloak-down flowable-up flowable-smoke flowable-down help +.PHONY: ci lint build unit mutation frontend integration verify verify-up verify-acl verify-nrc verify-projection verify-bff verify-domain verify-observability verify-tracing verify-metrics verify-notifications smoke up down local verify-local local-down changelog openzaak-up openzaak-smoke openzaak-seed openzaak-down stack-up stack-smoke stack-down keycloak-up keycloak-smoke keycloak-down flowable-up flowable-smoke flowable-down help ## ci: run the full pipeline — lint, build, unit, mutation, frontend, verify (mirrors Gitea Actions) ## `verify` is the live-stack stage (full stack up once → ACL + notification checks). @@ -180,6 +180,11 @@ verify-observability: verify-tracing: bash infra/run-tracing-check.sh +## verify-metrics: assert the services expose /metrics and Prometheus scrapes the golden +## signals (S-16c), against the already-running stack. +verify-metrics: + bash infra/run-metrics-check.sh + ## verify: local mirror of the CI verify-stack job — full stack up once, all checks, ## tear down (always). For fast single-concern local iteration use `integration` ## (oz-only) or `verify-notifications` (oz+nrc) instead. diff --git a/docs/architecture/adr-0024-prometheus-aspnetcore-exporter.md b/docs/architecture/adr-0024-prometheus-aspnetcore-exporter.md new file mode 100644 index 0000000..6a2b6d3 --- /dev/null +++ b/docs/architecture/adr-0024-prometheus-aspnetcore-exporter.md @@ -0,0 +1,53 @@ +# ADR-0024: Expose OTel metrics with the (prerelease) Prometheus AspNetCore exporter + +- **Status:** Accepted +- **Date:** 2026-07-24 +- **Deciders:** Respellion engineering +- **Slice:** S-16c (#124), last of the S-16 (#17) split + +## Context + +ADR-0023 already fixed the shape of metrics collection: **Prometheus scrapes each +service's `/metrics`** (pull, no collector). S-16c implements it. That needs a package +that turns the OpenTelemetry `MeterProvider` into a Prometheus scrape endpoint inside +ASP.NET Core. The canonical one is `OpenTelemetry.Exporter.Prometheus.AspNetCore` +(`AddPrometheusExporter()` + `app.MapPrometheusScrapingEndpoint()`). + +The catch: that exporter has **never had a stable release** — the whole OTel .NET +Prometheus exporter line is versioned `-beta` (we pin `1.17.0-beta.1`, matched to the +`1.17.0` core we already use). Adding it is a new dependency (CLAUDE.md §14), and taking +a prerelease package into all five services is the decision worth recording. + +## Decision + +**Add `OpenTelemetry.Exporter.Prometheus.AspNetCore` `1.17.0-beta.1` to the five .NET +services and expose `/metrics` with it.** + +- What it gives us: the OTel-native pull endpoint, so the meters we already register for + tracing-adjacent instrumentation surface as Prometheus text with zero extra plumbing. +- What we'd write to replace it: a hand-rolled `IMetricsListener`/`MeterListener` that + formats Prometheus exposition text — real work, and a reimplementation of a widely-used + library for no gain. +- Risk it adds: a prerelease API that can shift between betas. Contained: it is only + wired in `Program.cs` (two calls per service, excluded from mutation), the version is + pinned, and `verify-metrics` proves the endpoint + scrape actually work each CI run. + +The alternative — pushing metrics over OTLP to a collector that re-exposes them — was +already rejected in ADR-0023 (no collector hop). Not revisited here. + +## Consequences + +**Positive** + +- Golden-signal metrics on `/metrics` with the standard OTel names + (`http_server_request_duration_seconds`, `dotnet_*`), scraped straight by Prometheus. +- No collector, no bespoke exposition code. + +**Negative / costs** + +- A `-beta` package in production services. Mitigated by the pin + the `verify-metrics` + CI gate; upgrading tracks the OTel core version bumps. + +## Coupling rules touched (CLAUDE.md §8) + +None. Metrics are passive: Prometheus pulls; no service calls into the stack. diff --git a/docs/demo-script.md b/docs/demo-script.md index 02aa957..09d3a74 100644 --- a/docs/demo-script.md +++ b/docs/demo-script.md @@ -5,6 +5,34 @@ copy-pasteable walkthrough against a local `make up` stack. --- +## S-16c — Prometheus metrics + golden-signal Grafana dashboard (#124, ADR-0023) + +**Outcome:** the five .NET services now expose OpenTelemetry metrics in Prometheus format at `/metrics` +— ASP.NET Core + `HttpClient` instrumentation plus the built-in `System.Runtime` meter. Prometheus +scrapes each service (one job per service), and a **pre-built Grafana dashboard** — *Request path — +golden signals* — plots the four golden signals: **traffic** (req/s), **errors** (5xx/s), **latency** +(p95 request duration), and **saturation** (CPU cores in use), split by service. It populates under load. + +```bash +# 1. Automated (a CI verify-stack step): generate BFF traffic and assert Prometheus scraped the +# golden-signal metric from every service. +make verify-metrics # → OK — targets up: [...]; request metric scraped from: [...] + +# 2. By hand: drive the stack, generate some load, then open the dashboard. +make up +for i in $(seq 1 50); do curl -s localhost:8080/openbaar/register >/dev/null; done # BFF → projection-api +open http://localhost:3000 # Grafana → Dashboards → "Request path — golden signals" +open http://localhost:9090/targets # Prometheus → every service target UP +``` + +**The path:** each host adds `.WithMetrics(AddAspNetCoreInstrumentation + AddHttpClientInstrumentation + +AddMeter("System.Runtime") + AddPrometheusExporter)` and maps `/metrics`; Prometheus scrapes +`:8080/metrics` (config in `infra/observability/prometheus/prometheus.yml`); Grafana ships the +dashboard via provisioning against the fixed `prometheus` datasource uid. No metrics are pushed over +OTLP — Prometheus pulls, so there is no collector hop (ADR-0023). + +--- + ## S-16b — distributed traces across the .NET services (#123, ADR-0023) **Outcome:** the five .NET services (BFF, Domain, ACL, projection-api, event-subscriber) now emit diff --git a/infra/metrics-check.py b/infra/metrics-check.py new file mode 100755 index 0000000..237cee1 --- /dev/null +++ b/infra/metrics-check.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""S-16c (#124): prove the golden-signal metrics pipeline works end to end. + +Generate anonymous BFF traffic (GET /openbaar/register — no auth, no OpenZaak egress), +then query Prometheus and assert (1) every .NET service's scrape target is UP, and (2) +the http.server.request.duration histogram is actually being scraped — i.e. the services +expose /metrics AND Prometheus collects it, which is exactly what the golden-signal +dashboard reads. + +Stdlib only (urllib/json) so it runs in a bare python:3-slim container in-network. +""" +import json +import os +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + +BFF = os.environ["BFF"] # http://:8080 +PROM = os.environ["PROMETHEUS"] # http://:9090 +TIMEOUT = int(os.environ.get("METRICS_TIMEOUT", "90")) +SERVICES = {"acl", "domain", "bff", "event-subscriber", "projection-api"} + + +def _get(url): + with urllib.request.urlopen(url, timeout=10) as r: + return r.read() + + +def generate_traffic(): + for _ in range(3): + try: + _get(f"{BFF}/openbaar/register") + except urllib.error.HTTPError: + pass # a non-2xx still records an http.server metric + + +def query(promql): + q = urllib.parse.quote(promql) + try: + data = json.loads(_get(f"{PROM}/api/v1/query?query={q}")) + except Exception: + return [] + return data.get("data", {}).get("result", []) + + +def jobs_up(): + return {r["metric"].get("job") for r in query("up == 1")} + + +def jobs_with_request_metric(): + return {r["metric"].get("job") + for r in query("http_server_request_duration_seconds_count")} + + +def main(): + deadline = time.time() + TIMEOUT + while time.time() < deadline: + generate_traffic() + up = jobs_up() + scraped = jobs_with_request_metric() + if SERVICES.issubset(up) and SERVICES.issubset(scraped): + print(f"OK — targets up: {sorted(up & SERVICES)}; " + f"request metric scraped from: {sorted(scraped & SERVICES)}") + return 0 + time.sleep(3) + print(f"FAIL — up: {sorted(jobs_up() & SERVICES)}; " + f"request metric from: {sorted(jobs_with_request_metric() & SERVICES)}; " + f"expected all of {sorted(SERVICES)}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/infra/observability/grafana/Dockerfile b/infra/observability/grafana/Dockerfile index aefc6f7..c9709dc 100644 --- a/infra/observability/grafana/Dockerfile +++ b/infra/observability/grafana/Dockerfile @@ -1,4 +1,4 @@ -# Grafana with datasources baked in via provisioning (S-16a, ADR-0023). -# Dashboards (S-16c, #124) are added under provisioning/dashboards later. +# Grafana with datasources + the golden-signals dashboard baked in via provisioning +# (S-16a/S-16c, ADR-0023). Everything under provisioning/ is copied in below. FROM grafana/grafana:11.3.0 COPY provisioning/ /etc/grafana/provisioning/ diff --git a/infra/observability/grafana/provisioning/dashboards/dashboards.yaml b/infra/observability/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 0000000..9db21f5 --- /dev/null +++ b/infra/observability/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,13 @@ +# Dashboard provider (S-16c, ADR-0023): Grafana loads every *.json in this folder as a +# read-only, code-owned dashboard. The golden-signals board is versioned here, not +# clicked together in the UI. +apiVersion: 1 + +providers: + - name: register-referentie + type: file + disableDeletion: true + allowUiUpdates: false + options: + path: /etc/grafana/provisioning/dashboards + foldersFromFilesStructure: false diff --git a/infra/observability/grafana/provisioning/dashboards/golden-signals.json b/infra/observability/grafana/provisioning/dashboards/golden-signals.json new file mode 100644 index 0000000..292706d --- /dev/null +++ b/infra/observability/grafana/provisioning/dashboards/golden-signals.json @@ -0,0 +1,87 @@ +{ + "uid": "golden-signals", + "title": "Request path — golden signals", + "tags": ["s-16c", "golden-signals"], + "timezone": "browser", + "schemaVersion": 39, + "version": 1, + "editable": true, + "refresh": "10s", + "time": { "from": "now-15m", "to": "now" }, + "templating": { + "list": [ + { + "name": "job", + "type": "query", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "query": "label_values(http_server_request_duration_seconds_count, job)", + "includeAll": true, + "multi": true, + "current": { "text": "All", "value": "$__all" }, + "refresh": 2 + } + ] + }, + "panels": [ + { + "id": 1, + "title": "Traffic — requests/sec", + "type": "timeseries", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, + "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "line", "fillOpacity": 10 } }, "overrides": [] }, + "targets": [ + { + "refId": "A", + "expr": "sum by (job) (rate(http_server_request_duration_seconds_count{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "{{job}}" + } + ] + }, + { + "id": 2, + "title": "Errors — 5xx responses/sec", + "type": "timeseries", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, + "fieldConfig": { "defaults": { "unit": "reqps", "custom": { "drawStyle": "line", "fillOpacity": 10 }, "color": { "mode": "fixed", "fixedColor": "red" } }, "overrides": [] }, + "targets": [ + { + "refId": "A", + "expr": "sum by (job) (rate(http_server_request_duration_seconds_count{job=~\"$job\",http_response_status_code=~\"5..\"}[$__rate_interval]))", + "legendFormat": "{{job}}" + } + ] + }, + { + "id": 3, + "title": "Latency — p95 request duration", + "type": "timeseries", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, + "fieldConfig": { "defaults": { "unit": "s", "custom": { "drawStyle": "line", "fillOpacity": 10 } }, "overrides": [] }, + "targets": [ + { + "refId": "A", + "expr": "histogram_quantile(0.95, sum by (job, le) (rate(http_server_request_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])))", + "legendFormat": "{{job}} p95" + } + ] + }, + { + "id": 4, + "title": "Saturation — CPU cores in use", + "type": "timeseries", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, + "fieldConfig": { "defaults": { "unit": "none", "custom": { "drawStyle": "line", "fillOpacity": 10 } }, "overrides": [] }, + "targets": [ + { + "refId": "A", + "expr": "sum by (job) (rate(dotnet_process_cpu_time_seconds_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "{{job}}" + } + ] + } + ] +} diff --git a/infra/observability/prometheus/prometheus.yml b/infra/observability/prometheus/prometheus.yml index 4e0f800..d2f6ed5 100644 --- a/infra/observability/prometheus/prometheus.yml +++ b/infra/observability/prometheus/prometheus.yml @@ -1,6 +1,7 @@ -# Prometheus scrape config (S-16a, ADR-0023). For the backplane slice it scrapes -# only itself; the .NET services' /metrics scrape targets are added in S-16c -# (#124) when the services expose metrics. +# Prometheus scrape config (S-16c, ADR-0023). Each .NET service exposes OTel metrics +# at /metrics (Prometheus text format); one scrape job per service, so the service is +# identified by the `job` label in the golden-signal dashboard. Targets are reached by +# compose service name on the shared `cg` network (internal port 8080). global: scrape_interval: 15s @@ -8,3 +9,19 @@ scrape_configs: - job_name: prometheus static_configs: - targets: ['localhost:9090'] + + - job_name: acl + static_configs: + - targets: ['acl:8080'] + - job_name: domain + static_configs: + - targets: ['domain:8080'] + - job_name: bff + static_configs: + - targets: ['bff:8080'] + - job_name: event-subscriber + static_configs: + - targets: ['event-subscriber:8080'] + - job_name: projection-api + static_configs: + - targets: ['projection-api:8080'] diff --git a/infra/run-metrics-check.sh b/infra/run-metrics-check.sh new file mode 100755 index 0000000..48c0bca --- /dev/null +++ b/infra/run-metrics-check.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# +# S-16c (#124): assert the golden-signal metrics pipeline works — the .NET services expose +# /metrics and Prometheus scrapes them — against an ALREADY-RUNNING full stack. Runs the +# driver in a python:3-slim container on the stack network (services reached by container IP; +# the runner can't reach published ports — gitea-actions-gotchas.md §5/§6). Does NOT manage +# the stack lifecycle. +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +ip() { docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$1"; } + +bff="$(docker ps -q --filter 'name=[-_]bff[-_]' | head -1)" +prom="$(docker ps -q --filter 'name=[-_]prometheus[-_]' | head -1)" +[ -n "$bff" ] && [ -n "$prom" ] || { echo "ERROR: bff and/or prometheus not running — bring the stack up first" >&2; exit 1; } +net="$(docker inspect -f '{{range $k,$_ := .NetworkSettings.Networks}}{{$k}}{{"\n"}}{{end}}' "$bff" | head -1)" +bff_ip="$(ip "$bff")"; prom_ip="$(ip "$prom")" +echo ">> network=$net bff=$bff_ip prometheus=$prom_ip" + +cid="$(docker create --network "$net" \ + -e "BFF=http://$bff_ip:8080" -e "PROMETHEUS=http://$prom_ip:9090" \ + -e "METRICS_TIMEOUT=${METRICS_TIMEOUT:-90}" \ + python:3-slim python /metrics-check.py)" +docker cp "$here/metrics-check.py" "$cid:/metrics-check.py" >/dev/null +rc=0; docker start -a "$cid" || rc=$? +docker rm -f "$cid" >/dev/null +exit $rc diff --git a/services/acl/Acl.Api/Acl.Api.csproj b/services/acl/Acl.Api/Acl.Api.csproj index 316816e..5b27d67 100644 --- a/services/acl/Acl.Api/Acl.Api.csproj +++ b/services/acl/Acl.Api/Acl.Api.csproj @@ -7,6 +7,7 @@ + diff --git a/services/acl/Acl.Api/Program.cs b/services/acl/Acl.Api/Program.cs index 052e15d..7812234 100644 --- a/services/acl/Acl.Api/Program.cs +++ b/services/acl/Acl.Api/Program.cs @@ -1,5 +1,6 @@ using Acl.Application; using Acl.Infrastructure; +using OpenTelemetry.Metrics; using OpenTelemetry.Resources; using OpenTelemetry.Trace; @@ -14,7 +15,16 @@ builder.Services.AddOpenTelemetry() .WithTracing(tracing => tracing .AddAspNetCoreInstrumentation(o => o.Filter = ctx => ctx.Request.Path != "/health") .AddHttpClientInstrumentation() - .AddOtlpExporter()); + .AddOtlpExporter()) + // OpenTelemetry metrics (S-16c, ADR-0023): golden signals for the request path — + // http.server.request.duration (traffic/errors/latency) + http.client.* for downstream hops, plus + // the built-in System.Runtime meter for saturation (GC, CPU, thread pool). Prometheus scrapes these + // from /metrics (mapped below); metrics aren't pushed over OTLP, so no collector hop (ADR-0023). + .WithMetrics(metrics => metrics + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddMeter("System.Runtime") + .AddPrometheusExporter()); builder.Services.AddSingleton(); builder.Services.AddSingleton(sp => sp.GetRequiredService() @@ -32,6 +42,9 @@ var app = builder.Build(); app.MapGet("/health", () => "Healthy"); +// Prometheus scrape endpoint (S-16c): exposes the OTel metrics above in Prometheus text format. +app.MapPrometheusScrapingEndpoint(); + // The ACL's single operation, exposed as a service endpoint. app.MapPost("/zaken", async (OpenZaakRequest body, AclService acl, CancellationToken ct) => { diff --git a/services/domain/Big.Api/Big.Api.csproj b/services/domain/Big.Api/Big.Api.csproj index eed138a..9b215e0 100644 --- a/services/domain/Big.Api/Big.Api.csproj +++ b/services/domain/Big.Api/Big.Api.csproj @@ -7,6 +7,7 @@ + diff --git a/services/domain/Big.Api/Program.cs b/services/domain/Big.Api/Program.cs index e4167e0..466ce1d 100644 --- a/services/domain/Big.Api/Program.cs +++ b/services/domain/Big.Api/Program.cs @@ -1,6 +1,7 @@ using Big.Application; using Big.Domain; using Big.Infrastructure; +using OpenTelemetry.Metrics; using OpenTelemetry.Resources; using OpenTelemetry.Trace; using Quartz; @@ -18,7 +19,16 @@ builder.Services.AddOpenTelemetry() .WithTracing(tracing => tracing .AddAspNetCoreInstrumentation(o => o.Filter = ctx => ctx.Request.Path != "/health") .AddHttpClientInstrumentation() - .AddOtlpExporter()); + .AddOtlpExporter()) + // OpenTelemetry metrics (S-16c, ADR-0023): golden signals for the request path — + // http.server.request.duration (traffic/errors/latency) + http.client.* for downstream hops, plus + // the built-in System.Runtime meter for saturation (GC, CPU, thread pool). Prometheus scrapes these + // from /metrics (mapped below); metrics aren't pushed over OTLP, so no collector hop (ADR-0023). + .WithMetrics(metrics => metrics + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddMeter("System.Runtime") + .AddPrometheusExporter()); // Options bound from configuration (compose sets Flowable__* and Acl__* env vars). builder.Services.AddSingleton(sp => sp.GetRequiredService() @@ -84,6 +94,9 @@ var app = builder.Build(); app.MapGet("/health", () => "Healthy"); +// Prometheus scrape endpoint (S-16c): exposes the OTel metrics above in Prometheus text format. +app.MapPrometheusScrapingEndpoint(); + // Submit a registration. The aggregate is created (INGEDIEND) and the registratie process started; // the zaak is opened later, off the request path, by the worker — so this returns 202 Accepted with // a location to read the registration's progress (ADR-0009, eventual consistency). diff --git a/services/event-subscriber/EventSubscriber.Api/EventSubscriber.Api.csproj b/services/event-subscriber/EventSubscriber.Api/EventSubscriber.Api.csproj index 925003a..faa78ec 100644 --- a/services/event-subscriber/EventSubscriber.Api/EventSubscriber.Api.csproj +++ b/services/event-subscriber/EventSubscriber.Api/EventSubscriber.Api.csproj @@ -7,6 +7,7 @@ + diff --git a/services/event-subscriber/EventSubscriber.Api/Program.cs b/services/event-subscriber/EventSubscriber.Api/Program.cs index f21c8ca..07a4656 100644 --- a/services/event-subscriber/EventSubscriber.Api/Program.cs +++ b/services/event-subscriber/EventSubscriber.Api/Program.cs @@ -1,5 +1,6 @@ using System.Text.Json; using EventSubscriber.Application; +using OpenTelemetry.Metrics; using OpenTelemetry.Resources; using OpenTelemetry.Trace; using Projection.ReadModel; @@ -15,7 +16,16 @@ builder.Services.AddOpenTelemetry() .WithTracing(tracing => tracing .AddAspNetCoreInstrumentation(o => o.Filter = ctx => ctx.Request.Path != "/health") .AddHttpClientInstrumentation() - .AddOtlpExporter()); + .AddOtlpExporter()) + // OpenTelemetry metrics (S-16c, ADR-0023): golden signals for the request path — + // http.server.request.duration (traffic/errors/latency) + http.client.* for downstream hops, plus + // the built-in System.Runtime meter for saturation (GC, CPU, thread pool). Prometheus scrapes these + // from /metrics (mapped below); metrics aren't pushed over OTLP, so no collector hop (ADR-0023). + .WithMetrics(metrics => metrics + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddMeter("System.Runtime") + .AddPrometheusExporter()); var connectionString = builder.Configuration.GetConnectionString("Projection") ?? throw new InvalidOperationException("Missing connection string 'ConnectionStrings:Projection'"); @@ -41,6 +51,9 @@ await app.Services.MigrateProjectionAsync(); app.MapGet("/health", () => "Healthy"); +// Prometheus scrape endpoint (S-16c): exposes the OTel metrics above in Prometheus text format. +app.MapPrometheusScrapingEndpoint(); + // The NRC abonnement callback. Open Notificaties POSTs a notification here; we project it. // Auth-on-callback is mandatory: the auth check runs *before* the body is read, so NRC's // registration probe (a POST without the configured Authorization, and without a valid diff --git a/services/projection-api/ProjectionApi.Api/Program.cs b/services/projection-api/ProjectionApi.Api/Program.cs index 8a14c28..067aa89 100644 --- a/services/projection-api/ProjectionApi.Api/Program.cs +++ b/services/projection-api/ProjectionApi.Api/Program.cs @@ -1,4 +1,5 @@ using Microsoft.EntityFrameworkCore; +using OpenTelemetry.Metrics; using OpenTelemetry.Resources; using OpenTelemetry.Trace; using Projection.ReadModel; @@ -14,7 +15,16 @@ builder.Services.AddOpenTelemetry() .WithTracing(tracing => tracing .AddAspNetCoreInstrumentation(o => o.Filter = ctx => ctx.Request.Path != "/health") .AddHttpClientInstrumentation() - .AddOtlpExporter()); + .AddOtlpExporter()) + // OpenTelemetry metrics (S-16c, ADR-0023): golden signals for the request path — + // http.server.request.duration (traffic/errors/latency) + http.client.* for downstream hops, plus + // the built-in System.Runtime meter for saturation (GC, CPU, thread pool). Prometheus scrapes these + // from /metrics (mapped below); metrics aren't pushed over OTLP, so no collector hop (ADR-0023). + .WithMetrics(metrics => metrics + .AddAspNetCoreInstrumentation() + .AddHttpClientInstrumentation() + .AddMeter("System.Runtime") + .AddPrometheusExporter()); var connectionString = builder.Configuration.GetConnectionString("Projection") ?? throw new InvalidOperationException("Missing connection string 'ConnectionStrings:Projection'"); @@ -30,6 +40,9 @@ await app.Services.MigrateProjectionAsync(); app.MapGet("/health", () => "Healthy"); +// Prometheus scrape endpoint (S-16c): exposes the OTel metrics above in Prometheus text format. +app.MapPrometheusScrapingEndpoint(); + // The read side of the projection. Public-safe field filtering is tightened in S-09; for now // the minimal projection only carries id + status (bsn/naam deferred — ADR-0008). app.MapGet("/register", async (ProjectionDbContext db, CancellationToken ct) => diff --git a/services/projection-api/ProjectionApi.Api/ProjectionApi.Api.csproj b/services/projection-api/ProjectionApi.Api/ProjectionApi.Api.csproj index e9b681a..07d4b39 100644 --- a/services/projection-api/ProjectionApi.Api/ProjectionApi.Api.csproj +++ b/services/projection-api/ProjectionApi.Api/ProjectionApi.Api.csproj @@ -6,6 +6,7 @@ + -- 2.54.0