diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 84fbaea..7eabeb2 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -204,6 +204,9 @@ jobs: - name: Objecttypen API up + token authenticates id: objecttypen run: OBJECTTYPEN_TIMEOUT=120 make verify-objecttypen + - name: Objecten API up + token authenticates + trusts Objecttypen + id: objecten + run: OBJECTEN_TIMEOUT=120 make verify-objecten - name: ACL ↔ OpenZaak integration tests id: acl run: make verify-acl @@ -237,6 +240,7 @@ jobs: UP: ${{ steps.up.outcome }} OBS: ${{ steps.obs.outcome }} OBJECTTYPEN: ${{ steps.objecttypen.outcome }} + OBJECTEN: ${{ steps.objecten.outcome }} ACL: ${{ steps.acl.outcome }} NRC: ${{ steps.nrc.outcome }} PROJECTION: ${{ steps.projection.outcome }} @@ -256,6 +260,7 @@ jobs: echo "| Bring up + health | $(icon "$UP") |" echo "| Observability backplane | $(icon "$OBS") |" echo "| Objecttypen API + token | $(icon "$OBJECTTYPEN") |" + echo "| Objecten API + token | $(icon "$OBJECTEN") |" echo "| ACL ↔ OpenZaak | $(icon "$ACL") |" echo "| OpenZaak → NRC | $(icon "$NRC") |" echo "| NRC → Event Subscriber → projection | $(icon "$PROJECTION") |" @@ -275,7 +280,7 @@ jobs: # Log dump must precede teardown (which removes the containers). - name: Dump container logs on failure if: failure() - run: docker compose -f infra/docker-compose.yml logs --no-color --tail=100 oz-init openzaak nrc-init nrc-web nrc-celery nrc-beat flowable-db flowable-rest flowable-init keycloak acl bff domain projection-db event-subscriber projection-api self-service openbaar behandel beheer objecttypen-db objecttypen-redis objecttypen-init objecttypen tempo prometheus grafana 2>&1 || true + run: docker compose -f infra/docker-compose.yml logs --no-color --tail=100 oz-init openzaak nrc-init nrc-web nrc-celery nrc-beat flowable-db flowable-rest flowable-init keycloak acl bff domain projection-db event-subscriber projection-api self-service openbaar behandel beheer objecttypen-db objecttypen-redis objecttypen-init objecttypen objecten-db objecten-redis objecten-init objecten tempo prometheus grafana 2>&1 || true - name: Tear down if: always() run: make down diff --git a/BACKLOG.md b/BACKLOG.md index 998299b..d00afac 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -283,8 +283,8 @@ Split into independently deployable sub-slices (CLAUDE.md §13): Split into independently deployable sub-slices (CLAUDE.md §13): -- **S-18a** (#139) · Objecttypen API up in compose (own DB + seeded config + health + static token). -- **S-18b** (#140) · Objecten API up in compose, wired to Objecttypen. Depends on S-18a. +- **S-18a** (#139, ✅) · Objecttypen API up in compose (own DB + seeded config + health + static token). +- **S-18b** (#140, ✅) · Objecten API up in compose, wired to Objecttypen. Depends on S-18a. - **S-18c** (#141) · RegisterRecord objecttype defined + registered (public-safe JSON schema). Depends on S-18a/b. ### S-19 · ACL extension: write register-record to Objecten on approval diff --git a/Makefile b/Makefile index 0a5dd05..a399cbf 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ COMPOSE := infra/docker-compose.yml # Long-running services with a healthcheck — the smoke polls these for readiness # (infra/wait-healthy.sh). One-shot init jobs (oz-init, nrc-init, flowable-init) # are not polled; they only need to have run. See docs/runbooks/gitea-actions-gotchas.md. -WAIT_SVCS := openzaak nrc-web acl bff domain event-subscriber projection-api self-service openbaar behandel beheer objecttypen +WAIT_SVCS := openzaak nrc-web acl bff domain event-subscriber projection-api self-service openbaar behandel beheer objecttypen objecten # Config files (OpenZaak data.yaml, Keycloak realms, Flowable BPMN) are streamed # into external named volumes via `docker cp` (infra/seed-config.sh) instead of # bind-mounted, because bind mounts don't reach sibling containers on the @@ -18,7 +18,7 @@ WAIT_SVCS := openzaak nrc-web acl bff domain event-subscriber projection-api se # volumes are `external`, so compose won't remove them — CFG_VOLS lists them for # explicit teardown. See docs/runbooks/gitea-actions-gotchas.md. SEED := bash infra/seed-config.sh -CFG_VOLS := rr-oz-config rr-nrc-config rr-kc-realms rr-fl-bpmn rr-objecttypen-config +CFG_VOLS := rr-oz-config rr-nrc-config rr-kc-realms rr-fl-bpmn rr-objecttypen-config rr-objecten-config # Local-only stack: same services but config is bind-mounted (no seed step), so a # plain `docker compose -f infra/docker-compose.local.yml up` works on any local # engine. This is the no-make / Windows-friendly path. See that file's header. @@ -43,7 +43,7 @@ export DOCKER_HOST := unix://$(PODMAN_SOCK) endif endif -.PHONY: ci lint build unit mutation frontend integration verify verify-up verify-acl verify-nrc verify-projection verify-bff verify-domain verify-observability verify-tracing verify-metrics verify-objecttypen verify-notifications smoke up down local verify-local local-down changelog openzaak-up openzaak-smoke openzaak-seed openzaak-down stack-up stack-smoke stack-down keycloak-up keycloak-smoke keycloak-down flowable-up flowable-smoke flowable-down help +.PHONY: ci lint build unit mutation frontend integration verify verify-up verify-acl verify-nrc verify-projection verify-bff verify-domain verify-observability verify-tracing verify-metrics verify-objecttypen verify-objecten verify-notifications smoke up down local verify-local local-down changelog openzaak-up openzaak-smoke openzaak-seed openzaak-down stack-up stack-smoke stack-down keycloak-up keycloak-smoke keycloak-down flowable-up flowable-smoke flowable-down help ## ci: run the full pipeline — lint, build, unit, mutation, frontend, verify (mirrors Gitea Actions) ## `verify` is the live-stack stage (full stack up once → ACL + notification checks). @@ -94,14 +94,14 @@ mutation: # podman-compose, and needing no `--wait` flag or host port access. The one-shots # (oz-init, flowable-init) aren't polled; they just need to have run. smoke: - $(SEED) oz nrc kc fl objecttypen + $(SEED) oz nrc kc fl objecttypen objecten docker compose -f $(COMPOSE) up -d --build bash -c 'WAIT_TIMEOUT=420 bash infra/wait-healthy.sh $(WAIT_SVCS); rc=$$?; docker compose -f $(COMPOSE) down --volumes; docker volume rm -f $(CFG_VOLS) >/dev/null 2>&1; exit $$rc' ## up: seed config volumes and start the full stack (use instead of bare ## `docker compose up`, which can't self-seed the external config volumes) up: - $(SEED) oz nrc kc fl objecttypen + $(SEED) oz nrc kc fl objecttypen objecten docker compose -f $(COMPOSE) up -d --build ## down: stop and remove the local stack (incl. the external config volumes) @@ -139,7 +139,7 @@ changelog: ## verify-up: bring the FULL stack up and wait for health (CI verify-stack step 1; ## subsumes the old compose-smoke health gate — the DoD "up reaches green" check). verify-up: - $(SEED) oz nrc kc fl objecttypen + $(SEED) oz nrc kc fl objecttypen objecten docker compose -f $(COMPOSE) up -d --build WAIT_TIMEOUT=420 bash infra/wait-healthy.sh $(WAIT_SVCS) @@ -191,11 +191,16 @@ verify-metrics: verify-objecttypen: bash infra/run-objecttypen-check.sh +## verify-objecten: assert the Objecten API is up + its static token authenticates and it +## trusts the Objecttypen API (S-18b), against the already-running stack. +verify-objecten: + bash infra/run-objecten-check.sh + ## verify: local mirror of the CI verify-stack job — full stack up once, all checks, ## tear down (always). For fast single-concern local iteration use `integration` ## (oz-only) or `verify-notifications` (oz+nrc) instead. verify: - $(SEED) oz nrc kc fl objecttypen + $(SEED) oz nrc kc fl objecttypen objecten docker compose -f $(COMPOSE) up -d --build @bash -c 'set -e; rc=0; \ WAIT_TIMEOUT=420 bash infra/wait-healthy.sh $(WAIT_SVCS) \ diff --git a/docs/demo-script.md b/docs/demo-script.md index ce3d1f5..363a905 100644 --- a/docs/demo-script.md +++ b/docs/demo-script.md @@ -5,6 +5,38 @@ copy-pasteable walkthrough against a local `make up` stack. --- +## S-18b — Objecten API up in compose, wired to Objecttypen (#140) + +**Outcome:** the upstream Maykin **Objecten API** runs in the stack — own **PostGIS** DB + redis, +config seeded like the other CG modules (`objecten-init` runs `setup_configuration` from the +`rr-objecten-config` volume: migrate + provision a dev **static API token** + register the +**Objecttypen API** (S-18a) as a trusted service), a health-checked `objecten` web on host `:8021`. +An object can now reference its objecttype; the ACL writes register records here on approval (S-19). + +```bash +make up +# 1. The API is up; the seeded token authenticates (401 without, 200 with): +curl -s -o /dev/null -w "%{http_code}\n" http://localhost:8021/api/v2/objects # 401 +curl -s -o /dev/null -w "%{http_code}\n" -H "Authorization: Token 1234567890abcdef1234567890abcdef12345678" \ + http://localhost:8021/api/v2/objects # 200 +# +# 2. It trusts Objecttypen — the seeded zgw_consumers service points at the Objecttypen API: +docker exec infra-objecten-1 python src/manage.py shell -c \ + "from zgw_consumers.models import Service; print(*[(s.slug,s.api_root) for s in Service.objects.all()])" +# → ('objecttypen', 'http://objecttypen:8000/api/v2/') +# +# 3. Automated (a CI verify-stack step): asserts unauth 401 + token 200, against the running stack. +make verify-objecten # → OK — no-auth 401, token 200 +``` + +**The path:** verbatim upstream image (`maykinmedia/objects-api`, pinned 3.4.0) + the same seed +pattern as S-18a — `infra/seed-config.sh objecten` streams `data.yaml` into an external config +volume, `objecten-init` (RUN_SETUP_CONFIG) applies it. Its `zgw_consumers` step registers Objecttypen +(`api_type: orc`, api-key auth with the S-18a dev token). The RegisterRecord objecttype (S-18c) and +the ACL write path (S-19) build on this. + +--- + ## S-18a — Objecttypen API up in compose (#139) **Outcome:** the upstream Maykin **Objecttypen API** runs in the stack — own Postgres + redis, config diff --git a/infra/docker-compose.local.yml b/infra/docker-compose.local.yml index fd1d1c1..a03cacd 100644 --- a/infra/docker-compose.local.yml +++ b/infra/docker-compose.local.yml @@ -621,12 +621,76 @@ services: condition: service_completed_successfully networks: [cg] + # ── Objecten API (S-18b) — bind-mounted config (local variant) ───────────── + objecten-db: + image: docker.io/postgis/postgis:17-3.5 + environment: + POSTGRES_USER: objects + POSTGRES_PASSWORD: objects + POSTGRES_DB: objects + volumes: + - objecten-db:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U objects"] + interval: 5s + timeout: 3s + retries: 10 + networks: [cg] + + objecten-redis: + image: docker.io/library/redis:7 + networks: [cg] + + objecten-init: + image: docker.io/maykinmedia/objects-api:${OBJECTS_TAG:-3.4.0} + environment: &objecten-env-local + DJANGO_SETTINGS_MODULE: objects.conf.docker + SECRET_KEY: ${OBJECTS_SECRET_KEY:-dev-only-not-for-production} + DB_HOST: objecten-db + DB_NAME: objects + DB_USER: objects + DB_PASSWORD: objects + ALLOWED_HOSTS: "*" + CACHE_DEFAULT: objecten-redis:6379/0 + CACHE_AXES: objecten-redis:6379/0 + DISABLE_2FA: "true" + OTEL_SDK_DISABLED: "true" + RUN_SETUP_CONFIG: "true" + command: /setup_configuration.sh + volumes: + - ./objecten/setup_configuration:/app/setup_configuration:ro,z + depends_on: + objecten-db: + condition: service_healthy + objecten-redis: + condition: service_started + objecttypen: + condition: service_healthy + networks: [cg] + + objecten: + image: docker.io/maykinmedia/objects-api:${OBJECTS_TAG:-3.4.0} + environment: *objecten-env-local + healthcheck: + test: ["CMD", "python", "-c", "import requests,sys; sys.exit(0 if requests.head('http://localhost:8000/admin/').status_code in (200,302) else 1)"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + ports: + - "8021:8000" + depends_on: + objecten-init: + condition: service_completed_successfully + networks: [cg] + volumes: oz-db: nrc-db: flowable-db: projection-db: objecttypen-db: + objecten-db: # Carries the seed-generated acl.env (server-assigned zaaktype URLs) from local-seed to the ACL. seed-env: diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml index 468e57c..2d0b2a0 100644 --- a/infra/docker-compose.yml +++ b/infra/docker-compose.yml @@ -635,6 +635,76 @@ services: condition: service_completed_successfully networks: [cg] + # ── Objecten API (S-18b) — upstream Maykin image, verbatim ───────────────── + # The authoritative object store. Same shape as Objecttypen (own DB + redis, an `-init` that runs + # setup_configuration from the external config volume, a health-checked web). Two differences: the + # DB is PostGIS (objects carry geometry), and setup_configuration registers the Objecttypen API + # (S-18a) as a trusted service so an object can reference its objecttype. + objecten-db: + image: docker.io/postgis/postgis:17-3.5 + environment: + POSTGRES_USER: objects + POSTGRES_PASSWORD: objects + POSTGRES_DB: objects + volumes: + - objecten-db:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U objects"] + interval: 5s + timeout: 3s + retries: 10 + networks: [cg] + + objecten-redis: + image: docker.io/library/redis:7 + networks: [cg] + + objecten-init: + image: docker.io/maykinmedia/objects-api:${OBJECTS_TAG:-3.4.0} + environment: &objecten-env + DJANGO_SETTINGS_MODULE: objects.conf.docker + SECRET_KEY: ${OBJECTS_SECRET_KEY:-dev-only-not-for-production} + DB_HOST: objecten-db + DB_NAME: objects + DB_USER: objects + DB_PASSWORD: objects + ALLOWED_HOSTS: "*" + CACHE_DEFAULT: objecten-redis:6379/0 + CACHE_AXES: objecten-redis:6379/0 + DISABLE_2FA: "true" + OTEL_SDK_DISABLED: "true" + RUN_SETUP_CONFIG: "true" + command: /setup_configuration.sh + # data.yaml is streamed into this external volume by infra/seed-config.sh before start. + volumes: + - objecten-config:/app/setup_configuration:ro + depends_on: + objecten-db: + condition: service_healthy + objecten-redis: + condition: service_started + # Objecten's setup_configuration registers the Objecttypen service; that service only needs to + # exist as config, but wait for Objecttypen to be up so the register is meaningful end to end. + objecttypen: + condition: service_healthy + networks: [cg] + + objecten: + image: docker.io/maykinmedia/objects-api:${OBJECTS_TAG:-3.4.0} + environment: *objecten-env + healthcheck: + test: ["CMD", "python", "-c", "import requests,sys; sys.exit(0 if requests.head('http://localhost:8000/admin/').status_code in (200,302) else 1)"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + ports: + - "8021:8000" + depends_on: + objecten-init: + condition: service_completed_successfully + networks: [cg] + # ── Observability backplane (S-16a, ADR-0023) ────────────────────────────── # Grafana-native stack: Tempo ingests OTLP traces (the .NET services export # straight to it — no collector hop, S-16b), Prometheus scrapes service @@ -685,6 +755,7 @@ volumes: flowable-db: projection-db: objecttypen-db: + objecten-db: # Config volumes — created and populated out-of-band by infra/seed-config.sh # (docker cp), because bind mounts don't reach sibling containers on the CI # runner. `external` keeps the names deterministic; the seed step manages them. @@ -703,6 +774,9 @@ volumes: objecttypen-config: external: true name: rr-objecttypen-config + objecten-config: + external: true + name: rr-objecten-config networks: cg: diff --git a/infra/objecten-check.py b/infra/objecten-check.py new file mode 100644 index 0000000..3d5d1bc --- /dev/null +++ b/infra/objecten-check.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""S-18b (#140): prove the Objecten API is up and its static token authenticates. + +Assert an unauthenticated call to /api/v2/objects is 401 and an authenticated one (the seeded dev +token) is 200 — i.e. the service migrated, booted, and setup_configuration provisioned the token +and the Objecttypen service it trusts. Stdlib only so it runs in a bare python:3-slim container on +the compose network. +""" +import os +import sys +import time +import urllib.error +import urllib.request + +BASE = os.environ["OBJECTEN"] # http://:8000 +TOKEN = os.environ["OBJECTEN_TOKEN"] +TIMEOUT = int(os.environ.get("OBJECTEN_TIMEOUT", "60")) + + +def status(url, token=None): + req = urllib.request.Request(url) + if token: + req.add_header("Authorization", f"Token {token}") + try: + with urllib.request.urlopen(req, timeout=10) as r: + return r.status + except urllib.error.HTTPError as e: + return e.code + except Exception: + return 0 + + +def main(): + url = f"{BASE}/api/v2/objects" + deadline = time.time() + TIMEOUT + while time.time() < deadline: + unauth = status(url) + authed = status(url, TOKEN) + if unauth == 401 and authed == 200: + print(f"OK — {url}: no-auth {unauth}, token {authed}") + return 0 + time.sleep(3) + print(f"FAIL — {url}: expected no-auth 401 + token 200, got {status(url)} / {status(url, TOKEN)}", + file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/infra/objecten/setup_configuration/data.yaml b/infra/objecten/setup_configuration/data.yaml new file mode 100644 index 0000000..1a5b2a3 --- /dev/null +++ b/infra/objecten/setup_configuration/data.yaml @@ -0,0 +1,31 @@ +# Objecten API setup_configuration (S-18b). Streamed into the external rr-objecten-config volume by +# infra/seed-config.sh and applied by objecten-init (RUN_SETUP_CONFIG). Declarative + idempotent. +# +# Two things: (1) register the Objecttypen API (S-18a) as a trusted service so an object can +# reference its objecttype — authenticating with the dev static token Objecttypen provisioned; and +# (2) a dev static token so peers (the ACL, S-19) can write objects here. Dev-only, not for prod. + +# (1) Trust the Objecttypen API. `orc` = overige RESTful component (how zgw_consumers classifies the +# Objecttypen API). The RegisterRecord objecttype (S-18c) will reference an objecttype under this +# service by uuid. +zgw_consumers_config_enable: true +zgw_consumers: + services: + - identifier: objecttypen + label: Objecttypen API + api_type: orc + api_root: http://objecttypen:8000/api/v2/ + auth_type: api_key + header_key: Authorization + header_value: Token 0123456789abcdef0123456789abcdef01234567 + +# (2) Static API token peers use to write/read objects. +tokenauth_config_enable: true +tokenauth: + items: + - identifier: register-referentie + token: 1234567890abcdef1234567890abcdef12345678 + contact_person: Register Referentie + email: admin@localhost + organization: Respellion + is_superuser: true diff --git a/infra/run-objecten-check.sh b/infra/run-objecten-check.sh new file mode 100755 index 0000000..dbfc1bc --- /dev/null +++ b/infra/run-objecten-check.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# +# S-18b (#140): assert the Objecten API is healthy + its static token authenticates, against an +# ALREADY-RUNNING stack. Runs the check in a python:3-slim container on the stack network (the +# service is reached by container IP; the runner can't reach published ports — gitea-actions-gotchas.md +# §5/§6). Does NOT manage the stack lifecycle. +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# The dev token provisioned by infra/objecten/setup_configuration/data.yaml. +TOKEN="${OBJECTEN_TOKEN:-1234567890abcdef1234567890abcdef12345678}" + +ot="$(docker ps -q --filter 'name=objecten' --filter 'health=healthy' | head -1)" +[ -n "$ot" ] || ot="$(docker ps -q --filter 'name=[-_]objecten[-_]' | head -1)" +[ -n "$ot" ] || { echo "ERROR: no running objecten container — bring the stack up first" >&2; exit 1; } +net="$(docker inspect -f '{{range $k,$_ := .NetworkSettings.Networks}}{{$k}}{{"\n"}}{{end}}' "$ot" | head -1)" +ip="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$ot")" +echo ">> network=$net objecten=$ip" + +cid="$(docker create --network "$net" \ + -e "OBJECTEN=http://$ip:8000" -e "OBJECTEN_TOKEN=$TOKEN" \ + -e "OBJECTEN_TIMEOUT=${OBJECTEN_TIMEOUT:-60}" \ + python:3-slim python /objecten-check.py)" +docker cp "$here/objecten-check.py" "$cid:/objecten-check.py" >/dev/null +rc=0; docker start -a "$cid" || rc=$? +docker rm -f "$cid" >/dev/null +exit $rc diff --git a/infra/seed-config.sh b/infra/seed-config.sh index 6ee5907..6895d6c 100755 --- a/infra/seed-config.sh +++ b/infra/seed-config.sh @@ -13,7 +13,7 @@ # subcommand. Fixed-name `external` volumes keep the names deterministic across # both runtimes. See docs/runbooks/gitea-actions-gotchas.md. # -# Usage: seed-config.sh [ ...] where key ∈ { oz, kc, fl } +# Usage: seed-config.sh [ ...] where key ∈ { oz, nrc, kc, fl, objecttypen, objecten } set -euo pipefail here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -33,7 +33,7 @@ populate() { # volume source(file or dir/.) echo " seeded $vol" } -[ "$#" -gt 0 ] || { echo "usage: seed-config.sh ..." >&2; exit 2; } +[ "$#" -gt 0 ] || { echo "usage: seed-config.sh ..." >&2; exit 2; } # The registratie process (BPMN) and its diploma-eligibility DMN are deployed as SEPARATE Flowable # deployments — the process engine and the DMN engine each own theirs (S-13, ADR-0016). flowable-rest @@ -50,6 +50,7 @@ for key in "$@"; do nrc) populate rr-nrc-config "$here/opennotificaties/setup_configuration/." ;; kc) populate rr-kc-realms "$here/keycloak/realms/." ;; objecttypen) populate rr-objecttypen-config "$here/objecttypen/setup_configuration/." ;; + objecten) populate rr-objecten-config "$here/objecten/setup_configuration/." ;; fl) d="$(mktemp -d)"; stage_flowable_workflows "$d"; populate rr-fl-bpmn "$d/." ;; *) echo "unknown seed key: $key" >&2; exit 2 ;; esac