diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 2790001..84fbaea 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -201,6 +201,9 @@ jobs: - name: Observability backplane (Grafana + Tempo + Prometheus datasources) id: obs run: OBS_TIMEOUT=180 make verify-observability + - name: Objecttypen API up + token authenticates + id: objecttypen + run: OBJECTTYPEN_TIMEOUT=120 make verify-objecttypen - name: ACL ↔ OpenZaak integration tests id: acl run: make verify-acl @@ -233,6 +236,7 @@ jobs: env: UP: ${{ steps.up.outcome }} OBS: ${{ steps.obs.outcome }} + OBJECTTYPEN: ${{ steps.objecttypen.outcome }} ACL: ${{ steps.acl.outcome }} NRC: ${{ steps.nrc.outcome }} PROJECTION: ${{ steps.projection.outcome }} @@ -251,6 +255,7 @@ jobs: echo "| ----- | :----: |" echo "| Bring up + health | $(icon "$UP") |" echo "| Observability backplane | $(icon "$OBS") |" + echo "| Objecttypen API + token | $(icon "$OBJECTTYPEN") |" echo "| ACL ↔ OpenZaak | $(icon "$ACL") |" echo "| OpenZaak → NRC | $(icon "$NRC") |" echo "| NRC → Event Subscriber → projection | $(icon "$PROJECTION") |" @@ -270,7 +275,7 @@ jobs: # Log dump must precede teardown (which removes the containers). - name: Dump container logs on failure if: failure() - run: docker compose -f infra/docker-compose.yml logs --no-color --tail=100 oz-init openzaak nrc-init nrc-web nrc-celery nrc-beat flowable-db flowable-rest flowable-init keycloak acl bff domain projection-db event-subscriber projection-api self-service openbaar behandel beheer tempo prometheus grafana 2>&1 || true + run: docker compose -f infra/docker-compose.yml logs --no-color --tail=100 oz-init openzaak nrc-init nrc-web nrc-celery nrc-beat flowable-db flowable-rest flowable-init keycloak acl bff domain projection-db event-subscriber projection-api self-service openbaar behandel beheer objecttypen-db objecttypen-redis objecttypen-init objecttypen tempo prometheus grafana 2>&1 || true - name: Tear down if: always() run: make down diff --git a/Makefile b/Makefile index 69a5ade..0a5dd05 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ COMPOSE := infra/docker-compose.yml # Long-running services with a healthcheck — the smoke polls these for readiness # (infra/wait-healthy.sh). One-shot init jobs (oz-init, nrc-init, flowable-init) # are not polled; they only need to have run. See docs/runbooks/gitea-actions-gotchas.md. -WAIT_SVCS := openzaak nrc-web acl bff domain event-subscriber projection-api self-service openbaar behandel beheer +WAIT_SVCS := openzaak nrc-web acl bff domain event-subscriber projection-api self-service openbaar behandel beheer objecttypen # Config files (OpenZaak data.yaml, Keycloak realms, Flowable BPMN) are streamed # into external named volumes via `docker cp` (infra/seed-config.sh) instead of # bind-mounted, because bind mounts don't reach sibling containers on the @@ -18,7 +18,7 @@ WAIT_SVCS := openzaak nrc-web acl bff domain event-subscriber projection-api se # volumes are `external`, so compose won't remove them — CFG_VOLS lists them for # explicit teardown. See docs/runbooks/gitea-actions-gotchas.md. SEED := bash infra/seed-config.sh -CFG_VOLS := rr-oz-config rr-nrc-config rr-kc-realms rr-fl-bpmn +CFG_VOLS := rr-oz-config rr-nrc-config rr-kc-realms rr-fl-bpmn rr-objecttypen-config # Local-only stack: same services but config is bind-mounted (no seed step), so a # plain `docker compose -f infra/docker-compose.local.yml up` works on any local # engine. This is the no-make / Windows-friendly path. See that file's header. @@ -43,7 +43,7 @@ export DOCKER_HOST := unix://$(PODMAN_SOCK) endif endif -.PHONY: ci lint build unit mutation frontend integration verify verify-up verify-acl verify-nrc verify-projection verify-bff verify-domain verify-observability verify-tracing verify-metrics verify-notifications smoke up down local verify-local local-down changelog openzaak-up openzaak-smoke openzaak-seed openzaak-down stack-up stack-smoke stack-down keycloak-up keycloak-smoke keycloak-down flowable-up flowable-smoke flowable-down help +.PHONY: ci lint build unit mutation frontend integration verify verify-up verify-acl verify-nrc verify-projection verify-bff verify-domain verify-observability verify-tracing verify-metrics verify-objecttypen verify-notifications smoke up down local verify-local local-down changelog openzaak-up openzaak-smoke openzaak-seed openzaak-down stack-up stack-smoke stack-down keycloak-up keycloak-smoke keycloak-down flowable-up flowable-smoke flowable-down help ## ci: run the full pipeline — lint, build, unit, mutation, frontend, verify (mirrors Gitea Actions) ## `verify` is the live-stack stage (full stack up once → ACL + notification checks). @@ -94,14 +94,14 @@ mutation: # podman-compose, and needing no `--wait` flag or host port access. The one-shots # (oz-init, flowable-init) aren't polled; they just need to have run. smoke: - $(SEED) oz nrc kc fl + $(SEED) oz nrc kc fl objecttypen docker compose -f $(COMPOSE) up -d --build bash -c 'WAIT_TIMEOUT=420 bash infra/wait-healthy.sh $(WAIT_SVCS); rc=$$?; docker compose -f $(COMPOSE) down --volumes; docker volume rm -f $(CFG_VOLS) >/dev/null 2>&1; exit $$rc' ## up: seed config volumes and start the full stack (use instead of bare ## `docker compose up`, which can't self-seed the external config volumes) up: - $(SEED) oz nrc kc fl + $(SEED) oz nrc kc fl objecttypen docker compose -f $(COMPOSE) up -d --build ## down: stop and remove the local stack (incl. the external config volumes) @@ -139,7 +139,7 @@ changelog: ## verify-up: bring the FULL stack up and wait for health (CI verify-stack step 1; ## subsumes the old compose-smoke health gate — the DoD "up reaches green" check). verify-up: - $(SEED) oz nrc kc fl + $(SEED) oz nrc kc fl objecttypen docker compose -f $(COMPOSE) up -d --build WAIT_TIMEOUT=420 bash infra/wait-healthy.sh $(WAIT_SVCS) @@ -186,11 +186,16 @@ verify-tracing: verify-metrics: bash infra/run-metrics-check.sh +## verify-objecttypen: assert the Objecttypen API is up + its static token authenticates +## (S-18a), against the already-running stack. +verify-objecttypen: + bash infra/run-objecttypen-check.sh + ## verify: local mirror of the CI verify-stack job — full stack up once, all checks, ## tear down (always). For fast single-concern local iteration use `integration` ## (oz-only) or `verify-notifications` (oz+nrc) instead. verify: - $(SEED) oz nrc kc fl + $(SEED) oz nrc kc fl objecttypen docker compose -f $(COMPOSE) up -d --build @bash -c 'set -e; rc=0; \ WAIT_TIMEOUT=420 bash infra/wait-healthy.sh $(WAIT_SVCS) \ diff --git a/infra/docker-compose.local.yml b/infra/docker-compose.local.yml index 0bb7aad..fd1d1c1 100644 --- a/infra/docker-compose.local.yml +++ b/infra/docker-compose.local.yml @@ -560,11 +560,73 @@ services: condition: service_started networks: [cg] + # ── Objecttypen API (S-18a) — bind-mounted config (local variant) ────────── + objecttypen-db: + image: docker.io/library/postgres:17-alpine + environment: + POSTGRES_USER: objecttypes + POSTGRES_PASSWORD: objecttypes + POSTGRES_DB: objecttypes + volumes: + - objecttypen-db:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U objecttypes"] + interval: 5s + timeout: 3s + retries: 10 + networks: [cg] + + objecttypen-redis: + image: docker.io/library/redis:7 + networks: [cg] + + objecttypen-init: + image: docker.io/maykinmedia/objecttypes-api:${OBJECTTYPES_TAG:-3.4.2} + environment: &objecttypen-env-local + DJANGO_SETTINGS_MODULE: objecttypes.conf.docker + SECRET_KEY: ${OBJECTTYPES_SECRET_KEY:-dev-only-not-for-production} + DB_HOST: objecttypen-db + DB_NAME: objecttypes + DB_USER: objecttypes + DB_PASSWORD: objecttypes + ALLOWED_HOSTS: "*" + CACHE_DEFAULT: objecttypen-redis:6379/0 + CACHE_AXES: objecttypen-redis:6379/0 + DISABLE_2FA: "true" + OTEL_SDK_DISABLED: "true" + RUN_SETUP_CONFIG: "true" + command: /setup_configuration.sh + volumes: + - ./objecttypen/setup_configuration:/app/setup_configuration:ro,z + depends_on: + objecttypen-db: + condition: service_healthy + objecttypen-redis: + condition: service_started + networks: [cg] + + objecttypen: + image: docker.io/maykinmedia/objecttypes-api:${OBJECTTYPES_TAG:-3.4.2} + environment: *objecttypen-env-local + healthcheck: + test: ["CMD", "python", "-c", "import requests,sys; sys.exit(0 if requests.head('http://localhost:8000/admin/').status_code in (200,302) else 1)"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + ports: + - "8020:8000" + depends_on: + objecttypen-init: + condition: service_completed_successfully + networks: [cg] + volumes: oz-db: nrc-db: flowable-db: projection-db: + objecttypen-db: # Carries the seed-generated acl.env (server-assigned zaaktype URLs) from local-seed to the ACL. seed-env: diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml index 0004266..468e57c 100644 --- a/infra/docker-compose.yml +++ b/infra/docker-compose.yml @@ -569,6 +569,72 @@ services: condition: service_started networks: [cg] + # ── Objecttypen API (S-18a) — upstream Maykin image, verbatim ────────────── + # The register's objecttype catalogue. Same shape as the other CG modules: own DB + redis, an + # `-init` that runs setup_configuration (RUN_SETUP_CONFIG → migrate + provision a static API token) + # from the external config volume streamed in by infra/seed-config.sh, and a health-checked web + # service that depends on init completing. + objecttypen-db: + image: docker.io/library/postgres:17-alpine + environment: + POSTGRES_USER: objecttypes + POSTGRES_PASSWORD: objecttypes + POSTGRES_DB: objecttypes + volumes: + - objecttypen-db:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U objecttypes"] + interval: 5s + timeout: 3s + retries: 10 + networks: [cg] + + objecttypen-redis: + image: docker.io/library/redis:7 + networks: [cg] + + objecttypen-init: + image: docker.io/maykinmedia/objecttypes-api:${OBJECTTYPES_TAG:-3.4.2} + environment: &objecttypen-env + DJANGO_SETTINGS_MODULE: objecttypes.conf.docker + SECRET_KEY: ${OBJECTTYPES_SECRET_KEY:-dev-only-not-for-production} + DB_HOST: objecttypen-db + DB_NAME: objecttypes + DB_USER: objecttypes + DB_PASSWORD: objecttypes + ALLOWED_HOSTS: "*" + CACHE_DEFAULT: objecttypen-redis:6379/0 + CACHE_AXES: objecttypen-redis:6379/0 + DISABLE_2FA: "true" + OTEL_SDK_DISABLED: "true" + RUN_SETUP_CONFIG: "true" + command: /setup_configuration.sh + # data.yaml is streamed into this external volume by infra/seed-config.sh before start. + volumes: + - objecttypen-config:/app/setup_configuration:ro + depends_on: + objecttypen-db: + condition: service_healthy + objecttypen-redis: + condition: service_started + networks: [cg] + + objecttypen: + image: docker.io/maykinmedia/objecttypes-api:${OBJECTTYPES_TAG:-3.4.2} + environment: *objecttypen-env + healthcheck: + test: ["CMD", "python", "-c", "import requests,sys; sys.exit(0 if requests.head('http://localhost:8000/admin/').status_code in (200,302) else 1)"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + ports: + - "8020:8000" + depends_on: + objecttypen-init: + condition: service_completed_successfully + networks: [cg] + # ── Observability backplane (S-16a, ADR-0023) ────────────────────────────── # Grafana-native stack: Tempo ingests OTLP traces (the .NET services export # straight to it — no collector hop, S-16b), Prometheus scrapes service @@ -618,6 +684,7 @@ volumes: nrc-db: flowable-db: projection-db: + objecttypen-db: # Config volumes — created and populated out-of-band by infra/seed-config.sh # (docker cp), because bind mounts don't reach sibling containers on the CI # runner. `external` keeps the names deterministic; the seed step manages them. @@ -633,6 +700,9 @@ volumes: fl-bpmn: external: true name: rr-fl-bpmn + objecttypen-config: + external: true + name: rr-objecttypen-config networks: cg: diff --git a/infra/objecttypen-check.py b/infra/objecttypen-check.py new file mode 100644 index 0000000..f1f30f2 --- /dev/null +++ b/infra/objecttypen-check.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""S-18a (#139): prove the Objecttypen API is up and its static token authenticates. + +Assert an unauthenticated call to /api/v2/objecttypes is 401 and an authenticated one (the seeded +dev token) is 200 — i.e. the service migrated, booted, and setup_configuration provisioned the token. +Stdlib only so it runs in a bare python:3-slim container on the compose network. +""" +import os +import sys +import time +import urllib.error +import urllib.request + +BASE = os.environ["OBJECTTYPEN"] # http://:8000 +TOKEN = os.environ["OBJECTTYPEN_TOKEN"] +TIMEOUT = int(os.environ.get("OBJECTTYPEN_TIMEOUT", "60")) + + +def status(url, token=None): + req = urllib.request.Request(url) + if token: + req.add_header("Authorization", f"Token {token}") + try: + with urllib.request.urlopen(req, timeout=10) as r: + return r.status + except urllib.error.HTTPError as e: + return e.code + except Exception: + return 0 + + +def main(): + url = f"{BASE}/api/v2/objecttypes" + deadline = time.time() + TIMEOUT + while time.time() < deadline: + unauth = status(url) + authed = status(url, TOKEN) + if unauth == 401 and authed == 200: + print(f"OK — {url}: no-auth {unauth}, token {authed}") + return 0 + time.sleep(3) + print(f"FAIL — {url}: expected no-auth 401 + token 200, got {status(url)} / {status(url, TOKEN)}", + file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/infra/objecttypen/setup_configuration/data.yaml b/infra/objecttypen/setup_configuration/data.yaml new file mode 100644 index 0000000..b6391cc --- /dev/null +++ b/infra/objecttypen/setup_configuration/data.yaml @@ -0,0 +1,11 @@ +# Objecttypen API setup_configuration (S-18a). Streamed into the external rr-objecttypen-config +# volume by infra/seed-config.sh and applied by objecttypen-init (RUN_SETUP_CONFIG). Declarative + +# idempotent. Dev-only static token so peers (Objecten S-18b, the ACL) can authenticate. +tokenauth_config_enable: true +tokenauth: + items: + - identifier: register-referentie + token: 0123456789abcdef0123456789abcdef01234567 + contact_person: Register Referentie + email: admin@localhost + organization: Respellion diff --git a/infra/run-objecttypen-check.sh b/infra/run-objecttypen-check.sh new file mode 100644 index 0000000..fec8b2c --- /dev/null +++ b/infra/run-objecttypen-check.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# +# S-18a (#139): assert the Objecttypen API is healthy + its static token authenticates, against an +# ALREADY-RUNNING stack. Runs the check in a python:3-slim container on the stack network (the +# service is reached by container IP; the runner can't reach published ports — gitea-actions-gotchas.md +# §5/§6). Does NOT manage the stack lifecycle. +set -euo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# The dev token provisioned by infra/objecttypen/setup_configuration/data.yaml. +TOKEN="${OBJECTTYPEN_TOKEN:-0123456789abcdef0123456789abcdef01234567}" + +ot="$(docker ps -q --filter 'name=objecttypen' --filter 'health=healthy' | head -1)" +[ -n "$ot" ] || ot="$(docker ps -q --filter 'name=[-_]objecttypen[-_]' | head -1)" +[ -n "$ot" ] || { echo "ERROR: no running objecttypen container — bring the stack up first" >&2; exit 1; } +net="$(docker inspect -f '{{range $k,$_ := .NetworkSettings.Networks}}{{$k}}{{"\n"}}{{end}}' "$ot" | head -1)" +ip="$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$ot")" +echo ">> network=$net objecttypen=$ip" + +cid="$(docker create --network "$net" \ + -e "OBJECTTYPEN=http://$ip:8000" -e "OBJECTTYPEN_TOKEN=$TOKEN" \ + -e "OBJECTTYPEN_TIMEOUT=${OBJECTTYPEN_TIMEOUT:-60}" \ + python:3-slim python /objecttypen-check.py)" +docker cp "$here/objecttypen-check.py" "$cid:/objecttypen-check.py" >/dev/null +rc=0; docker start -a "$cid" || rc=$? +docker rm -f "$cid" >/dev/null +exit $rc diff --git a/infra/seed-config.sh b/infra/seed-config.sh index 52cf94d..6ee5907 100755 --- a/infra/seed-config.sh +++ b/infra/seed-config.sh @@ -33,7 +33,7 @@ populate() { # volume source(file or dir/.) echo " seeded $vol" } -[ "$#" -gt 0 ] || { echo "usage: seed-config.sh ..." >&2; exit 2; } +[ "$#" -gt 0 ] || { echo "usage: seed-config.sh ..." >&2; exit 2; } # The registratie process (BPMN) and its diploma-eligibility DMN are deployed as SEPARATE Flowable # deployments — the process engine and the DMN engine each own theirs (S-13, ADR-0016). flowable-rest @@ -49,6 +49,7 @@ for key in "$@"; do oz) populate rr-oz-config "$here/openzaak/setup_configuration/." ;; nrc) populate rr-nrc-config "$here/opennotificaties/setup_configuration/." ;; kc) populate rr-kc-realms "$here/keycloak/realms/." ;; + objecttypen) populate rr-objecttypen-config "$here/objecttypen/setup_configuration/." ;; fl) d="$(mktemp -d)"; stage_flowable_workflows "$d"; populate rr-fl-bpmn "$d/." ;; *) echo "unknown seed key: $key" >&2; exit 2 ;; esac