Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 not
|
6cfcc4cf83 | ||
|
not
|
9d7e8e5b65 | ||
|
not
|
17f1f2f809 | ||
|
not
|
1dd8bd4e1b | ||
|
not
|
d6b3f9764f | ||
|
not
|
8b206a005f |
@@ -41,6 +41,27 @@ jobs:
|
||||
nuget-${{ runner.os }}-
|
||||
- run: make lint
|
||||
|
||||
# The Helm chart's only automated gate: it renders and schema-checks the whole
|
||||
# stack, and checks it still describes the same stack as the compose file
|
||||
# (ADR-0033). No cluster involved — see docs/runbooks/kubernetes-talos.md.
|
||||
k8s:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: https://github.com/actions/checkout@v4
|
||||
# helm as its pinned static binary rather than a marketplace action: one URL,
|
||||
# the same one the Talos runbook §0 gives a developer, and no third-party
|
||||
# action to vet (CLAUDE.md §13). The drift check also needs `docker compose`,
|
||||
# which the runner already has (see docs/runbooks/ci.md).
|
||||
- name: Install helm
|
||||
run: |
|
||||
mkdir -p "$HOME/.local/bin"
|
||||
curl -sSL https://get.helm.sh/helm-v3.16.4-linux-amd64.tar.gz \
|
||||
| tar xz -O linux-amd64/helm > "$HOME/.local/bin/helm"
|
||||
chmod +x "$HOME/.local/bin/helm"
|
||||
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
||||
- run: make k8s-lint
|
||||
- run: make k8s-drift
|
||||
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
|
||||
@@ -0,0 +1,110 @@
|
||||
name: Deploy to Talos
|
||||
|
||||
# A merge to main ships the stack to the Talos cluster on the lab server
|
||||
# (docs/runbooks/kubernetes-talos.md §9). PR CI is the merge gate, so main is
|
||||
# green by construction — this workflow only deploys.
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# Queue deploys, never cancel one: a helm upgrade killed half-way leaves the
|
||||
# release in `pending-upgrade` and the next run has to be unwedged by hand.
|
||||
concurrency:
|
||||
group: deploy-talos
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
deploy:
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
# The Talos VM as seen from the Fedora host (libvirt guest IP), and the
|
||||
# address a browser uses to reach the cluster. `localhost` is deliberate:
|
||||
# the portals' PKCE needs a secure context, so they are reached over
|
||||
# `kubectl port-forward` — runbook §5. Override with repo variables.
|
||||
TALOS_VM_IP: ${{ vars.TALOS_VM_IP }}
|
||||
TALOS_HOST: ${{ vars.TALOS_HOST }}
|
||||
steps:
|
||||
- uses: https://github.com/actions/checkout@v4
|
||||
|
||||
# Pinned static binaries, the same URLs the Talos runbook §0 gives a
|
||||
# developer and the same helm the `k8s` CI job uses — no action to vet.
|
||||
- name: Install kubectl, helm and crane
|
||||
run: |
|
||||
set -euo pipefail
|
||||
bin="$HOME/.local/bin"; mkdir -p "$bin"
|
||||
curl -sSLo "$bin/kubectl" https://dl.k8s.io/release/v1.37.0/bin/linux/amd64/kubectl
|
||||
curl -sSL https://get.helm.sh/helm-v3.16.4-linux-amd64.tar.gz | tar xz -O linux-amd64/helm > "$bin/helm"
|
||||
curl -sSL https://github.com/google/go-containerregistry/releases/download/v0.20.2/go-containerregistry_Linux_x86_64.tar.gz | tar xz -O crane > "$bin/crane"
|
||||
chmod +x "$bin"/{kubectl,helm,crane}
|
||||
echo "$bin" >> "$GITHUB_PATH"
|
||||
|
||||
# The cluster's API and its registry are only reachable through the Fedora
|
||||
# host, so forward both to the runner. 30141 is the openbaar portal, for
|
||||
# the smoke at the end.
|
||||
- name: Tunnel the Talos API + registry through the Fedora host
|
||||
env:
|
||||
SSH_KEY: ${{ secrets.TALOS_SSH_KEY }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
: "${TALOS_VM_IP:=192.168.122.173}"
|
||||
umask 077
|
||||
printf '%s\n' "$SSH_KEY" > ~/.ssh_talos
|
||||
ssh -i ~/.ssh_talos -o StrictHostKeyChecking=no -o IdentitiesOnly=yes \
|
||||
-o ExitOnForwardFailure=yes -p 6667 -f -N \
|
||||
-L 6443:$TALOS_VM_IP:6443 \
|
||||
-L 30500:$TALOS_VM_IP:30500 \
|
||||
-L 30141:$TALOS_VM_IP:30141 \
|
||||
user@labs.respellion.tech
|
||||
|
||||
# The kubeconfig's server must be https://127.0.0.1:6443 — Talos puts
|
||||
# 127.0.0.1 in the apiserver cert SANs, so TLS verification still holds
|
||||
# through the tunnel.
|
||||
- name: Write the kubeconfig
|
||||
env:
|
||||
KUBECONFIG_B64: ${{ secrets.TALOS_KUBECONFIG }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
base64 -d <<< "$KUBECONFIG_B64" > "$RUNNER_TEMP/kubeconfig"
|
||||
echo "KUBECONFIG=$RUNNER_TEMP/kubeconfig" >> "$GITHUB_ENV"
|
||||
kubectl --kubeconfig "$RUNNER_TEMP/kubeconfig" get nodes
|
||||
|
||||
# Idempotent; also makes a first deploy onto a bare cluster work. The
|
||||
# registry's storage is an emptyDir, so a replaced pod loses the images —
|
||||
# which the push in the next step puts back anyway.
|
||||
- name: Ensure the in-cluster registry
|
||||
run: make k8s-registry
|
||||
|
||||
# Push through the tunnel (localhost), pull from the node's own NodePort
|
||||
# (the address in the Talos registry-mirror patch) — same registry, two
|
||||
# names, so the two `make` calls get different K8S_REGISTRY values.
|
||||
- name: Build and push the images
|
||||
run: make k8s-images K8S_REGISTRY=localhost:30500
|
||||
|
||||
# k8s-reseed = seed configmaps + helm upgrade + re-run the bootstrap jobs.
|
||||
# The jobs are idempotent, and deleting them first is what keeps a changed
|
||||
# Job template from wedging the upgrade (`cannot patch … with kind Job`).
|
||||
- name: Deploy the chart
|
||||
run: make k8s-reseed TALOS_HOST=${TALOS_HOST:-localhost} K8S_REGISTRY=${TALOS_VM_IP:-192.168.122.173}:30500
|
||||
|
||||
# `dev` is a mutable tag and helm sees an unchanged pod template, so the
|
||||
# new images only land on a restart (pullPolicy is already Always).
|
||||
- name: Roll the services onto the new images
|
||||
run: |
|
||||
set -euo pipefail
|
||||
svcs="acl domain bff event-subscriber projection-api self-service openbaar behandel beheer"
|
||||
kubectl -n big rollout restart deploy $svcs
|
||||
kubectl -n big rollout status --timeout=300s deploy $svcs
|
||||
|
||||
# Proves portal → Caddy → BFF → projection end to end. An empty register is
|
||||
# a pass; a 502 or a timeout is not.
|
||||
- name: Smoke the public register
|
||||
run: curl -fsS --retry 10 --retry-delay 6 --retry-all-errors http://localhost:30141/openbaar/register
|
||||
|
||||
- name: Pods on failure
|
||||
if: failure()
|
||||
run: kubectl -n big get pods,jobs || true
|
||||
@@ -43,7 +43,7 @@ export DOCKER_HOST := unix://$(PODMAN_SOCK)
|
||||
endif
|
||||
endif
|
||||
|
||||
.PHONY: ci lint build unit mutation frontend integration verify verify-up verify-acl verify-nrc verify-projection verify-bff verify-domain verify-observability verify-tracing verify-metrics verify-objecttypen verify-objecten verify-registerrecord verify-objecten-notifications verify-notifications smoke up down local verify-local local-down changelog openzaak-up openzaak-smoke openzaak-seed openzaak-down stack-up stack-smoke stack-down keycloak-up keycloak-smoke keycloak-down flowable-up flowable-smoke flowable-down help
|
||||
.PHONY: ci lint build unit mutation frontend integration verify verify-up verify-acl verify-nrc verify-projection verify-bff verify-domain verify-observability verify-tracing verify-metrics verify-objecttypen verify-objecten verify-registerrecord verify-objecten-notifications verify-notifications smoke up down local verify-local local-down changelog openzaak-up openzaak-smoke openzaak-seed openzaak-down stack-up stack-smoke stack-down keycloak-up keycloak-smoke keycloak-down flowable-up flowable-smoke flowable-down k8s-lint k8s-drift k8s-registry k8s-images k8s-seed k8s-up k8s-reseed k8s-portals k8s-down k8s-purge help
|
||||
|
||||
## ci: run the full pipeline — lint, build, unit, mutation, frontend, verify (mirrors Gitea Actions)
|
||||
## `verify` is the live-stack stage (full stack up once → ACL + notification checks).
|
||||
@@ -64,6 +64,9 @@ frontend:
|
||||
## lint: verify formatting (no changes)
|
||||
lint:
|
||||
dotnet format $(SLN) --verify-no-changes
|
||||
# Only pages in mkdocs.yml's nav are published, and mkdocs keeps a build green
|
||||
# when one is missing — so the nav is checked here rather than not at all.
|
||||
python3 infra/check-docs-nav.py
|
||||
|
||||
## build: release build
|
||||
build:
|
||||
@@ -71,8 +74,12 @@ build:
|
||||
|
||||
## unit: run unit tests (excludes the container-backed Integration lane)
|
||||
# TRX per test project (→ TestResults/) feeds the CI per-service summary (#136); harmless locally.
|
||||
# The CI reporting scripts are stdlib Python with their own assert-based self-checks (#161) — they
|
||||
# ride this lane so a broken job summary is caught by CI rather than by the next red pipeline.
|
||||
unit:
|
||||
dotnet test $(SLN) -c Release --filter "Category!=Integration" --logger trx --results-directory TestResults
|
||||
python3 infra/test_playwright_summary.py
|
||||
python3 infra/test_portal_caddyfiles.py
|
||||
|
||||
## mutation: run the Stryker.NET ratchet on each service with branching logic (fails below baseline)
|
||||
# Stryker is pinned as a local dotnet tool (.config/dotnet-tools.json); `tool restore`
|
||||
@@ -326,6 +333,98 @@ flowable-down:
|
||||
docker compose -f $(FL_COMPOSE) down --volumes
|
||||
-docker volume rm -f rr-fl-bpmn
|
||||
|
||||
# ── Kubernetes (single-node Talos) ─────────────────────────────────────────────
|
||||
# The Helm chart in infra/helm/big-reference is a port of infra/docker-compose.yml
|
||||
# (ADR-0033). Full walkthrough: docs/runbooks/kubernetes-talos.md.
|
||||
# TALOS_HOST the address the BROWSER uses — pins Keycloak's issuer and the portals'
|
||||
# OIDC authority. Use `localhost` with `make k8s-portals`: the OIDC
|
||||
# library needs crypto.subtle, which browsers only expose on a secure
|
||||
# context (https, or localhost) — see docs/runbooks/kubernetes-talos.md §5
|
||||
# K8S_REGISTRY the registry both sides use for this repo's images (see k8s-registry)
|
||||
K8S_NS ?= big
|
||||
K8S_CHART := infra/helm/big-reference
|
||||
K8S_REGISTRY ?=
|
||||
TALOS_HOST ?=
|
||||
# The images built from this repo — compose service name == image name == chart workload.
|
||||
K8S_IMAGES := acl domain bff event-subscriber projection-api self-service openbaar behandel beheer
|
||||
|
||||
## k8s-lint: render + schema-check the Helm chart (no cluster needed)
|
||||
k8s-lint:
|
||||
helm lint $(K8S_CHART)
|
||||
helm template big $(K8S_CHART) -n $(K8S_NS) --set images.registry=registry.invalid:5000 >/dev/null
|
||||
|
||||
## k8s-drift: fail if compose and the Helm chart describe different stacks
|
||||
# Compose is CI-canonical (ADR-0033) and the chart is a transcription of it; this
|
||||
# compares what each one deploys — workload names and resolved images. Needs
|
||||
# `docker compose` and `helm`, no cluster.
|
||||
k8s-drift:
|
||||
python3 infra/helm/check-drift.py
|
||||
|
||||
## k8s-registry: deploy the in-cluster image registry (NodePort 30500)
|
||||
k8s-registry:
|
||||
kubectl apply -f infra/helm/registry.yaml
|
||||
kubectl -n registry rollout status deploy/registry --timeout=180s
|
||||
|
||||
## k8s-images: build this repo's images (via compose) and push them to $(K8S_REGISTRY)
|
||||
# `docker save | crane push` rather than `docker push`: the registry speaks plain
|
||||
# HTTP, which the Docker daemon refuses without a root-level insecure-registries
|
||||
# entry, while crane just takes --insecure. Install: see docs/runbooks/kubernetes-talos.md.
|
||||
k8s-images:
|
||||
@command -v crane >/dev/null || { echo "crane not found — see docs/runbooks/kubernetes-talos.md §0" >&2; exit 2; }
|
||||
@test -n "$(K8S_REGISTRY)" || { echo "set K8S_REGISTRY=<registry host:port>" >&2; exit 2; }
|
||||
docker compose -f $(COMPOSE) build $(K8S_IMAGES)
|
||||
@tar=$$(mktemp -t rr-img-XXXX.tar); \
|
||||
for i in $(K8S_IMAGES); do \
|
||||
docker save register-referentie/$$i:dev -o $$tar; \
|
||||
crane push --insecure $$tar $(K8S_REGISTRY)/register-referentie/$$i:dev; \
|
||||
done; rm -f $$tar
|
||||
|
||||
## k8s-seed: create the ConfigMaps the chart mounts (upstream config + bootstrap scripts)
|
||||
k8s-seed:
|
||||
bash infra/helm/seed-configmaps.sh $(K8S_NS)
|
||||
|
||||
## k8s-up: seed the config and install/upgrade the release
|
||||
k8s-up: k8s-seed
|
||||
@test -n "$(TALOS_HOST)" || { echo "set TALOS_HOST=<node ip>" >&2; exit 2; }
|
||||
@test -n "$(K8S_REGISTRY)" || { echo "set K8S_REGISTRY=<registry the node can pull from>" >&2; exit 2; }
|
||||
helm upgrade --install big $(K8S_CHART) -n $(K8S_NS) --create-namespace \
|
||||
--set host=$(TALOS_HOST) --set images.registry=$(K8S_REGISTRY) $(K8S_SET)
|
||||
kubectl -n $(K8S_NS) get pods
|
||||
|
||||
## k8s-reseed: re-run the bootstrap jobs (after a database was wiped, or after
|
||||
## changing a Job in the chart — Job pod templates are immutable, so a plain
|
||||
## `helm upgrade` is rejected)
|
||||
k8s-reseed:
|
||||
kubectl -n $(K8S_NS) delete job -l app.kubernetes.io/component=init --ignore-not-found
|
||||
$(MAKE) k8s-up
|
||||
# The projection's schema is created on service start (Projection.ReadModel migrates in a
|
||||
# hosted service), so a wiped database also needs these two restarted — otherwise they keep
|
||||
# writing to a schema-less DB and fail with `relation "processed_notifications" does not exist`.
|
||||
kubectl -n $(K8S_NS) rollout restart deploy/event-subscriber deploy/projection-api
|
||||
kubectl -n $(K8S_NS) rollout status deploy/event-subscriber deploy/projection-api --timeout=180s
|
||||
|
||||
## k8s-portals: forward the browser-facing services to localhost (Ctrl-C stops them all)
|
||||
# The portals' OIDC flow needs a *secure context* for crypto.subtle (PKCE), and browsers
|
||||
# only grant that to https or localhost — a NodePort on the VM's IP is neither. Forwarding
|
||||
# to localhost on the same port numbers keeps Keycloak's pinned issuer valid. Deploy with
|
||||
# TALOS_HOST=localhost for this to line up.
|
||||
k8s-portals:
|
||||
@echo "self-service http://localhost:30140 · openbaar :30141 · behandel :30142 · beheer :30143 · keycloak :30180"
|
||||
@trap 'kill 0' INT TERM; \
|
||||
for f in self-service:30140:80 openbaar:30141:80 behandel:30142:80 beheer:30143:80 keycloak:30180:8080; do \
|
||||
svc=$${f%%:*}; rest=$${f#*:}; lport=$${rest%%:*}; rport=$${rest#*:}; \
|
||||
kubectl -n $(K8S_NS) port-forward --address 127.0.0.1 svc/$$svc $$lport:$$rport >/dev/null & \
|
||||
done; wait
|
||||
|
||||
## k8s-down: uninstall the release (database PVCs are kept)
|
||||
k8s-down:
|
||||
helm uninstall big -n $(K8S_NS)
|
||||
|
||||
## k8s-purge: uninstall AND drop the namespace, including the database volumes
|
||||
k8s-purge:
|
||||
-helm uninstall big -n $(K8S_NS)
|
||||
kubectl delete namespace $(K8S_NS) --ignore-not-found
|
||||
|
||||
## help: list available targets
|
||||
help:
|
||||
@grep -E '^## ' $(MAKEFILE_LIST) | sed 's/^## //'
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
:80 {
|
||||
# Same-origin API: behandelaars authenticate against the medewerker realm; the BFF validates it
|
||||
# for /behandel/* (S-12c).
|
||||
# `handle` blocks are mutually exclusive and matched most-specific-first, so the
|
||||
# SPA fallback below can never swallow an API call — unlike a bare `try_files`,
|
||||
# which Caddy sorts *before* reverse_proxy and would rewrite it to /index.html.
|
||||
#
|
||||
# No `resolver` stanza is needed: Caddy dials the upstream per
|
||||
# request through the system resolver, so it starts before the BFF is up, picks up
|
||||
# its restarts, and honours the DNS search domains in /etc/resolv.conf — which is
|
||||
# what lets the bare `bff` name resolve on Kubernetes as well as under compose.
|
||||
handle /behandel/* {
|
||||
reverse_proxy bff:8080
|
||||
}
|
||||
|
||||
# The Angular app. Client-side routing: an unknown path serves index.html.
|
||||
handle {
|
||||
root * /usr/share/caddy
|
||||
try_files {path} /index.html
|
||||
file_server
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
# Multi-stage build for the behandel portal (Angular → nginx).
|
||||
# Multi-stage build for the behandel portal (Angular → Caddy).
|
||||
# Build context is the repo root (the app needs the pnpm workspace + libs). See infra/docker-compose.yml.
|
||||
FROM node:24-slim AS build
|
||||
WORKDIR /src
|
||||
@@ -13,15 +13,12 @@ COPY apps/behandel apps/behandel
|
||||
COPY libs libs
|
||||
RUN pnpm nx build behandel
|
||||
|
||||
FROM nginx:1.27-alpine AS runtime
|
||||
COPY apps/behandel/nginx.conf /etc/nginx/conf.d/default.conf
|
||||
COPY --from=build /src/dist/apps/behandel/browser /usr/share/nginx/html
|
||||
FROM caddy:2-alpine AS runtime
|
||||
COPY apps/behandel/Caddyfile /etc/caddy/Caddyfile
|
||||
COPY --from=build /src/dist/apps/behandel/browser /usr/share/caddy
|
||||
# Compose-time OIDC config: the browser (Playwright, on the compose network) reaches Keycloak by
|
||||
# service name, so the token issuer matches the BFF's medewerker authority (host-consistent, ADR-0013).
|
||||
RUN printf '{ "authority": "http://keycloak:8080/realms/medewerker" }\n' > /usr/share/nginx/html/config.json
|
||||
# Make the reverse-proxy resolver engine-portable (Docker 127.0.0.11 vs podman aardvark); runs from
|
||||
# the nginx image's /docker-entrypoint.d before nginx starts.
|
||||
COPY apps/portal-nginx-resolver.sh /docker-entrypoint.d/40-resolver.sh
|
||||
RUN chmod +x /docker-entrypoint.d/40-resolver.sh
|
||||
# Kubernetes mounts a ConfigMap over this file with the node address instead (ADR-0033).
|
||||
RUN printf '{ "authority": "http://keycloak:8080/realms/medewerker" }\n' > /usr/share/caddy/config.json
|
||||
|
||||
EXPOSE 80
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
server {
|
||||
listen 80;
|
||||
server_name _;
|
||||
root /usr/share/nginx/html;
|
||||
index index.html;
|
||||
|
||||
# Resolve the BFF via Docker's embedded DNS at request time (variable proxy_pass), so nginx starts
|
||||
# even before the BFF is up and picks up restarts — instead of failing to load the config.
|
||||
resolver 127.0.0.11 ipv6=off valid=30s;
|
||||
|
||||
# Same-origin API: proxy the behandel endpoint group to the bff service. The api-client uses
|
||||
# relative URLs, so the browser calls this origin and nginx forwards to the BFF — no CORS, and the
|
||||
# medewerker token (same-origin) is attached by the app's interceptor (ADR-0013).
|
||||
location /behandel/ {
|
||||
set $bff http://bff:8080;
|
||||
proxy_pass $bff;
|
||||
proxy_set_header Host $host;
|
||||
}
|
||||
|
||||
# SPA fallback — Angular client-side routing.
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
}
|
||||
@@ -12,7 +12,7 @@ export interface RuntimeConfig {
|
||||
|
||||
/**
|
||||
* Route prefixes whose requests carry the medewerker token. These MUST match the **relative** URLs
|
||||
* the api-client actually calls (same-origin via the nginx proxy) — the interceptor matches on
|
||||
* the api-client actually calls (same-origin via the Caddy proxy) — the interceptor matches on
|
||||
* `req.url`, which stays relative, so an absolute origin would never match and the token would go
|
||||
* unattached. Only `/behandel/` is secured; the app calls no other endpoint group.
|
||||
*/
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
:80 {
|
||||
# Same-origin API: beheerders use the same medewerker realm as behandel (S-15a).
|
||||
# `handle` blocks are mutually exclusive and matched most-specific-first, so the
|
||||
# SPA fallback below can never swallow an API call — unlike a bare `try_files`,
|
||||
# which Caddy sorts *before* reverse_proxy and would rewrite it to /index.html.
|
||||
#
|
||||
# No `resolver` stanza is needed: Caddy dials the upstream per
|
||||
# request through the system resolver, so it starts before the BFF is up, picks up
|
||||
# its restarts, and honours the DNS search domains in /etc/resolv.conf — which is
|
||||
# what lets the bare `bff` name resolve on Kubernetes as well as under compose.
|
||||
handle /beheer/* {
|
||||
reverse_proxy bff:8080
|
||||
}
|
||||
|
||||
# The Angular app. Client-side routing: an unknown path serves index.html.
|
||||
handle {
|
||||
root * /usr/share/caddy
|
||||
try_files {path} /index.html
|
||||
file_server
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
# Multi-stage build for the beheer portal (Angular → nginx).
|
||||
# Multi-stage build for the beheer portal (Angular → Caddy).
|
||||
# Build context is the repo root (the app needs the pnpm workspace + libs). See infra/docker-compose.yml.
|
||||
FROM node:24-slim AS build
|
||||
WORKDIR /src
|
||||
@@ -13,15 +13,12 @@ COPY apps/beheer apps/beheer
|
||||
COPY libs libs
|
||||
RUN pnpm nx build beheer
|
||||
|
||||
FROM nginx:1.27-alpine AS runtime
|
||||
COPY apps/beheer/nginx.conf /etc/nginx/conf.d/default.conf
|
||||
COPY --from=build /src/dist/apps/beheer/browser /usr/share/nginx/html
|
||||
FROM caddy:2-alpine AS runtime
|
||||
COPY apps/beheer/Caddyfile /etc/caddy/Caddyfile
|
||||
COPY --from=build /src/dist/apps/beheer/browser /usr/share/caddy
|
||||
# Compose-time OIDC config: the browser (Playwright, on the compose network) reaches Keycloak by
|
||||
# service name, so the token issuer matches the BFF's medewerker authority (host-consistent, ADR-0013).
|
||||
RUN printf '{ "authority": "http://keycloak:8080/realms/medewerker" }\n' > /usr/share/nginx/html/config.json
|
||||
# Make the reverse-proxy resolver engine-portable (Docker 127.0.0.11 vs podman aardvark); runs from
|
||||
# the nginx image's /docker-entrypoint.d before nginx starts.
|
||||
COPY apps/portal-nginx-resolver.sh /docker-entrypoint.d/40-resolver.sh
|
||||
RUN chmod +x /docker-entrypoint.d/40-resolver.sh
|
||||
# Kubernetes mounts a ConfigMap over this file with the node address instead (ADR-0033).
|
||||
RUN printf '{ "authority": "http://keycloak:8080/realms/medewerker" }\n' > /usr/share/caddy/config.json
|
||||
|
||||
EXPOSE 80
|
||||
|
||||
@@ -1,24 +0,0 @@
|
||||
server {
|
||||
listen 80;
|
||||
server_name _;
|
||||
root /usr/share/nginx/html;
|
||||
index index.html;
|
||||
|
||||
# Resolve the BFF via Docker's embedded DNS at request time (variable proxy_pass), so nginx starts
|
||||
# even before the BFF is up and picks up restarts — instead of failing to load the config.
|
||||
resolver 127.0.0.11 ipv6=off valid=30s;
|
||||
|
||||
# Same-origin API: proxy the beheer endpoint group to the bff service. The api-client uses
|
||||
# relative URLs, so the browser calls this origin and nginx forwards to the BFF — no CORS, and the
|
||||
# medewerker token (same-origin) is attached by the app's interceptor (ADR-0013).
|
||||
location /beheer/ {
|
||||
set $bff http://bff:8080;
|
||||
proxy_pass $bff;
|
||||
proxy_set_header Host $host;
|
||||
}
|
||||
|
||||
# SPA fallback — Angular client-side routing.
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
}
|
||||
@@ -12,7 +12,7 @@ export interface RuntimeConfig {
|
||||
|
||||
/**
|
||||
* Route prefixes whose requests carry the medewerker token. These MUST match the **relative** URLs
|
||||
* the api-client actually calls (same-origin via the nginx proxy) — the interceptor matches on
|
||||
* the api-client actually calls (same-origin via the Caddy proxy) — the interceptor matches on
|
||||
* `req.url`, which stays relative, so an absolute origin would never match and the token would go
|
||||
* unattached. Only `/beheer/` is secured; the app calls no other endpoint group.
|
||||
*/
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
:80 {
|
||||
# Same-origin API: the public register is anonymous, but still reads through the BFF (S-09).
|
||||
# `handle` blocks are mutually exclusive and matched most-specific-first, so the
|
||||
# SPA fallback below can never swallow an API call — unlike a bare `try_files`,
|
||||
# which Caddy sorts *before* reverse_proxy and would rewrite it to /index.html.
|
||||
#
|
||||
# No `resolver` stanza is needed: Caddy dials the upstream per
|
||||
# request through the system resolver, so it starts before the BFF is up, picks up
|
||||
# its restarts, and honours the DNS search domains in /etc/resolv.conf — which is
|
||||
# what lets the bare `bff` name resolve on Kubernetes as well as under compose.
|
||||
handle /openbaar/* {
|
||||
reverse_proxy bff:8080
|
||||
}
|
||||
|
||||
# The Angular app. Client-side routing: an unknown path serves index.html.
|
||||
handle {
|
||||
root * /usr/share/caddy
|
||||
try_files {path} /index.html
|
||||
file_server
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
# Multi-stage build for the openbaar portal (Angular → nginx).
|
||||
# Multi-stage build for the openbaar portal (Angular → Caddy).
|
||||
# Build context is the repo root (the app needs the pnpm workspace + libs). See infra/docker-compose.yml.
|
||||
FROM node:24-slim AS build
|
||||
WORKDIR /src
|
||||
@@ -13,13 +13,9 @@ COPY apps/openbaar apps/openbaar
|
||||
COPY libs libs
|
||||
RUN pnpm nx build openbaar
|
||||
|
||||
FROM nginx:1.27-alpine AS runtime
|
||||
COPY apps/openbaar/nginx.conf /etc/nginx/conf.d/default.conf
|
||||
COPY --from=build /src/dist/apps/openbaar/browser /usr/share/nginx/html
|
||||
FROM caddy:2-alpine AS runtime
|
||||
COPY apps/openbaar/Caddyfile /etc/caddy/Caddyfile
|
||||
COPY --from=build /src/dist/apps/openbaar/browser /usr/share/caddy
|
||||
# No runtime config: the openbaar register is anonymous (no OIDC authority to inject).
|
||||
# Make the reverse-proxy resolver engine-portable (Docker 127.0.0.11 vs podman aardvark); runs from
|
||||
# the nginx image's /docker-entrypoint.d before nginx starts.
|
||||
COPY apps/portal-nginx-resolver.sh /docker-entrypoint.d/40-resolver.sh
|
||||
RUN chmod +x /docker-entrypoint.d/40-resolver.sh
|
||||
|
||||
EXPOSE 80
|
||||
|
||||
@@ -1,23 +0,0 @@
|
||||
server {
|
||||
listen 80;
|
||||
server_name _;
|
||||
root /usr/share/nginx/html;
|
||||
index index.html;
|
||||
|
||||
# Resolve the BFF via Docker's embedded DNS at request time (variable proxy_pass), so nginx starts
|
||||
# even before the BFF is up and picks up restarts — instead of failing to load the config.
|
||||
resolver 127.0.0.11 ipv6=off valid=30s;
|
||||
|
||||
# Same-origin API: proxy the anonymous openbaar endpoint group to the bff service. The api-client
|
||||
# uses relative URLs, so the browser calls this origin and nginx forwards to the BFF — no CORS.
|
||||
location /openbaar/ {
|
||||
set $bff http://bff:8080;
|
||||
proxy_pass $bff;
|
||||
proxy_set_header Host $host;
|
||||
}
|
||||
|
||||
# SPA fallback — Angular client-side routing.
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
}
|
||||
@@ -8,7 +8,7 @@ import { appRoutes } from './app.routes';
|
||||
|
||||
/**
|
||||
* The openbaar register is a public, anonymous read: no DigiD, no auth interceptor. The app is served
|
||||
* same-origin as the BFF (nginx proxies /openbaar), so the api-client's relative calls stay same-origin.
|
||||
* same-origin as the BFF (Caddy proxies /openbaar), so the api-client's relative calls stay same-origin.
|
||||
*/
|
||||
export const appConfig: ApplicationConfig = {
|
||||
providers: [
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
#!/bin/sh
|
||||
# Point nginx's reverse-proxy `resolver` at THIS container's real DNS server.
|
||||
#
|
||||
# The portal nginx configs use a variable proxy_pass, which needs a `resolver` so the BFF hostname is
|
||||
# resolved at request time (nginx can start before the BFF is up). The config hardcodes Docker's
|
||||
# embedded DNS (127.0.0.11) — correct on Docker/Docker Desktop, but rootless podman uses a
|
||||
# network-specific address (aardvark, e.g. 10.89.0.1), so proxied calls 502 there. Read the actual
|
||||
# nameserver from /etc/resolv.conf and substitute it, so the reverse proxy works on any engine.
|
||||
#
|
||||
# Runs from the nginx image's /docker-entrypoint.d/ before nginx starts. On Docker the nameserver IS
|
||||
# 127.0.0.11, so the substitution is a no-op. Guarded (no `set -e`) so it's safe whether the nginx
|
||||
# entrypoint executes or sources it.
|
||||
ns="$(awk '/^nameserver/{print $2; exit}' /etc/resolv.conf 2>/dev/null)"
|
||||
if [ -n "$ns" ] && [ "$ns" != "127.0.0.11" ]; then
|
||||
sed -i "s/resolver 127\.0\.0\.11/resolver $ns/" /etc/nginx/conf.d/default.conf 2>/dev/null || true
|
||||
echo "portal-nginx-resolver: set resolver to $ns"
|
||||
fi
|
||||
@@ -0,0 +1,26 @@
|
||||
:80 {
|
||||
# Same-origin API: the api-client uses relative URLs, so the browser calls this origin and Caddy
|
||||
# forwards to the BFF — no CORS, and the DigiD token is attached by the app interceptor
|
||||
# (S-08d/ADR-0010).
|
||||
# `handle` blocks are mutually exclusive and matched most-specific-first, so the
|
||||
# SPA fallback below can never swallow an API call — unlike a bare `try_files`,
|
||||
# which Caddy sorts *before* reverse_proxy and would rewrite it to /index.html.
|
||||
#
|
||||
# No `resolver` stanza is needed: Caddy dials the upstream per
|
||||
# request through the system resolver, so it starts before the BFF is up, picks up
|
||||
# its restarts, and honours the DNS search domains in /etc/resolv.conf — which is
|
||||
# what lets the bare `bff` name resolve on Kubernetes as well as under compose.
|
||||
handle /self-service/* {
|
||||
reverse_proxy bff:8080
|
||||
}
|
||||
handle /openbaar/* {
|
||||
reverse_proxy bff:8080
|
||||
}
|
||||
|
||||
# The Angular app. Client-side routing: an unknown path serves index.html.
|
||||
handle {
|
||||
root * /usr/share/caddy
|
||||
try_files {path} /index.html
|
||||
file_server
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
# Multi-stage build for the self-service portal (Angular → nginx).
|
||||
# Multi-stage build for the self-service portal (Angular → Caddy).
|
||||
# Build context is the repo root (the app needs the pnpm workspace + libs). See infra/docker-compose.yml.
|
||||
FROM node:24-slim AS build
|
||||
WORKDIR /src
|
||||
@@ -13,15 +13,12 @@ COPY apps/self-service apps/self-service
|
||||
COPY libs libs
|
||||
RUN pnpm nx build self-service
|
||||
|
||||
FROM nginx:1.27-alpine AS runtime
|
||||
COPY apps/self-service/nginx.conf /etc/nginx/conf.d/default.conf
|
||||
COPY --from=build /src/dist/apps/self-service/browser /usr/share/nginx/html
|
||||
FROM caddy:2-alpine AS runtime
|
||||
COPY apps/self-service/Caddyfile /etc/caddy/Caddyfile
|
||||
COPY --from=build /src/dist/apps/self-service/browser /usr/share/caddy
|
||||
# Compose-time OIDC config: the browser (Playwright, on the compose network) reaches Keycloak by
|
||||
# service name, so the token issuer matches the BFF's authority (host-consistent, ADR-0010).
|
||||
RUN printf '{ "authority": "http://keycloak:8080/realms/digid" }\n' > /usr/share/nginx/html/config.json
|
||||
# Make the reverse-proxy resolver engine-portable (Docker 127.0.0.11 vs podman aardvark); runs from
|
||||
# the nginx image's /docker-entrypoint.d before nginx starts.
|
||||
COPY apps/portal-nginx-resolver.sh /docker-entrypoint.d/40-resolver.sh
|
||||
RUN chmod +x /docker-entrypoint.d/40-resolver.sh
|
||||
# Kubernetes mounts a ConfigMap over this file with the node address instead (ADR-0033).
|
||||
RUN printf '{ "authority": "http://keycloak:8080/realms/digid" }\n' > /usr/share/caddy/config.json
|
||||
|
||||
EXPOSE 80
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
server {
|
||||
listen 80;
|
||||
server_name _;
|
||||
root /usr/share/nginx/html;
|
||||
index index.html;
|
||||
|
||||
# Resolve the BFF via Docker's embedded DNS at request time (variable proxy_pass), so nginx starts
|
||||
# even before the BFF is up and picks up restarts — instead of failing to load the config.
|
||||
resolver 127.0.0.11 ipv6=off valid=30s;
|
||||
|
||||
# Same-origin API: proxy the BFF endpoint groups to the bff service. The api-client uses relative
|
||||
# URLs, so the browser calls this origin and nginx forwards to the BFF — no CORS, and the DigiD
|
||||
# token (same-origin) is attached by the app's interceptor (S-08d/ADR-0010).
|
||||
location /self-service/ {
|
||||
set $bff http://bff:8080;
|
||||
proxy_pass $bff;
|
||||
proxy_set_header Host $host;
|
||||
}
|
||||
location /openbaar/ {
|
||||
set $bff http://bff:8080;
|
||||
proxy_pass $bff;
|
||||
proxy_set_header Host $host;
|
||||
}
|
||||
|
||||
# SPA fallback — Angular client-side routing.
|
||||
location / {
|
||||
try_files $uri $uri/ /index.html;
|
||||
}
|
||||
}
|
||||
@@ -15,7 +15,7 @@ export interface RuntimeConfig {
|
||||
|
||||
/**
|
||||
* Route prefixes whose requests carry the DigiD token. These MUST match the **relative** URLs the
|
||||
* api-client actually calls (same-origin via the nginx proxy) — the interceptor matches on `req.url`,
|
||||
* api-client actually calls (same-origin via the Caddy proxy) — the interceptor matches on `req.url`,
|
||||
* which stays relative, so an absolute origin would never match and the token would go unattached.
|
||||
* `/openbaar/` is deliberately excluded: it is the anonymous public register.
|
||||
*/
|
||||
|
||||
@@ -18,7 +18,7 @@ Three forces shape the mechanism:
|
||||
|
||||
- **Nothing notifies anyone.** The trigger lives in Flowable. The domain does not publish
|
||||
task events, and there is no bus between the domain and the BFF.
|
||||
- **The BFF is stateless** and sits behind each portal's nginx.
|
||||
- **The BFF is stateless** and sits behind each portal's reverse proxy.
|
||||
- **This is the repo's first live-updating view**, so the choice sets a precedent.
|
||||
|
||||
## Decision
|
||||
@@ -40,7 +40,7 @@ werkbak is readable at all.
|
||||
Neither buys freshness here, because **nothing notifies the BFF either**:
|
||||
|
||||
- **SSE** (`text/event-stream`) would mean a new streaming endpoint whose handler polls the
|
||||
domain and forwards diffs — the same latency, plus connection lifecycle, nginx
|
||||
domain and forwards diffs — the same latency, plus connection lifecycle, proxy
|
||||
buffering, and auth on a long-lived connection.
|
||||
- **WebSocket/SignalR** adds a dependency (CLAUDE.md §13) and makes the BFF stateful and
|
||||
sticky-session-bound. A genuine push path would *also* need the domain to publish task
|
||||
|
||||
@@ -0,0 +1,162 @@
|
||||
# ADR-0033: Kubernetes deployment is one values-driven Helm chart, not a chart per service
|
||||
|
||||
- **Status:** Accepted
|
||||
- **Date:** 2026-09-04
|
||||
- **Deciders:** Respellion engineering
|
||||
- **Slice:** #25 (S-24) — raised directly as a deployment-target request and matched to
|
||||
that issue afterwards; see the "Process note" at the end
|
||||
|
||||
## Context
|
||||
|
||||
The stack is defined once, in `infra/docker-compose.yml`: 30-odd containers made of six
|
||||
upstream Common Ground modules (OpenZaak, Open Notificaties, Objecten, Objecttypen,
|
||||
Keycloak, Flowable), their databases and workers, five .NET services, four portals, six
|
||||
one-shot bootstrap containers, and an observability backplane (off by default here). Compose is the
|
||||
CI-canonical stack: `make verify` and every `verify-*` script drive it.
|
||||
|
||||
We now also want the stack on Kubernetes — first target a **single-node Talos VM on a
|
||||
laptop**. Four properties of this particular stack shape the answer:
|
||||
|
||||
- **The upstream images are used verbatim** and read their configuration from a mounted
|
||||
directory (`setup_configuration/data.yaml`, Keycloak realm exports, BPMN/DMN). Compose
|
||||
streams those files into external volumes (`infra/seed-config.sh`) because bind mounts
|
||||
don't reach sibling containers on the CI runner. Kubernetes needs the same files as
|
||||
ConfigMaps — from *somewhere*.
|
||||
- **Django's `URLValidator` rejects single-label hosts.** Compose works around it by
|
||||
handing the ACL and the seeds a container *IP* (ADR-0009, ADR-0020, ADR-0029, and the
|
||||
`objecten.local` network alias). In Kubernetes a Service FQDN is already multi-label, so
|
||||
the workaround has a natural replacement — but the hosts have to line up exactly, since
|
||||
Objecten reflects the request Host into the URLs it publishes to NRC.
|
||||
- **The OIDC issuer must be one string** for both the browser and the BFF (ADR-0010).
|
||||
`infra/host-browser.yml` already solved this for a host browser: pin `KC_HOSTNAME`, keep
|
||||
backchannel discovery in-cluster, and mount a `config.json` per portal.
|
||||
- **Nothing here is highly available.** One replica of everything, on one node.
|
||||
|
||||
## Decision
|
||||
|
||||
**One chart — `infra/helm/big-reference` — whose `values.yaml` is a near-literal
|
||||
transcription of the compose file, rendered by three generic templates (Deployment, Job,
|
||||
Service) over a `workloads` map.** Adding a service is a values edit.
|
||||
|
||||
Consequences of that shape, each chosen deliberately:
|
||||
|
||||
- **Config files are not copied into the chart.** `infra/helm/seed-configmaps.sh` creates
|
||||
the ConfigMaps from the files that already live in the repo — the Kubernetes sibling of
|
||||
`infra/seed-config.sh`. The chart therefore needs `make k8s-seed` before `helm install`,
|
||||
which is the same two-step dance compose already has.
|
||||
- **Bootstrap one-shots become Jobs, with no ordering mechanism.** Every one is idempotent
|
||||
(ADR-0020); each waits for the TCP ports it needs via a busybox init container and
|
||||
Kubernetes retries the rest. `make k8s-reseed` re-runs them.
|
||||
- **The four Django services apply their own `setup_configuration`** —
|
||||
`args: [sh, -c, "/setup_configuration.sh && exec /start.sh"]` — instead of getting a
|
||||
separate `*-init` Job like compose. Both of those image scripts run
|
||||
`manage.py migrate`, and compose serialises them with
|
||||
`depends_on: service_completed_successfully`; Kubernetes has no such edge, so a Job and
|
||||
its web pod migrate the same database concurrently and Django dies with
|
||||
*"relation zgw_consumers_service already exists"*. Running the two steps in order inside
|
||||
the one container leaves exactly one migrator per database, and deletes four workloads.
|
||||
- **`args`, never `command`.** Compose's `command:` replaces the image's CMD; Kubernetes'
|
||||
`command:` replaces its ENTRYPOINT. Transcribing one to the other silently broke every
|
||||
upstream image that relies on its entrypoint — postgres ran as root and refused to
|
||||
start, Keycloak tried to exec `start-dev` as a binary. The chart now `fail`s at render
|
||||
time if a workload sets `command`, because the symptom (a crashloop three layers down)
|
||||
is nothing like the cause.
|
||||
- **Published ports are NodePorts.** No ingress controller, no LoadBalancer, no TLS. The
|
||||
four portals are the exception in *use*, not in wiring: PKCE needs `crypto.subtle`, which
|
||||
browsers expose only in a secure context, so a portal has to be reached over `localhost`
|
||||
(`make k8s-portals` forwards them) or eventually over HTTPS. `.Values.host` is therefore
|
||||
"the address the browser uses", not "the node's address" — it pins Keycloak's issuer and
|
||||
each portal's `config.json`, and both must agree with the URL bar (ADR-0010).
|
||||
- **Databases are `emptyDir` by default**, so the stack comes up on a cluster with no CSI
|
||||
driver; setting `persistence.storageClass` switches every database to a PVC.
|
||||
- **Only two hosts become FQDNs** — OpenZaak (for the ACL and the zaaktype seed) and
|
||||
Objecten (for the ACL's register writes), the two that Django validates as URLs.
|
||||
Everything else keeps the short compose service name, because the upstream
|
||||
`setup_configuration` files name those and Objecten matches an objecttype URL against the
|
||||
one it was configured with. The portals used to be a third case — nginx's `resolver` never
|
||||
appends search domains, so the bare `bff` upstream could not resolve on Kubernetes — which
|
||||
ADR-0034 removed by serving them with Caddy, whose resolver honours `/etc/resolv.conf`.
|
||||
- **Compose stays CI-canonical.** The chart is a second deployment target, not a
|
||||
replacement; the acceptance, verify and e2e lanes are unchanged.
|
||||
|
||||
### Alternatives considered
|
||||
|
||||
- **A chart per service, or an umbrella of 30 subcharts.** The conventional layout, and
|
||||
roughly 1,500 lines of near-identical YAML for a stack where 28 of 30 workloads are
|
||||
"one pod, one image, some env". It buys independent versioning we don't want (the stack
|
||||
is demoed as a whole) and costs the eye-diffability against the compose file that keeps
|
||||
the two stacks honest.
|
||||
- **`kompose convert`.** One-shot generation, no ongoing artefact to maintain — but it
|
||||
drops exactly the parts that carry the design (init ordering, the config volumes, the
|
||||
issuer pinning) and produces output nobody owns.
|
||||
- **Bitnami PostgreSQL/Redis subcharts.** Six more dependencies (CLAUDE.md §13) and a
|
||||
second way of expressing the same three-line database.
|
||||
- **ingress-nginx with hostname routing.** Needs a controller, `/etc/hosts` entries and a
|
||||
matching issuer host; NodePorts need none of it and reuse the mechanism
|
||||
`infra/host-browser.yml` already proves.
|
||||
- **A registry on the laptop** (the obvious home for images built there). Talos cannot
|
||||
side-load an image, so a registry is required either way — but reaching one on the host
|
||||
means opening an inbound port on firewalld's `libvirt` zone, which needs root, and
|
||||
pushing to it over plain HTTP means an `insecure-registries` entry in the Docker daemon,
|
||||
which needs root again. `infra/helm/registry.yaml` runs the registry *in* the cluster on
|
||||
a NodePort instead: pushing laptop → node is outbound and unfiltered, the node pulls from
|
||||
its own NodePort, and `docker save | crane push --insecure` needs no daemon
|
||||
configuration. Cost: one more (throwaway, `emptyDir`) workload, and a re-push if its pod
|
||||
is replaced.
|
||||
- **Helm hooks (`pre-install`/`post-install`) for bootstrap ordering.** Hooks run after
|
||||
`--wait`, which would deadlock: OpenZaak's readiness needs the migrations that the hook
|
||||
is supposed to run. Idempotent Jobs plus retries need no such sequencing.
|
||||
|
||||
- ponytail ceiling: single-node assumptions are baked in — one replica per workload,
|
||||
`Recreate` rollouts, ReadWriteOnce volumes, no PodDisruptionBudgets, no resource
|
||||
requests or limits (a laptop VM schedules everything or nothing), plain HTTP.
|
||||
Upgrade path for a real cluster: add requests/limits per workload (the field is already
|
||||
passed through), swap NodePorts for an Ingress with TLS, and give the databases a real
|
||||
StorageClass — none of which changes the workload graph.
|
||||
|
||||
## Consequences
|
||||
|
||||
**Positive**
|
||||
|
||||
- One file to read to see what the cluster runs, and it lines up with the compose file
|
||||
line for line.
|
||||
- The compose IP workarounds disappear: cluster DNS supplies multi-label hosts.
|
||||
- `make k8s-lint` renders and schema-checks the whole stack without a cluster.
|
||||
- The config inputs have exactly one home (the repo) for both stacks — no fork to drift.
|
||||
|
||||
**Negative / costs**
|
||||
|
||||
- A second deployment description to keep in step with compose. `make k8s-drift` (#168)
|
||||
now enforces the part that bites — the workload set and the resolved images, with the
|
||||
four deviations below declared — but not per-workload env, ports or volumes.
|
||||
- `helm install` alone is not enough — the ConfigMaps must be seeded first, and a missing
|
||||
one surfaces as `ContainerCreating`, not as a clear error.
|
||||
- Generic templates mean a values typo can render valid-but-wrong YAML; `k8s-lint` catches
|
||||
schema errors, not intent.
|
||||
- The verify/e2e lanes do not run against the chart, so the Kubernetes path is verified by
|
||||
hand (docs/runbooks/kubernetes-talos.md §5) rather than by CI.
|
||||
- The chart deviates from compose in four places now (args, self-configuring Django pods,
|
||||
FQDN hosts, NodePorts). Each is forced by the platform and commented where it appears,
|
||||
but it is four more things that can drift.
|
||||
|
||||
## Coupling rules touched (CLAUDE.md §8)
|
||||
|
||||
None. The chart deploys the same graph: portals reach only the BFF (§8.3), only the ACL
|
||||
holds ZGW credentials (§8.1), only the Workflow Client talks to Flowable (§8.2), each
|
||||
service keeps its own database (§8.5). No workload gained a peer it didn't have in compose.
|
||||
|
||||
## Verified
|
||||
|
||||
Brought up from scratch on a single-node Talos v1.14.0 VM (6 vCPU / 10 GB, virtio disk)
|
||||
under virt-manager: 29 pods ready and four bootstrap Jobs complete in under three minutes,
|
||||
with zero restarts, using ~4.4 GB of the VM's 10 GB. The smoke test in the runbook's §5
|
||||
walks the whole path — portal proxy → BFF → domain → Flowable → ACL → OpenZaak + Objecten →
|
||||
NRC → event-subscriber → projection → public register — plus a werkbak read with an
|
||||
MFA'd medewerker token. The browser flow itself was driven with Playwright against
|
||||
`http://localhost:30140`: secure context, PKCE, Keycloak form, login, no console errors.
|
||||
|
||||
## Process note
|
||||
|
||||
CLAUDE.md §14 wants the ADR proposal issue opened before the code, and §7 wants a slice
|
||||
issue behind the work. This landed the other way round — chart first, on request. The
|
||||
issue and the CI drift check are the outstanding follow-ups.
|
||||
@@ -0,0 +1,103 @@
|
||||
# ADR-0034: The portals are served by Caddy, not nginx
|
||||
|
||||
- **Status:** Accepted
|
||||
- **Date:** 2026-09-04
|
||||
- **Deciders:** Respellion engineering
|
||||
- **Slice:** _(none yet — raised directly alongside the Kubernetes deployment, ADR-0033)_
|
||||
|
||||
## Context
|
||||
|
||||
Each portal ships as one image that does two jobs: serve the built Angular app, and
|
||||
reverse-proxy *its own* BFF endpoint group so the browser calls a single origin (no CORS,
|
||||
and the DigiD/medewerker token rides along — ADR-0010, ADR-0013). Until now that was nginx
|
||||
with a hand-written `nginx.conf` per app.
|
||||
|
||||
Two workarounds had accumulated around nginx's resolver, both for the same root cause:
|
||||
**nginx resolves a variable `proxy_pass` upstream itself**, using only the `resolver`
|
||||
directive, and never the search domains in `/etc/resolv.conf`.
|
||||
|
||||
1. `resolver 127.0.0.11` (Docker's embedded DNS) is wrong on rootless podman, which uses a
|
||||
network-specific aardvark address — so `apps/portal-nginx-resolver.sh` rewrote the
|
||||
directive at container start by reading the pod's actual nameserver.
|
||||
2. On Kubernetes the bare `bff` name cannot resolve at all without the `svc.cluster.local`
|
||||
search domain, so the same script gained a `BFF_HOST` override that the Helm chart set
|
||||
per portal (ADR-0033).
|
||||
|
||||
Both existed only to tell the proxy how to resolve one hostname.
|
||||
|
||||
## Decision
|
||||
|
||||
**Serve the portals with `caddy:2-alpine` and a small `Caddyfile` per app, replacing the
|
||||
nginx runtime stage, the four `nginx.conf` files, and the resolver workaround.**
|
||||
|
||||
Caddy dials its upstream per request through Go's resolver, which reads
|
||||
`/etc/resolv.conf` — nameserver *and* search domains. So `reverse_proxy bff:8080` resolves
|
||||
correctly under Docker, rootless podman and Kubernetes with no per-engine configuration,
|
||||
and it still starts before the BFF exists and picks up its restarts (the property the
|
||||
variable `proxy_pass` was there to buy). `apps/portal-nginx-resolver.sh`, its unit test and
|
||||
the chart's `BFF_HOST` env are deleted.
|
||||
|
||||
The Caddyfile uses `handle` blocks rather than a bare `try_files`:
|
||||
|
||||
```
|
||||
handle /behandel/* { reverse_proxy bff:8080 }
|
||||
handle { root * /usr/share/caddy; try_files {path} /index.html; file_server }
|
||||
```
|
||||
|
||||
`handle` blocks are mutually exclusive and matched most-specific-first. This matters:
|
||||
Caddy's default directive order puts rewrites (`try_files`) *before* `reverse_proxy`, so a
|
||||
top-level `try_files {path} /index.html` would rewrite every API path to `/index.html`
|
||||
before the proxy ever saw it — the SPA fallback would silently eat the API. The `handle`
|
||||
form makes the routing explicit instead of relying on directive-order trivia.
|
||||
|
||||
`infra/test_portal_caddyfiles.py` (in `make unit`) asserts each portal proxies exactly its
|
||||
own endpoint groups and keeps the SPA fallback. The four files are near-identical, so a
|
||||
copy-paste slip is cheap to make and expensive to find: proxying another portal's group
|
||||
hands a browser an endpoint its token isn't for, and the failure surfaces as a 401 three
|
||||
services away.
|
||||
|
||||
### Alternatives considered
|
||||
|
||||
- **Keep nginx.** Zero migration, and it works — but the resolver workaround stays, and it
|
||||
had already grown a second head for Kubernetes. Both heads are nginx-specific.
|
||||
- **Keep nginx, hard-code the FQDN.** Would need a different config per deployment target
|
||||
(compose vs Kubernetes), which is exactly the fork the chart was written to avoid.
|
||||
- **Drop the proxy and use CORS.** Turns the same-origin design (ADR-0010) inside out:
|
||||
CORS preflights, an explicit origin allowlist in the BFF, and a token attached
|
||||
cross-origin. Not a serving decision — an architectural regression.
|
||||
- **Kubernetes Ingress in front of the portals.** Solves nothing about compose, adds a
|
||||
controller, and the portals would still need something to serve static files.
|
||||
|
||||
- ponytail ceiling: plain HTTP on `:80`, no compression, no cache headers beyond Caddy's
|
||||
defaults, and Caddy's automatic HTTPS deliberately unused (there is no hostname to get a
|
||||
certificate for). Upgrade path: `encode zstd gzip` and a cache policy for immutable
|
||||
Angular bundles; a real hostname makes TLS a one-line `Caddyfile` change, which is the
|
||||
main reason this is worth having in place.
|
||||
|
||||
## Consequences
|
||||
|
||||
**Positive**
|
||||
|
||||
- One resolver behaviour across compose, podman and Kubernetes; a script, a unit test and a
|
||||
chart env var are deleted rather than maintained.
|
||||
- The images gain `curl` for free (the alpine nginx image had only busybox `wget`), which
|
||||
the compose healthchecks can use.
|
||||
- Routing intent is readable: one `handle` block per endpoint group, one for the app.
|
||||
- TLS later is a one-line change instead of a new component.
|
||||
|
||||
**Negative / costs**
|
||||
|
||||
- A new runtime dependency in four images (CLAUDE.md §13): Caddy replaces nginx rather than
|
||||
joining it, so the count is unchanged, but it is a less familiar config language for
|
||||
anyone who has only read nginx configs.
|
||||
- The images grew: 90.6 MB against nginx's 75.7 MB, because `caddy:2-alpine` carries a
|
||||
bigger static binary than nginx's. Measured, not estimated.
|
||||
- Caddy's directive-order rule is a genuine footgun (see above); the `handle` form and the
|
||||
Caddyfile comments exist to keep the next person out of it.
|
||||
- Any operational note that says "the portal's nginx" is now wrong; the ones in `docs/` were
|
||||
updated with this ADR.
|
||||
|
||||
## Coupling rules touched (CLAUDE.md §8)
|
||||
|
||||
None. §8.3 is unchanged and unchanged in kind: the portals still talk only to the BFF, and
|
||||
the proxy is still the thing that makes that same-origin.
|
||||
+1
-1
@@ -389,7 +389,7 @@ make verify-e2e # → login as jan-burger → submit → "ontvangen" co
|
||||
open http://localhost:8140
|
||||
```
|
||||
|
||||
> The portal is served same-origin with the BFF (nginx proxies `/self-service` + `/openbaar`), so no
|
||||
> The portal is served same-origin with the BFF (Caddy proxies `/self-service` + `/openbaar`), so no
|
||||
> CORS; the OIDC authority comes from `/config.json` at runtime. See `docs/frontend-decisions.md`.
|
||||
|
||||
---
|
||||
|
||||
@@ -77,11 +77,13 @@ with the submit form (S-08c, #67); any deviation from NL DS will be recorded her
|
||||
|
||||
## Serving + e2e (S-08d, #68)
|
||||
|
||||
- **Served by nginx, same-origin as the BFF.** The compose `self-service` image serves the built app
|
||||
- **Served by Caddy, same-origin as the BFF.** The compose `self-service` image serves the built app
|
||||
and **reverse-proxies** `/self-service/*` + `/openbaar/*` to the `bff` service. Because the
|
||||
api-client uses **relative URLs**, the browser calls the app's own origin → nginx forwards to the
|
||||
BFF: **no CORS**, and the DigiD token (same-origin) is attached by the interceptor. nginx resolves
|
||||
the BFF at request time (a `resolver` + variable `proxy_pass`) so it starts before the BFF is up.
|
||||
api-client uses **relative URLs**, the browser calls the app's own origin → Caddy forwards to the
|
||||
BFF: **no CORS**, and the DigiD token (same-origin) is attached by the interceptor. Caddy dials
|
||||
the BFF per request through the system resolver, so it starts before the BFF is up, picks up its
|
||||
restarts, and resolves the bare `bff` name on every engine — compose, podman and Kubernetes
|
||||
(ADR-0034; the `Caddyfile` sits next to each app's `Dockerfile`).
|
||||
- **Runtime config.** The app fetches `/config.json` before bootstrap (`main.ts`); `appConfig` is a
|
||||
factory. The dev default (`public/config.json`) points at `localhost:8180`; the Docker image bakes
|
||||
the compose value (`keycloak:8080`). One build, per-environment OIDC authority.
|
||||
@@ -110,7 +112,7 @@ with the submit form (S-08c, #67); any deviation from NL DS will be recorded her
|
||||
`angular-auth-oidc-client`, no interceptor, and no `config.json` — `main.ts` bootstraps `appConfig`
|
||||
directly with just `provideHttpClient` + `provideRouter`. This is the deliberate contrast to
|
||||
self-service and keeps the app trivially cacheable/CDN-able.
|
||||
- **Same-origin via nginx, like self-service.** The compose `openbaar` image serves the built app and
|
||||
- **Same-origin via Caddy, like self-service.** The compose `openbaar` image serves the built app and
|
||||
reverse-proxies `/openbaar` to the BFF; the api-client's relative calls stay same-origin (no CORS).
|
||||
Served on `:8141`, health-checked over IPv4 (`127.0.0.1`), no Keycloak dependency.
|
||||
- **Public-safe by construction.** The portal only ever sees the BFF's `OpenbaarProjection.PublicView`
|
||||
@@ -138,7 +140,7 @@ frontend work is the medewerker realm auth and the werkbak/decide page. Wiring r
|
||||
**BFF remains the security boundary** (`behandelaar` policy, 401/403 on `/behandel/*`, ADR-0013);
|
||||
the frontend role signal is for display/UX, and the werkbak page surfaces a load failure (e.g. a
|
||||
403 for a non-behandelaar) rather than swallowing it.
|
||||
- **Same-origin via nginx, like the other portals.** The compose `behandel` image serves the built
|
||||
- **Same-origin via Caddy, like the other portals.** The compose `behandel` image serves the built
|
||||
app and reverse-proxies `/behandel` to the BFF (relative calls, no CORS). Served on `:8142`,
|
||||
health-checked over IPv4 (`127.0.0.1`), depends on Keycloak for the medewerker realm.
|
||||
- **Werkbak = decide-and-refresh.** `WerkbakPage` loads `GET /behandel/werkbak` on open and renders a
|
||||
|
||||
@@ -14,6 +14,8 @@ should teach.
|
||||
In Dutch; the strategic framing lives in `Respellion/innovation-lab`.
|
||||
- **[Working in Gitea](gitea-workflow.md)** — issues, milestones, branches, PRs.
|
||||
- **[CI runbook](runbooks/ci.md)** — the pipeline and the `make ci` local gate.
|
||||
- **[Kubernetes on Talos](runbooks/kubernetes-talos.md)** — the second deployment target:
|
||||
one Helm chart, a single-node cluster, and the parts that bite (ADR-0033).
|
||||
|
||||
## Quickstart
|
||||
|
||||
|
||||
+10
-2
@@ -2,8 +2,10 @@
|
||||
|
||||
> **Status: active.** The workflow `.gitea/workflows/ci.yaml` runs on Gitea's
|
||||
> hosted `ubuntu-latest` runner — no self-hosted runner required.
|
||||
> **`make ci` is still the local gate** — it runs the exact same checks
|
||||
> (the workflow calls the same `make` targets).
|
||||
> **`make ci` is still the local gate** — it runs the same checks via the same
|
||||
> `make` targets, with one exception: the `k8s` job's targets are not in `make ci`,
|
||||
> because `helm` is optional for everyone not deploying to Kubernetes. Run
|
||||
> `make k8s-lint k8s-drift` by hand after touching the chart or the compose file.
|
||||
|
||||
## The pipeline
|
||||
|
||||
@@ -16,6 +18,8 @@ and CI cannot drift:
|
||||
| `lint` | `make lint` → `dotnet format … --verify-no-changes` | .NET 10 SDK |
|
||||
| `build` | `make build` → `dotnet build … -c Release` | .NET 10 SDK |
|
||||
| `unit` | `make unit` → `dotnet test … -c Release --filter "Category!=Integration"` | .NET 10 SDK |
|
||||
| `frontend` | `make frontend` → Nx lint/test/build for the four portals | pnpm + Node |
|
||||
| `k8s` | `make k8s-lint` (render + schema-check the Helm chart) → `make k8s-drift` (chart still describes the same stack as `infra/docker-compose.yml`) | pinned `helm` binary + `docker compose` |
|
||||
| `mutation` | `make mutation` → `dotnet tool restore` → `dotnet stryker` (ACL); uploads the HTML report as an artifact | .NET 10 SDK |
|
||||
| `verify-stack` | the single live-stack stage — steps: `make verify-up` (full stack up + health, the DoD smoke) → `make verify-acl` (ACL ↔ OpenZaak) → `make verify-nrc` (OpenZaak → NRC delivery) → `make down` | container engine + egress (base images, nuget, `selectielijst.openzaak.nl`) |
|
||||
|
||||
@@ -27,6 +31,10 @@ and CI cannot drift:
|
||||
> services by **container IP** (the runner can't reach published ports — see
|
||||
> [gitea-actions-gotchas.md §5/§6](gitea-actions-gotchas.md)).
|
||||
|
||||
A second workflow, `.gitea/workflows/deploy.yaml`, deploys the stack to the Talos
|
||||
cluster on the lab server when a PR is merged to `main` — see
|
||||
[kubernetes-talos.md §9](kubernetes-talos.md) for its secrets and the SSH tunnel it needs.
|
||||
|
||||
All `uses:` references are absolute, tag-pinned URLs (`https://github.com/actions/checkout@v4`,
|
||||
`https://github.com/actions/setup-dotnet@v4`) per CLAUDE.md §8.7 and §15 — Gitea
|
||||
Actions resolves them from GitHub.
|
||||
|
||||
@@ -245,3 +245,47 @@ the verify-stack check table, and per-spec e2e results (`infra/playwright-summar
|
||||
- Getting a report out of the e2e container: Playwright writes `playwright-report.json`
|
||||
inside the container; `infra/run-e2e-check.sh` `docker cp`s it back to the host
|
||||
(capturing the test exit code first) so the summary step can read it.
|
||||
|
||||
---
|
||||
|
||||
## 9. `if: always()` does not survive the job being killed — bound the work itself
|
||||
|
||||
`if: always()` makes a step run when an *earlier step failed*. It does **not** help when
|
||||
the job as a whole is stopped: the run's remaining steps are simply never dispatched.
|
||||
|
||||
That is how #161 lost its diagnosis. `verify-stack` entered `make verify-e2e` at 09:48:17
|
||||
and the job ended at 10:14:54 — 26½ minutes later, mid-suite. Every step after the e2e
|
||||
shows a **0-second `failure`** stamped at that same instant:
|
||||
|
||||
```
|
||||
14 failure 09:48:17 -> 10:14:54 Self-service e2e (Playwright, login → submit → success)
|
||||
15 failure 10:14:54 -> 10:14:54 verify-stack check summary ← if: always()
|
||||
16 failure 10:14:54 -> 10:14:54 e2e spec summary ← if: always()
|
||||
17 failure 10:14:54 -> 10:14:54 Dump container logs on failure ← if: failure()
|
||||
18 failure 10:14:54 -> 10:14:54 Tear down ← if: always()
|
||||
```
|
||||
|
||||
So the per-spec summary, the container-log dump and the teardown never ran, and the job
|
||||
log — which also loses whatever the killed process had buffered — ended at a single `✘`
|
||||
line. A job that dies takes its own post-mortem with it.
|
||||
|
||||
**Read the step timings, not just the log.** `GET /api/v1/repos/{owner}/{repo}/actions/jobs/{id}`
|
||||
returns every step with `started_at`/`completed_at`; a row of identical zero-length
|
||||
steps at the end means *killed*, not *silent*. (Job ids come from
|
||||
`…/actions/runs/{run}/jobs`, and that route returns only the **latest attempt** — a
|
||||
re-run hides the failed one, so keep the failing job id from the original report. Logs:
|
||||
`…/actions/jobs/{id}/logs`, see also `gitea-ci-logs`.)
|
||||
|
||||
**Conventions that follow:**
|
||||
|
||||
- **Bound long-running work inside the tool**, where it can still report. Playwright's
|
||||
`globalTimeout` (`tests/e2e/playwright.config.ts`) ends the run, writes the JSON
|
||||
report and exits, so the summary and log-dump steps still get their turn. A
|
||||
`timeout-minutes` on the job would reproduce the very failure above.
|
||||
- **Never let an auto-waiting action be the timeout.** Playwright actions (`fill`,
|
||||
`click`) inherit the *test* timeout, not `expect.timeout`, so a missing element costs
|
||||
the full 90 s and reports `locator.fill: Test timeout …` — the symptom. Assert the
|
||||
element visible first with its own budget and a message (`tests/e2e/keycloak-login.ts`).
|
||||
- Remember `concurrency.cancel-in-progress: true` in `ci.yaml`: a new push to the same
|
||||
ref, or a re-run, kills the in-flight run the same way. Check `run_attempt` before
|
||||
concluding a job hung.
|
||||
|
||||
@@ -0,0 +1,414 @@
|
||||
# Deploying the stack to a single-node Talos cluster
|
||||
|
||||
The Helm chart in `infra/helm/big-reference` is a port of `infra/docker-compose.yml`
|
||||
(ADR-0033). This runbook is the walkthrough that was actually used to bring the stack up
|
||||
on a Talos VM under virt-manager on a laptop, including the parts that bite.
|
||||
|
||||
Compose remains the CI-canonical stack — `make verify`, the acceptance lane and the
|
||||
Playwright e2e all still drive it. Kubernetes is a second deployment target.
|
||||
|
||||
## 0. What you need
|
||||
|
||||
On the laptop, four static binaries, all installable to `~/.local/bin` without root:
|
||||
|
||||
```bash
|
||||
curl -sSLo ~/.local/bin/talosctl https://github.com/siderolabs/talos/releases/download/v1.14.0/talosctl-linux-amd64
|
||||
curl -sSLo ~/.local/bin/kubectl https://dl.k8s.io/release/v1.37.0/bin/linux/amd64/kubectl
|
||||
curl -sSL https://get.helm.sh/helm-v3.16.4-linux-amd64.tar.gz | tar xz -O linux-amd64/helm > ~/.local/bin/helm
|
||||
curl -sSL https://github.com/google/go-containerregistry/releases/download/v0.20.2/go-containerregistry_Linux_x86_64.tar.gz | tar xz -O crane > ~/.local/bin/crane
|
||||
chmod +x ~/.local/bin/{talosctl,kubectl,helm,crane}
|
||||
```
|
||||
|
||||
Match `talosctl` to the Talos ISO you booted (`talosctl version --insecure -n <ip>` reports
|
||||
the server's tag). `crane` is what pushes images to a plain-HTTP registry without a
|
||||
root-level Docker daemon change — see §2.
|
||||
|
||||
**VM sizing.** 6 vCPU / 10 GB RAM / 27 GB disk runs the whole stack with room to spare
|
||||
(measured: ~4.4 GB used, 5.4 GB available with all 29 pods up). 4 GB is not enough. The
|
||||
chart sets no resource requests or limits on purpose — on a single node the VM's RAM is the
|
||||
only budget there is. Resize a stopped VM with:
|
||||
|
||||
```bash
|
||||
virsh -c qemu:///system destroy talos # it's in maintenance mode; nothing is lost
|
||||
virsh -c qemu:///system setmaxmem talos 10G --config
|
||||
virsh -c qemu:///system setmem talos 10G --config
|
||||
virsh -c qemu:///system setvcpus talos 6 --config --maximum
|
||||
virsh -c qemu:///system setvcpus talos 6 --config
|
||||
```
|
||||
|
||||
Two addresses matter throughout:
|
||||
|
||||
| Name | Meaning | Example |
|
||||
|---|---|---|
|
||||
| `TALOS_HOST` | the VM's IP — used by the browser, `talosctl` and `kubectl` | `192.168.122.33` |
|
||||
| `K8S_REGISTRY` | `TALOS_HOST:30500` — the in-cluster registry (§2) | `192.168.122.33:30500` |
|
||||
|
||||
Find the VM's address with `virsh -c qemu:///system net-dhcp-leases default`.
|
||||
|
||||
## 1. Install Talos onto the VM
|
||||
|
||||
### The virt-manager trap
|
||||
|
||||
virt-manager treats the install ISO as one-shot: on the VM's **first shutdown** it ejects
|
||||
the CD and rewrites the boot order to `hd`. A Talos VM booted from `metal-amd64.iso` runs
|
||||
entirely in RAM, so the disk is still empty — the next start lands on
|
||||
`Boot failed: not a bootable disk`. Put the ISO back before installing:
|
||||
|
||||
```bash
|
||||
virsh -c qemu:///system change-media talos sda /path/to/metal-amd64.iso --config --insert
|
||||
virt-xml -c qemu:///system talos --edit --boot cdrom,hd
|
||||
virsh -c qemu:///system start talos
|
||||
```
|
||||
|
||||
Wait for the maintenance-mode API, then confirm the install disk's device name — on virtio
|
||||
it is `/dev/vda`, and Talos's default selector expects `/dev/sda`:
|
||||
|
||||
```bash
|
||||
talosctl get disks --insecure -n <TALOS_HOST> -e <TALOS_HOST>
|
||||
```
|
||||
|
||||
### Generate the machine config
|
||||
|
||||
Talos 1.14 moved several v1alpha1 fields into their own config documents. In particular
|
||||
`machine.install` is now `UnattendedInstallConfig`, and patching the old field is rejected
|
||||
with *"UnattendedInstallConfig config is incompatible with v1alpha1 config"*. Write
|
||||
`patch.yaml` as a multi-document patch:
|
||||
|
||||
```yaml
|
||||
machine:
|
||||
certSANs:
|
||||
- 192.168.122.33
|
||||
registries:
|
||||
mirrors:
|
||||
# The in-cluster registry (§2) speaks plain HTTP.
|
||||
"192.168.122.33:30500":
|
||||
endpoints:
|
||||
- http://192.168.122.33:30500
|
||||
---
|
||||
apiVersion: v1alpha1
|
||||
kind: UnattendedInstallConfig
|
||||
provisioning:
|
||||
diskSelector:
|
||||
match: disk.dev_path == "/dev/vda"
|
||||
```
|
||||
|
||||
```bash
|
||||
talosctl gen config big https://<TALOS_HOST>:6443 --output-dir ~/.talos/big --config-patch @patch.yaml
|
||||
talosctl apply-config --insecure -n <TALOS_HOST> -e <TALOS_HOST> --file ~/.talos/big/controlplane.yaml
|
||||
```
|
||||
|
||||
Talos installs to the disk and **kexecs straight into the installed system**, so the CD
|
||||
boot order doesn't get in the way here. Then point the client at the node and bootstrap:
|
||||
|
||||
```bash
|
||||
talosctl config merge ~/.talos/big/talosconfig
|
||||
talosctl config endpoint <TALOS_HOST>
|
||||
talosctl config node <TALOS_HOST>
|
||||
talosctl bootstrap # wait for `talosctl version` to answer first
|
||||
talosctl kubeconfig -f ~/.kube/config
|
||||
```
|
||||
|
||||
A single-node cluster must run workloads on the control plane, or CoreDNS never schedules:
|
||||
|
||||
```bash
|
||||
kubectl taint node --all node-role.kubernetes.io/control-plane-
|
||||
```
|
||||
|
||||
Finally, make a VM restart boot the installed system rather than the ISO (takes effect at
|
||||
the next full power cycle):
|
||||
|
||||
```bash
|
||||
virsh -c qemu:///system change-media talos sda --eject --config
|
||||
virt-xml -c qemu:///system talos --edit --boot hd
|
||||
```
|
||||
|
||||
## 2. A registry the node can pull from
|
||||
|
||||
Talos has no Docker daemon and no way to side-load an image, so this repo's images have to
|
||||
come from a registry. The registry runs **inside the cluster**, published on NodePort
|
||||
30500 (`infra/helm/registry.yaml`):
|
||||
|
||||
```bash
|
||||
make k8s-registry
|
||||
```
|
||||
|
||||
Why in-cluster rather than on the laptop: a laptop-side registry needs an inbound port
|
||||
opened on firewalld's `libvirt` zone (`sudo firewall-cmd --zone=libvirt --add-port=5000/tcp`),
|
||||
which needs root. Pushing from the laptop *to* the node is outbound and always allowed, and
|
||||
the node pulls from its own NodePort. If you do open that port, put a registry on the
|
||||
laptop instead and point `K8S_REGISTRY` at `<laptop-ip>:5000` — the mirror patch in §1 has
|
||||
an entry ready for it.
|
||||
|
||||
Its storage is `emptyDir`, so if the registry pod is ever replaced, re-run `make k8s-images`.
|
||||
|
||||
## 3. Build and push the images
|
||||
|
||||
```bash
|
||||
make k8s-images K8S_REGISTRY=<TALOS_HOST>:30500
|
||||
```
|
||||
|
||||
This builds the nine images with `docker compose build` — same contexts and Dockerfiles as
|
||||
compose, no second build definition — then `docker save | crane push --insecure` each one.
|
||||
`docker push` is not used: the registry speaks plain HTTP, which the Docker daemon refuses
|
||||
without a root-level `insecure-registries` entry, while crane just takes `--insecure`.
|
||||
|
||||
## 4. Deploy
|
||||
|
||||
```bash
|
||||
make k8s-up TALOS_HOST=<TALOS_HOST> K8S_REGISTRY=<TALOS_HOST>:30500
|
||||
```
|
||||
|
||||
That does two things:
|
||||
|
||||
1. `make k8s-seed` — creates the ConfigMaps the chart mounts, from the config files that
|
||||
already live in this repo (`infra/helm/seed-configmaps.sh`): the four
|
||||
`setup_configuration/data.yaml` files, the Keycloak realm exports, the BPMN + DMN, and
|
||||
the two bootstrap scripts. Re-run it after editing any of them.
|
||||
2. `helm upgrade --install` of the chart into namespace `big`.
|
||||
|
||||
First bring-up takes a few minutes: the four Django services migrate their databases and
|
||||
apply their `setup_configuration`, Flowable creates its schema, and the bootstrap Jobs
|
||||
deploy the BPMN/DMN, seed the zaaktype and register the NRC abonnement.
|
||||
|
||||
```bash
|
||||
kubectl -n big get pods -w
|
||||
kubectl -n big get jobs # all four must reach COMPLETIONS 1/1
|
||||
```
|
||||
|
||||
The Jobs are the stack's wiring; if one is not complete, the flow is broken somewhere
|
||||
specific:
|
||||
|
||||
| Job | What breaks without it |
|
||||
|---|---|
|
||||
| `flowable-init` | no `registratie` process, no diploma DMN |
|
||||
| `registerrecord-init` | the register has no RegisterRecord objecttype, so writes are refused |
|
||||
| `seed-zaaktype` | the ACL can't resolve `BIG-REGISTRATIE`, so no zaak is created |
|
||||
| `nrc-subscribe` | register writes never reach the projection — the public register stays empty |
|
||||
|
||||
## 5. Use it
|
||||
|
||||
### The portals must be reached over `localhost`
|
||||
|
||||
The portals' OIDC flow uses PKCE, which needs `crypto.subtle` — and browsers only expose
|
||||
that in a **secure context**: HTTPS, or an origin on `localhost`/`127.0.0.1`. A NodePort on
|
||||
the VM's IP is neither, so `http://<TALOS_HOST>:30140` fails before it can even build the
|
||||
authorize URL:
|
||||
|
||||
```
|
||||
ERROR TypeError: Cannot read properties of undefined (reading 'digest')
|
||||
at t.calcHash → t.generateCodeChallenge → t.createUrlCodeFlowAuthorize
|
||||
```
|
||||
|
||||
So deploy with `TALOS_HOST=localhost` — which pins Keycloak's issuer and the portals'
|
||||
`config.json` authority to `http://localhost:30180` — and forward the browser-facing
|
||||
services to those same ports:
|
||||
|
||||
```bash
|
||||
make k8s-up TALOS_HOST=localhost K8S_REGISTRY=<TALOS_HOST>:30500
|
||||
make k8s-portals # stays in the foreground; Ctrl-C stops all five forwards
|
||||
```
|
||||
|
||||
| URL (needs `make k8s-portals`) | What |
|
||||
|---|---|
|
||||
| `http://localhost:30140` | self-service portal (DigiD) |
|
||||
| `http://localhost:30141` | openbaar register (anonymous) |
|
||||
| `http://localhost:30142` | behandel portal (medewerker) |
|
||||
| `http://localhost:30143` | beheer portal (medewerker) |
|
||||
| `http://localhost:30180` | Keycloak (admin/admin) |
|
||||
|
||||
The port numbers are deliberately the NodePort numbers: Keycloak's issuer is one fixed
|
||||
string, so the port the browser uses has to match the one baked into `config.json`.
|
||||
|
||||
This is the same mechanism `infra/host-browser.yml` uses for the compose stack (which pins
|
||||
`localhost:8180`); only the addresses differ.
|
||||
|
||||
### The admin UIs work straight off the NodePorts
|
||||
|
||||
These are server-rendered and need no secure context, so they are reachable at the VM's
|
||||
address with no forwarding:
|
||||
|
||||
| URL | What |
|
||||
|---|---|
|
||||
| `http://<TALOS_HOST>:30000` | OpenZaak admin (admin/admin) |
|
||||
| `http://<TALOS_HOST>:30001` | Open Notificaties admin (admin/admin) |
|
||||
| `http://<TALOS_HOST>:30020` / `:30021` | Objecttypen / Objecten admin |
|
||||
| `http://<TALOS_HOST>:30080` | BFF (`/health`) |
|
||||
| `http://<TALOS_HOST>:30090` | Flowable REST (rest-admin/test) |
|
||||
|
||||
### Credentials
|
||||
|
||||
Log in with the test users from `docs/synthetic-data.md` (all password `test123`, e.g.
|
||||
`jan-burger` for self-service, `merel-behandelaar` for behandel). The `medewerker` realm
|
||||
enforces MFA (ADR-0031) — print a current code with
|
||||
`python3 infra/keycloak/check_realms.py otp`. Walk the flow in `docs/demo-script.md`.
|
||||
|
||||
`TALOS_HOST` is not cosmetic: it pins Keycloak's issuer (`KC_HOSTNAME`) and the portals'
|
||||
OIDC authority to the same string, which is what makes a browser token pass the BFF's
|
||||
validation (ADR-0010). Change it and you must re-run `make k8s-up` — the chart rolls the
|
||||
portals for you, because their `config.json` is a subPath mount and would otherwise keep
|
||||
serving the old authority.
|
||||
|
||||
### Smoke-test the whole chain without a browser
|
||||
|
||||
With the forwards running:
|
||||
|
||||
```bash
|
||||
TOK=$(curl -s -X POST http://localhost:30180/realms/digid/protocol/openid-connect/token \
|
||||
-d grant_type=password -d client_id=big-portal \
|
||||
-d username=jan-burger -d password=test123 -d scope=openid | jq -r .access_token)
|
||||
|
||||
# through the portal's Caddy, so this also proves the BFF reverse proxy
|
||||
curl -s -X POST http://localhost:30140/self-service/registrations \
|
||||
-H "Authorization: Bearer $TOK" -H 'Content-Length: 0'
|
||||
# → {"registrationId":"…","status":"Ingediend"}
|
||||
|
||||
curl -s http://localhost:30141/openbaar/register
|
||||
# → [{"id":"…","status":"INGEDIEND","reference":"<the registrationId>"}]
|
||||
```
|
||||
|
||||
The second call proves the whole Common Ground path: portal → BFF → domain → Flowable →
|
||||
ACL → OpenZaak + Objecten → NRC → event-subscriber → projection → openbaar register.
|
||||
|
||||
## 6. Keeping the databases (recommended if you iterate on the chart)
|
||||
|
||||
By default every database is an `emptyDir`: no CSI driver needed, and the data lives as
|
||||
long as the pod. Note what that means in practice — **any** change to a database pod's
|
||||
template (an image policy, an env value, a probe) recreates the pod and wipes it. The stack
|
||||
then needs its bootstrap re-run:
|
||||
|
||||
```bash
|
||||
make k8s-reseed TALOS_HOST=... K8S_REGISTRY=...
|
||||
```
|
||||
|
||||
which re-runs the four Jobs *and* restarts `event-subscriber` + `projection-api`, because
|
||||
those two create the projection schema on start and otherwise keep writing to a
|
||||
schema-less database (`relation "processed_notifications" does not exist`). For persistence, install Rancher's local-path-provisioner — on Talos it
|
||||
must write under `/var` and its namespace needs the privileged Pod Security label:
|
||||
|
||||
```yaml
|
||||
# kustomization.yaml
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- github.com/rancher/local-path-provisioner/deploy?ref=v0.0.31
|
||||
patches:
|
||||
- patch: |-
|
||||
kind: ConfigMap
|
||||
apiVersion: v1
|
||||
metadata:
|
||||
name: local-path-config
|
||||
namespace: local-path-storage
|
||||
data:
|
||||
config.json: |-
|
||||
{ "nodePathMap":[ { "node":"DEFAULT_PATH_FOR_NON_LISTED_NODES", "paths":["/var/local-path-provisioner"] } ] }
|
||||
- patch: |-
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: local-path-storage
|
||||
labels:
|
||||
pod-security.kubernetes.io/enforce: privileged
|
||||
```
|
||||
|
||||
```bash
|
||||
kubectl apply -k .
|
||||
make k8s-up TALOS_HOST=... K8S_REGISTRY=... K8S_SET='--set persistence.storageClass=local-path'
|
||||
```
|
||||
|
||||
The PVCs carry `helm.sh/resource-policy: keep`, so `make k8s-down` leaves the data behind;
|
||||
`make k8s-purge` drops the namespace and with it the volumes.
|
||||
|
||||
## 7. Day-to-day
|
||||
|
||||
```bash
|
||||
make k8s-lint # render + schema-check the chart, no cluster needed
|
||||
make k8s-drift # fail if compose and the chart describe different stacks
|
||||
make k8s-portals # forward the portals + Keycloak to localhost (browser access)
|
||||
make k8s-images K8S_REGISTRY=... # after changing a service or a portal
|
||||
make k8s-up TALOS_HOST=... K8S_REGISTRY=...
|
||||
make k8s-seed # after editing a data.yaml, a realm export, or the BPMN
|
||||
make k8s-reseed TALOS_HOST=... K8S_REGISTRY=... # re-run the bootstrap Jobs + reset the projection schema
|
||||
make k8s-down # uninstall, keep the database PVCs
|
||||
make k8s-purge # uninstall and drop the namespace
|
||||
```
|
||||
|
||||
This repo's images are pulled with `imagePullPolicy: Always` (the `dev` tag is mutable), so
|
||||
`kubectl -n big rollout restart deploy/<name>` after `make k8s-images` picks up a rebuild.
|
||||
Upstream images stay `IfNotPresent`: their tags are pinned, and keeping them out of the pod
|
||||
template avoids needless churn — a changed template makes a Job unpatchable.
|
||||
|
||||
`k8s-reseed` is also the path for *changing* a Job in the chart: a Job's pod template is
|
||||
immutable, so `helm upgrade` is rejected with `cannot patch "…" with kind Job`.
|
||||
|
||||
## 8. When it doesn't work
|
||||
|
||||
| Symptom | Cause |
|
||||
|---|---|
|
||||
| `Boot failed: not a bootable disk` | virt-manager ejected the install ISO on first shutdown — see §1 |
|
||||
| The VM comes back in maintenance mode after a restart | the ISO is still attached and boots first; eject it and set `--boot hd` (§1) |
|
||||
| `apply-config` rejects the patch with *"incompatible with v1alpha1"* | Talos ≥1.14 owns that field in its own config document — patch the document, not `machine.*` (§1) |
|
||||
| CoreDNS `Pending` forever | the control-plane taint is still on the only node (§1) |
|
||||
| `ImagePullBackOff` … `pull QPS exceeded` | transient: the kubelet rate-limits pulls when ~30 pods start at once. It recovers on retry |
|
||||
| `ImagePullBackOff` on a `register-referentie/*` image | the registry mirror patch is missing: `talosctl get registriesconfig` |
|
||||
| Pod stuck in `ContainerCreating`, event names a ConfigMap | `make k8s-seed` |
|
||||
| `seed-zaaktype` retrying | publishing a zaaktype validates the resultaattype against `selectielijst.openzaak.nl`, so this one Job needs outbound internet from the VM (ADR-0006) |
|
||||
| `TypeError: Cannot read properties of undefined (reading 'digest')` on a portal | not a secure context: `crypto.subtle` is absent on `http://<ip>`. Use `localhost` + `make k8s-portals` (§5) |
|
||||
| Login redirects but the portal stays logged out, or the BFF answers 401 | `TALOS_HOST` doesn't match the address in the browser's URL bar — issuer mismatch. Re-run `make k8s-up` with the right value |
|
||||
| A portal returns 502 on `/self-service/…` | the BFF is unreachable from the portal pod: check `kubectl -n big get svc bff` and the BFF's own readiness |
|
||||
| Public register empty after a submit | usually a wiped `emptyDir` database (§6): `make k8s-reseed`. Confirm with `kubectl -n big logs deploy/event-subscriber \| grep 42P01` |
|
||||
| `helm upgrade` fails with `cannot patch … with kind Job` | see §7 — use `make k8s-reseed` |
|
||||
| Pods `Evicted` / `OOMKilled` | the VM is too small (§0) |
|
||||
| A Job shows `BackoffLimitExceeded` | read it: `kubectl -n big logs job/<name>` |
|
||||
|
||||
## 9. Deploying on merge to main
|
||||
|
||||
`.gitea/workflows/deploy.yaml` runs the §3–§4 steps against the **lab server's** Talos VM
|
||||
every time a PR is squash-merged to `main` (and on demand via *Run workflow*). PR CI is the
|
||||
merge gate, so the workflow deploys without re-running the checks.
|
||||
|
||||
The cluster's API and registry are not exposed publicly, so the job forwards them over the
|
||||
same SSH hop the Gitea-runner pipeline uses:
|
||||
|
||||
```
|
||||
ssh -p 6667 user@labs.respellion.tech -L 6443 -L 30500 -L 30141 → <TALOS_VM_IP>
|
||||
```
|
||||
|
||||
Consequences worth knowing:
|
||||
|
||||
- Images are **pushed** to `localhost:30500` (the tunnel) and **pulled** by the node from
|
||||
`<TALOS_VM_IP>:30500` (its own NodePort, the address in the Talos registry-mirror patch).
|
||||
Same registry, two names — hence the two `K8S_REGISTRY` values in the workflow.
|
||||
- It calls `make k8s-reseed`, not `make k8s-up`: the bootstrap Jobs are idempotent, and
|
||||
deleting them first is what stops a changed Job template from wedging `helm upgrade` (§7).
|
||||
- `dev` is a mutable tag, so a `rollout restart` of the nine repo deployments is what
|
||||
actually puts the new images in the pods.
|
||||
- Deploys **queue** (`cancel-in-progress: false`): a helm upgrade killed half-way leaves the
|
||||
release in `pending-upgrade`, which has to be unwedged by hand.
|
||||
|
||||
Settings, all on the repository in Gitea:
|
||||
|
||||
| Kind | Name | What |
|
||||
|---|---|---|
|
||||
| Secret | `TALOS_SSH_KEY` | private key for `user@labs.respellion.tech` (the Fedora host) |
|
||||
| Secret | `TALOS_KUBECONFIG` | base64 of the kubeconfig, **`server: https://127.0.0.1:6443`** — Talos puts `127.0.0.1` in the apiserver cert SANs, so TLS still verifies through the tunnel |
|
||||
| Variable | `TALOS_VM_IP` | the VM's libvirt address (default `192.168.122.173`) |
|
||||
| Variable | `TALOS_HOST` | the browser-facing host baked into Keycloak's issuer (default `localhost`, see §5) |
|
||||
|
||||
The last step smokes `GET /openbaar/register` through the openbaar portal, which exercises
|
||||
portal → Caddy → BFF → projection. An empty register passes; a 502 does not.
|
||||
|
||||
Not covered: the portals still need `make k8s-portals` (or an SSH forward) to be usable in a
|
||||
browser, because PKCE needs a secure context (§5). Giving the server a hostname + TLS is the
|
||||
upgrade path.
|
||||
|
||||
## What is not ported
|
||||
|
||||
- **Observability** (Tempo, Prometheus, Grafana) is defined but disabled — those are built
|
||||
images too, so switching them on means pushing them as well:
|
||||
`K8S_SET='--set workloads.tempo.enabled=true --set workloads.prometheus.enabled=true --set workloads.grafana.enabled=true'`.
|
||||
The .NET services still export OTLP; the exporter fails harmlessly when Tempo is absent.
|
||||
- **The verify/e2e lanes.** `make verify*` and the Playwright e2e drive compose, not the
|
||||
chart. The Kubernetes path is verified with §5's smoke test. CI's `k8s` job runs the two
|
||||
clusterless checks (`k8s-lint`, `k8s-drift`) on every PR — a values typo or a compose
|
||||
image bump that skipped the chart fails there, but nothing deploys the chart in CI.
|
||||
- **Ingress, TLS, and resource requests.** See the ponytail ceiling in ADR-0033.
|
||||
Executable
+30
@@ -0,0 +1,30 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fail when a page under docs/ is missing from mkdocs.yml's nav.
|
||||
|
||||
docs/ is the source of truth (CLAUDE.md §12), but only the pages listed in the nav
|
||||
are published — and mkdocs' own `omitted_files: warn` keeps a build green while
|
||||
silently dropping them, which is how every ADR after 0010 and every runbook but
|
||||
ci.md fell off the site.
|
||||
|
||||
ponytail: a substring test, not a YAML parse — a page's path either appears in
|
||||
mkdocs.yml or it doesn't, and that needs no dependency.
|
||||
"""
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
nav = (ROOT / "mkdocs.yml").read_text()
|
||||
|
||||
missing = sorted(
|
||||
str(page.relative_to(ROOT / "docs"))
|
||||
for page in (ROOT / "docs").rglob("*.md")
|
||||
if str(page.relative_to(ROOT / "docs")) not in nav
|
||||
)
|
||||
|
||||
if missing:
|
||||
print(f"{len(missing)} page(s) under docs/ are not in mkdocs.yml's nav:")
|
||||
print("\n".join(f" {m}" for m in missing))
|
||||
sys.exit(1)
|
||||
|
||||
print("docs nav complete: every page under docs/ is published")
|
||||
@@ -510,7 +510,7 @@ services:
|
||||
networks: [cg]
|
||||
|
||||
# ── Portals (S-08/S-09/S-12) ──────────────────────────────────────────────
|
||||
# nginx serves each Angular app and reverse-proxies its endpoint group to the BFF (same-origin).
|
||||
# Caddy serves each Angular app and reverse-proxies its endpoint group to the BFF (same-origin).
|
||||
# The images bake config.json with the compose authority (keycloak:8080), which a HOST browser
|
||||
# can't resolve — so here we bind-mount a config.json pointing at the host-published localhost:8180
|
||||
# (matching KC_HOSTNAME). openbaar is anonymous and needs no config.
|
||||
@@ -522,7 +522,7 @@ services:
|
||||
ports:
|
||||
- "8140:80"
|
||||
volumes:
|
||||
- ./local-config/self-service.config.json:/usr/share/nginx/html/config.json:ro,z
|
||||
- ./local-config/self-service.config.json:/usr/share/caddy/config.json:ro,z
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1/ || exit 1"]
|
||||
interval: 5s
|
||||
@@ -562,7 +562,7 @@ services:
|
||||
ports:
|
||||
- "8142:80"
|
||||
volumes:
|
||||
- ./local-config/behandel.config.json:/usr/share/nginx/html/config.json:ro,z
|
||||
- ./local-config/behandel.config.json:/usr/share/caddy/config.json:ro,z
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1/ || exit 1"]
|
||||
interval: 5s
|
||||
|
||||
@@ -496,7 +496,7 @@ services:
|
||||
networks: [cg]
|
||||
|
||||
# ── Self-Service portal (S-08d) ────────────────────────────────────────────
|
||||
# nginx serves the Angular app and reverse-proxies /self-service + /openbaar to the BFF
|
||||
# Caddy serves the Angular app and reverse-proxies /self-service + /openbaar to the BFF
|
||||
# (same-origin, no CORS). The Playwright e2e drives it inside this network so the DigiD
|
||||
# token issuer (keycloak:8080) matches the BFF's authority (ADR-0010).
|
||||
self-service:
|
||||
@@ -507,7 +507,7 @@ services:
|
||||
ports:
|
||||
- "8140:80"
|
||||
healthcheck:
|
||||
# 127.0.0.1, not localhost: nginx listens on IPv4 only, but localhost resolves to ::1 first.
|
||||
# 127.0.0.1, not localhost: keeps the check on the interface Caddy is published on.
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1/ || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
@@ -520,7 +520,7 @@ services:
|
||||
condition: service_started
|
||||
networks: [cg]
|
||||
|
||||
# The openbaar (public) register portal: nginx serves the Angular app and reverse-proxies
|
||||
# The openbaar (public) register portal: Caddy serves the Angular app and reverse-proxies
|
||||
# /openbaar to the BFF. Anonymous — no DigiD, no Keycloak dependency (S-09).
|
||||
openbaar:
|
||||
build:
|
||||
@@ -530,7 +530,7 @@ services:
|
||||
ports:
|
||||
- "8141:80"
|
||||
healthcheck:
|
||||
# 127.0.0.1, not localhost: nginx listens on IPv4 only, but localhost resolves to ::1 first.
|
||||
# 127.0.0.1, not localhost: keeps the check on the interface Caddy is published on.
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1/ || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
@@ -541,7 +541,7 @@ services:
|
||||
condition: service_healthy
|
||||
networks: [cg]
|
||||
|
||||
# The behandel portal: nginx serves the Angular app and reverse-proxies /behandel to the BFF.
|
||||
# The behandel portal: Caddy serves the Angular app and reverse-proxies /behandel to the BFF.
|
||||
# Behandelaars log in against the Keycloak medewerker realm (ADR-0013; S-12).
|
||||
behandel:
|
||||
build:
|
||||
@@ -551,7 +551,7 @@ services:
|
||||
ports:
|
||||
- "8142:80"
|
||||
healthcheck:
|
||||
# 127.0.0.1, not localhost: nginx listens on IPv4 only, but localhost resolves to ::1 first.
|
||||
# 127.0.0.1, not localhost: keeps the check on the interface Caddy is published on.
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1/ || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
@@ -564,7 +564,7 @@ services:
|
||||
condition: service_started
|
||||
networks: [cg]
|
||||
|
||||
# The beheer portal: nginx serves the Angular app and reverse-proxies /beheer to the BFF.
|
||||
# The beheer portal: Caddy serves the Angular app and reverse-proxies /beheer to the BFF.
|
||||
# Beheerders log in against the Keycloak medewerker realm (same realm as behandel, S-15a).
|
||||
beheer:
|
||||
build:
|
||||
@@ -574,7 +574,7 @@ services:
|
||||
ports:
|
||||
- "8143:80"
|
||||
healthcheck:
|
||||
# 127.0.0.1, not localhost: nginx listens on IPv4 only, but localhost resolves to ::1 first.
|
||||
# 127.0.0.1, not localhost: keeps the check on the interface Caddy is published on.
|
||||
test: ["CMD-SHELL", "wget -q -O /dev/null http://127.0.0.1/ || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
apiVersion: v2
|
||||
name: big-reference
|
||||
description: >-
|
||||
The BIG reference stack (Common Ground) on Kubernetes — a port of
|
||||
infra/docker-compose.yml, aimed at a single-node Talos cluster.
|
||||
type: application
|
||||
version: 0.1.0
|
||||
appVersion: dev
|
||||
@@ -0,0 +1,25 @@
|
||||
{{ .Chart.Name }} {{ .Chart.Version }} deployed to namespace {{ .Release.Namespace }}.
|
||||
|
||||
Watch it converge (the upstream Django services migrate on first boot, so the
|
||||
first bring-up takes a few minutes):
|
||||
|
||||
kubectl -n {{ .Release.Namespace }} get pods -w
|
||||
kubectl -n {{ .Release.Namespace }} get jobs
|
||||
|
||||
Every bootstrap Job must reach Completions 1/1:
|
||||
{{- range $name, $w := .Values.workloads }}
|
||||
{{- if and (ne $w.enabled false) $w.job }}
|
||||
- {{ $name }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
Open in a browser (add {{ .Values.host }} to /etc/hosts if you use a name):
|
||||
{{- range $name, $port := .Values.nodePorts }}
|
||||
{{- $w := index $.Values.workloads $name }}
|
||||
{{- if ne $w.enabled false }}
|
||||
{{ printf "%-16s http://%s:%v" $name $.Values.host $port }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
Test users are in docs/synthetic-data.md. If a pod is stuck in
|
||||
ContainerCreating on a missing ConfigMap, run: make k8s-seed
|
||||
@@ -0,0 +1,142 @@
|
||||
{{/*
|
||||
One pod spec for every workload, Deployment and Job alike. The chart is
|
||||
values-driven on purpose: `.Values.workloads` is a near-literal transcription of
|
||||
infra/docker-compose.yml, so the two stacks can be diffed by eye instead of by
|
||||
archaeology. Adding a service is a values edit, not a template edit.
|
||||
|
||||
Called as: include "big.podspec" (dict "root" $ "name" $name "w" $w)
|
||||
*/}}
|
||||
{{- define "big.podspec" -}}
|
||||
{{- $root := .root -}}
|
||||
{{- $name := .name -}}
|
||||
{{- $w := .w -}}
|
||||
{{- with $root.Values.imagePullSecrets }}
|
||||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 2 }}
|
||||
{{- end }}
|
||||
{{- with $w.waitFor }}
|
||||
initContainers:
|
||||
- name: wait-for-deps
|
||||
image: {{ $root.Values.images.busybox }}
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
for t in {{ join " " . }}; do
|
||||
echo "waiting for $t"
|
||||
until nc -z "${t%:*}" "${t#*:}"; do sleep 2; done
|
||||
done
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: {{ $name }}
|
||||
image: {{ include "big.image" (dict "root" $root "name" $name "w" $w) }}
|
||||
# Only this repo's images get the configured policy: their `dev` tag is mutable.
|
||||
# Upstream tags are pinned, so IfNotPresent keeps them out of pod-template diffs —
|
||||
# which matters because a changed template makes a Job unpatchable (immutable).
|
||||
imagePullPolicy: {{ if $w.own }}{{ $root.Values.images.pullPolicy }}{{ else }}IfNotPresent{{ end }}
|
||||
{{- if $w.command }}
|
||||
{{- fail (printf "workload %s: use `args`, not `command` — compose's `command:` replaces CMD, but Kubernetes' `command:` replaces the image ENTRYPOINT (postgres would run as root, keycloak would exec `start-dev`)" $name) }}
|
||||
{{- end }}
|
||||
{{- with $w.args }}
|
||||
args:
|
||||
{{- toYaml . | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- with $w.envFrom }}
|
||||
envFrom:
|
||||
{{- range . }}
|
||||
- configMapRef:
|
||||
# optional: an env group whose feature is disabled (e.g. otel) simply
|
||||
# isn't rendered, and the pod must still start.
|
||||
name: {{ printf "%s-env" . }}
|
||||
optional: true
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- with $w.env }}
|
||||
env:
|
||||
{{- include "big.env" (list $root .) | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- with $w.ports }}
|
||||
ports:
|
||||
{{- range . }}
|
||||
- name: {{ .name }}
|
||||
containerPort: {{ .targetPort | default .port }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- with $w.probe }}
|
||||
readinessProbe:
|
||||
{{- toYaml . | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- with $w.resources }}
|
||||
resources:
|
||||
{{- toYaml . | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- if or $w.files $w.data }}
|
||||
volumeMounts:
|
||||
{{- range $w.files }}
|
||||
- name: {{ .configMap }}
|
||||
mountPath: {{ .mountPath }}
|
||||
{{- with .subPath }}
|
||||
subPath: {{ . }}
|
||||
{{- end }}
|
||||
readOnly: true
|
||||
{{- end }}
|
||||
{{- with $w.data }}
|
||||
- name: data
|
||||
mountPath: {{ .mountPath }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if or $w.files $w.data }}
|
||||
volumes:
|
||||
{{- range $w.files }}
|
||||
- name: {{ .configMap }}
|
||||
configMap:
|
||||
name: {{ .configMap }}
|
||||
{{- with .defaultMode }}
|
||||
defaultMode: {{ . }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- with $w.data }}
|
||||
- name: data
|
||||
{{- if $root.Values.persistence.storageClass }}
|
||||
persistentVolumeClaim:
|
||||
claimName: {{ $name }}-data
|
||||
{{- else }}
|
||||
# No StorageClass configured: the databases are emptyDir, so the stack needs
|
||||
# no CSI driver to come up. Data then lives as long as the pod does — see
|
||||
# docs/runbooks/kubernetes-talos.md for switching on local-path.
|
||||
emptyDir: {}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
|
||||
{{/* Image ref: `own: true` workloads are built from this repo, everything else is upstream. */}}
|
||||
{{- define "big.image" -}}
|
||||
{{- $root := .root -}}
|
||||
{{- $w := .w -}}
|
||||
{{- if $w.own -}}
|
||||
{{- $ref := printf "%s/%s:%s" $root.Values.images.repositoryPrefix .name $root.Values.images.tag -}}
|
||||
{{- with $root.Values.images.registry }}{{ printf "%s/%s" . $ref }}{{ else }}{{ $ref }}{{ end }}
|
||||
{{- else -}}
|
||||
{{- $w.image -}}
|
||||
{{- end -}}
|
||||
{{- end -}}
|
||||
|
||||
{{/*
|
||||
Env list from a map. Every value is run through `tpl`, so values.yaml can name
|
||||
cluster-internal hosts ({{ .Release.Namespace }}) and the node address
|
||||
({{ .Values.host }}) without the chart hard-coding either.
|
||||
*/}}
|
||||
{{- define "big.env" -}}
|
||||
{{- $root := index . 0 -}}
|
||||
{{- range $k, $v := index . 1 }}
|
||||
- name: {{ $k }}
|
||||
value: {{ tpl (toString $v) $root | quote }}
|
||||
{{- end }}
|
||||
{{- end -}}
|
||||
|
||||
{{- define "big.labels" -}}
|
||||
app.kubernetes.io/name: {{ .name }}
|
||||
app.kubernetes.io/instance: {{ .root.Release.Name }}
|
||||
app.kubernetes.io/managed-by: Helm
|
||||
{{- end -}}
|
||||
@@ -0,0 +1,44 @@
|
||||
{{- /*
|
||||
Shared env blocks — the Kubernetes equivalent of the YAML anchors in
|
||||
infra/docker-compose.yml (&oz-env, &nrc-env, &objecttypen-env, &objecten-env).
|
||||
A workload picks them up with `envFrom`, so the web/celery/init variants of an
|
||||
upstream image stay guaranteed-identical, and `kubectl get cm oz-env -o yaml`
|
||||
shows what a pod actually got.
|
||||
|
||||
The *file* inputs (setup_configuration data.yaml, Keycloak realms, BPMN/DMN, the
|
||||
seed scripts) are NOT here: they live in the repo and are turned into ConfigMaps
|
||||
by infra/helm/seed-configmaps.sh, exactly as infra/seed-config.sh streams them
|
||||
into the compose config volumes. Copying them into the chart would fork them.
|
||||
*/ -}}
|
||||
{{- range $group, $env := .Values.envGroups }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: {{ $group }}-env
|
||||
labels:
|
||||
{{- include "big.labels" (dict "root" $ "name" (printf "%s-env" $group)) | nindent 4 }}
|
||||
data:
|
||||
{{- range $k, $v := $env }}
|
||||
{{ $k }}: {{ tpl (toString $v) $ | quote }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- /*
|
||||
Portal OIDC config. The images bake config.json with the compose authority
|
||||
(keycloak:8080), which a browser outside the cluster cannot resolve; these
|
||||
ConfigMaps mount over it with the node address Keycloak's issuer is pinned to
|
||||
(KC_HOSTNAME below), so the token the browser gets and the issuer the BFF
|
||||
discovers are the same string. Same mechanism as infra/host-browser.yml.
|
||||
*/ -}}
|
||||
{{- range $realm := list "digid" "medewerker" }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: portal-config-{{ $realm }}
|
||||
labels:
|
||||
{{- include "big.labels" (dict "root" $ "name" (printf "portal-config-%s" $realm)) | nindent 4 }}
|
||||
data:
|
||||
config.json: |
|
||||
{ "authority": "{{ printf "http://%s:%v" $.Values.host (index $.Values.nodePorts "keycloak") }}/realms/{{ $realm }}" }
|
||||
{{- end }}
|
||||
@@ -0,0 +1,39 @@
|
||||
{{- range $name, $w := .Values.workloads }}
|
||||
{{- if and (ne $w.enabled false) (not $w.job) }}
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ $name }}
|
||||
labels:
|
||||
{{- include "big.labels" (dict "root" $ "name" $name) | nindent 4 }}
|
||||
spec:
|
||||
replicas: 1
|
||||
# Recreate, not RollingUpdate: single node, ReadWriteOnce volumes, and nothing
|
||||
# here is HA — a second pod would just fight the first for the disk.
|
||||
strategy:
|
||||
type: Recreate
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: {{ $name }}
|
||||
app.kubernetes.io/instance: {{ $.Release.Name }}
|
||||
template:
|
||||
metadata:
|
||||
{{- /*
|
||||
A ConfigMap mounted with subPath never picks up updates, so a portal whose
|
||||
config.json content changed has to be rolled. Hashing only the values that
|
||||
render it keeps the churn off the databases — an emptyDir database that is
|
||||
recreated for no reason loses its data (see the runbook §6).
|
||||
*/}}
|
||||
{{- range $w.files }}
|
||||
{{- if hasPrefix "portal-config-" .configMap }}
|
||||
annotations:
|
||||
checksum/portal-config: {{ printf "%s|%v" $.Values.host (index $.Values.nodePorts "keycloak") | sha256sum }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
labels:
|
||||
{{- include "big.labels" (dict "root" $ "name" $name) | nindent 8 }}
|
||||
spec:
|
||||
{{- include "big.podspec" (dict "root" $ "name" $name "w" $w) | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,29 @@
|
||||
{{- /*
|
||||
The one-shot bootstrap containers from compose (oz-init, nrc-init, flowable-init,
|
||||
the *-init setup_configuration runs, the zaaktype seed and the NRC abonnement)
|
||||
become Jobs. All of them are idempotent, so ordering is not enforced with hooks:
|
||||
each waits for the ports it needs (waitFor) and Kubernetes retries the rest.
|
||||
A wiped database is re-seeded by `make k8s-reseed`.
|
||||
*/ -}}
|
||||
{{- range $name, $w := .Values.workloads }}
|
||||
{{- if and (ne $w.enabled false) $w.job }}
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: {{ $name }}
|
||||
labels:
|
||||
{{- include "big.labels" (dict "root" $ "name" $name) | nindent 4 }}
|
||||
app.kubernetes.io/component: init
|
||||
spec:
|
||||
backoffLimit: 20
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
{{- include "big.labels" (dict "root" $ "name" $name) | nindent 8 }}
|
||||
app.kubernetes.io/component: init
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
{{- include "big.podspec" (dict "root" $ "name" $name "w" $w) | nindent 6 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,22 @@
|
||||
{{- if .Values.persistence.storageClass }}
|
||||
{{- range $name, $w := .Values.workloads }}
|
||||
{{- if and (ne $w.enabled false) $w.data }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: {{ $name }}-data
|
||||
labels:
|
||||
{{- include "big.labels" (dict "root" $ "name" $name) | nindent 4 }}
|
||||
# Keep the databases when the release is uninstalled; `make k8s-purge` drops them.
|
||||
annotations:
|
||||
helm.sh/resource-policy: keep
|
||||
spec:
|
||||
accessModes: [ReadWriteOnce]
|
||||
storageClassName: {{ $.Values.persistence.storageClass }}
|
||||
resources:
|
||||
requests:
|
||||
storage: {{ $w.data.size | default "2Gi" }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,35 @@
|
||||
{{- /*
|
||||
Service names are the compose service names, verbatim: the portals' Caddy
|
||||
proxies to http://bff:8080 and the upstream setup_configuration files name
|
||||
http://openzaak:8000 / http://nrc-web:8000, so in-cluster DNS has to answer to
|
||||
exactly those names. Do not rename a workload without checking both.
|
||||
|
||||
.Values.nodePorts is the single place a port is published outside the cluster;
|
||||
a workload listed there gets a NodePort on its first (only) port.
|
||||
*/ -}}
|
||||
{{- range $name, $w := .Values.workloads }}
|
||||
{{- if and (ne $w.enabled false) $w.ports }}
|
||||
{{- $nodePort := index $.Values.nodePorts $name }}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ $name }}
|
||||
labels:
|
||||
{{- include "big.labels" (dict "root" $ "name" $name) | nindent 4 }}
|
||||
spec:
|
||||
type: {{ if $nodePort }}NodePort{{ else }}ClusterIP{{ end }}
|
||||
selector:
|
||||
app.kubernetes.io/name: {{ $name }}
|
||||
app.kubernetes.io/instance: {{ $.Release.Name }}
|
||||
ports:
|
||||
{{- range $i, $p := $w.ports }}
|
||||
- name: {{ $p.name }}
|
||||
port: {{ $p.port }}
|
||||
targetPort: {{ $p.targetPort | default $p.port }}
|
||||
{{- if and $nodePort (eq $i 0) }}
|
||||
nodePort: {{ $nodePort }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -0,0 +1,608 @@
|
||||
# Values for the BIG reference stack on Kubernetes.
|
||||
#
|
||||
# `workloads` is a near-literal transcription of infra/docker-compose.yml — same
|
||||
# service names, same images, same env, same one-shots — so the two stacks can be
|
||||
# diffed by eye. Read that file's comments for the *why* behind each setting; only
|
||||
# the deviations forced by Kubernetes are re-explained here.
|
||||
#
|
||||
# Every env value is rendered with Helm's `tpl`, so it may use:
|
||||
# {{ .Release.Namespace }} — for a cluster-internal FQDN
|
||||
# {{ .Values.host }} — the node address a browser reaches the cluster on
|
||||
#
|
||||
# Deviations from compose, all of them consequences of the platform:
|
||||
# * The compose stack hands the ACL and the seeds OpenZaak's *container IP*,
|
||||
# because OpenZaak and NRC validate URLs with Django's URLValidator and a
|
||||
# single-label host ("openzaak") is rejected. In Kubernetes the service FQDN
|
||||
# (openzaak.<ns>.svc.cluster.local) is already multi-label, so the IP dance and
|
||||
# the `objecten.local` network alias both disappear.
|
||||
# * `depends_on: service_healthy` becomes a `waitFor` init container (TCP wait)
|
||||
# plus readiness probes. Ordering is otherwise not enforced: every bootstrap
|
||||
# job is idempotent and Kubernetes retries.
|
||||
# * The published ports are NodePorts (see `nodePorts`), not host ports.
|
||||
|
||||
# The address a browser outside the cluster uses to reach the node: your Talos
|
||||
# VM's IP. It pins Keycloak's issuer and the portals' OIDC authority to one
|
||||
# string, so browser tokens and the BFF's discovered issuer agree.
|
||||
host: 192.168.122.100
|
||||
|
||||
# Set when pulling from a private registry (e.g. the Gitea Container Registry).
|
||||
imagePullSecrets: []
|
||||
|
||||
images:
|
||||
# Where the images built from THIS repo live. Empty = the bare
|
||||
# `register-referentie/<svc>:dev` names, which only works if the node already
|
||||
# has them. On Talos it never does — point this at a registry the node can
|
||||
# reach (see docs/runbooks/kubernetes-talos.md).
|
||||
registry: ""
|
||||
repositoryPrefix: register-referentie
|
||||
tag: dev
|
||||
# Applies to this repo's images only (see _helpers.tpl). Always, because `dev`
|
||||
# is a mutable tag: with IfNotPresent the node keeps the first image it pulled
|
||||
# and `make k8s-images` would appear to do nothing. The registry is in-cluster,
|
||||
# so a re-pull is local and cheap — but the pods do depend on it being up.
|
||||
pullPolicy: Always
|
||||
busybox: docker.io/library/busybox:stable
|
||||
|
||||
persistence:
|
||||
# Empty = every database is an emptyDir, so the stack comes up on a bare
|
||||
# cluster with no CSI driver. Set to a StorageClass (e.g. `local-path`) to keep
|
||||
# the data across pod restarts.
|
||||
storageClass: ""
|
||||
|
||||
# The only place a port is published outside the cluster. A workload listed here
|
||||
# gets a NodePort on its single port; everything else stays ClusterIP.
|
||||
nodePorts:
|
||||
openzaak: 30000
|
||||
nrc-web: 30001
|
||||
objecttypen: 30020
|
||||
objecten: 30021
|
||||
bff: 30080
|
||||
flowable-rest: 30090
|
||||
self-service: 30140
|
||||
openbaar: 30141
|
||||
behandel: 30142
|
||||
beheer: 30143
|
||||
keycloak: 30180
|
||||
grafana: 30300
|
||||
|
||||
# ── Shared env blocks (the compose YAML anchors) ────────────────────────────────
|
||||
envGroups:
|
||||
|
||||
oz:
|
||||
UWSGI_PROCESSES: "1"
|
||||
UWSGI_THREADS: "2"
|
||||
DJANGO_SETTINGS_MODULE: openzaak.conf.docker
|
||||
SECRET_KEY: dev-only-not-for-production
|
||||
DB_HOST: oz-db
|
||||
DB_NAME: openzaak
|
||||
DB_USER: openzaak
|
||||
DB_PASSWORD: openzaak
|
||||
IS_HTTPS: "no"
|
||||
ALLOWED_HOSTS: "*"
|
||||
CACHE_DEFAULT: oz-redis:6379/0
|
||||
CACHE_AXES: oz-redis:6379/0
|
||||
CELERY_BROKER_URL: redis://oz-redis:6379/1
|
||||
CELERY_RESULT_BACKEND: redis://oz-redis:6379/1
|
||||
DISABLE_2FA: "true"
|
||||
NOTIFICATIONS_DISABLED: "false"
|
||||
OPENZAAK_SUPERUSER_USERNAME: admin
|
||||
DJANGO_SUPERUSER_PASSWORD: admin
|
||||
OPENZAAK_SUPERUSER_EMAIL: admin@localhost
|
||||
RUN_SETUP_CONFIG: "true"
|
||||
|
||||
nrc:
|
||||
UWSGI_PROCESSES: "1"
|
||||
UWSGI_THREADS: "2"
|
||||
DJANGO_SETTINGS_MODULE: nrc.conf.docker
|
||||
SECRET_KEY: dev-only-not-for-production
|
||||
DB_HOST: nrc-db
|
||||
DB_NAME: opennotificaties
|
||||
DB_USER: opennotificaties
|
||||
DB_PASSWORD: opennotificaties
|
||||
IS_HTTPS: "no"
|
||||
ALLOWED_HOSTS: "*"
|
||||
CACHE_DEFAULT: nrc-redis:6379/0
|
||||
CACHE_AXES: nrc-redis:6379/0
|
||||
CELERY_BROKER_URL: redis://nrc-redis:6379/1
|
||||
CELERY_RESULT_BACKEND: redis://nrc-redis:6379/1
|
||||
DISABLE_2FA: "true"
|
||||
OPENNOTIFICATIES_SUPERUSER_USERNAME: admin
|
||||
DJANGO_SUPERUSER_PASSWORD: admin
|
||||
OPENNOTIFICATIES_SUPERUSER_EMAIL: admin@localhost
|
||||
RUN_SETUP_CONFIG: "true"
|
||||
NOTIFICATION_SEC_INTERVAL: "5"
|
||||
|
||||
objecttypen:
|
||||
UWSGI_PROCESSES: "1"
|
||||
UWSGI_THREADS: "2"
|
||||
DJANGO_SETTINGS_MODULE: objecttypes.conf.docker
|
||||
SECRET_KEY: dev-only-not-for-production
|
||||
DB_HOST: objecttypen-db
|
||||
DB_NAME: objecttypes
|
||||
DB_USER: objecttypes
|
||||
DB_PASSWORD: objecttypes
|
||||
ALLOWED_HOSTS: "*"
|
||||
CACHE_DEFAULT: objecttypen-redis:6379/0
|
||||
CACHE_AXES: objecttypen-redis:6379/0
|
||||
DISABLE_2FA: "true"
|
||||
OTEL_SDK_DISABLED: "true"
|
||||
RUN_SETUP_CONFIG: "true"
|
||||
|
||||
objecten:
|
||||
UWSGI_PROCESSES: "1"
|
||||
UWSGI_THREADS: "2"
|
||||
DJANGO_SETTINGS_MODULE: objects.conf.docker
|
||||
SECRET_KEY: dev-only-not-for-production
|
||||
DB_HOST: objecten-db
|
||||
DB_NAME: objects
|
||||
DB_USER: objects
|
||||
DB_PASSWORD: objects
|
||||
ALLOWED_HOSTS: "*"
|
||||
CACHE_DEFAULT: objecten-redis:6379/0
|
||||
CACHE_AXES: objecten-redis:6379/0
|
||||
DISABLE_2FA: "true"
|
||||
OTEL_SDK_DISABLED: "true"
|
||||
CELERY_BROKER_URL: redis://objecten-redis:6379/1
|
||||
CELERY_RESULT_BACKEND: redis://objecten-redis:6379/1
|
||||
NOTIFICATIONS_DISABLED: "false"
|
||||
RUN_SETUP_CONFIG: "true"
|
||||
|
||||
# Traces for the .NET services. Always set, like compose: the exporter fails
|
||||
# harmlessly when Tempo is absent (services/*/Program.cs).
|
||||
otel:
|
||||
OTEL_EXPORTER_OTLP_ENDPOINT: http://tempo:4317
|
||||
OTEL_EXPORTER_OTLP_PROTOCOL: grpc
|
||||
|
||||
# ── Workloads ──────────────────────────────────────────────────────────────────
|
||||
# Per entry: image | own (built here) · args · envFrom (env groups) · env
|
||||
# ports · probe (a literal readinessProbe) · files (ConfigMap mounts) · data
|
||||
# (a database volume) · waitFor (host:port to wait for) · job · enabled
|
||||
#
|
||||
# `args` (never `command`) is the compose `command:` equivalent: compose replaces
|
||||
# the image's CMD, and so does Kubernetes' `args` — Kubernetes' `command` would
|
||||
# replace the ENTRYPOINT instead. The chart fails to render if you use `command`.
|
||||
workloads:
|
||||
|
||||
# ── OpenZaak (S-01) ─────────────────────────────────────────────────────────
|
||||
oz-db:
|
||||
image: docker.io/postgis/postgis:17-3.5
|
||||
args: [postgres, -c, max_connections=300]
|
||||
env:
|
||||
POSTGRES_USER: openzaak
|
||||
POSTGRES_PASSWORD: openzaak
|
||||
POSTGRES_DB: openzaak
|
||||
ports: [{ name: postgres, port: 5432 }]
|
||||
data: { mountPath: /var/lib/postgresql/data, size: 4Gi }
|
||||
probe:
|
||||
exec:
|
||||
command: [sh, -c, "pg_isready -U openzaak -d openzaak && psql -U openzaak -d openzaak -c 'SELECT PostGIS_Version();' -q"]
|
||||
periodSeconds: 5
|
||||
|
||||
oz-redis:
|
||||
image: docker.io/library/redis:7
|
||||
ports: [{ name: redis, port: 6379 }]
|
||||
probe: { tcpSocket: { port: 6379 } }
|
||||
openzaak:
|
||||
image: docker.io/openzaak/open-zaak:1.28.2
|
||||
# setup_configuration first, then the server — in ONE container, on purpose.
|
||||
# Both /setup_configuration.sh and /start.sh run `manage.py migrate`, so a
|
||||
# separate init Job (as compose has, ordered by depends_on) races this pod for
|
||||
# the same database and Django fails with "relation already exists".
|
||||
args: [sh, -c, "/setup_configuration.sh && exec /start.sh"]
|
||||
envFrom: [oz]
|
||||
ports: [{ name: http, port: 8000 }]
|
||||
# /admin/ answers 302 when Django is up — a redirect counts as ready.
|
||||
probe:
|
||||
httpGet: { path: /admin/, port: 8000 }
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
failureThreshold: 30
|
||||
files: [{ configMap: rr-oz-config, mountPath: /app/setup_configuration }]
|
||||
waitFor: [oz-db:5432, oz-redis:6379]
|
||||
|
||||
oz-celery:
|
||||
image: docker.io/openzaak/open-zaak:1.28.2
|
||||
args: [/celery_worker.sh]
|
||||
envFrom: [oz]
|
||||
waitFor: [oz-db:5432, oz-redis:6379]
|
||||
|
||||
# ── Open Notificaties / NRC (S-01-c) ────────────────────────────────────────
|
||||
nrc-db:
|
||||
image: docker.io/postgis/postgis:17-3.5
|
||||
args: [postgres, -c, max_connections=300]
|
||||
env:
|
||||
POSTGRES_USER: opennotificaties
|
||||
POSTGRES_PASSWORD: opennotificaties
|
||||
POSTGRES_DB: opennotificaties
|
||||
ports: [{ name: postgres, port: 5432 }]
|
||||
data: { mountPath: /var/lib/postgresql/data, size: 2Gi }
|
||||
probe:
|
||||
exec: { command: [pg_isready, -U, opennotificaties, -d, opennotificaties] }
|
||||
periodSeconds: 5
|
||||
|
||||
nrc-redis:
|
||||
image: docker.io/library/redis:7
|
||||
ports: [{ name: redis, port: 6379 }]
|
||||
probe: { tcpSocket: { port: 6379 } }
|
||||
nrc-web:
|
||||
image: docker.io/openzaak/open-notificaties:1.16.1
|
||||
# setup_configuration first, then the server — in ONE container, on purpose.
|
||||
# Both /setup_configuration.sh and /start.sh run `manage.py migrate`, so a
|
||||
# separate init Job (as compose has, ordered by depends_on) races this pod for
|
||||
# the same database and Django fails with "relation already exists".
|
||||
args: [sh, -c, "/setup_configuration.sh && exec /start.sh"]
|
||||
envFrom: [nrc]
|
||||
ports: [{ name: http, port: 8000 }]
|
||||
probe:
|
||||
httpGet: { path: /admin/, port: 8000 }
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
failureThreshold: 30
|
||||
files: [{ configMap: rr-nrc-config, mountPath: /app/setup_configuration }]
|
||||
waitFor: [nrc-db:5432, nrc-redis:6379, openzaak:8000]
|
||||
|
||||
nrc-celery:
|
||||
image: docker.io/openzaak/open-notificaties:1.16.1
|
||||
args: [/celery_worker.sh]
|
||||
envFrom: [nrc]
|
||||
waitFor: [nrc-db:5432, nrc-redis:6379]
|
||||
|
||||
# Without beat, notifications are accepted but never delivered (ADR-0007).
|
||||
nrc-beat:
|
||||
image: docker.io/openzaak/open-notificaties:1.16.1
|
||||
args: [/celery_beat.sh]
|
||||
envFrom: [nrc]
|
||||
waitFor: [nrc-db:5432, nrc-redis:6379]
|
||||
|
||||
# ── Keycloak (S-02) ─────────────────────────────────────────────────────────
|
||||
keycloak:
|
||||
image: quay.io/keycloak/keycloak:26.1
|
||||
args: [start-dev, --import-realm]
|
||||
env:
|
||||
KC_BOOTSTRAP_ADMIN_USERNAME: admin
|
||||
KC_BOOTSTRAP_ADMIN_PASSWORD: admin
|
||||
KEYCLOAK_ADMIN: admin
|
||||
KEYCLOAK_ADMIN_PASSWORD: admin
|
||||
KC_HEALTH_ENABLED: "true"
|
||||
KC_HTTP_ENABLED: "true"
|
||||
# Pin the issuer to the address the browser uses, and let backchannel calls
|
||||
# keep using keycloak:8080 — the BFF discovers metadata in-cluster and gets
|
||||
# this issuer back, which is what browser tokens carry (infra/host-browser.yml).
|
||||
KC_HOSTNAME: "http://{{ .Values.host }}:{{ index .Values.nodePorts \"keycloak\" }}"
|
||||
KC_HOSTNAME_BACKCHANNEL_DYNAMIC: "true"
|
||||
ports: [{ name: http, port: 8080 }]
|
||||
# TCP, not /health/ready on the management port: nothing here gates on realm
|
||||
# import, and a wrong health path would leave the Service with no endpoints.
|
||||
probe: { tcpSocket: { port: 8080 }, initialDelaySeconds: 15 }
|
||||
files: [{ configMap: rr-kc-realms, mountPath: /opt/keycloak/data/import }]
|
||||
|
||||
# ── Flowable (S-03) ─────────────────────────────────────────────────────────
|
||||
flowable-db:
|
||||
image: docker.io/library/postgres:16
|
||||
env:
|
||||
POSTGRES_USER: flowable
|
||||
POSTGRES_PASSWORD: flowable
|
||||
POSTGRES_DB: flowable
|
||||
ports: [{ name: postgres, port: 5432 }]
|
||||
data: { mountPath: /var/lib/postgresql/data, size: 2Gi }
|
||||
probe:
|
||||
exec: { command: [pg_isready, -U, flowable, -d, flowable] }
|
||||
periodSeconds: 5
|
||||
|
||||
flowable-rest:
|
||||
image: docker.io/flowable/flowable-rest:latest
|
||||
env:
|
||||
SPRING_DATASOURCE_DRIVER-CLASS-NAME: org.postgresql.Driver
|
||||
SPRING_DATASOURCE_URL: jdbc:postgresql://flowable-db:5432/flowable
|
||||
SPRING_DATASOURCE_USERNAME: flowable
|
||||
SPRING_DATASOURCE_PASSWORD: flowable
|
||||
ports: [{ name: http, port: 8080 }]
|
||||
# Every REST path needs basic auth, so an httpGet probe would read 401 as
|
||||
# not-ready. TCP is the honest signal here.
|
||||
probe: { tcpSocket: { port: 8080 }, initialDelaySeconds: 20 }
|
||||
waitFor: [flowable-db:5432]
|
||||
|
||||
# Deploys the BPMN to the process engine and the DMN to the DMN engine as two
|
||||
# separate deployments — flowable-rest does not cascade one into the other
|
||||
# (S-13, ADR-0016). Idempotent.
|
||||
flowable-init:
|
||||
job: true
|
||||
image: docker.io/curlimages/curl:latest
|
||||
args:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
svc=http://flowable-rest:8080/flowable-rest/service/repository/deployments
|
||||
dmn=http://flowable-rest:8080/flowable-rest/dmn-api/dmn-repository/deployments
|
||||
until curl -sf -u rest-admin:test "$svc" >/dev/null 2>&1; do echo "waiting for flowable-rest..."; sleep 3; done
|
||||
if curl -s -u rest-admin:test "$dmn" | grep -q '"name":"diploma-eligibility.dmn"'; then
|
||||
echo "diploma-eligibility DMN already deployed; skip"
|
||||
else
|
||||
curl -sf -u rest-admin:test -F 'file=@/work/diploma-eligibility.dmn;filename=diploma-eligibility.dmn' "$dmn" >/dev/null && echo "deployed diploma-eligibility DMN"
|
||||
fi
|
||||
if curl -s -u rest-admin:test "$svc?name=registratie" | grep -q '"name":"registratie"'; then
|
||||
echo "registratie BPMN already deployed; skip"
|
||||
else
|
||||
curl -sf -u rest-admin:test -F 'file=@/work/registratie.bpmn;filename=registratie.bpmn' "$svc" >/dev/null && echo "deployed registratie BPMN"
|
||||
fi
|
||||
files: [{ configMap: rr-fl-bpmn, mountPath: /work }]
|
||||
waitFor: [flowable-rest:8080]
|
||||
|
||||
# ── ACL ─────────────────────────────────────────────────────────────────────
|
||||
acl:
|
||||
own: true
|
||||
envFrom: [otel]
|
||||
env:
|
||||
OTEL_SERVICE_NAME: acl
|
||||
# The FQDN, not `openzaak`: OpenZaak rejects a single-label host on
|
||||
# zaak-create. It must be the same host the zaaktype was seeded through
|
||||
# (see the seed-zaaktype job) so the URLs stay host-consistent (ADR-0009).
|
||||
Acl__OpenZaak__BaseUrl: "http://openzaak.{{ .Release.Namespace }}.svc.cluster.local:8000/"
|
||||
Acl__OpenZaak__ClientId: big-reference-seed
|
||||
Acl__OpenZaak__Secret: insecure-dev-secret-change-me
|
||||
Acl__Defaults__Bronorganisatie: "517439943"
|
||||
Acl__Defaults__VerantwoordelijkeOrganisatie: "517439943"
|
||||
Acl__Defaults__Vertrouwelijkheidaanduiding: openbaar
|
||||
Acl__Defaults__ZaaktypeIdentificatie: BIG-REGISTRATIE
|
||||
Acl__Defaults__InformatieobjecttypeOmschrijving: Diploma
|
||||
# Objecten reflects the request Host into the object url it returns, and
|
||||
# publishes that url to NRC — which rejects a single-label host. The FQDN
|
||||
# replaces compose's `objecten.local` alias (ADR-0029).
|
||||
Acl__Objecten__BaseUrl: "http://objecten.{{ .Release.Namespace }}.svc.cluster.local:8000/"
|
||||
Acl__Objecten__Token: 1234567890abcdef1234567890abcdef12345678
|
||||
# Short name on purpose: Objecten only accepts an objecttype URL that
|
||||
# matches the one it was configured with (infra/objecten/setup_configuration
|
||||
# /data.yaml → http://objecttypen:8000/api/v2/).
|
||||
Acl__Objecten__ObjecttypenBaseUrl: http://objecttypen:8000/
|
||||
Acl__Objecten__ObjecttypenToken: 0123456789abcdef0123456789abcdef01234567
|
||||
Acl__Objecten__ObjecttypeName: RegisterRecord
|
||||
ports: [{ name: http, port: 8080 }]
|
||||
probe: { httpGet: { path: /health, port: 8080 }, periodSeconds: 5 }
|
||||
|
||||
# ── BIG Domain Service (S-05) ───────────────────────────────────────────────
|
||||
domain:
|
||||
own: true
|
||||
envFrom: [otel]
|
||||
env:
|
||||
OTEL_SERVICE_NAME: domain
|
||||
Flowable__BaseUrl: http://flowable-rest:8080/flowable-rest/
|
||||
Flowable__Username: rest-admin
|
||||
Flowable__Password: test
|
||||
Acl__BaseUrl: http://acl:8080/
|
||||
ports: [{ name: http, port: 8080 }]
|
||||
probe: { httpGet: { path: /health, port: 8080 }, periodSeconds: 5 }
|
||||
|
||||
# ── BFF ─────────────────────────────────────────────────────────────────────
|
||||
bff:
|
||||
own: true
|
||||
envFrom: [otel]
|
||||
env:
|
||||
OTEL_SERVICE_NAME: bff
|
||||
# In-cluster authority: Keycloak's discovery document returns the pinned
|
||||
# KC_HOSTNAME issuer, which is what browser tokens carry (ADR-0010).
|
||||
Keycloak__Authority: http://keycloak:8080/realms/digid
|
||||
Keycloak__MedewerkerAuthority: http://keycloak:8080/realms/medewerker
|
||||
Downstream__Domain__BaseUrl: http://domain:8080/
|
||||
Downstream__Projection__BaseUrl: http://projection-api:8080/
|
||||
Downstream__Acl__BaseUrl: http://acl:8080/
|
||||
ports: [{ name: http, port: 8080 }]
|
||||
probe: { httpGet: { path: /health, port: 8080 }, periodSeconds: 5 }
|
||||
|
||||
# ── Read projection (S-06) ──────────────────────────────────────────────────
|
||||
projection-db:
|
||||
image: docker.io/library/postgres:16
|
||||
env:
|
||||
POSTGRES_USER: projection
|
||||
POSTGRES_PASSWORD: projection
|
||||
POSTGRES_DB: projection
|
||||
ports: [{ name: postgres, port: 5432 }]
|
||||
data: { mountPath: /var/lib/postgresql/data, size: 2Gi }
|
||||
probe:
|
||||
exec: { command: [pg_isready, -U, projection, -d, projection] }
|
||||
periodSeconds: 5
|
||||
|
||||
event-subscriber:
|
||||
own: true
|
||||
envFrom: [otel]
|
||||
env:
|
||||
OTEL_SERVICE_NAME: event-subscriber
|
||||
ConnectionStrings__Projection: Host=projection-db;Database=projection;Username=projection;Password=projection
|
||||
Acl__BaseUrl: http://acl:8080/
|
||||
EventSubscriber__Webhook__AuthToken: Bearer big-reference-notifications
|
||||
ports: [{ name: http, port: 8080 }]
|
||||
probe: { httpGet: { path: /health, port: 8080 }, periodSeconds: 5 }
|
||||
# It migrates the projection schema on start and throws if the DB is absent.
|
||||
waitFor: [projection-db:5432]
|
||||
|
||||
projection-api:
|
||||
own: true
|
||||
envFrom: [otel]
|
||||
env:
|
||||
OTEL_SERVICE_NAME: projection-api
|
||||
ConnectionStrings__Projection: Host=projection-db;Database=projection;Username=projection;Password=projection
|
||||
ports: [{ name: http, port: 8080 }]
|
||||
probe: { httpGet: { path: /health, port: 8080 }, periodSeconds: 5 }
|
||||
waitFor: [projection-db:5432]
|
||||
|
||||
# ── Portals (S-08/S-09/S-12/S-15) ───────────────────────────────────────────
|
||||
# Caddy serves the Angular app and reverse-proxies its endpoint group to
|
||||
# http://bff:8080 — hence the Service must stay named `bff`. Caddy resolves that
|
||||
# name through the system resolver, so the DNS search domains apply and no
|
||||
# upstream rewriting is needed here (ADR-0034).
|
||||
self-service:
|
||||
own: true
|
||||
ports: [{ name: http, port: 80 }]
|
||||
probe: { httpGet: { path: /, port: 80 }, periodSeconds: 5 }
|
||||
files:
|
||||
- configMap: portal-config-digid
|
||||
mountPath: /usr/share/caddy/config.json
|
||||
subPath: config.json
|
||||
|
||||
openbaar:
|
||||
own: true
|
||||
ports: [{ name: http, port: 80 }]
|
||||
probe: { httpGet: { path: /, port: 80 }, periodSeconds: 5 }
|
||||
|
||||
behandel:
|
||||
own: true
|
||||
ports: [{ name: http, port: 80 }]
|
||||
probe: { httpGet: { path: /, port: 80 }, periodSeconds: 5 }
|
||||
files:
|
||||
- configMap: portal-config-medewerker
|
||||
mountPath: /usr/share/caddy/config.json
|
||||
subPath: config.json
|
||||
|
||||
beheer:
|
||||
own: true
|
||||
ports: [{ name: http, port: 80 }]
|
||||
probe: { httpGet: { path: /, port: 80 }, periodSeconds: 5 }
|
||||
files:
|
||||
- configMap: portal-config-medewerker
|
||||
mountPath: /usr/share/caddy/config.json
|
||||
subPath: config.json
|
||||
|
||||
# ── Objecttypen API (S-18a) ─────────────────────────────────────────────────
|
||||
objecttypen-db:
|
||||
image: docker.io/library/postgres:17-alpine
|
||||
env:
|
||||
POSTGRES_USER: objecttypes
|
||||
POSTGRES_PASSWORD: objecttypes
|
||||
POSTGRES_DB: objecttypes
|
||||
ports: [{ name: postgres, port: 5432 }]
|
||||
data: { mountPath: /var/lib/postgresql/data, size: 2Gi }
|
||||
probe:
|
||||
exec: { command: [pg_isready, -U, objecttypes] }
|
||||
periodSeconds: 5
|
||||
|
||||
objecttypen-redis:
|
||||
image: docker.io/library/redis:7
|
||||
ports: [{ name: redis, port: 6379 }]
|
||||
probe: { tcpSocket: { port: 6379 } }
|
||||
objecttypen:
|
||||
image: docker.io/maykinmedia/objecttypes-api:3.4.2
|
||||
# setup_configuration first, then the server — in ONE container, on purpose.
|
||||
# Both /setup_configuration.sh and /start.sh run `manage.py migrate`, so a
|
||||
# separate init Job (as compose has, ordered by depends_on) races this pod for
|
||||
# the same database and Django fails with "relation already exists".
|
||||
args: [sh, -c, "/setup_configuration.sh && exec /start.sh"]
|
||||
envFrom: [objecttypen]
|
||||
ports: [{ name: http, port: 8000 }]
|
||||
probe:
|
||||
httpGet: { path: /admin/, port: 8000 }
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
failureThreshold: 30
|
||||
files: [{ configMap: rr-objecttypen-config, mountPath: /app/setup_configuration }]
|
||||
waitFor: [objecttypen-db:5432, objecttypen-redis:6379]
|
||||
|
||||
# The RegisterRecord objecttype + published version, over the API (S-18c,
|
||||
# ADR-0020/ADR-0027). The uuid is pinned — Objecten identifies it by uuid.
|
||||
registerrecord-init:
|
||||
job: true
|
||||
image: docker.io/library/python:3-slim
|
||||
args: [python, /config/register.py]
|
||||
env:
|
||||
OBJECTTYPEN: http://objecttypen:8000
|
||||
OBJECTTYPEN_TOKEN: 0123456789abcdef0123456789abcdef01234567
|
||||
SCHEMA: /config/registerrecord.schema.json
|
||||
files: [{ configMap: rr-registerrecord-config, mountPath: /config }]
|
||||
waitFor: [objecttypen:8000]
|
||||
|
||||
# ── Objecten API (S-18b) ────────────────────────────────────────────────────
|
||||
objecten-db:
|
||||
image: docker.io/postgis/postgis:17-3.5
|
||||
env:
|
||||
POSTGRES_USER: objects
|
||||
POSTGRES_PASSWORD: objects
|
||||
POSTGRES_DB: objects
|
||||
ports: [{ name: postgres, port: 5432 }]
|
||||
data: { mountPath: /var/lib/postgresql/data, size: 2Gi }
|
||||
probe:
|
||||
exec: { command: [pg_isready, -U, objects] }
|
||||
periodSeconds: 5
|
||||
|
||||
objecten-redis:
|
||||
image: docker.io/library/redis:7
|
||||
ports: [{ name: redis, port: 6379 }]
|
||||
probe: { tcpSocket: { port: 6379 } }
|
||||
objecten:
|
||||
image: docker.io/maykinmedia/objects-api:3.4.0
|
||||
# setup_configuration first, then the server — in ONE container, on purpose.
|
||||
# Both /setup_configuration.sh and /start.sh run `manage.py migrate`, so a
|
||||
# separate init Job (as compose has, ordered by depends_on) races this pod for
|
||||
# the same database and Django fails with "relation already exists".
|
||||
args: [sh, -c, "/setup_configuration.sh && exec /start.sh"]
|
||||
envFrom: [objecten]
|
||||
ports: [{ name: http, port: 8000 }]
|
||||
probe:
|
||||
httpGet: { path: /admin/, port: 8000 }
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 10
|
||||
failureThreshold: 30
|
||||
files: [{ configMap: rr-objecten-config, mountPath: /app/setup_configuration }]
|
||||
waitFor: [objecten-db:5432, objecten-redis:6379, objecttypen:8000]
|
||||
|
||||
# Delivers Objecten's notifications to NRC; without it every register write is
|
||||
# silently undelivered (ADR-0029).
|
||||
objecten-celery:
|
||||
image: docker.io/maykinmedia/objects-api:3.4.0
|
||||
args: [/celery_worker.sh]
|
||||
envFrom: [objecten]
|
||||
waitFor: [objecten-db:5432, objecten-redis:6379]
|
||||
|
||||
# ── Bootstrap the flow, like the local compose stack does (S-B04, ADR-0020) ──
|
||||
# Seeds + publishes the BIG zaaktype through the same FQDN the ACL uses, so the
|
||||
# server-assigned URLs are host-consistent. The ACL then resolves them by
|
||||
# identificatie (S-27, ADR-0021) — nothing is injected back.
|
||||
# Publishing validates the resultaattype against the external Selectielijst
|
||||
# API, so the node needs outbound internet for this one job (ADR-0006).
|
||||
seed-zaaktype:
|
||||
job: true
|
||||
image: docker.io/library/python:3-slim
|
||||
args: [python, /seed/seed_catalogus.py]
|
||||
env:
|
||||
OZ_BASE: "http://openzaak.{{ .Release.Namespace }}.svc.cluster.local:8000"
|
||||
OZ_PUBLISH: "1"
|
||||
files: [{ configMap: rr-seed-scripts, mountPath: /seed }]
|
||||
waitFor: [openzaak:8000]
|
||||
|
||||
# Registers the NRC abonnement on the `objecten` kanaal pointing at the
|
||||
# event-subscriber, so register writes reach the projection (ADR-0030).
|
||||
# Without it the openbaar register stays empty. Restart-safe and idempotent.
|
||||
nrc-subscribe:
|
||||
job: true
|
||||
image: docker.io/library/python:3-slim
|
||||
args: [python, /seed/register-abonnement.py]
|
||||
env:
|
||||
NRC_BASE: http://nrc-web:8000
|
||||
# The script resolves this to an address for the callback URL; the FQDN
|
||||
# resolves to the Service's (stable) ClusterIP, which NRC's URLValidator
|
||||
# accepts — the compose stack uses the container IP for the same reason.
|
||||
SINK_HOST: "event-subscriber.{{ .Release.Namespace }}.svc.cluster.local"
|
||||
SINK_PORT: "8080"
|
||||
SINK_AUTH: Bearer big-reference-notifications
|
||||
files: [{ configMap: rr-seed-scripts, mountPath: /seed }]
|
||||
waitFor: [nrc-web:8000, event-subscriber:8080]
|
||||
|
||||
# ── Observability backplane (S-16a, ADR-0023) ───────────────────────────────
|
||||
# Off by default: these are built images too (config baked in), so switching
|
||||
# them on also means pushing three more images. Enable all three together.
|
||||
tempo:
|
||||
enabled: false
|
||||
own: true
|
||||
args: ["-config.file=/etc/tempo.yaml"]
|
||||
ports: [{ name: otlp, port: 4317 }, { name: http, port: 3200 }]
|
||||
|
||||
prometheus:
|
||||
enabled: false
|
||||
own: true
|
||||
ports: [{ name: http, port: 9090 }]
|
||||
|
||||
grafana:
|
||||
enabled: false
|
||||
own: true
|
||||
env:
|
||||
GF_SECURITY_ADMIN_USER: admin
|
||||
GF_SECURITY_ADMIN_PASSWORD: admin
|
||||
GF_AUTH_ANONYMOUS_ENABLED: "true"
|
||||
ports: [{ name: http, port: 3000 }]
|
||||
Executable
+116
@@ -0,0 +1,116 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Fail when the compose stack and the Helm chart stop describing the same stack.
|
||||
|
||||
`infra/docker-compose.yml` is CI-canonical; `infra/helm/big-reference` is a
|
||||
transcription of it (ADR-0033), and until now nothing kept the two in step — an
|
||||
upstream image bump or a new service applied to only one of them landed
|
||||
unnoticed. This compares what each side actually *deploys*, not the two files:
|
||||
the rendered chart against `docker compose config`. Both tools are already
|
||||
prerequisites of the `k8s-*` make targets.
|
||||
|
||||
Run it with `make k8s-drift`. No cluster needed.
|
||||
|
||||
ponytail: names and images only, as sets — no per-workload env/ports/volumes.
|
||||
Those differ by design in four documented places (ADR-0033), so comparing them
|
||||
would mean re-encoding every deviation field by field; a tag bump and a missing
|
||||
service are the drift that actually bites.
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
COMPOSE = ROOT / "infra/docker-compose.yml"
|
||||
CHART = ROOT / "infra/helm/big-reference"
|
||||
|
||||
# The busybox init container that every `waitFor` workload gets exists only in
|
||||
# the chart (compose has `depends_on`). Rendering it under a sentinel makes it
|
||||
# filterable without teaching the check what busybox is.
|
||||
BUSYBOX = "drift-check-ignored-init-image"
|
||||
|
||||
# Differences that Kubernetes forces, not drift (ADR-0033). A name listed here is
|
||||
# expected to be on exactly one side; anything else fails.
|
||||
DEVIATIONS = {
|
||||
# The four Django services apply their own setup_configuration in the web pod
|
||||
# (`args: [sh, -c, "/setup_configuration.sh && exec /start.sh"]`) rather than in a
|
||||
# separate init Job. Both that script and /start.sh run `manage.py migrate`, and
|
||||
# Kubernetes has no `depends_on: service_completed_successfully` to serialise them,
|
||||
# so the Job and its web pod migrated the same database concurrently.
|
||||
"oz-init": "folded into the openzaak pod",
|
||||
"nrc-init": "folded into the nrc-web pod",
|
||||
"objecttypen-init": "folded into the objecttypen pod",
|
||||
"objecten-init": "folded into the objecten pod",
|
||||
# Compose seeds these from the host — the verify scripts `docker cp` the two
|
||||
# scripts into a running container, and docker-compose.local.yml carries
|
||||
# `local-seed` + `nrc-subscribe` for `make local`. A cluster has no host to seed
|
||||
# from, so both became Jobs in the chart.
|
||||
"seed-zaaktype": "compose seeds the catalogus from the host (infra/openzaak/seed_catalogus.py)",
|
||||
"nrc-subscribe": "compose registers the abonnement from the host (infra/local/register-abonnement.py)",
|
||||
}
|
||||
|
||||
# Workloads the observability backplane adds. Off by default in both stacks'
|
||||
# defaults, so they are rendered on purpose here — otherwise their images drift
|
||||
# unwatched.
|
||||
OBSERVABILITY = ["tempo", "prometheus", "grafana"]
|
||||
|
||||
|
||||
def compose_services() -> dict[str, str]:
|
||||
"""Service name -> image, with ${TAG:-default} interpolation already applied."""
|
||||
out = run(["docker", "compose", "-f", str(COMPOSE), "config", "--format", "json"])
|
||||
return {name: svc.get("image", "") for name, svc in json.loads(out)["services"].items()}
|
||||
|
||||
|
||||
def chart_workloads() -> dict[str, str]:
|
||||
"""Workload name -> image, read back out of the rendered manifests."""
|
||||
out = run(
|
||||
["helm", "template", "big", str(CHART), "-n", "big", "--set", f"images.busybox={BUSYBOX}"]
|
||||
+ [f"--set=workloads.{w}.enabled=true" for w in OBSERVABILITY]
|
||||
)
|
||||
workloads = {}
|
||||
for doc in out.split("\n---"):
|
||||
if not re.search(r"^kind: (Deployment|Job)$", doc, re.M):
|
||||
continue
|
||||
name = re.search(r"^ name: (\S+)$", doc, re.M)[1]
|
||||
images = [i for i in re.findall(r"^\s+image: (\S+)$", doc, re.M) if i != BUSYBOX]
|
||||
workloads[name] = images[0]
|
||||
return workloads
|
||||
|
||||
|
||||
def run(argv: list[str]) -> str:
|
||||
proc = subprocess.run(argv, capture_output=True, text=True)
|
||||
if proc.returncode != 0:
|
||||
sys.exit(f"{argv[0]} failed:\n{proc.stderr}")
|
||||
return proc.stdout
|
||||
|
||||
|
||||
def main() -> int:
|
||||
compose, chart = compose_services(), chart_workloads()
|
||||
problems = []
|
||||
|
||||
for name in sorted(set(compose) - set(chart) - set(DEVIATIONS)):
|
||||
problems.append(f" {name}: in docker-compose.yml, not in the chart")
|
||||
for name in sorted(set(chart) - set(compose) - set(DEVIATIONS)):
|
||||
problems.append(f" {name}: in the chart, not in docker-compose.yml")
|
||||
for name in sorted(set(compose) & set(chart)):
|
||||
if compose[name] != chart[name]:
|
||||
problems.append(f" {name}: compose runs {compose[name]}, the chart runs {chart[name]}")
|
||||
|
||||
if problems:
|
||||
print("compose and the Helm chart describe different stacks:\n" + "\n".join(problems))
|
||||
print(
|
||||
"\nPort the change to the other stack, or — if the difference is forced by\n"
|
||||
"Kubernetes — declare it in DEVIATIONS in this file, with the reason."
|
||||
)
|
||||
return 1
|
||||
|
||||
print(f"no drift: {len(chart)} workloads, images identical on both stacks")
|
||||
for name, why in sorted(DEVIATIONS.items()):
|
||||
print(f" deviation (declared): {name} — {why}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,60 @@
|
||||
# Throwaway in-cluster OCI registry, published on NodePort 30500.
|
||||
#
|
||||
# Talos has no Docker daemon and no way to side-load an image, so the images built
|
||||
# from this repo must come from a registry. This one lives *inside* the cluster on
|
||||
# purpose: a registry on the laptop needs an inbound port opened on firewalld's
|
||||
# libvirt zone (root), while pushing from the laptop to the node is outbound and
|
||||
# always allowed. The node then pulls from its own NodePort.
|
||||
#
|
||||
# Talos must be told it speaks plain HTTP — see the machine.registries.mirrors
|
||||
# patch in docs/runbooks/kubernetes-talos.md. Storage is emptyDir: if this pod is
|
||||
# replaced, re-run `make k8s-images`.
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: registry
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: registry
|
||||
namespace: registry
|
||||
spec:
|
||||
replicas: 1
|
||||
strategy: { type: Recreate }
|
||||
selector:
|
||||
matchLabels: { app: registry }
|
||||
template:
|
||||
metadata:
|
||||
labels: { app: registry }
|
||||
spec:
|
||||
containers:
|
||||
- name: registry
|
||||
image: docker.io/library/registry:2
|
||||
env:
|
||||
- name: REGISTRY_STORAGE_DELETE_ENABLED
|
||||
value: "true"
|
||||
ports:
|
||||
- containerPort: 5000
|
||||
readinessProbe:
|
||||
httpGet: { path: /v2/, port: 5000 }
|
||||
volumeMounts:
|
||||
- name: data
|
||||
mountPath: /var/lib/registry
|
||||
volumes:
|
||||
- name: data
|
||||
emptyDir: {}
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: registry
|
||||
namespace: registry
|
||||
spec:
|
||||
type: NodePort
|
||||
selector: { app: registry }
|
||||
ports:
|
||||
- name: http
|
||||
port: 5000
|
||||
targetPort: 5000
|
||||
nodePort: 30500
|
||||
Executable
+46
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Turn the repo's config inputs into the ConfigMaps the Helm chart mounts.
|
||||
#
|
||||
# This is the Kubernetes sibling of infra/seed-config.sh: the upstream Common
|
||||
# Ground images are used verbatim and read their config from a mounted directory,
|
||||
# so the config has to be handed to the platform out-of-band. Compose gets it via
|
||||
# `docker cp` into external volumes; Kubernetes gets it as ConfigMaps created from
|
||||
# the files that already live in this repo. Copying those files into the chart
|
||||
# would fork them from the compose stack, so we don't.
|
||||
#
|
||||
# Idempotent: re-run after editing any data.yaml, then `make k8s-reseed`.
|
||||
#
|
||||
# Usage: seed-configmaps.sh [namespace] (default: big)
|
||||
set -euo pipefail
|
||||
|
||||
ns="${1:-big}"
|
||||
here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
repo="$(cd "$here/../.." && pwd)"
|
||||
|
||||
kubectl get namespace "$ns" >/dev/null 2>&1 || kubectl create namespace "$ns"
|
||||
|
||||
seed() { # name <kubectl --from-file args...>
|
||||
local name="$1"; shift
|
||||
kubectl create configmap "$name" -n "$ns" "$@" \
|
||||
--dry-run=client -o yaml | kubectl apply -f - >/dev/null
|
||||
echo " seeded configmap/$name"
|
||||
}
|
||||
|
||||
seed rr-oz-config --from-file="$repo/infra/openzaak/setup_configuration/"
|
||||
seed rr-nrc-config --from-file="$repo/infra/opennotificaties/setup_configuration/"
|
||||
seed rr-kc-realms --from-file="$repo/infra/keycloak/realms/"
|
||||
seed rr-objecttypen-config --from-file="$repo/infra/objecttypen/setup_configuration/"
|
||||
seed rr-objecten-config --from-file="$repo/infra/objecten/setup_configuration/"
|
||||
# register.py + the RegisterRecord JSON schema (the __pycache__ dir is skipped:
|
||||
# kubectl only takes regular files from a --from-file directory).
|
||||
seed rr-registerrecord-config --from-file="$repo/infra/objecttypen-registerrecord/"
|
||||
# The BPMN and the DMN are two separate Flowable deployments (S-13, ADR-0016).
|
||||
seed rr-fl-bpmn \
|
||||
--from-file="$repo/workflows/registratie.bpmn" \
|
||||
--from-file="$repo/workflows/diploma-eligibility.dmn"
|
||||
# The two bootstrap scripts the compose local stack runs as init containers
|
||||
# (S-B04, ADR-0020). Stdlib-only, so a plain python image can run them.
|
||||
seed rr-seed-scripts \
|
||||
--from-file="$repo/infra/openzaak/seed_catalogus.py" \
|
||||
--from-file="$repo/infra/local/register-abonnement.py"
|
||||
@@ -0,0 +1,20 @@
|
||||
# Overlay: make the CI compose stack usable from a HOST browser.
|
||||
# Same two mechanisms infra/docker-compose.local.yml already uses — pin Keycloak's issuer to the
|
||||
# host-published address, and point each portal's runtime config.json at it. The BFF needs no
|
||||
# change: it discovers metadata over keycloak:8080 and the discovered issuer is the pinned
|
||||
# localhost:8180, which is what browser tokens carry.
|
||||
services:
|
||||
keycloak:
|
||||
environment:
|
||||
KC_HOSTNAME: http://localhost:8180
|
||||
KC_HOSTNAME_BACKCHANNEL_DYNAMIC: "true"
|
||||
self-service:
|
||||
volumes:
|
||||
- ./local-config/self-service.config.json:/usr/share/caddy/config.json:ro,z
|
||||
behandel:
|
||||
volumes:
|
||||
- ./local-config/behandel.config.json:/usr/share/caddy/config.json:ro,z
|
||||
# beheer is the same medewerker realm as behandel, so it reuses behandel's config verbatim.
|
||||
beheer:
|
||||
volumes:
|
||||
- ./local-config/behandel.config.json:/usr/share/caddy/config.json:ro,z
|
||||
@@ -7,10 +7,34 @@ redirects it into $GITHUB_STEP_SUMMARY. Stdlib only.
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
STATUS_ICON = {"expected": "✅", "unexpected": "❌", "skipped": "⏭️", "flaky": "⚠️"}
|
||||
|
||||
# A verdict alone still costs a log dive, and a killed or truncated job leaves no log to dive into
|
||||
# (#161) — so a failing spec carries its first error into the table. Playwright errors are multi-line
|
||||
# with a "Call log:", which a markdown table cell cannot hold, so they are flattened and clipped.
|
||||
ERROR_CLIP = 300
|
||||
|
||||
|
||||
def first_error(spec):
|
||||
"""The first error message across a spec's test results, flattened for one table cell."""
|
||||
for test in spec.get("tests", []):
|
||||
for result in test.get("results", []):
|
||||
for error in result.get("errors", []):
|
||||
message = (error.get("message") or "").strip()
|
||||
if not message:
|
||||
continue
|
||||
# Strip ANSI colour, collapse to one line, and keep it inside the cell.
|
||||
message = re.sub(r"\x1b\[[0-9;]*m", "", message)
|
||||
message = " ".join(message.split())
|
||||
if len(message) > ERROR_CLIP:
|
||||
message = message[:ERROR_CLIP - 1].rstrip() + "…"
|
||||
# `|` would end the cell early.
|
||||
return message.replace("|", "\\|")
|
||||
return ""
|
||||
|
||||
|
||||
def walk(suite, out):
|
||||
for spec in suite.get("specs", []):
|
||||
@@ -22,7 +46,8 @@ def walk(suite, out):
|
||||
else "expected" if spec.get("ok", False)
|
||||
else "unexpected")
|
||||
out.append({"file": spec.get("file") or suite.get("file") or suite.get("title", ""),
|
||||
"title": spec.get("title", ""), "status": status})
|
||||
"title": spec.get("title", ""), "status": status,
|
||||
"error": first_error(spec) if status in ("unexpected", "flaky") else ""})
|
||||
for child in suite.get("suites", []):
|
||||
walk(child, out)
|
||||
|
||||
@@ -46,10 +71,17 @@ def main(path):
|
||||
if not specs:
|
||||
print("_No specs ran._")
|
||||
return 0
|
||||
print("| Spec | Result |")
|
||||
print("| ---- | :----: |")
|
||||
for s in specs:
|
||||
print(f"| {s['file']} › {s['title']} | {STATUS_ICON.get(s['status'], '❔')} |")
|
||||
# The failure column only earns its width when something failed.
|
||||
if any(s["error"] for s in specs):
|
||||
print("| Spec | Result | Why |")
|
||||
print("| ---- | :----: | --- |")
|
||||
for s in specs:
|
||||
print(f"| {s['file']} › {s['title']} | {STATUS_ICON.get(s['status'], '❔')} | {s['error']} |")
|
||||
else:
|
||||
print("| Spec | Result |")
|
||||
print("| ---- | :----: |")
|
||||
for s in specs:
|
||||
print(f"| {s['file']} › {s['title']} | {STATUS_ICON.get(s['status'], '❔')} |")
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Self-check for infra/playwright-summary.py — stdlib asserts, no framework.
|
||||
|
||||
Run: python3 infra/test_playwright_summary.py (also runs in `make unit`).
|
||||
|
||||
A red e2e is only useful if the job summary says WHY it failed: #161 lost a 36-minute
|
||||
verify-stack job whose only surviving output was one ✘ line with no assertion detail.
|
||||
"""
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from contextlib import redirect_stdout
|
||||
|
||||
# The script's filename is not a valid module name, so load it by path.
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"playwright_summary",
|
||||
os.path.join(os.path.dirname(os.path.abspath(__file__)), "playwright-summary.py"),
|
||||
)
|
||||
summary = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(summary)
|
||||
|
||||
|
||||
def render(report):
|
||||
"""Run the renderer over a report dict and return its markdown."""
|
||||
with tempfile.NamedTemporaryFile("w", suffix=".json", delete=False) as fh:
|
||||
json.dump(report, fh)
|
||||
path = fh.name
|
||||
try:
|
||||
out = io.StringIO()
|
||||
with redirect_stdout(out):
|
||||
summary.main(path)
|
||||
return out.getvalue()
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
||||
def spec_entry(title, status, errors=()):
|
||||
return {
|
||||
"title": title,
|
||||
"file": "catalogus.spec.ts",
|
||||
"ok": status == "expected",
|
||||
"tests": [{"status": status, "results": [{"errors": [{"message": m} for m in errors]}]}],
|
||||
}
|
||||
|
||||
|
||||
def test_failing_spec_reports_why():
|
||||
md = render({
|
||||
"stats": {"expected": 4, "unexpected": 1, "flaky": 0, "skipped": 0, "duration": 108_000},
|
||||
"suites": [{"file": "catalogus.spec.ts", "specs": [
|
||||
spec_entry("a beheerder sees the published zaaktypen in the catalogus", "unexpected",
|
||||
["locator.fill: Test timeout of 90000ms exceeded.\n"
|
||||
"Call log:\n - waiting for locator('#username')\n"]),
|
||||
]}],
|
||||
})
|
||||
assert "❌" in md, md
|
||||
# The point of the slice: the summary names the cause, not just the verdict.
|
||||
assert "Test timeout of 90000ms exceeded" in md, md
|
||||
assert "waiting for locator('#username')" in md, md
|
||||
# A multi-line Playwright error must not break out of its table row.
|
||||
assert not any(line.startswith("Call log:") for line in md.splitlines()), md
|
||||
|
||||
|
||||
def test_real_playwright_error_is_flattened():
|
||||
# A real report's message is multi-line and ANSI-coloured, and embeds the source snippet with
|
||||
# `|` gutters — all three would break the table cell. Shape verified against an actual
|
||||
# @playwright/test 1.61 JSON report.
|
||||
md = render({
|
||||
"stats": {"expected": 0, "unexpected": 1, "flaky": 0, "skipped": 0, "duration": 1_000},
|
||||
"suites": [{"file": "catalogus.spec.ts", "specs": [
|
||||
spec_entry("a beheerder sees the catalogus", "unexpected",
|
||||
["Error: expect(locator).toBeVisible() failed\n\n"
|
||||
"\x1b[2mLocator: \x1b[22mgetByRole('heading')\n"
|
||||
" 12 | await login(page);\n> 13 | await expect(heading).toBeVisible();\n"]),
|
||||
]}],
|
||||
})
|
||||
row = [line for line in md.splitlines() if line.startswith("| catalogus.spec.ts")][0]
|
||||
assert "\x1b" not in row, row
|
||||
assert "Locator: getByRole('heading')" in row, row
|
||||
# Every literal `|` from the snippet gutters is escaped, so the row keeps exactly 3 cells.
|
||||
assert row.count("|") - row.count("\\|") == 4, row
|
||||
|
||||
|
||||
def test_passing_run_stays_quiet():
|
||||
md = render({
|
||||
"stats": {"expected": 1, "unexpected": 0, "flaky": 0, "skipped": 0, "duration": 5_000},
|
||||
"suites": [{"file": "catalogus.spec.ts",
|
||||
"specs": [spec_entry("a beheerder sees the catalogus", "expected")]}],
|
||||
})
|
||||
assert "✅" in md, md
|
||||
assert "timeout" not in md.lower(), md
|
||||
|
||||
|
||||
def test_missing_report_is_not_a_crash():
|
||||
out = io.StringIO()
|
||||
with redirect_stdout(out):
|
||||
rc = summary.main("/nonexistent/playwright-report.json")
|
||||
assert rc == 0
|
||||
assert "did not reach the e2e step" in out.getvalue()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
for name, fn in sorted(globals().items()):
|
||||
if name.startswith("test_") and callable(fn):
|
||||
fn()
|
||||
print(f" ok {name}")
|
||||
print("playwright-summary self-check passed")
|
||||
@@ -0,0 +1,68 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Self-check for the portals' Caddyfiles — stdlib asserts, no framework.
|
||||
|
||||
Run: python3 infra/test_portal_caddyfiles.py (also runs in `make unit`).
|
||||
|
||||
Each portal serves its Angular app and reverse-proxies *its own* BFF endpoint group
|
||||
same-origin, so the browser never sees CORS and the DigiD token rides along (ADR-0010).
|
||||
The four files are near-identical, which makes a copy-paste slip cheap to introduce and
|
||||
expensive to find: proxying another portal's group hands a behandelaar's browser an
|
||||
endpoint its token isn't for, and the failure shows up as a 401 three services away.
|
||||
|
||||
What is asserted per portal: it proxies exactly its own groups to the BFF service, and it
|
||||
falls back to index.html so Angular's client-side routes survive a deep link / refresh.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
|
||||
APPS = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "apps")
|
||||
|
||||
# The self-service portal also renders the public register (S-09), so it proxies both.
|
||||
EXPECTED = {
|
||||
"self-service": {"/self-service/*", "/openbaar/*"},
|
||||
"openbaar": {"/openbaar/*"},
|
||||
"behandel": {"/behandel/*"},
|
||||
"beheer": {"/beheer/*"},
|
||||
}
|
||||
ALL_GROUPS = {g for groups in EXPECTED.values() for g in groups}
|
||||
|
||||
|
||||
def caddyfile(app):
|
||||
with open(os.path.join(APPS, app, "Caddyfile")) as fh:
|
||||
return fh.read()
|
||||
|
||||
|
||||
def proxied_groups(text):
|
||||
"""The path groups routed to the BFF: `handle <path> { reverse_proxy bff:8080 }`."""
|
||||
return {
|
||||
m.group(1)
|
||||
for m in re.finditer(r"handle\s+(\S+)\s*\{[^}]*reverse_proxy\s+bff:8080", text)
|
||||
}
|
||||
|
||||
|
||||
def test_each_portal_proxies_exactly_its_own_endpoint_groups():
|
||||
for app, expected in EXPECTED.items():
|
||||
got = proxied_groups(caddyfile(app))
|
||||
assert got == expected, f"{app}: proxies {got or '{}'}, expected {expected}"
|
||||
|
||||
|
||||
def test_no_portal_proxies_another_portals_group():
|
||||
for app, expected in EXPECTED.items():
|
||||
strays = proxied_groups(caddyfile(app)) & (ALL_GROUPS - expected)
|
||||
assert not strays, f"{app}: proxies another portal's group {strays}"
|
||||
|
||||
|
||||
def test_every_portal_falls_back_to_index_html():
|
||||
"""Angular routes client-side: an unknown path must serve the app, not a 404."""
|
||||
for app in EXPECTED:
|
||||
text = caddyfile(app)
|
||||
assert "try_files {path} /index.html" in text, f"{app}: no SPA fallback"
|
||||
assert "file_server" in text, f"{app}: nothing serves the built app"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
for name, fn in sorted(globals().items()):
|
||||
if name.startswith("test_") and callable(fn):
|
||||
fn()
|
||||
print(f" ok {name}")
|
||||
print("portal Caddyfile self-check passed")
|
||||
@@ -15,7 +15,7 @@ export interface DigiadAuthOptions {
|
||||
redirectUrl: string;
|
||||
/**
|
||||
* Route prefixes whose requests get the bearer token attached. The api-client calls the BFF with
|
||||
* **relative** URLs (same-origin via the nginx proxy), so these must be relative path prefixes
|
||||
* **relative** URLs (same-origin via the Caddy proxy), so these must be relative path prefixes
|
||||
* (e.g. `/self-service/`) — angular-auth-oidc-client matches `req.url.startsWith(route)`, and a
|
||||
* relative `req.url` never starts with an absolute origin.
|
||||
*/
|
||||
|
||||
@@ -10,7 +10,7 @@ export interface MedewerkerAuthOptions {
|
||||
redirectUrl: string;
|
||||
/**
|
||||
* Route prefixes whose requests get the bearer token attached. The api-client calls the BFF with
|
||||
* **relative** URLs (same-origin via the nginx proxy), so these must be relative path prefixes
|
||||
* **relative** URLs (same-origin via the Caddy proxy), so these must be relative path prefixes
|
||||
* (e.g. `/behandel/`) — angular-auth-oidc-client matches `req.url.startsWith(route)`, and a
|
||||
* relative `req.url` never starts with an absolute origin.
|
||||
*/
|
||||
|
||||
+31
@@ -32,6 +32,30 @@ nav:
|
||||
- "ADR-0008: Read projection store": architecture/adr-0008-read-projection-store.md
|
||||
- "ADR-0009: External-task job worker": architecture/adr-0009-external-task-job-worker.md
|
||||
- "ADR-0010: BFF OIDC validation": architecture/adr-0010-bff-oidc.md
|
||||
- "ADR-0011: Approval status flow": architecture/adr-0011-approval-status-flow.md
|
||||
- "ADR-0012: Citizen reference correlation": architecture/adr-0012-citizen-reference-correlation.md
|
||||
- "ADR-0013: Behandel-portal wiring": architecture/adr-0013-behandel-portal-wiring.md
|
||||
- "ADR-0014: Withdrawal cancels the process": architecture/adr-0014-withdrawal-cancels-the-process.md
|
||||
- "ADR-0015: Beoordeling escalation": architecture/adr-0015-beoordeling-escalation.md
|
||||
- "ADR-0016: Diploma eligibility DMN": architecture/adr-0016-diploma-eligibility-dmn.md
|
||||
- "ADR-0017: Document-wait timeout": architecture/adr-0017-document-wait-timeout-cancellation.md
|
||||
- "ADR-0018: Diploma upload via the ACL": architecture/adr-0018-diploma-upload-via-acl-documenten.md
|
||||
- "ADR-0019: Zaak cancellation on timeout": architecture/adr-0019-zaak-cancellation-on-timeout.md
|
||||
- "ADR-0020: Local stack self-seeds": architecture/adr-0020-local-stack-self-seeds.md
|
||||
- "ADR-0021: Zaaktype by identificatie": architecture/adr-0021-acl-resolves-zaaktype-by-identificatie.md
|
||||
- "ADR-0022: Quartz scheduler": architecture/adr-0022-quartz-scheduler.md
|
||||
- "ADR-0023: Observability stack": architecture/adr-0023-observability-stack.md
|
||||
- "ADR-0024: Prometheus AspNetCore exporter": architecture/adr-0024-prometheus-aspnetcore-exporter.md
|
||||
- "ADR-0025: BFF reads catalogus via the ACL": architecture/adr-0025-bff-reads-catalogus-via-acl.md
|
||||
- "ADR-0026: Mutable default-fill store": architecture/adr-0026-mutable-default-fill-store.md
|
||||
- "ADR-0027: RegisterRecord objecttype": architecture/adr-0027-registerrecord-objecttype-schema.md
|
||||
- "ADR-0028: Objecten holds the register": architecture/adr-0028-objecten-holds-the-register.md
|
||||
- "ADR-0029: Objecten publishes to NRC": architecture/adr-0029-objecten-publishes-to-nrc.md
|
||||
- "ADR-0030: Projection sourced from the register": architecture/adr-0030-projection-sourced-from-the-register.md
|
||||
- "ADR-0031: MFA on the medewerker realm": architecture/adr-0031-mfa-on-the-medewerker-realm.md
|
||||
- "ADR-0032: Werkbak live refresh": architecture/adr-0032-werkbak-live-refresh.md
|
||||
- "ADR-0033: Kubernetes via one Helm chart": architecture/adr-0033-kubernetes-via-one-helm-chart.md
|
||||
- "ADR-0034: Caddy serves the portals": architecture/adr-0034-caddy-serves-the-portals.md
|
||||
- FDS-architectuur:
|
||||
- Overzicht: architecture/fds/README.md
|
||||
- Componentview (L3): architecture/fds/c4-component-view.md
|
||||
@@ -46,8 +70,15 @@ nav:
|
||||
- Working in Gitea: gitea-workflow.md
|
||||
- Frontend decisions: frontend-decisions.md
|
||||
- Demo script: demo-script.md
|
||||
- Synthetic data: synthetic-data.md
|
||||
- Runbooks:
|
||||
- CI: runbooks/ci.md
|
||||
- OpenZaak: runbooks/openzaak.md
|
||||
- Open Notificaties (NRC): runbooks/opennotificaties.md
|
||||
- Keycloak: runbooks/keycloak.md
|
||||
- Flowable: runbooks/flowable.md
|
||||
- Kubernetes on Talos: runbooks/kubernetes-talos.md
|
||||
- Gitea Actions gotchas: runbooks/gitea-actions-gotchas.md
|
||||
|
||||
markdown_extensions:
|
||||
- admonition
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { expect, test } from '@playwright/test';
|
||||
import { loginMedewerker } from './medewerker-login';
|
||||
import { loginMedewerker } from './keycloak-login';
|
||||
|
||||
// S-15a walking skeleton: a beheerder logs in to the beheer portal (medewerker realm) and sees the
|
||||
// read-only ZTC catalogus. The verify stack seeds and publishes the BIG-REGISTRATIE zaaktype (the
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { expect, test } from '@playwright/test';
|
||||
import { loginMedewerker } from './medewerker-login';
|
||||
import { loginMedewerker } from './keycloak-login';
|
||||
|
||||
// S-15b: a beheerder edits the ACL default-fill in the beheer portal and gets a saved confirmation.
|
||||
// Runs against the shared verify stack; it edits + saves (the ACL store is in-memory, ADR-0026) and
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { expect, test } from '@playwright/test';
|
||||
import { OTP_PERIOD_MS, nextUnusedCounter } from './medewerker-login';
|
||||
import { OTP_PERIOD_MS, nextUnusedCounter } from './keycloak-login';
|
||||
|
||||
// Pure check of the TOTP counter guard in loginMedewerker — no browser, no stack. Keycloak refuses
|
||||
// a code it has already accepted (its otpPolicyCodeReusable defaults to false), so two logins as
|
||||
@@ -0,0 +1,99 @@
|
||||
import { createHmac } from 'node:crypto';
|
||||
import { readFileSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import { expect, type Page } from '@playwright/test';
|
||||
|
||||
// Every portal login in the suite goes through this module — citizen realms (mock DigiD) and the
|
||||
// medewerker realm alike — so the shared Keycloak form handling lives in exactly one place.
|
||||
|
||||
// The medewerker realm enforces MFA (S-15c), so a staff login is two steps: password, then a TOTP
|
||||
// code. The realm export seeds every medewerker with this fixture secret — Keycloak HMACs the raw
|
||||
// secret bytes — so the e2e can compute a valid code instead of enrolling an authenticator.
|
||||
const OTP_SECRET = 'BIGMEDEWERKEROTPSEED';
|
||||
|
||||
export const OTP_PERIOD_MS = 30_000;
|
||||
|
||||
/**
|
||||
* How long a Keycloak form gets to appear. Generous enough for a cold first browser launch and a
|
||||
* loaded stack, far short of the 90-second test timeout an auto-waiting action would otherwise eat.
|
||||
*/
|
||||
const FORM_TIMEOUT_MS = 20_000;
|
||||
const FORM_NEVER_APPEARED =
|
||||
'the Keycloak login form never appeared — the portal did not reach Keycloak (check its ' +
|
||||
'config.json fetch and the OIDC discovery on the authority it was built with)';
|
||||
const OTP_NEVER_APPEARED =
|
||||
'the Keycloak OTP form never appeared — the password step did not complete (check the ' +
|
||||
'medewerker realm seeded this user with both a password and a TOTP credential)';
|
||||
|
||||
// RFC 6238 TOTP: HMAC-SHA1 over the 30-second counter, dynamically truncated to 6 digits.
|
||||
export function totp(secret = OTP_SECRET, at = Date.now()): string {
|
||||
const counter = Buffer.alloc(8);
|
||||
counter.writeBigUInt64BE(BigInt(Math.floor(at / OTP_PERIOD_MS)));
|
||||
const mac = createHmac('sha1', secret).update(counter).digest();
|
||||
const offset = mac[mac.length - 1] & 0x0f;
|
||||
return String((mac.readUInt32BE(offset) & 0x7fffffff) % 1_000_000).padStart(6, '0');
|
||||
}
|
||||
|
||||
// Keycloak refuses a TOTP code it has already accepted (its otpPolicyCodeReusable defaults to
|
||||
// false), so two logins as the same medewerker inside one 30-second window would both submit the
|
||||
// same code and the second is rejected. Spend the first counter this medewerker has left.
|
||||
export function nextUnusedCounter(now: number, spent: number): number {
|
||||
return Math.max(Math.floor(now / OTP_PERIOD_MS), spent + 1);
|
||||
}
|
||||
|
||||
// The spent counter lives on disk rather than in module state: Playwright starts a fresh worker
|
||||
// process for a retry, which would otherwise forget it and resubmit the rejected code.
|
||||
function spendCounter(username: string): number {
|
||||
const file = join(tmpdir(), `otp-counter-${username}`);
|
||||
let spent = -1;
|
||||
try {
|
||||
spent = Number(readFileSync(file, 'utf8')) || -1;
|
||||
} catch {
|
||||
// first login as this medewerker in this run
|
||||
}
|
||||
const counter = nextUnusedCounter(Date.now(), spent);
|
||||
writeFileSync(file, String(counter));
|
||||
return counter;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fill Keycloak's login form. Every portal is guarded, so the first navigation redirects here; the
|
||||
* form ids are stable across themes.
|
||||
*
|
||||
* The form is asserted visible *before* it is filled. A portal that never reaches Keycloak — its
|
||||
* runtime `config.json` fetch or the OIDC discovery behind `authorize()` failed, so it never
|
||||
* bootstrapped and shows a blank page (main.ts only logs to the console) — would otherwise leave
|
||||
* `fill()` auto-waiting until the whole test times out: 90 seconds spent to report
|
||||
* `locator.fill: Test timeout of 90000ms exceeded`, naming the symptom and not the cause. That is
|
||||
* how #161's catalogus.spec burned 1.8 minutes. This fails in a quarter of the time and says which
|
||||
* step never happened.
|
||||
*/
|
||||
async function submitPassword(page: Page, username: string): Promise<void> {
|
||||
await expect(page.locator('#username'), FORM_NEVER_APPEARED).toBeVisible({ timeout: FORM_TIMEOUT_MS });
|
||||
await page.locator('#username').fill(username);
|
||||
await page.locator('#password').fill('test123');
|
||||
await page.locator('#kc-login').click();
|
||||
}
|
||||
|
||||
/** A citizen login on a mock-DigiD realm — no second factor (ADR-0031). */
|
||||
export async function loginBurger(page: Page, username: string): Promise<void> {
|
||||
await submitPassword(page, username);
|
||||
}
|
||||
|
||||
/** A staff login on the medewerker realm: password, then the enforced TOTP second factor. */
|
||||
export async function loginMedewerker(page: Page, username: string): Promise<void> {
|
||||
await submitPassword(page, username);
|
||||
|
||||
// Keycloak's conditional-OTP step. Same reasoning as the password form above: assert it arrived
|
||||
// rather than letting `fill()` swallow the test timeout.
|
||||
await expect(page.locator('#otp'), OTP_NEVER_APPEARED).toBeVisible({ timeout: FORM_TIMEOUT_MS });
|
||||
|
||||
// Wait out the rest of the window if the counter we may spend is still in the future; Keycloak's
|
||||
// lookAheadWindow would accept the code a moment early, but only by one counter — waiting keeps a
|
||||
// third login in the same window valid too.
|
||||
const counter = spendCounter(username);
|
||||
await page.waitForTimeout(Math.max(0, counter * OTP_PERIOD_MS - Date.now()));
|
||||
await page.locator('#otp').fill(totp(OTP_SECRET, counter * OTP_PERIOD_MS));
|
||||
await page.locator('#kc-login').click();
|
||||
}
|
||||
@@ -1,57 +0,0 @@
|
||||
import { createHmac } from 'node:crypto';
|
||||
import { readFileSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import { join } from 'node:path';
|
||||
import type { Page } from '@playwright/test';
|
||||
|
||||
// The medewerker realm enforces MFA (S-15c), so a staff login is two steps: password, then a TOTP
|
||||
// code. The realm export seeds every medewerker with this fixture secret — Keycloak HMACs the raw
|
||||
// secret bytes — so the e2e can compute a valid code instead of enrolling an authenticator.
|
||||
const OTP_SECRET = 'BIGMEDEWERKEROTPSEED';
|
||||
|
||||
export const OTP_PERIOD_MS = 30_000;
|
||||
|
||||
// RFC 6238 TOTP: HMAC-SHA1 over the 30-second counter, dynamically truncated to 6 digits.
|
||||
export function totp(secret = OTP_SECRET, at = Date.now()): string {
|
||||
const counter = Buffer.alloc(8);
|
||||
counter.writeBigUInt64BE(BigInt(Math.floor(at / OTP_PERIOD_MS)));
|
||||
const mac = createHmac('sha1', secret).update(counter).digest();
|
||||
const offset = mac[mac.length - 1] & 0x0f;
|
||||
return String((mac.readUInt32BE(offset) & 0x7fffffff) % 1_000_000).padStart(6, '0');
|
||||
}
|
||||
|
||||
// Keycloak refuses a TOTP code it has already accepted (its otpPolicyCodeReusable defaults to
|
||||
// false), so two logins as the same medewerker inside one 30-second window would both submit the
|
||||
// same code and the second is rejected. Spend the first counter this medewerker has left.
|
||||
export function nextUnusedCounter(now: number, spent: number): number {
|
||||
return Math.max(Math.floor(now / OTP_PERIOD_MS), spent + 1);
|
||||
}
|
||||
|
||||
// The spent counter lives on disk rather than in module state: Playwright starts a fresh worker
|
||||
// process for a retry, which would otherwise forget it and resubmit the rejected code.
|
||||
function spendCounter(username: string): number {
|
||||
const file = join(tmpdir(), `otp-counter-${username}`);
|
||||
let spent = -1;
|
||||
try {
|
||||
spent = Number(readFileSync(file, 'utf8')) || -1;
|
||||
} catch {
|
||||
// first login as this medewerker in this run
|
||||
}
|
||||
const counter = nextUnusedCounter(Date.now(), spent);
|
||||
writeFileSync(file, String(counter));
|
||||
return counter;
|
||||
}
|
||||
|
||||
export async function loginMedewerker(page: Page, username: string): Promise<void> {
|
||||
await page.locator('#username').fill(username);
|
||||
await page.locator('#password').fill('test123');
|
||||
await page.locator('#kc-login').click();
|
||||
|
||||
// Keycloak's conditional-OTP step. Wait out the rest of the window if the counter we may spend is
|
||||
// still in the future; its lookAheadWindow would accept the code a moment early, but only by one
|
||||
// counter — waiting keeps a third login in the same window valid too.
|
||||
const counter = spendCounter(username);
|
||||
await page.waitForTimeout(Math.max(0, counter * OTP_PERIOD_MS - Date.now()));
|
||||
await page.locator('#otp').fill(totp(OTP_SECRET, counter * OTP_PERIOD_MS));
|
||||
await page.locator('#kc-login').click();
|
||||
}
|
||||
@@ -15,6 +15,12 @@ export default defineConfig({
|
||||
timeout: 90_000,
|
||||
expect: { timeout: 15_000 },
|
||||
retries: 1,
|
||||
// Bound the whole run, not just each test (#161). A wedged suite used to run until CI killed the
|
||||
// job — which also killed the `if: always()` steps that would have said why: the per-spec summary
|
||||
// and the container-log dump never ran, leaving a 36-minute job whose entire surviving output was
|
||||
// one ✘ line. On `globalTimeout` Playwright stops and *reports*, so the JSON report is written and
|
||||
// those steps still run. Generous over the ~1-minute suite: this is a backstop, not a budget.
|
||||
globalTimeout: 12 * 60_000,
|
||||
// Run the specs serially. Each spec drives a full `channel: 'chromium'` browser, and the e2e
|
||||
// shares an 8 GB runner with the entire compose stack (OpenZaak, NRC, Keycloak, Flowable, 4×
|
||||
// Postgres, every service + 3 portals). Two parallel browsers exhaust memory and the renderer is
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { expect, request, test } from '@playwright/test';
|
||||
import { loginMedewerker } from './medewerker-login';
|
||||
import { loginBurger, loginMedewerker } from './keycloak-login';
|
||||
|
||||
// Walking-skeleton happy path (S-08d + S-09 + S-09b + S-12 + S-10a + S-19b-2): a zorgprofessional
|
||||
// logs in via mock DigiD and submits through the self-service portal → BFF → domain; the entry
|
||||
@@ -23,9 +23,7 @@ test('DigiD submit → public INGEDIEND → documenten → behandelaar goedkeurt
|
||||
// checks submit as jan-burger (bsn 123456782) before the e2e runs on the shared stack, and
|
||||
// resume-on-load (S-26) would otherwise restore one of those on login — so each self-service spec
|
||||
// uses a dedicated citizen no other actor touches.
|
||||
await page.locator('#username').fill('emma-burger');
|
||||
await page.locator('#password').fill('test123');
|
||||
await page.locator('#kc-login').click();
|
||||
await loginBurger(page, 'emma-burger');
|
||||
|
||||
// Back on the portal, authenticated.
|
||||
await expect(page.getByRole('heading', { name: /Zelfservice/i })).toBeVisible();
|
||||
@@ -100,7 +98,7 @@ test('DigiD submit → public INGEDIEND → documenten → behandelaar goedkeurt
|
||||
await expect(goedkeuren).toBeVisible({ timeout: 30_000 });
|
||||
|
||||
// Click and wait for the decide POST to finish (204) BEFORE leaving the page. `click()` only
|
||||
// dispatches the request; navigating away immediately cancels it in flight (nginx logs a 499) and
|
||||
// dispatches the request; navigating away immediately cancels it in flight (the proxy logs a client-cancelled request) and
|
||||
// the decision never reaches the domain — so the registration would stay INGEDIEND.
|
||||
const decided = staff.waitForResponse(
|
||||
(r) =>
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { expect, test } from '@playwright/test';
|
||||
import { loginBurger } from './keycloak-login';
|
||||
|
||||
// S-26: a zorgprofessional submits, then reloads the self-service portal. On load the portal asks the
|
||||
// BFF for the caller's current open registration (owner-scoped by the DigiD token's bsn) and restores
|
||||
@@ -9,9 +10,7 @@ test('DigiD submit → reload → self-service restores the existing registratio
|
||||
// Its own DigiD user (like every self-service spec): on the shared verify stack, resume-on-load
|
||||
// (S-26) restores any open registration for the bsn, so each spec uses a dedicated citizen that no
|
||||
// other spec or verify-* check touches. This one in particular leaves an open registration.
|
||||
await page.locator('#username').fill('sanne-burger');
|
||||
await page.locator('#password').fill('test123');
|
||||
await page.locator('#kc-login').click();
|
||||
await loginBurger(page, 'sanne-burger');
|
||||
|
||||
await expect(page.getByRole('heading', { name: /Zelfservice/i })).toBeVisible();
|
||||
await page.getByRole('button', { name: /indienen/i }).click();
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { expect, test } from '@playwright/test';
|
||||
import { loginBurger } from './keycloak-login';
|
||||
|
||||
// S-11 (Flow 3): a zorgprofessional logs in via mock DigiD, submits a registration, then withdraws
|
||||
// it ("trek aanvraag in") from the self-service portal. The withdrawal goes portal → BFF (owner-
|
||||
@@ -10,9 +11,7 @@ test('DigiD submit → trek aanvraag in → self-service confirms ingetrokken',
|
||||
|
||||
// Its own DigiD user — isolated from the verify-* checks (jan-burger/123456782) so resume-on-load
|
||||
// (S-26) can't restore someone else's registration on the shared stack.
|
||||
await page.locator('#username').fill('lars-burger');
|
||||
await page.locator('#password').fill('test123');
|
||||
await page.locator('#kc-login').click();
|
||||
await loginBurger(page, 'lars-burger');
|
||||
|
||||
await expect(page.getByRole('heading', { name: /Zelfservice/i })).toBeVisible();
|
||||
|
||||
|
||||
Reference in New Issue
Block a user