Compare commits
4
Commits
main
..
421597851a
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
421597851a | ||
|
|
70de3d0a4d | ||
|
|
fddf14e5f9 | ||
|
|
5000b749b9 |
@@ -1,118 +0,0 @@
|
|||||||
name: Deploy to Talos
|
|
||||||
|
|
||||||
# A merge to main ships the stack to the Talos cluster on the lab server
|
|
||||||
# (docs/runbooks/kubernetes-talos.md §9). PR CI is the merge gate, so main is
|
|
||||||
# green by construction — this workflow only deploys.
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches: [main]
|
|
||||||
workflow_dispatch:
|
|
||||||
|
|
||||||
permissions:
|
|
||||||
contents: read
|
|
||||||
|
|
||||||
# Queue deploys, never cancel one: a helm upgrade killed half-way leaves the
|
|
||||||
# release in `pending-upgrade` and the next run has to be unwedged by hand.
|
|
||||||
concurrency:
|
|
||||||
group: deploy-talos
|
|
||||||
cancel-in-progress: false
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
deploy:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
env:
|
|
||||||
# The Talos VM as seen from the Fedora host (libvirt guest IP), and the
|
|
||||||
# address a browser uses to reach the cluster. `localhost` is deliberate:
|
|
||||||
# the portals' PKCE needs a secure context, so they are reached over
|
|
||||||
# `kubectl port-forward` — runbook §5. Override with repo variables.
|
|
||||||
TALOS_VM_IP: ${{ vars.TALOS_VM_IP }}
|
|
||||||
TALOS_HOST: ${{ vars.TALOS_HOST }}
|
|
||||||
# Set it when the labs Caddy publishes the portals: Keycloak's public https
|
|
||||||
# origin, e.g. https://big-auth.labs.respellion.tech (runbook, "Publishing
|
|
||||||
# through the labs Caddy").
|
|
||||||
KEYCLOAK_URL: ${{ vars.KEYCLOAK_URL }}
|
|
||||||
steps:
|
|
||||||
- uses: https://github.com/actions/checkout@v4
|
|
||||||
|
|
||||||
# Pinned static binaries, the same URLs the Talos runbook §0 gives a
|
|
||||||
# developer and the same helm the `k8s` CI job uses — no action to vet.
|
|
||||||
- name: Install kubectl, helm and crane
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
bin="$HOME/.local/bin"; mkdir -p "$bin"
|
|
||||||
curl -sSLo "$bin/kubectl" https://dl.k8s.io/release/v1.37.0/bin/linux/amd64/kubectl
|
|
||||||
curl -sSL https://get.helm.sh/helm-v3.16.4-linux-amd64.tar.gz | tar xz -O linux-amd64/helm > "$bin/helm"
|
|
||||||
curl -sSL https://github.com/google/go-containerregistry/releases/download/v0.20.2/go-containerregistry_Linux_x86_64.tar.gz | tar xz -O crane > "$bin/crane"
|
|
||||||
chmod +x "$bin"/{kubectl,helm,crane}
|
|
||||||
echo "$bin" >> "$GITHUB_PATH"
|
|
||||||
|
|
||||||
# The cluster's API and its registry are only reachable through the Fedora
|
|
||||||
# host, so forward both to the runner. 30141 is the openbaar portal, for
|
|
||||||
# the smoke at the end.
|
|
||||||
- name: Tunnel the Talos API + registry through the Fedora host
|
|
||||||
env:
|
|
||||||
SSH_KEY: ${{ secrets.TALOS_SSH_KEY }}
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
: "${TALOS_VM_IP:=192.168.122.173}"
|
|
||||||
umask 077
|
|
||||||
printf '%s\n' "$SSH_KEY" > ~/.ssh_talos
|
|
||||||
ssh -i ~/.ssh_talos -o StrictHostKeyChecking=no -o IdentitiesOnly=yes \
|
|
||||||
-o ExitOnForwardFailure=yes -p 6667 -f -N \
|
|
||||||
-L 6443:$TALOS_VM_IP:6443 \
|
|
||||||
-L 30500:$TALOS_VM_IP:30500 \
|
|
||||||
-L 30141:$TALOS_VM_IP:30141 \
|
|
||||||
user@labs.respellion.tech
|
|
||||||
|
|
||||||
# The kubeconfig's server must be https://127.0.0.1:6443 — Talos puts
|
|
||||||
# 127.0.0.1 in the apiserver cert SANs, so TLS verification still holds
|
|
||||||
# through the tunnel.
|
|
||||||
- name: Write the kubeconfig
|
|
||||||
env:
|
|
||||||
KUBECONFIG_B64: ${{ secrets.TALOS_KUBECONFIG }}
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
umask 077
|
|
||||||
base64 -d <<< "$KUBECONFIG_B64" > "$RUNNER_TEMP/kubeconfig"
|
|
||||||
echo "KUBECONFIG=$RUNNER_TEMP/kubeconfig" >> "$GITHUB_ENV"
|
|
||||||
kubectl --kubeconfig "$RUNNER_TEMP/kubeconfig" get nodes
|
|
||||||
|
|
||||||
# Idempotent; also makes a first deploy onto a bare cluster work. The
|
|
||||||
# registry's storage is an emptyDir, so a replaced pod loses the images —
|
|
||||||
# which the push in the next step puts back anyway.
|
|
||||||
- name: Ensure the in-cluster registry
|
|
||||||
run: make k8s-registry
|
|
||||||
|
|
||||||
# Push through the tunnel (localhost), pull from the node's own NodePort
|
|
||||||
# (the address in the Talos registry-mirror patch) — same registry, two
|
|
||||||
# names, so the two `make` calls get different K8S_REGISTRY values.
|
|
||||||
- name: Build and push the images
|
|
||||||
run: make k8s-images K8S_REGISTRY=localhost:30500
|
|
||||||
|
|
||||||
# k8s-reseed = seed configmaps + helm upgrade + re-run the bootstrap jobs.
|
|
||||||
# The jobs are idempotent, and deleting them first is what keeps a changed
|
|
||||||
# Job template from wedging the upgrade (`cannot patch … with kind Job`).
|
|
||||||
- name: Deploy the chart
|
|
||||||
run: |
|
|
||||||
make k8s-reseed \
|
|
||||||
TALOS_HOST=${TALOS_HOST:-localhost} \
|
|
||||||
K8S_REGISTRY=${TALOS_VM_IP:-192.168.122.173}:30500 \
|
|
||||||
K8S_SET="${KEYCLOAK_URL:+--set keycloakUrl=$KEYCLOAK_URL}"
|
|
||||||
|
|
||||||
# `dev` is a mutable tag and helm sees an unchanged pod template, so the
|
|
||||||
# new images only land on a restart (pullPolicy is already Always).
|
|
||||||
- name: Roll the services onto the new images
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
svcs="acl domain bff event-subscriber projection-api self-service openbaar behandel beheer"
|
|
||||||
kubectl -n big rollout restart deploy $svcs
|
|
||||||
kubectl -n big rollout status --timeout=300s deploy $svcs
|
|
||||||
|
|
||||||
# Proves portal → Caddy → BFF → projection end to end. An empty register is
|
|
||||||
# a pass; a 502 or a timeout is not.
|
|
||||||
- name: Smoke the public register
|
|
||||||
run: curl -fsS --retry 10 --retry-delay 6 --retry-all-errors http://localhost:30141/openbaar/register
|
|
||||||
|
|
||||||
- name: Pods on failure
|
|
||||||
if: failure()
|
|
||||||
run: kubectl -n big get pods,jobs || true
|
|
||||||
@@ -31,10 +31,6 @@ and CI cannot drift:
|
|||||||
> services by **container IP** (the runner can't reach published ports — see
|
> services by **container IP** (the runner can't reach published ports — see
|
||||||
> [gitea-actions-gotchas.md §5/§6](gitea-actions-gotchas.md)).
|
> [gitea-actions-gotchas.md §5/§6](gitea-actions-gotchas.md)).
|
||||||
|
|
||||||
A second workflow, `.gitea/workflows/deploy.yaml`, deploys the stack to the Talos
|
|
||||||
cluster on the lab server when a PR is merged to `main` — see
|
|
||||||
[kubernetes-talos.md §9](kubernetes-talos.md) for its secrets and the SSH tunnel it needs.
|
|
||||||
|
|
||||||
All `uses:` references are absolute, tag-pinned URLs (`https://github.com/actions/checkout@v4`,
|
All `uses:` references are absolute, tag-pinned URLs (`https://github.com/actions/checkout@v4`,
|
||||||
`https://github.com/actions/setup-dotnet@v4`) per CLAUDE.md §8.7 and §15 — Gitea
|
`https://github.com/actions/setup-dotnet@v4`) per CLAUDE.md §8.7 and §15 — Gitea
|
||||||
Actions resolves them from GitHub.
|
Actions resolves them from GitHub.
|
||||||
|
|||||||
@@ -360,83 +360,6 @@ immutable, so `helm upgrade` is rejected with `cannot patch "…" with kind Job`
|
|||||||
| Pods `Evicted` / `OOMKilled` | the VM is too small (§0) |
|
| Pods `Evicted` / `OOMKilled` | the VM is too small (§0) |
|
||||||
| A Job shows `BackoffLimitExceeded` | read it: `kubectl -n big logs job/<name>` |
|
| A Job shows `BackoffLimitExceeded` | read it: `kubectl -n big logs job/<name>` |
|
||||||
|
|
||||||
## 9. Deploying on merge to main
|
|
||||||
|
|
||||||
`.gitea/workflows/deploy.yaml` runs the §3–§4 steps against the **lab server's** Talos VM
|
|
||||||
every time a PR is squash-merged to `main` (and on demand via *Run workflow*). PR CI is the
|
|
||||||
merge gate, so the workflow deploys without re-running the checks.
|
|
||||||
|
|
||||||
The cluster's API and registry are not exposed publicly, so the job forwards them over the
|
|
||||||
same SSH hop the Gitea-runner pipeline uses:
|
|
||||||
|
|
||||||
```
|
|
||||||
ssh -p 6667 user@labs.respellion.tech -L 6443 -L 30500 -L 30141 → <TALOS_VM_IP>
|
|
||||||
```
|
|
||||||
|
|
||||||
Consequences worth knowing:
|
|
||||||
|
|
||||||
- Images are **pushed** to `localhost:30500` (the tunnel) and **pulled** by the node from
|
|
||||||
`<TALOS_VM_IP>:30500` (its own NodePort, the address in the Talos registry-mirror patch).
|
|
||||||
Same registry, two names — hence the two `K8S_REGISTRY` values in the workflow.
|
|
||||||
- It calls `make k8s-reseed`, not `make k8s-up`: the bootstrap Jobs are idempotent, and
|
|
||||||
deleting them first is what stops a changed Job template from wedging `helm upgrade` (§7).
|
|
||||||
- `dev` is a mutable tag, so a `rollout restart` of the nine repo deployments is what
|
|
||||||
actually puts the new images in the pods.
|
|
||||||
- Deploys **queue** (`cancel-in-progress: false`): a helm upgrade killed half-way leaves the
|
|
||||||
release in `pending-upgrade`, which has to be unwedged by hand.
|
|
||||||
|
|
||||||
Settings, all on the repository in Gitea:
|
|
||||||
|
|
||||||
| Kind | Name | What |
|
|
||||||
|---|---|---|
|
|
||||||
| Secret | `TALOS_SSH_KEY` | private key for `user@labs.respellion.tech` (the Fedora host) |
|
|
||||||
| Secret | `TALOS_KUBECONFIG` | base64 of the kubeconfig, **`server: https://127.0.0.1:6443`** — Talos puts `127.0.0.1` in the apiserver cert SANs, so TLS still verifies through the tunnel |
|
|
||||||
| Variable | `TALOS_VM_IP` | the VM's libvirt address (default `192.168.122.173`) |
|
|
||||||
| Variable | `TALOS_HOST` | the browser-facing host baked into Keycloak's issuer (default `localhost`, see §5) |
|
|
||||||
|
|
||||||
The last step smokes `GET /openbaar/register` through the openbaar portal, which exercises
|
|
||||||
portal → Caddy → BFF → projection. An empty register passes; a 502 does not.
|
|
||||||
|
|
||||||
Not covered: the portals still need `make k8s-portals` (or an SSH forward) to be usable in a
|
|
||||||
browser, because PKCE needs a secure context (§5). Giving the server a hostname + TLS is the
|
|
||||||
upgrade path.
|
|
||||||
|
|
||||||
## Publishing through the labs Caddy
|
|
||||||
|
|
||||||
The portals can be reached on real hostnames through the Caddy that already fronts
|
|
||||||
`*.labs.respellion.tech` (repo `Infra`, `infra/development/`). The chain:
|
|
||||||
|
|
||||||
```
|
|
||||||
browser → Caddy (labs server, TLS) → openssh-server:3014x/30180
|
|
||||||
→ reverse SSH tunnel → Fedora host → <TALOS_VM_IP>:3014x/30180 (NodePorts)
|
|
||||||
```
|
|
||||||
|
|
||||||
| URL | NodePort |
|
|
||||||
|---|---|
|
|
||||||
| `https://big-register.labs.respellion.tech` | 30141 openbaar |
|
|
||||||
| `https://big-mijn.labs.respellion.tech` | 30140 self-service |
|
|
||||||
| `https://big-behandel.labs.respellion.tech` | 30142 behandel |
|
|
||||||
| `https://big-beheer.labs.respellion.tech` | 30143 beheer |
|
|
||||||
| `https://big-auth.labs.respellion.tech` | 30180 Keycloak (`/admin` blocked) |
|
|
||||||
|
|
||||||
HTTPS makes the portals a secure context, so PKCE works without port-forwards — but
|
|
||||||
Keycloak's issuer must be the public origin. Deploy with it:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
make k8s-up TALOS_HOST=localhost K8S_REGISTRY=<TALOS_HOST>:30500 \
|
|
||||||
K8S_SET="--set keycloakUrl=https://big-auth.labs.respellion.tech"
|
|
||||||
```
|
|
||||||
|
|
||||||
For deploy-on-merge, set the repository variable `KEYCLOAK_URL` to the same value.
|
|
||||||
With it set, the `localhost` port-forwards (§5) no longer log in: the issuer is one string.
|
|
||||||
|
|
||||||
One-time setup:
|
|
||||||
|
|
||||||
1. Fedora host: install `infra/development/big-portals-tunnel.service` from the Infra repo
|
|
||||||
(instructions in the file).
|
|
||||||
2. Labs server: deploy the Infra `Caddyfile` + `compose.yml` (Caddy joins the
|
|
||||||
`openssh_default` network to reach the tunnel ends).
|
|
||||||
|
|
||||||
## What is not ported
|
## What is not ported
|
||||||
|
|
||||||
- **Observability** (Tempo, Prometheus, Grafana) is defined but disabled — those are built
|
- **Observability** (Tempo, Prometheus, Grafana) is defined but disabled — those are built
|
||||||
|
|||||||
@@ -135,14 +135,6 @@ cluster-internal hosts ({{ .Release.Namespace }}) and the node address
|
|||||||
{{- end }}
|
{{- end }}
|
||||||
{{- end -}}
|
{{- end -}}
|
||||||
|
|
||||||
{{/*
|
|
||||||
The origin a browser reaches Keycloak on: the issuer Keycloak pins and the
|
|
||||||
authority the portals use, from one place so they cannot drift (ADR-0010).
|
|
||||||
*/}}
|
|
||||||
{{- define "big.keycloakUrl" -}}
|
|
||||||
{{- .Values.keycloakUrl | default (printf "http://%s:%v" .Values.host (index .Values.nodePorts "keycloak")) -}}
|
|
||||||
{{- end -}}
|
|
||||||
|
|
||||||
{{- define "big.labels" -}}
|
{{- define "big.labels" -}}
|
||||||
app.kubernetes.io/name: {{ .name }}
|
app.kubernetes.io/name: {{ .name }}
|
||||||
app.kubernetes.io/instance: {{ .root.Release.Name }}
|
app.kubernetes.io/instance: {{ .root.Release.Name }}
|
||||||
|
|||||||
@@ -40,5 +40,5 @@ metadata:
|
|||||||
{{- include "big.labels" (dict "root" $ "name" (printf "portal-config-%s" $realm)) | nindent 4 }}
|
{{- include "big.labels" (dict "root" $ "name" (printf "portal-config-%s" $realm)) | nindent 4 }}
|
||||||
data:
|
data:
|
||||||
config.json: |
|
config.json: |
|
||||||
{ "authority": "{{ include "big.keycloakUrl" $ }}/realms/{{ $realm }}" }
|
{ "authority": "{{ printf "http://%s:%v" $.Values.host (index $.Values.nodePorts "keycloak") }}/realms/{{ $realm }}" }
|
||||||
{{- end }}
|
{{- end }}
|
||||||
|
|||||||
@@ -28,7 +28,7 @@ spec:
|
|||||||
{{- range $w.files }}
|
{{- range $w.files }}
|
||||||
{{- if hasPrefix "portal-config-" .configMap }}
|
{{- if hasPrefix "portal-config-" .configMap }}
|
||||||
annotations:
|
annotations:
|
||||||
checksum/portal-config: {{ include "big.keycloakUrl" $ | sha256sum }}
|
checksum/portal-config: {{ printf "%s|%v" $.Values.host (index $.Values.nodePorts "keycloak") | sha256sum }}
|
||||||
{{- end }}
|
{{- end }}
|
||||||
{{- end }}
|
{{- end }}
|
||||||
labels:
|
labels:
|
||||||
|
|||||||
@@ -25,11 +25,6 @@
|
|||||||
# string, so browser tokens and the BFF's discovered issuer agree.
|
# string, so browser tokens and the BFF's discovered issuer agree.
|
||||||
host: 192.168.122.100
|
host: 192.168.122.100
|
||||||
|
|
||||||
# Set when a TLS proxy outside the cluster publishes Keycloak: the full origin, no
|
|
||||||
# trailing slash. It replaces `host` + Keycloak's NodePort as the issuer and the
|
|
||||||
# portals' authority (runbook, "Publishing through the labs Caddy").
|
|
||||||
keycloakUrl: ""
|
|
||||||
|
|
||||||
# Set when pulling from a private registry (e.g. the Gitea Container Registry).
|
# Set when pulling from a private registry (e.g. the Gitea Container Registry).
|
||||||
imagePullSecrets: []
|
imagePullSecrets: []
|
||||||
|
|
||||||
@@ -273,7 +268,7 @@ workloads:
|
|||||||
# Pin the issuer to the address the browser uses, and let backchannel calls
|
# Pin the issuer to the address the browser uses, and let backchannel calls
|
||||||
# keep using keycloak:8080 — the BFF discovers metadata in-cluster and gets
|
# keep using keycloak:8080 — the BFF discovers metadata in-cluster and gets
|
||||||
# this issuer back, which is what browser tokens carry (infra/host-browser.yml).
|
# this issuer back, which is what browser tokens carry (infra/host-browser.yml).
|
||||||
KC_HOSTNAME: '{{ include "big.keycloakUrl" . }}'
|
KC_HOSTNAME: "http://{{ .Values.host }}:{{ index .Values.nodePorts \"keycloak\" }}"
|
||||||
KC_HOSTNAME_BACKCHANNEL_DYNAMIC: "true"
|
KC_HOSTNAME_BACKCHANNEL_DYNAMIC: "true"
|
||||||
ports: [{ name: http, port: 8080 }]
|
ports: [{ name: http, port: 8080 }]
|
||||||
# TCP, not /health/ready on the management port: nothing here gates on realm
|
# TCP, not /health/ready on the management port: nothing here gates on realm
|
||||||
|
|||||||
Reference in New Issue
Block a user