From c7f06b35faf4a235aefa6a67c82c51edbb215b8e Mon Sep 17 00:00:00 2001 From: Niek Otten Date: Fri, 25 Sep 2026 12:09:08 +0200 Subject: [PATCH] fix(infra): cap celery workers at 2 so the shared node stops OOM-killing CI (refs #182) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unset CELERY_WORKER_CONCURRENCY makes oz-celery and nrc-celery fork one process per CPU — 22 each on the lab node, ~225 MB apiece — and Talos' OOM controller was killing the runner mid-verify-stack. Same lever as the uWSGI caps (#144/#145/#147); objecten-celery already defaults to 1. Co-Authored-By: Claude Opus 5.5 (1M context) --- infra/docker-compose.yml | 5 +++++ infra/helm/big-reference/values.yaml | 2 ++ 2 files changed, 7 insertions(+) diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml index 3a5a6da..6ac6fae 100644 --- a/infra/docker-compose.yml +++ b/infra/docker-compose.yml @@ -57,6 +57,9 @@ services: # share this anchor and ignore it — they don't run uwsgi. UWSGI_PROCESSES: "1" UWSGI_THREADS: "2" + # Same lever for oz-celery: unset, the worker forks one process per CPU (22 on the lab node, + # ~225 MB each), which OOM-killed the shared runner mid-verify-stack. Only celery reads it. + CELERY_WORKER_CONCURRENCY: "2" DJANGO_SETTINGS_MODULE: openzaak.conf.docker SECRET_KEY: ${OZ_SECRET_KEY:-dev-only-not-for-production} DB_HOST: oz-db @@ -144,6 +147,8 @@ services: # 1 uWSGI worker, not the image default of 4×4 (#147) — see the oz-env note above. UWSGI_PROCESSES: "1" UWSGI_THREADS: "2" + # Two celery workers, not one per CPU — see the oz-env note above. + CELERY_WORKER_CONCURRENCY: "2" DJANGO_SETTINGS_MODULE: nrc.conf.docker SECRET_KEY: ${NRC_SECRET_KEY:-dev-only-not-for-production} DB_HOST: nrc-db diff --git a/infra/helm/big-reference/values.yaml b/infra/helm/big-reference/values.yaml index 20e7361..31f4df4 100644 --- a/infra/helm/big-reference/values.yaml +++ b/infra/helm/big-reference/values.yaml @@ -76,6 +76,7 @@ envGroups: oz: UWSGI_PROCESSES: "1" UWSGI_THREADS: "2" + CELERY_WORKER_CONCURRENCY: "2" DJANGO_SETTINGS_MODULE: openzaak.conf.docker SECRET_KEY: dev-only-not-for-production DB_HOST: oz-db @@ -98,6 +99,7 @@ envGroups: nrc: UWSGI_PROCESSES: "1" UWSGI_THREADS: "2" + CELERY_WORKER_CONCURRENCY: "2" DJANGO_SETTINGS_MODULE: nrc.conf.docker SECRET_KEY: dev-only-not-for-production DB_HOST: nrc-db -- 2.54.0