rve/learning-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: heal migration-ledger mismatch and make Azure SSO deploy-proof (#18)
All checks were successful
On Pull Request to Main / test (pull_request) Successful in 39s
Details
On Pull Request to Main / publish (pull_request) Successful in 1m9s
Details
On Pull Request to Main / deploy-dev (pull_request) Successful in 3m1s
Details

Browse Source 
PocketBase on Labs crash-looped since the SSO deploy (PR #17): mounting
--migrationsDir for the first time replayed the entire migration history
against a database that was provisioned out-of-band (empty _migrations
ledger) and died on 1778948471_created_content.js. On top of that the
team_members->auth migration had its own crash paths and trapped OAuth2
config inside a one-shot, env-dependent migration.

- pb_migrations/1000000000_baseline_ledger_sync.js: detects an
  out-of-band provisioned DB (schema exists, ledger empty) and marks the
  79 historical migrations as applied; no-op on fresh or already-synced DBs
- pb_migrations/1781000000_team_members_to_auth.js: idempotency guard,
  relation->text conversion WITH data preservation (PocketBase diffs
  fields by id, so the column is backed up and restored via SQL), unique
  index rebuild, no silent catches, env only as fast-path
- pb_hooks/entra_oidc.pb.js + pb_hooks/utils.js: reconcile the Entra OIDC
  provider from ENTRA_* env on every bootstrap + cron tick
  (compare-before-save, warn-once); heals environments that migrated
  without secrets and supports secret rotation without re-apply
- pb_hooks/team_members.pb.js: require() pattern — JSVM runs callbacks as
  isolated programs, top-level helpers are not in scope (adopts Leroy's
  fix from the fix-sso branch)
- infra/*/site/deploy-playbook.yml: health-gate after compose up — the
  deploy fails loudly with container logs when PocketBase does not become
  healthy (runs #83-#88 were green while PB crash-looped)
- docker-compose.yml: .env.local is optional again
- docs/auth-spec.md + AI_AGENT.md: ledger/reconciler documentation, ADRs
  006-008, never-rename-applied-migrations warning

Verified locally against PocketBase v0.30.4 with a 7-scenario DoD matrix
(fresh DB +/- env, out-of-band DB with data incl. data preservation,
populated-ledger upgrade, late-secrets healing, re-run guard, hook
provisioning): 15/15 pass. npm test 112/112.

Closes #18

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
RaymondVerhoef
2026-07-11 11:14:09 +02:00
parent 89d3395a62
commit d79e69aad2
10 changed files with 478 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
pb_migrations/1000000000_baseline_ledger_sync.js Normal file
View File

@@ -0,0 +1,138 @@
/// <reference path="../pb_data/types.d.ts" />
//
// Issue #18 — Baseline ledger sync for out-of-band provisioned databases.
//
// The Labs/production PocketBase originally ran WITHOUT --migrationsDir: its
// schema was created by scripts/setup-pb-collections.mjs / the admin UI, so its
// _migrations ledger is empty. When the migrations dir was mounted for the
// first time (Azure SSO, PR #17), PocketBase tried to replay the entire
// migration history against that existing schema and crash-looped on the very
// first file ("Collection name must be unique").
//
// This migration sorts before every other file. When it detects that the
// legacy schema already exists but the ledger has never tracked it, it marks
// the historical migrations below as applied so the replay is skipped and the
// chain continues with the genuinely new migrations (team_members_to_auth,
// tighten_api_rules). On fresh databases and on databases that already have a
// populated ledger it is a no-op.
//
// NOTE for future agents: never rename a migration file after it has been
// applied anywhere — the ledger is filename-based. New migrations must sort
// after 1781100001 and be appended to environments through a normal deploy.
const HISTORICAL_MIGRATIONS = [
"1778948471_created_content.js",
"1778948471_created_quiz_banks.js",
"1778948471_created_relations.js",
"1778948471_created_sources.js",
"1778948471_created_team_members.js",
"1778948471_created_topics.js",
"1778948472_created_leaderboard.js",
"1778948472_created_learn_progress.js",
"1778948472_created_quiz_cache.js",
"1778948472_created_quiz_results.js",
"1778948472_created_settings.js",
"1778954289_created_test_col2.js",
"1778954310_deleted_relations.js",
"1778954310_deleted_topics.js",
"1778954310_deleted_users.js",
"1778954311_deleted_content.js",
"1778954311_deleted_leaderboard.js",
"1778954311_deleted_learn_progress.js",
"1778954311_deleted_quiz_banks.js",
"1778954311_deleted_quiz_cache.js",
"1778954311_deleted_quiz_results.js",
"1778954311_deleted_settings.js",
"1778954311_deleted_sources.js",
"1778954311_deleted_team_members.js",
"1778954311_deleted_test_col2.js",
"1778954317_created_content.js",
"1778954317_created_leaderboard.js",
"1778954317_created_learn_progress.js",
"1778954317_created_quiz_banks.js",
"1778954317_created_quiz_cache.js",
"1778954317_created_quiz_results.js",
"1778954317_created_relations.js",
"1778954317_created_settings.js",
"1778954317_created_sources.js",
"1778954317_created_team_members.js",
"1778954317_created_topics.js",
"1779005271_created_test_col3.js",
"1779005289_created_test_col4.js",
"1779005309_deleted_content.js",
"1779005309_deleted_learn_progress.js",
"1779005309_deleted_quiz_banks.js",
"1779005309_deleted_quiz_cache.js",
"1779005309_deleted_quiz_results.js",
"1779005309_deleted_relations.js",
"1779005309_deleted_sources.js",
"1779005309_deleted_team_members.js",
"1779005309_deleted_topics.js",
"1779005310_deleted_leaderboard.js",
"1779005310_deleted_settings.js",
"1779005310_deleted_test_col3.js",
"1779005310_deleted_test_col4.js",
"1779005316_created_content.js",
"1779005316_created_quiz_banks.js",
"1779005316_created_relations.js",
"1779005316_created_sources.js",
"1779005316_created_topics.js",
"1779005317_created_leaderboard.js",
"1779005317_created_learn_progress.js",
"1779005317_created_quiz_cache.js",
"1779005317_created_quiz_results.js",
"1779005317_created_settings.js",
"1779005317_created_team_members.js",
"1779127586_created_curriculum.js",
"1779127759_collections_snapshot.js",
"1779200000_updated_sources.js",
"1780000001_updated_topics.js",
"1780500000_updated_topics_relevance_locked.js",
"1780500001_normalize_relation_types.js",
"1780500002_created_llm_calls.js",
"1780600000_curriculum_v2.js",
"1780700000_sources_progress.js",
"1780800000_created_micro_learnings.js",
"1780800001_deleted_legacy_collections.js",
"1780800002_update_micro_learnings_rules.js",
"1780900000_team_members_enrollment.js",
"1780900001_created_test_results.js",
"1781000000_created_theme_sessions.js",
"1781100000_created_question_bank.js",
"1781100001_created_on_demand_attempts.js",
];
migrate((app) => {
// Fresh database? (no legacy schema) -> let the normal chain run.
try {
app.findCollectionByNameOrId("content");
} catch (_) {
return;
}
// Ledger already tracks the history? -> normally migrated DB, nothing to do.
const probe = new DynamicModel({ count: 0 });
app.db()
.newQuery("SELECT COUNT(*) AS count FROM _migrations WHERE file = {:file}")
.bind({ file: "1778948471_created_content.js" })
.one(probe);
if (probe.count > 0) {
return;
}
// Out-of-band provisioned database: schema exists, ledger is empty.
// Mark the historical migrations as applied so they are not replayed.
const applied = Date.now();
for (const file of HISTORICAL_MIGRATIONS) {
app.db()
.newQuery("INSERT OR IGNORE INTO _migrations (file, applied) VALUES ({:file}, {:applied})")
.bind({ file: file, applied: applied })
.execute();
}
console.log(
"baseline_ledger_sync: detected out-of-band provisioned database — marked " +
HISTORICAL_MIGRATIONS.length + " historical migrations as applied"
);
}, (_app) => {
// Down: intentionally a no-op. The ledger rows describe environment-specific
// bookkeeping; removing them would re-trigger the replay this fix prevents.
});

187
pb_migrations/1781000000_team_members_to_auth.js
View File

@@ -1,6 +1,6 @@
/// <reference path="../pb_data/types.d.ts" />
//
// Issue #16 — Azure (Entra ID) login.
// Issue #16 / #18 — Azure (Entra ID) login.
//
// Replaces the PIN-based `base` collection `team_members` with a PocketBase
// `auth` collection that authenticates against Azure Entra ID via OIDC.
@@ -10,53 +10,127 @@
// base, generated tests and micro-learnings live in OTHER collections and are
// left completely untouched by this migration.
//
// OIDC provider credentials are read from the environment at apply time, so no
// secrets are committed to git:
// ENTRA_TENANT_ID, ENTRA_CLIENT_ID, ENTRA_CLIENT_SECRET
// (set via the Ansible-rendered .env, see infra/*/site/compose.yml).
//
// To rotate credentials or change the tenant later, update the env and either
// re-configure the provider from the PocketBase admin UI (Settings → Auth
// providers) or ship a follow-up migration.
// This migration is STRUCTURE ONLY and does not depend on the environment:
// the OIDC provider credentials (ENTRA_TENANT_ID / ENTRA_CLIENT_ID /
// ENTRA_CLIENT_SECRET) are reconciled into the collection on every startup by
// pb_hooks/entra_oidc.pb.js. That keeps the one-shot migration deterministic:
// applying it while the secrets are absent can no longer permanently disable
// OAuth2 (issue #18). As a fast path the provider is also attached here when
// the env vars are already present at apply time.
migrate((app) => {
// 1. Drop the old PIN-based collection (takes the PIN records with it).
// Other collections (micro_learning_completions, theme_session_completions)
// have relation fields pointing to the old team_members — PocketBase refuses
// to delete a referenced collection. Remove those relation fields first,
// delete the old collection, create the new auth collection, then re-add
// the relation fields pointing to the new collection.
const relDeps = [
{ collection: "micro_learning_completions", fieldId: "rel_team_member_id", fieldName: "team_member_id" },
{ collection: "theme_session_completions", fieldId: "rel_tsc_team_member", fieldName: "team_member_id" },
];
// Remove blocking relation fields so the old collection can be deleted.
for (const dep of relDeps) {
try {
const col = app.findCollectionByNameOrId(dep.collection);
col.fields.removeById(dep.fieldId);
app.save(col);
} catch (_) { /* collection doesn't exist yet — skip */ }
}
// Idempotency guard: if team_members is already an auth collection (e.g. the
// conversion happened through an earlier partial rollout), leave it alone.
let existing = null;
try {
const old = app.findCollectionByNameOrId("team_members");
app.delete(old);
existing = app.findCollectionByNameOrId("team_members");
} catch (_) {
// Collection did not exist — nothing to drop.
existing = null; // collection absent — nothing to convert, only to create
}
if (existing && existing.type === "auth") {
console.log("team_members_to_auth: team_members is already an auth collection — skipping conversion.");
return;
}
const tenant = $os.getenv("ENTRA_TENANT_ID") || "common";
const clientId = $os.getenv("ENTRA_CLIENT_ID");
const clientSecret = $os.getenv("ENTRA_CLIENT_SECRET");
// 1. Detach relation fields that point at team_members. PocketBase refuses
// to delete a collection that other collections relate to, so before we
// can drop the old PIN-based team_members we convert those relations into
// plain text fields (the id is stored as text — consistent with how
// user_id is stored in test_results / on_demand_attempts). The column
// values are preserved by the conversion.
const deps = [
{ name: "micro_learning_completions", relId: "rel_team_member_id", textId: "txt_mlc_team_member" },
{ name: "theme_session_completions", relId: "rel_tsc_team_member", textId: "txt_tsc_team_member" },
];
for (const dep of deps) {
let c = null;
try {
c = app.findCollectionByNameOrId(dep.name);
} catch (_) {
console.log("team_members_to_auth: " + dep.name + " does not exist yet — nothing to detach.");
continue;
}
if (!c.fields.getById(dep.relId)) {
console.log("team_members_to_auth: " + dep.name + "." + dep.relId + " already detached — skipping.");
continue;
}
// PocketBase diffs fields by ID, so swapping the relation field for a text
// field drops and recreates the underlying column. Back the values up and
// restore them after the schema save — the ids must survive (issue #18).
const backup = "_issue18_tm_" + dep.name;
app.db().newQuery("DROP TABLE IF EXISTS `" + backup + "`").execute();
app.db().newQuery(
"CREATE TABLE `" + backup + "` AS SELECT `id`, `team_member_id` AS `v` FROM `" + dep.name + "`"
).execute();
const hasOidc = !!(clientId && clientSecret);
if (!hasOidc) {
console.log("WARN: ENTRA_CLIENT_ID / ENTRA_CLIENT_SECRET not set — OIDC provider will not be configured. " +
"Set the env vars and re-apply the migration to enable Azure login.");
// Drop any index that references the team_member_id column (it is rebuilt
// on the text column below).
c.indexes = (c.indexes || []).filter((idx) => idx.indexOf("team_member_id") === -1);
// Replace the relation field with a text field of the same name.
c.fields.removeById(dep.relId);
c.fields.add(new Field({
type: "text",
id: dep.textId,
name: "team_member_id",
required: false,
min: 0,
max: 0,
}));
app.save(c);
app.db().newQuery(
"UPDATE `" + dep.name + "` SET `team_member_id` = COALESCE(" +
"(SELECT `v` FROM `" + backup + "` WHERE `" + backup + "`.`id` = `" + dep.name + "`.`id`), '')"
).execute();
app.db().newQuery("DROP TABLE `" + backup + "`").execute();
}
// 2. Recreate `team_members` as an auth collection (same name → all existing
// Recreate the unique (team_member_id, session_week) guard on the text column.
try {
const tsc = app.findCollectionByNameOrId("theme_session_completions");
const idx = "CREATE UNIQUE INDEX `idx_theme_session_completions_user_week` ON `theme_session_completions` (`team_member_id`, `session_week`)";
if ((tsc.indexes || []).indexOf(idx) === -1) {
tsc.indexes.push(idx);
app.save(tsc);
}
} catch (_) {
console.log("team_members_to_auth: theme_session_completions does not exist yet — no index to rebuild.");
}
// 2. Drop the old PIN-based team_members (now unreferenced). NOT wrapped in
// a try/catch: if this fails there is an unknown relation left and the
// migration must fail loudly instead of masking the error (issue #18).
if (existing) {
app.delete(existing);
}
// Fast path: attach the OIDC provider right away when the credentials are
// already present at apply time. When they are absent the collection is
// created without a provider and pb_hooks/entra_oidc.pb.js configures it on
// the next startup / cron tick once the ENTRA_* env vars are supplied.
const tenant = $os.getenv("ENTRA_TENANT_ID") || "common";
const clientId = $os.getenv("ENTRA_CLIENT_ID") || "";
const clientSecret = $os.getenv("ENTRA_CLIENT_SECRET") || "";
const oidcProviders = (clientId && clientSecret) ? [
{
name: "oidc",
clientId: clientId,
clientSecret: clientSecret,
authURL: "https://login.microsoftonline.com/" + tenant + "/oauth2/v2.0/authorize",
tokenURL: "https://login.microsoftonline.com/" + tenant + "/oauth2/v2.0/token",
userInfoURL: "https://graph.microsoft.com/oidc/userinfo",
displayName: "Microsoft",
pkce: true,
},
] : [];
if (oidcProviders.length === 0) {
console.log(
"team_members_to_auth: ENTRA_CLIENT_ID / ENTRA_CLIENT_SECRET not set — creating the auth " +
"collection without an OIDC provider. pb_hooks/entra_oidc.pb.js will configure the provider " +
"automatically once the env vars are present (no re-apply needed)."
);
}
// 3. Recreate `team_members` as an auth collection (same name → all existing
// `user_id` references in test_results, on_demand_attempts,
// micro_learning_completions, leaderboard, theme_session_completions keep
// working unchanged).
@@ -83,7 +157,7 @@ migrate((app) => {
mfa: { enabled: false, duration: 600, rule: "" },
oauth2: {
enabled: hasOidc,
enabled: oidcProviders.length > 0,
// Map the OIDC userinfo claims onto our fields.
mappedFields: {
id: "",
@@ -91,18 +165,7 @@ migrate((app) => {
username: "",
avatarURL: "",
},
providers: hasOidc ? [
{
name: "oidc",
clientId: clientId,
clientSecret: clientSecret,
authURL: "https://login.microsoftonline.com/" + tenant + "/oauth2/v2.0/authorize",
tokenURL: "https://login.microsoftonline.com/" + tenant + "/oauth2/v2.0/token",
userInfoURL: "https://graph.microsoft.com/oidc/userinfo",
displayName: "Microsoft",
pkce: true,
},
] : [],
providers: oidcProviders,
},
fields: [
@@ -258,24 +321,6 @@ migrate((app) => {
});
app.save(collection);
// 3. Re-add the relation fields pointing to the new auth collection.
for (const dep of relDeps) {
try {
const col = app.findCollectionByNameOrId(dep.collection);
const newField = new Field({
id: dep.fieldId,
name: dep.fieldName,
type: "relation",
required: true,
collectionId: collection.id,
cascadeDelete: true,
maxSelect: 1,
});
col.fields.add(newField);
app.save(col);
} catch (_) { /* collection doesn't exist — skip */ }
}
}, (app) => {
// Down: drop the auth collection and recreate the original PIN-based base
// collection (without data — the original records are gone).