fix(projection): serialise concurrent migrators with a pg advisory lock (refs #75)
Some checks failed
CI / lint (pull_request) Successful in 1m25s
CI / build (pull_request) Successful in 1m21s
CI / unit (pull_request) Successful in 1m30s
CI / frontend (pull_request) Successful in 3m11s
CI / mutation (pull_request) Successful in 5m59s
CI / verify-stack (pull_request) Failing after 5m50s
Some checks failed
CI / lint (pull_request) Successful in 1m25s
CI / build (pull_request) Successful in 1m21s
CI / unit (pull_request) Successful in 1m30s
CI / frontend (pull_request) Successful in 3m11s
CI / mutation (pull_request) Successful in 5m59s
CI / verify-stack (pull_request) Failing after 5m50s
verify-up failed: the event-subscriber and projection-api both migrate the shared
projection DB on start, and EF releases its migrations-history lock between individual
migrations — harmless with one migration, but the new AddNotificationResource made a
second, so a migrator re-applied it in the window between the other's two migrations
("column resource already exists"). Hold a session pg_advisory_lock across the whole
MigrateAsync so the sequence runs exactly once; the second migrator then finds nothing
pending.
Verified by starting both images simultaneously against a fresh Postgres: both reach
health, the resource column is created once, and __EFMigrationsHistory has both rows.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -19,12 +19,46 @@ public static class ServiceCollectionExtensions
|
|||||||
return services;
|
return services;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A fixed application-scoped key for the migration advisory lock (any stable 64-bit constant).
|
||||||
|
private const long MigrationAdvisoryLockKey = 727501;
|
||||||
|
|
||||||
/// <summary>Apply any pending EF migrations. Called once on service start so a fresh stack
|
/// <summary>Apply any pending EF migrations. Called once on service start so a fresh stack
|
||||||
/// reaches a usable schema without a manual migration step (DoD: compose up reaches green).</summary>
|
/// reaches a usable schema without a manual migration step (DoD: compose up reaches green).
|
||||||
|
///
|
||||||
|
/// The Event Subscriber and the projection-api share this DB and both migrate on start. EF's
|
||||||
|
/// migrations-history lock is released between individual migrations, so with more than one pending
|
||||||
|
/// migration two migrators can interleave and one re-applies a migration the other just did
|
||||||
|
/// ("column already exists"). Hold a Postgres session <c>pg_advisory_lock</c> across the whole
|
||||||
|
/// sequence so it runs exactly once; the second migrator then finds nothing pending.</summary>
|
||||||
public static async Task MigrateProjectionAsync(this IServiceProvider services, CancellationToken ct = default)
|
public static async Task MigrateProjectionAsync(this IServiceProvider services, CancellationToken ct = default)
|
||||||
{
|
{
|
||||||
await using var scope = services.CreateAsyncScope();
|
await using var scope = services.CreateAsyncScope();
|
||||||
var db = scope.ServiceProvider.GetRequiredService<ProjectionDbContext>();
|
var db = scope.ServiceProvider.GetRequiredService<ProjectionDbContext>();
|
||||||
await db.Database.MigrateAsync(ct);
|
var connection = db.Database.GetDbConnection();
|
||||||
|
|
||||||
|
await connection.OpenAsync(ct);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await ExecuteAsync(connection, $"SELECT pg_advisory_lock({MigrationAdvisoryLockKey})", ct);
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await db.Database.MigrateAsync(ct);
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
await ExecuteAsync(connection, $"SELECT pg_advisory_unlock({MigrationAdvisoryLockKey})", ct);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
finally
|
||||||
|
{
|
||||||
|
await connection.CloseAsync();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static async Task ExecuteAsync(System.Data.Common.DbConnection connection, string sql, CancellationToken ct)
|
||||||
|
{
|
||||||
|
await using var command = connection.CreateCommand();
|
||||||
|
command.CommandText = sql;
|
||||||
|
await command.ExecuteNonQueryAsync(ct);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,8 +10,9 @@ builder.Services.AddProjectionReadModel(connectionString);
|
|||||||
|
|
||||||
var app = builder.Build();
|
var app = builder.Build();
|
||||||
|
|
||||||
// Ensure the schema exists before serving reads. EF serialises concurrent migrators via the
|
// Ensure the schema exists before serving reads. The Event Subscriber migrates this shared DB too;
|
||||||
// migrations-history lock, so it is safe that the Event Subscriber migrates too.
|
// MigrateProjectionAsync serialises concurrent migrators with a session advisory lock so the whole
|
||||||
|
// migration sequence runs exactly once (see ServiceCollectionExtensions).
|
||||||
await app.Services.MigrateProjectionAsync();
|
await app.Services.MigrateProjectionAsync();
|
||||||
|
|
||||||
app.MapGet("/health", () => "Healthy");
|
app.MapGet("/health", () => "Healthy");
|
||||||
|
|||||||
Reference in New Issue
Block a user