feat(openzaak): bounded retry + flagged write divergence (WP-60)

Local aanvraag/document writes and their paired ZGW writes aren't
transactional; a ZGW failure after the local write succeeds used to
diverge silently. ZgwHttpClient now retries transport-shaped failures
(not 500, which can follow a partial commit on the non-idempotent
statussen/rollen POSTs), and a ZGW failure that survives retry sets
Aanvraag.ZgwError plus a zgw:divergence audit row instead of failing
or diverging quietly. No outbox/reconcile job: three request-triggered
write paths don't justify a persisted queue that would also need to
carry citizen PII for the JWT audit claims.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

backend/src/BigRegister.Api/Zgw/OpenZaakDocumentSource.cs

@@ -4,6 +4,7 @@ using BigRegister.Api.Contracts;
 using BigRegister.Api.Data;
 using BigRegister.Domain.Authorization;
 using BigRegister.Stamdata;
+using Microsoft.Extensions.Logging;
 
 namespace BigRegister.Api.Zgw;
 
@@ -20,7 +21,9 @@ namespace BigRegister.Api.Zgw;
 /// <see cref="OpenZaakZaakSource"/> — creating a document needs write scope on Documenten;
 /// linking one to a zaak needs write scope on Zaken (the zaakinformatieobject resource).
 /// </summary>
-public sealed class OpenZaakDocumentSource(HttpClient http, ZgwTokenProvider tokens, ZgwOptions options) : IDocumentSource
+public sealed class OpenZaakDocumentSource(
+    HttpClient http, ZgwTokenProvider tokens, ZgwOptions options, ILogger<OpenZaakDocumentSource>? log = null)
+    : IDocumentSource
 {
   private readonly ZgwHttpClient zgw = new(http, tokens);
 
@@ -41,37 +44,54 @@ public sealed class OpenZaakDocumentSource(HttpClient http, ZgwTokenProvider tok
       UploadAsync(localId, categoryId, wizardId, fileName, contentType, content, caller)
           .GetAwaiter().GetResult();
 
+  // WP-60: once DocumentStore.Add (below) has committed, the local document is the record of
+  // truth (per the class doc above) — a ZGW failure past that point is caught, logged, and
+  // leaves DrcUrl null rather than throwing. DrcUrl == null is already the meaningful "not
+  // registered in ZGW yet" detector LinkToZaak skips on, so no separate flag column is needed
+  // here the way ApplicationStore.ZgwError is for the zaak side (see openzaak-integration.md's
+  // "Write resilience" section for why the two write paths differ).
   private async Task<UploadResponse> UploadAsync(
       string localId, string categoryId, string wizardId, string fileName, string contentType,
       byte[] content, CallerIdentity caller)
   {
     var doc = DocumentStore.Add(localId, categoryId, wizardId, fileName, contentType, content, caller.Bsn);
 
-    if (!options.InformatieobjecttypeUrls.TryGetValue(categoryId, out var informatieobjecttypeUrl))
-      throw new InvalidOperationException(
-          $"Zgw:InformatieobjecttypeUrls has no entry for category '{categoryId}'.");
+    try
+    {
+      if (!options.InformatieobjecttypeUrls.TryGetValue(categoryId, out var informatieobjecttypeUrl))
+        throw new InvalidOperationException(
+            $"Zgw:InformatieobjecttypeUrls has no entry for category '{categoryId}'.");
 
-    var eio = await zgw.PostAsync<Eio>($"{options.DrcBaseUrl}/enkelvoudiginformatieobjecten", new CreateEioRequest(
-        Bronorganisatie: options.Bronorganisatie,
-        Creatiedatum: DateOnly.FromDateTime(doc.UploadedAt.UtcDateTime),
-        Titel: fileName,
-        Auteur: options.UserRepresentation,
-        Taal: "nld",
-        Formaat: contentType,
-        Bestandsnaam: fileName,
-        Inhoud: Convert.ToBase64String(content),
-        Informatieobjecttype: informatieobjecttypeUrl,
-        Identificatie: doc.DocumentId,
-        Vertrouwelijkheidaanduiding: ConfidentialiteitFor(categoryId)), caller);
+      var eio = await zgw.PostAsync<Eio>($"{options.DrcBaseUrl}/enkelvoudiginformatieobjecten", new CreateEioRequest(
+          Bronorganisatie: options.Bronorganisatie,
+          Creatiedatum: DateOnly.FromDateTime(doc.UploadedAt.UtcDateTime),
+          Titel: fileName,
+          Auteur: options.UserRepresentation,
+          Taal: "nld",
+          Formaat: contentType,
+          Bestandsnaam: fileName,
+          Inhoud: Convert.ToBase64String(content),
+          Informatieobjecttype: informatieobjecttypeUrl,
+          Identificatie: doc.DocumentId,
+          Vertrouwelijkheidaanduiding: ConfidentialiteitFor(categoryId)), caller);
+
+      DocumentStore.SetDrcUrl(doc.DocumentId, eio.Url);
+    }
+    catch (Exception ex)
+    {
+      log?.LogError(ex, "zgw divergence document={DocumentId} category={CategoryId}", doc.DocumentId, categoryId);
+    }
 
-    DocumentStore.SetDrcUrl(doc.DocumentId, eio.Url);
     return new UploadResponse(doc.DocumentId, doc.LocalId);
   }
 
   /// <summary>Local link always happens (dual-write, same reasoning as upload); additionally,
   /// once a zaak exists, POST a zaakinformatieobject for every document that has a DRC url —
   /// documents uploaded before Zgw:Enabled was ever true (or under a config gap) simply have
-  /// no DrcUrl yet and are skipped, matching "nothing extra to link" for the local case.</summary>
+  /// no DrcUrl yet and are skipped, matching "nothing extra to link" for the local case.
+  /// WP-60: unlike Upload, a ZGW failure here still throws — DocumentStore.Link (the local
+  /// half) already ran above, so the caller (Program.cs's submit endpoint) catching this and
+  /// recording it as a flagged divergence is what closes the gap, not a try/catch in here.</summary>
   public void LinkToZaak(IReadOnlyList<string> documentIds, string? zaakUrl, CallerIdentity caller)
   {
     DocumentStore.Link(documentIds);

backend/src/BigRegister.Api/Zgw/OpenZaakZaakSource.cs

@@ -86,10 +86,12 @@ public sealed class OpenZaakZaakSource(HttpClient http, ZgwTokenProvider tokens,
   /// reason as <see cref="ListCases"/> (see the ponytail note there) — a submit is already a
   /// single request/response round trip, so no extra concurrency concern.
   ///
-  /// ponytail: no compensating transaction — if any ZGW call here throws, the aanvraag is
-  /// already marked Submitted locally (ApplicationStore.Submit already ran) but has no zaak.
-  /// Acceptable for a first write slice against a demo backend; a production arc would need a
-  /// retry/reconciliation story (or an outbox) before this dual-write can be trusted.
+  /// WP-60: still no compensating transaction — if any call here throws (after
+  /// <see cref="ZgwHttpClient"/>'s retry gives up), the aanvraag stays Submitted locally with
+  /// no zaak; rolling it back risks an orphan zaak if the failure landed after the zaak POST
+  /// succeeded. The caller (Program.cs's submit endpoint) catches this and records it as a
+  /// flagged divergence (Aanvraag.ZgwError) instead of letting it fail (or diverge) silently —
+  /// see openzaak-integration.md's "Write resilience" section.
   public (string Referentie, AanvraagStatusDto Status, string? ZaakUrl) CreateZaak(Aanvraag aanvraag, DateTimeOffset now, CallerIdentity caller) =>
       CreateZaakAsync(aanvraag, now, caller).GetAwaiter().GetResult();
 

backend/src/BigRegister.Api/Zgw/ZgwHttpClient.cs

@@ -1,3 +1,4 @@
+using System.Net;
 using System.Net.Http.Headers;
 using System.Net.Http.Json;
 using BigRegister.Domain.Authorization;
@@ -14,26 +15,77 @@ namespace BigRegister.Api.Zgw;
 /// </summary>
 internal sealed class ZgwHttpClient(HttpClient http, ZgwTokenProvider tokens)
 {
+  // WP-60: bounded retry for transport-shaped failures only (gateway restarts, timeouts) —
+  // never a substitute for reconciliation. 3 attempts, doubling from 200ms.
+  private const int MaxAttempts = 3;
+  private static readonly TimeSpan BaseDelay = TimeSpan.FromMilliseconds(200);
+
   public async Task<T> GetAsync<T>(string url, CallerIdentity? caller = null)
   {
-    using var req = new HttpRequestMessage(HttpMethod.Get, url);
-    Authorize(req, caller);
-    using var res = await http.SendAsync(req);
-    res.EnsureSuccessStatusCode();
+    using var res = await SendWithRetryAsync(() => new HttpRequestMessage(HttpMethod.Get, url), caller);
     return (await res.Content.ReadFromJsonAsync<T>())
         ?? throw new InvalidOperationException($"ZGW GET {url} returned null body.");
   }
 
   public async Task<T> PostAsync<T>(string url, object body, CallerIdentity? caller = null)
   {
-    using var req = new HttpRequestMessage(HttpMethod.Post, url) { Content = JsonContent.Create(body) };
-    Authorize(req, caller);
-    using var res = await http.SendAsync(req);
-    res.EnsureSuccessStatusCode();
+    using var res = await SendWithRetryAsync(
+        () => new HttpRequestMessage(HttpMethod.Post, url) { Content = JsonContent.Create(body) }, caller);
     return (await res.Content.ReadFromJsonAsync<T>())
         ?? throw new InvalidOperationException($"ZGW POST {url} returned null body.");
   }
 
+  /// <summary>
+  /// A fresh <see cref="HttpRequestMessage"/> (and JWT) per attempt — a sent request/content
+  /// cannot be resent. Only transport-shaped failures are retried (429/502/503/504/408, plus
+  /// connection errors and timeouts); 500 is deliberately excluded because it can follow a
+  /// partial commit on the two non-idempotent ZGW POSTs (<c>/statussen</c>, <c>/rollen</c>) and
+  /// retrying risks a duplicate write — the create-zaak/document POSTs are additionally
+  /// protected by OpenZaak's own uniqueness constraint on (bronorganisatie, identificatie).
+  /// A non-transient (or exhausted) failure throws with the status + a body snippet, which
+  /// <c>Program.cs</c>'s submit endpoint catches and records as a flagged divergence rather
+  /// than letting it diverge silently (see openzaak-integration.md's "Write resilience" section).
+  /// </summary>
+  private async Task<HttpResponseMessage> SendWithRetryAsync(Func<HttpRequestMessage> newRequest, CallerIdentity? caller)
+  {
+    for (var attempt = 1; ; attempt++)
+    {
+      using var req = newRequest();
+      Authorize(req, caller);
+
+      HttpResponseMessage res;
+      try
+      {
+        res = await http.SendAsync(req);
+      }
+      catch (Exception ex) when (attempt < MaxAttempts && ex is HttpRequestException or TaskCanceledException)
+      {
+        await Task.Delay(BaseDelay * (1 << (attempt - 1)));
+        continue;
+      }
+
+      if (res.IsSuccessStatusCode) return res;
+
+      if (attempt < MaxAttempts && IsTransient(res.StatusCode))
+      {
+        res.Dispose();
+        await Task.Delay(BaseDelay * (1 << (attempt - 1)));
+        continue;
+      }
+
+      var body = await res.Content.ReadAsStringAsync();
+      var snippet = body.Length > 500 ? body[..500] : body;
+      var message = $"ZGW {req.Method} {req.RequestUri} failed: {(int)res.StatusCode} {snippet}";
+      var status = res.StatusCode;
+      res.Dispose();
+      throw new HttpRequestException(message, null, status);
+    }
+  }
+
+  private static bool IsTransient(HttpStatusCode status) => status is
+      HttpStatusCode.RequestTimeout or HttpStatusCode.TooManyRequests or
+      HttpStatusCode.BadGateway or HttpStatusCode.ServiceUnavailable or HttpStatusCode.GatewayTimeout;
+
   private void Authorize(HttpRequestMessage req, CallerIdentity? caller)
   {
     req.Headers.Authorization = new AuthenticationHeaderValue("Bearer", caller is null ? tokens.Mint() : tokens.Mint(caller));