feat: phase 3 of AI pipeline hardening — extraction quality

Replace stateless one-shot extraction with a stateful, paced, cancellable
pipeline. Six subtasks:

- 3.1 Sentence-aware chunking with 800-char overlap (was paragraph-only
  at 4000 chars). Hard-split fallback for runaway sentences.
- 3.2 Stateful extraction: chunks 2+ receive an "already-extracted topic
  IDs" hint capped at 200 IDs, so the model reuses IDs instead of
  inventing variants like software-developer vs software-engineer.
- 3.3 Token-bucket limiter in llmRetry.js (extractionLimiter, 5 req/min).
  callLLM awaits the limiter before fetch; 429+Retry-After calls
  pauseUntil. Replaces hard setTimeout(12000) and setTimeout(15000).
- 3.4 relevance_locked column on topics — admin edits to relevance are
  sticky across re-extraction. Migration + merge respects the flag +
  unlock checkbox in KnowledgeGraph edit form.
- 3.5 Unify relation vocabulary — handbook prompt no longer mentions
  legacy "executes"; one-shot migration rewrites existing executes rows
  to executed_by with source/target swapped.
- 3.6 Cancellation — Cancel button on UploadZone wired to an
  AbortController threaded into callLLM; aborted runs persist status =
  "cancelled" rather than "failed".

Tests: 16 new unit tests for chunkText, buildKnownIdsHint, and
createLimiter. All 61 tests pass, 0 lint errors, build clean.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

src/lib/__tests__/extractionPipeline.test.js

@@ -0,0 +1,89 @@
+import { describe, expect, it, vi, beforeEach, afterEach } from 'vitest';
+
+vi.mock('../pb', () => ({
+  pb: { collection: () => ({ create: () => ({ catch: () => {} }) }) },
+}));
+
+import { chunkText, buildKnownIdsHint, MAX_CHUNK_CHARS, OVERLAP_CHARS } from '../extractionPipeline';
+
+describe('chunkText', () => {
+  let warnSpy;
+  beforeEach(() => { warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); });
+  afterEach(() => { warnSpy.mockRestore(); });
+
+  it('returns the original text as a single chunk when below maxChars', () => {
+    const result = chunkText('A short paragraph.');
+    expect(result).toEqual(['A short paragraph.']);
+  });
+
+  it('returns empty array for empty/whitespace input', () => {
+    expect(chunkText('')).toEqual([]);
+    expect(chunkText('   \n  ')).toEqual([]);
+  });
+
+  it('splits along sentence boundaries with overlap between adjacent chunks', () => {
+    const sentence = 'This is a sentence with exactly a known length. ';
+    const text = sentence.repeat(100); // ~5000 chars
+    const chunks = chunkText(text, { maxChars: 600, overlapChars: 150 });
+
+    expect(chunks.length).toBeGreaterThan(1);
+    for (const c of chunks) {
+      expect(c.length).toBeLessThanOrEqual(600);
+    }
+    // Adjacent chunks share trailing text — the overlap should be non-empty.
+    for (let i = 1; i < chunks.length; i++) {
+      const tail = chunks[i - 1].slice(-150);
+      // The new chunk must begin with content that appears at the tail of the prior chunk.
+      const firstHundred = chunks[i].slice(0, 100);
+      // At least one word from the tail should appear in the head of the next chunk.
+      const words = tail.split(/\s+/).filter((w) => w.length > 3);
+      const shared = words.some((w) => firstHundred.includes(w));
+      expect(shared).toBe(true);
+    }
+  });
+
+  it('hard-splits a single sentence that exceeds maxChars and logs a warning', () => {
+    const huge = 'word '.repeat(400).trim() + '.'; // ~2000 chars, no sentence break
+    const chunks = chunkText(huge, { maxChars: 500, overlapChars: 50 });
+    expect(chunks.length).toBeGreaterThan(1);
+    expect(warnSpy).toHaveBeenCalled();
+    for (const c of chunks) expect(c.length).toBeLessThanOrEqual(500);
+  });
+
+  it('handles paragraph-only splits when no sentence punctuation is present', () => {
+    const paragraphs = Array.from({ length: 10 }, (_, i) => `paragraph ${i} content here`).join('\n\n');
+    const chunks = chunkText(paragraphs, { maxChars: 80, overlapChars: 20 });
+    expect(chunks.length).toBeGreaterThan(1);
+  });
+
+  it('uses the documented defaults', () => {
+    expect(MAX_CHUNK_CHARS).toBe(8000);
+    expect(OVERLAP_CHARS).toBe(800);
+  });
+});
+
+describe('buildKnownIdsHint', () => {
+  it('returns empty string when no IDs are known', () => {
+    expect(buildKnownIdsHint([])).toBe('');
+    expect(buildKnownIdsHint(undefined)).toBe('');
+    expect(buildKnownIdsHint(null)).toBe('');
+  });
+
+  it('formats the known IDs as a bulleted list with a leading instruction', () => {
+    const hint = buildKnownIdsHint(['software-engineer', 'onboarding-buddy']);
+    expect(hint).toContain('Already-extracted topic IDs');
+    expect(hint).toContain('- software-engineer');
+    expect(hint).toContain('- onboarding-buddy');
+    expect(hint.endsWith('\n')).toBe(true);
+  });
+
+  it('caps the hint at the 200 most recent IDs', () => {
+    const ids = Array.from({ length: 250 }, (_, i) => `topic-${i}`);
+    const hint = buildKnownIdsHint(ids);
+    // The newest IDs must appear; the oldest must not.
+    expect(hint).toContain('topic-249');
+    expect(hint).toContain('topic-50');
+    expect(hint).not.toContain('topic-49');
+    expect(hint).not.toContain('topic-0\n');
+  });
+});

src/lib/__tests__/llmRetry.test.js

@@ -0,0 +1,72 @@
+import { describe, expect, it, vi, afterEach } from 'vitest';
+import { createLimiter, extractionLimiter } from '../llmRetry';
+
+afterEach(() => { vi.useRealTimers(); });
+
+describe('createLimiter', () => {
+  it('rejects an invalid rps', () => {
+    expect(() => createLimiter({ rps: 0 })).toThrow();
+    expect(() => createLimiter({ rps: -1 })).toThrow();
+  });
+
+  it('rejects an invalid burst', () => {
+    expect(() => createLimiter({ rps: 1, burst: 0 })).toThrow();
+  });
+
+  it('lets the first call through immediately (initial burst token)', async () => {
+    const limiter = createLimiter({ rps: 1, burst: 1 });
+    const start = Date.now();
+    await limiter.acquire();
+    expect(Date.now() - start).toBeLessThan(50);
+  });
+
+  it('queues subsequent calls to respect the spacing', async () => {
+    vi.useFakeTimers();
+    const limiter = createLimiter({ rps: 10, burst: 1 }); // 100ms spacing
+    await limiter.acquire(); // consume initial token
+
+    let resolved = false;
+    const p = limiter.acquire().then(() => { resolved = true; });
+
+    await vi.advanceTimersByTimeAsync(50);
+    expect(resolved).toBe(false);
+
+    await vi.advanceTimersByTimeAsync(100);
+    await p;
+    expect(resolved).toBe(true);
+  });
+
+  it('honours pauseUntil — no acquire returns before the pause expires', async () => {
+    vi.useFakeTimers();
+    const limiter = createLimiter({ rps: 100, burst: 5 });
+    limiter.pauseUntil(Date.now() + 1000);
+
+    let resolved = false;
+    const p = limiter.acquire().then(() => { resolved = true; });
+
+    await vi.advanceTimersByTimeAsync(500);
+    expect(resolved).toBe(false);
+
+    await vi.advanceTimersByTimeAsync(600);
+    await p;
+    expect(resolved).toBe(true);
+  });
+
+  it('aborts a queued acquire when the signal fires', async () => {
+    const limiter = createLimiter({ rps: 1, burst: 1 });
+    await limiter.acquire(); // consume
+
+    const ctl = new AbortController();
+    const p = limiter.acquire({ signal: ctl.signal });
+    ctl.abort();
+
+    await expect(p).rejects.toBeInstanceOf(DOMException);
+  });
+});
+
+describe('extractionLimiter', () => {
+  it('is exported and exposes the limiter shape', () => {
+    expect(typeof extractionLimiter.acquire).toBe('function');
+    expect(typeof extractionLimiter.pauseUntil).toBe('function');
+  });
+});