feat: phase 5 of AI pipeline hardening — R42 retrieval & telemetry

- Add dependency-free TF-IDF retrieval (src/lib/retrieval.js) with NL+EN
  stopwords and a WeakMap-cached index.
- Rewrite buildKbContext to ship the top-K relevant topics + verbatim-
  mentioned ids only, filter relations to the included set, and append a
  [kb_hash: <8 hex>] suffix so the ephemeral prompt cache busts when the
  graph changes. Returns { context, retrievedTopics, allTopics }.
- Add LOOKUP_TOPIC_TOOL and drive useChat through callLLM directly with a
  multi-hop tool_result loop capped at 3 hops; preserve Anthropic-provided
  tool_use ids through callLLM so the loop can echo correct tool_use_id.
- Truncate R42 history to the last 12 turns and prepend a single
  "(earlier conversation truncated)" assistant message.
- Set R42 chat defaults: temperature 0.3, maxTokens 2048.
- Add pb_migrations/1780500002_created_llm_calls.js (the best-effort
  logger in callLLM was already wired) and a new Admin → Diagnostics
  view showing the last 100 calls with token usage, cache-hit rate, and
  USD cost from a local Anthropic price table.
- Finalize AI_PIPELINE_HARDENING_PLAN.md: mark Phases 1–5 shipped and
  Phase 6 (eval harness) explicitly out of scope.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

src/components/chat/rag.js

@@ -1,68 +1,119 @@
 import * as db from '../../lib/db';
+import { buildIndex, retrieveTopK } from '../../lib/retrieval';
+
+const TOP_K = 10;
+
+async function sha256Hex(input) {
+  const enc = new TextEncoder().encode(input);
+  if (globalThis.crypto?.subtle?.digest) {
+    const buf = await globalThis.crypto.subtle.digest('SHA-256', enc);
+    return Array.from(new Uint8Array(buf)).map(b => b.toString(16).padStart(2, '0')).join('');
+  }
+  let h = 2166136261 >>> 0;
+  for (let i = 0; i < input.length; i++) {
+    h ^= input.charCodeAt(i);
+    h = Math.imul(h, 16777619);
+  }
+  return (h >>> 0).toString(16).padStart(8, '0');
+}
 
 /**
- * Build a compact knowledge-base context string to inject into the system prompt.
- * Reads topics + relations from PocketBase via db.js.
- * Topic-level content is loaded only when a topic id/label appears in the user's message.
+ * Build a retrieval-scoped KB context. Instead of dumping the whole graph,
+ * we pick the top-K topics by TF-IDF over `userMessage`, plus any topic
+ * whose id or label appears verbatim in the message. Relations are filtered
+ * to those that touch the included set.
  *
- * Returns { context: string, topics: Array } so callers can reuse the fetched topics
- * for validateDelta without a second round-trip.
+ * A `[kb_hash: …]` suffix is appended so the Anthropic ephemeral prompt
+ * cache automatically busts when topics are added/removed.
+ *
+ * Returns:
+ *   { context, retrievedTopics, allTopics }
+ * — `allTopics` is the full PocketBase list so callers can still run
+ * `validateDelta` against the entire current graph.
  */
 export async function buildKbContext(userMessage = '') {
-  const [topics, relations] = await Promise.all([
+  const [allTopics, allRelations] = await Promise.all([
     db.getTopics(),
     db.getRelations(),
   ]);
 
-  if (topics.length === 0) {
+  const sortedIds = allTopics.map(t => t.id).sort().join('|');
+  const fullHash = await sha256Hex(sortedIds);
+  const kbHash = fullHash.slice(0, 8);
+
+  if (allTopics.length === 0) {
     return {
-      context: 'KENNISGRAAF: (leeg — er zijn nog geen onderwerpen geëxtraheerd)',
-      topics: [],
+      context: `KENNISGRAAF: (leeg — er zijn nog geen onderwerpen geëxtraheerd)\n[kb_hash: ${kbHash}]`,
+      retrievedTopics: [],
+      allTopics: [],
     };
   }
 
-  const topicLines = topics.map(t => {
+  const lowered = userMessage.toLowerCase();
+  const mentionedIds = new Set();
+  for (const t of allTopics) {
+    const idHit = t.id && lowered.includes(t.id.toLowerCase());
+    const labelHit = t.label && lowered.includes(t.label.toLowerCase());
+    if (idHit || labelHit) mentionedIds.add(t.id);
+  }
+
+  const index = buildIndex(allTopics);
+  const retrieved = retrieveTopK(index, userMessage, TOP_K);
+
+  const includedById = new Map();
+  for (const id of mentionedIds) {
+    const t = allTopics.find(x => x.id === id);
+    if (t) includedById.set(id, t);
+  }
+  for (const t of retrieved) {
+    if (!includedById.has(t.id)) includedById.set(t.id, t);
+  }
+  const included = [...includedById.values()];
+
+  const topicLines = included.map(t => {
     const desc = (t.description || '').replace(/\s+/g, ' ').trim().slice(0, 200);
     return `- ${t.id} (${t.type || 'concept'}) "${t.label}": ${desc}`;
   });
 
-  const relLines = relations.map(r => {
+  const includedIds = new Set(included.map(t => t.id));
+  const relLines = [];
+  for (const r of allRelations) {
     const src = typeof r.source === 'object' ? r.source.id : r.source;
     const tgt = typeof r.target === 'object' ? r.target.id : r.target;
-    return `- ${src} --${r.type}--> ${tgt}`;
-  });
-
-  // Pull deep content for any topic explicitly mentioned in the user message.
-  const lowered = userMessage.toLowerCase();
-  const mentionedDeepContent = [];
-  for (const t of topics) {
-    const idHit = t.id && lowered.includes(t.id.toLowerCase());
-    const labelHit = t.label && lowered.includes(t.label.toLowerCase());
-    if (idHit || labelHit) {
-      const content = await db.getContent(t.id).catch(() => null);
-      if (content) {
-        let raw;
-        if (typeof content === 'string') raw = content;
-        else if (content.article) raw = content.article;
-        else raw = JSON.stringify(content);
-        const snippet = raw.replace(/\s+/g, ' ').trim().slice(0, 1200);
-        mentionedDeepContent.push(`### ${t.label}\n${snippet}`);
-      }
+    if (includedIds.has(src) && includedIds.has(tgt)) {
+      relLines.push(`- ${src} --${r.type}--> ${tgt}`);
     }
   }
 
+  const mentionedDeepContent = [];
+  for (const id of mentionedIds) {
+    const t = includedById.get(id);
+    if (!t) continue;
+    const content = await db.getContent(t.id).catch(() => null);
+    if (!content) continue;
+    let raw;
+    if (typeof content === 'string') raw = content;
+    else if (content.article) raw = typeof content.article === 'string' ? content.article : JSON.stringify(content.article);
+    else raw = JSON.stringify(content);
+    const snippet = raw.replace(/\s+/g, ' ').trim().slice(0, 1200);
+    mentionedDeepContent.push(`### ${t.label}\n${snippet}`);
+  }
+
   const context = [
-    `KENNISGRAAF — TOPICS:`,
+    `KENNISGRAAF — RELEVANTE TOPICS (top ${included.length} van ${allTopics.length}):`,
     topicLines.join('\n'),
     ``,
-    `KENNISGRAAF — RELATIES:`,
-    relLines.length ? relLines.join('\n') : '(geen relaties)',
+    `KENNISGRAAF — RELATIES (binnen deze selectie):`,
+    relLines.length ? relLines.join('\n') : '(geen relaties binnen deze selectie)',
     mentionedDeepContent.length
       ? `\n\nDIEPERE INHOUD (voor genoemde topics):\n${mentionedDeepContent.join('\n\n')}`
       : '',
+    ``,
+    `Als de relevante context hierboven te beperkt is, gebruik dan de tool "lookup_topic" om de volledige beschrijving en eventuele leerinhoud van een specifiek topic op te halen.`,
+    `[kb_hash: ${kbHash}]`,
   ].join('\n');
 
-  return { context, topics };
+  return { context, retrievedTopics: included, allTopics };
 }
 
 /**