feat: phase 2 of AI pipeline hardening — tool-based structured outputs + prompt caching

Every structured-output call now uses an Anthropic tool instead of
parsing JSON out of free-form prose, and stable system prompts are
sent as cacheable blocks. Behaviour-equivalent to phase 1 from the
caller's point of view; the savings show up in token usage and in the
absence of "AI returned non-JSON response" failure modes.

* src/lib/llmTools.js — single source of truth for tool definitions:
  emit_knowledge_graph, emit_handbook_delta, emit_learning_article /
  _slides / _infographic / _all, emit_custom_topic, emit_quiz_questions,
  emit_graph_actions, plus five article-patch tools (set_intro,
  set_section, add_section, remove_section, replace_takeaways).
* src/lib/articlePatches.js — pure applyArticlePatches +
  applyAndValidate; rebuilds the article from a sequence of patch tool
  calls and re-validates against learningArticleSchema. set_section
  falls back to appending when no matching heading exists so the
  model's intent is preserved rather than silently dropped.
* src/lib/llmSchemas.js — Zod schemas for the five patch ops,
  registered in toolSchemaRegistry so callLLM validates them
  automatically.
* src/lib/llm.js — simulation mode now returns a tool_use stub matching
  toolChoice.name, so the UI keeps working with Simulation Mode on
  after the structured-output migration.
* src/lib/extractionPipeline.js — processSourceText and
  analyzeHandbookDelta migrated to callLLM + tool use. System prompts
  sent as { cache_control: ephemeral } blocks. Handbook results pass
  through normalizeHandbookResult to collapse legacy "executes"
  relations into executed_by with swapped source/target.
* src/lib/learningService.js — generateLearningContent picks the right
  tool per selectedType; generateCustomTopic uses emit_custom_topic;
  refineLearningContent now drives the five patch tools with
  toolChoice 'any' and rejects the whole turn if the patched article
  fails validation. Article-only refinement is intentional for phase 2;
  refining a topic without an article surfaces a clear error.
* src/lib/testService.js — quiz generation via emit_quiz_questions.
* src/components/admin/KnowledgeGraph.jsx — analyzeGraph routed through
  the reasoning tier (Opus) since graph-wide consolidation benefits
  from a stronger reasoner.
* src/components/chat/prompts.js — buildSystemPrompt now returns three
  text blocks: stable preamble (cached), KB context (cached, hash-bust
  deferred to phase 5), per-turn user/admin tail (uncached).
* src/lib/__tests__/ — 13 new tests covering each patch op, multi-op
  sequencing, post-patch validation failure, and tool/registry shape.

Acceptance: lint and 45/45 tests green; build succeeds; no
`match(/\{[\s\S]*\}/)` JSON extraction left in src/. Live verification
of cache hits on a second extraction within 5 minutes is deferred to
manual smoke testing — needs real `/api/anthropic` traffic.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

src/lib/learningService.js

@@ -1,51 +1,37 @@
-import { anthropicApi } from './api';
 import * as db from './db';
+import { callLLM } from './llm';
+import {
+  EMIT_LEARNING_ARTICLE_TOOL,
+  EMIT_LEARNING_SLIDES_TOOL,
+  EMIT_LEARNING_INFOGRAPHIC_TOOL,
+  EMIT_LEARNING_ALL_TOOL,
+  EMIT_CUSTOM_TOPIC_TOOL,
+  ARTICLE_PATCH_TOOLS,
+} from './llmTools';
+import { applyAndValidate } from './articlePatches';
 import { getCurriculumTopic } from './curriculumService';
 
 const CONTENT_GENERATION_SYSTEM = `You are an expert learning content writer for Respellion, an internal IT company.
 You write training material for employees based on knowledge topics.
 Always write in clear, professional English.
-ALWAYS return valid JSON only — no markdown code blocks, no extra text.`;
 
-const CONTENT_SCHEMA_ARTICLE = `{
-  "article": {
-    "title": "Article title",
-    "intro": "Short intro of 1-2 sentences",
-    "sections": [
-      { "heading": "Section title", "body": "Section text of at least 3 sentences." }
-    ],
-    "keyTakeaways": ["Takeaway 1", "Takeaway 2", "Takeaway 3"]
-  }
-}`;
+Emit the requested content through the matching tool — do not return prose JSON.`;
 
-const CONTENT_SCHEMA_SLIDES = `{
-  "slides": [
-    { "title": "Slide title", "bullets": ["Point 1", "Point 2", "Point 3"], "speakerNote": "Speaker note for this slide." }
-  ]
-}`;
+const cachedSystem = (text) => [{ type: 'text', text, cache_control: { type: 'ephemeral' } }];
 
+const TOOL_BY_TYPE = {
+  article: EMIT_LEARNING_ARTICLE_TOOL,
+  slides: EMIT_LEARNING_SLIDES_TOOL,
+  infographic: EMIT_LEARNING_INFOGRAPHIC_TOOL,
+  all: EMIT_LEARNING_ALL_TOOL,
+};
 
-
-const CONTENT_SCHEMA_INFOGRAPHIC = `{
-  "infographic": {
-    "headline": "A short, punchy headline summarizing the topic (max 8 words)",
-    "tagline": "A subtitle of max 15 words",
-    "stats": [
-      { "value": "Number or %", "label": "Short description", "icon": "📊" }
-    ],
-    "steps": [
-      { "number": 1, "title": "Step title", "description": "One-sentence description.", "icon": "🔑" }
-    ],
-    "quote": "An inspiring or insightful quote about the topic.",
-    "colorTheme": "teal"
-  }
-}`;
-
-const CONTENT_SCHEMA_ALL = `{
-  "article": ${CONTENT_SCHEMA_ARTICLE.replace(/^\{|\}$/g, '').trim()},
-  "slides": ${CONTENT_SCHEMA_SLIDES.replace(/^\{|\}$/g, '').trim()},
-  "infographic": ${CONTENT_SCHEMA_INFOGRAPHIC.replace(/^\{|\}$/g, '').trim()}
-}`;
+const INSTRUCTIONS_BY_TYPE = {
+  article: 'Provide at least 3 article sections and at least 2 key takeaways.',
+  slides: 'Provide at least 4 slides.',
+  infographic: 'Provide at least 3 stats and 3 steps.',
+  all: 'Provide at least 3 article sections, 4 slides, 3 stats, and 3 steps in the infographic.',
+};
 
 /**
  * Get the assigned topic for a given week.
@@ -53,7 +39,6 @@ const CONTENT_SCHEMA_ALL = `{
  * Falls back to hash-based assignment if no curriculum is configured.
  */
 export async function getAssignedTopic(userId, weekNumber) {
-  // Try curriculum first
   try {
     const { topic } = await getCurriculumTopic(weekNumber);
     if (topic && topic.learning_relevance !== 'exclude') return topic;
@@ -61,9 +46,7 @@ export async function getAssignedTopic(userId, weekNumber) {
     console.warn('[Learn] Curriculum lookup failed, falling back to hash:', e.message);
   }
 
-  // Fallback: hash-based assignment (backwards compatible)
   const allTopics = await db.getTopics();
-  // Filter out 'fact' type topics and 'exclude' relevance topics
   const topics = allTopics.filter(t => t.type !== 'fact' && t.learning_relevance !== 'exclude');
   if (!topics || topics.length === 0) return null;
 
@@ -96,29 +79,15 @@ export async function generateLearningContent(topic, force = false, selectedType
   let cached = null;
   if (!force) {
     cached = await db.getContent(topic.id);
-    if (cached) {
-      if (cached[selectedType]) {
-        console.log(`[Learn] Cache hit for topic: ${topic.id} (${selectedType})`);
-        return cached;
-      }
+    if (cached && cached[selectedType]) {
+      console.log(`[Learn] Cache hit for topic: ${topic.id} (${selectedType})`);
+      return cached;
     }
   }
 
-  let schema = '';
-  let instructions = '';
-  if (selectedType === 'all') {
-    schema = CONTENT_SCHEMA_ALL;
-    instructions = 'Provide at least 3 article sections, 4 slides, 3 stats, and 3-5 steps in the infographic.';
-  } else if (selectedType === 'article') {
-    schema = CONTENT_SCHEMA_ARTICLE;
-    instructions = 'Provide at least 3 article sections.';
-  } else if (selectedType === 'slides') {
-    schema = CONTENT_SCHEMA_SLIDES;
-    instructions = 'Provide at least 4 slides.';
-  } else if (selectedType === 'infographic') {
-    schema = CONTENT_SCHEMA_INFOGRAPHIC;
-    instructions = 'Provide at least 3 stats, and 3-5 steps in the infographic.';
-  }
+  const tool = TOOL_BY_TYPE[selectedType];
+  if (!tool) throw new Error(`Unknown learning content type: ${selectedType}`);
+  const instructions = INSTRUCTIONS_BY_TYPE[selectedType];
 
   const prompt = `Generate a learning module piece for the following topic:
 
@@ -126,20 +95,20 @@ Label: ${topic.label}
 Type: ${topic.type}
 Description: ${topic.description}
 
-Return ONLY a JSON object with the following structure:
-${schema}
-
 ${instructions}`;
 
-  const responseText = await anthropicApi.generateContent(CONTENT_GENERATION_SYSTEM, prompt);
+  const result = await callLLM({
+    task: `learning.${selectedType}`,
+    tier: 'standard',
+    system: cachedSystem(CONTENT_GENERATION_SYSTEM),
+    user: prompt,
+    tools: [tool],
+    toolChoice: { type: 'tool', name: tool.name },
+    maxTokens: 8192,
+  });
 
-  let newContent;
-  try {
-    const jsonMatch = responseText.match(/\{[\s\S]*\}/);
-    newContent = JSON.parse(jsonMatch ? jsonMatch[0] : responseText);
-  } catch (e) {
-    throw new Error('AI could not generate valid learning content. Please try again.', { cause: e });
-  }
+  const newContent = result.toolUses[0]?.input;
+  if (!newContent) throw new Error('AI did not return learning content. Please try again.');
 
   const mergedContent = { ...(cached || {}), ...newContent };
   await db.setContent(topic.id, mergedContent);
@@ -148,28 +117,37 @@ ${instructions}`;
 
 export async function refineLearningContent(topic, refinementInstruction) {
   const existing = await db.getContent(topic.id);
+  if (!existing?.article) {
+    throw new Error('Refinement is currently only supported for the article. Generate an article for this topic first.');
+  }
 
-  const prompt = `You have previously generated the following learning module for the topic "${topic.label}":
+  const prompt = `You have previously generated the following article for the topic "${topic.label}":
 
-${JSON.stringify(existing, null, 2)}
+${JSON.stringify(existing.article, null, 2)}
 
 The admin has requested the following refinement:
 "${refinementInstruction}"
 
-Apply the refinement and return the complete updated JSON object using the same structure. Return ONLY valid JSON.`;
+Apply the refinement by calling one or more of the available patch tools. Make the smallest set of changes that satisfies the instruction — do not rewrite untouched sections.`;
 
-  const responseText = await anthropicApi.generateContent(CONTENT_GENERATION_SYSTEM, prompt);
+  const result = await callLLM({
+    task: 'learning.refine',
+    tier: 'standard',
+    system: cachedSystem(CONTENT_GENERATION_SYSTEM),
+    user: prompt,
+    tools: ARTICLE_PATCH_TOOLS,
+    toolChoice: { type: 'any' },
+    maxTokens: 4096,
+  });
 
-  let content;
-  try {
-    const jsonMatch = responseText.match(/\{[\s\S]*\}/);
-    content = JSON.parse(jsonMatch ? jsonMatch[0] : responseText);
-  } catch (e) {
-    throw new Error('AI could not process the refinement. Please try a different instruction.', { cause: e });
+  if (!result.toolUses.length) {
+    throw new Error('AI did not propose any changes for that instruction. Try a more specific request.');
   }
 
-  await db.setContent(topic.id, content);
-  return content;
+  const patchedArticle = applyAndValidate(existing.article, result.toolUses);
+  const merged = { ...existing, article: patchedArticle };
+  await db.setContent(topic.id, merged);
+  return merged;
 }
 
 export async function deleteCachedContent(topicId) {
@@ -177,30 +155,20 @@ export async function deleteCachedContent(topicId) {
 }
 
 export async function generateCustomTopic(label) {
-  const prompt = `A user wants to learn about "${label}".
-Create a short description (2-3 sentences) and categorize it.
+  const result = await callLLM({
+    task: 'topic.custom',
+    tier: 'standard',
+    system: cachedSystem('You are a knowledge graph AI categorising user-requested topics for the Respellion learning platform.'),
+    user: `A user wants to learn about "${label}". Provide a polished label, type, and 2–3 sentence description via the emit_custom_topic tool.`,
+    tools: [EMIT_CUSTOM_TOPIC_TOOL],
+    toolChoice: { type: 'tool', name: EMIT_CUSTOM_TOPIC_TOOL.name },
+    maxTokens: 1024,
+  });
 
-Return ONLY a JSON object with this structure:
-{
-  "label": "Polished topic title",
-  "type": "concept", // one of: concept, role, process
-  "description": "Short description"
-}`;
-
-  const responseText = await anthropicApi.generateContent(
-    "You are a knowledge graph AI categorizing topics.",
-    prompt
-  );
-
-  let newTopic;
-  try {
-    const jsonMatch = responseText.match(/\{[\s\S]*\}/);
-    newTopic = JSON.parse(jsonMatch ? jsonMatch[0] : responseText);
-    newTopic.id = 'custom_' + Date.now().toString(36);
-  } catch (e) {
-    throw new Error('Could not process custom topic. Please try again.', { cause: e });
-  }
+  const emitted = result.toolUses[0]?.input;
+  if (!emitted) throw new Error('Could not process custom topic. Please try again.');
 
+  const newTopic = { ...emitted, id: 'custom_' + Date.now().toString(36) };
   await db.upsertTopic(newTopic);
   return newTopic;
 }