From f838755991986ea77591b79f7f0c07db6b674fb0 Mon Sep 17 00:00:00 2001 From: RaymondVerhoef Date: Wed, 20 May 2026 15:47:20 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20phase=202=20of=20AI=20pipeline=20harden?= =?UTF-8?q?ing=20=E2=80=94=20tool-based=20structured=20outputs=20+=20promp?= =?UTF-8?q?t=20caching?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every structured-output call now uses an Anthropic tool instead of parsing JSON out of free-form prose, and stable system prompts are sent as cacheable blocks. Behaviour-equivalent to phase 1 from the caller's point of view; the savings show up in token usage and in the absence of "AI returned non-JSON response" failure modes. * src/lib/llmTools.js — single source of truth for tool definitions: emit_knowledge_graph, emit_handbook_delta, emit_learning_article / _slides / _infographic / _all, emit_custom_topic, emit_quiz_questions, emit_graph_actions, plus five article-patch tools (set_intro, set_section, add_section, remove_section, replace_takeaways). * src/lib/articlePatches.js — pure applyArticlePatches + applyAndValidate; rebuilds the article from a sequence of patch tool calls and re-validates against learningArticleSchema. set_section falls back to appending when no matching heading exists so the model's intent is preserved rather than silently dropped. * src/lib/llmSchemas.js — Zod schemas for the five patch ops, registered in toolSchemaRegistry so callLLM validates them automatically. * src/lib/llm.js — simulation mode now returns a tool_use stub matching toolChoice.name, so the UI keeps working with Simulation Mode on after the structured-output migration. * src/lib/extractionPipeline.js — processSourceText and analyzeHandbookDelta migrated to callLLM + tool use. System prompts sent as { cache_control: ephemeral } blocks. Handbook results pass through normalizeHandbookResult to collapse legacy "executes" relations into executed_by with swapped source/target. * src/lib/learningService.js — generateLearningContent picks the right tool per selectedType; generateCustomTopic uses emit_custom_topic; refineLearningContent now drives the five patch tools with toolChoice 'any' and rejects the whole turn if the patched article fails validation. Article-only refinement is intentional for phase 2; refining a topic without an article surfaces a clear error. * src/lib/testService.js — quiz generation via emit_quiz_questions. * src/components/admin/KnowledgeGraph.jsx — analyzeGraph routed through the reasoning tier (Opus) since graph-wide consolidation benefits from a stronger reasoner. * src/components/chat/prompts.js — buildSystemPrompt now returns three text blocks: stable preamble (cached), KB context (cached, hash-bust deferred to phase 5), per-turn user/admin tail (uncached). * src/lib/__tests__/ — 13 new tests covering each patch op, multi-op sequencing, post-patch validation failure, and tool/registry shape. Acceptance: lint and 45/45 tests green; build succeeds; no `match(/\{[\s\S]*\}/)` JSON extraction left in src/. Live verification of cache hits on a second extraction within 5 minutes is deferred to manual smoke testing — needs real `/api/anthropic` traffic. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/components/admin/KnowledgeGraph.jsx | 44 +-- src/components/chat/prompts.js | 63 +++-- src/lib/__tests__/articlePatches.test.js | 104 ++++++++ src/lib/__tests__/llmTools.test.js | 44 +++ src/lib/articlePatches.js | 80 ++++++ src/lib/extractionPipeline.js | 144 ++++------ src/lib/learningService.js | 180 ++++++------- src/lib/llm.js | 92 +++++-- src/lib/llmSchemas.js | 30 +++ src/lib/llmTools.js | 324 +++++++++++++++++++++++ src/lib/testService.js | 58 ++-- 11 files changed, 872 insertions(+), 291 deletions(-) create mode 100644 src/lib/__tests__/articlePatches.test.js create mode 100644 src/lib/__tests__/llmTools.test.js create mode 100644 src/lib/articlePatches.js create mode 100644 src/lib/llmTools.js diff --git a/src/components/admin/KnowledgeGraph.jsx b/src/components/admin/KnowledgeGraph.jsx index 934707d..40898ab 100644 --- a/src/components/admin/KnowledgeGraph.jsx +++ b/src/components/admin/KnowledgeGraph.jsx @@ -2,7 +2,8 @@ import { useCallback, useEffect, useRef, useState } from 'react'; import * as d3 from 'd3'; import { Trash2, Edit2, Save, X, RefreshCw, AlertCircle, Plus, Link as LinkIcon } from 'lucide-react'; import * as db from '../../lib/db'; -import { anthropicApi } from '../../lib/api'; +import { callLLM } from '../../lib/llm'; +import { EMIT_GRAPH_ACTIONS_TOOL } from '../../lib/llmTools'; import { analyzeHandbookDelta } from '../../lib/extractionPipeline'; import { getRepoFolder, getFileContent } from '../../lib/githubService'; import Button from '../ui/Button'; @@ -304,18 +305,18 @@ const KnowledgeGraph = () => { const currentTopics = await db.getTopics(); const currentRelations = await db.getRelations(); - const systemPrompt = `You are a strict Data Quality AI maintaining a Knowledge Graph. -Your goal is to evaluate the provided topics and relations, identify duplicates to merge, useless nodes to delete, and new logical relations to add. + const systemPrompt = `You are a strict Data Quality AI maintaining a Knowledge Graph for Respellion. +Evaluate the provided topics and relations and emit the actions to take via the emit_graph_actions tool. Rules: -1. Identify topics that mean exactly the same thing. Choose one to keep, and one to delete. -2. Identify topics that are too vague, irrelevant, or malformed to delete. -3. Identify missing logical relations (depends_on, part_of, related_to) if two topics are conceptually linked but missing a relation. -4. Evaluate the learning_relevance of each topic. If a topic is purely operational/mundane (like a printer guide), mark it as "exclude". If it's low priority, mark "peripheral". -5. Return ONLY a valid JSON object describing the ACTIONS to take. Do not return the entire graph. Do not wrap in markdown blocks.`; +1. Identify topics that mean exactly the same thing. Choose one to keep, one to delete (merges). +2. Identify topics that are too vague, irrelevant, or malformed (deletions). +3. Identify missing logical relations (depends_on, part_of, related_to, executed_by) between conceptually linked topics (newRelations). +4. Evaluate learning_relevance. Mark purely operational topics (printer guides, etc.) as "exclude"; low-priority as "peripheral" (relevanceUpdates). + +Do not return the entire graph — only the actions to take.`; // Send a compact representation to minimize token usage and avoid rate limits. - // The AI only needs id, label, type, and relevance to identify duplicates/merges and adjust relevance. const compactTopics = currentTopics.map(({ id, label, type, learning_relevance }) => ({ id, label, type, learning_relevance })); const compactRelations = currentRelations.map(r => ({ source: r.source?.id || r.source, @@ -324,21 +325,20 @@ Rules: })); const userPrompt = `Here is the current knowledge graph: -${JSON.stringify({ topics: compactTopics, relations: compactRelations })} +${JSON.stringify({ topics: compactTopics, relations: compactRelations })}`; -Analyze this graph and return ONLY the optimized JSON object with this EXACT structure: -{ - "merges": [ { "keepId": "id_to_keep", "deleteId": "id_to_delete" } ], - "deletions": [ "id_to_delete_completely" ], - "newRelations": [ { "source": "id1", "target": "id2", "type": "depends_on" } ], - "relevanceUpdates": [ { "id": "topic_id", "learning_relevance": "exclude" } ] -}`; + const llmResult = await callLLM({ + task: 'graph.analyze', + tier: 'reasoning', + system: [{ type: 'text', text: systemPrompt, cache_control: { type: 'ephemeral' } }], + user: userPrompt, + tools: [EMIT_GRAPH_ACTIONS_TOOL], + toolChoice: { type: 'tool', name: EMIT_GRAPH_ACTIONS_TOOL.name }, + maxTokens: 4096, + }); - const responseText = await anthropicApi.generateContent(systemPrompt, userPrompt); - const jsonMatch = responseText.match(/\{[\s\S]*\}/); - if (!jsonMatch) throw new Error('AI returned invalid format.'); - - const actions = JSON.parse(jsonMatch[0]); + const actions = llmResult.toolUses[0]?.input; + if (!actions) throw new Error('Graph analysis did not emit a tool result.'); let updatedTopics = [...currentTopics]; let updatedRelations = [...currentRelations]; diff --git a/src/components/chat/prompts.js b/src/components/chat/prompts.js index c771970..0268d44 100644 --- a/src/components/chat/prompts.js +++ b/src/components/chat/prompts.js @@ -23,32 +23,49 @@ export const STRINGS = { openAria: 'Open R42 chatbot', }; +const STABLE_PREAMBLE = [ + `Je bent R42, de chatbot-avatar van Respellion — een leerplatform voor microlearning, quizzen en kennisontwikkeling.`, + `Antwoord altijd in het Nederlands, kort en zakelijk-vriendelijk. Spreek de gebruiker aan met hun voornaam wanneer dat natuurlijk voelt.`, + ``, + `JE TAKEN:`, + `1. Leg onderwerpen uit die in de kennisbasis staan.`, + `2. Help de gebruiker quizvragen begrijpen ná afloop (niet tijdens een actieve quiz).`, + `3. Verwijs bij twijfel terug naar het bronmateriaal of zeg eerlijk dat je het niet weet.`, + ``, + `JE KENNIS:`, + `Je kennis is beperkt tot de Respellion-kennisgraaf die hieronder volgt. Als een vraag duidelijk buiten dit bereik valt, zeg dat dan eerlijk en stel voor dat de gebruiker de bron toevoegt via Admin → Sources.`, + ``, + `KENNISGRAAF VERFIJNEN:`, + `Wanneer de gebruiker iets noemt dat duidelijk een nieuw topic, nieuwe relatie, proces of rol is — en dat nog niet in de kennisgraaf staat — gebruik dan de tool "propose_graph_delta" om een voorstel te maken. Verzin niets: stel alleen iets voor als de gebruiker het concreet noemt. Stel maximaal 3 topics en 5 relaties per beurt voor.`, + ``, + `STIJL:`, + `- Houd antwoorden onder de 4 zinnen tenzij de gebruiker om uitleg vraagt.`, + `- Geen markdown-headers; gewone Nederlandse tekst.`, + `- Bij onzekerheid: "Ik weet het niet zeker — controleer dit in het handboek."`, +].join('\n'); + +/** + * Build the R42 system prompt as three cacheable blocks: + * 1. stable preamble (role, tasks, style) — cached + * 2. KB context (current topics + relations) — cached (hash-bust comes in Phase 5) + * 3. per-turn tail (user name + admin status) — NOT cached + * + * Returning an array lets `callLLM` pass it through unchanged so the + * Anthropic API caches each block with the 5-minute ephemeral TTL. + */ export function buildSystemPrompt({ userName, isAdmin, kbContext }) { - return [ - `Je bent R42, de chatbot-avatar van Respellion — een leerplatform voor microlearning, quizzen en kennisontwikkeling.`, - `Antwoord altijd in het Nederlands, kort en zakelijk-vriendelijk. Spreek de gebruiker aan met hun voornaam wanneer dat natuurlijk voelt (${userName}).`, - ``, - `JE TAKEN:`, - `1. Leg onderwerpen uit die in de kennisbasis staan.`, - `2. Help de gebruiker quizvragen begrijpen ná afloop (niet tijdens een actieve quiz).`, - `3. Verwijs bij twijfel terug naar het bronmateriaal of zeg eerlijk dat je het niet weet.`, - ``, - `JE KENNIS:`, - `Je kennis is beperkt tot de onderstaande Respellion-kennisgraaf. Als een vraag duidelijk buiten dit bereik valt, zeg dat dan eerlijk en stel voor dat de gebruiker de bron toevoegt via Admin → Sources.`, - ``, - kbContext, - ``, - `KENNISGRAAF VERFIJNEN:`, - `Wanneer de gebruiker iets noemt dat duidelijk een nieuw topic, nieuwe relatie, proces of rol is — en dat nog niet in de kennisgraaf staat — gebruik dan de tool "propose_graph_delta" om een voorstel te maken. Verzin niets: stel alleen iets voor als de gebruiker het concreet noemt. Stel maximaal 3 topics en 5 relaties per beurt voor.`, - ``, - `STIJL:`, - `- Houd antwoorden onder de 4 zinnen tenzij de gebruiker om uitleg vraagt.`, - `- Geen markdown-headers; gewone Nederlandse tekst.`, - `- Bij onzekerheid: "Ik weet het niet zeker — controleer dit in het handboek."`, + const tail = [ + `De gebruiker heet ${userName}.`, isAdmin - ? `\nDe gebruiker is beheerder; voorstellen die de tool genereert worden direct toegepast.` - : `\nDe gebruiker is geen beheerder; voorstellen worden in een goedkeuringswachtrij gezet.`, + ? `De gebruiker is beheerder; voorstellen die de tool genereert worden direct toegepast.` + : `De gebruiker is geen beheerder; voorstellen worden in een goedkeuringswachtrij gezet.`, ].join('\n'); + + return [ + { type: 'text', text: STABLE_PREAMBLE, cache_control: { type: 'ephemeral' } }, + { type: 'text', text: kbContext, cache_control: { type: 'ephemeral' } }, + { type: 'text', text: tail }, + ]; } export const PROPOSE_GRAPH_DELTA_TOOL = { diff --git a/src/lib/__tests__/articlePatches.test.js b/src/lib/__tests__/articlePatches.test.js new file mode 100644 index 0000000..79f2c8c --- /dev/null +++ b/src/lib/__tests__/articlePatches.test.js @@ -0,0 +1,104 @@ +import { describe, expect, it } from 'vitest'; +import { applyArticlePatches, applyAndValidate } from '../articlePatches'; + +const article = () => ({ + title: 'Onboarding', + intro: 'Old intro.', + sections: [ + { heading: 'Day one', body: 'First day body, three sentences long. Welcome. Read the handbook.' }, + { heading: 'Day two', body: 'Second day body. Three sentences. Meet your team.' }, + ], + keyTakeaways: ['Show up', 'Ask questions'], +}); + +describe('applyArticlePatches', () => { + it('does not mutate the input article', () => { + const original = article(); + const snapshot = JSON.parse(JSON.stringify(original)); + applyArticlePatches(original, [ + { name: 'set_intro', input: { intro: 'New intro.' } }, + ]); + expect(original).toEqual(snapshot); + }); + + it('set_intro replaces the intro', () => { + const result = applyArticlePatches(article(), [ + { name: 'set_intro', input: { intro: 'Punchier intro.' } }, + ]); + expect(result.intro).toBe('Punchier intro.'); + }); + + it('set_section replaces the matching section body (case-insensitive)', () => { + const result = applyArticlePatches(article(), [ + { name: 'set_section', input: { heading: 'DAY ONE', body: 'Rewritten body. With several sentences. Indeed.' } }, + ]); + expect(result.sections[0].body).toMatch(/Rewritten body/); + expect(result.sections[1].body).toMatch(/Second day body/); + }); + + it('add_section position=start prepends a new section', () => { + const result = applyArticlePatches(article(), [ + { name: 'add_section', input: { heading: 'Before', body: 'New intro section. Three sentences. Indeed.', position: 'start' } }, + ]); + expect(result.sections[0].heading).toBe('Before'); + expect(result.sections).toHaveLength(3); + }); + + it('add_section position=end appends a new section', () => { + const result = applyArticlePatches(article(), [ + { name: 'add_section', input: { heading: 'After', body: 'Closing section. Three sentences. Indeed.', position: 'end' } }, + ]); + expect(result.sections[2].heading).toBe('After'); + }); + + it('remove_section drops the matching section', () => { + const result = applyArticlePatches(article(), [ + { name: 'remove_section', input: { heading: 'Day one' } }, + ]); + expect(result.sections).toHaveLength(1); + expect(result.sections[0].heading).toBe('Day two'); + }); + + it('replace_takeaways swaps the key takeaways', () => { + const result = applyArticlePatches(article(), [ + { name: 'replace_takeaways', input: { items: ['First', 'Second', 'Third'] } }, + ]); + expect(result.keyTakeaways).toEqual(['First', 'Second', 'Third']); + }); + + it('applies multiple patches in order', () => { + const result = applyArticlePatches(article(), [ + { name: 'set_intro', input: { intro: 'Brand new intro.' } }, + { name: 'remove_section', input: { heading: 'Day one' } }, + { name: 'add_section', input: { heading: 'New', body: 'Body of the new section. Three sentences. Yes.', position: 'end' } }, + ]); + expect(result.intro).toBe('Brand new intro.'); + expect(result.sections.map(s => s.heading)).toEqual(['Day two', 'New']); + }); + + it('falls back to appending when set_section cannot find a matching heading', () => { + const result = applyArticlePatches(article(), [ + { name: 'set_section', input: { heading: 'Nonexistent', body: 'New body, with three sentences. Yes indeed. Foo.' } }, + ]); + expect(result.sections).toHaveLength(3); + expect(result.sections[2].heading).toBe('Nonexistent'); + }); +}); + +describe('applyAndValidate', () => { + it('returns the patched article when valid', () => { + const patched = applyAndValidate(article(), [ + { name: 'set_intro', input: { intro: 'Tighter intro.' } }, + ]); + expect(patched.intro).toBe('Tighter intro.'); + }); + + it('throws when patches strip the article to invalid', () => { + expect(() => + applyAndValidate(article(), [ + { name: 'remove_section', input: { heading: 'Day one' } }, + { name: 'remove_section', input: { heading: 'Day two' } }, + ]), + ).toThrow(/invalid article/i); + }); +}); diff --git a/src/lib/__tests__/llmTools.test.js b/src/lib/__tests__/llmTools.test.js new file mode 100644 index 0000000..2a3fe0c --- /dev/null +++ b/src/lib/__tests__/llmTools.test.js @@ -0,0 +1,44 @@ +import { describe, expect, it } from 'vitest'; +import { + EMIT_KNOWLEDGE_GRAPH_TOOL, + EMIT_HANDBOOK_DELTA_TOOL, + EMIT_LEARNING_ARTICLE_TOOL, + EMIT_LEARNING_SLIDES_TOOL, + EMIT_LEARNING_INFOGRAPHIC_TOOL, + EMIT_LEARNING_ALL_TOOL, + EMIT_CUSTOM_TOPIC_TOOL, + EMIT_QUIZ_QUESTIONS_TOOL, + EMIT_GRAPH_ACTIONS_TOOL, + ARTICLE_PATCH_TOOLS, +} from '../llmTools'; +import { toolSchemaRegistry } from '../llmSchemas'; + +const allTools = [ + EMIT_KNOWLEDGE_GRAPH_TOOL, + EMIT_HANDBOOK_DELTA_TOOL, + EMIT_LEARNING_ARTICLE_TOOL, + EMIT_LEARNING_SLIDES_TOOL, + EMIT_LEARNING_INFOGRAPHIC_TOOL, + EMIT_LEARNING_ALL_TOOL, + EMIT_CUSTOM_TOPIC_TOOL, + EMIT_QUIZ_QUESTIONS_TOOL, + EMIT_GRAPH_ACTIONS_TOOL, + ...ARTICLE_PATCH_TOOLS, +]; + +describe('llmTools', () => { + it('every tool has a name, description, and object input_schema', () => { + for (const t of allTools) { + expect(typeof t.name).toBe('string'); + expect(t.name.length).toBeGreaterThan(0); + expect(typeof t.description).toBe('string'); + expect(t.input_schema).toMatchObject({ type: 'object' }); + } + }); + + it('every tool has a matching Zod validator in toolSchemaRegistry', () => { + for (const t of allTools) { + expect(toolSchemaRegistry[t.name]).toBeTruthy(); + } + }); +}); diff --git a/src/lib/articlePatches.js b/src/lib/articlePatches.js new file mode 100644 index 0000000..65e1bc9 --- /dev/null +++ b/src/lib/articlePatches.js @@ -0,0 +1,80 @@ +/** + * Apply a sequence of patch operations (the tool_use calls returned by + * `refineLearningContent`) to an article object, in order. The returned + * article is a fresh object — the input is not mutated. + * + * Recognised tool names mirror `llmTools.js`: + * set_intro, set_section, add_section, remove_section, replace_takeaways. + * + * Unknown tool names are ignored on purpose; the caller validates the + * result against `learningArticleSchema` and rejects the whole turn if + * the patches produced an invalid article. + */ + +import { learningArticleSchema } from './llmSchemas'; + +function matchesHeading(section, heading) { + return (section.heading ?? '').trim().toLowerCase() === heading.trim().toLowerCase(); +} + +function cloneArticle(article) { + return { + ...article, + sections: article.sections.map((s) => ({ ...s })), + keyTakeaways: [...article.keyTakeaways], + }; +} + +export function applyArticlePatches(article, toolUses) { + let next = cloneArticle(article); + for (const tu of toolUses) { + switch (tu.name) { + case 'set_intro': + next = { ...next, intro: tu.input.intro }; + break; + case 'set_section': { + const idx = next.sections.findIndex((s) => matchesHeading(s, tu.input.heading)); + if (idx === -1) { + // No matching section — fall back to appending so the model's + // intent (provide that body) is preserved rather than lost. + next.sections = [...next.sections, { heading: tu.input.heading, body: tu.input.body }]; + } else { + next.sections = next.sections.map((s, i) => (i === idx ? { ...s, body: tu.input.body } : s)); + } + break; + } + case 'add_section': { + const newSection = { heading: tu.input.heading, body: tu.input.body }; + next.sections = tu.input.position === 'start' + ? [newSection, ...next.sections] + : [...next.sections, newSection]; + break; + } + case 'remove_section': + next.sections = next.sections.filter((s) => !matchesHeading(s, tu.input.heading)); + break; + case 'replace_takeaways': + next = { ...next, keyTakeaways: [...tu.input.items] }; + break; + default: + // Unknown patch op — ignore. + break; + } + } + return next; +} + +/** + * Apply the patches and re-validate against the article schema. Throws + * a clear error if the result is invalid. + */ +export function applyAndValidate(article, toolUses) { + const updated = applyArticlePatches(article, toolUses); + const parsed = learningArticleSchema.safeParse({ article: updated }); + if (!parsed.success) { + const err = new Error(`Refinement produced an invalid article: ${parsed.error.message}`); + err.cause = parsed.error; + throw err; + } + return parsed.data.article; +} diff --git a/src/lib/extractionPipeline.js b/src/lib/extractionPipeline.js index 4de6d20..e553af9 100644 --- a/src/lib/extractionPipeline.js +++ b/src/lib/extractionPipeline.js @@ -1,85 +1,62 @@ -import { anthropicApi } from './api'; import * as db from './db'; +import { callLLM } from './llm'; +import { EMIT_KNOWLEDGE_GRAPH_TOOL, EMIT_HANDBOOK_DELTA_TOOL } from './llmTools'; +import { normalizeHandbookResult } from './llmSchemas'; -const SYSTEM_PROMPT = `You are an AI knowledge extractor for Respellion, an IT company built on radical transparency. -You receive a source text. Your task is to extract all core concepts, roles, and processes from the text, and return them as a structured JSON Knowledge Graph. -Facts should be integrated into the descriptions of the other labels and NOT be extracted as unique topics. +const EXTRACTION_SYSTEM_PROMPT = `You are an AI knowledge extractor for Respellion, an IT company built on radical transparency. +You receive a source text. Extract every distinct concept, role, and process from it and emit them through the emit_knowledge_graph tool. CRITICAL INSTRUCTIONS FOR COMPLETENESS: -- You must extract EVERY SINGLE distinct role, process, and concept described or mentioned in the source text. -- DO NOT summarize, skip, truncate, or omit any items. -- If the document contains 29 roles, your JSON topics array must contain exactly 29 role topics. -- Completeness is of paramount importance. Failing to extract all topics will result in loss of critical company knowledge. -- Keep descriptions concise (max 3 sentences) to ensure you have enough output tokens to list everything. +- Extract EVERY SINGLE distinct role, process, and concept described or mentioned in the source text. +- DO NOT summarise, skip, truncate, or omit any items. +- If the document contains 29 roles, the topics array must contain exactly 29 role topics. +- Completeness is paramount. Failing to extract all topics loses critical company knowledge. +- Facts should be integrated into the descriptions of other topics — never extracted as standalone topics. +- Keep descriptions concise (max 3 sentences) so the response fits. -You MUST assign a learning_relevance to each topic: -- "core": Fundamental company knowledge. -- "standard": Normal learning topics. -- "peripheral": Good to know, but low priority. -- "exclude": Pure operational reference material (e.g., printer guides, wifi passwords) that should NEVER be tested. +Topic IDs are lowercase kebab-case slugs specific to the topic (e.g. "software-engineer", "data-quality-review"). Do not use generic IDs like "role-1" or "concept-2". -ALWAYS return a valid JSON object in the following format: -{ - "topics": [ - { - "id": "a-unique-lowercase-kebab-case-slug-specific-to-this-topic (e.g., 'software-engineer' or 'data-quality-review'). DO NOT use generic IDs like 'role-1' or 'concept-2'.", - "label": "Topic title", - "type": "concept | role | process", - "description": "A concise, clear explanation of max 3 sentences.", - "learning_relevance": "core | standard | peripheral | exclude" - } - ], - "relations": [ - { - "source": "topic-id-1", - "target": "topic-id-2", - "type": "related_to | depends_on | part_of | executed_by" - } - ] -} -Return JSON only. No markdown blocks or other text.`; +Assign a learning_relevance to every topic: +- "core": fundamental company knowledge. +- "standard": normal learning topics. +- "peripheral": good to know, low priority. +- "exclude": pure operational reference (printer guides, wifi passwords) that should never be tested. -const HANDBOOK_SYSTEM_PROMPT = `You are analyzing an update to the Respellion Employee Handbook. -Your task is to identify changes and extract structural knowledge. +Relation types: related_to | depends_on | part_of | executed_by. +`; -CRITICAL INSTRUCTION: -You must explicitly identify and create relations between Roles, Processes, and Concepts. -Every Process must have a Role attached (who does it). -Every Concept must have a relation to a Process or Role. +const HANDBOOK_SYSTEM_PROMPT = `You are analysing an update to the Respellion Employee Handbook. Emit the extracted topics and relations through the emit_handbook_delta tool. -You MUST assign a learning_relevance to each topic: -- "core": Fundamental company knowledge. -- "standard": Normal learning topics. -- "peripheral": Good to know, but low priority. -- "exclude": Pure operational reference material (e.g., printer guides, wifi passwords) that should NEVER be tested. +CRITICAL INSTRUCTIONS: +- Every process must have a role attached (the role that executes it). +- Every concept must connect to a process or role. +- Mark handbook topics with metadata.source = "github_handbook". +- Assign learning_relevance using the same scale as extraction: core | standard | peripheral | exclude. -Return a JSON object: -{ - "topics": [ - { "id": "...", "label": "...", "type": "role | process | concept", "description": "...", "learning_relevance": "standard", "metadata": { "source": "github_handbook" } } - ], - "relations": [ - { "source": "role-id", "target": "process-id", "type": "executes | related_to | depends_on | part_of", "description": "Brief metadata about this specific relation" } - ] -} -Return JSON only. No markdown blocks or other text.`; +Relation types: related_to | depends_on | part_of | executed_by. (Legacy "executes" relations are normalised by the client into executed_by with source/target swapped.) +`; + +const cachedSystem = (text) => [{ type: 'text', text, cache_control: { type: 'ephemeral' } }]; export async function analyzeHandbookDelta(fileContent, filePath) { - const responseText = await anthropicApi.generateContent(HANDBOOK_SYSTEM_PROMPT, `Analyze the following handbook file update (${filePath}):\n\n${fileContent}`); + const result = await callLLM({ + task: 'extract.handbook', + tier: 'standard', + system: cachedSystem(HANDBOOK_SYSTEM_PROMPT), + user: `Analyze the following handbook file update (${filePath}):\n\n${fileContent}`, + tools: [EMIT_HANDBOOK_DELTA_TOOL], + toolChoice: { type: 'tool', name: EMIT_HANDBOOK_DELTA_TOOL.name }, + maxTokens: 8192, + }); - let extractedData; - try { - const jsonMatch = responseText.match(/\{[\s\S]*\}/); - const jsonStr = jsonMatch ? jsonMatch[0] : responseText; - extractedData = JSON.parse(jsonStr); - } catch (e) { - console.error('[Pipeline] AI returned non-JSON response for handbook delta:', responseText?.substring(0, 500)); - throw new Error(`AI response was not valid JSON. The model responded with: "${responseText?.substring(0, 120)}..."`, { cause: e }); - } + const raw = result.toolUses[0]?.input; + if (!raw) throw new Error('Handbook extraction did not emit a tool result.'); + const extractedData = normalizeHandbookResult(raw); await mergeKnowledgeGraph(extractedData); return { success: true, data: extractedData }; } + function chunkText(text, maxChunkSize = 4000) { const paragraphs = text.split(/\n+/); const chunks = []; @@ -98,7 +75,6 @@ function chunkText(text, maxChunkSize = 4000) { } export async function processSourceText(textContent, sourceName) { - // Deduplicate: skip if a source with the same name was already successfully processed const existing = await db.getSources(); const alreadyDone = existing.find( s => s.name === sourceName && s.status === 'completed' @@ -124,21 +100,18 @@ export async function processSourceText(textContent, sourceName) { } console.log(`[Pipeline] Processing chunk ${i + 1}/${chunks.length} (${chunks[i].length} chars)...`); - const responseText = await anthropicApi.generateContent( - SYSTEM_PROMPT, - `Analyze this part of the document (${i + 1}/${chunks.length}):\n\n${chunks[i]}` - ); - console.log(`[Pipeline] Raw AI response for chunk ${i + 1}:`, responseText); + const result = await callLLM({ + task: 'extract.source', + tier: 'standard', + system: cachedSystem(EXTRACTION_SYSTEM_PROMPT), + user: `Analyze this part of the document (${i + 1}/${chunks.length}):\n\n${chunks[i]}`, + tools: [EMIT_KNOWLEDGE_GRAPH_TOOL], + toolChoice: { type: 'tool', name: EMIT_KNOWLEDGE_GRAPH_TOOL.name }, + maxTokens: 8192, + }); - let extractedData; - try { - const jsonMatch = responseText.match(/\{[\s\S]*\}/); - const jsonStr = jsonMatch ? jsonMatch[0] : responseText; - extractedData = JSON.parse(jsonStr); - } catch (e) { - console.error(`[Pipeline] AI returned non-JSON response for chunk ${i + 1}:`, responseText?.substring(0, 500)); - throw new Error(`AI response for chunk ${i + 1} was not valid JSON.`, { cause: e }); - } + const extractedData = result.toolUses[0]?.input; + if (!extractedData) throw new Error(`Extraction did not emit a tool result for chunk ${i + 1}.`); if (extractedData.topics && Array.isArray(extractedData.topics)) { allExtractedTopics.push(...extractedData.topics); @@ -148,7 +121,6 @@ export async function processSourceText(textContent, sourceName) { } } - // Merge everything together await mergeKnowledgeGraph({ topics: allExtractedTopics, relations: allExtractedRelations }); await db.updateSourceStatus(sourceId, 'completed'); @@ -169,13 +141,11 @@ async function mergeKnowledgeGraph(newData) { if (newData.topics && Array.isArray(newData.topics)) { for (const t of newData.topics) { if (topicsMap.has(t.id)) { - // Upsert: merge new data into existing topic const existing = topicsMap.get(t.id); - topicsMap.set(t.id, { - ...existing, - ...t, - // Keep existing description if new one is empty, or combine them if needed. Here we prefer the new one. - description: t.description || existing.description + topicsMap.set(t.id, { + ...existing, + ...t, + description: t.description || existing.description, }); } else { topicsMap.set(t.id, t); diff --git a/src/lib/learningService.js b/src/lib/learningService.js index a8ca691..788846b 100644 --- a/src/lib/learningService.js +++ b/src/lib/learningService.js @@ -1,51 +1,37 @@ -import { anthropicApi } from './api'; import * as db from './db'; +import { callLLM } from './llm'; +import { + EMIT_LEARNING_ARTICLE_TOOL, + EMIT_LEARNING_SLIDES_TOOL, + EMIT_LEARNING_INFOGRAPHIC_TOOL, + EMIT_LEARNING_ALL_TOOL, + EMIT_CUSTOM_TOPIC_TOOL, + ARTICLE_PATCH_TOOLS, +} from './llmTools'; +import { applyAndValidate } from './articlePatches'; import { getCurriculumTopic } from './curriculumService'; const CONTENT_GENERATION_SYSTEM = `You are an expert learning content writer for Respellion, an internal IT company. You write training material for employees based on knowledge topics. Always write in clear, professional English. -ALWAYS return valid JSON only — no markdown code blocks, no extra text.`; -const CONTENT_SCHEMA_ARTICLE = `{ - "article": { - "title": "Article title", - "intro": "Short intro of 1-2 sentences", - "sections": [ - { "heading": "Section title", "body": "Section text of at least 3 sentences." } - ], - "keyTakeaways": ["Takeaway 1", "Takeaway 2", "Takeaway 3"] - } -}`; +Emit the requested content through the matching tool — do not return prose JSON.`; -const CONTENT_SCHEMA_SLIDES = `{ - "slides": [ - { "title": "Slide title", "bullets": ["Point 1", "Point 2", "Point 3"], "speakerNote": "Speaker note for this slide." } - ] -}`; +const cachedSystem = (text) => [{ type: 'text', text, cache_control: { type: 'ephemeral' } }]; +const TOOL_BY_TYPE = { + article: EMIT_LEARNING_ARTICLE_TOOL, + slides: EMIT_LEARNING_SLIDES_TOOL, + infographic: EMIT_LEARNING_INFOGRAPHIC_TOOL, + all: EMIT_LEARNING_ALL_TOOL, +}; - -const CONTENT_SCHEMA_INFOGRAPHIC = `{ - "infographic": { - "headline": "A short, punchy headline summarizing the topic (max 8 words)", - "tagline": "A subtitle of max 15 words", - "stats": [ - { "value": "Number or %", "label": "Short description", "icon": "📊" } - ], - "steps": [ - { "number": 1, "title": "Step title", "description": "One-sentence description.", "icon": "🔑" } - ], - "quote": "An inspiring or insightful quote about the topic.", - "colorTheme": "teal" - } -}`; - -const CONTENT_SCHEMA_ALL = `{ - "article": ${CONTENT_SCHEMA_ARTICLE.replace(/^\{|\}$/g, '').trim()}, - "slides": ${CONTENT_SCHEMA_SLIDES.replace(/^\{|\}$/g, '').trim()}, - "infographic": ${CONTENT_SCHEMA_INFOGRAPHIC.replace(/^\{|\}$/g, '').trim()} -}`; +const INSTRUCTIONS_BY_TYPE = { + article: 'Provide at least 3 article sections and at least 2 key takeaways.', + slides: 'Provide at least 4 slides.', + infographic: 'Provide at least 3 stats and 3 steps.', + all: 'Provide at least 3 article sections, 4 slides, 3 stats, and 3 steps in the infographic.', +}; /** * Get the assigned topic for a given week. @@ -53,7 +39,6 @@ const CONTENT_SCHEMA_ALL = `{ * Falls back to hash-based assignment if no curriculum is configured. */ export async function getAssignedTopic(userId, weekNumber) { - // Try curriculum first try { const { topic } = await getCurriculumTopic(weekNumber); if (topic && topic.learning_relevance !== 'exclude') return topic; @@ -61,9 +46,7 @@ export async function getAssignedTopic(userId, weekNumber) { console.warn('[Learn] Curriculum lookup failed, falling back to hash:', e.message); } - // Fallback: hash-based assignment (backwards compatible) const allTopics = await db.getTopics(); - // Filter out 'fact' type topics and 'exclude' relevance topics const topics = allTopics.filter(t => t.type !== 'fact' && t.learning_relevance !== 'exclude'); if (!topics || topics.length === 0) return null; @@ -96,29 +79,15 @@ export async function generateLearningContent(topic, force = false, selectedType let cached = null; if (!force) { cached = await db.getContent(topic.id); - if (cached) { - if (cached[selectedType]) { - console.log(`[Learn] Cache hit for topic: ${topic.id} (${selectedType})`); - return cached; - } + if (cached && cached[selectedType]) { + console.log(`[Learn] Cache hit for topic: ${topic.id} (${selectedType})`); + return cached; } } - let schema = ''; - let instructions = ''; - if (selectedType === 'all') { - schema = CONTENT_SCHEMA_ALL; - instructions = 'Provide at least 3 article sections, 4 slides, 3 stats, and 3-5 steps in the infographic.'; - } else if (selectedType === 'article') { - schema = CONTENT_SCHEMA_ARTICLE; - instructions = 'Provide at least 3 article sections.'; - } else if (selectedType === 'slides') { - schema = CONTENT_SCHEMA_SLIDES; - instructions = 'Provide at least 4 slides.'; - } else if (selectedType === 'infographic') { - schema = CONTENT_SCHEMA_INFOGRAPHIC; - instructions = 'Provide at least 3 stats, and 3-5 steps in the infographic.'; - } + const tool = TOOL_BY_TYPE[selectedType]; + if (!tool) throw new Error(`Unknown learning content type: ${selectedType}`); + const instructions = INSTRUCTIONS_BY_TYPE[selectedType]; const prompt = `Generate a learning module piece for the following topic: @@ -126,20 +95,20 @@ Label: ${topic.label} Type: ${topic.type} Description: ${topic.description} -Return ONLY a JSON object with the following structure: -${schema} - ${instructions}`; - const responseText = await anthropicApi.generateContent(CONTENT_GENERATION_SYSTEM, prompt); + const result = await callLLM({ + task: `learning.${selectedType}`, + tier: 'standard', + system: cachedSystem(CONTENT_GENERATION_SYSTEM), + user: prompt, + tools: [tool], + toolChoice: { type: 'tool', name: tool.name }, + maxTokens: 8192, + }); - let newContent; - try { - const jsonMatch = responseText.match(/\{[\s\S]*\}/); - newContent = JSON.parse(jsonMatch ? jsonMatch[0] : responseText); - } catch (e) { - throw new Error('AI could not generate valid learning content. Please try again.', { cause: e }); - } + const newContent = result.toolUses[0]?.input; + if (!newContent) throw new Error('AI did not return learning content. Please try again.'); const mergedContent = { ...(cached || {}), ...newContent }; await db.setContent(topic.id, mergedContent); @@ -148,28 +117,37 @@ ${instructions}`; export async function refineLearningContent(topic, refinementInstruction) { const existing = await db.getContent(topic.id); + if (!existing?.article) { + throw new Error('Refinement is currently only supported for the article. Generate an article for this topic first.'); + } - const prompt = `You have previously generated the following learning module for the topic "${topic.label}": + const prompt = `You have previously generated the following article for the topic "${topic.label}": -${JSON.stringify(existing, null, 2)} +${JSON.stringify(existing.article, null, 2)} The admin has requested the following refinement: "${refinementInstruction}" -Apply the refinement and return the complete updated JSON object using the same structure. Return ONLY valid JSON.`; +Apply the refinement by calling one or more of the available patch tools. Make the smallest set of changes that satisfies the instruction — do not rewrite untouched sections.`; - const responseText = await anthropicApi.generateContent(CONTENT_GENERATION_SYSTEM, prompt); + const result = await callLLM({ + task: 'learning.refine', + tier: 'standard', + system: cachedSystem(CONTENT_GENERATION_SYSTEM), + user: prompt, + tools: ARTICLE_PATCH_TOOLS, + toolChoice: { type: 'any' }, + maxTokens: 4096, + }); - let content; - try { - const jsonMatch = responseText.match(/\{[\s\S]*\}/); - content = JSON.parse(jsonMatch ? jsonMatch[0] : responseText); - } catch (e) { - throw new Error('AI could not process the refinement. Please try a different instruction.', { cause: e }); + if (!result.toolUses.length) { + throw new Error('AI did not propose any changes for that instruction. Try a more specific request.'); } - await db.setContent(topic.id, content); - return content; + const patchedArticle = applyAndValidate(existing.article, result.toolUses); + const merged = { ...existing, article: patchedArticle }; + await db.setContent(topic.id, merged); + return merged; } export async function deleteCachedContent(topicId) { @@ -177,30 +155,20 @@ export async function deleteCachedContent(topicId) { } export async function generateCustomTopic(label) { - const prompt = `A user wants to learn about "${label}". -Create a short description (2-3 sentences) and categorize it. + const result = await callLLM({ + task: 'topic.custom', + tier: 'standard', + system: cachedSystem('You are a knowledge graph AI categorising user-requested topics for the Respellion learning platform.'), + user: `A user wants to learn about "${label}". Provide a polished label, type, and 2–3 sentence description via the emit_custom_topic tool.`, + tools: [EMIT_CUSTOM_TOPIC_TOOL], + toolChoice: { type: 'tool', name: EMIT_CUSTOM_TOPIC_TOOL.name }, + maxTokens: 1024, + }); -Return ONLY a JSON object with this structure: -{ - "label": "Polished topic title", - "type": "concept", // one of: concept, role, process - "description": "Short description" -}`; - - const responseText = await anthropicApi.generateContent( - "You are a knowledge graph AI categorizing topics.", - prompt - ); - - let newTopic; - try { - const jsonMatch = responseText.match(/\{[\s\S]*\}/); - newTopic = JSON.parse(jsonMatch ? jsonMatch[0] : responseText); - newTopic.id = 'custom_' + Date.now().toString(36); - } catch (e) { - throw new Error('Could not process custom topic. Please try again.', { cause: e }); - } + const emitted = result.toolUses[0]?.input; + if (!emitted) throw new Error('Could not process custom topic. Please try again.'); + const newTopic = { ...emitted, id: 'custom_' + Date.now().toString(36) }; await db.upsertTopic(newTopic); return newTopic; } diff --git a/src/lib/llm.js b/src/lib/llm.js index a9b2edc..9e69f59 100644 --- a/src/lib/llm.js +++ b/src/lib/llm.js @@ -125,7 +125,7 @@ function isChatLikeTask(task) { return task === 'legacy.chat' || task.startsWith('chat.') || task.startsWith('r42.'); } -const SIMULATION_EXTRACTION_PAYLOAD = JSON.stringify({ +const SIMULATION_EXTRACTION_GRAPH = { topics: [ { id: 'radicale-transparantie', label: 'Radicale Transparantie', type: 'concept', description: 'De kernwaarde van Respellion waarbij alle informatie publiek toegankelijk is.', learning_relevance: 'core' }, { id: 'kennisbeheer', label: 'Kennisbeheer', type: 'process', description: 'Het proces van het vastleggen en ontsluiten van organisatiekennis.', learning_relevance: 'standard' }, @@ -135,28 +135,66 @@ const SIMULATION_EXTRACTION_PAYLOAD = JSON.stringify({ { source: 'kennisbeheer', target: 'radicale-transparantie', type: 'depends_on' }, { source: 'wekelijkse-sessie', target: 'kennisbeheer', type: 'part_of' }, ], -}); +}; + +const SIMULATION_EXTRACTION_PAYLOAD = JSON.stringify(SIMULATION_EXTRACTION_GRAPH); const SIMULATION_CHAT_TEXT = 'Simulatiemodus staat aan — vraag een beheerder om Simulation Mode uit te zetten in Admin → Settings om met R42 te chatten.'; -async function simulatedResponse({ task }) { - await new Promise((r) => setTimeout(r, 400)); - if (isChatLikeTask(task)) { - return { - text: SIMULATION_CHAT_TEXT, - toolUses: [], - stopReason: 'end_turn', - usage: { input_tokens: 0, output_tokens: 0, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 }, - requestId: null, - model: 'simulation', - durationMs: 400, - }; - } +const SIMULATION_ARTICLE = { + title: 'Voorbeeld leermodule', + intro: 'Dit is een simulatie. Schakel Simulation Mode uit om echte content te genereren.', + sections: [ + { heading: 'Wat dit is', body: 'Dit is een placeholder-sectie die alleen verschijnt wanneer simulatiemodus aan staat. Hij illustreert de structuur van het artikel zonder een echte API-aanroep te doen. Dat is handig voor UI-werk.' }, + ], + keyTakeaways: ['Simulatiemodus levert geen echte inhoud.', 'Schakel uit voor productie.'], +}; + +const SIMULATION_SLIDE = { + title: 'Voorbeeldslide', + bullets: ['Eerste punt', 'Tweede punt'], + speakerNote: 'Spreker-notitie ter illustratie.', +}; + +const SIMULATION_INFOGRAPHIC = { + headline: 'Simulatie', + tagline: 'Vervang door echte content', + stats: [{ value: '100%', label: 'simulatie', icon: '📊' }], + steps: [{ number: 1, title: 'Schakel uit', description: 'Zet simulatiemodus uit in Admin → Settings.', icon: '🔧' }], + quote: 'Een simulatie vertelt niets nieuws.', + colorTheme: 'teal', +}; + +const SIMULATION_TOOL_STUBS = { + emit_knowledge_graph: SIMULATION_EXTRACTION_GRAPH, + emit_handbook_delta: SIMULATION_EXTRACTION_GRAPH, + emit_learning_article: { article: SIMULATION_ARTICLE }, + emit_learning_slides: { slides: [SIMULATION_SLIDE] }, + emit_learning_infographic: { infographic: SIMULATION_INFOGRAPHIC }, + emit_learning_all: { article: SIMULATION_ARTICLE, slides: [SIMULATION_SLIDE], infographic: SIMULATION_INFOGRAPHIC }, + emit_custom_topic: { label: 'Simulatie onderwerp', type: 'concept', description: 'Een placeholder-onderwerp gegenereerd in simulatiemodus.' }, + emit_quiz_questions: { + questions: [ + { + id: 'sim-q1', + question: 'Wat doet simulatiemodus?', + topicLabel: 'Simulatie', + options: ['Echte API-aanroepen', 'Stub-data tonen', 'Niets', 'Crasht de app'], + correctIndex: 1, + explanation: 'Simulatiemodus retourneert vaste stub-data zonder de API te raken.', + }, + ], + }, + emit_graph_actions: { merges: [], deletions: [], newRelations: [], relevanceUpdates: [] }, + set_intro: { intro: 'Bijgewerkte intro (simulatie).' }, +}; + +function stubResponse({ stopReason = 'end_turn', text = '', toolUses = [] }) { return { - text: SIMULATION_EXTRACTION_PAYLOAD, - toolUses: [], - stopReason: 'end_turn', + text, + toolUses, + stopReason, usage: { input_tokens: 0, output_tokens: 0, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 }, requestId: null, model: 'simulation', @@ -164,6 +202,22 @@ async function simulatedResponse({ task }) { }; } +async function simulatedResponse({ task, toolChoice }) { + await new Promise((r) => setTimeout(r, 400)); + + if (toolChoice?.type === 'tool' && SIMULATION_TOOL_STUBS[toolChoice.name]) { + return stubResponse({ + stopReason: 'tool_use', + toolUses: [{ name: toolChoice.name, input: SIMULATION_TOOL_STUBS[toolChoice.name] }], + }); + } + + if (isChatLikeTask(task)) { + return stubResponse({ text: SIMULATION_CHAT_TEXT }); + } + return stubResponse({ text: SIMULATION_EXTRACTION_PAYLOAD }); +} + function linkSignals(userSignal, timeoutSignal) { const controller = new AbortController(); const abort = (reason) => controller.abort(reason); @@ -241,7 +295,7 @@ export async function callLLM(options) { if (!task) throw new Error('callLLM requires a `task` label.'); const useSimulation = storage.get('admin:use_simulation') === true; - if (useSimulation) return simulatedResponse({ task }); + if (useSimulation) return simulatedResponse({ task, toolChoice }); const model = resolveModel(tier); const messagesPayload = buildMessages({ messages, user }); diff --git a/src/lib/llmSchemas.js b/src/lib/llmSchemas.js index 17fa773..052fd85 100644 --- a/src/lib/llmSchemas.js +++ b/src/lib/llmSchemas.js @@ -183,6 +183,31 @@ export const proposeGraphDeltaSchema = z.object({ relations: z.array(deltaRelationSchema).max(5).optional(), }); +// ── Article patch operation schemas (Phase 2.4) ────────────────────────────── + +export const setIntroPatchSchema = z.object({ + intro: z.string().min(1), +}); + +export const setSectionPatchSchema = z.object({ + heading: z.string().min(1), + body: z.string().min(1), +}); + +export const addSectionPatchSchema = z.object({ + heading: z.string().min(1), + body: z.string().min(1), + position: z.enum(['start', 'end']), +}); + +export const removeSectionPatchSchema = z.object({ + heading: z.string().min(1), +}); + +export const replaceTakeawaysPatchSchema = z.object({ + items: z.array(z.string().min(1)).min(1), +}); + /** * Registry mapping known tool names to their input schemas. `callLLM` * consults this when the caller does not pass an explicit `toolSchemas` @@ -199,4 +224,9 @@ export const toolSchemaRegistry = { emit_custom_topic: customTopicSchema, emit_graph_actions: graphActionsSchema, propose_graph_delta: proposeGraphDeltaSchema, + set_intro: setIntroPatchSchema, + set_section: setSectionPatchSchema, + add_section: addSectionPatchSchema, + remove_section: removeSectionPatchSchema, + replace_takeaways: replaceTakeawaysPatchSchema, }; diff --git a/src/lib/llmTools.js b/src/lib/llmTools.js new file mode 100644 index 0000000..630f38d --- /dev/null +++ b/src/lib/llmTools.js @@ -0,0 +1,324 @@ +/** + * Anthropic tool definitions used by every structured-output flow. + * + * Each `tool_use` reply the model emits is validated against the matching + * Zod schema in `llmSchemas.js` (see `toolSchemaRegistry`). The two stay + * in lock-step on purpose — JSON Schema here drives the model, Zod there + * defends the application. + */ + +const TOPIC_TYPES = ['concept', 'role', 'process']; +const LEARNING_RELEVANCE = ['core', 'standard', 'peripheral', 'exclude']; +const RELATION_TYPES_STRICT = ['related_to', 'depends_on', 'part_of', 'executed_by']; +const RELATION_TYPES_LOOSE = ['related_to', 'depends_on', 'part_of', 'executed_by', 'executes']; + +const extractionTopicSchema = { + type: 'object', + properties: { + id: { type: 'string', description: 'kebab-case slug specific to the topic. Reuse existing IDs when the same concept recurs.' }, + label: { type: 'string' }, + type: { type: 'string', enum: TOPIC_TYPES }, + description: { type: 'string', description: 'Max 3 sentences.' }, + learning_relevance: { type: 'string', enum: LEARNING_RELEVANCE }, + }, + required: ['id', 'label', 'type', 'description', 'learning_relevance'], +}; + +const extractionRelationSchema = { + type: 'object', + properties: { + source: { type: 'string', description: 'Topic id.' }, + target: { type: 'string', description: 'Topic id.' }, + type: { type: 'string', enum: RELATION_TYPES_STRICT }, + }, + required: ['source', 'target', 'type'], +}; + +export const EMIT_KNOWLEDGE_GRAPH_TOOL = { + name: 'emit_knowledge_graph', + description: 'Return the complete knowledge graph extracted from the supplied source text — every distinct role, process and concept as a topic, plus the relations between them.', + input_schema: { + type: 'object', + properties: { + topics: { type: 'array', items: extractionTopicSchema }, + relations: { type: 'array', items: extractionRelationSchema }, + }, + required: ['topics', 'relations'], + }, +}; + +const handbookTopicSchema = { + type: 'object', + properties: { + ...extractionTopicSchema.properties, + metadata: { + type: 'object', + properties: { source: { type: 'string' } }, + }, + }, + required: extractionTopicSchema.required, +}; + +const handbookRelationSchema = { + type: 'object', + properties: { + source: { type: 'string' }, + target: { type: 'string' }, + type: { type: 'string', enum: RELATION_TYPES_LOOSE }, + description: { type: 'string' }, + }, + required: ['source', 'target', 'type'], +}; + +export const EMIT_HANDBOOK_DELTA_TOOL = { + name: 'emit_handbook_delta', + description: 'Return the topics and relations extracted from a handbook file update. Every process must have a role attached; every concept must connect to a process or role.', + input_schema: { + type: 'object', + properties: { + topics: { type: 'array', items: handbookTopicSchema }, + relations: { type: 'array', items: handbookRelationSchema }, + }, + required: ['topics', 'relations'], + }, +}; + +const articleSectionSchema = { + type: 'object', + properties: { + heading: { type: 'string' }, + body: { type: 'string', description: 'At least three sentences.' }, + }, + required: ['heading', 'body'], +}; + +const articleBodySchema = { + type: 'object', + properties: { + title: { type: 'string' }, + intro: { type: 'string', description: 'One or two sentences.' }, + sections: { type: 'array', items: articleSectionSchema, minItems: 1 }, + keyTakeaways: { type: 'array', items: { type: 'string' }, minItems: 1 }, + }, + required: ['title', 'intro', 'sections', 'keyTakeaways'], +}; + +const slideSchema = { + type: 'object', + properties: { + title: { type: 'string' }, + bullets: { type: 'array', items: { type: 'string' }, minItems: 1 }, + speakerNote: { type: 'string' }, + }, + required: ['title', 'bullets', 'speakerNote'], +}; + +const infographicStatSchema = { + type: 'object', + properties: { + value: { type: 'string' }, + label: { type: 'string' }, + icon: { type: 'string' }, + }, + required: ['value', 'label', 'icon'], +}; + +const infographicStepSchema = { + type: 'object', + properties: { + number: { type: 'integer', minimum: 1 }, + title: { type: 'string' }, + description: { type: 'string' }, + icon: { type: 'string' }, + }, + required: ['number', 'title', 'description', 'icon'], +}; + +const infographicBodySchema = { + type: 'object', + properties: { + headline: { type: 'string', description: 'Punchy, max 8 words.' }, + tagline: { type: 'string', description: 'Max 15 words.' }, + stats: { type: 'array', items: infographicStatSchema, minItems: 1 }, + steps: { type: 'array', items: infographicStepSchema, minItems: 1 }, + quote: { type: 'string' }, + colorTheme: { type: 'string', description: 'Tailwind colour token (e.g. "teal").' }, + }, + required: ['headline', 'tagline', 'stats', 'steps', 'quote', 'colorTheme'], +}; + +export const EMIT_LEARNING_ARTICLE_TOOL = { + name: 'emit_learning_article', + description: 'Return the article body for a learning module. At least three sections.', + input_schema: { + type: 'object', + properties: { article: articleBodySchema }, + required: ['article'], + }, +}; + +export const EMIT_LEARNING_SLIDES_TOOL = { + name: 'emit_learning_slides', + description: 'Return the slide deck for a learning module. At least four slides.', + input_schema: { + type: 'object', + properties: { slides: { type: 'array', items: slideSchema, minItems: 1 } }, + required: ['slides'], + }, +}; + +export const EMIT_LEARNING_INFOGRAPHIC_TOOL = { + name: 'emit_learning_infographic', + description: 'Return the infographic for a learning module. At least three stats and three steps.', + input_schema: { + type: 'object', + properties: { infographic: infographicBodySchema }, + required: ['infographic'], + }, +}; + +export const EMIT_LEARNING_ALL_TOOL = { + name: 'emit_learning_all', + description: 'Return article, slides and infographic for a learning module in one call.', + input_schema: { + type: 'object', + properties: { + article: articleBodySchema, + slides: { type: 'array', items: slideSchema, minItems: 1 }, + infographic: infographicBodySchema, + }, + required: ['article', 'slides', 'infographic'], + }, +}; + +export const EMIT_CUSTOM_TOPIC_TOOL = { + name: 'emit_custom_topic', + description: 'Return a polished label, type and short description for a user-requested topic.', + input_schema: { + type: 'object', + properties: { + label: { type: 'string' }, + type: { type: 'string', enum: TOPIC_TYPES }, + description: { type: 'string', description: 'Two or three sentences.' }, + }, + required: ['label', 'type', 'description'], + }, +}; + +const quizQuestionSchema = { + type: 'object', + properties: { + id: { type: 'string' }, + question: { type: 'string' }, + topicLabel: { type: 'string' }, + options: { type: 'array', items: { type: 'string' }, minItems: 4, maxItems: 4 }, + correctIndex: { type: 'integer', minimum: 0, maximum: 3 }, + explanation: { type: 'string', description: 'Why the correct answer is correct (1–2 sentences).' }, + }, + required: ['id', 'question', 'topicLabel', 'options', 'correctIndex', 'explanation'], +}; + +export const EMIT_QUIZ_QUESTIONS_TOOL = { + name: 'emit_quiz_questions', + description: 'Return a batch of multiple-choice questions for a topic. Exactly four options each; correctIndex is 0-based.', + input_schema: { + type: 'object', + properties: { questions: { type: 'array', items: quizQuestionSchema, minItems: 1 } }, + required: ['questions'], + }, +}; + +export const EMIT_GRAPH_ACTIONS_TOOL = { + name: 'emit_graph_actions', + description: 'Return the actions to take on the knowledge graph: merges, deletions, new relations and relevance updates. Do not return the entire graph.', + input_schema: { + type: 'object', + properties: { + merges: { + type: 'array', + items: { + type: 'object', + properties: { keepId: { type: 'string' }, deleteId: { type: 'string' } }, + required: ['keepId', 'deleteId'], + }, + }, + deletions: { type: 'array', items: { type: 'string' } }, + newRelations: { type: 'array', items: extractionRelationSchema }, + relevanceUpdates: { + type: 'array', + items: { + type: 'object', + properties: { id: { type: 'string' }, learning_relevance: { type: 'string', enum: LEARNING_RELEVANCE } }, + required: ['id', 'learning_relevance'], + }, + }, + }, + }, +}; + +// ── Patch tools for refineLearningContent (Phase 2.4) ───────────────────────── + +export const SET_INTRO_TOOL = { + name: 'set_intro', + description: 'Replace the article intro with a new one or two sentences.', + input_schema: { + type: 'object', + properties: { intro: { type: 'string', description: 'New intro text.' } }, + required: ['intro'], + }, +}; + +export const SET_SECTION_TOOL = { + name: 'set_section', + description: 'Replace the body of an existing section, matched by its heading (case-insensitive). Use add_section if no section with that heading exists.', + input_schema: { + type: 'object', + properties: { + heading: { type: 'string', description: 'Heading of the section to replace.' }, + body: { type: 'string', description: 'New body for that section, at least three sentences.' }, + }, + required: ['heading', 'body'], + }, +}; + +export const ADD_SECTION_TOOL = { + name: 'add_section', + description: 'Insert a new section into the article at the start or end.', + input_schema: { + type: 'object', + properties: { + heading: { type: 'string' }, + body: { type: 'string', description: 'At least three sentences.' }, + position: { type: 'string', enum: ['start', 'end'] }, + }, + required: ['heading', 'body', 'position'], + }, +}; + +export const REMOVE_SECTION_TOOL = { + name: 'remove_section', + description: 'Delete a section from the article, matched by its heading (case-insensitive).', + input_schema: { + type: 'object', + properties: { heading: { type: 'string' } }, + required: ['heading'], + }, +}; + +export const REPLACE_TAKEAWAYS_TOOL = { + name: 'replace_takeaways', + description: 'Replace the key takeaways list with a new one.', + input_schema: { + type: 'object', + properties: { items: { type: 'array', items: { type: 'string' }, minItems: 1 } }, + required: ['items'], + }, +}; + +export const ARTICLE_PATCH_TOOLS = [ + SET_INTRO_TOOL, + SET_SECTION_TOOL, + ADD_SECTION_TOOL, + REMOVE_SECTION_TOOL, + REPLACE_TAKEAWAYS_TOOL, +]; diff --git a/src/lib/testService.js b/src/lib/testService.js index b581a27..b6daa36 100644 --- a/src/lib/testService.js +++ b/src/lib/testService.js @@ -1,11 +1,15 @@ -import { anthropicApi } from './api'; import * as db from './db'; +import { callLLM } from './llm'; +import { EMIT_QUIZ_QUESTIONS_TOOL } from './llmTools'; import { getCurriculumTopic, getQuarterForWeek } from './curriculumService'; const QUIZ_SYSTEM = `You are a quiz generator for Respellion, an internal IT company learning platform. You generate multiple-choice questions to test employee knowledge on specific topics. Always write in clear, professional English. -ALWAYS return valid JSON only — no markdown code blocks, no extra text.`; + +Emit questions through the emit_quiz_questions tool. Each question has exactly four options; correctIndex is 0-based; mix difficulty roughly 4 easy / 4 medium / 2 hard.`; + +const cachedSystem = (text) => [{ type: 'text', text, cache_control: { type: 'ephemeral' } }]; async function selectTestTopics(userId, weekNumber) { const allTopics = await db.getTopics(); @@ -66,45 +70,31 @@ export async function getCachedQuiz(userId, weekNumber) { export async function forceGenerateTopicQuestions(topic, count = 10) { let bank = await db.getQuizBank(topic.id); - const prompt = `Generate exactly ${count} multiple-choice quiz questions based on this knowledge topic: + const prompt = `Generate exactly ${count} multiple-choice quiz questions for this knowledge topic and emit them via the emit_quiz_questions tool: Topic: ${topic.label} Type: ${topic.type} Description: ${topic.description} -Return ONLY a JSON object with this structure: -{ - "questions": [ - { - "id": "unique-id-string", - "question": "The question text", - "topicLabel": "${topic.label}", - "options": ["A) First option", "B) Second option", "C) Third option", "D) Fourth option"], - "correctIndex": 0, - "explanation": "A clear 1-2 sentence explanation of why the correct answer is correct." - } - ] -} +Options must be prefixed "A) ", "B) ", "C) ", "D) ". Make questions specific and practical, not trivial.`; -Rules: -- Each question must have exactly 4 options. -- correctIndex is 0-based (0=A, 1=B, 2=C, 3=D). -- Mix difficulty: 4 easy, 4 medium, 2 hard. -- Make questions specific and practical, not trivial.`; + const result = await callLLM({ + task: 'quiz.generate', + tier: 'standard', + system: cachedSystem(QUIZ_SYSTEM), + user: prompt, + tools: [EMIT_QUIZ_QUESTIONS_TOOL], + toolChoice: { type: 'tool', name: EMIT_QUIZ_QUESTIONS_TOOL.name }, + maxTokens: 4096, + }); - const responseText = await anthropicApi.generateContent(QUIZ_SYSTEM, prompt); - let newQuestions; - try { - const jsonMatch = responseText.match(/\{[\s\S]*\}/); - const parsed = JSON.parse(jsonMatch ? jsonMatch[0] : responseText); - newQuestions = parsed.questions || []; - newQuestions.forEach(q => { - q.id = `${topic.id}-${Math.random().toString(36).substr(2, 9)}`; - }); - } catch (e) { - console.error('Failed to generate questions for topic', topic.label, e); - throw new Error(`Could not generate questions for ${topic.label}`, { cause: e }); - } + const emitted = result.toolUses[0]?.input; + if (!emitted) throw new Error(`Could not generate questions for ${topic.label}`); + + const newQuestions = (emitted.questions || []).map(q => ({ + ...q, + id: `${topic.id}-${Math.random().toString(36).slice(2, 11)}`, + })); bank = [...bank, ...newQuestions]; await db.setQuizBank(topic.id, bank); -- 2.49.1