feat: phase 1 of AI pipeline hardening — single LLM client + tier-aware models
Implements phase 1 of AI_PIPELINE_HARDENING_PLAN.md. Every Anthropic call now goes through one module that owns retry, timeout, abort, structured- output parsing, schema validation, and best-effort call telemetry. * src/lib/llm.js — single callLLM entry point. Resolves model per tier (fast / standard / reasoning) with admin:model legacy fallback for the standard tier; 60s default timeout via AbortController; balanced-brace JSON extraction; LLMHttpError, LLMTruncatedError, LLMOutputError, and LLMValidationError surface clearly distinct failure modes. * src/lib/llmRetry.js — exponential backoff with full jitter, retries only on transient HTTP statuses, honours Retry-After up to 60s, never retries on AbortError. * src/lib/llmSchemas.js — Zod schemas for every structured task plus normalizeHandbookResult (collapses legacy "executes" relations into the canonical "executed_by" vocabulary). * src/lib/api.js — thin shim over callLLM so existing callers (extraction pipeline, learning, quiz, R42, knowledge graph) keep working unchanged. * src/lib/__tests__/ — 32 Vitest cases covering parse paths, error surfaces, simulation mode, model resolution, and schema validation. * src/pages/Admin/index.jsx — three model inputs (fast / standard / reasoning) replacing the single legacy field; legacy value falls back for the standard tier so existing overrides survive. Adds Zod and Vitest, plus an "npm run test" script. Also cleans up the pre-existing repo-wide ESLint failures so phase 1's "npm run lint passes" acceptance criterion can be checked: drops unused React imports across the JSX tree (React 19 JSX runtime auto-imports), attaches cause to rethrown errors in the service modules, ignores pb_migrations in the ESLint config (PocketBase JSVM globals), and removes one dead handleCreateCustom function in Leren.jsx. A real behaviour bug surfaced in Testen.jsx — the quiz timer captured a stale finishQuiz via setInterval closure; now updated via finishQuizRef so the timer always invokes the latest callback. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
RaymondVerhoef
232
src/lib/__tests__/llm.test.js
Normal file
232
src/lib/__tests__/llm.test.js
Normal file
@@ -0,0 +1,232 @@
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest';
|
||||
import { z } from 'zod';
|
||||
|
||||
vi.mock('../storage', () => ({
|
||||
storage: {
|
||||
_data: new Map(),
|
||||
get(key, fallback = null) {
|
||||
return this._data.has(key) ? this._data.get(key) : fallback;
|
||||
},
|
||||
set(key, value) { this._data.set(key, value); },
|
||||
remove(key) { this._data.delete(key); },
|
||||
getKeysByPrefix() { return []; },
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock('../pb', () => ({
|
||||
pb: { collection: () => ({ create: () => ({ catch: () => {} }) }) },
|
||||
}));
|
||||
|
||||
import {
|
||||
callLLM,
|
||||
LLMHttpError,
|
||||
LLMOutputError,
|
||||
LLMTruncatedError,
|
||||
LLMValidationError,
|
||||
parseStructuredText,
|
||||
resolveModel,
|
||||
} from '../llm';
|
||||
import { storage } from '../storage';
|
||||
|
||||
const originalFetch = globalThis.fetch;
|
||||
|
||||
afterEach(() => {
|
||||
globalThis.fetch = originalFetch;
|
||||
storage._data.clear();
|
||||
});
|
||||
|
||||
describe('parseStructuredText', () => {
|
||||
it('extracts an object from raw JSON', () => {
|
||||
expect(parseStructuredText('{"a":1}')).toEqual({ a: 1 });
|
||||
});
|
||||
|
||||
it('extracts an object from a json-fenced block', () => {
|
||||
const fenced = '```json\n{"hello":"world"}\n```';
|
||||
expect(parseStructuredText(fenced)).toEqual({ hello: 'world' });
|
||||
});
|
||||
|
||||
it('extracts an object surrounded by prose', () => {
|
||||
const messy = 'Sure! Here you go:\n{"id":"x","label":"X"}\nLet me know if you want changes.';
|
||||
expect(parseStructuredText(messy)).toEqual({ id: 'x', label: 'X' });
|
||||
});
|
||||
|
||||
it('extracts an array when it is the top-level value', () => {
|
||||
expect(parseStructuredText('[1,2,3]')).toEqual([1, 2, 3]);
|
||||
});
|
||||
|
||||
it('ignores braces inside string literals', () => {
|
||||
const tricky = '{"text":"this { is not } a brace"}';
|
||||
expect(parseStructuredText(tricky)).toEqual({ text: 'this { is not } a brace' });
|
||||
});
|
||||
|
||||
it('throws LLMOutputError when no balanced JSON is present', () => {
|
||||
expect(() => parseStructuredText('no json here, just words')).toThrow(LLMOutputError);
|
||||
});
|
||||
});
|
||||
|
||||
describe('resolveModel', () => {
|
||||
it('falls back to tier defaults when no override is set', () => {
|
||||
expect(resolveModel('fast')).toBe('claude-haiku-4-5-20251001');
|
||||
expect(resolveModel('standard')).toBe('claude-sonnet-4-6');
|
||||
expect(resolveModel('reasoning')).toBe('claude-opus-4-7');
|
||||
});
|
||||
|
||||
it('honours an explicit tier override', () => {
|
||||
storage.set('admin:model:reasoning', 'claude-opus-9-future');
|
||||
expect(resolveModel('reasoning')).toBe('claude-opus-9-future');
|
||||
});
|
||||
|
||||
it('uses the legacy admin:model setting as a standard-tier fallback', () => {
|
||||
storage.set('admin:model', 'claude-some-legacy-id');
|
||||
expect(resolveModel('standard')).toBe('claude-some-legacy-id');
|
||||
});
|
||||
|
||||
it('prefers the tier-specific override over the legacy fallback', () => {
|
||||
storage.set('admin:model', 'claude-legacy');
|
||||
storage.set('admin:model:standard', 'claude-new');
|
||||
expect(resolveModel('standard')).toBe('claude-new');
|
||||
});
|
||||
});
|
||||
|
||||
function mockJsonResponse(body, { status = 200, headers = {} } = {}) {
|
||||
const h = new Headers({ 'content-type': 'application/json', ...headers });
|
||||
return new Response(JSON.stringify(body), { status, headers: h });
|
||||
}
|
||||
|
||||
describe('callLLM happy path', () => {
|
||||
it('returns parsed tool input when toolChoice forces a tool', async () => {
|
||||
globalThis.fetch = vi.fn(async () =>
|
||||
mockJsonResponse({
|
||||
id: 'msg_1',
|
||||
model: 'claude-sonnet-4-6',
|
||||
stop_reason: 'tool_use',
|
||||
usage: { input_tokens: 10, output_tokens: 20 },
|
||||
content: [
|
||||
{
|
||||
type: 'tool_use',
|
||||
name: 'emit_custom_topic',
|
||||
input: { label: 'Pair Programming', type: 'process', description: 'Two engineers, one keyboard.' },
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
const result = await callLLM({
|
||||
task: 'learning.custom_topic',
|
||||
tier: 'standard',
|
||||
user: 'Pair programming',
|
||||
tools: [{ name: 'emit_custom_topic', description: 'x', input_schema: { type: 'object' } }],
|
||||
toolChoice: { type: 'tool', name: 'emit_custom_topic' },
|
||||
});
|
||||
|
||||
expect(result.toolUses).toHaveLength(1);
|
||||
expect(result.toolUses[0].input.label).toBe('Pair Programming');
|
||||
});
|
||||
|
||||
it('parses and validates plain text against a Zod schema', async () => {
|
||||
globalThis.fetch = vi.fn(async () =>
|
||||
mockJsonResponse({
|
||||
id: 'msg_2',
|
||||
model: 'claude-sonnet-4-6',
|
||||
stop_reason: 'end_turn',
|
||||
usage: { input_tokens: 5, output_tokens: 7 },
|
||||
content: [{ type: 'text', text: '```json\n{"value":42}\n```' }],
|
||||
}),
|
||||
);
|
||||
|
||||
const schema = z.object({ value: z.number() });
|
||||
const result = await callLLM({
|
||||
task: 'demo.json',
|
||||
user: 'give me a number',
|
||||
schema,
|
||||
});
|
||||
expect(result.parsed).toEqual({ value: 42 });
|
||||
});
|
||||
});
|
||||
|
||||
describe('callLLM error paths', () => {
|
||||
it('throws LLMTruncatedError when stop_reason is max_tokens and a tool was requested', async () => {
|
||||
globalThis.fetch = vi.fn(async () =>
|
||||
mockJsonResponse({
|
||||
stop_reason: 'max_tokens',
|
||||
usage: { input_tokens: 1, output_tokens: 1 },
|
||||
content: [],
|
||||
}),
|
||||
);
|
||||
await expect(
|
||||
callLLM({
|
||||
task: 'extract.source',
|
||||
user: 'x',
|
||||
tools: [{ name: 'emit_knowledge_graph', description: 'x', input_schema: { type: 'object' } }],
|
||||
toolChoice: { type: 'tool', name: 'emit_knowledge_graph' },
|
||||
}),
|
||||
).rejects.toBeInstanceOf(LLMTruncatedError);
|
||||
});
|
||||
|
||||
it('throws LLMTruncatedError when stop_reason is max_tokens and a schema was requested', async () => {
|
||||
globalThis.fetch = vi.fn(async () =>
|
||||
mockJsonResponse({
|
||||
stop_reason: 'max_tokens',
|
||||
usage: { input_tokens: 1, output_tokens: 1 },
|
||||
content: [{ type: 'text', text: 'partial...' }],
|
||||
}),
|
||||
);
|
||||
const schema = z.object({ value: z.number() });
|
||||
await expect(
|
||||
callLLM({ task: 'demo.json', user: 'x', schema }),
|
||||
).rejects.toBeInstanceOf(LLMTruncatedError);
|
||||
});
|
||||
|
||||
it('throws LLMValidationError when tool input fails schema validation', async () => {
|
||||
globalThis.fetch = vi.fn(async () =>
|
||||
mockJsonResponse({
|
||||
stop_reason: 'tool_use',
|
||||
usage: {},
|
||||
content: [
|
||||
{ type: 'tool_use', name: 'emit_custom_topic', input: { label: 'X', type: 'concept' } },
|
||||
],
|
||||
}),
|
||||
);
|
||||
await expect(
|
||||
callLLM({
|
||||
task: 'learning.custom_topic',
|
||||
user: 'x',
|
||||
tools: [{ name: 'emit_custom_topic', description: 'x', input_schema: { type: 'object' } }],
|
||||
toolChoice: { type: 'tool', name: 'emit_custom_topic' },
|
||||
}),
|
||||
).rejects.toBeInstanceOf(LLMValidationError);
|
||||
});
|
||||
|
||||
it('surfaces a non-retryable HTTP error as LLMHttpError', async () => {
|
||||
globalThis.fetch = vi.fn(async () =>
|
||||
new Response(JSON.stringify({ error: 'bad request' }), {
|
||||
status: 400,
|
||||
headers: { 'content-type': 'application/json' },
|
||||
}),
|
||||
);
|
||||
await expect(callLLM({ task: 'demo', user: 'x' })).rejects.toBeInstanceOf(LLMHttpError);
|
||||
});
|
||||
|
||||
it('detects an auth portal HTML response and raises a clear message', async () => {
|
||||
globalThis.fetch = vi.fn(async () =>
|
||||
new Response('<html>login</html>', { status: 200, headers: { 'content-type': 'text/html' } }),
|
||||
);
|
||||
await expect(callLLM({ task: 'demo', user: 'x' })).rejects.toThrow(/session has expired/i);
|
||||
});
|
||||
});
|
||||
|
||||
describe('callLLM simulation mode', () => {
|
||||
it('returns the chat stub when admin:use_simulation is true and task is chat-like', async () => {
|
||||
storage.set('admin:use_simulation', true);
|
||||
const result = await callLLM({ task: 'chat.r42', user: 'hello' });
|
||||
expect(result.stopReason).toBe('end_turn');
|
||||
expect(result.text).toMatch(/Simulatiemodus/);
|
||||
});
|
||||
|
||||
it('returns the extraction stub for other tasks in simulation mode', async () => {
|
||||
storage.set('admin:use_simulation', true);
|
||||
const result = await callLLM({ task: 'extract.source', user: 'doc' });
|
||||
expect(() => JSON.parse(result.text)).not.toThrow();
|
||||
expect(JSON.parse(result.text)).toHaveProperty('topics');
|
||||
});
|
||||
});
|
||||
197
src/lib/__tests__/llmSchemas.test.js
Normal file
197
src/lib/__tests__/llmSchemas.test.js
Normal file
@@ -0,0 +1,197 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
extractionResultSchema,
|
||||
handbookResultSchema,
|
||||
normalizeHandbookResult,
|
||||
learningArticleSchema,
|
||||
learningSlidesSchema,
|
||||
learningInfographicSchema,
|
||||
learningAllSchema,
|
||||
quizQuestionsSchema,
|
||||
customTopicSchema,
|
||||
graphActionsSchema,
|
||||
proposeGraphDeltaSchema,
|
||||
} from '../llmSchemas';
|
||||
|
||||
const sampleTopic = {
|
||||
id: 'software-engineer',
|
||||
label: 'Software Engineer',
|
||||
type: 'role',
|
||||
description: 'Builds and maintains the platform.',
|
||||
learning_relevance: 'core',
|
||||
};
|
||||
|
||||
const sampleRelation = {
|
||||
source: 'software-engineer',
|
||||
target: 'onboarding',
|
||||
type: 'part_of',
|
||||
};
|
||||
|
||||
const sampleArticle = {
|
||||
title: 'Onboarding 101',
|
||||
intro: 'A short intro.',
|
||||
sections: [{ heading: 'Day one', body: 'Welcome to the team.' }],
|
||||
keyTakeaways: ['Show up', 'Ask questions'],
|
||||
};
|
||||
|
||||
const sampleSlide = {
|
||||
title: 'Welcome',
|
||||
bullets: ['Meet your buddy', 'Read the handbook'],
|
||||
speakerNote: 'Greet new joiners warmly.',
|
||||
};
|
||||
|
||||
const sampleInfographic = {
|
||||
headline: 'Onboarding flow',
|
||||
tagline: 'From hire to productive in 30 days',
|
||||
stats: [{ value: '30', label: 'days', icon: '📅' }],
|
||||
steps: [{ number: 1, title: 'Sign in', description: 'Use the welcome email.', icon: '🔑' }],
|
||||
quote: 'A great start beats a great recovery.',
|
||||
colorTheme: 'teal',
|
||||
};
|
||||
|
||||
describe('extractionResultSchema', () => {
|
||||
it('accepts a minimal extraction result', () => {
|
||||
const parsed = extractionResultSchema.parse({
|
||||
topics: [sampleTopic],
|
||||
relations: [sampleRelation],
|
||||
});
|
||||
expect(parsed.topics).toHaveLength(1);
|
||||
expect(parsed.relations[0].type).toBe('part_of');
|
||||
});
|
||||
});
|
||||
|
||||
describe('handbookResultSchema', () => {
|
||||
it('accepts the loose vocabulary including executes', () => {
|
||||
const parsed = handbookResultSchema.parse({
|
||||
topics: [{ ...sampleTopic, metadata: { source: 'github_handbook' } }],
|
||||
relations: [{ source: 'software-engineer', target: 'code-review', type: 'executes' }],
|
||||
});
|
||||
expect(parsed.relations[0].type).toBe('executes');
|
||||
});
|
||||
|
||||
it('normalises executes into executed_by with swapped source/target', () => {
|
||||
const parsed = handbookResultSchema.parse({
|
||||
topics: [sampleTopic],
|
||||
relations: [{ source: 'software-engineer', target: 'code-review', type: 'executes' }],
|
||||
});
|
||||
const normalised = normalizeHandbookResult(parsed);
|
||||
expect(normalised.relations[0]).toMatchObject({
|
||||
source: 'code-review',
|
||||
target: 'software-engineer',
|
||||
type: 'executed_by',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('learning schemas', () => {
|
||||
it('accepts an article payload', () => {
|
||||
expect(() => learningArticleSchema.parse({ article: sampleArticle })).not.toThrow();
|
||||
});
|
||||
|
||||
it('accepts a slides payload', () => {
|
||||
expect(() => learningSlidesSchema.parse({ slides: [sampleSlide] })).not.toThrow();
|
||||
});
|
||||
|
||||
it('accepts an infographic payload', () => {
|
||||
expect(() => learningInfographicSchema.parse({ infographic: sampleInfographic })).not.toThrow();
|
||||
});
|
||||
|
||||
it('accepts a combined "all" payload', () => {
|
||||
expect(() =>
|
||||
learningAllSchema.parse({
|
||||
article: sampleArticle,
|
||||
slides: [sampleSlide],
|
||||
infographic: sampleInfographic,
|
||||
}),
|
||||
).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('quizQuestionsSchema', () => {
|
||||
it('accepts a quiz with four options and a valid correctIndex', () => {
|
||||
const parsed = quizQuestionsSchema.parse({
|
||||
questions: [
|
||||
{
|
||||
id: 'q-1',
|
||||
question: 'What is the buddy system?',
|
||||
topicLabel: 'Onboarding',
|
||||
options: ['A', 'B', 'C', 'D'],
|
||||
correctIndex: 2,
|
||||
explanation: 'C describes the buddy system best.',
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(parsed.questions[0].options).toHaveLength(4);
|
||||
});
|
||||
|
||||
it('rejects three options or an out-of-range correctIndex', () => {
|
||||
expect(() =>
|
||||
quizQuestionsSchema.parse({
|
||||
questions: [
|
||||
{
|
||||
id: 'q',
|
||||
question: 'q',
|
||||
topicLabel: 't',
|
||||
options: ['A', 'B', 'C'],
|
||||
correctIndex: 0,
|
||||
explanation: 'e',
|
||||
},
|
||||
],
|
||||
}),
|
||||
).toThrow();
|
||||
expect(() =>
|
||||
quizQuestionsSchema.parse({
|
||||
questions: [
|
||||
{
|
||||
id: 'q',
|
||||
question: 'q',
|
||||
topicLabel: 't',
|
||||
options: ['A', 'B', 'C', 'D'],
|
||||
correctIndex: 4,
|
||||
explanation: 'e',
|
||||
},
|
||||
],
|
||||
}),
|
||||
).toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('customTopicSchema', () => {
|
||||
it('accepts a polished custom topic', () => {
|
||||
expect(() =>
|
||||
customTopicSchema.parse({
|
||||
label: 'Pair Programming',
|
||||
type: 'process',
|
||||
description: 'Two engineers, one keyboard.',
|
||||
}),
|
||||
).not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('graphActionsSchema', () => {
|
||||
it('fills missing arrays with empty defaults', () => {
|
||||
const parsed = graphActionsSchema.parse({});
|
||||
expect(parsed.merges).toEqual([]);
|
||||
expect(parsed.deletions).toEqual([]);
|
||||
expect(parsed.newRelations).toEqual([]);
|
||||
expect(parsed.relevanceUpdates).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe('proposeGraphDeltaSchema', () => {
|
||||
it('accepts a reason-only delta', () => {
|
||||
expect(() => proposeGraphDeltaSchema.parse({ reason: 'Nothing to add.' })).not.toThrow();
|
||||
});
|
||||
|
||||
it('caps topics at three and relations at five', () => {
|
||||
const bigTopics = Array.from({ length: 4 }, (_, i) => ({
|
||||
id: `t-${i}`,
|
||||
label: `Topic ${i}`,
|
||||
type: 'concept',
|
||||
description: 'desc',
|
||||
}));
|
||||
expect(() =>
|
||||
proposeGraphDeltaSchema.parse({ reason: 'too many', topics: bigTopics }),
|
||||
).toThrow();
|
||||
});
|
||||
});
|
||||
160
src/lib/api.js
160
src/lib/api.js
@@ -1,135 +1,39 @@
|
||||
import { storage } from './storage';
|
||||
|
||||
/**
|
||||
* Anthropic API Service
|
||||
* Handles communication with the /v1/messages endpoint via Nginx proxy.
|
||||
* Back-compatibility shim for the legacy `anthropicApi` interface.
|
||||
*
|
||||
* All real work lives in `./llm.js`. Existing callers (extractionPipeline,
|
||||
* learningService, testService, KnowledgeGraph, useChat) keep working
|
||||
* unchanged; new code should import `callLLM` from `./llm.js` directly.
|
||||
*/
|
||||
|
||||
const DEFAULT_MODEL = 'claude-sonnet-4-20250514';
|
||||
import { callLLM } from './llm';
|
||||
|
||||
export const anthropicApi = {
|
||||
async generateContent(systemPrompt, userMessage, maxRetries = 1) {
|
||||
// Check if simulation mode is on
|
||||
const useSimulation = storage.get('admin:use_simulation') === true;
|
||||
if (useSimulation) {
|
||||
console.log('[API] Simulation mode active. Mock data will be returned.');
|
||||
return await simulateResponse();
|
||||
}
|
||||
async generateContent(systemPrompt, userMessage /*, maxRetries */) {
|
||||
const { text } = await callLLM({
|
||||
task: 'legacy.generateContent',
|
||||
tier: 'standard',
|
||||
system: systemPrompt,
|
||||
user: userMessage,
|
||||
maxTokens: 8192,
|
||||
temperature: 0,
|
||||
});
|
||||
return text;
|
||||
},
|
||||
|
||||
// The API key is now securely injected by the Caddy reverse proxy via environment variables.
|
||||
|
||||
// Model is configurable from Admin > Settings, defaults to the original spec model
|
||||
const model = storage.get('admin:model') || DEFAULT_MODEL;
|
||||
console.log(`[API] Calling with model: ${model}`);
|
||||
|
||||
let retries = 0;
|
||||
while (retries <= maxRetries) {
|
||||
try {
|
||||
const response = await fetch('/api/anthropic/v1/messages', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'anthropic-version': '2023-06-01',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model: model,
|
||||
max_tokens: 8192,
|
||||
temperature: 0,
|
||||
system: systemPrompt,
|
||||
messages: [{ role: 'user', content: userMessage }]
|
||||
})
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errData = await response.json().catch(() => ({}));
|
||||
console.error('[API] Error response:', errData);
|
||||
throw new Error(`API Error: ${response.status} ${response.statusText} - ${JSON.stringify(errData)}`);
|
||||
}
|
||||
|
||||
// Detect auth portal session expiry: the portal returns HTML instead of JSON
|
||||
const contentType = response.headers.get('content-type') || '';
|
||||
if (!contentType.includes('application/json')) {
|
||||
throw new Error('Your session has expired. Please refresh the page and log in again.');
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
return data.content[0].text;
|
||||
} catch (error) {
|
||||
console.error('API call failed:', error);
|
||||
retries++;
|
||||
if (retries > maxRetries) throw error;
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
}
|
||||
}
|
||||
}
|
||||
async chat(systemPrompt, messages, opts = {}) {
|
||||
const r = await callLLM({
|
||||
task: 'legacy.chat',
|
||||
tier: 'standard',
|
||||
system: systemPrompt,
|
||||
messages,
|
||||
tools: opts.tools,
|
||||
maxTokens: 1024,
|
||||
temperature: 0.3,
|
||||
});
|
||||
const content = [];
|
||||
if (r.text) content.push({ type: 'text', text: r.text });
|
||||
for (const tu of r.toolUses) content.push({ type: 'tool_use', name: tu.name, input: tu.input });
|
||||
return { content, stop_reason: r.stopReason };
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Multi-turn chat with optional tool use.
|
||||
* Returns the raw Anthropic response so callers can read both `text` and
|
||||
* `tool_use` content blocks.
|
||||
*
|
||||
* @param {string} systemPrompt
|
||||
* @param {Array<{role: 'user'|'assistant', content: string}>} messages
|
||||
* @param {{tools?: Array}} opts
|
||||
* @returns {Promise<{content: Array, stop_reason: string}>}
|
||||
*/
|
||||
anthropicApi.chat = async function chat(systemPrompt, messages, opts = {}) {
|
||||
const useSimulation = storage.get('admin:use_simulation') === true;
|
||||
if (useSimulation) {
|
||||
await new Promise(r => setTimeout(r, 600));
|
||||
return {
|
||||
content: [{
|
||||
type: 'text',
|
||||
text: 'Simulatiemodus staat aan — vraag een beheerder om Simulation Mode uit te zetten in Admin → Settings om met R42 te chatten.',
|
||||
}],
|
||||
stop_reason: 'end_turn',
|
||||
};
|
||||
}
|
||||
|
||||
const model = storage.get('admin:model') || DEFAULT_MODEL;
|
||||
|
||||
const body = {
|
||||
model,
|
||||
max_tokens: 1024,
|
||||
system: systemPrompt,
|
||||
messages,
|
||||
};
|
||||
if (opts.tools && opts.tools.length) body.tools = opts.tools;
|
||||
|
||||
const response = await fetch('/api/anthropic/v1/messages', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'anthropic-version': '2023-06-01',
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errData = await response.json().catch(() => ({}));
|
||||
throw new Error(`API Error: ${response.status} ${response.statusText} - ${JSON.stringify(errData)}`);
|
||||
}
|
||||
|
||||
const contentType = response.headers.get('content-type') || '';
|
||||
if (!contentType.includes('application/json')) {
|
||||
throw new Error('Your session has expired. Please refresh the page and log in again.');
|
||||
}
|
||||
|
||||
return await response.json();
|
||||
};
|
||||
|
||||
async function simulateResponse() {
|
||||
await new Promise(r => setTimeout(r, 2000));
|
||||
return JSON.stringify({
|
||||
topics: [
|
||||
{ id: "radicale-transparantie", label: "Radicale Transparantie", type: "concept", description: "De kernwaarde van Respellion waarbij alle informatie publiek toegankelijk is." },
|
||||
{ id: "kennisbeheer", label: "Kennisbeheer", type: "process", description: "Het proces van het vastleggen en ontsluiten van organisatiekennis." },
|
||||
{ id: "wekelijkse-sessie", label: "Wekelijkse Leersessie", type: "process", description: "Elke week leren medewerkers via AI-gegenereerde vragen en quizzen." }
|
||||
],
|
||||
relations: [
|
||||
{ source: "kennisbeheer", target: "radicale-transparantie", type: "depends_on" },
|
||||
{ source: "wekelijkse-sessie", target: "kennisbeheer", type: "part_of" }
|
||||
]
|
||||
});
|
||||
}
|
||||
|
||||
@@ -184,10 +184,7 @@ export async function autoGenerateCurriculum(year) {
|
||||
const weeks = [];
|
||||
const reviewWeeks = [13, 26, 39, 52];
|
||||
|
||||
// Calculate available weeks (52 total minus review weeks)
|
||||
const availableWeeks = 52 - reviewWeeks.length; // 48
|
||||
|
||||
// Distribute topics across available weeks
|
||||
// Distribute topics across the 48 non-review weeks.
|
||||
let topicIndex = 0;
|
||||
|
||||
for (let w = 1; w <= 52; w++) {
|
||||
|
||||
@@ -65,24 +65,20 @@ Return a JSON object:
|
||||
Return JSON only. No markdown blocks or other text.`;
|
||||
|
||||
export async function analyzeHandbookDelta(fileContent, filePath) {
|
||||
const responseText = await anthropicApi.generateContent(HANDBOOK_SYSTEM_PROMPT, `Analyze the following handbook file update (${filePath}):\n\n${fileContent}`);
|
||||
|
||||
let extractedData;
|
||||
try {
|
||||
const responseText = await anthropicApi.generateContent(HANDBOOK_SYSTEM_PROMPT, `Analyze the following handbook file update (${filePath}):\n\n${fileContent}`);
|
||||
|
||||
let extractedData;
|
||||
try {
|
||||
const jsonMatch = responseText.match(/\{[\s\S]*\}/);
|
||||
const jsonStr = jsonMatch ? jsonMatch[0] : responseText;
|
||||
extractedData = JSON.parse(jsonStr);
|
||||
} catch (e) {
|
||||
console.error('[Pipeline] AI returned non-JSON response for handbook delta:', responseText?.substring(0, 500));
|
||||
throw new Error(`AI response was not valid JSON. The model responded with: "${responseText?.substring(0, 120)}..."`);
|
||||
}
|
||||
|
||||
await mergeKnowledgeGraph(extractedData);
|
||||
return { success: true, data: extractedData };
|
||||
} catch (error) {
|
||||
throw error;
|
||||
const jsonMatch = responseText.match(/\{[\s\S]*\}/);
|
||||
const jsonStr = jsonMatch ? jsonMatch[0] : responseText;
|
||||
extractedData = JSON.parse(jsonStr);
|
||||
} catch (e) {
|
||||
console.error('[Pipeline] AI returned non-JSON response for handbook delta:', responseText?.substring(0, 500));
|
||||
throw new Error(`AI response was not valid JSON. The model responded with: "${responseText?.substring(0, 120)}..."`, { cause: e });
|
||||
}
|
||||
|
||||
await mergeKnowledgeGraph(extractedData);
|
||||
return { success: true, data: extractedData };
|
||||
}
|
||||
function chunkText(text, maxChunkSize = 4000) {
|
||||
const paragraphs = text.split(/\n+/);
|
||||
@@ -141,7 +137,7 @@ export async function processSourceText(textContent, sourceName) {
|
||||
extractedData = JSON.parse(jsonStr);
|
||||
} catch (e) {
|
||||
console.error(`[Pipeline] AI returned non-JSON response for chunk ${i + 1}:`, responseText?.substring(0, 500));
|
||||
throw new Error(`AI response for chunk ${i + 1} was not valid JSON.`);
|
||||
throw new Error(`AI response for chunk ${i + 1} was not valid JSON.`, { cause: e });
|
||||
}
|
||||
|
||||
if (extractedData.topics && Array.isArray(extractedData.topics)) {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { anthropicApi } from './api';
|
||||
import * as db from './db';
|
||||
import { getCurriculumTopic, getCurriculumYear } from './curriculumService';
|
||||
import { getCurriculumTopic } from './curriculumService';
|
||||
|
||||
const CONTENT_GENERATION_SYSTEM = `You are an expert learning content writer for Respellion, an internal IT company.
|
||||
You write training material for employees based on knowledge topics.
|
||||
@@ -138,7 +138,7 @@ ${instructions}`;
|
||||
const jsonMatch = responseText.match(/\{[\s\S]*\}/);
|
||||
newContent = JSON.parse(jsonMatch ? jsonMatch[0] : responseText);
|
||||
} catch (e) {
|
||||
throw new Error('AI could not generate valid learning content. Please try again.');
|
||||
throw new Error('AI could not generate valid learning content. Please try again.', { cause: e });
|
||||
}
|
||||
|
||||
const mergedContent = { ...(cached || {}), ...newContent };
|
||||
@@ -165,7 +165,7 @@ Apply the refinement and return the complete updated JSON object using the same
|
||||
const jsonMatch = responseText.match(/\{[\s\S]*\}/);
|
||||
content = JSON.parse(jsonMatch ? jsonMatch[0] : responseText);
|
||||
} catch (e) {
|
||||
throw new Error('AI could not process the refinement. Please try a different instruction.');
|
||||
throw new Error('AI could not process the refinement. Please try a different instruction.', { cause: e });
|
||||
}
|
||||
|
||||
await db.setContent(topic.id, content);
|
||||
@@ -198,7 +198,7 @@ Return ONLY a JSON object with this structure:
|
||||
newTopic = JSON.parse(jsonMatch ? jsonMatch[0] : responseText);
|
||||
newTopic.id = 'custom_' + Date.now().toString(36);
|
||||
} catch (e) {
|
||||
throw new Error('Could not process custom topic. Please try again.');
|
||||
throw new Error('Could not process custom topic. Please try again.', { cause: e });
|
||||
}
|
||||
|
||||
await db.upsertTopic(newTopic);
|
||||
|
||||
366
src/lib/llm.js
Normal file
366
src/lib/llm.js
Normal file
@@ -0,0 +1,366 @@
|
||||
/**
|
||||
* Single Anthropic client used by every service module.
|
||||
*
|
||||
* Centralises model selection, retry, timeout/abort, structured-output
|
||||
* parsing, schema validation, and best-effort call telemetry. Callers
|
||||
* import `callLLM` from here — they must not reach `/api/anthropic` on
|
||||
* their own.
|
||||
*/
|
||||
|
||||
import { storage } from './storage';
|
||||
import { withRetry, RetryableError, parseRetryAfter, isRetryableStatus } from './llmRetry';
|
||||
import { toolSchemaRegistry } from './llmSchemas';
|
||||
import { pb } from './pb';
|
||||
|
||||
const ANTHROPIC_URL = '/api/anthropic/v1/messages';
|
||||
const ANTHROPIC_VERSION = '2023-06-01';
|
||||
const DEFAULT_TIMEOUT_MS = 60_000;
|
||||
|
||||
const TIER_DEFAULTS = {
|
||||
fast: 'claude-haiku-4-5-20251001',
|
||||
standard: 'claude-sonnet-4-6',
|
||||
reasoning: 'claude-opus-4-7',
|
||||
};
|
||||
|
||||
export class LLMHttpError extends Error {
|
||||
constructor(status, statusText, body) {
|
||||
super(`API Error: ${status} ${statusText} - ${typeof body === 'string' ? body : JSON.stringify(body)}`);
|
||||
this.name = 'LLMHttpError';
|
||||
this.status = status;
|
||||
this.body = body;
|
||||
}
|
||||
}
|
||||
|
||||
export class LLMTruncatedError extends Error {
|
||||
constructor(task) {
|
||||
super(`LLM response truncated (stop_reason: max_tokens) for task "${task}". Increase max_tokens or shorten the input.`);
|
||||
this.name = 'LLMTruncatedError';
|
||||
}
|
||||
}
|
||||
|
||||
export class LLMOutputError extends Error {
|
||||
constructor(message) {
|
||||
super(message);
|
||||
this.name = 'LLMOutputError';
|
||||
}
|
||||
}
|
||||
|
||||
export class LLMValidationError extends Error {
|
||||
constructor(task, zodError) {
|
||||
super(`LLM output failed schema validation for task "${task}": ${zodError?.message ?? zodError}`);
|
||||
this.name = 'LLMValidationError';
|
||||
this.cause = zodError;
|
||||
}
|
||||
}
|
||||
|
||||
export function resolveModel(tier) {
|
||||
const key = `admin:model:${tier}`;
|
||||
const override = storage.get(key);
|
||||
if (override) return String(override).trim();
|
||||
if (tier === 'standard') {
|
||||
const legacy = storage.get('admin:model');
|
||||
if (legacy) return String(legacy).trim();
|
||||
}
|
||||
return TIER_DEFAULTS[tier] ?? TIER_DEFAULTS.standard;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the outermost balanced JSON value (object or array) from arbitrary
|
||||
* model output. Strips ```json fences first. Brace-matching ignores braces
|
||||
* inside strings; escapes inside strings are skipped.
|
||||
*/
|
||||
export function parseStructuredText(raw) {
|
||||
if (typeof raw !== 'string') throw new LLMOutputError('LLM returned no text.');
|
||||
let text = raw.trim();
|
||||
text = text.replace(/```(?:json)?\s*/gi, '').replace(/```/g, '');
|
||||
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
const ch = text[i];
|
||||
if (ch !== '{' && ch !== '[') continue;
|
||||
const open = ch;
|
||||
const close = ch === '{' ? '}' : ']';
|
||||
let depth = 0;
|
||||
let inString = false;
|
||||
for (let j = i; j < text.length; j++) {
|
||||
const c = text[j];
|
||||
if (inString) {
|
||||
if (c === '\\') { j++; continue; }
|
||||
if (c === '"') inString = false;
|
||||
continue;
|
||||
}
|
||||
if (c === '"') { inString = true; continue; }
|
||||
if (c === open) depth++;
|
||||
else if (c === close) {
|
||||
depth--;
|
||||
if (depth === 0) {
|
||||
const slice = text.slice(i, j + 1);
|
||||
try {
|
||||
return JSON.parse(slice);
|
||||
} catch {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
throw new LLMOutputError('No balanced JSON value found in LLM output.');
|
||||
}
|
||||
|
||||
function buildMessages({ messages, user }) {
|
||||
if (Array.isArray(messages) && messages.length) return messages;
|
||||
if (typeof user === 'string' && user.length) return [{ role: 'user', content: user }];
|
||||
throw new Error('callLLM requires either `messages` or `user`.');
|
||||
}
|
||||
|
||||
function logLlmCall(record) {
|
||||
try {
|
||||
pb.collection('llm_calls').create(record).catch(() => {});
|
||||
} catch {
|
||||
/* collection may not exist yet — swallow */
|
||||
}
|
||||
}
|
||||
|
||||
function isChatLikeTask(task) {
|
||||
if (!task) return false;
|
||||
return task === 'legacy.chat' || task.startsWith('chat.') || task.startsWith('r42.');
|
||||
}
|
||||
|
||||
const SIMULATION_EXTRACTION_PAYLOAD = JSON.stringify({
|
||||
topics: [
|
||||
{ id: 'radicale-transparantie', label: 'Radicale Transparantie', type: 'concept', description: 'De kernwaarde van Respellion waarbij alle informatie publiek toegankelijk is.', learning_relevance: 'core' },
|
||||
{ id: 'kennisbeheer', label: 'Kennisbeheer', type: 'process', description: 'Het proces van het vastleggen en ontsluiten van organisatiekennis.', learning_relevance: 'standard' },
|
||||
{ id: 'wekelijkse-sessie', label: 'Wekelijkse Leersessie', type: 'process', description: 'Elke week leren medewerkers via AI-gegenereerde vragen en quizzen.', learning_relevance: 'standard' },
|
||||
],
|
||||
relations: [
|
||||
{ source: 'kennisbeheer', target: 'radicale-transparantie', type: 'depends_on' },
|
||||
{ source: 'wekelijkse-sessie', target: 'kennisbeheer', type: 'part_of' },
|
||||
],
|
||||
});
|
||||
|
||||
const SIMULATION_CHAT_TEXT =
|
||||
'Simulatiemodus staat aan — vraag een beheerder om Simulation Mode uit te zetten in Admin → Settings om met R42 te chatten.';
|
||||
|
||||
async function simulatedResponse({ task }) {
|
||||
await new Promise((r) => setTimeout(r, 400));
|
||||
if (isChatLikeTask(task)) {
|
||||
return {
|
||||
text: SIMULATION_CHAT_TEXT,
|
||||
toolUses: [],
|
||||
stopReason: 'end_turn',
|
||||
usage: { input_tokens: 0, output_tokens: 0, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 },
|
||||
requestId: null,
|
||||
model: 'simulation',
|
||||
durationMs: 400,
|
||||
};
|
||||
}
|
||||
return {
|
||||
text: SIMULATION_EXTRACTION_PAYLOAD,
|
||||
toolUses: [],
|
||||
stopReason: 'end_turn',
|
||||
usage: { input_tokens: 0, output_tokens: 0, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 },
|
||||
requestId: null,
|
||||
model: 'simulation',
|
||||
durationMs: 400,
|
||||
};
|
||||
}
|
||||
|
||||
function linkSignals(userSignal, timeoutSignal) {
|
||||
const controller = new AbortController();
|
||||
const abort = (reason) => controller.abort(reason);
|
||||
if (userSignal) {
|
||||
if (userSignal.aborted) controller.abort(userSignal.reason);
|
||||
else userSignal.addEventListener('abort', () => abort(userSignal.reason), { once: true });
|
||||
}
|
||||
if (timeoutSignal) {
|
||||
if (timeoutSignal.aborted) controller.abort(timeoutSignal.reason);
|
||||
else timeoutSignal.addEventListener('abort', () => abort(timeoutSignal.reason), { once: true });
|
||||
}
|
||||
return controller.signal;
|
||||
}
|
||||
|
||||
function extractToolUses(content) {
|
||||
if (!Array.isArray(content)) return [];
|
||||
return content
|
||||
.filter((b) => b?.type === 'tool_use')
|
||||
.map((b) => ({ name: b.name, input: b.input }));
|
||||
}
|
||||
|
||||
function extractText(content) {
|
||||
if (!Array.isArray(content)) return '';
|
||||
return content
|
||||
.filter((b) => b?.type === 'text' && typeof b.text === 'string')
|
||||
.map((b) => b.text)
|
||||
.join('');
|
||||
}
|
||||
|
||||
function validateToolInputs(toolUses, task, toolSchemas) {
|
||||
const registry = { ...toolSchemaRegistry, ...(toolSchemas || {}) };
|
||||
for (const tu of toolUses) {
|
||||
const schema = registry[tu.name];
|
||||
if (!schema) continue;
|
||||
const result = schema.safeParse(tu.input);
|
||||
if (!result.success) throw new LLMValidationError(`${task}:${tu.name}`, result.error);
|
||||
tu.input = result.data;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @typedef {Object} CallLLMOptions
|
||||
* @property {string} task Logging label, e.g. 'extract.source'.
|
||||
* @property {'fast'|'standard'|'reasoning'} [tier='standard']
|
||||
* @property {string|Array<{type:'text',text:string,cache_control?:{type:'ephemeral'}}>} [system]
|
||||
* @property {Array<{role:'user'|'assistant',content:any}>} [messages]
|
||||
* @property {string} [user] Shorthand for a single user message.
|
||||
* @property {Array<object>} [tools] Anthropic tool definitions.
|
||||
* @property {object} [toolChoice] e.g. { type: 'tool', name: 'emit_knowledge_graph' }.
|
||||
* @property {import('zod').ZodTypeAny} [schema] For text→JSON validation.
|
||||
* @property {Record<string, import('zod').ZodTypeAny>} [toolSchemas] Overrides for tool_use input validation.
|
||||
* @property {number} [maxTokens=4096]
|
||||
* @property {number} [temperature=0]
|
||||
* @property {AbortSignal} [signal]
|
||||
*/
|
||||
|
||||
/**
|
||||
* @param {CallLLMOptions} options
|
||||
*/
|
||||
export async function callLLM(options) {
|
||||
const {
|
||||
task,
|
||||
tier = 'standard',
|
||||
system,
|
||||
messages,
|
||||
user,
|
||||
tools,
|
||||
toolChoice,
|
||||
schema,
|
||||
toolSchemas,
|
||||
maxTokens = 4096,
|
||||
temperature = 0,
|
||||
signal,
|
||||
} = options;
|
||||
if (!task) throw new Error('callLLM requires a `task` label.');
|
||||
|
||||
const useSimulation = storage.get('admin:use_simulation') === true;
|
||||
if (useSimulation) return simulatedResponse({ task });
|
||||
|
||||
const model = resolveModel(tier);
|
||||
const messagesPayload = buildMessages({ messages, user });
|
||||
|
||||
const body = {
|
||||
model,
|
||||
max_tokens: maxTokens,
|
||||
temperature,
|
||||
messages: messagesPayload,
|
||||
};
|
||||
if (system !== undefined) body.system = system;
|
||||
if (tools && tools.length) body.tools = tools;
|
||||
if (toolChoice) body.tool_choice = toolChoice;
|
||||
|
||||
const start = Date.now();
|
||||
let result;
|
||||
try {
|
||||
result = await withRetry(
|
||||
async () => {
|
||||
const timeoutCtl = signal ? null : new AbortController();
|
||||
const timer = timeoutCtl ? setTimeout(() => timeoutCtl.abort(new DOMException('Timeout', 'AbortError')), DEFAULT_TIMEOUT_MS) : null;
|
||||
const fetchSignal = linkSignals(signal, timeoutCtl?.signal);
|
||||
|
||||
try {
|
||||
const response = await fetch(ANTHROPIC_URL, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'anthropic-version': ANTHROPIC_VERSION,
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
signal: fetchSignal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errBody = await response.json().catch(() => ({}));
|
||||
if (isRetryableStatus(response.status)) {
|
||||
const retryAfterMs = parseRetryAfter(response.headers.get('Retry-After'));
|
||||
throw new RetryableError(response.status, retryAfterMs, `HTTP ${response.status}`);
|
||||
}
|
||||
throw new LLMHttpError(response.status, response.statusText, errBody);
|
||||
}
|
||||
|
||||
const contentType = response.headers.get('content-type') || '';
|
||||
if (!contentType.includes('application/json')) {
|
||||
throw new Error('Your session has expired. Please refresh the page and log in again.');
|
||||
}
|
||||
|
||||
return await response.json();
|
||||
} finally {
|
||||
if (timer) clearTimeout(timer);
|
||||
}
|
||||
},
|
||||
{ signal },
|
||||
);
|
||||
} catch (err) {
|
||||
logLlmCall({
|
||||
task,
|
||||
model,
|
||||
tier,
|
||||
duration_ms: Date.now() - start,
|
||||
input_tokens: 0,
|
||||
output_tokens: 0,
|
||||
cache_read_tokens: 0,
|
||||
cache_create_tokens: 0,
|
||||
stop_reason: '',
|
||||
ok: false,
|
||||
error_msg: String(err?.message ?? err).slice(0, 500),
|
||||
});
|
||||
throw err;
|
||||
}
|
||||
|
||||
const stopReason = result.stop_reason || '';
|
||||
const toolUses = extractToolUses(result.content);
|
||||
const text = extractText(result.content);
|
||||
const usage = result.usage || {};
|
||||
|
||||
const truncationRequiresFailure =
|
||||
stopReason === 'max_tokens' && (Boolean(schema) || Boolean(toolChoice));
|
||||
|
||||
logLlmCall({
|
||||
task,
|
||||
model,
|
||||
tier,
|
||||
duration_ms: Date.now() - start,
|
||||
input_tokens: usage.input_tokens ?? 0,
|
||||
output_tokens: usage.output_tokens ?? 0,
|
||||
cache_read_tokens: usage.cache_read_input_tokens ?? 0,
|
||||
cache_create_tokens: usage.cache_creation_input_tokens ?? 0,
|
||||
stop_reason: stopReason,
|
||||
ok: !truncationRequiresFailure,
|
||||
error_msg: truncationRequiresFailure ? 'max_tokens' : '',
|
||||
});
|
||||
|
||||
if (truncationRequiresFailure) throw new LLMTruncatedError(task);
|
||||
|
||||
if (toolUses.length) validateToolInputs(toolUses, task, toolSchemas);
|
||||
|
||||
let parsedFromText;
|
||||
if (schema && !toolUses.length) {
|
||||
const value = parseStructuredText(text);
|
||||
const parsed = schema.safeParse(value);
|
||||
if (!parsed.success) throw new LLMValidationError(task, parsed.error);
|
||||
parsedFromText = parsed.data;
|
||||
}
|
||||
|
||||
return {
|
||||
text,
|
||||
toolUses,
|
||||
stopReason,
|
||||
usage: {
|
||||
input_tokens: usage.input_tokens ?? 0,
|
||||
output_tokens: usage.output_tokens ?? 0,
|
||||
cache_creation_input_tokens: usage.cache_creation_input_tokens ?? 0,
|
||||
cache_read_input_tokens: usage.cache_read_input_tokens ?? 0,
|
||||
},
|
||||
requestId: result.id ?? null,
|
||||
model: result.model ?? model,
|
||||
durationMs: Date.now() - start,
|
||||
parsed: parsedFromText,
|
||||
};
|
||||
}
|
||||
96
src/lib/llmRetry.js
Normal file
96
src/lib/llmRetry.js
Normal file
@@ -0,0 +1,96 @@
|
||||
/**
|
||||
* Retry policy for LLM calls.
|
||||
*
|
||||
* Exponential backoff with full jitter, base 1000ms, cap 16000ms. Only
|
||||
* retries on transient HTTP statuses (408, 425, 429, 5xx, 529). Honours a
|
||||
* `Retry-After` hint up to 60 seconds; longer waits fail fast. Never retries
|
||||
* on AbortError.
|
||||
*/
|
||||
|
||||
const RETRYABLE_STATUSES = new Set([408, 425, 429, 500, 502, 503, 504, 529]);
|
||||
const BASE_DELAY_MS = 1000;
|
||||
const CAP_DELAY_MS = 16000;
|
||||
const MAX_RETRY_AFTER_MS = 60 * 1000;
|
||||
|
||||
export class RetryableError extends Error {
|
||||
constructor(status, retryAfterMs = null, message) {
|
||||
super(message || `Retryable HTTP ${status}`);
|
||||
this.name = 'RetryableError';
|
||||
this.status = status;
|
||||
this.retryAfterMs = retryAfterMs;
|
||||
}
|
||||
}
|
||||
|
||||
export function isRetryableStatus(status) {
|
||||
return RETRYABLE_STATUSES.has(status);
|
||||
}
|
||||
|
||||
function backoffWithJitter(attempt) {
|
||||
const exp = Math.min(CAP_DELAY_MS, BASE_DELAY_MS * 2 ** attempt);
|
||||
return Math.floor(Math.random() * exp);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse a `Retry-After` header. Returns null when absent or unusable, or
|
||||
* a millisecond delay otherwise. Supports both seconds and HTTP-date forms.
|
||||
*/
|
||||
export function parseRetryAfter(value, now = Date.now()) {
|
||||
if (value == null) return null;
|
||||
const s = String(value).trim();
|
||||
if (!s) return null;
|
||||
if (/^\d+$/.test(s)) return Number(s) * 1000;
|
||||
const dateMs = Date.parse(s);
|
||||
if (Number.isFinite(dateMs)) {
|
||||
const delta = dateMs - now;
|
||||
return delta > 0 ? delta : 0;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function sleep(ms, signal) {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (signal?.aborted) return reject(signal.reason ?? new DOMException('Aborted', 'AbortError'));
|
||||
const t = setTimeout(() => {
|
||||
signal?.removeEventListener('abort', onAbort);
|
||||
resolve();
|
||||
}, ms);
|
||||
const onAbort = () => {
|
||||
clearTimeout(t);
|
||||
reject(signal.reason ?? new DOMException('Aborted', 'AbortError'));
|
||||
};
|
||||
signal?.addEventListener('abort', onAbort, { once: true });
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Run `fn(attempt)` with retry. `fn` may throw a `RetryableError` to request
|
||||
* a retry, or any other error to fail immediately.
|
||||
*
|
||||
* @template T
|
||||
* @param {(attempt:number) => Promise<T>} fn
|
||||
* @param {{ maxRetries?: number, signal?: AbortSignal }} [opts]
|
||||
* @returns {Promise<T>}
|
||||
*/
|
||||
export async function withRetry(fn, { maxRetries = 4, signal } = {}) {
|
||||
let attempt = 0;
|
||||
for (;;) {
|
||||
if (signal?.aborted) throw signal.reason ?? new DOMException('Aborted', 'AbortError');
|
||||
try {
|
||||
return await fn(attempt);
|
||||
} catch (err) {
|
||||
if (err?.name === 'AbortError') throw err;
|
||||
const retryable = err instanceof RetryableError;
|
||||
if (!retryable || attempt >= maxRetries) throw err;
|
||||
|
||||
let delayMs;
|
||||
if (err.retryAfterMs != null) {
|
||||
if (err.retryAfterMs > MAX_RETRY_AFTER_MS) throw err;
|
||||
delayMs = err.retryAfterMs;
|
||||
} else {
|
||||
delayMs = backoffWithJitter(attempt);
|
||||
}
|
||||
await sleep(delayMs, signal);
|
||||
attempt++;
|
||||
}
|
||||
}
|
||||
}
|
||||
202
src/lib/llmSchemas.js
Normal file
202
src/lib/llmSchemas.js
Normal file
@@ -0,0 +1,202 @@
|
||||
/**
|
||||
* Zod schemas for every structured LLM output the platform consumes.
|
||||
*
|
||||
* Field names mirror what callers already produce — do not rename them
|
||||
* without migrating the corresponding service module.
|
||||
*/
|
||||
|
||||
import { z } from 'zod';
|
||||
|
||||
const topicTypeEnum = z.enum(['concept', 'role', 'process']);
|
||||
const relationTypeStrict = z.enum(['related_to', 'depends_on', 'part_of', 'executed_by']);
|
||||
const relationTypeLoose = z.enum(['related_to', 'depends_on', 'part_of', 'executed_by', 'executes']);
|
||||
const learningRelevanceEnum = z.enum(['core', 'standard', 'peripheral', 'exclude']);
|
||||
|
||||
const extractionTopicSchema = z.object({
|
||||
id: z.string().min(1),
|
||||
label: z.string().min(1),
|
||||
type: topicTypeEnum,
|
||||
description: z.string().min(1),
|
||||
learning_relevance: learningRelevanceEnum,
|
||||
});
|
||||
|
||||
const extractionRelationSchema = z.object({
|
||||
source: z.string().min(1),
|
||||
target: z.string().min(1),
|
||||
type: relationTypeStrict,
|
||||
});
|
||||
|
||||
export const extractionResultSchema = z.object({
|
||||
topics: z.array(extractionTopicSchema),
|
||||
relations: z.array(extractionRelationSchema),
|
||||
});
|
||||
|
||||
const handbookTopicSchema = extractionTopicSchema.extend({
|
||||
metadata: z.object({ source: z.string() }).optional(),
|
||||
});
|
||||
|
||||
const handbookRelationSchema = z.object({
|
||||
source: z.string().min(1),
|
||||
target: z.string().min(1),
|
||||
type: relationTypeLoose,
|
||||
description: z.string().optional(),
|
||||
});
|
||||
|
||||
export const handbookResultSchema = z.object({
|
||||
topics: z.array(handbookTopicSchema),
|
||||
relations: z.array(handbookRelationSchema),
|
||||
});
|
||||
|
||||
/**
|
||||
* Normalise legacy `executes` relations into the canonical `executed_by`
|
||||
* vocabulary by swapping source and target. The handbook prompt previously
|
||||
* emitted `role → executes → process`; the canonical form is
|
||||
* `process → executed_by → role`.
|
||||
*/
|
||||
export function normalizeHandbookResult(parsed) {
|
||||
return {
|
||||
...parsed,
|
||||
relations: parsed.relations.map((r) =>
|
||||
r.type === 'executes'
|
||||
? { ...r, type: 'executed_by', source: r.target, target: r.source }
|
||||
: r,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
const articleSectionSchema = z.object({
|
||||
heading: z.string().min(1),
|
||||
body: z.string().min(1),
|
||||
});
|
||||
|
||||
const articleBodySchema = z.object({
|
||||
title: z.string().min(1),
|
||||
intro: z.string().min(1),
|
||||
sections: z.array(articleSectionSchema).min(1),
|
||||
keyTakeaways: z.array(z.string().min(1)).min(1),
|
||||
});
|
||||
|
||||
export const learningArticleSchema = z.object({
|
||||
article: articleBodySchema,
|
||||
});
|
||||
|
||||
const slideSchema = z.object({
|
||||
title: z.string().min(1),
|
||||
bullets: z.array(z.string().min(1)).min(1),
|
||||
speakerNote: z.string().min(1),
|
||||
});
|
||||
|
||||
export const learningSlidesSchema = z.object({
|
||||
slides: z.array(slideSchema).min(1),
|
||||
});
|
||||
|
||||
const infographicStatSchema = z.object({
|
||||
value: z.string().min(1),
|
||||
label: z.string().min(1),
|
||||
icon: z.string().min(1),
|
||||
});
|
||||
|
||||
const infographicStepSchema = z.object({
|
||||
number: z.number().int().min(1),
|
||||
title: z.string().min(1),
|
||||
description: z.string().min(1),
|
||||
icon: z.string().min(1),
|
||||
});
|
||||
|
||||
const infographicBodySchema = z.object({
|
||||
headline: z.string().min(1),
|
||||
tagline: z.string().min(1),
|
||||
stats: z.array(infographicStatSchema).min(1),
|
||||
steps: z.array(infographicStepSchema).min(1),
|
||||
quote: z.string().min(1),
|
||||
colorTheme: z.string().min(1),
|
||||
});
|
||||
|
||||
export const learningInfographicSchema = z.object({
|
||||
infographic: infographicBodySchema,
|
||||
});
|
||||
|
||||
export const learningAllSchema = z.object({
|
||||
article: articleBodySchema,
|
||||
slides: z.array(slideSchema).min(1),
|
||||
infographic: infographicBodySchema,
|
||||
});
|
||||
|
||||
const quizQuestionSchema = z.object({
|
||||
id: z.string().min(1),
|
||||
question: z.string().min(1),
|
||||
topicLabel: z.string().min(1),
|
||||
options: z.array(z.string().min(1)).length(4),
|
||||
correctIndex: z.number().int().min(0).max(3),
|
||||
explanation: z.string().min(1),
|
||||
});
|
||||
|
||||
export const quizQuestionsSchema = z.object({
|
||||
questions: z.array(quizQuestionSchema).min(1),
|
||||
});
|
||||
|
||||
export const customTopicSchema = z.object({
|
||||
label: z.string().min(1),
|
||||
type: topicTypeEnum,
|
||||
description: z.string().min(1),
|
||||
});
|
||||
|
||||
const mergeActionSchema = z.object({
|
||||
keepId: z.string().min(1),
|
||||
deleteId: z.string().min(1),
|
||||
});
|
||||
|
||||
const newRelationSchema = z.object({
|
||||
source: z.string().min(1),
|
||||
target: z.string().min(1),
|
||||
type: relationTypeStrict,
|
||||
});
|
||||
|
||||
const relevanceUpdateSchema = z.object({
|
||||
id: z.string().min(1),
|
||||
learning_relevance: learningRelevanceEnum,
|
||||
});
|
||||
|
||||
export const graphActionsSchema = z.object({
|
||||
merges: z.array(mergeActionSchema).optional().default([]),
|
||||
deletions: z.array(z.string().min(1)).optional().default([]),
|
||||
newRelations: z.array(newRelationSchema).optional().default([]),
|
||||
relevanceUpdates: z.array(relevanceUpdateSchema).optional().default([]),
|
||||
});
|
||||
|
||||
const deltaTopicSchema = z.object({
|
||||
id: z.string().min(1),
|
||||
label: z.string().min(1),
|
||||
type: topicTypeEnum,
|
||||
description: z.string().min(1),
|
||||
});
|
||||
|
||||
const deltaRelationSchema = z.object({
|
||||
source: z.string().min(1),
|
||||
target: z.string().min(1),
|
||||
type: relationTypeStrict,
|
||||
});
|
||||
|
||||
export const proposeGraphDeltaSchema = z.object({
|
||||
reason: z.string().min(1),
|
||||
topics: z.array(deltaTopicSchema).max(3).optional(),
|
||||
relations: z.array(deltaRelationSchema).max(5).optional(),
|
||||
});
|
||||
|
||||
/**
|
||||
* Registry mapping known tool names to their input schemas. `callLLM`
|
||||
* consults this when the caller does not pass an explicit `toolSchemas`
|
||||
* override.
|
||||
*/
|
||||
export const toolSchemaRegistry = {
|
||||
emit_knowledge_graph: extractionResultSchema,
|
||||
emit_handbook_delta: handbookResultSchema,
|
||||
emit_learning_article: learningArticleSchema,
|
||||
emit_learning_slides: learningSlidesSchema,
|
||||
emit_learning_infographic: learningInfographicSchema,
|
||||
emit_learning_all: learningAllSchema,
|
||||
emit_quiz_questions: quizQuestionsSchema,
|
||||
emit_custom_topic: customTopicSchema,
|
||||
emit_graph_actions: graphActionsSchema,
|
||||
propose_graph_delta: proposeGraphDeltaSchema,
|
||||
};
|
||||
@@ -93,7 +93,7 @@ Rules:
|
||||
- Make questions specific and practical, not trivial.`;
|
||||
|
||||
const responseText = await anthropicApi.generateContent(QUIZ_SYSTEM, prompt);
|
||||
let newQuestions = [];
|
||||
let newQuestions;
|
||||
try {
|
||||
const jsonMatch = responseText.match(/\{[\s\S]*\}/);
|
||||
const parsed = JSON.parse(jsonMatch ? jsonMatch[0] : responseText);
|
||||
@@ -103,7 +103,7 @@ Rules:
|
||||
});
|
||||
} catch (e) {
|
||||
console.error('Failed to generate questions for topic', topic.label, e);
|
||||
throw new Error(`Could not generate questions for ${topic.label}`);
|
||||
throw new Error(`Could not generate questions for ${topic.label}`, { cause: e });
|
||||
}
|
||||
|
||||
bank = [...bank, ...newQuestions];
|
||||
|
||||
Reference in New Issue
Block a user