feat: phase 1 of AI pipeline hardening — single LLM client + tier-aware models
Implements phase 1 of AI_PIPELINE_HARDENING_PLAN.md. Every Anthropic call now goes through one module that owns retry, timeout, abort, structured- output parsing, schema validation, and best-effort call telemetry. * src/lib/llm.js — single callLLM entry point. Resolves model per tier (fast / standard / reasoning) with admin:model legacy fallback for the standard tier; 60s default timeout via AbortController; balanced-brace JSON extraction; LLMHttpError, LLMTruncatedError, LLMOutputError, and LLMValidationError surface clearly distinct failure modes. * src/lib/llmRetry.js — exponential backoff with full jitter, retries only on transient HTTP statuses, honours Retry-After up to 60s, never retries on AbortError. * src/lib/llmSchemas.js — Zod schemas for every structured task plus normalizeHandbookResult (collapses legacy "executes" relations into the canonical "executed_by" vocabulary). * src/lib/api.js — thin shim over callLLM so existing callers (extraction pipeline, learning, quiz, R42, knowledge graph) keep working unchanged. * src/lib/__tests__/ — 32 Vitest cases covering parse paths, error surfaces, simulation mode, model resolution, and schema validation. * src/pages/Admin/index.jsx — three model inputs (fast / standard / reasoning) replacing the single legacy field; legacy value falls back for the standard tier so existing overrides survive. Adds Zod and Vitest, plus an "npm run test" script. Also cleans up the pre-existing repo-wide ESLint failures so phase 1's "npm run lint passes" acceptance criterion can be checked: drops unused React imports across the JSX tree (React 19 JSX runtime auto-imports), attaches cause to rethrown errors in the service modules, ignores pb_migrations in the ESLint config (PocketBase JSVM globals), and removes one dead handleCreateCustom function in Leren.jsx. A real behaviour bug surfaced in Testen.jsx — the quiz timer captured a stale finishQuiz via setInterval closure; now updated via finishQuizRef so the timer always invokes the latest callback. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
RaymondVerhoef
366
src/lib/llm.js
Normal file
366
src/lib/llm.js
Normal file
@@ -0,0 +1,366 @@
|
||||
/**
|
||||
* Single Anthropic client used by every service module.
|
||||
*
|
||||
* Centralises model selection, retry, timeout/abort, structured-output
|
||||
* parsing, schema validation, and best-effort call telemetry. Callers
|
||||
* import `callLLM` from here — they must not reach `/api/anthropic` on
|
||||
* their own.
|
||||
*/
|
||||
|
||||
import { storage } from './storage';
|
||||
import { withRetry, RetryableError, parseRetryAfter, isRetryableStatus } from './llmRetry';
|
||||
import { toolSchemaRegistry } from './llmSchemas';
|
||||
import { pb } from './pb';
|
||||
|
||||
const ANTHROPIC_URL = '/api/anthropic/v1/messages';
|
||||
const ANTHROPIC_VERSION = '2023-06-01';
|
||||
const DEFAULT_TIMEOUT_MS = 60_000;
|
||||
|
||||
const TIER_DEFAULTS = {
|
||||
fast: 'claude-haiku-4-5-20251001',
|
||||
standard: 'claude-sonnet-4-6',
|
||||
reasoning: 'claude-opus-4-7',
|
||||
};
|
||||
|
||||
export class LLMHttpError extends Error {
|
||||
constructor(status, statusText, body) {
|
||||
super(`API Error: ${status} ${statusText} - ${typeof body === 'string' ? body : JSON.stringify(body)}`);
|
||||
this.name = 'LLMHttpError';
|
||||
this.status = status;
|
||||
this.body = body;
|
||||
}
|
||||
}
|
||||
|
||||
export class LLMTruncatedError extends Error {
|
||||
constructor(task) {
|
||||
super(`LLM response truncated (stop_reason: max_tokens) for task "${task}". Increase max_tokens or shorten the input.`);
|
||||
this.name = 'LLMTruncatedError';
|
||||
}
|
||||
}
|
||||
|
||||
export class LLMOutputError extends Error {
|
||||
constructor(message) {
|
||||
super(message);
|
||||
this.name = 'LLMOutputError';
|
||||
}
|
||||
}
|
||||
|
||||
export class LLMValidationError extends Error {
|
||||
constructor(task, zodError) {
|
||||
super(`LLM output failed schema validation for task "${task}": ${zodError?.message ?? zodError}`);
|
||||
this.name = 'LLMValidationError';
|
||||
this.cause = zodError;
|
||||
}
|
||||
}
|
||||
|
||||
export function resolveModel(tier) {
|
||||
const key = `admin:model:${tier}`;
|
||||
const override = storage.get(key);
|
||||
if (override) return String(override).trim();
|
||||
if (tier === 'standard') {
|
||||
const legacy = storage.get('admin:model');
|
||||
if (legacy) return String(legacy).trim();
|
||||
}
|
||||
return TIER_DEFAULTS[tier] ?? TIER_DEFAULTS.standard;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract the outermost balanced JSON value (object or array) from arbitrary
|
||||
* model output. Strips ```json fences first. Brace-matching ignores braces
|
||||
* inside strings; escapes inside strings are skipped.
|
||||
*/
|
||||
export function parseStructuredText(raw) {
|
||||
if (typeof raw !== 'string') throw new LLMOutputError('LLM returned no text.');
|
||||
let text = raw.trim();
|
||||
text = text.replace(/```(?:json)?\s*/gi, '').replace(/```/g, '');
|
||||
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
const ch = text[i];
|
||||
if (ch !== '{' && ch !== '[') continue;
|
||||
const open = ch;
|
||||
const close = ch === '{' ? '}' : ']';
|
||||
let depth = 0;
|
||||
let inString = false;
|
||||
for (let j = i; j < text.length; j++) {
|
||||
const c = text[j];
|
||||
if (inString) {
|
||||
if (c === '\\') { j++; continue; }
|
||||
if (c === '"') inString = false;
|
||||
continue;
|
||||
}
|
||||
if (c === '"') { inString = true; continue; }
|
||||
if (c === open) depth++;
|
||||
else if (c === close) {
|
||||
depth--;
|
||||
if (depth === 0) {
|
||||
const slice = text.slice(i, j + 1);
|
||||
try {
|
||||
return JSON.parse(slice);
|
||||
} catch {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
throw new LLMOutputError('No balanced JSON value found in LLM output.');
|
||||
}
|
||||
|
||||
function buildMessages({ messages, user }) {
|
||||
if (Array.isArray(messages) && messages.length) return messages;
|
||||
if (typeof user === 'string' && user.length) return [{ role: 'user', content: user }];
|
||||
throw new Error('callLLM requires either `messages` or `user`.');
|
||||
}
|
||||
|
||||
function logLlmCall(record) {
|
||||
try {
|
||||
pb.collection('llm_calls').create(record).catch(() => {});
|
||||
} catch {
|
||||
/* collection may not exist yet — swallow */
|
||||
}
|
||||
}
|
||||
|
||||
function isChatLikeTask(task) {
|
||||
if (!task) return false;
|
||||
return task === 'legacy.chat' || task.startsWith('chat.') || task.startsWith('r42.');
|
||||
}
|
||||
|
||||
const SIMULATION_EXTRACTION_PAYLOAD = JSON.stringify({
|
||||
topics: [
|
||||
{ id: 'radicale-transparantie', label: 'Radicale Transparantie', type: 'concept', description: 'De kernwaarde van Respellion waarbij alle informatie publiek toegankelijk is.', learning_relevance: 'core' },
|
||||
{ id: 'kennisbeheer', label: 'Kennisbeheer', type: 'process', description: 'Het proces van het vastleggen en ontsluiten van organisatiekennis.', learning_relevance: 'standard' },
|
||||
{ id: 'wekelijkse-sessie', label: 'Wekelijkse Leersessie', type: 'process', description: 'Elke week leren medewerkers via AI-gegenereerde vragen en quizzen.', learning_relevance: 'standard' },
|
||||
],
|
||||
relations: [
|
||||
{ source: 'kennisbeheer', target: 'radicale-transparantie', type: 'depends_on' },
|
||||
{ source: 'wekelijkse-sessie', target: 'kennisbeheer', type: 'part_of' },
|
||||
],
|
||||
});
|
||||
|
||||
const SIMULATION_CHAT_TEXT =
|
||||
'Simulatiemodus staat aan — vraag een beheerder om Simulation Mode uit te zetten in Admin → Settings om met R42 te chatten.';
|
||||
|
||||
async function simulatedResponse({ task }) {
|
||||
await new Promise((r) => setTimeout(r, 400));
|
||||
if (isChatLikeTask(task)) {
|
||||
return {
|
||||
text: SIMULATION_CHAT_TEXT,
|
||||
toolUses: [],
|
||||
stopReason: 'end_turn',
|
||||
usage: { input_tokens: 0, output_tokens: 0, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 },
|
||||
requestId: null,
|
||||
model: 'simulation',
|
||||
durationMs: 400,
|
||||
};
|
||||
}
|
||||
return {
|
||||
text: SIMULATION_EXTRACTION_PAYLOAD,
|
||||
toolUses: [],
|
||||
stopReason: 'end_turn',
|
||||
usage: { input_tokens: 0, output_tokens: 0, cache_creation_input_tokens: 0, cache_read_input_tokens: 0 },
|
||||
requestId: null,
|
||||
model: 'simulation',
|
||||
durationMs: 400,
|
||||
};
|
||||
}
|
||||
|
||||
function linkSignals(userSignal, timeoutSignal) {
|
||||
const controller = new AbortController();
|
||||
const abort = (reason) => controller.abort(reason);
|
||||
if (userSignal) {
|
||||
if (userSignal.aborted) controller.abort(userSignal.reason);
|
||||
else userSignal.addEventListener('abort', () => abort(userSignal.reason), { once: true });
|
||||
}
|
||||
if (timeoutSignal) {
|
||||
if (timeoutSignal.aborted) controller.abort(timeoutSignal.reason);
|
||||
else timeoutSignal.addEventListener('abort', () => abort(timeoutSignal.reason), { once: true });
|
||||
}
|
||||
return controller.signal;
|
||||
}
|
||||
|
||||
function extractToolUses(content) {
|
||||
if (!Array.isArray(content)) return [];
|
||||
return content
|
||||
.filter((b) => b?.type === 'tool_use')
|
||||
.map((b) => ({ name: b.name, input: b.input }));
|
||||
}
|
||||
|
||||
function extractText(content) {
|
||||
if (!Array.isArray(content)) return '';
|
||||
return content
|
||||
.filter((b) => b?.type === 'text' && typeof b.text === 'string')
|
||||
.map((b) => b.text)
|
||||
.join('');
|
||||
}
|
||||
|
||||
function validateToolInputs(toolUses, task, toolSchemas) {
|
||||
const registry = { ...toolSchemaRegistry, ...(toolSchemas || {}) };
|
||||
for (const tu of toolUses) {
|
||||
const schema = registry[tu.name];
|
||||
if (!schema) continue;
|
||||
const result = schema.safeParse(tu.input);
|
||||
if (!result.success) throw new LLMValidationError(`${task}:${tu.name}`, result.error);
|
||||
tu.input = result.data;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @typedef {Object} CallLLMOptions
|
||||
* @property {string} task Logging label, e.g. 'extract.source'.
|
||||
* @property {'fast'|'standard'|'reasoning'} [tier='standard']
|
||||
* @property {string|Array<{type:'text',text:string,cache_control?:{type:'ephemeral'}}>} [system]
|
||||
* @property {Array<{role:'user'|'assistant',content:any}>} [messages]
|
||||
* @property {string} [user] Shorthand for a single user message.
|
||||
* @property {Array<object>} [tools] Anthropic tool definitions.
|
||||
* @property {object} [toolChoice] e.g. { type: 'tool', name: 'emit_knowledge_graph' }.
|
||||
* @property {import('zod').ZodTypeAny} [schema] For text→JSON validation.
|
||||
* @property {Record<string, import('zod').ZodTypeAny>} [toolSchemas] Overrides for tool_use input validation.
|
||||
* @property {number} [maxTokens=4096]
|
||||
* @property {number} [temperature=0]
|
||||
* @property {AbortSignal} [signal]
|
||||
*/
|
||||
|
||||
/**
|
||||
* @param {CallLLMOptions} options
|
||||
*/
|
||||
export async function callLLM(options) {
|
||||
const {
|
||||
task,
|
||||
tier = 'standard',
|
||||
system,
|
||||
messages,
|
||||
user,
|
||||
tools,
|
||||
toolChoice,
|
||||
schema,
|
||||
toolSchemas,
|
||||
maxTokens = 4096,
|
||||
temperature = 0,
|
||||
signal,
|
||||
} = options;
|
||||
if (!task) throw new Error('callLLM requires a `task` label.');
|
||||
|
||||
const useSimulation = storage.get('admin:use_simulation') === true;
|
||||
if (useSimulation) return simulatedResponse({ task });
|
||||
|
||||
const model = resolveModel(tier);
|
||||
const messagesPayload = buildMessages({ messages, user });
|
||||
|
||||
const body = {
|
||||
model,
|
||||
max_tokens: maxTokens,
|
||||
temperature,
|
||||
messages: messagesPayload,
|
||||
};
|
||||
if (system !== undefined) body.system = system;
|
||||
if (tools && tools.length) body.tools = tools;
|
||||
if (toolChoice) body.tool_choice = toolChoice;
|
||||
|
||||
const start = Date.now();
|
||||
let result;
|
||||
try {
|
||||
result = await withRetry(
|
||||
async () => {
|
||||
const timeoutCtl = signal ? null : new AbortController();
|
||||
const timer = timeoutCtl ? setTimeout(() => timeoutCtl.abort(new DOMException('Timeout', 'AbortError')), DEFAULT_TIMEOUT_MS) : null;
|
||||
const fetchSignal = linkSignals(signal, timeoutCtl?.signal);
|
||||
|
||||
try {
|
||||
const response = await fetch(ANTHROPIC_URL, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'anthropic-version': ANTHROPIC_VERSION,
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
signal: fetchSignal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const errBody = await response.json().catch(() => ({}));
|
||||
if (isRetryableStatus(response.status)) {
|
||||
const retryAfterMs = parseRetryAfter(response.headers.get('Retry-After'));
|
||||
throw new RetryableError(response.status, retryAfterMs, `HTTP ${response.status}`);
|
||||
}
|
||||
throw new LLMHttpError(response.status, response.statusText, errBody);
|
||||
}
|
||||
|
||||
const contentType = response.headers.get('content-type') || '';
|
||||
if (!contentType.includes('application/json')) {
|
||||
throw new Error('Your session has expired. Please refresh the page and log in again.');
|
||||
}
|
||||
|
||||
return await response.json();
|
||||
} finally {
|
||||
if (timer) clearTimeout(timer);
|
||||
}
|
||||
},
|
||||
{ signal },
|
||||
);
|
||||
} catch (err) {
|
||||
logLlmCall({
|
||||
task,
|
||||
model,
|
||||
tier,
|
||||
duration_ms: Date.now() - start,
|
||||
input_tokens: 0,
|
||||
output_tokens: 0,
|
||||
cache_read_tokens: 0,
|
||||
cache_create_tokens: 0,
|
||||
stop_reason: '',
|
||||
ok: false,
|
||||
error_msg: String(err?.message ?? err).slice(0, 500),
|
||||
});
|
||||
throw err;
|
||||
}
|
||||
|
||||
const stopReason = result.stop_reason || '';
|
||||
const toolUses = extractToolUses(result.content);
|
||||
const text = extractText(result.content);
|
||||
const usage = result.usage || {};
|
||||
|
||||
const truncationRequiresFailure =
|
||||
stopReason === 'max_tokens' && (Boolean(schema) || Boolean(toolChoice));
|
||||
|
||||
logLlmCall({
|
||||
task,
|
||||
model,
|
||||
tier,
|
||||
duration_ms: Date.now() - start,
|
||||
input_tokens: usage.input_tokens ?? 0,
|
||||
output_tokens: usage.output_tokens ?? 0,
|
||||
cache_read_tokens: usage.cache_read_input_tokens ?? 0,
|
||||
cache_create_tokens: usage.cache_creation_input_tokens ?? 0,
|
||||
stop_reason: stopReason,
|
||||
ok: !truncationRequiresFailure,
|
||||
error_msg: truncationRequiresFailure ? 'max_tokens' : '',
|
||||
});
|
||||
|
||||
if (truncationRequiresFailure) throw new LLMTruncatedError(task);
|
||||
|
||||
if (toolUses.length) validateToolInputs(toolUses, task, toolSchemas);
|
||||
|
||||
let parsedFromText;
|
||||
if (schema && !toolUses.length) {
|
||||
const value = parseStructuredText(text);
|
||||
const parsed = schema.safeParse(value);
|
||||
if (!parsed.success) throw new LLMValidationError(task, parsed.error);
|
||||
parsedFromText = parsed.data;
|
||||
}
|
||||
|
||||
return {
|
||||
text,
|
||||
toolUses,
|
||||
stopReason,
|
||||
usage: {
|
||||
input_tokens: usage.input_tokens ?? 0,
|
||||
output_tokens: usage.output_tokens ?? 0,
|
||||
cache_creation_input_tokens: usage.cache_creation_input_tokens ?? 0,
|
||||
cache_read_input_tokens: usage.cache_read_input_tokens ?? 0,
|
||||
},
|
||||
requestId: result.id ?? null,
|
||||
model: result.model ?? model,
|
||||
durationMs: Date.now() - start,
|
||||
parsed: parsedFromText,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user