import { afterEach, describe, expect, it, vi } from 'vitest'; import { z } from 'zod'; vi.mock('../storage', () => ({ storage: { _data: new Map(), get(key, fallback = null) { return this._data.has(key) ? this._data.get(key) : fallback; }, set(key, value) { this._data.set(key, value); }, remove(key) { this._data.delete(key); }, getKeysByPrefix() { return []; }, }, })); vi.mock('../pb', () => ({ pb: { collection: () => ({ create: () => ({ catch: () => {} }) }) }, })); import { callLLM, LLMHttpError, LLMOutputError, LLMTruncatedError, LLMValidationError, parseStructuredText, resolveModel, } from '../llm'; import { storage } from '../storage'; const originalFetch = globalThis.fetch; afterEach(() => { globalThis.fetch = originalFetch; storage._data.clear(); }); describe('parseStructuredText', () => { it('extracts an object from raw JSON', () => { expect(parseStructuredText('{"a":1}')).toEqual({ a: 1 }); }); it('extracts an object from a json-fenced block', () => { const fenced = '```json\n{"hello":"world"}\n```'; expect(parseStructuredText(fenced)).toEqual({ hello: 'world' }); }); it('extracts an object surrounded by prose', () => { const messy = 'Sure! Here you go:\n{"id":"x","label":"X"}\nLet me know if you want changes.'; expect(parseStructuredText(messy)).toEqual({ id: 'x', label: 'X' }); }); it('extracts an array when it is the top-level value', () => { expect(parseStructuredText('[1,2,3]')).toEqual([1, 2, 3]); }); it('ignores braces inside string literals', () => { const tricky = '{"text":"this { is not } a brace"}'; expect(parseStructuredText(tricky)).toEqual({ text: 'this { is not } a brace' }); }); it('throws LLMOutputError when no balanced JSON is present', () => { expect(() => parseStructuredText('no json here, just words')).toThrow(LLMOutputError); }); }); describe('resolveModel', () => { it('falls back to tier defaults when no override is set', () => { expect(resolveModel('fast')).toBe('claude-haiku-4-5-20251001'); expect(resolveModel('standard')).toBe('claude-sonnet-4-6'); expect(resolveModel('reasoning')).toBe('claude-opus-4-7'); }); it('honours an explicit tier override', () => { storage.set('admin:model:reasoning', 'claude-opus-9-future'); expect(resolveModel('reasoning')).toBe('claude-opus-9-future'); }); it('uses the legacy admin:model setting as a standard-tier fallback', () => { storage.set('admin:model', 'claude-some-legacy-id'); expect(resolveModel('standard')).toBe('claude-some-legacy-id'); }); it('prefers the tier-specific override over the legacy fallback', () => { storage.set('admin:model', 'claude-legacy'); storage.set('admin:model:standard', 'claude-new'); expect(resolveModel('standard')).toBe('claude-new'); }); }); function mockJsonResponse(body, { status = 200, headers = {} } = {}) { const h = new Headers({ 'content-type': 'application/json', ...headers }); return new Response(JSON.stringify(body), { status, headers: h }); } describe('callLLM happy path', () => { it('returns parsed tool input when toolChoice forces a tool', async () => { globalThis.fetch = vi.fn(async () => mockJsonResponse({ id: 'msg_1', model: 'claude-sonnet-4-6', stop_reason: 'tool_use', usage: { input_tokens: 10, output_tokens: 20 }, content: [ { type: 'tool_use', name: 'emit_custom_topic', input: { label: 'Pair Programming', type: 'process', description: 'Two engineers, one keyboard.' }, }, ], }), ); const result = await callLLM({ task: 'learning.custom_topic', tier: 'standard', user: 'Pair programming', tools: [{ name: 'emit_custom_topic', description: 'x', input_schema: { type: 'object' } }], toolChoice: { type: 'tool', name: 'emit_custom_topic' }, }); expect(result.toolUses).toHaveLength(1); expect(result.toolUses[0].input.label).toBe('Pair Programming'); }); it('parses and validates plain text against a Zod schema', async () => { globalThis.fetch = vi.fn(async () => mockJsonResponse({ id: 'msg_2', model: 'claude-sonnet-4-6', stop_reason: 'end_turn', usage: { input_tokens: 5, output_tokens: 7 }, content: [{ type: 'text', text: '```json\n{"value":42}\n```' }], }), ); const schema = z.object({ value: z.number() }); const result = await callLLM({ task: 'demo.json', user: 'give me a number', schema, }); expect(result.parsed).toEqual({ value: 42 }); }); }); describe('callLLM error paths', () => { it('throws LLMTruncatedError when stop_reason is max_tokens and a tool was requested', async () => { globalThis.fetch = vi.fn(async () => mockJsonResponse({ stop_reason: 'max_tokens', usage: { input_tokens: 1, output_tokens: 1 }, content: [], }), ); await expect( callLLM({ task: 'extract.source', user: 'x', tools: [{ name: 'emit_knowledge_graph', description: 'x', input_schema: { type: 'object' } }], toolChoice: { type: 'tool', name: 'emit_knowledge_graph' }, }), ).rejects.toBeInstanceOf(LLMTruncatedError); }); it('throws LLMTruncatedError when stop_reason is max_tokens and a schema was requested', async () => { globalThis.fetch = vi.fn(async () => mockJsonResponse({ stop_reason: 'max_tokens', usage: { input_tokens: 1, output_tokens: 1 }, content: [{ type: 'text', text: 'partial...' }], }), ); const schema = z.object({ value: z.number() }); await expect( callLLM({ task: 'demo.json', user: 'x', schema }), ).rejects.toBeInstanceOf(LLMTruncatedError); }); it('throws LLMValidationError when tool input fails schema validation', async () => { globalThis.fetch = vi.fn(async () => mockJsonResponse({ stop_reason: 'tool_use', usage: {}, content: [ { type: 'tool_use', name: 'emit_custom_topic', input: { label: 'X', type: 'concept' } }, ], }), ); await expect( callLLM({ task: 'learning.custom_topic', user: 'x', tools: [{ name: 'emit_custom_topic', description: 'x', input_schema: { type: 'object' } }], toolChoice: { type: 'tool', name: 'emit_custom_topic' }, }), ).rejects.toBeInstanceOf(LLMValidationError); }); it('surfaces a non-retryable HTTP error as LLMHttpError', async () => { globalThis.fetch = vi.fn(async () => new Response(JSON.stringify({ error: 'bad request' }), { status: 400, headers: { 'content-type': 'application/json' }, }), ); await expect(callLLM({ task: 'demo', user: 'x' })).rejects.toBeInstanceOf(LLMHttpError); }); it('detects an auth portal HTML response and raises a clear message', async () => { globalThis.fetch = vi.fn(async () => new Response('login', { status: 200, headers: { 'content-type': 'text/html' } }), ); await expect(callLLM({ task: 'demo', user: 'x' })).rejects.toThrow(/session has expired/i); }); }); describe('callLLM simulation mode', () => { it('returns the chat stub when admin:use_simulation is true and task is chat-like', async () => { storage.set('admin:use_simulation', true); const result = await callLLM({ task: 'chat.r42', user: 'hello' }); expect(result.stopReason).toBe('end_turn'); expect(result.text).toMatch(/Simulatiemodus/); }); it('returns the extraction stub for other tasks in simulation mode', async () => { storage.set('admin:use_simulation', true); const result = await callLLM({ task: 'extract.source', user: 'doc' }); expect(() => JSON.parse(result.text)).not.toThrow(); expect(JSON.parse(result.text)).toHaveProperty('topics'); }); });