learning-platform/src/lib/__tests__/llm.test.js

import { afterEach, describe, expect, it, vi } from 'vitest';
import { z } from 'zod';

vi.mock('../storage', () => ({
  storage: {
    _data: new Map(),
    get(key, fallback = null) {
      return this._data.has(key) ? this._data.get(key) : fallback;
    },
    set(key, value) { this._data.set(key, value); },
    remove(key) { this._data.delete(key); },
    getKeysByPrefix() { return []; },
  },
}));

vi.mock('../pb', () => ({
  pb: { collection: () => ({ create: () => ({ catch: () => {} }) }) },
}));

import {
  callLLM,
  LLMHttpError,
  LLMOutputError,
  LLMTruncatedError,
  LLMValidationError,
  parseStructuredText,
  resolveModel,
} from '../llm';
import { storage } from '../storage';

const originalFetch = globalThis.fetch;

afterEach(() => {
  globalThis.fetch = originalFetch;
  storage._data.clear();
});

describe('parseStructuredText', () => {
  it('extracts an object from raw JSON', () => {
    expect(parseStructuredText('{"a":1}')).toEqual({ a: 1 });
  });

  it('extracts an object from a json-fenced block', () => {
    const fenced = '```json\n{"hello":"world"}\n```';
    expect(parseStructuredText(fenced)).toEqual({ hello: 'world' });
  });

  it('extracts an object surrounded by prose', () => {
    const messy = 'Sure! Here you go:\n{"id":"x","label":"X"}\nLet me know if you want changes.';
    expect(parseStructuredText(messy)).toEqual({ id: 'x', label: 'X' });
  });

  it('extracts an array when it is the top-level value', () => {
    expect(parseStructuredText('[1,2,3]')).toEqual([1, 2, 3]);
  });

  it('ignores braces inside string literals', () => {
    const tricky = '{"text":"this { is not } a brace"}';
    expect(parseStructuredText(tricky)).toEqual({ text: 'this { is not } a brace' });
  });

  it('throws LLMOutputError when no balanced JSON is present', () => {
    expect(() => parseStructuredText('no json here, just words')).toThrow(LLMOutputError);
  });
});

describe('resolveModel', () => {
  it('falls back to tier defaults when no override is set', () => {
    expect(resolveModel('fast')).toBe('claude-haiku-4-5-20251001');
    expect(resolveModel('standard')).toBe('claude-sonnet-4-6');
    expect(resolveModel('reasoning')).toBe('claude-opus-4-7');
  });

  it('honours an explicit tier override', () => {
    storage.set('admin:model:reasoning', 'claude-opus-9-future');
    expect(resolveModel('reasoning')).toBe('claude-opus-9-future');
  });

  it('uses the legacy admin:model setting as a standard-tier fallback', () => {
    storage.set('admin:model', 'claude-some-legacy-id');
    expect(resolveModel('standard')).toBe('claude-some-legacy-id');
  });

  it('prefers the tier-specific override over the legacy fallback', () => {
    storage.set('admin:model', 'claude-legacy');
    storage.set('admin:model:standard', 'claude-new');
    expect(resolveModel('standard')).toBe('claude-new');
  });
});

function mockJsonResponse(body, { status = 200, headers = {} } = {}) {
  const h = new Headers({ 'content-type': 'application/json', ...headers });
  return new Response(JSON.stringify(body), { status, headers: h });
}

describe('callLLM happy path', () => {
  it('returns parsed tool input when toolChoice forces a tool', async () => {
    globalThis.fetch = vi.fn(async () =>
      mockJsonResponse({
        id: 'msg_1',
        model: 'claude-sonnet-4-6',
        stop_reason: 'tool_use',
        usage: { input_tokens: 10, output_tokens: 20 },
        content: [
          {
            type: 'tool_use',
            name: 'emit_custom_topic',
            input: { label: 'Pair Programming', type: 'process', description: 'Two engineers, one keyboard.' },
          },
        ],
      }),
    );

    const result = await callLLM({
      task: 'learning.custom_topic',
      tier: 'standard',
      user: 'Pair programming',
      tools: [{ name: 'emit_custom_topic', description: 'x', input_schema: { type: 'object' } }],
      toolChoice: { type: 'tool', name: 'emit_custom_topic' },
    });

    expect(result.toolUses).toHaveLength(1);
    expect(result.toolUses[0].input.label).toBe('Pair Programming');
  });

  it('parses and validates plain text against a Zod schema', async () => {
    globalThis.fetch = vi.fn(async () =>
      mockJsonResponse({
        id: 'msg_2',
        model: 'claude-sonnet-4-6',
        stop_reason: 'end_turn',
        usage: { input_tokens: 5, output_tokens: 7 },
        content: [{ type: 'text', text: '```json\n{"value":42}\n```' }],
      }),
    );

    const schema = z.object({ value: z.number() });
    const result = await callLLM({
      task: 'demo.json',
      user: 'give me a number',
      schema,
    });
    expect(result.parsed).toEqual({ value: 42 });
  });
});

describe('callLLM error paths', () => {
  it('throws LLMTruncatedError when stop_reason is max_tokens and a tool was requested', async () => {
    globalThis.fetch = vi.fn(async () =>
      mockJsonResponse({
        stop_reason: 'max_tokens',
        usage: { input_tokens: 1, output_tokens: 1 },
        content: [],
      }),
    );
    await expect(
      callLLM({
        task: 'extract.source',
        user: 'x',
        tools: [{ name: 'emit_knowledge_graph', description: 'x', input_schema: { type: 'object' } }],
        toolChoice: { type: 'tool', name: 'emit_knowledge_graph' },
      }),
    ).rejects.toBeInstanceOf(LLMTruncatedError);
  });

  it('throws LLMTruncatedError when stop_reason is max_tokens and a schema was requested', async () => {
    globalThis.fetch = vi.fn(async () =>
      mockJsonResponse({
        stop_reason: 'max_tokens',
        usage: { input_tokens: 1, output_tokens: 1 },
        content: [{ type: 'text', text: 'partial...' }],
      }),
    );
    const schema = z.object({ value: z.number() });
    await expect(
      callLLM({ task: 'demo.json', user: 'x', schema }),
    ).rejects.toBeInstanceOf(LLMTruncatedError);
  });

  it('throws LLMValidationError when tool input fails schema validation', async () => {
    globalThis.fetch = vi.fn(async () =>
      mockJsonResponse({
        stop_reason: 'tool_use',
        usage: {},
        content: [
          { type: 'tool_use', name: 'emit_custom_topic', input: { label: 'X', type: 'concept' } },
        ],
      }),
    );
    await expect(
      callLLM({
        task: 'learning.custom_topic',
        user: 'x',
        tools: [{ name: 'emit_custom_topic', description: 'x', input_schema: { type: 'object' } }],
        toolChoice: { type: 'tool', name: 'emit_custom_topic' },
      }),
    ).rejects.toBeInstanceOf(LLMValidationError);
  });

  it('surfaces a non-retryable HTTP error as LLMHttpError', async () => {
    globalThis.fetch = vi.fn(async () =>
      new Response(JSON.stringify({ error: 'bad request' }), {
        status: 400,
        headers: { 'content-type': 'application/json' },
      }),
    );
    await expect(callLLM({ task: 'demo', user: 'x' })).rejects.toBeInstanceOf(LLMHttpError);
  });

  it('detects an auth portal HTML response and raises a clear message', async () => {
    globalThis.fetch = vi.fn(async () =>
      new Response('<html>login</html>', { status: 200, headers: { 'content-type': 'text/html' } }),
    );
    await expect(callLLM({ task: 'demo', user: 'x' })).rejects.toThrow(/session has expired/i);
  });
});

describe('callLLM simulation mode', () => {
  it('returns the chat stub when admin:use_simulation is true and task is chat-like', async () => {
    storage.set('admin:use_simulation', true);
    const result = await callLLM({ task: 'chat.r42', user: 'hello' });
    expect(result.stopReason).toBe('end_turn');
    expect(result.text).toMatch(/Simulatiemodus/);
  });

  it('returns the extraction stub for other tasks in simulation mode', async () => {
    storage.set('admin:use_simulation', true);
    const result = await callLLM({ task: 'extract.source', user: 'doc' });
    expect(() => JSON.parse(result.text)).not.toThrow();
    expect(JSON.parse(result.text)).toHaveProperty('topics');
  });
});