From 182fb2357f424f090eed31d87c997c7cb673a44b Mon Sep 17 00:00:00 2001 From: RaymondVerhoef Date: Mon, 13 Jul 2026 09:38:46 +0200 Subject: [PATCH] fix: coerce stringified arrays in onboarding overview output (#32) Opening an onboarding theme could fail with "LLM output failed schema validation": the fast-tier model occasionally emits the key_points and topics_covered tool fields as JSON-encoded strings instead of real arrays. Same failure class the quiz schema already guards against with z.preprocess. - onboardingOverviewSchema: parse "[...]" strings back to arrays before validation; split a bullet/newline string of key points as a fallback; keep the first 5 key points on overage instead of failing the user. Genuinely bad output (too few points, non-JSON strings, wrong types) still fails validation. - emit_onboarding_overview tool: descriptions now state the fields must be JSON arrays, never strings (prompt-side nudge). - 5 regression tests reproducing the exact #32 payload shapes (npm test 139/139). Closes #32 Co-Authored-By: Claude Fable 5 --- src/lib/__tests__/llmSchemas.test.js | 54 ++++++++++++++++++++++++++++ src/lib/llmSchemas.js | 37 ++++++++++++++++--- src/lib/llmTools.js | 4 +-- 3 files changed, 89 insertions(+), 6 deletions(-) diff --git a/src/lib/__tests__/llmSchemas.test.js b/src/lib/__tests__/llmSchemas.test.js index deb65c1..4e81a4d 100644 --- a/src/lib/__tests__/llmSchemas.test.js +++ b/src/lib/__tests__/llmSchemas.test.js @@ -9,6 +9,7 @@ import { customTopicSchema, graphActionsSchema, proposeGraphDeltaSchema, + onboardingOverviewSchema, } from '../llmSchemas'; const sampleTopic = { @@ -192,3 +193,56 @@ describe('proposeGraphDeltaSchema', () => { ).toThrow(); }); }); + +describe('onboardingOverviewSchema (issue #32 — fast-tier stringified arrays)', () => { + const base = { + title: 'Privacy', + what_it_is: 'How we handle data.', + why_it_matters: 'You touch personal data weekly.', + }; + const points = ['Point one', 'Point two', 'Point three']; + const topics = [{ topic_id: 'avg', label: 'AVG' }]; + + it('accepts well-formed output', () => { + const r = onboardingOverviewSchema.parse({ ...base, key_points: points, topics_covered: topics }); + expect(r.key_points).toEqual(points); + expect(r.topics_covered).toEqual(topics); + }); + + it('coerces JSON-stringified arrays back to arrays (the exact #32 failure)', () => { + const r = onboardingOverviewSchema.parse({ + ...base, + key_points: JSON.stringify(points), + topics_covered: JSON.stringify(topics), + }); + expect(r.key_points).toEqual(points); + expect(r.topics_covered).toEqual(topics); + }); + + it('splits a bullet/newline string of key points as a fallback', () => { + const r = onboardingOverviewSchema.parse({ + ...base, + key_points: '- Point one\n• Point two\n3. Point three', + topics_covered: topics, + }); + expect(r.key_points).toEqual(points); + }); + + it('keeps the first 5 key points instead of failing on overage', () => { + const seven = Array.from({ length: 7 }, (_, i) => `P${i + 1}`); + const r = onboardingOverviewSchema.parse({ ...base, key_points: seven, topics_covered: topics }); + expect(r.key_points).toEqual(['P1', 'P2', 'P3', 'P4', 'P5']); + }); + + it('still rejects genuinely bad output', () => { + expect(() => + onboardingOverviewSchema.parse({ ...base, key_points: ['only', 'two'], topics_covered: topics }), + ).toThrow(); + expect(() => + onboardingOverviewSchema.parse({ ...base, key_points: points, topics_covered: 'not json at all' }), + ).toThrow(); + expect(() => + onboardingOverviewSchema.parse({ ...base, key_points: 42, topics_covered: topics }), + ).toThrow(); + }); +}); diff --git a/src/lib/llmSchemas.js b/src/lib/llmSchemas.js index 412e326..d9931cd 100644 --- a/src/lib/llmSchemas.js +++ b/src/lib/llmSchemas.js @@ -229,14 +229,43 @@ export const themeSessionSchema = z.object({ keyTakeaways: z.array(z.string().min(1)).min(3), }); +// Fast-tier models occasionally emit array tool fields as a JSON-encoded +// string ("[\"a\",\"b\"]") instead of a real array (issue #32). Parse those +// back to arrays before validating; leave anything else untouched so real +// type errors still fail validation. +function coerceStringifiedArray(v) { + if (typeof v === 'string') { + const s = v.trim(); + if (s.startsWith('[')) { + try { return JSON.parse(s); } catch { /* keep original, let Zod report */ } + } + } + return v; +} + +const onboardingKeyPoints = z.preprocess((v) => { + let out = coerceStringifiedArray(v); + // Fallback: a bullet/newline list as one string → split into points. + if (typeof out === 'string' && out.includes('\n')) { + out = out + .split('\n') + .map((line) => line.replace(/^\s*[-•*\d.]+\s*/, '').trim()) + .filter(Boolean); + } + // Overage is trivial — keep the first 5 rather than failing the user. + if (Array.isArray(out) && out.length > 5) out = out.slice(0, 5); + return out; +}, z.array(z.string().min(1)).min(3).max(5)); + export const onboardingOverviewSchema = z.object({ title: z.string().min(1), what_it_is: z.string().min(1), why_it_matters: z.string().min(1), - key_points: z.array(z.string().min(1)).min(3).max(5), - topics_covered: z - .array(z.object({ topic_id: z.string().min(1), label: z.string().min(1) })) - .min(1), + key_points: onboardingKeyPoints, + topics_covered: z.preprocess( + coerceStringifiedArray, + z.array(z.object({ topic_id: z.string().min(1), label: z.string().min(1) })).min(1), + ), }); /** diff --git a/src/lib/llmTools.js b/src/lib/llmTools.js index e8dcb18..1b45c6f 100644 --- a/src/lib/llmTools.js +++ b/src/lib/llmTools.js @@ -470,11 +470,11 @@ export const EMIT_ONBOARDING_OVERVIEW_TOOL = { items: { type: 'string' }, minItems: 3, maxItems: 5, - description: '3–5 short, concrete takeaways a newcomer should remember about this theme.', + description: '3–5 short, concrete takeaways a newcomer should remember about this theme. Must be a JSON array of strings — never a single string.', }, topics_covered: { type: 'array', - description: 'The topics that make up this theme. Reuse the exact topic_id you were given so the UI can link back.', + description: 'The topics that make up this theme. Reuse the exact topic_id you were given so the UI can link back. Must be a JSON array of objects — never a string.', items: { type: 'object', properties: { -- 2.49.1