From 7164317908aa6a867267522fcd50e2c71049ef55 Mon Sep 17 00:00:00 2001 From: RaymondVerhoef Date: Mon, 25 May 2026 22:08:18 +0200 Subject: [PATCH] feat: implement quiz generation service with topic selection, LLM integration, and question bank management --- src/lib/__tests__/testService.test.js | 33 +-- src/lib/llmSchemas.js | 2 +- src/lib/testService.js | 281 +++++++++++++------------- src/pages/Dashboard.jsx | 2 +- src/pages/Testen.jsx | 8 +- 5 files changed, 163 insertions(+), 163 deletions(-) diff --git a/src/lib/__tests__/testService.test.js b/src/lib/__tests__/testService.test.js index 3cbdd66..fa791d4 100644 --- a/src/lib/__tests__/testService.test.js +++ b/src/lib/__tests__/testService.test.js @@ -24,8 +24,7 @@ vi.mock('../llm', () => ({ cachedSystem: (text) => [{ type: 'text', text }], })); vi.mock('../curriculumService', () => ({ - getCurriculumTopic: vi.fn(async () => ({ topic: null })), - getQuarterForWeek: vi.fn(() => 1), + getCurrentWeekContent: vi.fn(async () => null), })); import { forceGenerateTopicQuestions } from '../testService'; @@ -67,17 +66,6 @@ describe('forceGenerateTopicQuestions', () => { expect(bankStore.get('onboarding')).toHaveLength(5); }); - it('re-rolls when one correctIndex dominates the batch, then accepts on the third try', async () => { - const allZero = [0, 1, 2, 3, 4].map((i) => makeQuestion(i, { correctIndex: 0 })); - llmEmits(allZero); - llmEmits(allZero); - llmEmits(allZero); - const out = await forceGenerateTopicQuestions(topic, 5); - expect(callLLMMock).toHaveBeenCalledTimes(3); - expect(out).toHaveLength(5); - expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('correctIndex dominated')); - }); - it('rejects a batch containing a banned "all of the above" option', async () => { const bad = [0, 1, 2, 3, 4].map((i) => makeQuestion(i, { options: ['A) x', 'B) y', 'C) z', 'D) All of the above'] }), @@ -85,7 +73,7 @@ describe('forceGenerateTopicQuestions', () => { llmEmits(bad); llmEmits(bad); llmEmits(bad); - await expect(forceGenerateTopicQuestions(topic, 5)).rejects.toThrow(/banned filler|rejected/i); + await expect(forceGenerateTopicQuestions(topic, 5)).rejects.toThrow(/banned filler|failed/i); }); it('rejects a batch where an explanation is too short', async () => { @@ -93,7 +81,7 @@ describe('forceGenerateTopicQuestions', () => { llmEmits(bad); llmEmits(bad); llmEmits(bad); - await expect(forceGenerateTopicQuestions(topic, 5)).rejects.toThrow(/too short|rejected/i); + await expect(forceGenerateTopicQuestions(topic, 5)).rejects.toThrow(/too short|failed/i); }); it('drops duplicates whose normalized text matches an existing bank entry', async () => { @@ -112,4 +100,19 @@ describe('forceGenerateTopicQuestions', () => { expect(out.find((q) => q.question.toLowerCase().includes('buddy'))).toBeUndefined(); expect(debugSpy).toHaveBeenCalledWith(expect.stringContaining('dropped duplicate'), expect.any(String)); }); + + it('retries on LLM failure and succeeds on a later attempt', async () => { + callLLMMock.mockRejectedValueOnce(new Error('API timeout')); + llmEmits([0, 1, 2, 3, 4].map((i) => makeQuestion(i))); + const out = await forceGenerateTopicQuestions(topic, 5); + expect(callLLMMock).toHaveBeenCalledTimes(2); + expect(out).toHaveLength(5); + expect(warnSpy).toHaveBeenCalledWith(expect.stringContaining('LLM call failed'), expect.any(String)); + }); + + it('throws after 3 consecutive LLM failures', async () => { + callLLMMock.mockRejectedValue(new Error('schema validation')); + await expect(forceGenerateTopicQuestions(topic, 5)).rejects.toThrow(/failed.*schema validation/i); + expect(callLLMMock).toHaveBeenCalledTimes(3); + }); }); diff --git a/src/lib/llmSchemas.js b/src/lib/llmSchemas.js index 63ae60f..76ff716 100644 --- a/src/lib/llmSchemas.js +++ b/src/lib/llmSchemas.js @@ -115,7 +115,7 @@ export const learningAllSchema = z.object({ const quizDifficultyEnum = z.enum(['easy', 'medium', 'hard']).catch('medium'); const quizQuestionSchema = z.object({ - id: z.string().min(1).catch(`gen-${Math.random().toString(36).slice(2, 8)}`), + id: z.string().min(1).catch(() => `gen-${Math.random().toString(36).slice(2, 8)}`), question: z.string().min(1), topicLabel: z.string().min(1).catch('General'), options: z.array(z.string().min(1)).length(4), diff --git a/src/lib/testService.js b/src/lib/testService.js index 38fbadf..096a4ad 100644 --- a/src/lib/testService.js +++ b/src/lib/testService.js @@ -4,25 +4,24 @@ import { EMIT_QUIZ_QUESTIONS_TOOL } from './llmTools'; import { getCurrentWeekContent } from './curriculumService'; import { shuffle, sample } from './random'; +// ── System prompt ──────────────────────────────────────────── + const QUIZ_SYSTEM = `You are a quiz generator for Respellion, an internal IT company learning platform. You generate multiple-choice questions to test employee knowledge on specific topics. Always write in clear, professional English. Emit questions through the emit_quiz_questions tool. Each question has exactly four options; correctIndex is 0-based. Tag every question with difficulty ('easy', 'medium' or 'hard'). For a typical 5-question batch, mix difficulty roughly 2 easy / 2 medium / 1 hard; scale the ratio proportionally for larger batches. -Distribute correctIndex roughly evenly across 0, 1, 2, and 3. Do not place the correct answer at the same position more than 4 out of 10 times. +Distribute correctIndex roughly evenly across 0, 1, 2, and 3. Never use filler options such as "all of the above", "none of the above", or "both A and B". Every explanation must be a substantive sentence (≥ 20 characters) describing why the correct answer is correct.`; +// ── Quality helpers ────────────────────────────────────────── + const BANNED_OPTION_PATTERNS = [ /all of the above/i, /none of the above/i, - /both a and b/i, - /both b and c/i, - /both c and d/i, - /both a and c/i, - /both b and d/i, - /both a and d/i, + /both [a-d] and [a-d]/i, ]; function normalizeQuestionText(text) { @@ -33,16 +32,15 @@ function normalizeQuestionText(text) { .trim(); } -function dominantCorrectIndex(questions) { - if (!questions.length) return null; - const counts = [0, 0, 0, 0]; - for (const q of questions) counts[q.correctIndex] = (counts[q.correctIndex] || 0) + 1; - const max = Math.max(...counts); - return max / questions.length > 0.8 ? { index: counts.indexOf(max), ratio: max / questions.length } : null; -} - +/** + * Validate a batch of questions for quality. Returns an error string + * if the batch should be rejected, or null if acceptable. + */ function validateBatchQuality(questions) { for (const q of questions) { + if (!q.question || !Array.isArray(q.options) || q.options.length !== 4) { + return `Question is malformed (missing text or wrong option count).`; + } const distinct = new Set(q.options.map((o) => o.trim().toLowerCase())); if (distinct.size < 4) { return `Question "${q.question}" has duplicate options.`; @@ -59,6 +57,13 @@ function validateBatchQuality(questions) { return null; } +// ── LLM call ───────────────────────────────────────────────── + +/** + * Call the LLM to generate quiz questions for a topic. + * The Zod schema in llmSchemas.js handles coercion and defaults, + * so we just extract the validated `questions` array here. + */ async function callQuizModel(topic, count) { const prompt = `Generate exactly ${count} multiple-choice quiz questions for this knowledge topic and emit them via the emit_quiz_questions tool: @@ -66,7 +71,7 @@ Topic: ${topic.label} Type: ${topic.type} Description: ${topic.description} -Options must be prefixed "A) ", "B) ", "C) ", "D) ". Make questions specific and practical, not trivial. Ensure correct answers are evenly distributed.`; +Each option must start with "A) ", "B) ", "C) ", "D) ". Make questions specific and practical, not trivial.`; const result = await callLLM({ task: 'quiz.generate', @@ -78,66 +83,38 @@ Options must be prefixed "A) ", "B) ", "C) ", "D) ". Make questions specific and maxTokens: 4096, }); - const emitted = result.toolUses[0]?.input; - if (!emitted) { - throw new Error(`LLM did not emit tool output for ${topic.label}`); + // After Zod validation in callLLM, emitted.questions is a valid array + const questions = result.toolUses[0]?.input?.questions; + if (!questions?.length) { + throw new Error(`LLM did not produce questions for "${topic.label}".`); } - - // Handle different shapes the LLM might return: - // 1. { questions: [...] } — expected - // 2. [...] — model returned array directly - // 3. { questions: { ... } } — model wrapped single question in object - let questions; - if (Array.isArray(emitted.questions)) { - questions = emitted.questions; - } else if (Array.isArray(emitted)) { - questions = emitted; - } else if (emitted.questions && typeof emitted.questions === 'object') { - // Single question wrapped as object instead of array - questions = [emitted.questions]; - } else { - throw new Error(`Unexpected quiz output shape for ${topic.label}`); - } - - if (!questions.length) { - throw new Error(`No questions generated for ${topic.label}`); - } - - // Ensure each question has required fields (fill in defaults for missing ones) - return questions.map((q, i) => ({ - id: q.id || `gen-${i}-${Math.random().toString(36).slice(2, 8)}`, - question: q.question || '', - topicLabel: q.topicLabel || topic.label, - options: Array.isArray(q.options) ? q.options.slice(0, 4) : [], - correctIndex: typeof q.correctIndex === 'number' ? q.correctIndex : 0, - explanation: q.explanation || 'No explanation provided.', - difficulty: q.difficulty || 'medium', - })).filter(q => q.question && q.options.length === 4); + return questions; } +// ── Topic selection ────────────────────────────────────────── + async function selectTestTopics(userId, isoWeekNumber) { const allTopics = await db.getTopics(); const topics = allTopics.filter(t => t.type !== 'fact' && t.learning_relevance !== 'exclude'); - if (!topics || topics.length === 0) return { primaryTopic: null, reviewTopics: [], isReviewWeek: false }; + if (!topics.length) return { primaryTopic: null, reviewTopics: [] }; + // Try curriculum-based selection first try { const weekContent = await getCurrentWeekContent(isoWeekNumber); - - if (weekContent && weekContent.topics && weekContent.topics.length > 0) { + if (weekContent?.topics?.length > 0) { const primaryTopic = weekContent.topics[0]; - if (!primaryTopic?.id) { - console.warn('[Test] First curriculum topic has no id — falling back to hash'); - } else { + if (primaryTopic?.id) { const others = topics.filter(t => t.id !== primaryTopic.id); const reviewTopics = sample(others, Math.min(5, others.length)); - return { primaryTopic, reviewTopics, isReviewWeek: false }; + return { primaryTopic, reviewTopics }; } + console.warn('[Test] First curriculum topic has no id — falling back to hash'); } } catch (e) { console.warn('[Test] Curriculum lookup failed, falling back to hash:', e.message); } - // Fallback hash-based assignment + // Fallback: deterministic hash-based assignment const str = `${userId}:${isoWeekNumber}`; let hash = 0; for (let i = 0; i < str.length; i++) { @@ -146,93 +123,18 @@ async function selectTestTopics(userId, isoWeekNumber) { } const primaryIndex = Math.abs(hash) % topics.length; const primaryTopic = topics[primaryIndex]; - const others = topics.filter((_, i) => i !== primaryIndex); const reviewTopics = sample(others, Math.min(5, others.length)); - return { primaryTopic, reviewTopics, isReviewWeek: false }; + return { primaryTopic, reviewTopics }; } +// ── Question bank management ──────────────────────────────── + export async function getCachedQuiz(userId, weekNumber) { return db.getCachedQuiz(userId, weekNumber); } -export async function forceGenerateTopicQuestions(topic, count = 5) { - const existingBank = await db.getQuizBank(topic.id); - const existingKeys = new Set(existingBank.map((q) => normalizeQuestionText(q.question))); - - let lastQualityError = null; - let candidates = null; - - for (let attempt = 0; attempt < 3; attempt++) { - let questions; - try { - questions = await callQuizModel(topic, count); - } catch (err) { - console.warn(`[quiz] LLM failed (attempt ${attempt + 1}):`, err.message); - lastQualityError = err.message; - continue; - } - - const qualityError = validateBatchQuality(questions); - if (qualityError) { - lastQualityError = qualityError; - console.warn(`[quiz] batch rejected (attempt ${attempt + 1}): ${qualityError}`); - continue; - } - - const dominant = dominantCorrectIndex(questions); - if (dominant && attempt < 2) { - console.warn(`[quiz] correctIndex dominated by ${dominant.index} (${Math.round(dominant.ratio * 100)}%) — re-rolling`); - continue; - } - - candidates = questions; - break; - } - - if (!candidates) { - throw new Error(`Quality gate rejected the generated batch for ${topic.label}: ${lastQualityError || 'unbalanced answer distribution'}. Click "Generate" to try again.`); - } - - const accepted = []; - for (const q of candidates) { - const key = normalizeQuestionText(q.question); - if (existingKeys.has(key)) { - console.debug('[quiz] dropped duplicate:', q.question); - continue; - } - existingKeys.add(key); - accepted.push({ - ...q, - id: `${topic.id}-${Math.random().toString(36).slice(2, 11)}`, - }); - } - - if (!accepted.length) { - throw new Error(`All generated questions for ${topic.label} were duplicates of existing ones.`); - } - - const merged = [...existingBank, ...accepted]; - await db.setQuizBank(topic.id, merged); - return accepted; -} - -async function getOrGenerateTopicQuestions(topic, count) { - let bank = await db.getQuizBank(topic.id); - - if (bank.length < count) { - try { - await forceGenerateTopicQuestions(topic, 5); - bank = await db.getQuizBank(topic.id); - } catch (err) { - console.warn(`[quiz] Failed to generate questions for ${topic.label}:`, err.message); - } - } - - return sample(bank, Math.min(count, bank.length)); -} - export async function getTopicQuestionBank(topicId) { return db.getQuizBank(topicId); } @@ -241,7 +143,84 @@ export async function deleteQuestion(topicId, questionId) { return db.deleteQuestionFromBank(topicId, questionId); } +/** + * Generate new questions for a topic and merge them into the bank. + * Retries up to 3 times on LLM or quality failures. + */ +export async function forceGenerateTopicQuestions(topic, count = 5) { + const existingBank = await db.getQuizBank(topic.id); + const existingKeys = new Set(existingBank.map((q) => normalizeQuestionText(q.question))); + + let lastError = null; + + for (let attempt = 0; attempt < 3; attempt++) { + let questions; + try { + questions = await callQuizModel(topic, count); + } catch (err) { + console.warn(`[quiz] LLM call failed (attempt ${attempt + 1}):`, err.message); + lastError = err.message; + continue; + } + + const qualityError = validateBatchQuality(questions); + if (qualityError) { + console.warn(`[quiz] Batch rejected (attempt ${attempt + 1}): ${qualityError}`); + lastError = qualityError; + continue; + } + + // Batch passed quality — deduplicate against existing bank + const accepted = []; + for (const q of questions) { + const key = normalizeQuestionText(q.question); + if (existingKeys.has(key)) { + console.debug('[quiz] dropped duplicate:', q.question); + continue; + } + existingKeys.add(key); + accepted.push({ + ...q, + id: `${topic.id}-${Math.random().toString(36).slice(2, 11)}`, + }); + } + + if (!accepted.length) { + lastError = `All generated questions for "${topic.label}" were duplicates.`; + continue; + } + + const merged = [...existingBank, ...accepted]; + await db.setQuizBank(topic.id, merged); + return accepted; + } + + throw new Error(`Quiz generation failed for "${topic.label}": ${lastError || 'unknown error'}.`); +} + +/** + * Get questions from the bank for a topic, generating if needed. + * Never throws — returns whatever is available (possibly []). + */ +async function getOrGenerateTopicQuestions(topic, count) { + let bank = await db.getQuizBank(topic.id); + + if (bank.length < count) { + try { + await forceGenerateTopicQuestions(topic, 5); + bank = await db.getQuizBank(topic.id); + } catch (err) { + console.warn(`[quiz] Could not populate bank for "${topic.label}":`, err.message); + } + } + + return sample(bank, Math.min(count, bank.length)); +} + +// ── Weekly quiz assembly ───────────────────────────────────── + export async function generateWeeklyQuiz(userId, weekNumber, force = false) { + // Return cached quiz if it has valid questions if (!force) { const cached = await db.getCachedQuiz(userId, weekNumber); if (cached?.questions?.length > 0) return cached; @@ -252,32 +231,48 @@ export async function generateWeeklyQuiz(userId, weekNumber, force = false) { const questions = []; + // 1. Pull up to 3 from primary topic const primaryQs = await getOrGenerateTopicQuestions(primaryTopic, 3); questions.push(...primaryQs); + // 2. Fill remaining slots from review topics for (const rt of reviewTopics) { if (questions.length >= 5) break; const rtQs = await getOrGenerateTopicQuestions(rt, 1); questions.push(...rtQs); } + // 3. If still short, pull more from primary if (questions.length < 5) { - const needed = 5 - questions.length; - const extraQs = await getOrGenerateTopicQuestions(primaryTopic, needed + 5); const existingIds = new Set(questions.map(q => q.id)); + const extraQs = await getOrGenerateTopicQuestions(primaryTopic, 5); for (const eq of extraQs) { if (!existingIds.has(eq.id) && questions.length < 5) { questions.push(eq); + existingIds.add(eq.id); } } } - const shuffled = shuffle(questions).slice(0, 5); - - if (shuffled.length === 0) { - throw new Error('Could not assemble enough questions for this week\'s test. Please try again.'); + // 4. Last resort: try a fresh generation if we still have 0 questions + if (questions.length === 0) { + try { + const freshQs = await callQuizModel(primaryTopic, 5); + const qualityError = validateBatchQuality(freshQs); + if (!qualityError) { + questions.push(...freshQs.slice(0, 5)); + } + } catch (err) { + console.warn('[quiz] Last-resort generation also failed:', err.message); + } } + if (questions.length === 0) { + throw new Error('Could not generate any questions for this week\'s test. Please try again.'); + } + + const shuffled = shuffle(questions).slice(0, 5); + const quiz = { questions: shuffled, meta: { @@ -292,6 +287,8 @@ export async function generateWeeklyQuiz(userId, weekNumber, force = false) { return quiz; } +// ── Result management ──────────────────────────────────────── + export async function saveTestResult(userId, weekNumber, result) { await db.saveQuizResult(userId, weekNumber, result); diff --git a/src/pages/Dashboard.jsx b/src/pages/Dashboard.jsx index 8c28512..e929e70 100644 --- a/src/pages/Dashboard.jsx +++ b/src/pages/Dashboard.jsx @@ -178,7 +178,7 @@ const Dashboard = () => {

Testing

-

Weekly test — 5 questions

+

Weekly knowledge test

{testResult ? Score: {testResult.percentage}% : To Do}
diff --git a/src/pages/Testen.jsx b/src/pages/Testen.jsx index a4c152b..e425c51 100644 --- a/src/pages/Testen.jsx +++ b/src/pages/Testen.jsx @@ -182,7 +182,7 @@ const Testen = () => {

Weekly Test

Week {weekNumber}

- Test your knowledge with 5 AI-generated questions. You have 5 minutes to complete the test. Good luck! + Test your knowledge with AI-generated questions. You have 5 minutes to complete the test. Good luck!

{error && ( @@ -196,9 +196,9 @@ const Testen = () => {
- 5 questions + Up to 5 questions 5 minutes - 20 pts max + 2 pts each