From 9ea5d5444df4c0727f26d7d7b1b21126297eba41 Mon Sep 17 00:00:00 2001 From: RaymondVerhoef Date: Tue, 26 May 2026 13:22:44 +0200 Subject: [PATCH] fix: generate weekly quiz in a single batch LLM call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The weekly quiz path made up to six sequential LLM calls (one per topic) with three retries each, blowing past the 30s budget. Worse, the quiz_banks collection has been dropped, so getQuizBank/setQuizBank are no-ops and every generated batch was thrown away — the assembled quiz ended up empty and surfaced as "Could not assemble enough questions." Replace the per-topic loop with a single batched call on the fast tier that emits all five questions in one round-trip, using the review topics as prompt context instead of separate calls. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/lib/testService.js | 115 +++++++++++++++++++++++++++++------------ 1 file changed, 83 insertions(+), 32 deletions(-) diff --git a/src/lib/testService.js b/src/lib/testService.js index 38fbadf..dd39eed 100644 --- a/src/lib/testService.js +++ b/src/lib/testService.js @@ -4,6 +4,8 @@ import { EMIT_QUIZ_QUESTIONS_TOOL } from './llmTools'; import { getCurrentWeekContent } from './curriculumService'; import { shuffle, sample } from './random'; +const QUIZ_BATCH_TIMEOUT_MS = 25_000; + const QUIZ_SYSTEM = `You are a quiz generator for Respellion, an internal IT company learning platform. You generate multiple-choice questions to test employee knowledge on specific topics. Always write in clear, professional English. @@ -218,19 +220,61 @@ export async function forceGenerateTopicQuestions(topic, count = 5) { return accepted; } -async function getOrGenerateTopicQuestions(topic, count) { - let bank = await db.getQuizBank(topic.id); +// Single-call batch generator used by the weekly quiz path. Generates the +// entire 5-question test in one LLM round-trip instead of looping topic by +// topic — the persisted quiz_banks collection is deprecated so caching can't +// amortise multiple calls, and 6 sequential calls blew through the <30s budget. +async function callQuizBatchModel(primaryTopic, reviewTopics, count) { + const reviewBlock = reviewTopics.length + ? `\n\nReview topics — draw ${Math.max(0, count - 3)} questions from these to broaden the test:\n${reviewTopics + .map((t, i) => `${i + 1}. ${t.label} (${t.type}) — ${t.description}`) + .join('\n')}` + : ''; - if (bank.length < count) { - try { - await forceGenerateTopicQuestions(topic, 5); - bank = await db.getQuizBank(topic.id); - } catch (err) { - console.warn(`[quiz] Failed to generate questions for ${topic.label}:`, err.message); - } - } + const prompt = `Generate exactly ${count} multiple-choice quiz questions for this week's test and emit them via the emit_quiz_questions tool. - return sample(bank, Math.min(count, bank.length)); +Primary topic — focus 3 questions here: +- Label: ${primaryTopic.label} +- Type: ${primaryTopic.type} +- Description: ${primaryTopic.description}${reviewBlock} + +Rules: +- Set "topicLabel" to the label of the topic the question covers. +- Prefix the four options with "A) ", "B) ", "C) ", "D) ". +- Make questions specific and practical, not trivial recall. +- Distribute correctIndex roughly evenly across 0, 1, 2, 3 within the batch.`; + + const result = await callLLM({ + task: 'quiz.batch_generate', + tier: 'fast', + system: cachedSystem(QUIZ_SYSTEM), + user: prompt, + tools: [EMIT_QUIZ_QUESTIONS_TOOL], + toolChoice: { type: 'tool', name: EMIT_QUIZ_QUESTIONS_TOOL.name }, + maxTokens: 4096, + timeoutMs: QUIZ_BATCH_TIMEOUT_MS, + }); + + const emitted = result.toolUses[0]?.input; + if (!emitted) throw new Error('LLM did not emit tool output.'); + + let questions; + if (Array.isArray(emitted.questions)) questions = emitted.questions; + else if (Array.isArray(emitted)) questions = emitted; + else if (emitted.questions && typeof emitted.questions === 'object') questions = [emitted.questions]; + else throw new Error('Unexpected quiz output shape.'); + + if (!questions.length) throw new Error('LLM emitted zero questions.'); + + return questions.map((q, i) => ({ + id: `${primaryTopic.id}-${Date.now().toString(36)}-${i}`, + question: q.question || '', + topicLabel: q.topicLabel || primaryTopic.label, + options: Array.isArray(q.options) ? q.options.slice(0, 4) : [], + correctIndex: typeof q.correctIndex === 'number' ? q.correctIndex : 0, + explanation: q.explanation || 'No explanation provided.', + difficulty: q.difficulty || 'medium', + })).filter((q) => q.question && q.options.length === 4); } export async function getTopicQuestionBank(topicId) { @@ -250,34 +294,41 @@ export async function generateWeeklyQuiz(userId, weekNumber, force = false) { const { primaryTopic, reviewTopics } = await selectTestTopics(userId, weekNumber); if (!primaryTopic) throw new Error('No topics available to generate a quiz.'); - const questions = []; + // Cap review-topic context to keep the prompt focused and the call fast. + const reviewContext = (reviewTopics || []).slice(0, 3); - const primaryQs = await getOrGenerateTopicQuestions(primaryTopic, 3); - questions.push(...primaryQs); + let accepted = []; + let lastError = null; - for (const rt of reviewTopics) { - if (questions.length >= 5) break; - const rtQs = await getOrGenerateTopicQuestions(rt, 1); - questions.push(...rtQs); - } - - if (questions.length < 5) { - const needed = 5 - questions.length; - const extraQs = await getOrGenerateTopicQuestions(primaryTopic, needed + 5); - const existingIds = new Set(questions.map(q => q.id)); - for (const eq of extraQs) { - if (!existingIds.has(eq.id) && questions.length < 5) { - questions.push(eq); - } + for (let attempt = 0; attempt < 2; attempt++) { + let candidates; + try { + candidates = await callQuizBatchModel(primaryTopic, reviewContext, 5); + } catch (err) { + lastError = err.message; + console.warn(`[quiz] batch attempt ${attempt + 1} failed:`, err.message); + continue; } + + const qualityError = validateBatchQuality(candidates); + if (qualityError) { + lastError = qualityError; + console.warn(`[quiz] batch quality rejected (attempt ${attempt + 1}): ${qualityError}`); + continue; + } + + accepted = candidates; + break; } - const shuffled = shuffle(questions).slice(0, 5); - - if (shuffled.length === 0) { - throw new Error('Could not assemble enough questions for this week\'s test. Please try again.'); + if (accepted.length < 5) { + throw new Error( + `Could not generate a 5-question test for ${primaryTopic.label}${lastError ? `: ${lastError}` : ''}. Please try again.`, + ); } + const shuffled = shuffle(accepted).slice(0, 5); + const quiz = { questions: shuffled, meta: {