feat: phase 4 of AI pipeline hardening — quiz & content quality

- src/lib/random.js: Fisher–Yates shuffle/sample/pickInt; replace every biased .sort(() => 0.5 - Math.random()) site in testService. - testService: debias correctIndex via prompt + runtime re-roll (up to 2x when one position holds >50%); quality gate rejecting <4 distinct options, banned filler ("all of the above" etc) and explanations shorter than 20 chars; dedup new questions against the existing bank via normalised question text. - Quiz schema/tool/prompt require difficulty ('easy'|'medium'|'hard'); db.getQuizBank defaults legacy records to 'medium' on read. - learningService.generateCustomTopic: kebab-case slug ID from the polished label with collision suffixes; default learning_relevance 'standard' when the model omits it. - Tests for random helpers, dedup/quality-gate behaviour and the extended quiz schema. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-20 19:22:10 +02:00
parent c82e4fc3a1
commit 66e0c275da
9 changed files with 407 additions and 74 deletions
--- a/src/lib/testService.js
+++ b/src/lib/testService.js
@@ -2,81 +2,73 @@ import * as db from './db';
 import { callLLM } from './llm';
 import { EMIT_QUIZ_QUESTIONS_TOOL } from './llmTools';
 import { getCurriculumTopic, getQuarterForWeek } from './curriculumService';
+import { shuffle, sample } from './random';

 const QUIZ_SYSTEM = `You are a quiz generator for Respellion, an internal IT company learning platform.
 You generate multiple-choice questions to test employee knowledge on specific topics.
 Always write in clear, professional English.

-Emit questions through the emit_quiz_questions tool. Each question has exactly four options; correctIndex is 0-based. For a typical 5-question batch, mix difficulty roughly 2 easy / 2 medium / 1 hard; scale the ratio proportionally for larger batches.`;
+Emit questions through the emit_quiz_questions tool. Each question has exactly four options; correctIndex is 0-based. Tag every question with difficulty ('easy', 'medium' or 'hard'). For a typical 5-question batch, mix difficulty roughly 2 easy / 2 medium / 1 hard; scale the ratio proportionally for larger batches.
+
+Distribute correctIndex roughly evenly across 0, 1, 2, and 3. Do not place the correct answer at the same position more than 4 out of 10 times.
+
+Never use filler options such as "all of the above", "none of the above", or "both A and B". Every explanation must be a substantive sentence (≥ 20 characters) describing why the correct answer is correct.`;
+
+const BANNED_OPTION_PATTERNS = [
+  /all of the above/i,
+  /none of the above/i,
+  /both a and b/i,
+  /both b and c/i,
+  /both c and d/i,
+  /both a and c/i,
+  /both b and d/i,
+  /both a and d/i,
+];

 const cachedSystem = (text) => [{ type: 'text', text, cache_control: { type: 'ephemeral' } }];

-async function selectTestTopics(userId, weekNumber) {
-  const allTopics = await db.getTopics();
-  const topics = allTopics.filter(t => t.type !== 'fact' && t.learning_relevance !== 'exclude');
-  if (!topics || topics.length === 0) return { primaryTopic: null, reviewTopics: [], isReviewWeek: false };
-
-  // Try curriculum-based selection first
-  try {
-    const { topic, curriculumEntry } = await getCurriculumTopic(weekNumber);
-
-    if (curriculumEntry?.is_review_week) {
-      // Review week: pull topics from the whole quarter
-      const quarter = getQuarterForWeek(weekNumber);
-      const curriculum = await db.getCurriculum(new Date().getFullYear());
-      const quarterTopicIds = curriculum
-        .filter(w => w.quarter === quarter && w.topic_id && !w.is_review_week)
-        .map(w => w.topic_id);
-      const quarterTopics = topics.filter(t => quarterTopicIds.includes(t.id));
-      // Use all quarter topics as review topics (no single primary)
-      return {
-        primaryTopic: quarterTopics[0] || topics[0],
-        reviewTopics: quarterTopics.slice(1),
-        isReviewWeek: true,
-      };
-    }
-
-    if (topic) {
-      const others = topics.filter(t => t.id !== topic.id);
-      const shuffled = others.sort(() => 0.5 - Math.random());
-      const reviewTopics = shuffled.slice(0, Math.min(5, shuffled.length));
-      return { primaryTopic: topic, reviewTopics, isReviewWeek: false };
-    }
-  } catch (e) {
-    console.warn('[Test] Curriculum lookup failed, falling back to hash:', e.message);
-  }
-
-  // Fallback: hash-based selection
-  const str = `${userId}:${weekNumber}`;
-  let hash = 0;
-  for (let i = 0; i < str.length; i++) {
-    hash = (hash << 5) - hash + str.charCodeAt(i);
-    hash |= 0;
-  }
-  const primaryIndex = Math.abs(hash) % topics.length;
-  const primaryTopic = topics[primaryIndex];
-
-  const others = topics.filter((_, i) => i !== primaryIndex);
-  const shuffled = others.sort(() => 0.5 - Math.random());
-  const reviewTopics = shuffled.slice(0, Math.min(5, shuffled.length));
-
-  return { primaryTopic, reviewTopics, isReviewWeek: false };
+function normalizeQuestionText(text) {
+  return String(text || '')
+    .toLowerCase()
+    .replace(/[\p{P}\p{S}]/gu, ' ')
+    .replace(/\s+/g, ' ')
+    .trim();
 }

-export async function getCachedQuiz(userId, weekNumber) {
-  return db.getCachedQuiz(userId, weekNumber);
+function dominantCorrectIndex(questions) {
+  if (!questions.length) return null;
+  const counts = [0, 0, 0, 0];
+  for (const q of questions) counts[q.correctIndex] = (counts[q.correctIndex] || 0) + 1;
+  const max = Math.max(...counts);
+  return max / questions.length > 0.5 ? { index: counts.indexOf(max), ratio: max / questions.length } : null;
 }

-export async function forceGenerateTopicQuestions(topic, count = 5) {
-  let bank = await db.getQuizBank(topic.id);
+function validateBatchQuality(questions) {
+  for (const q of questions) {
+    const distinct = new Set(q.options.map((o) => o.trim().toLowerCase()));
+    if (distinct.size < 4) {
+      return `Question "${q.question}" has duplicate options.`;
+    }
+    for (const opt of q.options) {
+      if (BANNED_OPTION_PATTERNS.some((re) => re.test(opt))) {
+        return `Question "${q.question}" uses a banned filler option ("${opt}").`;
+      }
+    }
+    if (!q.explanation || q.explanation.trim().length < 20) {
+      return `Question "${q.question}" has an explanation that is too short.`;
+    }
+  }
+  return null;
+}

+async function callQuizModel(topic, count) {
  const prompt = `Generate exactly ${count} multiple-choice quiz questions for this knowledge topic and emit them via the emit_quiz_questions tool:

 Topic: ${topic.label}
 Type: ${topic.type}
 Description: ${topic.description}

-Options must be prefixed "A) ", "B) ", "C) ", "D) ". Make questions specific and practical, not trivial.`;
+Options must be prefixed "A) ", "B) ", "C) ", "D) ". Make questions specific and practical, not trivial. Example: a question whose correct answer is option C uses "correctIndex": 2.`;

  const result = await callLLM({
    task: 'quiz.generate',
@@ -89,30 +81,125 @@ Options must be prefixed "A) ", "B) ", "C) ", "D) ". Make questions specific and
  });

  const emitted = result.toolUses[0]?.input;
-  if (!emitted) throw new Error(`Could not generate questions for ${topic.label}`);
+  if (!emitted?.questions?.length) {
+    throw new Error(`Could not generate questions for ${topic.label}`);
+  }
+  return emitted.questions;
+}

-  const newQuestions = (emitted.questions || []).map(q => ({
-    ...q,
-    id: `${topic.id}-${Math.random().toString(36).slice(2, 11)}`,
-  }));
+async function selectTestTopics(userId, weekNumber) {
+  const allTopics = await db.getTopics();
+  const topics = allTopics.filter(t => t.type !== 'fact' && t.learning_relevance !== 'exclude');
+  if (!topics || topics.length === 0) return { primaryTopic: null, reviewTopics: [], isReviewWeek: false };

-  bank = [...bank, ...newQuestions];
-  await db.setQuizBank(topic.id, bank);
-  return newQuestions;
+  try {
+    const { topic, curriculumEntry } = await getCurriculumTopic(weekNumber);
+
+    if (curriculumEntry?.is_review_week) {
+      const quarter = getQuarterForWeek(weekNumber);
+      const curriculum = await db.getCurriculum(new Date().getFullYear());
+      const quarterTopicIds = curriculum
+        .filter(w => w.quarter === quarter && w.topic_id && !w.is_review_week)
+        .map(w => w.topic_id);
+      const quarterTopics = topics.filter(t => quarterTopicIds.includes(t.id));
+      return {
+        primaryTopic: quarterTopics[0] || topics[0],
+        reviewTopics: quarterTopics.slice(1),
+        isReviewWeek: true,
+      };
+    }
+
+    if (topic) {
+      const others = topics.filter(t => t.id !== topic.id);
+      const reviewTopics = sample(others, Math.min(5, others.length));
+      return { primaryTopic: topic, reviewTopics, isReviewWeek: false };
+    }
+  } catch (e) {
+    console.warn('[Test] Curriculum lookup failed, falling back to hash:', e.message);
+  }
+
+  const str = `${userId}:${weekNumber}`;
+  let hash = 0;
+  for (let i = 0; i < str.length; i++) {
+    hash = (hash << 5) - hash + str.charCodeAt(i);
+    hash |= 0;
+  }
+  const primaryIndex = Math.abs(hash) % topics.length;
+  const primaryTopic = topics[primaryIndex];
+
+  const others = topics.filter((_, i) => i !== primaryIndex);
+  const reviewTopics = sample(others, Math.min(5, others.length));
+
+  return { primaryTopic, reviewTopics, isReviewWeek: false };
+}
+
+export async function getCachedQuiz(userId, weekNumber) {
+  return db.getCachedQuiz(userId, weekNumber);
+}
+
+export async function forceGenerateTopicQuestions(topic, count = 5) {
+  const existingBank = await db.getQuizBank(topic.id);
+  const existingKeys = new Set(existingBank.map((q) => normalizeQuestionText(q.question)));
+
+  let lastQualityError = null;
+  let candidates = null;
+
+  for (let attempt = 0; attempt < 3; attempt++) {
+    const questions = await callQuizModel(topic, count);
+
+    const qualityError = validateBatchQuality(questions);
+    if (qualityError) {
+      lastQualityError = qualityError;
+      console.warn(`[quiz] batch rejected (attempt ${attempt + 1}): ${qualityError}`);
+      continue;
+    }
+
+    const dominant = dominantCorrectIndex(questions);
+    if (dominant && attempt < 2) {
+      console.warn(`[quiz] correctIndex dominated by ${dominant.index} (${Math.round(dominant.ratio * 100)}%) — re-rolling`);
+      continue;
+    }
+
+    candidates = questions;
+    break;
+  }
+
+  if (!candidates) {
+    throw new Error(`Quality gate rejected the generated batch for ${topic.label}: ${lastQualityError || 'unbalanced answer distribution'}. Click "Generate" to try again.`);
+  }
+
+  const accepted = [];
+  for (const q of candidates) {
+    const key = normalizeQuestionText(q.question);
+    if (existingKeys.has(key)) {
+      console.debug('[quiz] dropped duplicate:', q.question);
+      continue;
+    }
+    existingKeys.add(key);
+    accepted.push({
+      ...q,
+      id: `${topic.id}-${Math.random().toString(36).slice(2, 11)}`,
+    });
+  }
+
+  if (!accepted.length) {
+    throw new Error(`All generated questions for ${topic.label} were duplicates of existing ones.`);
+  }
+
+  const merged = [...existingBank, ...accepted];
+  await db.setQuizBank(topic.id, merged);
+  return accepted;
 }

 async function getOrGenerateTopicQuestions(topic, count) {
  let bank = await db.getQuizBank(topic.id);

  if (bank.length < count) {
-    // Seed an empty/low bank with a small initial batch; admins can grow it
-    // explicitly via TestManager when they want more depth per topic.
    await forceGenerateTopicQuestions(topic, 5);
    bank = await db.getQuizBank(topic.id);
  }

-  const shuffled = [...bank].sort(() => 0.5 - Math.random());
-  return shuffled.slice(0, Math.min(count, shuffled.length));
+  return sample(bank, Math.min(count, bank.length));
 }

 export async function getTopicQuestionBank(topicId) {
@@ -153,10 +240,10 @@ export async function generateWeeklyQuiz(userId, weekNumber, force = false) {
    }
  }

-  questions.sort(() => 0.5 - Math.random());
+  const shuffled = shuffle(questions);

  const quiz = {
-    questions,
+    questions: shuffled,
    meta: {
      userId,
      weekNumber,