309 lines
10 KiB
JavaScript
309 lines
10 KiB
JavaScript
import * as db from './db';
|
|
import { callLLM, cachedSystem } from './llm';
|
|
import { EMIT_QUIZ_QUESTIONS_TOOL } from './llmTools';
|
|
import { getCurrentWeekContent } from './curriculumService';
|
|
import { shuffle, sample } from './random';
|
|
|
|
const QUIZ_SYSTEM = `You are a quiz generator for Respellion, an internal IT company learning platform.
|
|
You generate multiple-choice questions to test employee knowledge on specific topics.
|
|
Always write in clear, professional English.
|
|
|
|
Emit questions through the emit_quiz_questions tool. Each question has exactly four options; correctIndex is 0-based. Tag every question with difficulty ('easy', 'medium' or 'hard'). For a typical 5-question batch, mix difficulty roughly 2 easy / 2 medium / 1 hard; scale the ratio proportionally for larger batches.
|
|
|
|
Distribute correctIndex roughly evenly across 0, 1, 2, and 3. Do not place the correct answer at the same position more than 4 out of 10 times.
|
|
|
|
Never use filler options such as "all of the above", "none of the above", or "both A and B". Every explanation must be a substantive sentence (≥ 20 characters) describing why the correct answer is correct.`;
|
|
|
|
const BANNED_OPTION_PATTERNS = [
|
|
/all of the above/i,
|
|
/none of the above/i,
|
|
/both a and b/i,
|
|
/both b and c/i,
|
|
/both c and d/i,
|
|
/both a and c/i,
|
|
/both b and d/i,
|
|
/both a and d/i,
|
|
];
|
|
|
|
function normalizeQuestionText(text) {
|
|
return String(text || '')
|
|
.toLowerCase()
|
|
.replace(/[\p{P}\p{S}]/gu, ' ')
|
|
.replace(/\s+/g, ' ')
|
|
.trim();
|
|
}
|
|
|
|
function dominantCorrectIndex(questions) {
|
|
if (!questions.length) return null;
|
|
const counts = [0, 0, 0, 0];
|
|
for (const q of questions) counts[q.correctIndex] = (counts[q.correctIndex] || 0) + 1;
|
|
const max = Math.max(...counts);
|
|
return max / questions.length > 0.8 ? { index: counts.indexOf(max), ratio: max / questions.length } : null;
|
|
}
|
|
|
|
function validateBatchQuality(questions) {
|
|
for (const q of questions) {
|
|
const distinct = new Set(q.options.map((o) => o.trim().toLowerCase()));
|
|
if (distinct.size < 4) {
|
|
return `Question "${q.question}" has duplicate options.`;
|
|
}
|
|
for (const opt of q.options) {
|
|
if (BANNED_OPTION_PATTERNS.some((re) => re.test(opt))) {
|
|
return `Question "${q.question}" uses a banned filler option ("${opt}").`;
|
|
}
|
|
}
|
|
if (!q.explanation || q.explanation.trim().length < 20) {
|
|
return `Question "${q.question}" has an explanation that is too short.`;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function callQuizModel(topic, count) {
|
|
const prompt = `Generate exactly ${count} multiple-choice quiz questions for this knowledge topic and emit them via the emit_quiz_questions tool:
|
|
|
|
Topic: ${topic.label}
|
|
Type: ${topic.type}
|
|
Description: ${topic.description}
|
|
|
|
Options must be prefixed "A) ", "B) ", "C) ", "D) ". Make questions specific and practical, not trivial. Ensure correct answers are evenly distributed.`;
|
|
|
|
const result = await callLLM({
|
|
task: 'quiz.generate',
|
|
tier: 'standard',
|
|
system: cachedSystem(QUIZ_SYSTEM),
|
|
user: prompt,
|
|
tools: [EMIT_QUIZ_QUESTIONS_TOOL],
|
|
toolChoice: { type: 'tool', name: EMIT_QUIZ_QUESTIONS_TOOL.name },
|
|
maxTokens: 4096,
|
|
});
|
|
|
|
const emitted = result.toolUses[0]?.input;
|
|
if (!emitted) {
|
|
throw new Error(`LLM did not emit tool output for ${topic.label}`);
|
|
}
|
|
|
|
// Handle different shapes the LLM might return:
|
|
// 1. { questions: [...] } — expected
|
|
// 2. [...] — model returned array directly
|
|
// 3. { questions: { ... } } — model wrapped single question in object
|
|
let questions;
|
|
if (Array.isArray(emitted.questions)) {
|
|
questions = emitted.questions;
|
|
} else if (Array.isArray(emitted)) {
|
|
questions = emitted;
|
|
} else if (emitted.questions && typeof emitted.questions === 'object') {
|
|
// Single question wrapped as object instead of array
|
|
questions = [emitted.questions];
|
|
} else {
|
|
throw new Error(`Unexpected quiz output shape for ${topic.label}`);
|
|
}
|
|
|
|
if (!questions.length) {
|
|
throw new Error(`No questions generated for ${topic.label}`);
|
|
}
|
|
|
|
// Ensure each question has required fields (fill in defaults for missing ones)
|
|
return questions.map((q, i) => ({
|
|
id: q.id || `gen-${i}-${Math.random().toString(36).slice(2, 8)}`,
|
|
question: q.question || '',
|
|
topicLabel: q.topicLabel || topic.label,
|
|
options: Array.isArray(q.options) ? q.options.slice(0, 4) : [],
|
|
correctIndex: typeof q.correctIndex === 'number' ? q.correctIndex : 0,
|
|
explanation: q.explanation || 'No explanation provided.',
|
|
difficulty: q.difficulty || 'medium',
|
|
})).filter(q => q.question && q.options.length === 4);
|
|
}
|
|
|
|
async function selectTestTopics(userId, isoWeekNumber) {
|
|
const allTopics = await db.getTopics();
|
|
const topics = allTopics.filter(t => t.type !== 'fact' && t.learning_relevance !== 'exclude');
|
|
if (!topics || topics.length === 0) return { primaryTopic: null, reviewTopics: [], isReviewWeek: false };
|
|
|
|
try {
|
|
const weekContent = await getCurrentWeekContent(isoWeekNumber);
|
|
|
|
if (weekContent && weekContent.topics && weekContent.topics.length > 0) {
|
|
const primaryTopic = weekContent.topics[0];
|
|
if (!primaryTopic?.id) {
|
|
console.warn('[Test] First curriculum topic has no id — falling back to hash');
|
|
} else {
|
|
const others = topics.filter(t => t.id !== primaryTopic.id);
|
|
const reviewTopics = sample(others, Math.min(5, others.length));
|
|
return { primaryTopic, reviewTopics, isReviewWeek: false };
|
|
}
|
|
}
|
|
} catch (e) {
|
|
console.warn('[Test] Curriculum lookup failed, falling back to hash:', e.message);
|
|
}
|
|
|
|
// Fallback hash-based assignment
|
|
const str = `${userId}:${isoWeekNumber}`;
|
|
let hash = 0;
|
|
for (let i = 0; i < str.length; i++) {
|
|
hash = (hash << 5) - hash + str.charCodeAt(i);
|
|
hash |= 0;
|
|
}
|
|
const primaryIndex = Math.abs(hash) % topics.length;
|
|
const primaryTopic = topics[primaryIndex];
|
|
|
|
const others = topics.filter((_, i) => i !== primaryIndex);
|
|
const reviewTopics = sample(others, Math.min(5, others.length));
|
|
|
|
return { primaryTopic, reviewTopics, isReviewWeek: false };
|
|
}
|
|
|
|
export async function getCachedQuiz(userId, weekNumber) {
|
|
return db.getCachedQuiz(userId, weekNumber);
|
|
}
|
|
|
|
export async function forceGenerateTopicQuestions(topic, count = 5) {
|
|
const existingBank = await db.getQuizBank(topic.id);
|
|
const existingKeys = new Set(existingBank.map((q) => normalizeQuestionText(q.question)));
|
|
|
|
let lastQualityError = null;
|
|
let candidates = null;
|
|
|
|
for (let attempt = 0; attempt < 3; attempt++) {
|
|
let questions;
|
|
try {
|
|
questions = await callQuizModel(topic, count);
|
|
} catch (err) {
|
|
console.warn(`[quiz] LLM failed (attempt ${attempt + 1}):`, err.message);
|
|
lastQualityError = err.message;
|
|
continue;
|
|
}
|
|
|
|
const qualityError = validateBatchQuality(questions);
|
|
if (qualityError) {
|
|
lastQualityError = qualityError;
|
|
console.warn(`[quiz] batch rejected (attempt ${attempt + 1}): ${qualityError}`);
|
|
continue;
|
|
}
|
|
|
|
const dominant = dominantCorrectIndex(questions);
|
|
if (dominant && attempt < 2) {
|
|
console.warn(`[quiz] correctIndex dominated by ${dominant.index} (${Math.round(dominant.ratio * 100)}%) — re-rolling`);
|
|
continue;
|
|
}
|
|
|
|
candidates = questions;
|
|
break;
|
|
}
|
|
|
|
if (!candidates) {
|
|
throw new Error(`Quality gate rejected the generated batch for ${topic.label}: ${lastQualityError || 'unbalanced answer distribution'}. Click "Generate" to try again.`);
|
|
}
|
|
|
|
const accepted = [];
|
|
for (const q of candidates) {
|
|
const key = normalizeQuestionText(q.question);
|
|
if (existingKeys.has(key)) {
|
|
console.debug('[quiz] dropped duplicate:', q.question);
|
|
continue;
|
|
}
|
|
existingKeys.add(key);
|
|
accepted.push({
|
|
...q,
|
|
id: `${topic.id}-${Math.random().toString(36).slice(2, 11)}`,
|
|
});
|
|
}
|
|
|
|
if (!accepted.length) {
|
|
throw new Error(`All generated questions for ${topic.label} were duplicates of existing ones.`);
|
|
}
|
|
|
|
const merged = [...existingBank, ...accepted];
|
|
await db.setQuizBank(topic.id, merged);
|
|
return accepted;
|
|
}
|
|
|
|
async function getOrGenerateTopicQuestions(topic, count) {
|
|
let bank = await db.getQuizBank(topic.id);
|
|
|
|
if (bank.length < count) {
|
|
try {
|
|
await forceGenerateTopicQuestions(topic, 5);
|
|
bank = await db.getQuizBank(topic.id);
|
|
} catch (err) {
|
|
console.warn(`[quiz] Failed to generate questions for ${topic.label}:`, err.message);
|
|
}
|
|
}
|
|
|
|
return sample(bank, Math.min(count, bank.length));
|
|
}
|
|
|
|
export async function getTopicQuestionBank(topicId) {
|
|
return db.getQuizBank(topicId);
|
|
}
|
|
|
|
export async function deleteQuestion(topicId, questionId) {
|
|
return db.deleteQuestionFromBank(topicId, questionId);
|
|
}
|
|
|
|
export async function generateWeeklyQuiz(userId, weekNumber, force = false) {
|
|
if (!force) {
|
|
const cached = await db.getCachedQuiz(userId, weekNumber);
|
|
if (cached?.questions?.length > 0) return cached;
|
|
}
|
|
|
|
const { primaryTopic, reviewTopics } = await selectTestTopics(userId, weekNumber);
|
|
if (!primaryTopic) throw new Error('No topics available to generate a quiz.');
|
|
|
|
const questions = [];
|
|
|
|
const primaryQs = await getOrGenerateTopicQuestions(primaryTopic, 3);
|
|
questions.push(...primaryQs);
|
|
|
|
for (const rt of reviewTopics) {
|
|
if (questions.length >= 5) break;
|
|
const rtQs = await getOrGenerateTopicQuestions(rt, 1);
|
|
questions.push(...rtQs);
|
|
}
|
|
|
|
if (questions.length < 5) {
|
|
const needed = 5 - questions.length;
|
|
const extraQs = await getOrGenerateTopicQuestions(primaryTopic, needed + 5);
|
|
const existingIds = new Set(questions.map(q => q.id));
|
|
for (const eq of extraQs) {
|
|
if (!existingIds.has(eq.id) && questions.length < 5) {
|
|
questions.push(eq);
|
|
}
|
|
}
|
|
}
|
|
|
|
const shuffled = shuffle(questions).slice(0, 5);
|
|
|
|
if (shuffled.length === 0) {
|
|
throw new Error('Could not assemble enough questions for this week\'s test. Please try again.');
|
|
}
|
|
|
|
const quiz = {
|
|
questions: shuffled,
|
|
meta: {
|
|
userId,
|
|
weekNumber,
|
|
generatedAt: new Date().toISOString(),
|
|
primaryTopic: primaryTopic.label,
|
|
},
|
|
};
|
|
|
|
await db.setCachedQuiz(userId, weekNumber, quiz);
|
|
return quiz;
|
|
}
|
|
|
|
export async function saveTestResult(userId, weekNumber, result) {
|
|
await db.saveQuizResult(userId, weekNumber, result);
|
|
|
|
const pointsEarned = result.score * 2;
|
|
const members = await db.getTeamMembers();
|
|
const member = members.find(m => m.id === userId);
|
|
await db.upsertLeaderboardEntry(userId, member?.name || 'Unknown', pointsEarned, 1);
|
|
|
|
return { pointsEarned };
|
|
}
|
|
|
|
export async function getTestResult(userId, weekNumber) {
|
|
return db.getQuizResult(userId, weekNumber);
|
|
}
|