feat: implement knowledge testing system with leaderboard, quiz generation, and PocketBase integration

src/lib/extractionPipeline.js

@@ -1,5 +1,5 @@
 import { anthropicApi } from './api';
-import { storage } from './storage';
+import * as db from './db';
 
 const SYSTEM_PROMPT = `You are an AI knowledge extractor for Respellion, an IT company built on radical transparency.
 You receive a source text. Your task is to extract core concepts, roles, and processes, and return them as a structured JSON Knowledge Graph.
@@ -25,32 +25,15 @@ ALWAYS return a valid JSON object in the following format:
 }
 Return JSON only. No markdown blocks or other text.`;
 
-/**
- * Voert tekst door de Anthropic API en slaat de resulterende topics op.
- * @param {string} textContent De te analyseren tekst.
- * @param {string} sourceName Naam van de bron (voor tracking).
- */
 export async function processSourceText(textContent, sourceName) {
-  // 1. Sla de bron eerst op als "processing"
-  const sourceId = `src_${Date.now()}`;
-  const newSource = {
-    id: sourceId,
-    name: sourceName,
-    status: 'processing',
-    date: new Date().toISOString()
-  };
-  
-  const sources = storage.get('admin:sources', []);
-  storage.set('admin:sources', [newSource, ...sources]);
+  const rec = await db.addSource({ name: sourceName, status: 'processing' });
+  const sourceId = rec.id;
 
   try {
-    // 2. Roep Anthropic API aan
-  const responseText = await anthropicApi.generateContent(SYSTEM_PROMPT, `Analyze the following text:\n\n${textContent}`);
-    
-    // 3. Parse JSON (veilig)
+    const responseText = await anthropicApi.generateContent(SYSTEM_PROMPT, `Analyze the following text:\n\n${textContent}`);
+
     let extractedData;
     try {
-      // Zoek naar JSON in de response (indien het toch in markdown was verpakt)
       const jsonMatch = responseText.match(/\{[\s\S]*\}/);
       const jsonStr = jsonMatch ? jsonMatch[0] : responseText;
       extractedData = JSON.parse(jsonStr);
@@ -58,36 +41,23 @@ export async function processSourceText(textContent, sourceName) {
       throw new Error('AI response was not valid JSON.');
     }
 
-    // 4. Deduplicatie en opslag van Topics en Relaties
-    mergeKnowledgeGraph(extractedData);
-
-    // 5. Markeer bron als voltooid
-    updateSourceStatus(sourceId, 'completed');
+    await mergeKnowledgeGraph(extractedData);
+    await db.updateSourceStatus(sourceId, 'completed');
 
     return { success: true, data: extractedData };
 
   } catch (error) {
-    // Markeer bron als mislukt
-    updateSourceStatus(sourceId, 'failed', error.message);
+    await db.updateSourceStatus(sourceId, 'failed', error.message);
     throw error;
   }
 }
 
-function updateSourceStatus(sourceId, status, errorMsg = '') {
-  const sources = storage.get('admin:sources', []);
-  const updated = sources.map(s => 
-    s.id === sourceId ? { ...s, status, error: errorMsg } : s
-  );
-  storage.set('admin:sources', updated);
-}
+async function mergeKnowledgeGraph(newData) {
+  const existingTopics = await db.getTopics();
+  const existingRelations = await db.getRelations();
 
-function mergeKnowledgeGraph(newData) {
-  const existingTopics = storage.get('kb:topics', []);
-  const existingRelations = storage.get('kb:relations', []);
-
-  // Simpele deduplicatie op ID
   const topicsMap = new Map(existingTopics.map(t => [t.id, t]));
-  
+
   if (newData.topics && Array.isArray(newData.topics)) {
     for (const t of newData.topics) {
       if (!topicsMap.has(t.id)) {
@@ -96,11 +66,9 @@ function mergeKnowledgeGraph(newData) {
     }
   }
 
-  // Voeg relaties toe (we gaan er nu vanuit dat ze uniek genoeg zijn of dubbele negeren)
   const newRelations = [...existingRelations];
   if (newData.relations && Array.isArray(newData.relations)) {
     for (const r of newData.relations) {
-      // Simpele check om exacte duplicaten te voorkomen
       const isDup = newRelations.some(ex => ex.source === r.source && ex.target === r.target && ex.type === r.type);
       if (!isDup) {
         newRelations.push(r);
@@ -108,6 +76,6 @@ function mergeKnowledgeGraph(newData) {
     }
   }
 
-  storage.set('kb:topics', Array.from(topicsMap.values()));
-  storage.set('kb:relations', newRelations);
+  await db.saveTopics(Array.from(topicsMap.values()));
+  await db.saveRelations(newRelations);
 }