From d5655d2232da42d05c38f238ae337ba15ae13ef5 Mon Sep 17 00:00:00 2001 From: RaymondVerhoef Date: Wed, 20 May 2026 08:55:27 +0200 Subject: [PATCH] feat: implement automated knowledge graph extraction pipeline and visualization component --- src/components/admin/KnowledgeGraph.jsx | 19 +++++++++++++++---- src/lib/db.js | 6 +++++- src/lib/extractionPipeline.js | 17 +++++++++++++++-- 3 files changed, 35 insertions(+), 7 deletions(-) diff --git a/src/components/admin/KnowledgeGraph.jsx b/src/components/admin/KnowledgeGraph.jsx index 17886a9..9e585cc 100644 --- a/src/components/admin/KnowledgeGraph.jsx +++ b/src/components/admin/KnowledgeGraph.jsx @@ -311,11 +311,12 @@ Rules: 1. Identify topics that mean exactly the same thing. Choose one to keep, and one to delete. 2. Identify topics that are too vague, irrelevant, or malformed to delete. 3. Identify missing logical relations (depends_on, part_of, related_to) if two topics are conceptually linked but missing a relation. -4. Return ONLY a valid JSON object describing the ACTIONS to take. Do not return the entire graph. Do not wrap in markdown blocks.`; +4. Evaluate the learning_relevance of each topic. If a topic is purely operational/mundane (like a printer guide), mark it as "exclude". If it's low priority, mark "peripheral". +5. Return ONLY a valid JSON object describing the ACTIONS to take. Do not return the entire graph. Do not wrap in markdown blocks.`; // Send a compact representation to minimize token usage and avoid rate limits. - // The AI only needs id, label, and type to identify duplicates/merges. - const compactTopics = currentTopics.map(({ id, label, type }) => ({ id, label, type })); + // The AI only needs id, label, type, and relevance to identify duplicates/merges and adjust relevance. + const compactTopics = currentTopics.map(({ id, label, type, learning_relevance }) => ({ id, label, type, learning_relevance })); const compactRelations = currentRelations.map(r => ({ source: r.source?.id || r.source, target: r.target?.id || r.target, @@ -329,7 +330,8 @@ Analyze this graph and return ONLY the optimized JSON object with this EXACT str { "merges": [ { "keepId": "id_to_keep", "deleteId": "id_to_delete" } ], "deletions": [ "id_to_delete_completely" ], - "newRelations": [ { "source": "id1", "target": "id2", "type": "depends_on" } ] + "newRelations": [ { "source": "id1", "target": "id2", "type": "depends_on" } ], + "relevanceUpdates": [ { "id": "topic_id", "learning_relevance": "exclude" } ] }`; const responseText = await anthropicApi.generateContent(systemPrompt, userPrompt); @@ -364,6 +366,15 @@ Analyze this graph and return ONLY the optimized JSON object with this EXACT str } } + if (actions.relevanceUpdates && Array.isArray(actions.relevanceUpdates)) { + for (const update of actions.relevanceUpdates) { + const topicIndex = updatedTopics.findIndex(t => t.id === update.id); + if (topicIndex !== -1) { + updatedTopics[topicIndex] = { ...updatedTopics[topicIndex], learning_relevance: update.learning_relevance }; + } + } + } + // Ensure all relations only reference existing nodes and are normalized to string IDs const finalTopicIds = new Set(updatedTopics.map(t => t.id)); updatedRelations = updatedRelations diff --git a/src/lib/db.js b/src/lib/db.js index 5539844..5d0536a 100644 --- a/src/lib/db.js +++ b/src/lib/db.js @@ -27,7 +27,11 @@ export async function upsertTopic(topic) { await pb.collection('topics').getOne(topic.id); return await pb.collection('topics').update(topic.id, topic); } catch { - return await pb.collection('topics').create({ id: topic.id, ...topic }); + return await pb.collection('topics').create({ + id: topic.id, + learning_relevance: 'standard', + ...topic + }); } } diff --git a/src/lib/extractionPipeline.js b/src/lib/extractionPipeline.js index f2026c3..83d06bb 100644 --- a/src/lib/extractionPipeline.js +++ b/src/lib/extractionPipeline.js @@ -6,6 +6,12 @@ You receive a source text. Your task is to extract core concepts, roles, and pro Facts should be integrated into the descriptions of the other labels and NOT be extracted as unique topics. CRITICAL: To ensure the response fits within limits, extract a maximum of 15 key topics and their most important relations. Keep descriptions very concise. +You MUST assign a learning_relevance to each topic: +- "core": Fundamental company knowledge. +- "standard": Normal learning topics. +- "peripheral": Good to know, but low priority. +- "exclude": Pure operational reference material (e.g., printer guides, wifi passwords) that should NEVER be tested. + ALWAYS return a valid JSON object in the following format: { "topics": [ @@ -13,7 +19,8 @@ ALWAYS return a valid JSON object in the following format: "id": "unique-slug", "label": "Topic title", "type": "concept | role | process", - "description": "A concise, clear explanation of max 3 sentences." + "description": "A concise, clear explanation of max 3 sentences.", + "learning_relevance": "core | standard | peripheral | exclude" } ], "relations": [ @@ -35,10 +42,16 @@ You must explicitly identify and create relations between Roles, Processes, and Every Process must have a Role attached (who does it). Every Concept must have a relation to a Process or Role. +You MUST assign a learning_relevance to each topic: +- "core": Fundamental company knowledge. +- "standard": Normal learning topics. +- "peripheral": Good to know, but low priority. +- "exclude": Pure operational reference material (e.g., printer guides, wifi passwords) that should NEVER be tested. + Return a JSON object: { "topics": [ - { "id": "...", "label": "...", "type": "role | process | concept", "description": "...", "metadata": { "source": "github_handbook" } } + { "id": "...", "label": "...", "type": "role | process | concept", "description": "...", "learning_relevance": "standard", "metadata": { "source": "github_handbook" } } ], "relations": [ { "source": "role-id", "target": "process-id", "type": "executes | related_to | depends_on | part_of", "description": "Brief metadata about this specific relation" }