From ca8945ea5bb7c1f915b4802d69a40bf8fb1780b5 Mon Sep 17 00:00:00 2001 From: RaymondVerhoef Date: Fri, 22 May 2026 19:45:14 +0200 Subject: [PATCH] refactor: remove handbook sync state and related functionality --- .../1780000000_created_handbook_sync_state.js | 90 -------- src/components/admin/KnowledgeGraph.jsx | 124 ----------- src/components/admin/UploadZone.jsx | 197 ++++++++++++------ src/lib/__tests__/extractionPipeline.test.js | 21 +- src/lib/__tests__/llmSchemas.test.js | 24 --- src/lib/__tests__/llmTools.test.js | 2 - src/lib/db.js | 20 +- src/lib/extractionPipeline.js | 64 ++---- src/lib/githubService.js | 88 -------- src/lib/llmSchemas.js | 34 --- src/lib/llmTools.js | 36 ---- src/pages/Admin/index.jsx | 11 - 12 files changed, 159 insertions(+), 552 deletions(-) delete mode 100644 pb_migrations/1780000000_created_handbook_sync_state.js delete mode 100644 src/lib/githubService.js diff --git a/pb_migrations/1780000000_created_handbook_sync_state.js b/pb_migrations/1780000000_created_handbook_sync_state.js deleted file mode 100644 index 23f7d0e..0000000 --- a/pb_migrations/1780000000_created_handbook_sync_state.js +++ /dev/null @@ -1,90 +0,0 @@ -/// -migrate((app) => { - const collection = new Collection({ - "createRule": "", - "deleteRule": "", - "fields": [ - { - "autogeneratePattern": "[a-z0-9]{15}", - "help": "", - "hidden": false, - "id": "text3208210256", - "max": 15, - "min": 15, - "name": "id", - "pattern": "^[a-z0-9]+$", - "presentable": false, - "primaryKey": true, - "required": true, - "system": true, - "type": "text" - }, - { - "autogeneratePattern": "", - "help": "", - "hidden": false, - "id": "text_path123", - "max": 0, - "min": 0, - "name": "path", - "pattern": "", - "presentable": false, - "primaryKey": false, - "required": true, - "system": false, - "type": "text" - }, - { - "autogeneratePattern": "", - "help": "", - "hidden": false, - "id": "text_sha123", - "max": 0, - "min": 0, - "name": "sha", - "pattern": "", - "presentable": false, - "primaryKey": false, - "required": true, - "system": false, - "type": "text" - }, - { - "hidden": false, - "id": "autodate2990389176", - "name": "created", - "onCreate": true, - "onUpdate": false, - "presentable": false, - "system": false, - "type": "autodate" - }, - { - "hidden": false, - "id": "autodate3332085495", - "name": "updated", - "onCreate": true, - "onUpdate": true, - "presentable": false, - "system": false, - "type": "autodate" - } - ], - "id": "pbc_handbooksync123", - "indexes": [ - "CREATE UNIQUE INDEX `idx_handbook_path` ON `handbook_sync_state` (`path`)" - ], - "listRule": "", - "name": "handbook_sync_state", - "system": false, - "type": "base", - "updateRule": "", - "viewRule": "" - }); - - return app.save(collection); -}, (app) => { - const collection = app.findCollectionByNameOrId("pbc_handbooksync123"); - - return app.delete(collection); -}) diff --git a/src/components/admin/KnowledgeGraph.jsx b/src/components/admin/KnowledgeGraph.jsx index 7426f65..40db606 100644 --- a/src/components/admin/KnowledgeGraph.jsx +++ b/src/components/admin/KnowledgeGraph.jsx @@ -4,8 +4,6 @@ import { Trash2, Edit2, Save, X, RefreshCw, AlertCircle, Plus, Link as LinkIcon import * as db from '../../lib/db'; import { callLLM } from '../../lib/llm'; import { EMIT_GRAPH_ACTIONS_TOOL } from '../../lib/llmTools'; -import { analyzeHandbookDelta } from '../../lib/extractionPipeline'; -import { getRepoFolder, getFileContent } from '../../lib/githubService'; import Button from '../ui/Button'; import SuggestionsQueue from './SuggestionsQueue'; @@ -20,10 +18,6 @@ const KnowledgeGraph = () => { const [analyzeError, setAnalyzeError] = useState(null); const [newRelData, setNewRelData] = useState({ target: '', type: 'related_to' }); - const [isSyncing, setIsSyncing] = useState(false); - const [syncResult, setSyncResult] = useState(null); - const [syncError, setSyncError] = useState(null); - const [syncProgress, setSyncProgress] = useState(null); const [topics, setTopics] = useState([]); const [relations, setRelations] = useState([]); @@ -237,87 +231,6 @@ const KnowledgeGraph = () => { ))); }; - const syncHandbook = async () => { - setIsSyncing(true); - setSyncError(null); - setSyncResult(null); - - try { - const files = await getRepoFolder('respellion', 'employee-handbook', 'docs'); - const syncStates = await db.getHandbookSyncStates(); - - const added = []; - const modified = []; - const unchanged = []; - - files.forEach(file => { - const state = syncStates.find(s => s.path === file.path); - if (!state) { - added.push(file); - } else if (state.sha !== file.sha) { - modified.push(file); - } else { - unchanged.push(file); - } - }); - - setSyncResult({ added, modified, unchanged, failed: [] }); - - const filesToProcess = [...added, ...modified]; - if (filesToProcess.length > 0) { - const total = filesToProcess.length; - let done = 0; - const failed = []; - setSyncProgress(`Processing 0 of ${total} files...`); - - // Run files in parallel with bounded concurrency. The extractionLimiter - // inside analyzeHandbookDelta still governs the actual API request rate; - // parallelism here just hides GitHub fetch latency and overlaps with the - // limiter's spacing instead of waiting serially. - const CONCURRENCY = 4; - const queue = filesToProcess.slice(); - - async function worker() { - while (queue.length > 0) { - const file = queue.shift(); - if (!file) return; - try { - const rawContent = await getFileContent('respellion', 'employee-handbook', file.path); - // Skip near-empty files — they only burn an LLM call to extract nothing. - if (rawContent.trim().length < 50) { - await db.updateHandbookSyncState(file.path, file.sha); - } else { - await analyzeHandbookDelta(rawContent, file.path); - await db.updateHandbookSyncState(file.path, file.sha); - } - } catch (err) { - console.error('Failed to process file:', file.path, err); - failed.push({ path: file.path, message: err?.message || String(err) }); - } finally { - done++; - setSyncProgress(`Processing ${done} of ${total} files (${failed.length} failed)...`); - } - } - } - - await Promise.all(Array.from({ length: Math.min(CONCURRENCY, total) }, worker)); - - setSyncResult({ added, modified, unchanged, failed }); - setSyncProgress( - failed.length === 0 - ? 'Sync Complete! Click "Analyze & Optimize Graph" above to clean up and merge.' - : `Sync finished with ${failed.length} failure(s). See console for details.` - ); - reloadKb(); - } - - } catch (e) { - setSyncError(e.message || 'Failed to check GitHub for updates.'); - } finally { - setIsSyncing(false); - setTimeout(() => setSyncProgress(null), 10000); // Clear progress after 10s - } - }; const analyzeGraph = async () => { if (!confirm('This will send the entire graph to the AI to evaluate content, merge duplicates, and create new logical relations. This may take a moment. Proceed?')) return; @@ -466,43 +379,6 @@ ${JSON.stringify({ topics: compactTopics, relations: compactRelations })}`;

)} - -
- - {syncError && ( -

- - {syncError} -

- )} - {syncResult && ( -
-

GitHub Sync Status:

-

Added files: {syncResult.added.length}

-

Modified files: {syncResult.modified.length}

-

Unchanged: {syncResult.unchanged.length}

- {syncResult.failed?.length > 0 && ( -

Failed: {syncResult.failed.length}

- )} -
- )} - {syncProgress && ( -
-

- {syncProgress !== 'Sync Complete!' && } - {syncProgress} -

-
- )} -
{/* R42 chatbot suggestions queue */} diff --git a/src/components/admin/UploadZone.jsx b/src/components/admin/UploadZone.jsx index 63d4029..c6a3fb9 100644 --- a/src/components/admin/UploadZone.jsx +++ b/src/components/admin/UploadZone.jsx @@ -1,5 +1,5 @@ -import { useState, useRef } from 'react'; -import { UploadCloud, AlertCircle, CheckCircle, X } from 'lucide-react'; +import { useEffect, useRef, useState } from 'react'; +import { UploadCloud, CheckCircle, AlertCircle, Loader2, X, Clock, Minus } from 'lucide-react'; import { processSourceText } from '../../lib/extractionPipeline'; import Card from '../ui/Card'; import Button from '../ui/Button'; @@ -7,122 +7,187 @@ import Button from '../ui/Button'; // ── Main Component ──────────────────────────────────────────────────────────── const UploadZone = ({ onUploadComplete }) => { - const [isDragging, setIsDragging] = useState(false); - const [isProcessing, setIsProcessing] = useState(false); - const [status, setStatus] = useState(null); + const [isDragging, setIsDragging] = useState(false); + const [queue, setQueue] = useState([]); + const [processingId, setProcessingId] = useState(null); + const [rejectedNote, setRejectedNote] = useState(null); const fileInputRef = useRef(null); const abortRef = useRef(null); - // ── File upload (drag & drop / browse) ──────────────────────────────────── + // ── Processing loop ──────────────────────────────────────────────────────── + + useEffect(() => { + if (processingId !== null) return; + const next = queue.find((q) => q.status === 'pending'); + if (!next) return; + + setProcessingId(next.id); + setQueue((q) => q.map((item) => item.id === next.id ? { ...item, status: 'processing' } : item)); - const processFile = async (file) => { - setIsProcessing(true); - setStatus(null); const controller = new AbortController(); abortRef.current = controller; - try { - const text = await file.text(); - await processSourceText(text, file.name, { signal: controller.signal }); - setStatus({ type: 'success', msg: `Successfully processed: ${file.name}` }); - if (onUploadComplete) onUploadComplete(); - } catch (error) { - if (error?.name === 'AbortError') { - setStatus({ type: 'error', msg: `Cancelled: ${file.name}` }); + + next.file.text() + .then((text) => processSourceText(text, next.name, { signal: controller.signal })) + .then(() => { + setQueue((q) => q.map((item) => item.id === next.id ? { ...item, status: 'done' } : item)); + if (onUploadComplete) onUploadComplete(); + }) + .catch((err) => { + const isCancelled = err?.name === 'AbortError'; + setQueue((q) => q.map((item) => + item.id === next.id + ? { ...item, status: isCancelled ? 'cancelled' : 'failed', error: isCancelled ? null : err.message } + : item + )); + }) + .finally(() => { + abortRef.current = null; + setProcessingId(null); + }); + }, [processingId, queue, onUploadComplete]); + + // ── File ingestion ───────────────────────────────────────────────────────── + + const addFiles = (fileList) => { + const accepted = []; + let rejected = 0; + for (const file of fileList) { + if (file.type === 'text/plain' || file.name.endsWith('.md')) { + accepted.push({ id: crypto.randomUUID(), file, name: file.name, status: 'pending', error: null }); } else { - setStatus({ type: 'error', msg: `Error processing file: ${error.message}` }); + rejected++; } - } finally { - abortRef.current = null; - setIsProcessing(false); + } + if (accepted.length > 0) setQueue((q) => [...q, ...accepted]); + if (rejected > 0) { + setRejectedNote(`${rejected} file${rejected > 1 ? 's' : ''} skipped — only .txt and .md are supported.`); + setTimeout(() => setRejectedNote(null), 4000); } }; - const cancelProcessing = () => { + const cancelCurrent = () => { abortRef.current?.abort(new DOMException('Cancelled by user', 'AbortError')); }; + const clearDone = () => { + setQueue((q) => q.filter((item) => item.status === 'pending' || item.status === 'processing')); + }; + + // ── Drag & drop / browse ─────────────────────────────────────────────────── + const handleDragOver = (e) => { e.preventDefault(); setIsDragging(true); }; const handleDragLeave = () => setIsDragging(false); const handleDrop = (e) => { e.preventDefault(); setIsDragging(false); - if (e.dataTransfer.files?.length > 0) { - const file = e.dataTransfer.files[0]; - if (file.type === 'text/plain' || file.name.endsWith('.md')) { - processFile(file); - } else { - setStatus({ type: 'error', msg: 'Only .txt and .md files are currently supported.' }); - } - } + if (e.dataTransfer.files?.length > 0) addFiles(e.dataTransfer.files); }; const handleFileSelect = (e) => { - if (e.target.files?.length > 0) processFile(e.target.files[0]); + if (e.target.files?.length > 0) { + addFiles(e.target.files); + e.target.value = ''; + } }; + // ── Derived state ────────────────────────────────────────────────────────── + + const hasDone = queue.some((q) => ['done', 'failed', 'cancelled'].includes(q.status)); + // ── Render ───────────────────────────────────────────────────────────────── return ( -
+
- {/* ─── Drag & Drop Upload ─── */} + {/* ─── Drag & Drop Zone ─── */} fileInputRef.current?.click()} > - -

Drag files here

-

Supports .txt and .md (max 5MB)

- + +

Drag files here

+

Supports .txt and .md · max 5 MB each

+ - {/* Cancel sits outside the upload card so pointer-events-none doesn't disable it. */} - {isProcessing && ( -
- -
+ {/* ─── Rejected file note ─── */} + {rejectedNote && ( +

{rejectedNote}

)} - {/* ─── Status messages ─── */} - {status && ( -
- {status.type === 'error' - ? - : } -
-

{status.type === 'error' ? 'Error' : 'Success'}

-

{status.msg}

-
+ {/* ─── Queue ─── */} + {queue.length > 0 && ( +
+ {queue.map((item) => ( +
+ + {item.name} + + {item.status === 'processing' && 'Extracting…'} + {item.status === 'pending' && 'Pending'} + {item.status === 'done' && 'Done'} + {item.status === 'cancelled' && 'Cancelled'} + {item.status === 'failed' && ( + Failed + )} + + {item.status === 'processing' && ( + + )} +
+ ))} + + {hasDone && ( +
+ +
+ )}
)}
); }; +// ── Status icon ─────────────────────────────────────────────────────────────── + +const StatusIcon = ({ status }) => { + if (status === 'processing') return ; + if (status === 'done') return ; + if (status === 'failed') return ; + if (status === 'cancelled') return ; + return ; +}; + export default UploadZone; diff --git a/src/lib/__tests__/extractionPipeline.test.js b/src/lib/__tests__/extractionPipeline.test.js index 5efc45b..23759b6 100644 --- a/src/lib/__tests__/extractionPipeline.test.js +++ b/src/lib/__tests__/extractionPipeline.test.js @@ -69,18 +69,21 @@ describe('buildKnownIdsHint', () => { expect(buildKnownIdsHint(null)).toBe(''); }); - it('formats the known IDs as a bulleted list with a leading instruction', () => { - const hint = buildKnownIdsHint(['software-engineer', 'onboarding-buddy']); - expect(hint).toContain('Already-extracted topic IDs'); - expect(hint).toContain('- software-engineer'); - expect(hint).toContain('- onboarding-buddy'); + it('formats topics as id: "label" pairs with a leading instruction', () => { + const hint = buildKnownIdsHint([ + { id: 'software-engineer', label: 'Software Engineer' }, + { id: 'onboarding-buddy', label: 'Onboarding Buddy' }, + ]); + expect(hint).toContain('Already-extracted topics'); + expect(hint).toContain('- software-engineer: "Software Engineer"'); + expect(hint).toContain('- onboarding-buddy: "Onboarding Buddy"'); expect(hint.endsWith('\n')).toBe(true); }); - it('caps the hint at the 200 most recent IDs', () => { - const ids = Array.from({ length: 250 }, (_, i) => `topic-${i}`); - const hint = buildKnownIdsHint(ids); - // The newest IDs must appear; the oldest must not. + it('caps the hint at the 200 most recent topics', () => { + const topics = Array.from({ length: 250 }, (_, i) => ({ id: `topic-${i}`, label: `Topic ${i}` })); + const hint = buildKnownIdsHint(topics); + // The newest topics must appear; the oldest must not. expect(hint).toContain('topic-249'); expect(hint).toContain('topic-50'); expect(hint).not.toContain('topic-49'); diff --git a/src/lib/__tests__/llmSchemas.test.js b/src/lib/__tests__/llmSchemas.test.js index b036138..ae79799 100644 --- a/src/lib/__tests__/llmSchemas.test.js +++ b/src/lib/__tests__/llmSchemas.test.js @@ -1,8 +1,6 @@ import { describe, expect, it } from 'vitest'; import { extractionResultSchema, - handbookResultSchema, - normalizeHandbookResult, learningArticleSchema, learningSlidesSchema, learningInfographicSchema, @@ -60,28 +58,6 @@ describe('extractionResultSchema', () => { }); }); -describe('handbookResultSchema', () => { - it('accepts the loose vocabulary including executes', () => { - const parsed = handbookResultSchema.parse({ - topics: [{ ...sampleTopic, metadata: { source: 'github_handbook' } }], - relations: [{ source: 'software-engineer', target: 'code-review', type: 'executes' }], - }); - expect(parsed.relations[0].type).toBe('executes'); - }); - - it('normalises executes into executed_by with swapped source/target', () => { - const parsed = handbookResultSchema.parse({ - topics: [sampleTopic], - relations: [{ source: 'software-engineer', target: 'code-review', type: 'executes' }], - }); - const normalised = normalizeHandbookResult(parsed); - expect(normalised.relations[0]).toMatchObject({ - source: 'code-review', - target: 'software-engineer', - type: 'executed_by', - }); - }); -}); describe('learning schemas', () => { it('accepts an article payload', () => { diff --git a/src/lib/__tests__/llmTools.test.js b/src/lib/__tests__/llmTools.test.js index 2a3fe0c..d2715aa 100644 --- a/src/lib/__tests__/llmTools.test.js +++ b/src/lib/__tests__/llmTools.test.js @@ -1,7 +1,6 @@ import { describe, expect, it } from 'vitest'; import { EMIT_KNOWLEDGE_GRAPH_TOOL, - EMIT_HANDBOOK_DELTA_TOOL, EMIT_LEARNING_ARTICLE_TOOL, EMIT_LEARNING_SLIDES_TOOL, EMIT_LEARNING_INFOGRAPHIC_TOOL, @@ -15,7 +14,6 @@ import { toolSchemaRegistry } from '../llmSchemas'; const allTools = [ EMIT_KNOWLEDGE_GRAPH_TOOL, - EMIT_HANDBOOK_DELTA_TOOL, EMIT_LEARNING_ARTICLE_TOOL, EMIT_LEARNING_SLIDES_TOOL, EMIT_LEARNING_INFOGRAPHIC_TOOL, diff --git a/src/lib/db.js b/src/lib/db.js index 53d8aec..7e81c05 100644 --- a/src/lib/db.js +++ b/src/lib/db.js @@ -349,11 +349,10 @@ export async function getRecentLlmCalls(limit = 100) { * * `team_members`, `settings`, and `llm_calls` are intentionally preserved. * - * @param {{ includeHandbookState?: boolean, includeProgress?: boolean }} [opts] + * @param {{ includeProgress?: boolean }} [opts] * @returns {Promise>} */ export async function resetForSmokeTest({ - includeHandbookState = true, includeProgress = false, } = {}) { const collections = [ @@ -366,7 +365,6 @@ export async function resetForSmokeTest({ 'sources', 'topics', ]; - if (includeHandbookState) collections.push('handbook_sync_state'); if (includeProgress) collections.push('learn_progress', 'leaderboard'); const results = []; @@ -388,20 +386,4 @@ export async function resetForSmokeTest({ return results; } -// ── Handbook Sync State ─────────────────────────────────────────────────────── - -export async function getHandbookSyncStates() { - try { - return await pb.collection('handbook_sync_state').getFullList(); - } catch { return []; } -} - -export async function updateHandbookSyncState(path, sha) { - try { - const r = await pb.collection('handbook_sync_state').getFirstListItem(`path="${path}"`); - return await pb.collection('handbook_sync_state').update(r.id, { sha }); - } catch { - return await pb.collection('handbook_sync_state').create({ path, sha }); - } -} diff --git a/src/lib/extractionPipeline.js b/src/lib/extractionPipeline.js index 0a4be2f..38afae1 100644 --- a/src/lib/extractionPipeline.js +++ b/src/lib/extractionPipeline.js @@ -1,24 +1,21 @@ import * as db from './db'; import { callLLM } from './llm'; import { extractionLimiter } from './llmRetry'; -import { EMIT_KNOWLEDGE_GRAPH_TOOL, EMIT_HANDBOOK_DELTA_TOOL } from './llmTools'; -import { normalizeHandbookResult } from './llmSchemas'; +import { EMIT_KNOWLEDGE_GRAPH_TOOL } from './llmTools'; -const MAX_KNOWN_IDS_HINT = 200; +const MAX_KNOWN_TOPICS_HINT = 200; /** - * Build the "already-extracted topic IDs" hint that prepends every chunk - * after the first. Capped at the most-recent `MAX_KNOWN_IDS_HINT` IDs so - * the prompt stays a bounded size; the model uses this list to reuse IDs - * rather than invent variants like `software-developer` for - * `software-engineer`. + * Build the "already-extracted topics" hint included in every chunk prompt. + * Passes both ID and label so the model can match concepts by name and reuse + * the exact ID + label rather than inventing near-duplicate variants. */ -export function buildKnownIdsHint(ids) { - if (!ids || !ids.length) return ''; - const recent = ids.slice(-MAX_KNOWN_IDS_HINT); +export function buildKnownIdsHint(topics) { + if (!topics || !topics.length) return ''; + const recent = topics.slice(-MAX_KNOWN_TOPICS_HINT); return [ - 'Already-extracted topic IDs (do NOT create new IDs for these — reuse them if the same concept appears here):', - ...recent.map((id) => `- ${id}`), + 'Already-extracted topics (reuse their ID and label exactly if the same concept appears here):', + ...recent.map((t) => `- ${t.id}: "${t.label}"`), '', ].join('\n'); } @@ -45,41 +42,8 @@ Assign a learning_relevance to every topic: Relation types: related_to | depends_on | part_of | executed_by. `; -const HANDBOOK_SYSTEM_PROMPT = `You are analysing an update to the Respellion Employee Handbook. Emit the extracted topics and relations through the emit_handbook_delta tool. - -CRITICAL INSTRUCTIONS: -- Every process must have a role attached. Express this as: process --executed_by--> role. -- Every concept must connect to a process or role. -- Mark handbook topics with metadata.source = "github_handbook". -- Assign learning_relevance using the same scale as extraction: core | standard | peripheral | exclude. - -Relation types: related_to | depends_on | part_of | executed_by. -`; - const cachedSystem = (text) => [{ type: 'text', text, cache_control: { type: 'ephemeral' } }]; -export async function analyzeHandbookDelta(fileContent, filePath, { signal } = {}) { - const result = await callLLM({ - task: 'extract.handbook', - tier: 'standard', - system: cachedSystem(HANDBOOK_SYSTEM_PROMPT), - user: `Analyze the following handbook file update (${filePath}):\n\n${fileContent}`, - tools: [EMIT_HANDBOOK_DELTA_TOOL], - toolChoice: { type: 'tool', name: EMIT_HANDBOOK_DELTA_TOOL.name }, - maxTokens: 8192, - timeoutMs: 180_000, - limiter: extractionLimiter, - signal, - }); - - const raw = result.toolUses[0]?.input; - if (!raw) throw new Error('Handbook extraction did not emit a tool result.'); - const extractedData = normalizeHandbookResult(raw); - - await mergeKnowledgeGraph(extractedData); - return { success: true, data: extractedData }; -} - /** * Sentence-aware chunker with overlap. * @@ -169,7 +133,7 @@ export async function processSourceText(textContent, sourceName, { signal } = {} console.log(`[Pipeline] Split "${sourceName}" into ${chunks.length} chunks for processing.`); const existingTopics = await db.getTopics(); - const knownIds = existingTopics.map((t) => t.id); + const knownTopics = existingTopics.map((t) => ({ id: t.id, label: t.label })); const allExtractedTopics = []; const allExtractedRelations = []; @@ -178,7 +142,7 @@ export async function processSourceText(textContent, sourceName, { signal } = {} if (signal?.aborted) throw signal.reason ?? new DOMException('Aborted', 'AbortError'); console.log(`[Pipeline] Processing chunk ${i + 1}/${chunks.length} (${chunks[i].length} chars)...`); - const hint = i > 0 ? buildKnownIdsHint(knownIds) : ''; + const hint = buildKnownIdsHint(knownTopics); const result = await callLLM({ task: 'extract.source', tier: 'standard', @@ -197,7 +161,9 @@ export async function processSourceText(textContent, sourceName, { signal } = {} if (Array.isArray(extractedData.topics)) { allExtractedTopics.push(...extractedData.topics); for (const t of extractedData.topics) { - if (t?.id && !knownIds.includes(t.id)) knownIds.push(t.id); + if (t?.id && !knownTopics.some((k) => k.id === t.id)) { + knownTopics.push({ id: t.id, label: t.label }); + } } } if (Array.isArray(extractedData.relations)) { diff --git a/src/lib/githubService.js b/src/lib/githubService.js deleted file mode 100644 index 9da13e9..0000000 --- a/src/lib/githubService.js +++ /dev/null @@ -1,88 +0,0 @@ -/** - * GitHub API Service - * Fetches files from a public GitHub repository directly from the browser. - * No proxy or token required for public repos. - */ - -const GITHUB_API = 'https://api.github.com'; -const GITHUB_RAW = 'https://raw.githubusercontent.com'; - -/** - * List markdown/text files in a GitHub repository folder. - * @param {string} owner - e.g. "respellion" - * @param {string} repo - e.g. "employee-handbook" - * @param {string} folder - e.g. "docs" or "" for root - * @param {string} branch - e.g. "main" - * @returns {Promise>} - */ -export async function getRepoFolder(owner, repo, folder = '', branch = 'main') { - // Use the GitHub Git Trees API to fetch all files recursively in one request - const url = `${GITHUB_API}/repos/${owner}/${repo}/git/trees/${branch}?recursive=1`; - - const res = await fetch(url, { - headers: { Accept: 'application/vnd.github+json' }, - }); - - if (!res.ok) { - const err = await res.json().catch(() => ({})); - throw new Error(`GitHub API ${res.status}: ${err.message || 'Could not list folder'}`); - } - - const data = await res.json(); - - // If folder is 'docs', we only want paths starting with 'docs/' - const folderPrefix = folder ? folder.replace(/^\/|\/$/g, '') + '/' : ''; - - // Return only files (blobs) that match the folder prefix and file extensions - return data.tree - .filter(item => - item.type === 'blob' && - item.path.startsWith(folderPrefix) && - (item.path.endsWith('.md') || item.path.endsWith('.txt')) - ) - .map(item => ({ - name: item.path.split('/').pop(), - path: item.path, - sha: item.sha - })); -} - -/** - * Fetch the raw text content of a file. - * Uses raw.githubusercontent.com — no rate limits for public content. - * @param {string} owner - * @param {string} repo - * @param {string} path - e.g. "docs/ROLES.md" - * @param {string} branch - * @returns {Promise} - */ -export async function getFileContent(owner, repo, path, branch = 'main') { - const url = `${GITHUB_RAW}/${owner}/${repo}/${branch}/${path}`; - const res = await fetch(url); - - if (!res.ok) { - throw new Error(`Could not fetch "${path}" from GitHub (${res.status})`); - } - - return res.text(); -} - -/** - * Parse a GitHub tree URL into its parts. - * e.g. "https://github.com/respellion/employee-handbook/tree/main/docs" - * → { owner: 'respellion', repo: 'employee-handbook', branch: 'main', folder: 'docs' } - */ -export function parseGitHubUrl(url) { - try { - const match = url.match(/github\.com\/([^/]+)\/([^/]+)(?:\/tree\/([^/]+)(?:\/(.+))?)?/); - if (!match) return null; - return { - owner: match[1], - repo: match[2], - branch: match[3] || 'main', - folder: match[4] || '', - }; - } catch { - return null; - } -} diff --git a/src/lib/llmSchemas.js b/src/lib/llmSchemas.js index e3f5932..359faad 100644 --- a/src/lib/llmSchemas.js +++ b/src/lib/llmSchemas.js @@ -9,7 +9,6 @@ import { z } from 'zod'; const topicTypeEnum = z.enum(['concept', 'role', 'process']); const relationTypeStrict = z.enum(['related_to', 'depends_on', 'part_of', 'executed_by']); -const relationTypeLoose = z.enum(['related_to', 'depends_on', 'part_of', 'executed_by', 'executes']); const learningRelevanceEnum = z.enum(['core', 'standard', 'peripheral', 'exclude']); const extractionTopicSchema = z.object({ @@ -31,38 +30,6 @@ export const extractionResultSchema = z.object({ relations: z.array(extractionRelationSchema), }); -const handbookTopicSchema = extractionTopicSchema.extend({ - metadata: z.object({ source: z.string() }).optional(), -}); - -const handbookRelationSchema = z.object({ - source: z.string().min(1), - target: z.string().min(1), - type: relationTypeLoose, - description: z.string().optional(), -}); - -export const handbookResultSchema = z.object({ - topics: z.array(handbookTopicSchema), - relations: z.array(handbookRelationSchema), -}); - -/** - * Normalise legacy `executes` relations into the canonical `executed_by` - * vocabulary by swapping source and target. The handbook prompt previously - * emitted `role → executes → process`; the canonical form is - * `process → executed_by → role`. - */ -export function normalizeHandbookResult(parsed) { - return { - ...parsed, - relations: parsed.relations.map((r) => - r.type === 'executes' - ? { ...r, type: 'executed_by', source: r.target, target: r.source } - : r, - ), - }; -} const articleSectionSchema = z.object({ heading: z.string().min(1), @@ -218,7 +185,6 @@ export const replaceTakeawaysPatchSchema = z.object({ */ export const toolSchemaRegistry = { emit_knowledge_graph: extractionResultSchema, - emit_handbook_delta: handbookResultSchema, emit_learning_article: learningArticleSchema, emit_learning_slides: learningSlidesSchema, emit_learning_infographic: learningInfographicSchema, diff --git a/src/lib/llmTools.js b/src/lib/llmTools.js index 000e9a5..b7f03db 100644 --- a/src/lib/llmTools.js +++ b/src/lib/llmTools.js @@ -10,7 +10,6 @@ const TOPIC_TYPES = ['concept', 'role', 'process']; const LEARNING_RELEVANCE = ['core', 'standard', 'peripheral', 'exclude']; const RELATION_TYPES_STRICT = ['related_to', 'depends_on', 'part_of', 'executed_by']; -const RELATION_TYPES_LOOSE = ['related_to', 'depends_on', 'part_of', 'executed_by', 'executes']; const extractionTopicSchema = { type: 'object', @@ -47,41 +46,6 @@ export const EMIT_KNOWLEDGE_GRAPH_TOOL = { }, }; -const handbookTopicSchema = { - type: 'object', - properties: { - ...extractionTopicSchema.properties, - metadata: { - type: 'object', - properties: { source: { type: 'string' } }, - }, - }, - required: extractionTopicSchema.required, -}; - -const handbookRelationSchema = { - type: 'object', - properties: { - source: { type: 'string' }, - target: { type: 'string' }, - type: { type: 'string', enum: RELATION_TYPES_LOOSE }, - description: { type: 'string' }, - }, - required: ['source', 'target', 'type'], -}; - -export const EMIT_HANDBOOK_DELTA_TOOL = { - name: 'emit_handbook_delta', - description: 'Return the topics and relations extracted from a handbook file update. Every process must have a role attached; every concept must connect to a process or role.', - input_schema: { - type: 'object', - properties: { - topics: { type: 'array', items: handbookTopicSchema }, - relations: { type: 'array', items: handbookRelationSchema }, - }, - required: ['topics', 'relations'], - }, -}; const articleSectionSchema = { type: 'object', diff --git a/src/pages/Admin/index.jsx b/src/pages/Admin/index.jsx index fbfcf5e..8ddb76a 100644 --- a/src/pages/Admin/index.jsx +++ b/src/pages/Admin/index.jsx @@ -31,7 +31,6 @@ const Admin = () => { const [saveStatus, setSaveStatus] = useState(null); const [resetConfirmText, setResetConfirmText] = useState(''); - const [resetIncludeHandbook, setResetIncludeHandbook] = useState(true); const [resetIncludeProgress, setResetIncludeProgress] = useState(false); const [isResetting, setIsResetting] = useState(false); const [resetReport, setResetReport] = useState(null); @@ -69,7 +68,6 @@ const Admin = () => { setResetReport(null); try { const report = await db.resetForSmokeTest({ - includeHandbookState: resetIncludeHandbook, includeProgress: resetIncludeProgress, }); setResetReport(report); @@ -317,15 +315,6 @@ const Admin = () => {
-