rve/learning-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: remove handbook sync state and related functionality
All checks were successful
On Push to Main / test (push) Successful in 31s
Details
On Push to Main / publish (push) Successful in 1m4s
Details
On Push to Main / deploy-dev (push) Successful in 1m31s
Details

Browse Source
This commit is contained in:
RaymondVerhoef
2026-05-22 19:45:14 +02:00
parent dc628644b6
commit ca8945ea5b
12 changed files with 159 additions and 552 deletions
Download Patch File Download Diff File Expand all files Collapse all files

90
pb_migrations/1780000000_created_handbook_sync_state.js
View File

@@ -1,90 +0,0 @@
/// <reference path="../pb_data/types.d.ts" />
migrate((app) => {
const collection = new Collection({
"createRule": "",
"deleteRule": "",
"fields": [
{
"autogeneratePattern": "[a-z0-9]{15}",
"help": "",
"hidden": false,
"id": "text3208210256",
"max": 15,
"min": 15,
"name": "id",
"pattern": "^[a-z0-9]+$",
"presentable": false,
"primaryKey": true,
"required": true,
"system": true,
"type": "text"
},
{
"autogeneratePattern": "",
"help": "",
"hidden": false,
"id": "text_path123",
"max": 0,
"min": 0,
"name": "path",
"pattern": "",
"presentable": false,
"primaryKey": false,
"required": true,
"system": false,
"type": "text"
},
{
"autogeneratePattern": "",
"help": "",
"hidden": false,
"id": "text_sha123",
"max": 0,
"min": 0,
"name": "sha",
"pattern": "",
"presentable": false,
"primaryKey": false,
"required": true,
"system": false,
"type": "text"
},
{
"hidden": false,
"id": "autodate2990389176",
"name": "created",
"onCreate": true,
"onUpdate": false,
"presentable": false,
"system": false,
"type": "autodate"
},
{
"hidden": false,
"id": "autodate3332085495",
"name": "updated",
"onCreate": true,
"onUpdate": true,
"presentable": false,
"system": false,
"type": "autodate"
}
],
"id": "pbc_handbooksync123",
"indexes": [
"CREATE UNIQUE INDEX `idx_handbook_path` ON `handbook_sync_state` (`path`)"
],
"listRule": "",
"name": "handbook_sync_state",
"system": false,
"type": "base",
"updateRule": "",
"viewRule": ""
});
return app.save(collection);
}, (app) => {
const collection = app.findCollectionByNameOrId("pbc_handbooksync123");
return app.delete(collection);
})

124
src/components/admin/KnowledgeGraph.jsx
View File

@@ -4,8 +4,6 @@ import { Trash2, Edit2, Save, X, RefreshCw, AlertCircle, Plus, Link as LinkIcon
import * as db from '../../lib/db'; import * as db from '../../lib/db';
import { callLLM } from '../../lib/llm'; import { callLLM } from '../../lib/llm';
import { EMIT_GRAPH_ACTIONS_TOOL } from '../../lib/llmTools'; import { EMIT_GRAPH_ACTIONS_TOOL } from '../../lib/llmTools';
import { analyzeHandbookDelta } from '../../lib/extractionPipeline';
import { getRepoFolder, getFileContent } from '../../lib/githubService';
import Button from '../ui/Button'; import Button from '../ui/Button';
import SuggestionsQueue from './SuggestionsQueue'; import SuggestionsQueue from './SuggestionsQueue';
@@ -20,10 +18,6 @@ const KnowledgeGraph = () => {
const [analyzeError, setAnalyzeError] = useState(null); const [analyzeError, setAnalyzeError] = useState(null);
const [newRelData, setNewRelData] = useState({ target: '', type: 'related_to' }); const [newRelData, setNewRelData] = useState({ target: '', type: 'related_to' });
const [isSyncing, setIsSyncing] = useState(false);
const [syncResult, setSyncResult] = useState(null);
const [syncError, setSyncError] = useState(null);
const [syncProgress, setSyncProgress] = useState(null);
const [topics, setTopics] = useState([]); const [topics, setTopics] = useState([]);
const [relations, setRelations] = useState([]); const [relations, setRelations] = useState([]);
@@ -237,87 +231,6 @@ const KnowledgeGraph = () => {
))); )));
}; };
const syncHandbook = async () => {
setIsSyncing(true);
setSyncError(null);
setSyncResult(null);
try {
const files = await getRepoFolder('respellion', 'employee-handbook', 'docs');
const syncStates = await db.getHandbookSyncStates();
const added = [];
const modified = [];
const unchanged = [];
files.forEach(file => {
const state = syncStates.find(s => s.path === file.path);
if (!state) {
added.push(file);
} else if (state.sha !== file.sha) {
modified.push(file);
} else {
unchanged.push(file);
}
});
setSyncResult({ added, modified, unchanged, failed: [] });
const filesToProcess = [...added, ...modified];
if (filesToProcess.length > 0) {
const total = filesToProcess.length;
let done = 0;
const failed = [];
setSyncProgress(`Processing 0 of ${total} files...`);
// Run files in parallel with bounded concurrency. The extractionLimiter
// inside analyzeHandbookDelta still governs the actual API request rate;
// parallelism here just hides GitHub fetch latency and overlaps with the
// limiter's spacing instead of waiting serially.
const CONCURRENCY = 4;
const queue = filesToProcess.slice();
async function worker() {
while (queue.length > 0) {
const file = queue.shift();
if (!file) return;
try {
const rawContent = await getFileContent('respellion', 'employee-handbook', file.path);
// Skip near-empty files — they only burn an LLM call to extract nothing.
if (rawContent.trim().length < 50) {
await db.updateHandbookSyncState(file.path, file.sha);
} else {
await analyzeHandbookDelta(rawContent, file.path);
await db.updateHandbookSyncState(file.path, file.sha);
}
} catch (err) {
console.error('Failed to process file:', file.path, err);
failed.push({ path: file.path, message: err?.message || String(err) });
} finally {
done++;
setSyncProgress(`Processing ${done} of ${total} files (${failed.length} failed)...`);
}
}
}
await Promise.all(Array.from({ length: Math.min(CONCURRENCY, total) }, worker));
setSyncResult({ added, modified, unchanged, failed });
setSyncProgress(
failed.length === 0
? 'Sync Complete! Click "Analyze & Optimize Graph" above to clean up and merge.'
: `Sync finished with ${failed.length} failure(s). See console for details.`
);
reloadKb();
}
} catch (e) {
setSyncError(e.message || 'Failed to check GitHub for updates.');
} finally {
setIsSyncing(false);
setTimeout(() => setSyncProgress(null), 10000); // Clear progress after 10s
}
};
const analyzeGraph = async () => { const analyzeGraph = async () => {
if (!confirm('This will send the entire graph to the AI to evaluate content, merge duplicates, and create new logical relations. This may take a moment. Proceed?')) return; if (!confirm('This will send the entire graph to the AI to evaluate content, merge duplicates, and create new logical relations. This may take a moment. Proceed?')) return;
@@ -466,43 +379,6 @@ ${JSON.stringify({ topics: compactTopics, relations: compactRelations })}`;
</p> </p>
)} )}
</div> </div>
<div>
<Button
onClick={syncHandbook}
disabled={isSyncing}
variant="outline"
className="w-full flex justify-center items-center gap-2"
>
<RefreshCw size={16} className={isSyncing ? "animate-spin" : ""} />
{isSyncing ? 'Checking for updates...' : 'Sync Employee Handbook'}
</Button>
{syncError && (
<p className="mt-2 text-xs text-red-600 flex items-start gap-1">
<AlertCircle size={14} className="shrink-0 mt-0.5" />
{syncError}
</p>
)}
{syncResult && (
<div className="mt-2 text-xs text-fg-muted space-y-1 p-2 bg-bg rounded">
<p className="font-medium text-fg">GitHub Sync Status:</p>
<p>Added files: {syncResult.added.length}</p>
<p>Modified files: {syncResult.modified.length}</p>
<p>Unchanged: {syncResult.unchanged.length}</p>
{syncResult.failed?.length > 0 && (
<p className="text-red-600">Failed: {syncResult.failed.length}</p>
)}
</div>
)}
{syncProgress && (
<div className="mt-2 text-xs text-teal p-2 bg-teal/10 border border-teal/20 rounded">
<p className="font-medium flex items-center gap-2">
{syncProgress !== 'Sync Complete!' && <RefreshCw size={12} className="animate-spin" />}
{syncProgress}
</p>
</div>
)}
</div>
</div> </div>
{/* R42 chatbot suggestions queue */} {/* R42 chatbot suggestions queue */}

187
src/components/admin/UploadZone.jsx
View File

@@ -1,5 +1,5 @@
import { useState, useRef } from 'react'; import { useEffect, useRef, useState } from 'react';
import { UploadCloud, AlertCircle, CheckCircle, X } from 'lucide-react'; import { UploadCloud, CheckCircle, AlertCircle, Loader2, X, Clock, Minus } from 'lucide-react';
import { processSourceText } from '../../lib/extractionPipeline'; import { processSourceText } from '../../lib/extractionPipeline';
import Card from '../ui/Card'; import Card from '../ui/Card';
import Button from '../ui/Button'; import Button from '../ui/Button';
@@ -8,121 +8,186 @@ import Button from '../ui/Button';
const UploadZone = ({ onUploadComplete }) => { const UploadZone = ({ onUploadComplete }) => {
const [isDragging, setIsDragging] = useState(false); const [isDragging, setIsDragging] = useState(false);
const [isProcessing, setIsProcessing] = useState(false); const [queue, setQueue] = useState([]);
const [status, setStatus] = useState(null); const [processingId, setProcessingId] = useState(null);
const [rejectedNote, setRejectedNote] = useState(null);
const fileInputRef = useRef(null); const fileInputRef = useRef(null);
const abortRef = useRef(null); const abortRef = useRef(null);
// ── File upload (drag & drop / browse) ──────────────────────────────────── // ── Processing loop ────────────────────────────────────────────────────────
useEffect(() => {
if (processingId !== null) return;
const next = queue.find((q) => q.status === 'pending');
if (!next) return;
setProcessingId(next.id);
setQueue((q) => q.map((item) => item.id === next.id ? { ...item, status: 'processing' } : item));
const processFile = async (file) => {
setIsProcessing(true);
setStatus(null);
const controller = new AbortController(); const controller = new AbortController();
abortRef.current = controller; abortRef.current = controller;
try {
const text = await file.text(); next.file.text()
await processSourceText(text, file.name, { signal: controller.signal }); .then((text) => processSourceText(text, next.name, { signal: controller.signal }))
setStatus({ type: 'success', msg: `Successfully processed: ${file.name}` }); .then(() => {
setQueue((q) => q.map((item) => item.id === next.id ? { ...item, status: 'done' } : item));
if (onUploadComplete) onUploadComplete(); if (onUploadComplete) onUploadComplete();
} catch (error) { })
if (error?.name === 'AbortError') { .catch((err) => {
setStatus({ type: 'error', msg: `Cancelled: ${file.name}` }); const isCancelled = err?.name === 'AbortError';
} else { setQueue((q) => q.map((item) =>
setStatus({ type: 'error', msg: `Error processing file: ${error.message}` }); item.id === next.id
} ? { ...item, status: isCancelled ? 'cancelled' : 'failed', error: isCancelled ? null : err.message }
} finally { : item
));
})
.finally(() => {
abortRef.current = null; abortRef.current = null;
setIsProcessing(false); setProcessingId(null);
});
}, [processingId, queue, onUploadComplete]);
// ── File ingestion ─────────────────────────────────────────────────────────
const addFiles = (fileList) => {
const accepted = [];
let rejected = 0;
for (const file of fileList) {
if (file.type === 'text/plain' || file.name.endsWith('.md')) {
accepted.push({ id: crypto.randomUUID(), file, name: file.name, status: 'pending', error: null });
} else {
rejected++;
}
}
if (accepted.length > 0) setQueue((q) => [...q, ...accepted]);
if (rejected > 0) {
setRejectedNote(`${rejected} file${rejected > 1 ? 's' : ''} skipped — only .txt and .md are supported.`);
setTimeout(() => setRejectedNote(null), 4000);
} }
}; };
const cancelProcessing = () => { const cancelCurrent = () => {
abortRef.current?.abort(new DOMException('Cancelled by user', 'AbortError')); abortRef.current?.abort(new DOMException('Cancelled by user', 'AbortError'));
}; };
const clearDone = () => {
setQueue((q) => q.filter((item) => item.status === 'pending' || item.status === 'processing'));
};
// ── Drag & drop / browse ───────────────────────────────────────────────────
const handleDragOver = (e) => { e.preventDefault(); setIsDragging(true); }; const handleDragOver = (e) => { e.preventDefault(); setIsDragging(true); };
const handleDragLeave = () => setIsDragging(false); const handleDragLeave = () => setIsDragging(false);
const handleDrop = (e) => { const handleDrop = (e) => {
e.preventDefault(); e.preventDefault();
setIsDragging(false); setIsDragging(false);
if (e.dataTransfer.files?.length > 0) { if (e.dataTransfer.files?.length > 0) addFiles(e.dataTransfer.files);
const file = e.dataTransfer.files[0];
if (file.type === 'text/plain' || file.name.endsWith('.md')) {
processFile(file);
} else {
setStatus({ type: 'error', msg: 'Only .txt and .md files are currently supported.' });
}
}
}; };
const handleFileSelect = (e) => { const handleFileSelect = (e) => {
if (e.target.files?.length > 0) processFile(e.target.files[0]); if (e.target.files?.length > 0) {
addFiles(e.target.files);
e.target.value = '';
}
}; };
// ── Derived state ──────────────────────────────────────────────────────────
const hasDone = queue.some((q) => ['done', 'failed', 'cancelled'].includes(q.status));
// ── Render ───────────────────────────────────────────────────────────────── // ── Render ─────────────────────────────────────────────────────────────────
return ( return (
<div className="space-y-6"> <div className="space-y-4">
{/* ─── Drag & Drop Upload ─── */} {/* ─── Drag & Drop Zone ─── */}
<Card <Card
className={`border-2 border-dashed transition-colors flex flex-col items-center justify-center py-12 cursor-pointer ${ className={`border-2 border-dashed transition-colors flex flex-col items-center justify-center py-10 cursor-pointer ${
isDragging ? 'border-teal bg-teal/5' : 'border-bg-warm bg-bg-warm/20' isDragging ? 'border-teal bg-teal/5' : 'border-bg-warm bg-bg-warm/20'
} ${isProcessing ? 'opacity-50 pointer-events-none' : ''}`} }`}
onDragOver={handleDragOver} onDragOver={handleDragOver}
onDragLeave={handleDragLeave} onDragLeave={handleDragLeave}
onDrop={handleDrop} onDrop={handleDrop}
onClick={() => fileInputRef.current?.click()} onClick={() => fileInputRef.current?.click()}
> >
<UploadCloud size={48} className="text-teal/40 mb-4" /> <UploadCloud size={40} className="text-teal/40 mb-3" />
<p className="font-medium text-lg">Drag files here</p> <p className="font-medium">Drag files here</p>
<p className="text-sm text-fg-muted mb-4">Supports .txt and .md (max 5MB)</p> <p className="text-sm text-fg-muted mb-4">Supports .txt and .md · max 5 MB each</p>
<Button variant="outline" type="button" disabled={isProcessing}> <Button variant="outline" type="button">Browse files</Button>
{isProcessing ? 'AI extraction in progress...' : 'Browse files'}
</Button>
<input <input
type="file" type="file"
ref={fileInputRef} ref={fileInputRef}
onChange={handleFileSelect} onChange={handleFileSelect}
accept=".txt,.md" accept=".txt,.md"
multiple
className="hidden" className="hidden"
/> />
</Card> </Card>
{/* Cancel sits outside the upload card so pointer-events-none doesn't disable it. */} {/* ─── Rejected file note ─── */}
{isProcessing && ( {rejectedNote && (
<div className="flex justify-center"> <p className="text-xs text-amber-600">{rejectedNote}</p>
)}
{/* ─── Queue ─── */}
{queue.length > 0 && (
<div className="space-y-1">
{queue.map((item) => (
<div
key={item.id}
className="flex items-center gap-3 px-3 py-2 rounded-[var(--r-sm)] bg-bg-warm/30 text-sm"
>
<StatusIcon status={item.status} />
<span className="flex-1 truncate text-fg">{item.name}</span>
<span className="text-xs text-fg-muted shrink-0">
{item.status === 'processing' && 'Extracting…'}
{item.status === 'pending' && 'Pending'}
{item.status === 'done' && 'Done'}
{item.status === 'cancelled' && 'Cancelled'}
{item.status === 'failed' && (
<span className="text-red-500" title={item.error}>Failed</span>
)}
</span>
{item.status === 'processing' && (
<button <button
type="button" type="button"
onClick={cancelProcessing} onClick={cancelCurrent}
className="flex items-center gap-1 text-sm text-red-600 hover:text-red-700 underline" className="text-red-500 hover:text-red-600 shrink-0"
title="Cancel"
> >
<X size={14} /> Cancel extraction <X size={14} />
</button>
)}
</div>
))}
{hasDone && (
<div className="flex justify-end pt-1">
<button
type="button"
onClick={clearDone}
className="text-xs text-fg-muted hover:text-fg underline"
>
Clear done / failed
</button> </button>
</div> </div>
)} )}
{/* ─── Status messages ─── */}
{status && (
<div className={`p-4 rounded-[var(--r-sm)] flex items-start gap-3 ${
status.type === 'error'
? 'bg-red-50 text-red-900 border border-red-200'
: 'bg-teal-50 text-teal-900 border border-teal-200'
}`}>
{status.type === 'error'
? <AlertCircle className="text-red-500 flex-shrink-0" />
: <CheckCircle className="text-teal-600 flex-shrink-0" />}
<div>
<p className="font-medium">{status.type === 'error' ? 'Error' : 'Success'}</p>
<p className="text-sm">{status.msg}</p>
</div>
</div> </div>
)} )}
</div> </div>
); );
}; };
// ── Status icon ───────────────────────────────────────────────────────────────
const StatusIcon = ({ status }) => {
if (status === 'processing') return <Loader2 size={14} className="text-teal animate-spin shrink-0" />;
if (status === 'done') return <CheckCircle size={14} className="text-teal shrink-0" />;
if (status === 'failed') return <AlertCircle size={14} className="text-red-500 shrink-0" />;
if (status === 'cancelled') return <Minus size={14} className="text-fg-muted shrink-0" />;
return <Clock size={14} className="text-fg-muted shrink-0" />;
};
export default UploadZone; export default UploadZone;

21
src/lib/__tests__/extractionPipeline.test.js
View File

@@ -69,18 +69,21 @@ describe('buildKnownIdsHint', () => {
expect(buildKnownIdsHint(null)).toBe(''); expect(buildKnownIdsHint(null)).toBe('');
}); });
it('formats the known IDs as a bulleted list with a leading instruction', () => { it('formats topics as id: "label" pairs with a leading instruction', () => {
const hint = buildKnownIdsHint(['software-engineer', 'onboarding-buddy']); const hint = buildKnownIdsHint([
expect(hint).toContain('Already-extracted topic IDs'); { id: 'software-engineer', label: 'Software Engineer' },
expect(hint).toContain('- software-engineer'); { id: 'onboarding-buddy', label: 'Onboarding Buddy' },
expect(hint).toContain('- onboarding-buddy'); ]);
expect(hint).toContain('Already-extracted topics');
expect(hint).toContain('- software-engineer: "Software Engineer"');
expect(hint).toContain('- onboarding-buddy: "Onboarding Buddy"');
expect(hint.endsWith('\n')).toBe(true); expect(hint.endsWith('\n')).toBe(true);
}); });
it('caps the hint at the 200 most recent IDs', () => { it('caps the hint at the 200 most recent topics', () => {
const ids = Array.from({ length: 250 }, (_, i) => `topic-${i}`); const topics = Array.from({ length: 250 }, (_, i) => ({ id: `topic-${i}`, label: `Topic ${i}` }));
const hint = buildKnownIdsHint(ids); const hint = buildKnownIdsHint(topics);
// The newest IDs must appear; the oldest must not. // The newest topics must appear; the oldest must not.
expect(hint).toContain('topic-249'); expect(hint).toContain('topic-249');
expect(hint).toContain('topic-50'); expect(hint).toContain('topic-50');
expect(hint).not.toContain('topic-49'); expect(hint).not.toContain('topic-49');

24
src/lib/__tests__/llmSchemas.test.js
View File

@@ -1,8 +1,6 @@
import { describe, expect, it } from 'vitest'; import { describe, expect, it } from 'vitest';
import { import {
extractionResultSchema, extractionResultSchema,
handbookResultSchema,
normalizeHandbookResult,
learningArticleSchema, learningArticleSchema,
learningSlidesSchema, learningSlidesSchema,
learningInfographicSchema, learningInfographicSchema,
@@ -60,28 +58,6 @@ describe('extractionResultSchema', () => {
}); });
}); });
describe('handbookResultSchema', () => {
it('accepts the loose vocabulary including executes', () => {
const parsed = handbookResultSchema.parse({
topics: [{ ...sampleTopic, metadata: { source: 'github_handbook' } }],
relations: [{ source: 'software-engineer', target: 'code-review', type: 'executes' }],
});
expect(parsed.relations[0].type).toBe('executes');
});
it('normalises executes into executed_by with swapped source/target', () => {
const parsed = handbookResultSchema.parse({
topics: [sampleTopic],
relations: [{ source: 'software-engineer', target: 'code-review', type: 'executes' }],
});
const normalised = normalizeHandbookResult(parsed);
expect(normalised.relations[0]).toMatchObject({
source: 'code-review',
target: 'software-engineer',
type: 'executed_by',
});
});
});
describe('learning schemas', () => { describe('learning schemas', () => {
it('accepts an article payload', () => { it('accepts an article payload', () => {

2
src/lib/__tests__/llmTools.test.js
View File

@@ -1,7 +1,6 @@
import { describe, expect, it } from 'vitest'; import { describe, expect, it } from 'vitest';
import { import {
EMIT_KNOWLEDGE_GRAPH_TOOL, EMIT_KNOWLEDGE_GRAPH_TOOL,
EMIT_HANDBOOK_DELTA_TOOL,
EMIT_LEARNING_ARTICLE_TOOL, EMIT_LEARNING_ARTICLE_TOOL,
EMIT_LEARNING_SLIDES_TOOL, EMIT_LEARNING_SLIDES_TOOL,
EMIT_LEARNING_INFOGRAPHIC_TOOL, EMIT_LEARNING_INFOGRAPHIC_TOOL,
@@ -15,7 +14,6 @@ import { toolSchemaRegistry } from '../llmSchemas';
const allTools = [ const allTools = [
EMIT_KNOWLEDGE_GRAPH_TOOL, EMIT_KNOWLEDGE_GRAPH_TOOL,
EMIT_HANDBOOK_DELTA_TOOL,
EMIT_LEARNING_ARTICLE_TOOL, EMIT_LEARNING_ARTICLE_TOOL,
EMIT_LEARNING_SLIDES_TOOL, EMIT_LEARNING_SLIDES_TOOL,
EMIT_LEARNING_INFOGRAPHIC_TOOL, EMIT_LEARNING_INFOGRAPHIC_TOOL,

20
src/lib/db.js
View File

@@ -349,11 +349,10 @@ export async function getRecentLlmCalls(limit = 100) {
* *
* `team_members`, `settings`, and `llm_calls` are intentionally preserved. * `team_members`, `settings`, and `llm_calls` are intentionally preserved.
* *
* @param {{ includeHandbookState?: boolean, includeProgress?: boolean }} [opts] * @param {{ includeProgress?: boolean }} [opts]
* @returns {Promise<Array<{collection: string, deleted: number, error?: string}>>} * @returns {Promise<Array<{collection: string, deleted: number, error?: string}>>}
*/ */
export async function resetForSmokeTest({ export async function resetForSmokeTest({
includeHandbookState = true,
includeProgress = false, includeProgress = false,
} = {}) { } = {}) {
const collections = [ const collections = [
@@ -366,7 +365,6 @@ export async function resetForSmokeTest({
'sources', 'sources',
'topics', 'topics',
]; ];
if (includeHandbookState) collections.push('handbook_sync_state');
if (includeProgress) collections.push('learn_progress', 'leaderboard'); if (includeProgress) collections.push('learn_progress', 'leaderboard');
const results = []; const results = [];
@@ -388,20 +386,4 @@ export async function resetForSmokeTest({
return results; return results;
} }
// ── Handbook Sync State ───────────────────────────────────────────────────────
export async function getHandbookSyncStates() {
try {
return await pb.collection('handbook_sync_state').getFullList();
} catch { return []; }
}
export async function updateHandbookSyncState(path, sha) {
try {
const r = await pb.collection('handbook_sync_state').getFirstListItem(`path="${path}"`);
return await pb.collection('handbook_sync_state').update(r.id, { sha });
} catch {
return await pb.collection('handbook_sync_state').create({ path, sha });
}
}

64
src/lib/extractionPipeline.js
View File

@@ -1,24 +1,21 @@
import * as db from './db'; import * as db from './db';
import { callLLM } from './llm'; import { callLLM } from './llm';
import { extractionLimiter } from './llmRetry'; import { extractionLimiter } from './llmRetry';
import { EMIT_KNOWLEDGE_GRAPH_TOOL, EMIT_HANDBOOK_DELTA_TOOL } from './llmTools'; import { EMIT_KNOWLEDGE_GRAPH_TOOL } from './llmTools';
import { normalizeHandbookResult } from './llmSchemas';
const MAX_KNOWN_IDS_HINT = 200; const MAX_KNOWN_TOPICS_HINT = 200;
/** /**
* Build the "already-extracted topic IDs" hint that prepends every chunk * Build the "already-extracted topics" hint included in every chunk prompt.
* after the first. Capped at the most-recent `MAX_KNOWN_IDS_HINT` IDs so * Passes both ID and label so the model can match concepts by name and reuse
* the prompt stays a bounded size; the model uses this list to reuse IDs * the exact ID + label rather than inventing near-duplicate variants.
* rather than invent variants like `software-developer` for
* `software-engineer`.
*/ */
export function buildKnownIdsHint(ids) { export function buildKnownIdsHint(topics) {
if (!ids || !ids.length) return ''; if (!topics || !topics.length) return '';
const recent = ids.slice(-MAX_KNOWN_IDS_HINT); const recent = topics.slice(-MAX_KNOWN_TOPICS_HINT);
return [ return [
'Already-extracted topic IDs (do NOT create new IDs for these — reuse them if the same concept appears here):', 'Already-extracted topics (reuse their ID and label exactly if the same concept appears here):',
...recent.map((id) => `- ${id}`), ...recent.map((t) => `- ${t.id}: "${t.label}"`),
'', '',
].join('\n'); ].join('\n');
} }
@@ -45,41 +42,8 @@ Assign a learning_relevance to every topic:
Relation types: related_to | depends_on | part_of | executed_by. Relation types: related_to | depends_on | part_of | executed_by.
`; `;
const HANDBOOK_SYSTEM_PROMPT = `You are analysing an update to the Respellion Employee Handbook. Emit the extracted topics and relations through the emit_handbook_delta tool.
CRITICAL INSTRUCTIONS:
- Every process must have a role attached. Express this as: process --executed_by--> role.
- Every concept must connect to a process or role.
- Mark handbook topics with metadata.source = "github_handbook".
- Assign learning_relevance using the same scale as extraction: core | standard | peripheral | exclude.
Relation types: related_to | depends_on | part_of | executed_by.
`;
const cachedSystem = (text) => [{ type: 'text', text, cache_control: { type: 'ephemeral' } }]; const cachedSystem = (text) => [{ type: 'text', text, cache_control: { type: 'ephemeral' } }];
export async function analyzeHandbookDelta(fileContent, filePath, { signal } = {}) {
const result = await callLLM({
task: 'extract.handbook',
tier: 'standard',
system: cachedSystem(HANDBOOK_SYSTEM_PROMPT),
user: `Analyze the following handbook file update (${filePath}):\n\n${fileContent}`,
tools: [EMIT_HANDBOOK_DELTA_TOOL],
toolChoice: { type: 'tool', name: EMIT_HANDBOOK_DELTA_TOOL.name },
maxTokens: 8192,
timeoutMs: 180_000,
limiter: extractionLimiter,
signal,
});
const raw = result.toolUses[0]?.input;
if (!raw) throw new Error('Handbook extraction did not emit a tool result.');
const extractedData = normalizeHandbookResult(raw);
await mergeKnowledgeGraph(extractedData);
return { success: true, data: extractedData };
}
/** /**
* Sentence-aware chunker with overlap. * Sentence-aware chunker with overlap.
* *
@@ -169,7 +133,7 @@ export async function processSourceText(textContent, sourceName, { signal } = {}
console.log(`[Pipeline] Split "${sourceName}" into ${chunks.length} chunks for processing.`); console.log(`[Pipeline] Split "${sourceName}" into ${chunks.length} chunks for processing.`);
const existingTopics = await db.getTopics(); const existingTopics = await db.getTopics();
const knownIds = existingTopics.map((t) => t.id); const knownTopics = existingTopics.map((t) => ({ id: t.id, label: t.label }));
const allExtractedTopics = []; const allExtractedTopics = [];
const allExtractedRelations = []; const allExtractedRelations = [];
@@ -178,7 +142,7 @@ export async function processSourceText(textContent, sourceName, { signal } = {}
if (signal?.aborted) throw signal.reason ?? new DOMException('Aborted', 'AbortError'); if (signal?.aborted) throw signal.reason ?? new DOMException('Aborted', 'AbortError');
console.log(`[Pipeline] Processing chunk ${i + 1}/${chunks.length} (${chunks[i].length} chars)...`); console.log(`[Pipeline] Processing chunk ${i + 1}/${chunks.length} (${chunks[i].length} chars)...`);
const hint = i > 0 ? buildKnownIdsHint(knownIds) : ''; const hint = buildKnownIdsHint(knownTopics);
const result = await callLLM({ const result = await callLLM({
task: 'extract.source', task: 'extract.source',
tier: 'standard', tier: 'standard',
@@ -197,7 +161,9 @@ export async function processSourceText(textContent, sourceName, { signal } = {}
if (Array.isArray(extractedData.topics)) { if (Array.isArray(extractedData.topics)) {
allExtractedTopics.push(...extractedData.topics); allExtractedTopics.push(...extractedData.topics);
for (const t of extractedData.topics) { for (const t of extractedData.topics) {
if (t?.id && !knownIds.includes(t.id)) knownIds.push(t.id); if (t?.id && !knownTopics.some((k) => k.id === t.id)) {
knownTopics.push({ id: t.id, label: t.label });
}
} }
} }
if (Array.isArray(extractedData.relations)) { if (Array.isArray(extractedData.relations)) {

88
src/lib/githubService.js
View File

@@ -1,88 +0,0 @@
/**
* GitHub API Service
* Fetches files from a public GitHub repository directly from the browser.
* No proxy or token required for public repos.
*/
const GITHUB_API = 'https://api.github.com';
const GITHUB_RAW = 'https://raw.githubusercontent.com';
/**
* List markdown/text files in a GitHub repository folder.
* @param {string} owner - e.g. "respellion"
* @param {string} repo - e.g. "employee-handbook"
* @param {string} folder - e.g. "docs" or "" for root
* @param {string} branch - e.g. "main"
* @returns {Promise<Array<{name, sha, path}>>}
*/
export async function getRepoFolder(owner, repo, folder = '', branch = 'main') {
// Use the GitHub Git Trees API to fetch all files recursively in one request
const url = `${GITHUB_API}/repos/${owner}/${repo}/git/trees/${branch}?recursive=1`;
const res = await fetch(url, {
headers: { Accept: 'application/vnd.github+json' },
});
if (!res.ok) {
const err = await res.json().catch(() => ({}));
throw new Error(`GitHub API ${res.status}: ${err.message || 'Could not list folder'}`);
}
const data = await res.json();
// If folder is 'docs', we only want paths starting with 'docs/'
const folderPrefix = folder ? folder.replace(/^\/|\/$/g, '') + '/' : '';
// Return only files (blobs) that match the folder prefix and file extensions
return data.tree
.filter(item =>
item.type === 'blob' &&
item.path.startsWith(folderPrefix) &&
(item.path.endsWith('.md') || item.path.endsWith('.txt'))
)
.map(item => ({
name: item.path.split('/').pop(),
path: item.path,
sha: item.sha
}));
}
/**
* Fetch the raw text content of a file.
* Uses raw.githubusercontent.com — no rate limits for public content.
* @param {string} owner
* @param {string} repo
* @param {string} path - e.g. "docs/ROLES.md"
* @param {string} branch
* @returns {Promise<string>}
*/
export async function getFileContent(owner, repo, path, branch = 'main') {
const url = `${GITHUB_RAW}/${owner}/${repo}/${branch}/${path}`;
const res = await fetch(url);
if (!res.ok) {
throw new Error(`Could not fetch "${path}" from GitHub (${res.status})`);
}
return res.text();
}
/**
* Parse a GitHub tree URL into its parts.
* e.g. "https://github.com/respellion/employee-handbook/tree/main/docs"
* → { owner: 'respellion', repo: 'employee-handbook', branch: 'main', folder: 'docs' }
*/
export function parseGitHubUrl(url) {
try {
const match = url.match(/github\.com\/([^/]+)\/([^/]+)(?:\/tree\/([^/]+)(?:\/(.+))?)?/);
if (!match) return null;
return {
owner: match[1],
repo: match[2],
branch: match[3] || 'main',
folder: match[4] || '',
};
} catch {
return null;
}
}

34
src/lib/llmSchemas.js
View File

@@ -9,7 +9,6 @@ import { z } from 'zod';
const topicTypeEnum = z.enum(['concept', 'role', 'process']); const topicTypeEnum = z.enum(['concept', 'role', 'process']);
const relationTypeStrict = z.enum(['related_to', 'depends_on', 'part_of', 'executed_by']); const relationTypeStrict = z.enum(['related_to', 'depends_on', 'part_of', 'executed_by']);
const relationTypeLoose = z.enum(['related_to', 'depends_on', 'part_of', 'executed_by', 'executes']);
const learningRelevanceEnum = z.enum(['core', 'standard', 'peripheral', 'exclude']); const learningRelevanceEnum = z.enum(['core', 'standard', 'peripheral', 'exclude']);
const extractionTopicSchema = z.object({ const extractionTopicSchema = z.object({
@@ -31,38 +30,6 @@ export const extractionResultSchema = z.object({
relations: z.array(extractionRelationSchema), relations: z.array(extractionRelationSchema),
}); });
const handbookTopicSchema = extractionTopicSchema.extend({
metadata: z.object({ source: z.string() }).optional(),
});
const handbookRelationSchema = z.object({
source: z.string().min(1),
target: z.string().min(1),
type: relationTypeLoose,
description: z.string().optional(),
});
export const handbookResultSchema = z.object({
topics: z.array(handbookTopicSchema),
relations: z.array(handbookRelationSchema),
});
/**
* Normalise legacy `executes` relations into the canonical `executed_by`
* vocabulary by swapping source and target. The handbook prompt previously
* emitted `role → executes → process`; the canonical form is
* `process → executed_by → role`.
*/
export function normalizeHandbookResult(parsed) {
return {
...parsed,
relations: parsed.relations.map((r) =>
r.type === 'executes'
? { ...r, type: 'executed_by', source: r.target, target: r.source }
: r,
),
};
}
const articleSectionSchema = z.object({ const articleSectionSchema = z.object({
heading: z.string().min(1), heading: z.string().min(1),
@@ -218,7 +185,6 @@ export const replaceTakeawaysPatchSchema = z.object({
*/ */
export const toolSchemaRegistry = { export const toolSchemaRegistry = {
emit_knowledge_graph: extractionResultSchema, emit_knowledge_graph: extractionResultSchema,
emit_handbook_delta: handbookResultSchema,
emit_learning_article: learningArticleSchema, emit_learning_article: learningArticleSchema,
emit_learning_slides: learningSlidesSchema, emit_learning_slides: learningSlidesSchema,
emit_learning_infographic: learningInfographicSchema, emit_learning_infographic: learningInfographicSchema,

36
src/lib/llmTools.js
View File

@@ -10,7 +10,6 @@
const TOPIC_TYPES = ['concept', 'role', 'process']; const TOPIC_TYPES = ['concept', 'role', 'process'];
const LEARNING_RELEVANCE = ['core', 'standard', 'peripheral', 'exclude']; const LEARNING_RELEVANCE = ['core', 'standard', 'peripheral', 'exclude'];
const RELATION_TYPES_STRICT = ['related_to', 'depends_on', 'part_of', 'executed_by']; const RELATION_TYPES_STRICT = ['related_to', 'depends_on', 'part_of', 'executed_by'];
const RELATION_TYPES_LOOSE = ['related_to', 'depends_on', 'part_of', 'executed_by', 'executes'];
const extractionTopicSchema = { const extractionTopicSchema = {
type: 'object', type: 'object',
@@ -47,41 +46,6 @@ export const EMIT_KNOWLEDGE_GRAPH_TOOL = {
}, },
}; };
const handbookTopicSchema = {
type: 'object',
properties: {
...extractionTopicSchema.properties,
metadata: {
type: 'object',
properties: { source: { type: 'string' } },
},
},
required: extractionTopicSchema.required,
};
const handbookRelationSchema = {
type: 'object',
properties: {
source: { type: 'string' },
target: { type: 'string' },
type: { type: 'string', enum: RELATION_TYPES_LOOSE },
description: { type: 'string' },
},
required: ['source', 'target', 'type'],
};
export const EMIT_HANDBOOK_DELTA_TOOL = {
name: 'emit_handbook_delta',
description: 'Return the topics and relations extracted from a handbook file update. Every process must have a role attached; every concept must connect to a process or role.',
input_schema: {
type: 'object',
properties: {
topics: { type: 'array', items: handbookTopicSchema },
relations: { type: 'array', items: handbookRelationSchema },
},
required: ['topics', 'relations'],
},
};
const articleSectionSchema = { const articleSectionSchema = {
type: 'object', type: 'object',

11
src/pages/Admin/index.jsx
View File

@@ -31,7 +31,6 @@ const Admin = () => {
const [saveStatus, setSaveStatus] = useState(null); const [saveStatus, setSaveStatus] = useState(null);
const [resetConfirmText, setResetConfirmText] = useState(''); const [resetConfirmText, setResetConfirmText] = useState('');
const [resetIncludeHandbook, setResetIncludeHandbook] = useState(true);
const [resetIncludeProgress, setResetIncludeProgress] = useState(false); const [resetIncludeProgress, setResetIncludeProgress] = useState(false);
const [isResetting, setIsResetting] = useState(false); const [isResetting, setIsResetting] = useState(false);
const [resetReport, setResetReport] = useState(null); const [resetReport, setResetReport] = useState(null);
@@ -69,7 +68,6 @@ const Admin = () => {
setResetReport(null); setResetReport(null);
try { try {
const report = await db.resetForSmokeTest({ const report = await db.resetForSmokeTest({
includeHandbookState: resetIncludeHandbook,
includeProgress: resetIncludeProgress, includeProgress: resetIncludeProgress,
}); });
setResetReport(report); setResetReport(report);
@@ -317,15 +315,6 @@ const Admin = () => {
</div> </div>
<div className="space-y-2"> <div className="space-y-2">
<label className="flex items-center gap-2 text-sm cursor-pointer">
<input
type="checkbox"
checked={resetIncludeHandbook}
onChange={(e) => setResetIncludeHandbook(e.target.checked)}
className="rounded bg-bg-warm border-transparent focus:ring-0 text-teal"
/>
Also clear handbook sync state (so &quot;Sync Employee Handbook&quot; re-processes every file)
</label>
<label className="flex items-center gap-2 text-sm cursor-pointer"> <label className="flex items-center gap-2 text-sm cursor-pointer">
<input <input
type="checkbox" type="checkbox"