Add comprehensive documentation for employee learning platform
- Created handover document outlining design decisions and application functionality. - Developed implementation plan detailing phased approach for service development. - Specified ingestion service responsibilities, API surface, and processing pipeline.
This commit is contained in:
RaymondVerhoef
402
app/services/ingestion/src/migrations/001_initial_schema.ts
Normal file
402
app/services/ingestion/src/migrations/001_initial_schema.ts
Normal file
@@ -0,0 +1,402 @@
|
||||
import 'dotenv/config';
|
||||
import PocketBase, { type CollectionModel } from 'pocketbase';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Env validation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const POCKETBASE_URL = process.env['POCKETBASE_URL'] ?? '';
|
||||
const POCKETBASE_ADMIN_EMAIL = process.env['POCKETBASE_ADMIN_EMAIL'] ?? '';
|
||||
const POCKETBASE_ADMIN_PASSWORD = process.env['POCKETBASE_ADMIN_PASSWORD'] ?? '';
|
||||
|
||||
if (!POCKETBASE_URL || !POCKETBASE_ADMIN_EMAIL || !POCKETBASE_ADMIN_PASSWORD) {
|
||||
console.error('Missing env vars: POCKETBASE_URL, POCKETBASE_ADMIN_EMAIL, POCKETBASE_ADMIN_PASSWORD');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Field types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface FieldDef {
|
||||
name: string;
|
||||
type: string;
|
||||
required: boolean;
|
||||
options: Record<string, unknown>;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Field helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const field = {
|
||||
text: (name: string, required = false): FieldDef => ({
|
||||
name, type: 'text', required,
|
||||
options: { min: null, max: null, pattern: '' },
|
||||
}),
|
||||
|
||||
number: (name: string, required = false): FieldDef => ({
|
||||
name, type: 'number', required,
|
||||
options: { min: null, max: null, noDecimal: false },
|
||||
}),
|
||||
|
||||
select: (name: string, values: string[], required = false, maxSelect = 1): FieldDef => ({
|
||||
name, type: 'select', required,
|
||||
options: { maxSelect, values },
|
||||
}),
|
||||
|
||||
json: (name: string): FieldDef => ({
|
||||
name, type: 'json', required: false, options: {},
|
||||
}),
|
||||
|
||||
date: (name: string): FieldDef => ({
|
||||
name, type: 'date', required: false, options: { min: '', max: '' },
|
||||
}),
|
||||
|
||||
editor: (name: string): FieldDef => ({
|
||||
name, type: 'editor', required: false, options: {},
|
||||
}),
|
||||
|
||||
file: (name: string, mimeTypes: string[] = [], maxSelect = 1): FieldDef => ({
|
||||
name, type: 'file', required: false,
|
||||
options: { maxSelect, maxSize: 52428800, mimeTypes, thumbs: [], protected: false },
|
||||
}),
|
||||
|
||||
relation: (name: string, collectionId: string, maxSelect: number | null = 1, required = false): FieldDef => ({
|
||||
name, type: 'relation', required,
|
||||
options: { collectionId, cascadeDelete: false, minSelect: null, maxSelect, displayFields: null },
|
||||
}),
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Badge seed data (from data-model.md + handover.md)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const BADGES = [
|
||||
// Bronze — beginner milestones
|
||||
{ key: 'first_commit', tier: 'bronze', label: 'First Commit', description: 'Complete your first topic', icon: '🔰' },
|
||||
{ key: 'week_shipped', tier: 'bronze', label: 'Week Shipped', description: 'Complete your first full week', icon: '📦' },
|
||||
{ key: 'streak_3', tier: 'bronze', label: '3-Week Streak', description: 'Maintain a 3-week learning streak', icon: '🔥' },
|
||||
{ key: 'quiz_taker', tier: 'bronze', label: 'Quiz Taker', description: 'Complete your first scenario quiz', icon: '❓' },
|
||||
{ key: 'flashcard_fan', tier: 'bronze', label: 'Flashcard Fan', description: 'Complete your first flashcard set', icon: '🃏' },
|
||||
{ key: 'reached_junior', tier: 'bronze', label: 'Junior Dev', description: 'Reach Junior level', icon: '👶' },
|
||||
// Silver — intermediate
|
||||
{ key: 'streak_13', tier: 'silver', label: '13-Week Streak', description: 'Maintain a 13-week learning streak', icon: '💥' },
|
||||
{ key: 'half_cycle', tier: 'silver', label: 'Half Cycle', description: 'Complete week 13 — halfway through the cycle', icon: '🔄' },
|
||||
{ key: 'case_student', tier: 'silver', label: 'Case Student', description: 'Complete 5 case studies', icon: '📋' },
|
||||
{ key: 'reached_medior', tier: 'silver', label: 'Medior Dev', description: 'Reach Medior level', icon: '💼' },
|
||||
{ key: 'reached_senior', tier: 'silver', label: 'Senior Dev', description: 'Reach Senior level', icon: '🏅' },
|
||||
// Gold — advanced
|
||||
{ key: 'full_cycle', tier: 'gold', label: 'Full Cycle', description: 'Complete a full 26-week cycle', icon: '🏆' },
|
||||
{ key: 'type_explorer', tier: 'gold', label: 'Type Explorer', description: 'Use all 10 micro learning types at least once', icon: '🗺️' },
|
||||
{ key: 'reached_staff', tier: 'gold', label: 'Staff Eng', description: 'Reach Staff level', icon: '⭐' },
|
||||
// Legendary — exceptional
|
||||
{ key: 'streak_26', tier: 'legendary', label: 'Unbroken', description: 'Complete all 26 weeks without breaking a streak', icon: '⚡' },
|
||||
{ key: 'second_cycle', tier: 'legendary', label: 'Second Cycle', description: 'Complete two full cycles', icon: '🌀' },
|
||||
{ key: 'reached_principal', tier: 'legendary', label: 'Principal', description: 'Reach Principal level — maximum rank', icon: '👑' },
|
||||
// Content — domain mastery
|
||||
{ key: 'governance_nerd', tier: 'content', label: 'Governance Nerd', description: 'Complete all published governance topics', icon: '⚖️' },
|
||||
{ key: 'process_architect', tier: 'content', label: 'Process Architect', description: 'Complete all published process topics', icon: '🏗️' },
|
||||
{ key: 'deep_reader', tier: 'content', label: 'Deep Reader', description: 'Complete 50 or more unique topics', icon: '📚' },
|
||||
] as const;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function requireId(ids: Map<string, string>, name: string): string {
|
||||
const id = ids.get(name);
|
||||
if (id === undefined) throw new Error(`Collection ID not found: ${name}`);
|
||||
return id;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Migration
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function run(): Promise<void> {
|
||||
console.log('Connecting to PocketBase...');
|
||||
const pb = new PocketBase(POCKETBASE_URL);
|
||||
await pb.admins.authWithPassword(POCKETBASE_ADMIN_EMAIL, POCKETBASE_ADMIN_PASSWORD);
|
||||
console.log('Authenticated.\n');
|
||||
|
||||
// Snapshot of existing collections before we begin
|
||||
const existingCollections = await pb.collections.getFullList();
|
||||
const ids = new Map<string, string>();
|
||||
for (const col of existingCollections) {
|
||||
ids.set(col.name, col.id);
|
||||
}
|
||||
const existingNames = new Set(ids.keys());
|
||||
|
||||
// Create a collection only if it doesn't already exist
|
||||
async function ensureCollection(params: { name: string; type: string; schema: FieldDef[] }): Promise<void> {
|
||||
if (ids.has(params.name)) {
|
||||
console.log(` skip ${params.name}`);
|
||||
return;
|
||||
}
|
||||
const created = await pb.collections.create(params as Record<string, unknown>);
|
||||
ids.set(created.name, created.id);
|
||||
console.log(` create ${created.name}`);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Extend users (auth collection — already exists in PocketBase)
|
||||
// ---------------------------------------------------------------------------
|
||||
console.log('users:');
|
||||
const usersCol = await pb.collections.getFirstListItem<CollectionModel>('name="users"');
|
||||
ids.set('users', usersCol.id);
|
||||
|
||||
const hasRole = usersCol.schema.some(s => s.name === 'role');
|
||||
if (!hasRole) {
|
||||
const updateBody: Record<string, unknown> = {
|
||||
schema: [
|
||||
...usersCol.schema,
|
||||
field.select('role', ['admin', 'employee'], true),
|
||||
field.text('display_name'),
|
||||
field.file('avatar', ['image/jpeg', 'image/png', 'image/webp']),
|
||||
],
|
||||
};
|
||||
await pb.collections.update(usersCol.id, updateBody);
|
||||
console.log(' extended with role, display_name, avatar');
|
||||
} else {
|
||||
console.log(' skip users (already extended)');
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// source_documents
|
||||
// ---------------------------------------------------------------------------
|
||||
console.log('\ncollections:');
|
||||
await ensureCollection({
|
||||
name: 'source_documents',
|
||||
type: 'base',
|
||||
schema: [
|
||||
field.text('filename', true),
|
||||
field.file('file', ['application/pdf', 'text/markdown', 'text/x-markdown', 'text/plain']),
|
||||
field.select('format', ['pdf', 'md', 'txt'], true),
|
||||
field.select('status', ['processing', 'processed', 'failed'], true),
|
||||
field.date('ingested_at'),
|
||||
field.number('chunk_count'),
|
||||
field.relation('created_by', requireId(ids, 'users')),
|
||||
],
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// themes
|
||||
// ---------------------------------------------------------------------------
|
||||
await ensureCollection({
|
||||
name: 'themes',
|
||||
type: 'base',
|
||||
schema: [
|
||||
field.text('title', true),
|
||||
field.text('description'),
|
||||
field.select('status', ['draft', 'published'], true),
|
||||
field.relation('source_documents', requireId(ids, 'source_documents'), null),
|
||||
field.relation('approved_by', requireId(ids, 'users')),
|
||||
field.date('approved_at'),
|
||||
],
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// topics — create without self-referential fields first
|
||||
// ---------------------------------------------------------------------------
|
||||
const topicsBaseSchema: FieldDef[] = [
|
||||
field.relation('theme', requireId(ids, 'themes'), 1, true),
|
||||
field.text('title', true),
|
||||
field.editor('body'),
|
||||
field.select('difficulty', ['introductory', 'intermediate', 'advanced'], true),
|
||||
field.number('complexity_weight'),
|
||||
field.select('status', ['draft', 'published'], true),
|
||||
field.json('key_terms'),
|
||||
field.json('qdrant_chunk_ids'),
|
||||
];
|
||||
const topicsIsNew = !existingNames.has('topics');
|
||||
await ensureCollection({ name: 'topics', type: 'base', schema: topicsBaseSchema });
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// micro_learnings
|
||||
// ---------------------------------------------------------------------------
|
||||
await ensureCollection({
|
||||
name: 'micro_learnings',
|
||||
type: 'base',
|
||||
schema: [
|
||||
field.relation('topic', requireId(ids, 'topics'), 1, true),
|
||||
field.select('type', [
|
||||
'concept_explainer', 'scenario_quiz', 'misconceptions', 'how_to',
|
||||
'comparison_card', 'reflection_prompt', 'flashcard_set', 'case_study',
|
||||
'glossary_anchor', 'myth_vs_evidence',
|
||||
], true),
|
||||
field.json('content'),
|
||||
field.select('status', ['queued', 'generated', 'published', 'rejected'], true),
|
||||
field.text('generation_model'),
|
||||
field.date('generated_at'),
|
||||
field.date('published_at'),
|
||||
],
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// curriculum_versions
|
||||
// ---------------------------------------------------------------------------
|
||||
await ensureCollection({
|
||||
name: 'curriculum_versions',
|
||||
type: 'base',
|
||||
schema: [
|
||||
field.number('version', true),
|
||||
field.select('status', ['draft', 'active', 'superseded'], true),
|
||||
field.date('generated_at'),
|
||||
field.relation('approved_by', requireId(ids, 'users')),
|
||||
field.date('approved_at'),
|
||||
field.text('generation_notes'),
|
||||
],
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// curriculum_weeks
|
||||
// ---------------------------------------------------------------------------
|
||||
await ensureCollection({
|
||||
name: 'curriculum_weeks',
|
||||
type: 'base',
|
||||
schema: [
|
||||
field.relation('curriculum_version', requireId(ids, 'curriculum_versions'), 1, true),
|
||||
field.number('week_number', true),
|
||||
field.relation('theme', requireId(ids, 'themes'), 1, true),
|
||||
field.relation('topics', requireId(ids, 'topics'), null),
|
||||
field.json('topic_order'),
|
||||
field.number('estimated_duration_minutes'),
|
||||
field.text('admin_notes'),
|
||||
],
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// employee_curriculum_state
|
||||
// ---------------------------------------------------------------------------
|
||||
await ensureCollection({
|
||||
name: 'employee_curriculum_state',
|
||||
type: 'base',
|
||||
schema: [
|
||||
field.relation('user', requireId(ids, 'users'), 1, true),
|
||||
field.number('current_cycle', true),
|
||||
field.number('current_week', true),
|
||||
field.date('start_date'),
|
||||
field.relation('active_version', requireId(ids, 'curriculum_versions')),
|
||||
],
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// session_completions
|
||||
// ---------------------------------------------------------------------------
|
||||
await ensureCollection({
|
||||
name: 'session_completions',
|
||||
type: 'base',
|
||||
schema: [
|
||||
field.relation('user', requireId(ids, 'users'), 1, true),
|
||||
field.relation('topic', requireId(ids, 'topics'), 1, true),
|
||||
field.relation('micro_learning', requireId(ids, 'micro_learnings'), 1, true),
|
||||
field.number('week_number', true),
|
||||
field.number('cycle', true),
|
||||
field.date('completed_at'),
|
||||
],
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// gamification_profiles
|
||||
// ---------------------------------------------------------------------------
|
||||
await ensureCollection({
|
||||
name: 'gamification_profiles',
|
||||
type: 'base',
|
||||
schema: [
|
||||
field.relation('user', requireId(ids, 'users'), 1, true),
|
||||
field.number('total_commits'),
|
||||
field.select('current_level', ['intern', 'junior', 'medior', 'senior', 'staff', 'principal']),
|
||||
field.number('current_streak_weeks'),
|
||||
field.number('longest_streak_weeks'),
|
||||
field.json('types_used'),
|
||||
field.number('last_active_week'),
|
||||
],
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// badges
|
||||
// ---------------------------------------------------------------------------
|
||||
await ensureCollection({
|
||||
name: 'badges',
|
||||
type: 'base',
|
||||
schema: [
|
||||
field.text('key', true),
|
||||
field.select('tier', ['bronze', 'silver', 'gold', 'legendary', 'content'], true),
|
||||
field.text('label', true),
|
||||
field.text('description'),
|
||||
field.text('icon'),
|
||||
],
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// employee_badges
|
||||
// ---------------------------------------------------------------------------
|
||||
await ensureCollection({
|
||||
name: 'employee_badges',
|
||||
type: 'base',
|
||||
schema: [
|
||||
field.relation('user', requireId(ids, 'users'), 1, true),
|
||||
field.relation('badge', requireId(ids, 'badges'), 1, true),
|
||||
field.date('earned_at'),
|
||||
field.number('cycle'),
|
||||
],
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// milestone_cards
|
||||
// ---------------------------------------------------------------------------
|
||||
await ensureCollection({
|
||||
name: 'milestone_cards',
|
||||
type: 'base',
|
||||
schema: [
|
||||
field.relation('user', requireId(ids, 'users'), 1, true),
|
||||
field.number('cycle', true),
|
||||
field.number('week', true),
|
||||
field.number('total_commits'),
|
||||
field.number('streak_weeks'),
|
||||
field.json('badge_keys'),
|
||||
],
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Add self-referential relations to topics (requires topics ID to exist first)
|
||||
// ---------------------------------------------------------------------------
|
||||
if (topicsIsNew) {
|
||||
console.log('\ntopics self-refs:');
|
||||
const topicsId = requireId(ids, 'topics');
|
||||
const updateBody: Record<string, unknown> = {
|
||||
schema: [
|
||||
...topicsBaseSchema,
|
||||
field.relation('related_topics', topicsId, null),
|
||||
field.relation('prerequisite_topics', topicsId, null),
|
||||
field.relation('contrast_topics', topicsId, null),
|
||||
],
|
||||
};
|
||||
await pb.collections.update(topicsId, updateBody);
|
||||
console.log(' updated topics with related_topics, prerequisite_topics, contrast_topics');
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Seed badges
|
||||
// ---------------------------------------------------------------------------
|
||||
console.log('\nbadges:');
|
||||
const existingBadges = await pb.collection('badges').getFullList<{ key: string }>();
|
||||
const existingBadgeKeys = new Set(existingBadges.map(b => b.key));
|
||||
|
||||
for (const badge of BADGES) {
|
||||
if (existingBadgeKeys.has(badge.key)) {
|
||||
console.log(` skip ${badge.key}`);
|
||||
continue;
|
||||
}
|
||||
await pb.collection('badges').create(badge);
|
||||
console.log(` seed ${badge.key}`);
|
||||
}
|
||||
|
||||
console.log('\nDone.');
|
||||
}
|
||||
|
||||
run().catch((err: unknown) => {
|
||||
console.error('Migration failed:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
76
app/services/ingestion/src/migrations/002_qdrant_setup.ts
Normal file
76
app/services/ingestion/src/migrations/002_qdrant_setup.ts
Normal file
@@ -0,0 +1,76 @@
|
||||
import 'dotenv/config';
|
||||
import { QdrantClient } from '@qdrant/js-client-rest';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Env validation
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const QDRANT_URL = process.env['QDRANT_URL'] ?? '';
|
||||
const QDRANT_API_KEY = process.env['QDRANT_API_KEY'] ?? '';
|
||||
|
||||
if (!QDRANT_URL) {
|
||||
console.error('Missing env var: QDRANT_URL');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Collection definitions
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const VECTOR_SIZE = 1536; // text-embedding-3-small
|
||||
const DISTANCE = 'Cosine' as const;
|
||||
|
||||
const COLLECTIONS = [
|
||||
{
|
||||
name: 'source_chunks',
|
||||
// Payload indices for R42 context-weighted retrieval (boost by theme_id)
|
||||
payloadIndices: ['theme_id', 'topic_id', 'source_document_id', 'format'],
|
||||
},
|
||||
{
|
||||
name: 'topic_summaries',
|
||||
payloadIndices: ['theme_id', 'topic_id'],
|
||||
},
|
||||
] as const;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Setup
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
async function run(): Promise<void> {
|
||||
console.log('Connecting to Qdrant...');
|
||||
const client = new QdrantClient({
|
||||
url: QDRANT_URL,
|
||||
...(QDRANT_API_KEY ? { apiKey: QDRANT_API_KEY } : {}),
|
||||
});
|
||||
|
||||
const { collections } = await client.getCollections();
|
||||
const existingNames = new Set(collections.map(c => c.name));
|
||||
|
||||
for (const col of COLLECTIONS) {
|
||||
if (existingNames.has(col.name)) {
|
||||
console.log(` skip ${col.name}`);
|
||||
continue;
|
||||
}
|
||||
|
||||
await client.createCollection(col.name, {
|
||||
vectors: { size: VECTOR_SIZE, distance: DISTANCE },
|
||||
});
|
||||
console.log(` create ${col.name}`);
|
||||
|
||||
// Keyword payload indices for efficient filtering
|
||||
for (const field of col.payloadIndices) {
|
||||
await client.createPayloadIndex(col.name, {
|
||||
field_name: field,
|
||||
field_schema: 'keyword',
|
||||
});
|
||||
console.log(` index ${col.name}.${field}`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log('\nDone.');
|
||||
}
|
||||
|
||||
run().catch((err: unknown) => {
|
||||
console.error('Qdrant setup failed:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user