Add comprehensive documentation for employee learning platform

- Created handover document outlining design decisions and application functionality.
- Developed implementation plan detailing phased approach for service development.
- Specified ingestion service responsibilities, API surface, and processing pipeline.

app/services/ingestion/src/jobs/queue.ts

@@ -0,0 +1,102 @@
+import { v4 as uuid } from 'uuid';
+import { extract } from '../pipeline/extract.js';
+import { chunk } from '../pipeline/chunk.js';
+import { clean } from '../pipeline/clean.js';
+import { extractStructure } from '../pipeline/structure.js';
+import { writeToPocketBase, markDocumentFailed } from '../pipeline/write.js';
+import { embedAndStore } from '../pipeline/embed.js';
+import type { Job, JobProgress, IngestBody } from '../types.js';
+
+// ---------------------------------------------------------------------------
+// In-memory store
+// ---------------------------------------------------------------------------
+
+const jobs = new Map<string, Job>();
+
+const DEFAULT_PROGRESS: JobProgress = {
+  chunksTotal: 0,
+  chunksEmbedded: 0,
+  themesFound: 0,
+  topicsFound: 0,
+};
+
+export function createJob(params: IngestBody): Job {
+  const job: Job = {
+    id: uuid(),
+    documentId: params.documentId,
+    filename: params.filename,
+    format: params.format,
+    filePath: params.filePath,
+    status: 'queued',
+    progress: { ...DEFAULT_PROGRESS },
+    error: null,
+    createdAt: new Date(),
+    updatedAt: new Date(),
+  };
+  jobs.set(job.id, job);
+  void runPipeline(job.id);
+  return job;
+}
+
+export function getJob(id: string): Job | undefined {
+  return jobs.get(id);
+}
+
+function updateJob(id: string, updates: Partial<Omit<Job, 'id' | 'createdAt'>>): void {
+  const job = jobs.get(id);
+  if (!job) return;
+  jobs.set(id, { ...job, ...updates, updatedAt: new Date() });
+}
+
+function mergeProgress(id: string, partial: Partial<JobProgress>): void {
+  const job = jobs.get(id);
+  if (!job) return;
+  updateJob(id, { progress: { ...job.progress, ...partial } });
+}
+
+// ---------------------------------------------------------------------------
+// Pipeline orchestration
+// ---------------------------------------------------------------------------
+
+async function runPipeline(jobId: string): Promise<void> {
+  const job = jobs.get(jobId);
+  if (!job) return;
+
+  try {
+    // Stage 1: text extraction
+    updateJob(jobId, { status: 'extracting' });
+    const text = await extract(job.filePath, job.format);
+
+    // Stages 2–3: chunking + cleaning
+    updateJob(jobId, { status: 'chunking' });
+    const rawChunks = chunk(text, job.format, job.documentId);
+    const cleanChunks = clean(rawChunks);
+    mergeProgress(jobId, { chunksTotal: cleanChunks.length });
+
+    // Stage 4: structure extraction (AI)
+    updateJob(jobId, { status: 'structuring' });
+    const draftKB = await extractStructure(cleanChunks, job.filename, job.format);
+    const topicsFound = draftKB.themes.reduce((n, t) => n + t.topics.length, 0);
+    mergeProgress(jobId, { themesFound: draftKB.themes.length, topicsFound });
+
+    // Stage 5: PocketBase write
+    updateJob(jobId, { status: 'writing' });
+    const writtenTopics = await writeToPocketBase(draftKB, job.documentId, cleanChunks.length);
+
+    // Stage 6: embeddings + Qdrant write
+    updateJob(jobId, { status: 'embedding' });
+    await embedAndStore(cleanChunks, writtenTopics, embedded => {
+      mergeProgress(jobId, { chunksEmbedded: embedded });
+    });
+
+    updateJob(jobId, { status: 'done' });
+  } catch (err: unknown) {
+    const reason = err instanceof Error ? err.message : String(err);
+    updateJob(jobId, { status: 'failed', error: reason });
+
+    const j = jobs.get(jobId);
+    if (j) {
+      await markDocumentFailed(j.documentId, reason).catch(() => undefined);
+    }
+  }
+}