feat: add new page and migration scripts for access rules and collections

- Introduced new page component for library topics with type checks.
- Added migration scripts to update access rules for various collections including badges, curriculum versions, and themes.
- Implemented PocketBase integration for managing collection access rules dynamically.
- Ensured proper type validation for page props and metadata generation functions.

app/services/ingestion/src/index.ts

@@ -1,5 +1,6 @@
 import 'dotenv/config';
 import Fastify from 'fastify';
+import cors from '@fastify/cors';
 import documentRoutes from './routes/documents.js';
 
 const PORT = parseInt(process.env['INGESTION_PORT'] ?? '3001', 10);
@@ -7,6 +8,7 @@ const PORT = parseInt(process.env['INGESTION_PORT'] ?? '3001', 10);
 async function start(): Promise<void> {
   const app = Fastify({ logger: true });
 
+  await app.register(cors, { origin: true });
   await app.register(documentRoutes);
 
   await app.listen({ port: PORT, host: '0.0.0.0' });

app/services/ingestion/src/jobs/queue.ts

@@ -1,4 +1,7 @@
 import { v4 as uuid } from 'uuid';
+import { tmpdir } from 'os';
+import { join } from 'path';
+import { writeFile, unlink } from 'fs/promises';
 import { extract } from '../pipeline/extract.js';
 import { chunk } from '../pipeline/chunk.js';
 import { clean } from '../pipeline/clean.js';
@@ -26,7 +29,8 @@ export function createJob(params: IngestBody): Job {
     documentId: params.documentId,
     filename: params.filename,
     format: params.format,
-    filePath: params.filePath,
+    fileUrl: params.fileUrl,
+    filePath: '',
     status: 'queued',
     progress: { ...DEFAULT_PROGRESS },
     error: null,
@@ -62,10 +66,20 @@ async function runPipeline(jobId: string): Promise<void> {
   const job = jobs.get(jobId);
   if (!job) return;
 
+  let tempPath: string | null = null;
+
   try {
+    // Stage 0: download file from PocketBase to temp dir
+    const res = await fetch(job.fileUrl);
+    if (!res.ok) throw new Error(`Failed to download file: ${res.status}`);
+    const buffer = Buffer.from(await res.arrayBuffer());
+    tempPath = join(tmpdir(), `ingest-${jobId}-${job.filename}`);
+    await writeFile(tempPath, buffer);
+    updateJob(jobId, { filePath: tempPath });
+
     // Stage 1: text extraction
     updateJob(jobId, { status: 'extracting' });
-    const text = await extract(job.filePath, job.format);
+    const text = await extract(tempPath, job.format);
 
     // Stages 2–3: chunking + cleaning
     updateJob(jobId, { status: 'chunking' });
@@ -98,5 +112,9 @@ async function runPipeline(jobId: string): Promise<void> {
     if (j) {
       await markDocumentFailed(j.documentId, reason).catch(() => undefined);
     }
+  } finally {
+    if (tempPath) {
+      await unlink(tempPath).catch(() => undefined);
+    }
   }
 }

app/services/ingestion/src/migrations/003_access_rules.ts

@@ -0,0 +1,60 @@
+import 'dotenv/config'
+import PocketBase from 'pocketbase'
+
+const POCKETBASE_URL = process.env['POCKETBASE_URL'] ?? ''
+const POCKETBASE_ADMIN_EMAIL = process.env['POCKETBASE_ADMIN_EMAIL'] ?? ''
+const POCKETBASE_ADMIN_PASSWORD = process.env['POCKETBASE_ADMIN_PASSWORD'] ?? ''
+
+if (!POCKETBASE_URL || !POCKETBASE_ADMIN_EMAIL || !POCKETBASE_ADMIN_PASSWORD) {
+  console.error('Missing env vars')
+  process.exit(1)
+}
+
+const AUTH_ANY = '@request.auth.id != ""'
+const AUTH_ADMIN = '@request.auth.role = "admin"'
+const AUTH_OWN = '@request.auth.id = user'
+
+// [listRule, viewRule, createRule, updateRule, deleteRule]
+const RULES: Record<string, [string, string, string, string, string]> = {
+  source_documents:          [AUTH_ADMIN,  AUTH_ADMIN,  AUTH_ADMIN,  AUTH_ADMIN,  AUTH_ADMIN],
+  themes:                    [AUTH_ANY,    AUTH_ANY,    AUTH_ADMIN,  AUTH_ADMIN,  AUTH_ADMIN],
+  topics:                    [AUTH_ANY,    AUTH_ANY,    AUTH_ADMIN,  AUTH_ADMIN,  AUTH_ADMIN],
+  micro_learnings:           [AUTH_ANY,    AUTH_ANY,    AUTH_ADMIN,  AUTH_ADMIN,  AUTH_ADMIN],
+  curriculum_versions:       [AUTH_ANY,    AUTH_ANY,    AUTH_ADMIN,  AUTH_ADMIN,  AUTH_ADMIN],
+  curriculum_weeks:          [AUTH_ANY,    AUTH_ANY,    AUTH_ADMIN,  AUTH_ADMIN,  AUTH_ADMIN],
+  employee_curriculum_state: [AUTH_ANY,    AUTH_ANY,    AUTH_ANY,    AUTH_OWN,    AUTH_ADMIN],
+  session_completions:       [AUTH_ANY,    AUTH_ANY,    AUTH_ANY,    '',          ''],
+  gamification_profiles:     [AUTH_ANY,    AUTH_ANY,    AUTH_ADMIN,  AUTH_ADMIN,  AUTH_ADMIN],
+  badges:                    [AUTH_ANY,    AUTH_ANY,    AUTH_ADMIN,  AUTH_ADMIN,  AUTH_ADMIN],
+  employee_badges:           [AUTH_ANY,    AUTH_ANY,    AUTH_ADMIN,  AUTH_ADMIN,  AUTH_ADMIN],
+  milestone_cards:           [AUTH_ANY,    AUTH_ANY,    AUTH_ADMIN,  AUTH_ADMIN,  AUTH_ADMIN],
+}
+
+async function main() {
+  const pb = new PocketBase(POCKETBASE_URL)
+  await pb.admins.authWithPassword(POCKETBASE_ADMIN_EMAIL, POCKETBASE_ADMIN_PASSWORD)
+
+  const collections = await pb.collections.getFullList()
+
+  for (const col of collections) {
+    const rules = RULES[col.name]
+    if (!rules) continue
+
+    const [listRule, viewRule, createRule, updateRule, deleteRule] = rules
+    await pb.collections.update(col.id, {
+      listRule,
+      viewRule,
+      createRule,
+      updateRule,
+      deleteRule,
+    })
+    console.log(`✓ ${col.name}`)
+  }
+
+  console.log('\nAccess rules applied.')
+}
+
+main().catch((err) => {
+  console.error(err)
+  process.exit(1)
+})

app/services/ingestion/src/routes/documents.ts

@@ -12,7 +12,7 @@ const documentRoutes: FastifyPluginAsync = async (app) => {
     return reply.status(202).send({ jobId: job.id, status: job.status });
   });
 
-  app.get<{ Params: { jobId: string } }>('/status/:jobId', async (request, reply) => {
+  app.get<{ Params: { jobId: string } }>('/ingest/status/:jobId', async (request, reply) => {
     const job = getJob(request.params.jobId);
     if (!job) {
       return reply.status(404).send({ error: 'Job not found' });
@@ -20,8 +20,11 @@ const documentRoutes: FastifyPluginAsync = async (app) => {
     return reply.send({
       jobId: job.id,
       status: job.status,
-      progress: job.progress,
-      error: job.error,
+      chunksTotal: job.progress.chunksTotal,
+      chunksEmbedded: job.progress.chunksEmbedded,
+      themesFound: job.progress.themesFound,
+      topicsFound: job.progress.topicsFound,
+      reason: job.error ?? undefined,
     });
   });
 };

app/services/ingestion/src/types.ts

@@ -125,6 +125,7 @@ export interface Job {
   documentId: string;
   filename: string;
   format: DocumentFormat;
+  fileUrl: string;
   filePath: string;
   status: JobStatus;
   progress: JobProgress;
@@ -141,7 +142,7 @@ export const IngestBodySchema = z.object({
   documentId: z.string().min(1),
   filename: z.string().min(1),
   format: z.enum(['pdf', 'md', 'txt']),
-  filePath: z.string().min(1),
+  fileUrl: z.string().url(),
 });
 
 export type IngestBody = z.infer<typeof IngestBodySchema>;