feat: add new page and migration scripts for access rules and collections

- Introduced new page component for library topics with type checks.
- Added migration scripts to update access rules for various collections including badges, curriculum versions, and themes.
- Implemented PocketBase integration for managing collection access rules dynamically.
- Ensured proper type validation for page props and metadata generation functions.
This commit is contained in:
RaymondVerhoef
2026-05-23 22:26:40 +02:00
parent 14286d6cb1
commit 8684ffa35b
109 changed files with 2065 additions and 114 deletions

View File

@@ -1,4 +1,7 @@
import { v4 as uuid } from 'uuid';
import { tmpdir } from 'os';
import { join } from 'path';
import { writeFile, unlink } from 'fs/promises';
import { extract } from '../pipeline/extract.js';
import { chunk } from '../pipeline/chunk.js';
import { clean } from '../pipeline/clean.js';
@@ -26,7 +29,8 @@ export function createJob(params: IngestBody): Job {
documentId: params.documentId,
filename: params.filename,
format: params.format,
filePath: params.filePath,
fileUrl: params.fileUrl,
filePath: '',
status: 'queued',
progress: { ...DEFAULT_PROGRESS },
error: null,
@@ -62,10 +66,20 @@ async function runPipeline(jobId: string): Promise<void> {
const job = jobs.get(jobId);
if (!job) return;
let tempPath: string | null = null;
try {
// Stage 0: download file from PocketBase to temp dir
const res = await fetch(job.fileUrl);
if (!res.ok) throw new Error(`Failed to download file: ${res.status}`);
const buffer = Buffer.from(await res.arrayBuffer());
tempPath = join(tmpdir(), `ingest-${jobId}-${job.filename}`);
await writeFile(tempPath, buffer);
updateJob(jobId, { filePath: tempPath });
// Stage 1: text extraction
updateJob(jobId, { status: 'extracting' });
const text = await extract(job.filePath, job.format);
const text = await extract(tempPath, job.format);
// Stages 23: chunking + cleaning
updateJob(jobId, { status: 'chunking' });
@@ -98,5 +112,9 @@ async function runPipeline(jobId: string): Promise<void> {
if (j) {
await markDocumentFailed(j.documentId, reason).catch(() => undefined);
}
} finally {
if (tempPath) {
await unlink(tempPath).catch(() => undefined);
}
}
}