feat: add new page and migration scripts for access rules and collections
- Introduced new page component for library topics with type checks. - Added migration scripts to update access rules for various collections including badges, curriculum versions, and themes. - Implemented PocketBase integration for managing collection access rules dynamically. - Ensured proper type validation for page props and metadata generation functions.
This commit is contained in:
@@ -1,4 +1,7 @@
|
||||
import { v4 as uuid } from 'uuid';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { writeFile, unlink } from 'fs/promises';
|
||||
import { extract } from '../pipeline/extract.js';
|
||||
import { chunk } from '../pipeline/chunk.js';
|
||||
import { clean } from '../pipeline/clean.js';
|
||||
@@ -26,7 +29,8 @@ export function createJob(params: IngestBody): Job {
|
||||
documentId: params.documentId,
|
||||
filename: params.filename,
|
||||
format: params.format,
|
||||
filePath: params.filePath,
|
||||
fileUrl: params.fileUrl,
|
||||
filePath: '',
|
||||
status: 'queued',
|
||||
progress: { ...DEFAULT_PROGRESS },
|
||||
error: null,
|
||||
@@ -62,10 +66,20 @@ async function runPipeline(jobId: string): Promise<void> {
|
||||
const job = jobs.get(jobId);
|
||||
if (!job) return;
|
||||
|
||||
let tempPath: string | null = null;
|
||||
|
||||
try {
|
||||
// Stage 0: download file from PocketBase to temp dir
|
||||
const res = await fetch(job.fileUrl);
|
||||
if (!res.ok) throw new Error(`Failed to download file: ${res.status}`);
|
||||
const buffer = Buffer.from(await res.arrayBuffer());
|
||||
tempPath = join(tmpdir(), `ingest-${jobId}-${job.filename}`);
|
||||
await writeFile(tempPath, buffer);
|
||||
updateJob(jobId, { filePath: tempPath });
|
||||
|
||||
// Stage 1: text extraction
|
||||
updateJob(jobId, { status: 'extracting' });
|
||||
const text = await extract(job.filePath, job.format);
|
||||
const text = await extract(tempPath, job.format);
|
||||
|
||||
// Stages 2–3: chunking + cleaning
|
||||
updateJob(jobId, { status: 'chunking' });
|
||||
@@ -98,5 +112,9 @@ async function runPipeline(jobId: string): Promise<void> {
|
||||
if (j) {
|
||||
await markDocumentFailed(j.documentId, reason).catch(() => undefined);
|
||||
}
|
||||
} finally {
|
||||
if (tempPath) {
|
||||
await unlink(tempPath).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user