dda20612e9
- Created handover document outlining design decisions and application functionality. - Developed implementation plan detailing phased approach for service development. - Specified ingestion service responsibilities, API surface, and processing pipeline.
77 lines
2.2 KiB
TypeScript
77 lines
2.2 KiB
TypeScript
import 'dotenv/config';
|
|
import { QdrantClient } from '@qdrant/js-client-rest';
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Env validation
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const QDRANT_URL = process.env['QDRANT_URL'] ?? '';
|
|
const QDRANT_API_KEY = process.env['QDRANT_API_KEY'] ?? '';
|
|
|
|
if (!QDRANT_URL) {
|
|
console.error('Missing env var: QDRANT_URL');
|
|
process.exit(1);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Collection definitions
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const VECTOR_SIZE = 1536; // text-embedding-3-small
|
|
const DISTANCE = 'Cosine' as const;
|
|
|
|
const COLLECTIONS = [
|
|
{
|
|
name: 'source_chunks',
|
|
// Payload indices for R42 context-weighted retrieval (boost by theme_id)
|
|
payloadIndices: ['theme_id', 'topic_id', 'source_document_id', 'format'],
|
|
},
|
|
{
|
|
name: 'topic_summaries',
|
|
payloadIndices: ['theme_id', 'topic_id'],
|
|
},
|
|
] as const;
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Setup
|
|
// ---------------------------------------------------------------------------
|
|
|
|
async function run(): Promise<void> {
|
|
console.log('Connecting to Qdrant...');
|
|
const client = new QdrantClient({
|
|
url: QDRANT_URL,
|
|
...(QDRANT_API_KEY ? { apiKey: QDRANT_API_KEY } : {}),
|
|
});
|
|
|
|
const { collections } = await client.getCollections();
|
|
const existingNames = new Set(collections.map(c => c.name));
|
|
|
|
for (const col of COLLECTIONS) {
|
|
if (existingNames.has(col.name)) {
|
|
console.log(` skip ${col.name}`);
|
|
continue;
|
|
}
|
|
|
|
await client.createCollection(col.name, {
|
|
vectors: { size: VECTOR_SIZE, distance: DISTANCE },
|
|
});
|
|
console.log(` create ${col.name}`);
|
|
|
|
// Keyword payload indices for efficient filtering
|
|
for (const field of col.payloadIndices) {
|
|
await client.createPayloadIndex(col.name, {
|
|
field_name: field,
|
|
field_schema: 'keyword',
|
|
});
|
|
console.log(` index ${col.name}.${field}`);
|
|
}
|
|
}
|
|
|
|
console.log('\nDone.');
|
|
}
|
|
|
|
run().catch((err: unknown) => {
|
|
console.error('Qdrant setup failed:', err);
|
|
process.exit(1);
|
|
});
|