feat: phase 3 of AI pipeline hardening — extraction quality
Replace stateless one-shot extraction with a stateful, paced, cancellable pipeline. Six subtasks: - 3.1 Sentence-aware chunking with 800-char overlap (was paragraph-only at 4000 chars). Hard-split fallback for runaway sentences. - 3.2 Stateful extraction: chunks 2+ receive an "already-extracted topic IDs" hint capped at 200 IDs, so the model reuses IDs instead of inventing variants like software-developer vs software-engineer. - 3.3 Token-bucket limiter in llmRetry.js (extractionLimiter, 5 req/min). callLLM awaits the limiter before fetch; 429+Retry-After calls pauseUntil. Replaces hard setTimeout(12000) and setTimeout(15000). - 3.4 relevance_locked column on topics — admin edits to relevance are sticky across re-extraction. Migration + merge respects the flag + unlock checkbox in KnowledgeGraph edit form. - 3.5 Unify relation vocabulary — handbook prompt no longer mentions legacy "executes"; one-shot migration rewrites existing executes rows to executed_by with source/target swapped. - 3.6 Cancellation — Cancel button on UploadZone wired to an AbortController threaded into callLLM; aborted runs persist status = "cancelled" rather than "failed". Tests: 16 new unit tests for chunkText, buildKnownIdsHint, and createLimiter. All 61 tests pass, 0 lint errors, build clean. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
23
pb_migrations/1780500000_updated_topics_relevance_locked.js
Normal file
23
pb_migrations/1780500000_updated_topics_relevance_locked.js
Normal file
@@ -0,0 +1,23 @@
|
||||
/// <reference path="../pb_data/types.d.ts" />
|
||||
migrate((app) => {
|
||||
const collection = app.findCollectionByNameOrId("pbc_2800040823")
|
||||
|
||||
// add field — relevance_locked is set to true whenever an admin edits
|
||||
// learning_relevance via the UI; mergeKnowledgeGraph must never overwrite
|
||||
// learning_relevance on a locked topic during re-extraction.
|
||||
collection.fields.addAt(5, new Field({
|
||||
"hidden": false,
|
||||
"id": "bool_relevance_locked",
|
||||
"name": "relevance_locked",
|
||||
"presentable": false,
|
||||
"required": false,
|
||||
"system": false,
|
||||
"type": "bool"
|
||||
}))
|
||||
|
||||
return app.save(collection)
|
||||
}, (app) => {
|
||||
const collection = app.findCollectionByNameOrId("pbc_2800040823")
|
||||
collection.fields.removeById("bool_relevance_locked")
|
||||
return app.save(collection)
|
||||
})
|
||||
43
pb_migrations/1780500001_normalize_relation_types.js
Normal file
43
pb_migrations/1780500001_normalize_relation_types.js
Normal file
@@ -0,0 +1,43 @@
|
||||
/// <reference path="../pb_data/types.d.ts" />
|
||||
// One-shot data migration: rewrite legacy "executes" relations to the
|
||||
// canonical "executed_by" vocabulary by swapping source and target.
|
||||
// Previously `role --executes--> process`; canonical is
|
||||
// `process --executed_by--> role`.
|
||||
migrate((app) => {
|
||||
const records = app.findRecordsByFilter(
|
||||
"pbc_1883724256", // relations collection
|
||||
'type = "executes"',
|
||||
'',
|
||||
0,
|
||||
0,
|
||||
)
|
||||
|
||||
for (const rec of records) {
|
||||
const source = rec.get("source")
|
||||
const target = rec.get("target")
|
||||
rec.set("type", "executed_by")
|
||||
rec.set("source", target)
|
||||
rec.set("target", source)
|
||||
app.save(rec)
|
||||
}
|
||||
}, (app) => {
|
||||
// Reverse: turn executed_by back into executes (best-effort — only those
|
||||
// created before this migration would have been "executes"; rolling back
|
||||
// will affect any newer executed_by rows too).
|
||||
const records = app.findRecordsByFilter(
|
||||
"pbc_1883724256",
|
||||
'type = "executed_by"',
|
||||
'',
|
||||
0,
|
||||
0,
|
||||
)
|
||||
|
||||
for (const rec of records) {
|
||||
const source = rec.get("source")
|
||||
const target = rec.get("target")
|
||||
rec.set("type", "executes")
|
||||
rec.set("source", target)
|
||||
rec.set("target", source)
|
||||
app.save(rec)
|
||||
}
|
||||
})
|
||||
Reference in New Issue
Block a user