feat: phase 5 of AI pipeline hardening — R42 retrieval & telemetry
- Add dependency-free TF-IDF retrieval (src/lib/retrieval.js) with NL+EN
stopwords and a WeakMap-cached index.
- Rewrite buildKbContext to ship the top-K relevant topics + verbatim-
mentioned ids only, filter relations to the included set, and append a
[kb_hash: <8 hex>] suffix so the ephemeral prompt cache busts when the
graph changes. Returns { context, retrievedTopics, allTopics }.
- Add LOOKUP_TOPIC_TOOL and drive useChat through callLLM directly with a
multi-hop tool_result loop capped at 3 hops; preserve Anthropic-provided
tool_use ids through callLLM so the loop can echo correct tool_use_id.
- Truncate R42 history to the last 12 turns and prepend a single
"(earlier conversation truncated)" assistant message.
- Set R42 chat defaults: temperature 0.3, maxTokens 2048.
- Add pb_migrations/1780500002_created_llm_calls.js (the best-effort
logger in callLLM was already wired) and a new Admin → Diagnostics
view showing the last 100 calls with token usage, cache-hit rate, and
USD cost from a local Anthropic price table.
- Finalize AI_PIPELINE_HARDENING_PLAN.md: mark Phases 1–5 shipped and
Phase 6 (eval harness) explicitly out of scope.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
RaymondVerhoef
162
src/components/admin/Diagnostics.jsx
Normal file
162
src/components/admin/Diagnostics.jsx
Normal file
@@ -0,0 +1,162 @@
|
||||
import { useEffect, useState } from 'react';
|
||||
import { RefreshCw, AlertCircle, CheckCircle2 } from 'lucide-react';
|
||||
import Card from '../ui/Card';
|
||||
import Button from '../ui/Button';
|
||||
import Tag from '../ui/Tag';
|
||||
import * as db from '../../lib/db';
|
||||
|
||||
// Public Anthropic pricing per 1M tokens. Update manually when prices change.
|
||||
const PRICES = {
|
||||
'claude-haiku-4-5-20251001': { input: 1.0, output: 5.0, cache_read: 0.10, cache_write: 1.25 },
|
||||
'claude-haiku-4-5': { input: 1.0, output: 5.0, cache_read: 0.10, cache_write: 1.25 },
|
||||
'claude-sonnet-4-6': { input: 3.0, output: 15.0, cache_read: 0.30, cache_write: 3.75 },
|
||||
'claude-opus-4-7': { input: 15.0, output: 75.0, cache_read: 1.50, cache_write: 18.75 },
|
||||
};
|
||||
|
||||
function pricesFor(model) {
|
||||
if (!model) return null;
|
||||
if (PRICES[model]) return PRICES[model];
|
||||
if (model.includes('haiku')) return PRICES['claude-haiku-4-5'];
|
||||
if (model.includes('sonnet')) return PRICES['claude-sonnet-4-6'];
|
||||
if (model.includes('opus')) return PRICES['claude-opus-4-7'];
|
||||
return null;
|
||||
}
|
||||
|
||||
function costUsd(row) {
|
||||
const p = pricesFor(row.model);
|
||||
if (!p) return null;
|
||||
const inTok = (row.input_tokens || 0) - (row.cache_read_tokens || 0) - (row.cache_create_tokens || 0);
|
||||
const out = row.output_tokens || 0;
|
||||
const cr = row.cache_read_tokens || 0;
|
||||
const cc = row.cache_create_tokens || 0;
|
||||
const usd = (Math.max(inTok, 0) * p.input + out * p.output + cr * p.cache_read + cc * p.cache_write) / 1_000_000;
|
||||
return usd;
|
||||
}
|
||||
|
||||
function fmtUsd(n) {
|
||||
if (n == null) return '—';
|
||||
if (n < 0.0001) return '<$0.0001';
|
||||
return `$${n.toFixed(4)}`;
|
||||
}
|
||||
|
||||
function fmtMs(n) {
|
||||
if (n == null) return '—';
|
||||
if (n < 1000) return `${Math.round(n)}ms`;
|
||||
return `${(n / 1000).toFixed(1)}s`;
|
||||
}
|
||||
|
||||
const Diagnostics = () => {
|
||||
const [rows, setRows] = useState([]);
|
||||
const [loading, setLoading] = useState(false);
|
||||
|
||||
const load = async () => {
|
||||
setLoading(true);
|
||||
try {
|
||||
const r = await db.getRecentLlmCalls(100);
|
||||
setRows(r);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
useEffect(() => { load(); }, []);
|
||||
|
||||
const totals = rows.reduce((acc, r) => {
|
||||
acc.input += r.input_tokens || 0;
|
||||
acc.output += r.output_tokens || 0;
|
||||
acc.cacheRead += r.cache_read_tokens || 0;
|
||||
acc.cacheCreate += r.cache_create_tokens || 0;
|
||||
const c = costUsd(r);
|
||||
if (c != null) acc.cost += c;
|
||||
acc.duration += r.duration_ms || 0;
|
||||
if (r.ok) acc.ok++;
|
||||
else acc.fail++;
|
||||
return acc;
|
||||
}, { input: 0, output: 0, cacheRead: 0, cacheCreate: 0, cost: 0, duration: 0, ok: 0, fail: 0 });
|
||||
|
||||
const cacheHitRate = totals.input > 0
|
||||
? Math.round((totals.cacheRead / totals.input) * 100)
|
||||
: 0;
|
||||
|
||||
return (
|
||||
<div>
|
||||
<div className="flex items-center justify-between mb-4">
|
||||
<p className="text-fg-muted text-sm">
|
||||
Laatste 100 LLM-aanroepen. Kosten worden lokaal berekend met publieke Anthropic-prijzen (handmatig verversen).
|
||||
</p>
|
||||
<Button onClick={load} disabled={loading}>
|
||||
<RefreshCw size={16} className={`mr-2 ${loading ? 'animate-spin' : ''}`} /> Vernieuwen
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-2 md:grid-cols-4 gap-3 mb-6">
|
||||
<Card className="p-4 border border-bg-warm">
|
||||
<p className="text-xs text-fg-muted uppercase">Calls</p>
|
||||
<p className="text-2xl font-bold mt-1">{rows.length}</p>
|
||||
<p className="text-xs text-fg-muted mt-1">{totals.ok} ok · {totals.fail} fail</p>
|
||||
</Card>
|
||||
<Card className="p-4 border border-bg-warm">
|
||||
<p className="text-xs text-fg-muted uppercase">Tokens (in/out)</p>
|
||||
<p className="text-2xl font-bold mt-1">{totals.input.toLocaleString()} / {totals.output.toLocaleString()}</p>
|
||||
<p className="text-xs text-fg-muted mt-1">cache read: {totals.cacheRead.toLocaleString()} ({cacheHitRate}%)</p>
|
||||
</Card>
|
||||
<Card className="p-4 border border-bg-warm">
|
||||
<p className="text-xs text-fg-muted uppercase">Geschatte kosten</p>
|
||||
<p className="text-2xl font-bold mt-1">{fmtUsd(totals.cost)}</p>
|
||||
<p className="text-xs text-fg-muted mt-1">over deze 100 aanroepen</p>
|
||||
</Card>
|
||||
<Card className="p-4 border border-bg-warm">
|
||||
<p className="text-xs text-fg-muted uppercase">Gem. duur</p>
|
||||
<p className="text-2xl font-bold mt-1">{rows.length ? fmtMs(totals.duration / rows.length) : '—'}</p>
|
||||
</Card>
|
||||
</div>
|
||||
|
||||
<Card className="p-0 border border-bg-warm overflow-hidden">
|
||||
<div className="overflow-x-auto">
|
||||
<table className="w-full text-sm">
|
||||
<thead className="bg-bg-warm/40 text-fg-muted text-xs uppercase">
|
||||
<tr>
|
||||
<th className="text-left p-3">Tijd</th>
|
||||
<th className="text-left p-3">Task</th>
|
||||
<th className="text-left p-3">Tier</th>
|
||||
<th className="text-left p-3">Model</th>
|
||||
<th className="text-right p-3">In</th>
|
||||
<th className="text-right p-3">Out</th>
|
||||
<th className="text-right p-3">Cache</th>
|
||||
<th className="text-right p-3">$</th>
|
||||
<th className="text-right p-3">Duur</th>
|
||||
<th className="text-left p-3">Status</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody className="divide-y divide-bg-warm">
|
||||
{rows.length === 0 ? (
|
||||
<tr><td colSpan={10} className="p-8 text-center text-fg-muted">Nog geen aanroepen geregistreerd.</td></tr>
|
||||
) : rows.map(r => (
|
||||
<tr key={r.id} className="hover:bg-bg-warm/20">
|
||||
<td className="p-3 text-xs text-fg-muted whitespace-nowrap">
|
||||
{r.created ? new Date(r.created).toLocaleString() : '—'}
|
||||
</td>
|
||||
<td className="p-3 font-mono text-xs">{r.task || '—'}</td>
|
||||
<td className="p-3 text-xs">{r.tier || '—'}</td>
|
||||
<td className="p-3 font-mono text-xs">{r.model || '—'}</td>
|
||||
<td className="p-3 text-right">{(r.input_tokens || 0).toLocaleString()}</td>
|
||||
<td className="p-3 text-right">{(r.output_tokens || 0).toLocaleString()}</td>
|
||||
<td className="p-3 text-right text-fg-muted">{(r.cache_read_tokens || 0).toLocaleString()}</td>
|
||||
<td className="p-3 text-right">{fmtUsd(costUsd(r))}</td>
|
||||
<td className="p-3 text-right">{fmtMs(r.duration_ms)}</td>
|
||||
<td className="p-3">
|
||||
{r.ok
|
||||
? <Tag variant="success" className="flex items-center gap-1 w-fit"><CheckCircle2 size={12}/> ok</Tag>
|
||||
: <Tag variant="dark" className="bg-red-100 text-red-800 flex items-center gap-1 w-fit"><AlertCircle size={12}/> {(r.error_msg || r.stop_reason || 'fail').slice(0, 24)}</Tag>}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</Card>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
export default Diagnostics;
|
||||
Reference in New Issue
Block a user