learning-platform/src/lib/retrieval.js

/**
 * Lightweight, dependency-free TF-IDF retrieval over the knowledge graph.
 *
 * `buildIndex(topics)` tokenises the `label + description` of each topic and
 * computes document-frequency stats so queries can be scored with TF-IDF in
 * `retrieveTopK`. The index is cached against the `topics` array reference,
 * so repeated calls with the same array don't rebuild.
 *
 * Tokeniser: lowercase, split on `[^a-zA-Z0-9-]`, drop short tokens and a
 * small Dutch/English stopword list.
 */

const STOPWORDS = new Set([
  // English
  'a', 'an', 'and', 'are', 'as', 'at', 'be', 'by', 'for', 'from', 'has', 'have',
  'how', 'i', 'in', 'is', 'it', 'its', 'of', 'on', 'or', 'than', 'that', 'the',
  'this', 'to', 'was', 'were', 'what', 'when', 'where', 'which', 'who', 'why',
  'with', 'you', 'your', 'do', 'does', 'did',
  // Dutch
  'de', 'het', 'een', 'en', 'of', 'in', 'op', 'aan', 'bij', 'voor', 'naar',
  'met', 'uit', 'om', 'door', 'over', 'tegen', 'ook', 'er', 'is', 'zijn',
  'was', 'waren', 'wat', 'wie', 'hoe', 'waar', 'wanneer', 'welke', 'die',
  'dat', 'deze', 'dit', 'ik', 'jij', 'hij', 'zij', 'we', 'wij', 'jullie',
  'als', 'dan', 'maar', 'want', 'omdat', 'niet', 'wel', 'heeft', 'hebben',
  'word', 'wordt', 'worden', 'kan', 'kunnen', 'mag', 'moet', 'moeten',
  'zal', 'zou', 'zouden', 'al', 'ook', 'nog', 'naar',
]);

export function tokenize(text) {
  if (!text) return [];
  return String(text)
    .toLowerCase()
    .split(/[^a-z0-9-]+/i)
    .filter(t => t.length >= 2 && !STOPWORDS.has(t));
}

const indexCache = new WeakMap();

export function buildIndex(topics) {
  if (!Array.isArray(topics) || topics.length === 0) {
    return { topics: [], docFreq: new Map(), termsByDoc: [], N: 0 };
  }
  const cached = indexCache.get(topics);
  if (cached) return cached;

  const termsByDoc = topics.map(t => {
    const text = `${t.label || ''} ${t.description || ''}`;
    const tokens = tokenize(text);
    const tf = new Map();
    for (const tk of tokens) tf.set(tk, (tf.get(tk) || 0) + 1);
    return tf;
  });

  const docFreq = new Map();
  for (const tf of termsByDoc) {
    for (const term of tf.keys()) {
      docFreq.set(term, (docFreq.get(term) || 0) + 1);
    }
  }

  const index = { topics, docFreq, termsByDoc, N: topics.length };
  indexCache.set(topics, index);
  return index;
}

// Compound-word matching. Dutch is heavily compounding, so a user's word
// (`pensioenafspraken`) is a *different* token than the graph's labels
// (`pensioenregeling`, `partnerpensioen`), even though they share the stem
// `pensioen`. Exact TF-IDF scores those pairs at 0, so the relevant topics are
// never retrieved. These heuristics recover that recall at a reduced weight,
// so exact matches still dominate the ranking.
const PARTIAL_MIN_QUERY_LEN = 6; // only expand meaty query tokens
const PARTIAL_MIN_OVERLAP = 6;   // shared stem / substring must be this long
const PARTIAL_WEIGHT = 0.4;      // discount vs. an exact term hit

/** True when two distinct tokens share a long stem or one contains the other. */
function partialMatch(q, d) {
  if (q === d) return false;
  const shorter = q.length <= d.length ? q : d;
  const longer = q.length <= d.length ? d : q;
  if (shorter.length < PARTIAL_MIN_OVERLAP) return false;
  if (longer.includes(shorter)) return true;
  let n = 0;
  const m = shorter.length;
  while (n < m && q[n] === d[n]) n++;
  return n >= PARTIAL_MIN_OVERLAP;
}

export function retrieveTopK(index, query, k = 10) {
  if (!index || !index.N || !query) return [];
  const qTokens = tokenize(query);
  if (qTokens.length === 0) return [];

  const idf = (term) => {
    const df = index.docFreq.get(term) || 0;
    if (df === 0) return 0;
    return Math.log((index.N + 1) / (df + 1)) + 1;
  };

  const scores = new Array(index.N);
  for (let i = 0; i < index.N; i++) {
    const tf = index.termsByDoc[i];
    let s = 0;
    for (const t of qTokens) {
      const f = tf.get(t);
      if (f) {
        s += (1 + Math.log(f)) * idf(t);
        continue;
      }
      // No exact hit — try a compound-word match against this doc's terms.
      if (t.length < PARTIAL_MIN_QUERY_LEN) continue;
      let best = 0;
      for (const [term, tf2] of tf) {
        if (!partialMatch(t, term)) continue;
        const w = PARTIAL_WEIGHT * (1 + Math.log(tf2)) * idf(term);
        if (w > best) best = w;
      }
      s += best;
    }
    scores[i] = s;
  }

  const ranked = [];
  for (let i = 0; i < index.N; i++) {
    if (scores[i] > 0) ranked.push({ i, s: scores[i] });
  }
  ranked.sort((a, b) => b.s - a.s);
  return ranked.slice(0, k).map(r => index.topics[r.i]);
}