How do you evaluate a RAG pipeline end to end?

// End-to-end RAG eval harness on a tiny handbook. Retriever = word-overlap
// ranking (a stand-in for BM25 or vectors); generator = deterministic stub that
// answers from the first retrieved chunk sharing >= 2 words with the question,
// else says "not found". The harness scores each stage separately, then attributes failures.
const STOP = new Set("a an the is are be can do does i to of in on for and or with have has it you my what how many when long often each get".split(" "));
const stem = (w) => (w.length > 4 && w.endsWith("ed") ? w.slice(0, -2) : w.length > 3 && w.endsWith("s") ? w.slice(0, -1) : w);
const words = (s) => new Set(s.toLowerCase().replace(/[^a-z0-9 ]/g, " ").split(/\s+/).filter((w) => w && !STOP.has(w)).map(stem));
const overlap = (a, b) => { let n = 0; for (const w of a) if (b.has(w)) n++; return n; };

const DOCS = [
  ["d1", "Employees accrue 20 vacation days per year."],
  ["d2", "Up to 5 unused vacation days roll over into the next year."],
  ["d3", "Parental leave is 16 weeks paid for all employees."],
  ["d4", "Sick leave needs a doctor note after 3 consecutive sick days."],
  ["d5", "The office opens at 8 am and closes at 7 pm."],
  ["d6", "Remote work is allowed up to 3 days per week with manager approval."],
  ["d7", "Expense reports must be filed within 30 days of purchase."],
  ["d8", "Laptops are refreshed every 3 years by the IT team."],
].map(([id, text]) => ({ id, text, w: words(text) }));

// gold = the chunk that answers it; null = not answerable from the corpus.
const QUERIES = [
  ["q01", "How many vacation days do employees get per year?", "d1"],
  ["q02", "Can unused vacation roll over to next year?", "d2"],
  ["q03", "How long is paid parental leave?", "d3"],
  ["q04", "When do I need a doctor note for sick days?", "d4"],
  ["q05", "What time does the office close?", "d5"],
  ["q06", "How many days per week can I work remotely?", "d6"],
  ["q07", "What is the deadline to submit receipts?", "d7"],
  ["q08", "How often are laptops replaced?", "d8"],
  ["q09", "How many vacation days can I carry forward each year?", "d2"],
  ["q10", "How many sick days per year do employees get?", null],
  ["q11", "What is the dress code?", null],
].map(([id, q, gold]) => ({ id, q, gold, w: words(q) }));

function retrieve(qw, k) {
  return DOCS.map((d, i) => ({ d, s: overlap(qw, d.w), i }))
    .filter((x) => x.s > 0)
    .sort((a, b) => b.s - a.s || a.i - b.i)
    .slice(0, k).map((x) => x.d);
}
function generate(qw, ctx) {
  for (const d of ctx) if (overlap(qw, d.w) >= 2) return d.id; // answers from this chunk
  return null; // "not found in the handbook"
}

const K = 3;
const answerable = QUERIES.filter((q) => q.gold);
let r1 = 0, r3 = 0, mrr = 0, oracleOk = 0, e2eOk = 0;
const buckets = {};
const rows = [];
for (const q of QUERIES) {
  const ctx = retrieve(q.w, K);
  const ids = ctx.map((d) => d.id);
  const rank = q.gold ? ids.indexOf(q.gold) + 1 : 0;
  if (q.gold) { if (rank === 1) r1++; if (rank >= 1) { r3++; mrr += 1 / rank; } }
  // Stage 2 in isolation: give the generator the gold chunk only (oracle context).
  const oracle = q.gold ? generate(q.w, DOCS.filter((d) => d.id === q.gold)) : null;
  if (q.gold && oracle === q.gold) oracleOk++;
  const ans = generate(q.w, ctx);
  let label;
  if (!q.gold) label = ans ? "answered unanswerable" : "correct abstain";
  else if (ans === q.gold) label = "correct";
  else if (rank === 0) label = "retrieval miss";
  else if (ans === null) label = "over-abstain (gold retrieved)";
  else label = "wrong chunk used (gold retrieved)";
  if (label === "correct" || label === "correct abstain") e2eOk++;
  buckets[label] = (buckets[label] || 0) + 1;
  rows.push(`${q.id} top${K}=[${ids.join(",")}] gold=${q.gold || "-"} answer=${ans || "not found"} -> ${label}`);
}
rows.forEach((r) => console.log(r));
const n = answerable.length, f = (x) => x.toFixed(3);
console.log(`retrieval (${n} answerable): recall@1=${f(r1 / n)} recall@${K}=${f(r3 / n)} MRR@${K}=${f(mrr / n)}`);
console.log(`generation with oracle context: ${oracleOk}/${n} = ${f(oracleOk / n)}`);
console.log(`end to end (${QUERIES.length} queries incl. unanswerable): ${e2eOk}/${QUERIES.length} = ${f(e2eOk / QUERIES.length)}`);
console.log("failure attribution:");
for (const k of Object.keys(buckets).sort()) console.log(`  ${k.padEnd(34)} ${buckets[k]}`);