Evaluating RAG with Ragas: faithfulness, answer relevancy, context precision?

// The four core Ragas formulas, computed by hand. Where Ragas would ask an LLM
// (split into claims, judge support, write questions back from the answer) a
// deterministic stub does it: "supported" means every content word appears in the text.
const STOP = new Set("a an the is are be can do i to of in on for and or with within have has it you your my this that how what".split(" "));
const stem = (w) => (w.length > 4 && w.endsWith("ed") ? w.slice(0, -2) : w.length > 3 && w.endsWith("s") ? w.slice(0, -1) : w);
const words = (s) => s.toLowerCase().replace(/[^a-z0-9 ]/g, " ").split(/\s+/).filter((w) => w && !STOP.has(w)).map(stem);
const claims = (s) => s.split(". ").map((x) => x.replace(/\.$/, "")).filter(Boolean);
const supportedBy = (claim, text) => { const t = new Set(words(text)); return words(claim).every((w) => t.has(w)); };

const CHUNKS = {
  R: "Laptops can be returned within 14 days of delivery.",
  F: "Opened laptops carry a 15 percent restocking fee.",
  S: "Standard shipping takes 3 to 5 business days.",
  W: "Every laptop includes a one year limited warranty.",
};
const QUESTION = "How many days do I have to return a laptop?";
const REFERENCE = "Laptops can be returned within 14 days. Opened laptops carry a 15 percent restocking fee.";

// Each scenario: retrieved chunk ids in rank order, the answer, and the questions
// the judge would generate back from that answer (for answer relevancy).
const SCENARIOS = [
  { name: "1 good", ctx: ["R", "F", "S"], answer: "Laptops can be returned within 14 days. Opened laptops carry a 15 percent restocking fee.",
    gen: ["How many days to return a laptop?", "Is there a fee to return an opened laptop?", "Can laptops be returned?"] },
  { name: "2 relevant ranked low", ctx: ["S", "W", "R", "F"], answer: "Laptops can be returned within 14 days. Opened laptops carry a 15 percent restocking fee.",
    gen: ["How many days to return a laptop?", "Is there a fee to return an opened laptop?", "Can laptops be returned?"] },
  { name: "3 hallucinated claim", ctx: ["R", "F", "S"], answer: "Laptops can be returned within 14 days. Return shipping is always free.",
    gen: ["How many days to return a laptop?", "Is return shipping free?", "Can laptops be returned?"] },
  { name: "4 fee chunk missing", ctx: ["R", "S", "W"], answer: "Laptops can be returned within 14 days.",
    gen: ["How many days to return a laptop?", "Can laptops be returned?", "What is the laptop return window?"] },
  { name: "5 off-topic answer", ctx: ["R", "F", "S"], answer: "Standard shipping takes 3 to 5 business days.",
    gen: ["How long does standard shipping take?", "How many business days is shipping?", "What is the shipping time?"] },
];

function faithfulness(answer, ctx) {
  const text = ctx.map((id) => CHUNKS[id]).join(" ");
  const cs = claims(answer);
  return cs.filter((c) => supportedBy(c, text)).length / cs.length;
}
function vec(s) { const m = new Map(); for (const w of words(s)) m.set(w, (m.get(w) || 0) + 1); return m; }
function cosine(a, b) {
  let dot = 0, na = 0, nb = 0;
  for (const [w, x] of a) { na += x * x; if (b.has(w)) dot += x * b.get(w); }
  for (const [, y] of b) nb += y * y;
  return dot / Math.sqrt(na * nb);
}
function answerRelevancy(gen) { const q = vec(QUESTION); return gen.reduce((s, g) => s + cosine(vec(g), q), 0) / gen.length; }
// Context precision@K = sum over k of (precision@k * v_k) / (relevant items in top K).
function contextPrecision(ctx) {
  const refClaims = claims(REFERENCE);
  const v = ctx.map((id) => (refClaims.some((c) => supportedBy(c, CHUNKS[id])) ? 1 : 0));
  let hits = 0, sum = 0;
  v.forEach((rel, i) => { if (rel) { hits++; sum += hits / (i + 1); } });
  return { score: hits ? sum / hits : 0, v: v.join("") };
}
// Context recall = reference claims supported by the retrieved context / all reference claims.
function contextRecall(ctx) {
  const text = ctx.map((id) => CHUNKS[id]).join(" ");
  const rc = claims(REFERENCE);
  return rc.filter((c) => supportedBy(c, text)).length / rc.length;
}

const f = (x) => x.toFixed(3);
console.log(`question: ${QUESTION}`);
console.log("scenario                 faithful  ans_rel  ctx_prec  ctx_recall  relevance-by-rank");
for (const s of SCENARIOS) {
  const cp = contextPrecision(s.ctx);
  console.log(`${s.name.padEnd(24)} ${f(faithfulness(s.answer, s.ctx)).padStart(8)} ${f(answerRelevancy(s.gen)).padStart(8)} ${f(cp.score).padStart(9)} ${f(contextRecall(s.ctx)).padStart(11)}  ${cp.v}`);
}
// Show the claim-level verdicts for the hallucination case, which is what to read when the number drops.
const s3 = SCENARIOS[2], text3 = s3.ctx.map((id) => CHUNKS[id]).join(" ");
for (const c of claims(s3.answer)) console.log(`  scenario 3 claim "${c}" -> ${supportedBy(c, text3) ? "supported" : "NOT supported"}`);