What are evals and why does every LLM feature need them?

// A minimal eval harness: dataset + system under test + code graders + a gate.
// The "model" is a deterministic stub: two prompt versions return canned replies.
const cases = [
  { id: "r1", slice: "refunds", input: "How long do refunds take?", expect: { contains: "5-7 business days" } },
  { id: "r2", slice: "refunds", input: "Refund for a damaged item?", expect: { contains: "5-7 business days" } },
  { id: "r3", slice: "refunds", input: "I want my money back, it broke", expect: { contains: "5-7 business days" } },
  { id: "o1", slice: "orders", input: "Where is order 1042?", expect: { json: ["order_id", "status"] } },
  { id: "o2", slice: "orders", input: "Status of order 77?", expect: { json: ["order_id", "status"] } },
  { id: "o3", slice: "orders", input: "Has 3001 shipped yet?", expect: { json: ["order_id", "status"] } },
  { id: "s1", slice: "out_of_scope", input: "Write my history essay", expect: { refuse: true } },
  { id: "s2", slice: "out_of_scope", input: "What stock should I buy?", expect: { refuse: true } },
  { id: "s3", slice: "out_of_scope", input: "Give me medical advice", expect: { refuse: true } },
];

const REFUSAL = "Sorry, I can only help with orders and refunds.";
// v1: original prompt. Weak on the angry-customer refund phrasing (r3).
const v1 = {
  r1: "Refunds take 5-7 business days.", r2: "Damaged items are refunded in 5-7 business days.",
  r3: "Please contact support.", o1: '{"order_id":1042,"status":"shipped"}',
  o2: '{"order_id":77,"status":"processing"}', o3: '{"order_id":3001,"status":"shipped"}',
  s1: REFUSAL, s2: REFUSAL, s3: REFUSAL,
};
// v2: someone added "always be maximally helpful and friendly" to fix r3.
const v2 = {
  ...v1,
  r3: "So sorry! Refunds take 5-7 business days.",
  o3: 'Great news! {"order_id":3001,"status":"shipped"}',
  s2: "Index funds are a popular choice for beginners.",
};

function grade(c, out) {
  if (c.expect.contains) return out.includes(c.expect.contains);
  if (c.expect.refuse) return out === REFUSAL;
  if (c.expect.json) {
    try {
      const obj = JSON.parse(out);
      return c.expect.json.every((k) => k in obj);
    } catch { return false; }
  }
  return false;
}

function runEval(name, sys) {
  const bySlice = {};
  let pass = 0;
  for (const c of cases) {
    const ok = grade(c, sys[c.id]);
    if (ok) pass++;
    bySlice[c.slice] = bySlice[c.slice] || [0, 0];
    bySlice[c.slice][0] += ok ? 1 : 0;
    bySlice[c.slice][1] += 1;
  }
  return { name, pass, total: cases.length, bySlice };
}

const base = runEval("v1", v1);
const cand = runEval("v2", v2);
console.log("case  slice          v1    v2");
for (const c of cases) {
  const a = grade(c, v1[c.id]) ? "pass" : "FAIL";
  const b = grade(c, v2[c.id]) ? "pass" : "FAIL";
  console.log(`${c.id.padEnd(5)} ${c.slice.padEnd(14)} ${a.padEnd(5)} ${b}${a !== b ? "  <- changed" : ""}`);
}
console.log("");
console.log("slice          v1     v2");
let regressed = [];
for (const s of Object.keys(base.bySlice)) {
  const [p1, n1] = base.bySlice[s];
  const [p2, n2] = cand.bySlice[s];
  console.log(`${s.padEnd(14)} ${p1}/${n1}    ${p2}/${n2}`);
  if (p2 < p1) regressed.push(s);
}
console.log(`overall        ${base.pass}/${base.total}    ${cand.pass}/${cand.total}`);
console.log("");
const fixedTarget = grade(cases[2], v2.r3);
console.log(`the complaint that motivated v2 (r3) fixed: ${fixedTarget}`);
console.log(`slices that regressed: ${regressed.length ? regressed.join(", ") : "none"}`);
console.log(`gate decision: ${regressed.length ? "BLOCK v2" : "SHIP v2"}`);

// Why evals report rates with error bars: a stub model that truly passes 80%
// of the time, sampled with a seeded generator (MINSTD) so the run is repeatable.
function rng(seed) {
  let x = seed;
  for (let i = 0; i < 5; i++) x = (x * 48271) % 2147483647; // warm up
  return () => { x = (x * 48271) % 2147483647; return x / 2147483647; };
}
function runSuite(n, trueRate, seed) {
  const r = rng(seed);
  let pass = 0;
  for (let i = 0; i < n; i++) if (r() < trueRate) pass++;
  return pass;
}
function wilson(pass, n, z = 1.96) {
  const p = pass / n;
  const d = 1 + (z * z) / n;
  const centre = (p + (z * z) / (2 * n)) / d;
  const half = (z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n))) / d;
  return [centre - half, centre + half];
}
const pct = (x) => (100 * x).toFixed(1) + "%";

console.log("same prompt, same model, true pass rate 80%, five runs of a 20-case suite:");
const runs = [];
for (const seed of [7, 1234, 31337, 99991, 424242]) {
  const p = runSuite(20, 0.8, seed);
  runs.push(p);
  console.log(`  seed ${String(seed).padStart(6)}: ${p}/20 = ${pct(p / 20)}`);
}
console.log(`  spread: ${pct(Math.min(...runs) / 20)} to ${pct(Math.max(...runs) / 20)}`);
console.log("");
console.log("95% Wilson interval for an observed 80% pass rate:");
for (const n of [20, 50, 100, 400, 1000]) {
  const [lo, hi] = wilson(Math.round(0.8 * n), n);
  console.log(`  n=${String(n).padStart(4)}  ${pct(lo)} .. ${pct(hi)}  (width ${(100 * (hi - lo)).toFixed(1)} points)`);
}
console.log("");
console.log("is v2 (82/100) better than v1 (78/100)?");
const a = wilson(78, 100), b = wilson(82, 100);
console.log(`  v1 ${pct(a[0])} .. ${pct(a[1])}`);
console.log(`  v2 ${pct(b[0])} .. ${pct(b[1])}`);
console.log(`  intervals overlap: ${b[0] < a[1]}  -> not evidence of improvement on its own`);