How do you build a golden dataset for evaluating an LLM feature?

// Building a golden dataset from production traces (simulated in memory):
// normalise + dedupe, cluster near-duplicates, stratify by intent with a floor
// for rare intents, keep user-flagged failures, split dev/test by cluster.
const phrasings = {
  order_status: ["where is my order N", "where is my order N please", "wheres my order N", "has order N shipped",
    "has my order N shipped yet", "track package N", "when will order N arrive", "order N still not here", "is order N delayed"],
  refund: ["how do i get a refund", "how do i get a refund for order N", "refund for a damaged item",
    "refund for damaged item please", "how long do refunds take", "i was charged twice", "money back for order N"],
  cancel: ["cancel order N", "please cancel order N", "can i cancel my subscription", "stop my subscription"],
  account: ["reset my password", "change my email address", "delete my account"],
  legal: ["i will report you to consumer protection", "this is a chargeback threat"],
  out_of_scope: ["write my history essay", "what stock should i buy"],
};
const volume = { order_status: 120, refund: 50, cancel: 20, account: 6, legal: 2, out_of_scope: 2 };
const intents = Object.keys(volume);

let seed = 2024;
const rand = () => { seed = (seed * 48271) % 2147483647; return seed / 2147483647; };
const traces = [];
for (const intent of intents) {
  const pool = phrasings[intent];
  for (let i = 0; i < volume[intent]; i++) {
    const p = i < pool.length ? pool[i] : pool[Math.floor(rand() * pool.length)];
    let text = p.replace("N", String(1000 + Math.floor(rand() * 9000)));
    if (rand() < 0.3) text = text.toUpperCase();
    if (rand() < 0.2) text += "??";
    traces.push({ intent, text, flagged: rand() < 0.04 });
  }
}
console.log(`raw traces: ${traces.length}`);

const norm = (s) => s.toLowerCase().replace(/[0-9]+/g, "N").replace(/[^a-zN ]/g, "").replace(/\s+/g, " ").trim();
const byKey = new Map();
for (const t of traces) {
  const k = norm(t.text);
  if (!byKey.has(k)) byKey.set(k, { key: k, intent: t.intent, count: 0, flagged: false });
  const u = byKey.get(k);
  u.count++;
  u.flagged = u.flagged || t.flagged;
}
const unique = [...byKey.values()];
console.log(`after normalise + exact dedupe: ${unique.length} distinct requests`);

const jaccard = (a, b) => {
  const A = new Set(a.split(" ")), B = new Set(b.split(" "));
  let i = 0;
  for (const w of A) if (B.has(w)) i++;
  return i / (A.size + B.size - i);
};
const NEAR = 0.6;
const clusters = [];
for (const u of unique) {
  const c = clusters.find((cl) => cl.intent === u.intent && cl.members.some((m) => jaccard(m.key, u.key) >= NEAR));
  if (c) { c.members.push(u); u.cluster = c.id; } else { u.cluster = clusters.length; clusters.push({ id: clusters.length, intent: u.intent, members: [u] }); }
}
console.log(`near-duplicate clusters (jaccard >= ${NEAR}): ${clusters.length}`);
console.log("");

const TARGET = 20, FLOOR = 2;
const rank = (a, b) => (b.flagged - a.flagged) || (b.count - a.count) || (a.key < b.key ? -1 : a.key > b.key ? 1 : 0);
console.log("intent         traffic  share  clusters  proportional  stratified");
const golden = [];
for (const intent of intents) {
  const share = volume[intent] / traces.length;
  const cl = clusters.filter((c) => c.intent === intent);
  const prop = Math.min(cl.length, Math.round(share * TARGET));
  const strat = Math.min(cl.length, Math.max(FLOOR, prop));
  const reps = cl.map((c) => c.members.slice().sort(rank)[0]).sort(rank);
  golden.push(...reps.slice(0, strat));
  console.log(`${intent.padEnd(14)} ${String(volume[intent]).padStart(7)} ${(100 * share).toFixed(1).padStart(5)}%  ${String(cl.length).padStart(8)}  ${String(prop).padStart(12)}  ${String(strat).padStart(10)}`);
}
console.log(`golden set: ${golden.length} cases, ${golden.filter((g) => g.flagged).length} contain a user-flagged failure`);
console.log("");

const leaks = (test, dev) => test.filter((x) => dev.some((y) => x.intent === y.intent && jaccard(x.key, y.key) >= NEAR)).length;
const nDev = unique.filter((u, i) => i % 2 === 0), nTest = unique.filter((u, i) => i % 2 === 1);
console.log(`naive split of distinct requests: ${leaks(nTest, nDev)} of ${nTest.length} test cases have a near-twin in dev`);
const gDev = [], gTest = [];
for (const intent of intents) golden.filter((g) => g.intent === intent).forEach((g, i) => (i % 2 === 0 ? gDev : gTest).push(g));
console.log(`one case per cluster, split within each intent: dev ${gDev.length}, test ${gTest.length}, leaks ${leaks(gTest, gDev)}`);
console.log("");
console.log("rare-intent cases kept by the floor:");
for (const g of golden.filter((x) => volume[x.intent] < 10)) console.log(`  [${g.intent}] "${g.key}" seen ${g.count}x${g.flagged ? ", flagged" : ""}`);

// Are the golden labels trustworthy? Two annotators label the same 20 answers
// pass (P) or fail (F). Raw agreement flatters; Cohen's kappa corrects for chance.
function kappa(a, b) {
  const n = a.length;
  let agree = 0, aP = 0, bP = 0;
  for (let i = 0; i < n; i++) {
    if (a[i] === b[i]) agree++;
    if (a[i] === "P") aP++;
    if (b[i] === "P") bP++;
  }
  const po = agree / n;
  const pe = (aP / n) * (bP / n) + (1 - aP / n) * (1 - bP / n);
  return { po, pe, k: pe === 1 ? 0 : (po - pe) / (1 - pe) };
}
function report(name, a, b) {
  const { po, pe, k } = kappa(a, b);
  const band = k <= 0.001 ? "no better than chance" : k < 0.2 ? "slight" : k < 0.4 ? "fair" : k < 0.6 ? "moderate" : k < 0.8 ? "substantial" : "almost perfect";
  console.log(`${name.padEnd(34)} agreement ${(100 * po).toFixed(0).padStart(3)}%  chance ${(100 * pe).toFixed(1).padStart(5)}%  kappa ${k.toFixed(2).padStart(5)}  (${band})`);
}
const disagreements = (a, b) => [...a].map((x, i) => (x !== b[i] ? i + 1 : 0)).filter(Boolean);

// round 1: rubric says only "is the answer helpful?"
const r1a = "PPPPPPPPFPPPPPPPFPPP";
const r1b = "PPPFPPPPPPPPPFPPFPPP";
// a lazy annotator who marks everything pass
const lazy = "PPPPPPPPPPPPPPPPPPPP";
// round 2: rubric rewritten with anchored criteria after discussing round-1 disagreements
// ("states the 5-7 day refund window", "no promise the policy does not allow")
const r2a = "PPPFPPPPFPPFPPPPFPPP";
const r2b = "PPPFPPPPFPPFPPFPFPPP";

console.log("round 1, vague rubric");
report("  annotator A vs B", r1a, r1b);
report("  annotator A vs always-pass", r1a, lazy);
console.log(`  cases to adjudicate: ${disagreements(r1a, r1b).join(", ")}`);
console.log("");
console.log("round 2, anchored rubric");
report("  annotator A vs B", r2a, r2b);
console.log(`  cases to adjudicate: ${disagreements(r2a, r2b).join(", ")}`);
console.log("");
const failsA1 = [...r1a].filter((x) => x === "F").length;
const failsA2 = [...r2a].filter((x) => x === "F").length;
console.log(`failures annotator A found: round 1 ${failsA1}, round 2 ${failsA2}`);