How do you evaluate and debug agent trajectories?

// Scoring recorded agent trajectories: outcome (final state), trajectory match
// against a reference (strict / unordered / subset / superset), process flags,
// pass@k vs pass^k over repeated runs, and the first step where a failing run diverges.
// The traces are recorded fixtures; replaying them needs no model and no real tools.
const REF = [["lookup_order", { id: 123 }], ["check_policy", { reason: "damaged" }], ["issue_refund", { id: 123, amount: 49.99 }]];
const L = (id, ok = true) => ["lookup_order", { id }, ok ? "ok" : "error: not found"];
const P = ["check_policy", { reason: "damaged" }, "ok"];
const R = (amount) => ["issue_refund", { id: 123, amount }, "ok"];
const RUNS = {
  v1: [[L(123), P, R(49.99)], [P, L(123), R(49.99)], [L(123), L(123), L(123), R(49.99)], [L(123), P, R(4999)], [L(132, false), L(123), P, R(49.99)]],
  v2: [[L(123), P, R(49.99)], [L(123), P, R(49.99)], [L(123), P], [L(123), P, R(49.99)], [L(123), P, R(49.99), R(49.99)]],
};

const key = ([tool, args]) => `${tool}(${JSON.stringify(args)})`;
function count(list) { const m = new Map(); for (const k of list) m.set(k, (m.get(k) || 0) + 1); return m; }
function contains(big, small) { for (const [k, n] of small) if ((big.get(k) || 0) < n) return false; return true; }
function match(run) {
  const a = run.map(key), r = REF.map(key), ca = count(a), cr = count(r);
  return {
    strict: a.length === r.length && a.every((k, i) => k === r[i]),
    unordered: contains(ca, cr) && contains(cr, ca),
    subset: contains(cr, ca),   // agent used only reference calls
    superset: contains(ca, cr), // agent made every reference call (extras allowed)
  };
}
// Outcome = the final state the tools left behind: exactly one refund of 49.99 on order 123.
function outcome(run) {
  const refunds = run.filter(([t, , res]) => t === "issue_refund" && res === "ok").map(([, a]) => a);
  return refunds.length === 1 && refunds[0].id === 123 && refunds[0].amount === 49.99;
}
function process(run) {
  const flags = [];
  for (let i = 1; i < run.length; i++) if (key(run[i]) === key(run[i - 1])) { flags.push("repeated call"); break; }
  if (run.some((s) => s[2].startsWith("error"))) flags.push(outcome(run) ? "error recovered" : "error");
  if (!run.some((s) => s[0] === "check_policy")) flags.push("skipped policy");
  return flags;
}
const C = (n, k) => { if (k > n) return 0; let r = 1; for (let i = 0; i < k; i++) r = (r * (n - i)) / (i + 1); return r; };
const f = (x) => x.toFixed(3);
const yn = (b) => (b ? "Y" : "-");

console.log("run    ok  steps  strict unord subset super  process flags");
for (const [ver, runs] of Object.entries(RUNS)) {
  const ok = runs.map(outcome);
  runs.forEach((run, i) => {
    const m = match(run);
    console.log(`${ver}.r${i + 1}  ${yn(ok[i])}   ${String(run.length).padStart(2)}     ${yn(m.strict)}      ${yn(m.unordered)}     ${yn(m.subset)}      ${yn(m.superset)}     ${process(run).join(", ") || "none"}`);
  });
  const n = runs.length, c = ok.filter(Boolean).length;
  const strictRate = runs.filter((r) => match(r).strict).length / n;
  const parts = [1, 3].map((k) => `pass@${k}=${f(1 - C(n - c, k) / C(n, k))} pass^${k}=${f(C(c, k) / C(n, k))}`);
  console.log(`${ver}: success ${c}/${n}, strict-match rate ${f(strictRate)}, ${parts.join(", ")}`);
}

// Debugging: diff a failing run against a passing one and report the first divergent step.
function firstDivergence(good, bad) {
  for (let i = 0; i < Math.max(good.length, bad.length); i++) {
    const g = good[i] ? key(good[i]) : "(end)", b = bad[i] ? key(bad[i]) : "(end)";
    if (g !== b) return `step ${i + 1}: expected ${g} got ${b}`;
  }
  return "identical";
}
console.log(`v1.r4 vs v1.r1 -> ${firstDivergence(RUNS.v1[0], RUNS.v1[3])}`);
console.log(`v2.r3 vs v2.r1 -> ${firstDivergence(RUNS.v2[0], RUNS.v2[2])}`);
console.log(`v2.r5 vs v2.r1 -> ${firstDivergence(RUNS.v2[0], RUNS.v2[4])}`);