How do you monitor LLM quality drift in production?

// Daily drift monitor: PSI on the input topic mix, and a one-sided CUSUM on the
// sampled LLM-judge pass rate, compared with a fixed threshold and a daily z alarm.
// 21 simulated days, 200 judged conversations per day. No deploy happens.
const N = 200;
const TOPICS = ["billing", "bug", "account", "other", "promo"];
const BASE_MIX = [80, 60, 40, 20, 0];          // the mix the golden set was built from
const PROMO_MIX = [72, 54, 36, 18, 20];        // a marketing campaign from day 10
const WIGGLE = [0, 3, -2, 1, -4, 2, -1, 3, 0, -3, 1, 2, -2, 0, 2, -1, 0, 3, -2, 1, -1];

function topicCounts(day) {
  const mix = (day >= 10 ? PROMO_MIX : BASE_MIX).slice();
  mix[0] += WIGGLE[day - 1];                    // small day-to-day noise
  mix[1] -= WIGGLE[day - 1];
  return mix;
}
// Judge passes per day: about 90% until day 15, about 87% after (an upstream document went stale).
function passes(day) {
  return (day >= 15 ? 174 : 180) + WIGGLE[(day * 7) % 21];
}

function psi(expected, actual) {
  const eT = expected.reduce((a, b) => a + b, 0), aT = actual.reduce((a, b) => a + b, 0);
  let s = 0;
  for (let i = 0; i < expected.length; i++) {
    const e = Math.max(expected[i] / eT, 0.001), a = Math.max(actual[i] / aT, 0.001);
    s += (a - e) * Math.log(a / e);
  }
  return s;
}

// Baseline from the first 7 days (a rolling window in a real system).
let base = 0;
for (let d = 1; d <= 7; d++) base += passes(d);
const p0 = base / (7 * N);
const sigma = Math.sqrt((p0 * (1 - p0)) / N);
const K = 0.5, H = 4;                          // CUSUM slack and alarm level, in sigmas
console.log(`baseline pass rate ${(100 * p0).toFixed(1)}%, daily sigma ${(100 * sigma).toFixed(2)} pts, ` +
  `95% interval for one day +/-${(196 * sigma).toFixed(1)} pts`);
console.log("day  pass%     z  cusum   psi  alarms");
let S = 0;
const first = {};
for (let day = 1; day <= 21; day++) {
  const p = passes(day) / N;
  const z = (p - p0) / sigma;
  S = Math.max(0, S + (p0 - p) / sigma - K);
  const drift = psi(BASE_MIX, topicCounts(day));
  const alarms = [];
  if (p < 0.8) alarms.push("FIXED<80%");
  if (z < -3) alarms.push("Z<-3");
  if (S > H) alarms.push("CUSUM");
  if (drift > 0.25) alarms.push("PSI>0.25");
  else if (drift > 0.1) alarms.push("PSI>0.1");
  for (const a of alarms) if (!(a in first)) first[a] = day;
  console.log(`${String(day).padStart(3)}  ${(100 * p).toFixed(1).padStart(5)} ${z.toFixed(2).padStart(5)}` +
    ` ${S.toFixed(2).padStart(6)} ${drift.toFixed(3)}${alarms.length ? "  " + alarms.join(" ") : ""}`);
}
console.log("first alarm day per detector:");
for (const a of ["FIXED<80%", "Z<-3", "CUSUM", "PSI>0.25"]) console.log(`  ${a.padEnd(9)} ${a in first ? "day " + first[a] : "never"}`);

// Slice view on day 18: the drop is concentrated, which points at the cause.
const slices = { billing: [69, 75], bug: [47, 51], account: [34, 36], other: [15, 18], promo: [9, 20] };
const slicesBefore = { billing: [66, 72], bug: [50, 54], account: [34, 36], other: [15, 18], promo: [17, 20] };
console.log("day 18 by topic (day 14 -> day 18):");
for (const t of TOPICS) {
  const [b, bn] = slicesBefore[t], [a, an] = slices[t];
  console.log(`  ${t.padEnd(8)} ${(100 * b / bn).toFixed(1)}% -> ${(100 * a / an).toFixed(1)}%`);
}
// How many judged samples a day to see a 3-point drop in one day (z=1.96, 80% power z=0.84)?
const n3 = Math.ceil(((1.96 + 0.84) ** 2 * p0 * (1 - p0)) / 0.03 ** 2);
console.log(`samples per day to detect a 3-point drop in one day: ${n3}`);