How would you design a metrics monitoring and alerting system?

// SLO alerting: a naive threshold vs multiwindow, multi-burn-rate alerts, plus label cardinality arithmetic.
// Times are minutes since the trouble started. SLO 99.9% over 30 days, 1000 requests/minute, 4 simulated days per scenario, trouble starts at minute 600.
const RPM = 1000, MINUTES = 4 * 24 * 60, START = 600;
const BUDGET = (RPM * 30 * 24 * 60) / 1000; // 0.1% of 30 days of requests
// burn is in tenths (144 = 14.4x); windows in minutes; both windows must exceed the burn rate
const RULES = [
  ["page  14.4x 1h+5m", 144, 60, 5],
  ["page  6x    6h+30m", 60, 360, 30],
  ["ticket 1x   3d+6h", 10, 4320, 360],
];
const SCENARIOS = [
  ["blip: 20% errors for 2 min", (t) => (t >= START && t < START + 2 ? 200 : 0)],
  ["outage: 10% errors for 60 min", (t) => (t >= START && t < START + 60 ? 100 : 0)],
  ["slow burn: 0.8% errors from then on", (t) => (t >= START ? 8 : 0)],
  ["trickle: 0.3% errors from then on", (t) => (t >= START ? 3 : 0)],
];
const tenths = (num, den) => { const x = Math.floor((num * 10 + Math.floor(den / 2)) / den); return `${Math.floor(x / 10)}.${x % 10}`; };

function evaluate(firing) {
  let first = -1, fires = 0, cleared = -1, prev = false;
  for (let t = 0; t < MINUTES; t++) {
    const f = firing(t);
    if (f && !prev) { fires++; if (first < 0) first = t; }
    if (!f && prev && cleared < 0) cleared = t;
    prev = f;
  }
  if (first < 0) return "never";
  return `fires after ${first - START + 1} min, ${fires} firing(s)` + (cleared >= 0 ? `, clears after ${cleared - START + 1} min` : ", still firing at the end");
}

for (const [name, errAt] of SCENARIOS) {
  const prefix = [0];
  for (let t = 0; t < MINUTES; t++) prefix.push(prefix[t] + errAt(t));
  const errorsIn = (t, w) => prefix[t + 1] - prefix[Math.max(0, t + 1 - w)];
  const requestsIn = (t, w) => RPM * w; // the service ran cleanly for weeks before the simulation
  // error ratio > burn x 0.1%  <=>  errors * 10000 > burnTenths * requests
  const over = (t, w, burn) => errorsIn(t, w) * 10000 > burn * requestsIn(t, w);
  console.log(`== ${name} ==`);
  console.log(`  error budget used: ${tenths(prefix[MINUTES] * 100, BUDGET)}% of the 30-day budget (${prefix[MINUTES]} of ${BUDGET} errors)`);
  console.log(`  naive >1% in 1m     ${evaluate((t) => errorsIn(t, 1) * 100 > RPM)}`);
  for (const [label, burn, long, short] of RULES) {
    console.log(`  ${label.padEnd(19)} ${evaluate((t) => over(t, long, burn) && over(t, short, burn))}`);
  }
}

console.log("\n== Cardinality: every label multiplies the number of series ==");
const services = 200, pods = 10, endpoints = 20, statuses = 4, histSeries = 12 + 2;
const series = services * pods * endpoints * statuses * histSeries;
const perDay = series * ((24 * 60 * 60) / 15);
console.log(`  ${services} services x ${pods} pods x ${endpoints} endpoints x ${statuses} status classes x ${histSeries} histogram series = ${series} series`);
console.log(`  scraped every 15 s: ${perDay} samples/day, ${tenths(perDay * 137, 100 * 1000000000)} GB/day at 1.37 B/sample`);
console.log(`  add a user_id label with 100000 active users: ${series * 100000} series`);

// Gorilla-style compression (delta-of-delta timestamps + XOR float values): count the bits for
// one 2-hour block (480 samples at a 15 s scrape interval) of four kinds of series.
let seed = 99;
function rand() { seed = (seed * 48271) % 2147483647; return seed / 2147483647; }
const N = 480;

function timestampBits(ts) {
  let bits = 64 + 14, prevDelta = ts[1] - ts[0]; // header timestamp + first delta
  for (let i = 2; i < ts.length; i++) {
    const delta = ts[i] - ts[i - 1], dod = delta - prevDelta; prevDelta = delta;
    if (dod === 0) bits += 1;
    else if (dod >= -63 && dod <= 64) bits += 2 + 7;
    else if (dod >= -255 && dod <= 256) bits += 3 + 9;
    else if (dod >= -2047 && dod <= 2048) bits += 4 + 12;
    else bits += 4 + 32;
  }
  return bits;
}
const view = new DataView(new ArrayBuffer(8));
function f64bits(v) { view.setFloat64(0, v); return view.getBigUint64(0); }
function lz(x) { return 64 - x.toString(2).length; }
function tz(x) { let n = 0; while ((x & 1n) === 0n) { x >>= 1n; n++; } return n; }

function valueBits(vals) {
  let bits = 64, prev = f64bits(vals[0]), pl = -1, pt = -1, zero = 0, reuse = 0, fresh = 0;
  for (let i = 1; i < vals.length; i++) {
    const cur = f64bits(vals[i]), x = cur ^ prev; prev = cur;
    if (x === 0n) { bits += 1; zero++; continue; }
    const l = Math.min(lz(x), 31), t = tz(x);
    if (pl >= 0 && l >= pl && t >= pt) { bits += 2 + (64 - pl - pt); reuse++; }
    else { bits += 2 + 5 + 6 + (64 - l - t); pl = l; pt = t; fresh++; }
  }
  return { bits, zero, reuse, fresh };
}

const ts = [1700000000];
for (let i = 1; i < N; i++) ts.push(ts[i - 1] + 15 + (rand() < 0.05 ? (rand() < 0.5 ? -1 : 1) : 0));
const series = { counter: [], constant: [], gauge_tenths: [], random_float: [] };
const LOOKS = { counter: "43, 94, 137, 187 ...", constant: "1, 1, 1 ...", gauge_tenths: "45.1, 45.2, 45.1 ...", random_float: "0.4999..., 0.3657 ..." };
let total = 0, tenths = 450;
for (let i = 0; i < N; i++) {
  total += 40 + Math.floor(rand() * 20); series.counter.push(total);
  series.constant.push(1);
  tenths += rand() < 0.5 ? -1 : 1; series.gauge_tenths.push(tenths / 10);
  series.random_float.push(rand());
}
const tBits = timestampBits(ts);
console.log(`timestamps: ${N} samples, jittered 15 s interval -> ${tBits} bits (${(tBits / N).toFixed(2)} bits/sample)`);
console.log("series          example values             zero-xor reuse  new  bytes/sample  vs 16 B raw");
for (const [name, vals] of Object.entries(series)) {
  const v = valueBits(vals);
  const bytes = Math.ceil((tBits + v.bits) / 8);
  console.log(`${name.padEnd(15)} ${LOOKS[name].padEnd(26)} ${String(v.zero).padStart(7)} ${String(v.reuse).padStart(5)} ${String(v.fresh).padStart(4)} ${(bytes / N).toFixed(2).padStart(12)} ${((16 * N) / bytes).toFixed(1).padStart(9)}x`);
}