How do you build resilient LLM calls with rate limits, retries and fallbacks?

// A resilient LLM call: retry only transient errors, honour Retry-After,
// back off with full jitter, stop at a deadline, trip a circuit breaker,
// then fall back down a chain. The providers are stubs on a fake clock.
let now = 0; // fake clock in ms
let seed = 2026; // Park-Miller generator, identical in every language
function rand() { seed = (seed * 48271) % 2147483647; return seed / 2147483647; }
const s = (ms) => (ms / 1000).toFixed(3) + "s";

class StubProvider {
  constructor(name) { this.name = name; this.queue = []; this.fallback = { ok: true }; }
  call() { now += 300; return this.queue.length ? this.queue.shift() : this.fallback; }
}

class Breaker {
  constructor(threshold, cooldownMs) { this.threshold = threshold; this.cooldown = cooldownMs; this.fails = 0; this.state = "closed"; this.openedAt = 0; }
  allow() {
    if (this.state === "open" && now - this.openedAt >= this.cooldown) this.state = "half-open";
    return this.state !== "open";
  }
  record(ok) {
    if (ok) { this.fails = 0; this.state = "closed"; return; }
    this.fails += 1;
    if (this.state === "half-open" || this.fails >= this.threshold) { this.state = "open"; this.openedAt = now; }
  }
}

const RETRYABLE = new Set([408, 409, 429, 500, 502, 503, 504, 529]);
const MAX_ATTEMPTS = 3, BASE = 500, CAP = 8000, DEADLINE = 20000;

function callWithRetries(p, breaker, start, log) {
  for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
    if (!breaker.allow()) { log.push(`  t=${s(now)} ${p.name} breaker open, skipped`); return { ok: false, retryable: true }; }
    const probe = breaker.state === "half-open" ? " (half-open probe)" : "";
    const r = p.call();
    // a 4xx client error says nothing about provider health: do not count it
    if (r.ok || RETRYABLE.has(r.status)) breaker.record(!!r.ok);
    if (r.ok) { log.push(`  t=${s(now)} ${p.name} attempt ${attempt} -> 200${probe}`); return { ok: true }; }
    log.push(`  t=${s(now)} ${p.name} attempt ${attempt} -> ${r.status}${breaker.state === "open" ? " (breaker opens)" : ""}`);
    if (!RETRYABLE.has(r.status)) return { ok: false, retryable: false, status: r.status };
    if (attempt === MAX_ATTEMPTS || breaker.state === "open") break;
    let wait, why;
    if (r.retryAfter !== undefined) { wait = r.retryAfter * 1000; why = "retry-after header"; }
    else { wait = Math.floor(rand() * Math.min(CAP, BASE * 2 ** attempt)); why = `full jitter, cap ${BASE * 2 ** attempt}ms`; }
    if (now + wait - start > DEADLINE) { log.push(`  t=${s(now)} wait ${s(wait)} would pass the deadline, give up`); break; }
    log.push(`  t=${s(now)} wait ${s(wait)} (${why})`);
    now += wait;
  }
  return { ok: false, retryable: true };
}

const chain = [new StubProvider("primary"), new StubProvider("region-b"), new StubProvider("small-model")];
const breakers = new Map(chain.map((p) => [p.name, new Breaker(5, 30000)]));

function resilientCall(title) {
  const start = now, log = [];
  console.log(`== ${title} ==`);
  let result = "all tiers failed: show the degraded mode";
  for (const p of chain) {
    const r = callWithRetries(p, breakers.get(p.name), start, log);
    if (r.ok) { result = `ok from ${p.name}`; break; }
    if (!r.retryable) { result = `error ${r.status} returned to caller (a bad request fails on every tier)`; break; }
  }
  for (const line of log) console.log(line);
  console.log(`  result: ${result}, took ${s(now - start)}`);
}

const [primary] = chain;
primary.queue = [{ status: 429, retryAfter: 2 }];
resilientCall("1) 429 with retry-after: 2");
primary.queue = [{ status: 503 }, { status: 503 }];
resilientCall("2) two 503s, then success");
primary.queue = [{ status: 400 }];
resilientCall("3) 400 invalid request");
primary.queue = [{ status: 429, retryAfter: 30 }];
resilientCall("4) retry-after: 30 is longer than the 20s budget");
primary.fallback = { status: 503 };
resilientCall("5) primary outage begins");
resilientCall("6) outage continues");
resilientCall("7) breaker is open");
now += 30000;
primary.fallback = { ok: true };
resilientCall("8) 30s later: half-open probe");
console.log(`primary breaker: ${breakers.get("primary").state}`);

// Part 1: 1000 clients fail at the same moment. How bunched are their retries?
// Part 2: a client-side token bucket keeps a burst under a tokens-per-minute limit.
let seed = 7; // Park-Miller generator, identical in every language
function rand() { seed = (seed * 48271) % 2147483647; return seed / 2147483647; }

const CLIENTS = 1000, BASE = 1000, BUCKET_MS = 100;
const strategies = {
  "no jitter": (cap) => cap,
  "equal jitter": (cap) => cap / 2 + Math.floor(rand() * (cap / 2)),
  "full jitter": (cap) => Math.floor(rand() * cap),
};
console.log("part 1: 1000 clients fail at t=0, retry twice (waits capped at 1s, then 2s)");
for (const [name, waitFor] of Object.entries(strategies)) {
  const hits = new Map(); // 100 ms window -> retries that land in it
  for (let c = 0; c < CLIENTS; c++) {
    let t = 0;
    for (let attempt = 1; attempt <= 2; attempt++) {
      t += waitFor(BASE * 2 ** (attempt - 1));
      const w = Math.floor(t / BUCKET_MS);
      hits.set(w, (hits.get(w) || 0) + 1);
    }
  }
  const peak = Math.max(...hits.values());
  console.log(`  ${name.padEnd(13)} peak ${String(peak).padStart(4)} retries in one 100ms window, spread over ${hits.size} windows`);
}

console.log("part 2: 30 requests of 3000 tokens arrive at once; limit 60000 tokens/minute");
const LIMIT = 60000, RATE = 1; // the bucket refills 1 token per ms (60000 per minute)
function makeBucket() { return { tokens: LIMIT, t: 0 }; }
function refill(b, now) { b.tokens = Math.min(LIMIT, b.tokens + (now - b.t) * RATE); b.t = now; }

// a) fire everything: the provider bucket answers 429 when it runs dry
const server = makeBucket();
let ok = 0, rejected = 0;
for (let i = 0; i < 30; i++) {
  refill(server, 0);
  if (server.tokens >= 3000) { server.tokens -= 3000; ok++; } else rejected++;
}
console.log(`  a) no client limiter: ${ok} accepted, ${rejected} rejected with 429 (each one now retries)`);

// b) mirror the bucket on the client and wait for tokens before sending
const local = makeBucket(), server2 = makeBucket();
let now = 0, rejected2 = 0;
const sendTimes = [];
for (let i = 0; i < 30; i++) {
  refill(local, now);
  if (local.tokens < 3000) { now += Math.ceil((3000 - local.tokens) / RATE); refill(local, now); }
  local.tokens -= 3000;
  refill(server2, now);
  if (server2.tokens >= 3000) server2.tokens -= 3000; else rejected2++;
  sendTimes.push(now);
}
console.log(`  b) client token bucket: ${30 - rejected2} accepted, ${rejected2} rejected`);
console.log(`     first 20 sent at t=0s, then one every ${(sendTimes[21] - sendTimes[20]) / 1000}s, last at t=${sendTimes[29] / 1000}s`);