How do you design for high availability?

// Availability arithmetic for a web service: nines, serial chains, redundancy,
// failover time, correlated (same-zone) failures and N+1 capacity.
// All component numbers are assumptions for the example, not vendor SLAs.
const YEAR_MIN = 365 * 24 * 60, MONTH_MIN = 30 * 24 * 60;
const pct = (a) => (a * 100).toFixed(6) + "%";
const down = (a) => { const m = YEAR_MIN * (1 - a); return m >= 1 ? m.toFixed(1) + " min/yr" : (m * 60).toFixed(2) + " s/yr"; };
const serial = (xs) => xs.reduce((p, x) => p * x, 1);
const parallel = (a, n) => 1 - Math.pow(1 - a, n);

console.log("== 1. what each target allows ==");
for (const [label, a] of [["99%", 0.99], ["99.9%", 0.999], ["99.95%", 0.9995], ["99.99%", 0.9999], ["99.999%", 0.99999]])
  console.log(label.padEnd(8) + " " + (YEAR_MIN * (1 - a)).toFixed(1).padStart(7) + " min/yr  " + (MONTH_MIN * (1 - a)).toFixed(1).padStart(6) + " min per 30 days");

console.log("");
console.log("== 2. one copy of everything (serial chain) ==");
const parts = { dns: 0.9999, lb: 0.9999, app: 0.999, db: 0.9995, cache: 0.999 };
const chain = serial(Object.values(parts));
console.log("dns x lb x app x db x cache = " + pct(chain) + " -> " + down(chain));
console.log("weakest part alone: app " + pct(parts.app) + "; the chain is worse than every part");

console.log("");
console.log("== 3. redundancy: independent copies in parallel ==");
for (const n of [1, 2, 3]) console.log("app x" + n + " = " + pct(parallel(parts.app, n)) + " -> " + down(parallel(parts.app, n)));

console.log("");
console.log("== 4. a standby does not help until failover finishes ==");
const FAILS = 4; // primary database failures per year (assumption)
for (const [name, detect, ttl, promote] of [["manual restore", 0, 0, 3600], ["health check 30s x3", 90, 60, 30], ["health check 10s x3", 30, 60, 30]]) {
  const sec = FAILS * (detect + ttl + promote), a = 1 - sec / (YEAR_MIN * 60);
  console.log(name.padEnd(20) + " " + String(detect + ttl + promote).padStart(4) + " s per failover x " + FAILS + " = " + String(sec).padStart(5) + " s/yr -> db " + pct(a));
}

console.log("");
console.log("== 5. correlated failure: three app copies in one zone vs three zones ==");
const zone = 0.9995;
const oneZone = zone * parallel(parts.app, 3), threeZones = parallel(zone * parts.app, 3);
console.log("3 copies, 1 zone  = " + pct(oneZone) + " -> " + down(oneZone));
console.log("3 copies, 3 zones = " + pct(threeZones) + " -> " + down(threeZones));

console.log("");
console.log("== 6. N+1 capacity: surviving a loss without overload ==");
const PEAK = 1200, PER = 400, ZONES = 3;
const base = Math.ceil(PEAK / PER);
console.log("peak " + PEAK + " rps, " + PER + " rps per instance: " + base + " needed, " + (base + 1) + " to survive one instance");
const perZone = Math.ceil(PEAK / (PER * (ZONES - 1)));
console.log("survive a whole zone: " + perZone + " per zone x " + ZONES + " zones = " + perZone * ZONES + " instances, " + Math.round((100 * PEAK) / (perZone * ZONES * PER)) + "% busy at peak");
console.log("same 6 instances if one zone dies: " + Math.round((100 * PEAK) / (perZone * (ZONES - 1) * PER)) + "% busy; with only " + (base + 1) + " instances spread 2/1/1, losing the 2-instance zone puts the other 2 at " + Math.round((100 * PEAK) / ((base - 1) * PER)) + "% of capacity");

console.log("");
console.log("== 7. the redesigned system ==");
const dbHA = 1 - (FAILS * 120) / (YEAR_MIN * 60);
const total = serial([parts.dns, parts.lb, threeZones, dbHA, parallel(parts.cache, 2)]);
console.log("dns x lb x app(3 zones) x db(10s checks) x cache x2 = " + pct(total) + " -> " + down(total));