How do you make a payment system reliable against retries?

// Payment retries in miniature: a fake PSP (card processor) that honours its own
// idempotency keys, a payment service with a payments table + ledger, and injected faults.

class PSP {
  constructor() { this.charges = []; this.keys = new Map(); this.loseNext = 0; }
  charge(key, amountCents) {
    if (key && this.keys.has(key)) return { id: this.keys.get(key), replayed: true };
    const id = `ch_${this.charges.length + 1}`;
    this.charges.push({ id, amountCents });
    if (key) this.keys.set(key, id);
    if (this.loseNext > 0) { this.loseNext--; return null; } // charged, but the reply is lost
    return { id, replayed: false };
  }
}

class PaymentService {
  constructor(psp, { apiKeys, pspKeys }) {
    this.psp = psp; this.apiKeys = apiKeys; this.pspKeys = pspKeys;
    this.payments = new Map(); // idempotency key -> row
    this.ledger = []; this.seq = 0; this.crashAfterPsp = false;
  }
  pay(key, amountCents) {
    if (this.apiKeys) {
      const row = this.payments.get(key);
      if (row && row.state === "SUCCEEDED") return { status: 200, body: `${row.id} ${row.chargeId} (replayed)` };
      if (row) return { status: 409, body: `${row.id} still ${row.state}, retry later` };
    }
    const row = { id: `pay_${++this.seq}`, state: "PENDING_PSP", amountCents, chargeId: null };
    if (this.apiKeys) this.payments.set(key, row); // INSERT ... unique(key): first writer wins
    for (let attempt = 1; attempt <= 3; attempt++) {
      const res = this.psp.charge(this.pspKeys ? row.id : null, amountCents);
      if (res === null) { console.log(`    psp attempt ${attempt}: timeout`); continue; }
      if (this.crashAfterPsp) { this.crashAfterPsp = false; console.log("    service crashed after PSP said ok"); return null; }
      this.succeed(row, res.id);
      return { status: 201, body: `${row.id} ${res.id}${res.replayed ? " (psp replayed)" : ""}` };
    }
    row.state = "UNKNOWN"; return { status: 502, body: `${row.id} outcome unknown` };
  }
  succeed(row, chargeId) {
    row.state = "SUCCEEDED"; row.chargeId = chargeId;
    this.ledger.push({ account: "customer", cents: -row.amountCents, ref: row.id });
    this.ledger.push({ account: "merchant", cents: row.amountCents, ref: row.id });
  }
  recover() { // background job: finish rows stuck in PENDING_PSP or UNKNOWN using the same PSP key
    for (const row of this.payments.values()) {
      if (row.state === "SUCCEEDED") continue;
      const res = this.psp.charge(row.id, row.amountCents);
      console.log(`    recovery: ${row.id} -> ${res.id}${res.replayed ? " (psp replayed)" : ""}`);
      this.succeed(row, res.id);
    }
  }
}

function report(svc, psp) {
  const charged = psp.charges.reduce((s, c) => s + c.amountCents, 0);
  const sum = svc.ledger.reduce((s, e) => s + e.cents, 0);
  const merchant = svc.ledger.filter((e) => e.account === "merchant").reduce((s, e) => s + e.cents, 0);
  console.log(`  card charged ${psp.charges.length}x = $${(charged / 100).toFixed(2)} | ledger merchant $${(merchant / 100).toFixed(2)}, entries sum to ${sum} | reconciled=${charged === merchant}`);
}

function client(svc, key, attempts) {
  for (let i = 1; i <= attempts; i++) {
    const r = svc.pay(key, 5000);
    if (r === null) { console.log(`  client attempt ${i}: timeout`); continue; }
    console.log(`  client attempt ${i}: ${r.status} ${r.body}`);
    if (r.status < 300) return;
  }
}

console.log("== A. No keys anywhere; attempt 1 dies after the charge, client sees a timeout ==");
let psp = new PSP(), svc = new PaymentService(psp, { apiKeys: false, pspKeys: false });
svc.crashAfterPsp = true; // the client never hears back from the first attempt
client(svc, "k1", 3); report(svc, psp);

console.log("\n== B. Key on our API only; our call to the PSP times out once ==");
psp = new PSP(); psp.loseNext = 1; svc = new PaymentService(psp, { apiKeys: true, pspKeys: false });
client(svc, "k1", 3); report(svc, psp);

console.log("\n== C. Keys end to end (API key + PSP key = payment id) ==");
psp = new PSP(); psp.loseNext = 1; svc = new PaymentService(psp, { apiKeys: true, pspKeys: true });
client(svc, "k1", 3); client(svc, "k1", 1); report(svc, psp);

console.log("\n== D. Crash after the PSP charged, before we recorded it ==");
psp = new PSP(); svc = new PaymentService(psp, { apiKeys: true, pspKeys: true });
svc.crashAfterPsp = true;
client(svc, "k1", 2);
svc.recover();
client(svc, "k1", 1); report(svc, psp);

console.log("\n== E. Retry schedule: exponential backoff, full jitter, base 100 ms, cap 2000 ms ==");
let seed = 42;
const rand = () => (seed = (seed * 48271) % 2147483647) / 2147483647;
const delays = [];
for (let attempt = 0; attempt < 6; attempt++) {
  const ceiling = Math.min(2000, 100 * 2 ** attempt);
  delays.push(`${Math.floor(rand() * ceiling)}/${ceiling}`);
}
console.log(`  delay/ceiling ms: ${delays.join("  ")}`);