How should an agent handle tool errors, timeouts and retries?

// Tool-call wrapper: classify the error, retry only transient faults with backoff and
// jitter, respect Retry-After, a per-attempt timeout, a total deadline and a circuit
// breaker, and hand the model a short, safe, actionable message. Fake clock, seeded jitter.
let now = 0; // fake clock in ms
let seed = 12345;
function rand() { seed = (seed * 48271) % 2147483647; return seed / 2147483647; }

const POLICY = { timeoutMs: 3000, maxAttempts: 4, baseMs: 250, capMs: 4000, deadlineMs: 10000 };
const RETRYABLE = new Set(["timeout", 429, 500, 502, 503]);
const breakers = {};

function scripted(outcomes) { let i = 0; return () => outcomes[Math.min(i++, outcomes.length - 1)]; }
const TOOLS = {
  search: scripted([
    { ms: 120, status: 503, body: "Error: ECONNRESET at /srv/search/node_modules/pg/client.js:41" },
    { ms: 80, status: 429, retryAfterMs: 2000, body: "rate limited" },
    { ms: 150, status: 200, body: "3 results for 'refund window'" },
  ]),
  create_ticket: (args) => ["low", "normal", "high"].includes(args.priority)
    ? { ms: 90, status: 200, body: "ticket T-77 created" }
    : { ms: 60, status: 400, body: "priority must be one of low, normal, high (got " + args.priority + ")" },
  build_report: scripted([{ ms: 5000, status: 200, body: "report ready" }]),
};

function callTool(name, args) {
  const br = breakers[name] || (breakers[name] = { fails: 0, openUntil: -1 });
  const start = now;
  console.log("call " + name + JSON.stringify(args) + " at t=" + now);
  if (now < br.openUntil) return toModel(true, name + " is unavailable (circuit open for " + (br.openUntil - now) + " ms). Do not retry it; continue without it or tell the user.");
  for (let attempt = 1; ; attempt++) {
    const o = TOOLS[name](args);
    let status;
    if (o.ms > POLICY.timeoutMs) { now += POLICY.timeoutMs; status = "timeout"; }
    else { now += o.ms; status = o.status; }
    console.log("  attempt " + attempt + " -> " + status + " at t=" + now);
    if (status === 200) { br.fails = 0; return toModel(false, o.body); }
    if (!RETRYABLE.has(status)) return toModel(true, "Invalid request: " + o.body + ". Fix the arguments and call again.");
    br.fails += 1;
    if (br.fails >= 3) br.openUntil = now + 30000;
    let delay = Math.floor(rand() * Math.min(POLICY.capMs, POLICY.baseMs * 2 ** attempt));
    if (o.retryAfterMs) delay = Math.max(delay, o.retryAfterMs);
    const outOfTime = now + delay + POLICY.timeoutMs > start + POLICY.deadlineMs;
    if (attempt >= POLICY.maxAttempts || outOfTime || br.fails >= 3) {
      const why = br.fails >= 3 ? "circuit opened" : outOfTime ? "deadline reached" : "attempts used up";
      return toModel(true, name + " failed: " + status + ", " + why + ", attempts=" + attempt + ", elapsed " + (now - start) +
        " ms. It is temporary: try a smaller request later, or answer without it.");
    }
    console.log("  wait " + delay + " ms" + (o.retryAfterMs ? " (Retry-After)" : " (backoff+jitter)"));
    now += delay;
  }
}
function toModel(isError, content) {
  const block = { type: "tool_result", is_error: isError, content: content };
  console.log("  model sees: " + JSON.stringify(block));
  return block;
}

callTool("search", { q: "refund window" });
callTool("create_ticket", { priority: "urgent" });
callTool("create_ticket", { priority: "high" }); // the model fixed its own arguments
callTool("build_report", { month: "2026-09" });
now += 1000;
callTool("build_report", { month: "2026-09" }); // the model tries again later
callTool("build_report", { month: "2026-09" }); // and again: now it fails fast

// A timeout on a write is ambiguous: did it commit? Retrying blindly can double-charge.
// An idempotency key lets the server recognise the retry and replay the first result.
function makeServer() {
  const ledger = [];          // committed charges (stands in for the payments database)
  const seenKeys = new Map(); // idempotency key -> { fingerprint, response }
  let dropNextResponse = true; // the first response is lost on the way back
  return {
    ledger,
    charge(req) {
      const fingerprint = req.customer + ":" + req.cents;
      if (req.key && seenKeys.has(req.key)) {
        const prev = seenKeys.get(req.key);
        if (prev.fingerprint !== fingerprint) return { status: 422, body: "key reused with different parameters" };
        return { status: 200, body: prev.response.body, replayed: true };
      }
      const id = "ch_" + (ledger.length + 1);
      ledger.push({ id: id, customer: req.customer, cents: req.cents });
      const response = { status: 200, body: id };
      if (req.key) seenKeys.set(req.key, { fingerprint: fingerprint, response: response });
      if (dropNextResponse) { dropNextResponse = false; return { status: "timeout" }; }
      return response;
    },
  };
}

// The tool the agent calls. The key is derived from the agent run and the step, so a
// retry of the same step reuses it, and a deliberately new charge gets a new one.
function chargeTool(server, args, ctx, useKey) {
  const key = useKey ? ctx.runId + ":step" + ctx.step + ":charge" : null;
  for (let attempt = 1; attempt <= 3; attempt++) {
    const res = server.charge({ customer: args.customer, cents: args.cents, key: key });
    console.log("  attempt " + attempt + " key=" + (key || "none") + " -> " + res.status +
      (res.body ? " " + res.body : "") + (res.replayed ? " (replayed, no new charge)" : ""));
    if (res.status === 200) return res.body;
    if (res.status !== "timeout") return "error: " + res.body;
  }
  return "error: unknown outcome, check before retrying";
}

for (const useKey of [false, true]) {
  console.log("== retry " + (useKey ? "with" : "without") + " an idempotency key");
  const server = makeServer();
  const result = chargeTool(server, { customer: "cus_9", cents: 4999 }, { runId: "run42", step: 3 }, useKey);
  const total = server.ledger.reduce((n, c) => n + c.cents, 0);
  console.log("  tool returns " + result + "; ledger has " + server.ledger.length + " charge(s), total " + (total / 100).toFixed(2) + " USD");
}

console.log("== same key, different amount (a bug or a confused model)");
const s = makeServer();
s.charge({ customer: "cus_9", cents: 4999, key: "run42:step3:charge" });
console.log("  " + JSON.stringify(s.charge({ customer: "cus_9", cents: 9999, key: "run42:step3:charge" })));