How do you estimate and control LLM costs in production?

// Estimate a monthly LLM bill from traffic assumptions, then apply levers.
// Prices are a HYPOTHETICAL card in $ per million tokens; use your provider's page.
const PRICE = { large: { in: 3.0, out: 15.0 }, small: { in: 0.8, out: 4.0 } };
const CACHE_WRITE = 1.25, CACHE_READ = 0.1, BATCH = 0.5; // multipliers on input price
const DAYS = 30, HIT = 0.98; // assume 98% of prefix lookups hit a warm cache

// one-call features: stable prefix + variable part + output, per request
const features = {
  chat: { perDay: 40000, prefix: 2500, variable: 3500, out: 350, model: "large" },
  tagging: { perDay: 100000, prefix: 600, variable: 200, out: 10, model: "large" },
  summaries: { perDay: 5000, prefix: 500, variable: 7500, out: 500, model: "large" },
};
const agent = { perDay: 2000, steps: 10, prefix: 4000, toolResult: 1500, out: 200 };

function callCost(model, uncached, written, read, out, batch) {
  const p = PRICE[model];
  const input = (uncached + written * CACHE_WRITE + read * CACHE_READ) * p.in;
  return ((input + out * p.out) / 1e6) * (batch ? BATCH : 1);
}

function monthly(cfg) {
  const rows = {};
  for (const [name, f] of Object.entries(features)) {
    const model = name === "tagging" && cfg.route ? "small" : f.model;
    const out = name === "chat" && cfg.concise ? 250 : f.out;
    const batch = name === "summaries" && cfg.batch;
    let c;
    if (cfg.cache) {
      const hit = callCost(model, f.variable, 0, f.prefix, out, batch);
      const miss = callCost(model, f.variable, f.prefix, 0, out, batch);
      c = HIT * hit + (1 - HIT) * miss;
    } else c = callCost(model, f.prefix + f.variable, 0, 0, out, batch);
    rows[name] = c * f.perDay * DAYS;
  }
  // the agent re-sends the whole transcript every step
  const grow = (cfg.compact ? 400 : agent.toolResult) + agent.out;
  let task = 0;
  for (let i = 1; i <= agent.steps; i++) {
    const history = (i - 1) * grow;
    if (!cfg.cache) task += callCost("large", agent.prefix + history, 0, 0, agent.out, false);
    else {
      // prefix and older turns are read from cache; the newest turn is written
      const newest = i === 1 ? 0 : grow;
      task += callCost("large", 0, newest, agent.prefix + history - newest, agent.out, false);
    }
  }
  rows.agent = task * agent.perDay * DAYS;
  return rows;
}

const total = (r) => Object.values(r).reduce((a, b) => a + b, 0);
const usd = (x) => "$" + x.toFixed(0).padStart(7);

console.log("agent task, input tokens per step (the transcript is re-sent):");
let sum = 0;
const steps = [];
for (let i = 1; i <= agent.steps; i++) {
  const t = agent.prefix + (i - 1) * (agent.toolResult + agent.out);
  sum += t; steps.push(t);
}
console.log(`  ${steps.join(" ")}`);
console.log(`  total ${sum} tokens vs naive 10 x ${agent.prefix} = ${10 * agent.prefix} (${(sum / (10 * agent.prefix)).toFixed(2)}x)`);
console.log("");
const base = monthly({});
console.log("baseline monthly bill by feature:");
for (const [k, v] of Object.entries(base)) console.log(`  ${k.padEnd(10)} ${usd(v)}`);
console.log(`  ${"total".padEnd(10)} ${usd(total(base))}`);
console.log("");
console.log("levers applied one after another:        monthly   saved");
const levers = [
  ["route tagging to the small model", { route: true }],
  ["prompt-cache stable prefixes", { cache: true }],
  ["batch API for nightly summaries", { batch: true }],
  ["concise chat answers (350 -> 250)", { concise: true }],
  ["compact agent tool results (1500 -> 400)", { compact: true }],
];
let cfg = {}, prev = total(base);
for (const [name, add] of levers) {
  cfg = { ...cfg, ...add };
  const t = total(monthly(cfg));
  console.log(`  ${name.padEnd(40)} ${usd(t)}  ${((1 - t / prev) * 100).toFixed(1).padStart(4)}%`);
  prev = t;
}
console.log(`  overall: ${((1 - prev / total(base)) * 100).toFixed(1)}% lower than baseline`);

// Metering: price every call from its usage record, then attribute the cost
// to features, tenants and outcomes. Hypothetical prices, $ per million tokens.
const PRICE = { large: { in: 3.0, out: 15.0, read: 0.3 }, small: { in: 0.8, out: 4.0, read: 0.08 } };
// usage records as an API returns them, tagged by our gateway
// [feature, tenant, model, input, cachedRead, output, purpose, ticketId, resolved]
const log = [
  ["support", "acme", "large", 6200, 2500, 340, "answer", "T1", true],
  ["support", "acme", "large", 6900, 2500, 410, "answer", "T1", true],
  ["support", "globex", "large", 5800, 0, 300, "answer", "T2", false],
  ["support", "globex", "large", 6100, 2500, 900, "answer", "T3", true],
  ["support", "globex", "large", 6100, 2500, 880, "json-repair", "T3", true],
  ["support", "initech", "large", 7400, 2500, 350, "answer", "T4", false],
  ["tagging", "acme", "small", 800, 600, 8, "answer", null, false],
  ["tagging", "globex", "small", 820, 600, 9, "answer", null, false],
  ["tagging", "initech", "small", 790, 0, 7, "answer", null, false],
  ["agent", "initech", "large", 19300, 15900, 200, "answer", "T5", true],
  ["agent", "initech", "large", 21000, 17600, 220, "retry-after-timeout", "T5", true],
];

function cost([, , model, input, read, output]) {
  const p = PRICE[model];
  return ((input - read) * p.in + read * p.read + output * p.out) / 1e6;
}
function groupBy(idx) {
  const m = new Map();
  for (const r of log) m.set(r[idx], (m.get(r[idx]) || 0) + cost(r));
  return [...m.entries()].sort((a, b) => b[1] - a[1]);
}
const fmt = (x) => "$" + x.toFixed(4);
const all = log.reduce((a, r) => a + cost(r), 0);
console.log(`${log.length} calls, total ${fmt(all)}`);
console.log("by feature:");
for (const [k, v] of groupBy(0)) console.log(`  ${k.padEnd(8)} ${fmt(v)}  ${((v / all) * 100).toFixed(1).padStart(5)}%`);
console.log("by tenant:");
for (const [k, v] of groupBy(1)) console.log(`  ${k.padEnd(8)} ${fmt(v)}  ${((v / all) * 100).toFixed(1).padStart(5)}%`);

const overhead = log.filter((r) => r[6] !== "answer").reduce((a, r) => a + cost(r), 0);
console.log(`calls that were repairs or retries: ${fmt(overhead)} (${((overhead / all) * 100).toFixed(1)}% of spend)`);

// cost per outcome: all spend on a ticket, divided over resolved tickets
const tickets = new Map();
for (const r of log) if (r[7]) {
  const t = tickets.get(r[7]) || { cost: 0, resolved: r[8] };
  t.cost += cost(r); tickets.set(r[7], t);
}
let ticketSpend = 0, resolved = 0;
for (const t of tickets.values()) { ticketSpend += t.cost; if (t.resolved) resolved++; }
console.log(`tickets: ${tickets.size}, resolved: ${resolved}`);
console.log(`  cost per ticket touched ${fmt(ticketSpend / tickets.size)}, cost per resolved ticket ${fmt(ticketSpend / resolved)}`);
const cacheShare = log.reduce((a, r) => a + r[4], 0) / log.reduce((a, r) => a + r[3], 0);
console.log(`share of input tokens read from cache: ${(cacheShare * 100).toFixed(1)}%`);