How does streaming work in LLM APIs and why does it matter for UX?

// Blocking vs streaming on a fake clock: when does the user first see text,
// what does a buffering proxy do to that, and what does cancellation save?
// All times are simulated milliseconds; nothing here calls a real model.

const TTFT_MS = 700;        // queueing + prefill before the first output token
const MS_PER_TOKEN = 30;    // decode speed: about 33 tokens per second
const ANSWER_TOKENS = 300;  // length of the full answer
const CHARS_PER_TOKEN = 4;  // rough English average
const SSE_OVERHEAD = 60;    // bytes of "data: {...}" framing around each delta

// Time at which output token i (0-based) leaves the model.
const tokenAt = (i) => TTFT_MS + i * MS_PER_TOKEN;
const lastTokenAt = tokenAt(ANSWER_TOKENS - 1);

// A proxy that holds bytes until its buffer is full (or the response ends).
function firstFlushThroughBuffer(bufferBytes) {
  let held = 0;
  for (let i = 0; i < ANSWER_TOKENS; i++) {
    held += CHARS_PER_TOKEN + SSE_OVERHEAD;
    if (held >= bufferBytes) return { at: tokenAt(i), tokens: i + 1 };
  }
  return { at: lastTokenAt, tokens: ANSWER_TOKENS };
}

const pad = (s, n) => String(s).padEnd(n);
const row = (name, first, full, note) =>
  console.log(pad(name, 36) + pad(first + " ms", 12) + pad(full + " ms", 12) + note);

console.log("Answer: " + ANSWER_TOKENS + " tokens, TTFT " + TTFT_MS + " ms, " + MS_PER_TOKEN + " ms/token");
console.log("");
console.log(pad("delivery", 36) + pad("first text", 12) + pad("complete", 12) + "note");
row("blocking (stream: false)", lastTokenAt, lastTokenAt, "spinner the whole time");
row("streaming, direct", tokenAt(0), lastTokenAt, "first token shown at once");
const f4 = firstFlushThroughBuffer(4096);
row("streaming via 4 KB buffering proxy", f4.at, lastTokenAt, "first flush carries " + f4.tokens + " tokens");
row("streaming via full-buffer proxy", lastTokenAt, lastTokenAt, "same as blocking");

console.log("");
console.log("Same total time in every row: streaming changes when text appears, not when it ends.");
console.log("Share of the wait spent before first text (direct streaming): " +
  (100 * tokenAt(0) / lastTokenAt).toFixed(1) + "%");

// The user reads the first part and presses Stop at t = 3000 ms.
const STOP_AT = 3000;
const generatedByStop = Math.floor((STOP_AT - TTFT_MS) / MS_PER_TOKEN) + 1;
console.log("");
console.log("User presses Stop at " + STOP_AT + " ms:");
console.log("  abort propagated to the provider -> " + generatedByStop + " output tokens generated");
console.log("  abort ignored (server keeps reading) -> " + ANSWER_TOKENS + " output tokens generated");
console.log("  wasted by ignoring it: " + (ANSWER_TOKENS - generatedByStop) + " tokens (" +
  (100 * (ANSWER_TOKENS - generatedByStop) / ANSWER_TOKENS).toFixed(1) + "% of the answer)");

// Reassemble a streamed answer from server-sent events shaped like the
// Anthropic Messages API stream (event names and fields follow its docs).
// The stream is a canned string: no network, no API key.

const ev = (name, obj) => "event: " + name + "\ndata: " + JSON.stringify(obj) + "\n\n";
const RAW =
  ev("message_start", { type: "message_start", message: { id: "msg_demo", role: "assistant", content: [], stop_reason: null, usage: { input_tokens: 25, output_tokens: 1 } } }) +
  ev("content_block_start", { type: "content_block_start", index: 0, content_block: { type: "text", text: "" } }) +
  ev("ping", { type: "ping" }) +
  ev("content_block_delta", { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: "Checking" } }) +
  ev("content_block_delta", { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: " the weather" } }) +
  ev("content_block_delta", { type: "content_block_delta", index: 0, delta: { type: "text_delta", text: " for Paris." } }) +
  ev("content_block_stop", { type: "content_block_stop", index: 0 }) +
  ev("content_block_start", { type: "content_block_start", index: 1, content_block: { type: "tool_use", id: "toolu_demo", name: "get_weather", input: {} } }) +
  ev("content_block_delta", { type: "content_block_delta", index: 1, delta: { type: "input_json_delta", partial_json: "" } }) +
  ev("content_block_delta", { type: "content_block_delta", index: 1, delta: { type: "input_json_delta", partial_json: "{\"city\":" } }) +
  ev("content_block_delta", { type: "content_block_delta", index: 1, delta: { type: "input_json_delta", partial_json: " \"Par" } }) +
  ev("content_block_delta", { type: "content_block_delta", index: 1, delta: { type: "input_json_delta", partial_json: "is\", \"unit\"" } }) +
  ev("content_block_delta", { type: "content_block_delta", index: 1, delta: { type: "input_json_delta", partial_json: ": \"celsius\"}" } }) +
  ev("content_block_stop", { type: "content_block_stop", index: 1 }) +
  ev("message_delta", { type: "message_delta", delta: { stop_reason: "tool_use", stop_sequence: null }, usage: { output_tokens: 42 } }) +
  ev("message_stop", { type: "message_stop" });

function parseJsonOrNull(s) {
  try { return JSON.parse(s); } catch { return null; }
}

const blocks = [];   // index -> { type, text | json, input }
let stopReason = null;
let usage = {};

for (const frame of RAW.split("\n\n")) {
  if (!frame) continue;
  const data = frame.split("\n").find((l) => l.startsWith("data: ")).slice(6);
  const e = JSON.parse(data);
  if (e.type === "ping") { console.log("ping                 (keep-alive, ignored)"); continue; }
  if (e.type === "message_start") { usage = { ...e.message.usage }; console.log("message_start        id=" + e.message.id); }
  if (e.type === "content_block_start") {
    const b = e.content_block;
    blocks[e.index] = b.type === "text" ? { type: "text", text: "" } : { type: "tool_use", name: b.name, json: "" };
    console.log("content_block_start  #" + e.index + " " + b.type);
  }
  if (e.type === "content_block_delta") {
    const b = blocks[e.index];
    if (e.delta.type === "text_delta") {
      b.text += e.delta.text;
      console.log("text_delta           #" + e.index + " screen now shows: " + JSON.stringify(b.text));
    } else {
      b.json += e.delta.partial_json;
      const ok = parseJsonOrNull(b.json) !== null;
      console.log("input_json_delta     #" + e.index + " buffer " + JSON.stringify(b.json) + (ok ? " parses" : " -> not valid JSON yet"));
    }
  }
  if (e.type === "content_block_stop") {
    const b = blocks[e.index];
    if (b.type === "tool_use") b.input = JSON.parse(b.json);
    console.log("content_block_stop   #" + e.index + (b.type === "tool_use" ? " tool input = " + JSON.stringify(b.input) : ""));
  }
  if (e.type === "message_delta") {
    stopReason = e.delta.stop_reason;
    usage.output_tokens = e.usage.output_tokens; // cumulative count, so replace, do not add
    console.log("message_delta        stop_reason=" + stopReason);
  }
  if (e.type === "message_stop") console.log("message_stop");
}

console.log("");
console.log("final text:   " + blocks[0].text);
console.log("tool call:    " + blocks[1].name + " " + JSON.stringify(blocks[1].input));
console.log("stop_reason:  " + stopReason + "  -> run the tool, then send its result back");
console.log("usage:        input " + usage.input_tokens + ", output " + usage.output_tokens);