Chunking in RAG: how do you pick size, overlap, and strategy?

// Sweep chunk size x overlap on one document with 8 labelled questions, then pick a config.
// Tokens are words here; a real pipeline counts with the embedding model's tokenizer.
const FACTS = [
  ["The backup job for the billing database runs every night at 02:00 UTC.", "When does the billing database backup run?"],
  ["Customer passwords are hashed with argon2id using a per user salt.", "How are customer passwords hashed?"],
  ["Invoices older than seven years are archived to cold storage and then deleted.", "What happens to invoices older than seven years?"],
  ["The public API rejects payloads larger than 2 megabytes with status 413.", "What is the maximum API payload size?"],
  ["On call engineers must acknowledge a paging alert within fifteen minutes.", "How fast must on call engineers acknowledge a page?"],
  ["Feature flags are evaluated at the edge and cached for thirty seconds.", "How long are feature flags cached?"],
  ["Refund requests above 500 euros need approval from a finance manager.", "Who approves large refund requests?"],
  ["Staging data is refreshed from an anonymised production snapshot every Monday.", "How is staging data refreshed?"],
];
const FILLER = [
  "This section was reviewed by the platform group last quarter.",
  "Teams should read the related runbook before making changes.",
  "Questions about this topic go to the internal help channel.",
  "Older versions of this page are kept in the wiki history.",
];
// Build the document: 2 filler sentences, then a fact, repeated.
const tokens = [], spans = [];
FACTS.forEach(([fact], i) => {
  for (const f of [FILLER[i % 4], FILLER[(i + 1) % 4]]) tokens.push(...f.split(" "));
  spans.push([tokens.length, tokens.length + fact.split(" ").length]);
  tokens.push(...fact.split(" "));
});
const N = tokens.length;
const norm = (w) => w.toLowerCase().replace(/[^a-z0-9]/g, "");
const STOP = new Set("the a an is are and for of to at with by from in on be this how what when who must".split(" "));
const terms = (ws) => ws.map(norm).filter((w) => w && !STOP.has(w));

function chunk(size, overlap) {
  const out = [], stride = size - overlap;
  for (let s = 0; ; s += stride) { out.push([s, Math.min(s + size, N)]); if (s + size >= N) break; }
  return out;
}
function topK(chunks, query, k) {
  const docs = chunks.map(([a, b]) => terms(tokens.slice(a, b)));
  const df = new Map();
  for (const d of docs) for (const w of new Set(d)) df.set(w, (df.get(w) || 0) + 1);
  const vec = (t) => { const v = new Map(); for (const w of t) v.set(w, (v.get(w) || 0) + Math.log(1 + docs.length / (df.get(w) || docs.length))); return v; };
  const len = (v) => Math.sqrt([...v.values()].reduce((a, x) => a + x * x, 0)) || 1;
  const q = vec(terms(query.split(" ")));
  return docs.map((d, i) => {
    const v = vec(d); let dot = 0;
    for (const [w, x] of q) dot += x * (v.get(w) || 0);
    return { i, s: Math.round((dot / (len(q) * len(v))) * 1e9) };
  }).sort((a, b) => b.s - a.s || a.i - b.i).slice(0, k).map((h) => chunks[h.i]);
}

const K = 2, rows = [];
console.log(`document: ${N} tokens, ${FACTS.length} facts, longest fact ${Math.max(...spans.map(([a, b]) => b - a))} tokens, top-k = ${K}`);
console.log("size   overlap  chunks  stored  dup   facts whole in a chunk  recall@2  ctx tok/query");
for (const size of [12, 24, 48, 96]) {
  for (const pct of [0, 25, 50]) {
    const overlap = Math.floor((size * pct) / 100);
    const chunks = chunk(size, overlap);
    const stored = chunks.reduce((a, [x, y]) => a + y - x, 0);
    const whole = spans.filter(([a, b]) => chunks.some(([x, y]) => x <= a && b <= y)).length;
    let hit = 0, ctx = 0;
    FACTS.forEach(([, q], i) => {
      const got = topK(chunks, q, K);
      ctx += got.reduce((a, [x, y]) => a + y - x, 0);
      if (got.some(([x, y]) => x <= spans[i][0] && spans[i][1] <= y)) hit++;
    });
    rows.push({ size, overlap, pct, recall: hit / FACTS.length, ctx: ctx / FACTS.length, stored });
    console.log(`${String(size).padStart(4)}  ${(overlap + " (" + pct + "%)").padStart(8)}  ${String(chunks.length).padStart(6)}  ${String(stored).padStart(6)}  ${(stored / N).toFixed(2)}  ${(whole + "/" + FACTS.length).padStart(22)}  ${(hit + "/" + FACTS.length).padStart(8)}  ${(ctx / FACTS.length).toFixed(1).padStart(13)}`);
  }
}
const ok = rows.filter((r) => r.recall === 1).sort((a, b) => a.ctx - b.ctx || a.stored - b.stored);
const p = ok[0];
console.log(`\npick: the cheapest config with recall@2 = 100% is size ${p.size}, overlap ${p.overlap} (${p.pct}%): ${p.ctx.toFixed(1)} context tokens per query`);
const big = rows.find((r) => r.size === 96 && r.pct === 0);
console.log(`size 96 also reaches ${Math.round(big.recall * 100)}% but sends ${big.ctx.toFixed(1)} tokens per query (${(big.ctx / p.ctx).toFixed(1)}x)`);
console.log("\nrule: a fact of L tokens always fits whole in some window when overlap >= L - 1");
for (const [size, overlap] of [[24, 6], [24, 12], [24, 13]]) {
  const chunks = chunk(size, overlap);
  let worst = 0; // longest span that is guaranteed to fit, found by brute force over all positions
  for (let L = 1; L <= size; L++) {
    let all = true;
    for (let a = 0; a + L <= N && all; a++) all = chunks.some(([x, y]) => x <= a && a + L <= y);
    if (all) worst = L; else break;
  }
  console.log(`  size ${size}, overlap ${String(overlap).padStart(2)}: every span up to ${worst} tokens fits whole (overlap + 1 = ${overlap + 1})`);
}