How do you scope retrieval with metadata filtering and multi-tenancy?

// Scoped retrieval: post-filter vs pre-filter vs iterative scan, and where the filter comes from.
let seed = 7;
function rand() { seed = (seed * 48271) % 2147483647; return seed / 2147483647; }
const DIM = 16;
const vec = () => Array.from({ length: DIM }, () => rand() * 2 - 1);
const dot = (a, b) => a.reduce((s, x, i) => s + x * b[i], 0);

// 2,000 chunks shared by four tenants of very different size, each with an ACL.
const CHUNKS = [];
for (let id = 0; id < 2000; id++) {
  const r = rand();
  const tenant = r < 0.7 ? "acme" : r < 0.95 ? "globex" : r < 0.995 ? "initech" : "tiny";
  const g = rand();
  const groups = g < 0.1 ? null : g < 0.75 ? ["all"] : g < 0.9 ? ["hr"] : ["finance", "hr"];
  CHUNKS.push({ id, tenant, groups, v: vec() });
}
const query = vec();
// Stand-in for an ANN index: candidates come out in global similarity order, in pages.
const ORDER = CHUNKS.map((c) => ({ c, s: dot(query, c.v) })).sort((a, b) => b.s - a.s || a.c.id - b.c.id).map((x) => x.c);
const K = 5, EF = 40, MAX_SCAN = 400;

function postFilter(ok) { const page = ORDER.slice(0, EF); return { hits: page.filter(ok).slice(0, K), scanned: EF }; }
function preFilter(ok) { const subset = CHUNKS.filter(ok); const hits = subset.map((c) => ({ c, s: dot(query, c.v) })).sort((a, b) => b.s - a.s || a.c.id - b.c.id).slice(0, K).map((x) => x.c); return { hits, scanned: subset.length }; }
function iterative(ok) {
  const hits = []; let scanned = 0;
  while (hits.length < K && scanned < MAX_SCAN && scanned < ORDER.length) {
    for (const c of ORDER.slice(scanned, scanned + EF)) if (ok(c) && hits.length < K) hits.push(c);
    scanned += EF;
  }
  return { hits, scanned };
}

console.log(`1) top ${K} for each tenant (index page ef = ${EF}, iterative cap = ${MAX_SCAN})`);
console.log("   tenant    chunks  share   post-filter  pre-filter (scanned)  iterative (scanned)");
for (const t of ["acme", "globex", "initech", "tiny"]) {
  const ok = (c) => c.tenant === t;
  const n = CHUNKS.filter(ok).length;
  const a = postFilter(ok), b = preFilter(ok), c = iterative(ok);
  const same = b.hits.map((x) => x.id).join() === c.hits.map((x) => x.id).join();
  console.log(`   ${t.padEnd(8)}  ${String(n).padStart(6)}  ${(n / 20).toFixed(1).padStart(4)}%  ${String(a.hits.length).padStart(5)} found  ${String(b.hits.length).padStart(4)} found (${String(b.scanned).padStart(4)})  ${String(c.hits.length).padStart(4)} found (${String(c.scanned).padStart(4)})${same ? "  same as exact" : ""}`);
}

console.log("");
console.log("2) where the filter comes from (user u1 belongs to acme, groups: all)");
const SESSIONS = { tok_u1: { user: "u1", tenant: "acme", groups: ["all"] } };
const request = { token: "tok_u1", question: "salary bands", filter: { tenant: "globex" } };
const fromBody = (req) => (c) => c.tenant === req.filter.tenant;
function fromSession(req) {
  const s = SESSIONS[req.token];
  if (!s) throw new Error("no session");
  return (c) => c.tenant === s.tenant && c.groups !== null && c.groups.some((g) => s.groups.includes(g));
}
const unsafe = iterative(fromBody(request)).hits;
const safe = iterative(fromSession(request)).hits;
console.log(`   filter from request body: ids ${unsafe.map((c) => c.id).join(", ")}  tenants ${[...new Set(unsafe.map((c) => c.tenant))].join(", ")}  <- leak`);
console.log(`   filter from session:      ids ${safe.map((c) => c.id).join(", ")}  tenants ${[...new Set(safe.map((c) => c.tenant))].join(", ")}`);
const acme = CHUNKS.filter((c) => c.tenant === "acme");
const untagged = acme.filter((c) => c.groups === null).length;
const restricted = acme.filter((c) => c.groups !== null && !c.groups.includes("all")).length;
const visible = acme.filter(fromSession(request)).length;
console.log(`   acme chunks: ${acme.length}  visible to u1: ${visible}  hidden by ACL: ${restricted}  no ACL, denied by default: ${untagged}`);
const allowIfMissing = acme.filter((c) => c.groups === null || c.groups.includes("all")).length;
console.log(`   if a missing ACL meant allow, u1 could retrieve ${allowIfMissing} chunks (${allowIfMissing - visible} never cleared for u1)`);

console.log("");
console.log("3) routing rule: exact scan when the filtered set is small, index otherwise");
for (const t of ["acme", "tiny"]) {
  const ok = (c) => c.tenant === t;
  const n = CHUNKS.filter(ok).length;
  const r = n < 1000 ? preFilter(ok) : iterative(ok);
  console.log(`   ${t.padEnd(5)} ${String(n).padStart(4)} chunks -> ${n < 1000 ? "exact scan" : "iterative index scan"}: ${r.hits.length} found, ${r.scanned} scanned, ids ${r.hits.map((c) => c.id).join(", ")}`);
}