How would you design a Web Crawler?

// Web crawler core: URL frontier with per-host politeness, robots.txt, URL and content dedup, trap limits.
const crypto = require("crypto");

console.log("== capacity estimate ==");
const pagesPerMonth = 1000000000, kbPerPage = 100, delaySec = 1;
const pagesPerSec = Math.ceil(pagesPerMonth / (30 * 86400));
console.log("pages/s: " + pagesPerSec + ", download: " + (pagesPerSec * kbPerPage) / 1000 + " MB/s, raw storage/month: " + (pagesPerMonth * kbPerPage) / 1000000000 + " TB");
console.log("at 1 request per " + delaySec + " s per host, hosts that must be in flight at once: >= " + pagesPerSec * delaySec);

// The simulated web: url -> { links, text }. c.com/calendar links to the next day forever (a crawler trap).
const web = {
  "http://a.com/": { links: ["http://A.com/news#top", "http://a.com/private/admin", "http://b.com/", "http://a.com/about/"], text: "a home" },
  "http://a.com/news": { links: ["http://a.com/news/1", "http://a.com/news/2", "http://c.com/copy"], text: "a news" },
  "http://a.com/news/1": { links: ["http://a.com/"], text: "story one" },
  "http://a.com/news/2": { links: ["http://a.com/news"], text: "story two" },
  "http://a.com/about": { links: [], text: "about a" },
  "http://b.com/": { links: ["http://b.com/shop", "http://c.com/calendar?day=1"], text: "b home" },
  "http://b.com/shop": { links: ["http://b.com/"], text: "b shop" },
  "http://c.com/copy": { links: [], text: "  Story ONE " },
};
for (let d = 1; d <= 50; d++) web["http://c.com/calendar?day=" + d] = { links: ["http://c.com/calendar?day=" + (d + 1)], text: "day " + d };
const robots = { "a.com": { disallow: ["/private"], delayMs: 1000 }, "b.com": { disallow: [], delayMs: 2000 }, "c.com": { disallow: [], delayMs: 1000 } };
const MAX_PER_HOST = 5, FETCH_MS = 100;

function normalize(url) {
  let u = url.split("#")[0];
  const m = /^(\w+):\/\/([^/]+)(.*)$/.exec(u);
  let path = m[3] || "/";
  if (path.length > 1 && path.endsWith("/")) path = path.slice(0, -1);
  return m[1].toLowerCase() + "://" + m[2].toLowerCase() + path;
}
const hostOf = (url) => /^\w+:\/\/([^/]+)/.exec(url)[1];
const pathOf = (url) => url.slice(url.indexOf("/", url.indexOf("//") + 2)) || "/";
const fingerprint = (text) => crypto.createHash("sha256").update(text.trim().toLowerCase()).digest("hex").slice(0, 8);

function crawl(polite) {
  const queues = new Map(), nextAllowed = new Map(), seenUrls = new Set(), seenContent = new Map(), perHost = new Map();
  const stats = { fetched: 0, robots: 0, dupUrl: 0, dupContent: 0, budget: 0 };
  const lines = [], times = [];
  let clock = 0;
  function add(raw) {
    const url = normalize(raw), host = hostOf(url);
    if (seenUrls.has(url)) { stats.dupUrl++; return; }
    seenUrls.add(url);
    if (robots[host].disallow.some((p) => pathOf(url).startsWith(p))) { stats.robots++; return; }
    if (!queues.has(host)) { queues.set(host, []); nextAllowed.set(host, 0); }
    queues.get(host).push(url);
  }
  add("http://a.com/");
  for (;;) {
    let best = null;
    for (const [host, q] of queues) if (q.length && (best === null || nextAllowed.get(host) < nextAllowed.get(best))) best = host;
    if (best === null) break;
    const url = queues.get(best).shift();
    if ((perHost.get(best) || 0) >= MAX_PER_HOST) { stats.budget++; continue; }
    if (polite) clock = Math.max(clock, nextAllowed.get(best));
    const page = web[url];
    times.push([best, clock]);
    perHost.set(best, (perHost.get(best) || 0) + 1);
    stats.fetched++;
    const fp = fingerprint(page.text);
    if (seenContent.has(fp)) { stats.dupContent++; lines.push("t=" + clock + "ms " + url + " duplicate of " + seenContent.get(fp)); }
    else { seenContent.set(fp, url); lines.push("t=" + clock + "ms " + url + " stored, links found: " + page.links.length); for (const l of page.links) add(l); }
    clock += FETCH_MS;
    nextAllowed.set(best, clock + (polite ? robots[best].delayMs : 0));
  }
  let worst = 0;
  for (const [h, t] of times) worst = Math.max(worst, times.filter(([h2, t2]) => h2 === h && t2 >= t && t2 < t + 1000).length);
  return { lines, stats, worst, clock };
}

const run = crawl(true);
console.log("== polite crawl ==");
for (const l of run.lines) console.log(l);
const s = run.stats;
console.log("fetched " + s.fetched + ", blocked by robots.txt " + s.robots + ", duplicate URLs skipped " + s.dupUrl + ", duplicate content " + s.dupContent + ", cut by per-host budget " + s.budget);
console.log("finished at t=" + run.clock + "ms, max fetches to one host in any 1 s window: " + run.worst);
const naive = crawl(false);
console.log("== same crawl with no politeness delay ==");
console.log("finished at t=" + naive.clock + "ms, max fetches to one host in any 1 s window: " + naive.worst);