nx_web_crawl_step.nx source

1// nx_web_crawl_step.nx -- CRAWL AT SCALE: the frontier-PERSISTED successor to nx_web_ingest's one-shot BFS. 2// The frontier lives IN the web shard itself (seg_store-native, no side files): key fr:<cid(url)> whose 3// value is "P<url>" (pending) or "D<url>" (done) -- append-only, ss_hget last-wins flips state. Each run: 4// 1. enumerate pending frontier rows (keys-index walk, dedup, latest state wins) 5// 2. fetch up to <max_pages> of them (sovereign TLS; seq628 pool: WC_POOL_N forked workers on 6// DISTINCT hosts, parent-enforced WC_FETCH_BUDGET_MS wall clock, dead-host deferral) 7// 3. ingest kept pages (doc:<cid> + url:<cid>, the nx_web_ingest mechanics) 8// 4. bank every NEWLY-DISCOVERED content link as a pending fr: row (filtered: no queries/fragments/ 9// binaries/wiki-meta) and mark fetched urls done (even failures -- a dead url never wedges the loop) 10// So the crawl RESUMES where it stopped: run it any number of times and the index only grows. A seed url 11// is only needed to BOOTSTRAP an empty frontier (or inject a new start). 12// usage: nx_web_crawl_step <max_pages> [seed-url] 13// license_tier: ORIGINAL 14import "nx_heavyio_lib.nx" // the estate-wide heavy-I/O concurrency bound (2026-09-03): a crawl step fans out fetch children and seg-store writes, so it asks BEFORE the certdata load 15import "nx_corpus_ingest.nx" // ci_hash / ci_mkurlkey / dss_prefix / dss_mkkey / seg_store / nx_html_to_text 16import "nx_x509_trust_store.nx" 17import "nx_trust_store_load_from_certdata.nx" 18import "nx_https_fetch_follow.nx" 19import "nx_simhash.nx" 20import "nx_itoa_lib.nx" // nxi_out -- THE canonical integer emitter (runtime/ layer, importable from here) 21import "nx_crawl_pace.nx" 22import "nx_robots.nx" // R11: RFC 9309 allow/disallow + Crawl-delay (was imported by ONE organ) // adaptive per-host pacing (replaces the fixed 350ms; 429 backoff at scale) 23import "nx_feed_extract.nx" // nx_feed_discover + nx_feed_item_at -- seed the frontier with clean article URLs 24import "nx_js_eval.nx" // R-JS-HEADLESS: js_render_page + js_has_inline_script (hydrate JS pages for indexing) 25import "nx_outlink_harvest.nx" 26import "nx_store_seed_lib.nx" // sts_load -> the urlpolicy- plane (R10: the denylist is DATA, not code) // olh_urlok / olh_outkey / olh_resolve_root / olh_scan -- CANONICAL url filter + 27import "nx_block_density.nx" // R6 (2026-08-24, /compare/webscraping bd_fit_text): the index stores the ARTICLE, not the chrome around it 28import "nx_crawl_sufficiency.nx" // R5 (cs_confidence): query-scoped crawl stops when coverage+consistency+saturation clear the bar 29import "nx_antibot.nx" // R4 (abt_classify): bot walls classified from marker ROWS and named by vendor, never solved 30const WC_MAGIC_1024: i64 = 1024 // NX-UNNAMED: mechanical, collision-free placeholder -- this name RESTATES the value and so carries no meaning. Replace it with a domain term that says WHY, or move the number to a config row. 31const WC_MAGIC_1089: i64 = 1089 32const WC_MAGIC_4000: i64 = 4000 33// seq1002: JS hydration is an ENHANCEMENT gated behind an UNBUDGETED interpreter. 0 = off (safe default, 34// crawler always terminates); 1 = on (only once nx_js_eval enforces an execution budget). Named const so 35// re-enabling is a deliberate one-line decision with this comment attached, not a silent edit. 36const WC_JS_HYDRATE: i64 = 0 37const WC_MAGIC_2000: i64 = 2000 38const WC_MAGIC_2048: i64 = 2048 39const WC_MAGIC_8192: i64 = 8192 40const WC_MAGIC_4194304: i64 = 4194304 41const WC_MAGIC_4096: i64 = 4096 42const WC_MAGIC_8388608: i64 = 8388608 43const WC_MAGIC_1048576: i64 = 1048576 44const WC_MAGIC_5381: i64 = 5381 45 // link-graph edge semantics, SHARED with nx_cc_ingest (one node-identity policy) 46 47const WC_MAXPEND: i64 = 2048 // pending urls pulled per run. MEASURED 2026-08-04: at 256 the pull 48 // window sat 93% full of skipped-dead-host rows (~240 recur every run 49 // in stable walk order, deliberately left pending) -> fetched=13-48 of a 50 // 400-page budget, ~300-600 pages/day vs the ~7-8K the beat intends. 2048 51 // leaves ~1800 live-host candidates after the rot blockade; all buffers 52 // (urls mmap, seenk, seed shift) scale off this const. The rot itself 53 // still needs a persistent host-health retire policy (debt filed). 54 // STALE AS OF 2026-08-25 -- READ WC_HD_RETIRE IMMEDIATELY BELOW BEFORE 55 // ACTING ON THE LINE ABOVE. That persistent host-health retire policy WAS 56 // IMPLEMENTED: wc_hd_streak gates it at :1268 and nx_web_crawl_step_gate 57 // T8 proves the P->D retire actually fires. The sentence is kept, not 58 // deleted, because it records WHY 2048 was chosen -- but a growth-analysis 59 // lane read it as a LIVE gap and ranked "implement dead-host retire" as the 60 // number-one blocker, for a fix that already ships. A HEADER IS NOT A 61 // MEASUREMENT, and a stale one costs the next reader the whole investigation. 62 // The measured blocker is WC_MAXDISC below, pinned at its 300 ceiling in 63 // 58 of 60 recent runs while the frontier sits flat at ~1900 -- discovery 64 // exactly offsets consumption, so the frontier cannot build a backlog. 65const WC_HD_RETIRE: i64 = 3 // R3 host-health: retire (P->D) a skipped row once its host's PERSISTED 66 // dead-run streak reaches this (streak increments once per dead run, 67 // resets on any completed response; retire fires on run streak+1). 68const WC_MAXSEEDS: i64 = 256 // seed-file urls held for the retire EXEMPTION + the R4 canonical lane + 69 // the R5 re-queue. Was 96 while the seed file sat at 95 rows -- ONE row 70 // from a SILENT cliff: the 97th line would have been dropped from the 71 // retire exemption with no log line, and a retired seed is PERMANENT 72 // host loss (wc_seed_add refuses ANY existing fr: row). Raised 73 // 2026-08-05 (debt 1785936869); the drop is now LOUD (seed parse below). 74const WC_SEED_REFRESH: i64 = 8 // R5: already-done seed urls re-marked pending per run. The persisted 75 // sr:cursor rotates the whole seed list, so every hub is re-fetched 76 // roughly every nseeds/8 runs and re-harvests its outlinks -- the 77 // frontier REPLENISHES instead of burning down. 78const WC_MAXDISC: i64 = 300 // FLOOR for the per-run discovery budget -- see wc_disc_budget below. 79// R17 SPLIT (2026-08-25). WC_MAXDISC was serving TWO UNRELATED PURPOSES: the per-run POLICY budget, and 80// the DIMENSION of the runseen dedup buffer (sys_mmap(32 * (WC_MAXDISC + 8)), and nrs[0] < WC_MAXDISC+7). 81// ONE CONSTANT SERVING TWO PURPOSES CAN NEVER BE TUNED FOR EITHER: raising the budget silently overruns 82// the buffer, so the discovery policy was frozen in place by a memory bound nobody was thinking about 83// when they chose 300. Splitting them is what makes the policy adjustable at all. 84// WC_DISC_CEIL is now the buffer dimension AND the hard ceiling. Derived from WC_MAXPEND because banking 85// more rows than one pull window can hold buys nothing -- they cannot be pulled next run either way. 86// No new guessed number: the ceiling IS the window. 87const WC_DISC_CEIL: i64 = WC_MAXPEND 88const WC_CANON_RESERVE: i64 = 384 // R4 CANONICAL LANE (2026-08-05): head slots of each run's work list 89 // reserved for canonical-seed hosts. MEASURED that day: an 89,000-doc 90 // corpus held 64 en.wikipedia.org / 0 who.int / 1 python.org / 2 irs.gov 91 // docs -- every one of them seeded and answering HTTP 200, none blocked 92 // and none retired. They were STARVED, not refused. 93const WC_MAXCANH: i64 = 96 // distinct canonical HOSTS parsed from the seed file (mirrors WC_MAXSEEDS) 94const WC_MAXURL: i64 = 1500 // urls longer than this are skipped (junk/tracking monsters) 95const WC_MININDEX: i64 = 500 // min extracted chars to INDEX a page (not to harvest its links). Measured 96 // bimodal gap 2026-07-05: junk<=56 chars, real content>=1251, nothing between 97 // -> 500 filters thin junk (person/tag stubs) with huge margin. (>CI_MINDOC=20.) 98const WC_HOSTCAP_CANON: i64 = 40 // R4b DEPTH CAP FOR CURATED HOSTS (2026-08-05). WC_HOSTCAP below exists to 99 // stop ONE junk host exploding into N near-duplicate variants -- that is a 100 // defence against UNCURATED breadth, and it is exactly backwards for a 101 // canonical seed host, where DEPTH is the whole objective. MEASURED: at 6 102 // pages/host/run en.wikipedia.org gains at most 432 docs/day, so climbing 103 // out of 64 docs would take about three weeks; at 40 it is a few days. 104 // Still bounded, and still spread across ~60 canonical hosts. 105const WC_HOSTCAP: i64 = 6 // max pages INDEXED per host per crawl step (diversity). Measured 2026-07-06: 106 // one CC0 deed page exploded into 16 language variants (deed.fr/de/fi/...) that 107 // simhash misses (translated = different fingerprint). A per-host cap defends 108 // against that + faceted-nav/calendar/pagination traps. Harvest is NOT capped. 109 110// ---- seq628 PARALLEL FETCH POOL (2026-07-29) ---- 111// The crawl was SEQUENTIAL: one unbounded fetch at a time, so a frontier of mostly-dead hosts burned 112// minutes per url on connect timeouts (measured 2026-07-23: a 30-page crawl stalled 10+ min). Fix shape: 113// fetch in BATCHES of WC_POOL_N forked children (network wait overlaps), each child hard-bounded by a 114// PARENT-enforced wall clock (stronger than a connect timeout: it bounds connect+TLS+read+redirects), 115// one DISTINCT host per in-flight slot (politeness: never two concurrent fetches to one host), and a 116// host that times out or hard-fails is DEFERRED for the rest of the run (its urls stay PENDING, not 117// done -- recency/priority, retried next run at one-probe cost). Ingest/harvest stay single-process in 118// the parent: the seg-store writer keeps ONE writer by construction. 119const WC_POOL_N: i64 = 8 // in-flight fetch children per batch. Bounded by spool RAM (8 x 8MiB 120 // fetch caps = 64MiB peak) and by typical distinct-host diversity in 121 // a 256-url frontier pull; speedup ~ min(N, distinct hosts). 122const WC_FETCH_BUDGET_MS: i64 = 20000 // per-fetch wall clock. Healthy fetches measure <1s recv with 123 // worst-case ~5s TLS handshake flakes; 20s = ~4x the worst GOOD 124 // fetch observed, where a dead host used to cost minutes. 125const WC_POLL_MS: i64 = 100 // parent reap-poll quantum while children are in flight 126const WC_US_PER_MS: i64 = 1000 127const WC_WNOHANG: i64 = 1 128const WC_SIGKILL: i64 = 9 129// spool header: [i64 http-status][i64 body-bytes][i64 redirect-hops] 130// WIDENED 2026-08-16 to carry REDIRECT DEPTH across the fork. The fetch happens in a CHILD, so 131// nx_https_last_hops() read in the parent would report a static the child never touched -- a 132// constant-zero field with an authoritative name. The count has to ride the spool or it does not 133// exist downstream at all. 134// SAFE BY CONSTRUCTION, not by convention: the child is a fork() of this same binary, so producer and 135// consumer are the same struct in the same image and cannot disagree about layout. (The usual 136// wire-format hazard -- a reader that misparses a widened record -- needs two SEPARATELY deployed 137// programs; it does not apply to a process and its own fork.) 138// WHY IT MATTERS: a doorway that redirects onward and never serves the artifact is the most 139// goal-defeating result a search engine can return -- it looks like an answer and hands over nothing. 140// Captured at INGEST, so detecting it later never costs a re-fetch. 141const WC_SPOOL_HDRB: i64 = 24 142// fp: row scratch. Lazy-allocated ONCE per process, never per ingest -- an mmap inside the ingest 143// loop would burn a page for every page indexed (the estate has paid that bill before: 262KB x 2400 144// files = 640MB for one census). 145static wc_fpk_g: *u8 146static wc_fpv_g: *u8 147static wc_fek_g: *u8 // S12: the fe:<cid> key buffer (one, reused per page like wc_fpk_g) 148static wc_fev_g: *u8 // S12: the fetch epoch's decimal bytes 149// NEAR-DUP COMPARISON WINDOW. The dedup loop compared against at most this many of the run's kept 150// fingerprints and said NOTHING when it truncated -- past the window, duplicates pass unnoticed and 151// the run reports a clean ingest it never actually checked. A cap reached in silence becomes a 152// measurement nobody knows is partial. Named here and ANNOUNCED at the call site. 153const WC_DEDUP_WINDOW: i64 = 128 154// simhash Hamming distance at or below which two documents are the same page. 64-bit fingerprint, so 155// this is ~6% of the bit width -- the standard near-dup band, not a tuned knob. DECLARED IMPRECISION: 156// nx_simhash misses TRANSLATED duplicates (different tokens -> different fingerprint), which is why 157// the per-host cap exists as a second defence; see the note at WC_HOSTCAP. 158const WC_SIMHASH_HAM: i64 = 4 159static wc_dwarn_g: i64 // 1 once the window-truncation notice fired (announce ONCE per run, not per page) 160const WC_SPOOLPATH_B: i64 = 128 161const WC_MODE_RW: i64 = 420 // 0644 spool file mode 162const WC_DEADHOSTS: i64 = 128 // dead-host slots per run (frontier pull is 256 urls; hosts <= urls) 163const WC_HASH_MASK: i64 = 0x7fffffffffffffff 164 165// ==== R7 FRONTIER PRIORITY + AGING (2026-08-24, /compare/webscraping contract wc_priority) ================== 166// MEASURED 2026-08-04 (the WC_MAXPEND note above): the pull window sat 93% full of the SAME skipped rows in 167// stable walk order, so rows beyond the window never got a turn. Two mechanisms, both pure and gate-tested: 168// ROTATION: a persisted cursor (pc:cur row, the sr:cursor encoding) -- wc_pending skips the first `cursor` 169// pending rows (phase 0) and, if the window is not full, wraps to the head (phase 1); the cursor advances by 170// the rows taken and wraps at the pending population, so every pending row is offered in turn. 171// RANKING: the general window is ordered by (query-term hits in the url DESC, path depth ASC), stable. 172// Shallow-first is Cho, Garcia-Molina and Page 1998 (breadth-first order finds high-PageRank pages early); 173// the query bonus is the crawl4ai KeywordRelevanceScorer shape. No weight constants: one query hit outranks 174// any depth because a url is at most WC_MAXURL bytes and therefore at most WC_MAXURL levels deep. 175// The canonical lane (R4) is untouched: its own wc_pending pass runs with rotation off (skip=stop=0). 176static wc_qh_g: *i64 177static wc_qn_g: i64 178static wc_qtok_g: *i64 179static wc_pc_skip_g: i64 180static wc_pc_stop_g: i64 181static wc_pc_seen_g: i64 182static wc_pcc_key_g: *u8 183static wc_pcc_pp_g: *i64 184static wc_pcc_pl_g: *i64 185static wc_pcp_key_g: *u8 186static wc_pcp_val_g: *u8 187static wc_cs_g: *i64 188static wc_cs_stop_g: i64 189static wc_fit_pages_g: i64 190static wc_fit_saved_g: i64 191static wc_fit_fallback_g: i64 192const WC_FIT_TEXT: i64 = 1 // R6: index the density-fit text (nx_block_density); 0 = the pre-R6 full render 193func wc_query_set(q: *u8) -> i64 { 194 if (wc_qh_g as i64) == 0 { wc_qh_g = sys_mmap(CS_QMAX * 8) as *i64 } 195 if (wc_qtok_g as i64) == 0 { wc_qtok_g = sys_mmap(WC_MAXURL * 8) as *i64 } 196 var n: i64 = cs_tokens(q, wc_len(q), wc_qh_g, CS_QMAX) 197 if n > CS_QMAX { n = CS_QMAX } 198 wc_qn_g = n 199 return n 200} 201// path depth: levels below the host. https://a.org/ -> 0, /x -> 1, /x/y/z -> 3; a doubled or trailing slash opens no level 202func wc_url_depth(u: *u8, ul: i64) -> i64 { 203 var i: i64 = 0 204 var s: i64 = 0 - 1 205 var f: i64 = 0 206 while f == 0 { if i + 2 >= ul { f = 1 } else { if u[i] == (58 as u8) { if u[i + 1] == (47 as u8) { if u[i + 2] == (47 as u8) { s = i + 3; f = 1 } } } if f == 0 { i = i + 1 } } } 207 if s < 0 { s = 0 } 208 var g: i64 = 0 209 while g == 0 { if s >= ul { g = 1 } else { if u[s] == (47 as u8) { g = 1 } else { s = s + 1 } } } 210 var depth: i64 = 0 211 var p: i64 = s 212 while p < ul { 213 if u[p] == (47 as u8) { if p + 1 < ul { if u[p + 1] != (47 as u8) { depth = depth + 1 } } } 214 p = p + 1 215 } 216 return depth 217} 218// how many DISTINCT query terms the url names (0 without a query) 219func wc_url_qhits(u: *u8, ul: i64) -> i64 { 220 if wc_qn_g <= 0 { return 0 } 221 var n: i64 = cs_tokens(u, ul, wc_qtok_g, WC_MAXURL) 222 if n > WC_MAXURL { n = WC_MAXURL } 223 var hits: i64 = 0 224 var q: i64 = 0 225 while q < wc_qn_g { 226 var i: i64 = 0 227 var f: i64 = 0 228 while i < n { if wc_qtok_g[i] == wc_qh_g[q] { f = 1; i = n } else { i = i + 1 } } 229 hits = hits + f 230 q = q + 1 231 } 232 return hits 233} 234func wc_priority(u: *u8, ul: i64) -> i64 { return wc_url_qhits(u, ul) * WC_MAXURL - wc_url_depth(u, ul) } 235// order the window's 2KB slots by priority DESC, stable: rank an index array, then permute ONCE through a copy 236func wc_rank_window(urls: *u8, n: i64) -> i64 { 237 if n <= 1 { return 0 } 238 let pr: *i64 = sys_mmap(n * 8) as *i64 239 let ix: *i64 = sys_mmap(n * 8) as *i64 240 var i: i64 = 0 241 while i < n { let sp: *u8 = (urls as i64 + i * WC_MAGIC_2048) as *u8; pr[i] = wc_priority(sp, wc_len(sp)); ix[i] = i; i = i + 1 } 242 var a: i64 = 1 243 while a < n { 244 var j: i64 = a 245 while j > 0 { 246 if pr[ix[j - 1]] < pr[ix[j]] { let t: i64 = ix[j - 1]; ix[j - 1] = ix[j]; ix[j] = t; j = j - 1 } else { j = 0 } 247 } 248 a = a + 1 249 } 250 let cp: *u8 = sys_mmap(n * WC_MAGIC_2048) 251 var k: i64 = 0 252 while k < n { 253 let sp2: *u8 = (urls as i64 + ix[k] * WC_MAGIC_2048) as *u8 254 let dp2: *u8 = (cp as i64 + k * WC_MAGIC_2048) as *u8 255 var c: i64 = 0 256 var go: i64 = 1 257 while go == 1 { dp2[c] = sp2[c]; if sp2[c] == (0 as u8) { go = 0 } else { c = c + 1 } } 258 k = k + 1 259 } 260 k = 0 261 while k < n { 262 let sp3: *u8 = (cp as i64 + k * WC_MAGIC_2048) as *u8 263 let dp3: *u8 = (urls as i64 + k * WC_MAGIC_2048) as *u8 264 var c3: i64 = 0 265 var go3: i64 = 1 266 while go3 == 1 { dp3[c3] = sp3[c3]; if sp3[c3] == (0 as u8) { go3 = 0 } else { c3 = c3 + 1 } } 267 k = k + 1 268 } 269 sys_munmap(cp, n * WC_MAGIC_2048) 270 sys_munmap(pr as *u8, n * 8) 271 sys_munmap(ix as *u8, n * 8) 272 return n 273} 274// the persisted rotation cursor: pc:cur row, ASCII digits, last-wins (the sr:cursor pattern) 275func wc_pckey(out: *u8) -> i64 { 276 out[0] = 112 as u8; out[1] = 99 as u8; out[2] = 58 as u8 277 out[3] = 99 as u8; out[4] = 117 as u8; out[5] = 114 as u8 278 out[6] = 0 as u8 279 return 6 280} 281func wc_pc_cursor(h: *i64) -> i64 { 282 if (h as i64) == 0 { return 0 } 283 if (wc_pcc_key_g as i64) == 0 { wc_pcc_key_g = sys_mmap(64) } 284 if (wc_pcc_pp_g as i64) == 0 { wc_pcc_pp_g = sys_mmap(16) as *i64 } 285 if (wc_pcc_pl_g as i64) == 0 { wc_pcc_pl_g = sys_mmap(16) as *i64 } 286 wc_pckey(wc_pcc_key_g) 287 if ss_hget(h, wc_pcc_key_g, wc_pcc_pp_g, wc_pcc_pl_g) != 1 { return 0 } 288 let vp: *u8 = wc_pcc_pp_g[0] as *u8 289 var v: i64 = 0 290 var i: i64 = 0 291 while i < wc_pcc_pl_g[0] { let c: i64 = vp[i] as i64; if c >= 48 { if c <= 57 { v = v * 10 + (c - 48) } } i = i + 1 } 292 return v 293} 294func wc_pc_put(w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, v: i64) -> i64 { 295 if (wc_pcp_key_g as i64) == 0 { wc_pcp_key_g = sys_mmap(64) } 296 if (wc_pcp_val_g as i64) == 0 { wc_pcp_val_g = sys_mmap(28) } 297 wc_pckey(wc_pcp_key_g) 298 let vl: i64 = ccz_cat_num(wc_pcp_val_g, 0, v) 299 if ss_add(w, 1, wc_pcp_key_g, wc_pcp_val_g, vl) < 0 { 300 if ss_commit(prefix, w, segidbox[0]) == 0 { segsbox[0] = segsbox[0] + 1 } 301 segidbox[0] = segidbox[0] + 1 302 w[1] = 0 303 ss_add(w, 1, wc_pcp_key_g, wc_pcp_val_g, vl) 304 } 305 return 0 306} 307// next cursor: advance by the rows taken; wrap to 0 at the pending population (or when the cursor is already past it) 308func wc_pc_next(cur: i64, took: i64, seen: i64) -> i64 { 309 if seen <= 0 { return 0 } 310 if cur >= seen { return 0 } 311 let nx: i64 = cur + took 312 if nx >= seen { return 0 } 313 return nx 314} 315// wc_pending's admission under rotation: counts every pending row it meets (the population, an upper bound -- 316// a key living in two segments is met twice), takes a row only when the window has room and the row is inside 317// R13 FRONTIER GAUGE, EXTRACTED 2026-08-25 so it can be gated. It was inline inside the done-line's 318// wc_num() call, which meant the one number every capacity decision is read off had no tooth at all. 319// A row pulled into this run's window leaves the frontier by exactly FOUR routes, and each one marks it 320// D: it was fetched, it was retired as dead, robots refused it, or the selection-time host cap dropped it. 321// Everything else pulled stays PENDING and must NOT be subtracted -- deadhost SKIPS (that did not reach 322// the retire bar) and R13 pace DEFERRALS both stay P on purpose, and rows the batch loop never reached 323// because the page budget ran out are still P as well. 324// A PARTITION IS A CLAIM: THESE PARTS SUM, and wc_frontier_left below is the other half of the claim. 325func wc_frontier_remaining(npend: i64, fetched: i64, retired: i64, rbskip: i64, hostcapped: i64, disc: i64) -> i64 { 326 return npend - wc_frontier_left(fetched, retired, rbskip, hostcapped) + disc 327} 328// The outflow alone: the rows that were pulled AND left the pending state this run. Named separately so a 329// tooth can assert the partition rather than re-deriving the same expression it is supposed to be checking 330// (a check that restates its subject proves only that addition is commutative). 331func wc_frontier_left(fetched: i64, retired: i64, rbskip: i64, hostcapped: i64) -> i64 { 332 return fetched + retired + rbskip + hostcapped 333} 334// the phase's band. With skip=stop=0 this is exactly the old `npend < maxout`. 335func wc_pc_take(npend: i64, maxout: i64) -> i64 { 336 wc_pc_seen_g = wc_pc_seen_g + 1 337 if npend >= maxout { return 0 } 338 if wc_pc_skip_g > 0 { if wc_pc_seen_g <= wc_pc_skip_g { return 0 } } 339 if wc_pc_stop_g > 0 { if wc_pc_seen_g > wc_pc_stop_g { return 0 } } 340 return 1 341} 342 343func wc_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 } 344// HOISTED 2026-08-08: wc_num is the NUMBER PRINTER and it ran TWO sys_mmap(28) per call. sys_mmap is 345// page-granular, so EVERY NUMBER THIS ORGAN HAS EVER LOGGED COST 8 KB. Not per href, so it survived 346// three passes that were all looking at the crawl path -- a leak inside the INSTRUMENTATION is invisible 347// to anyone reading the instrumentation. Both buffers are fully consumed by sys_write before return. 348// DELEGATED 2026-08-08 -- SUPERSEDES MY OWN HOIST FROM THIS MORNING, and the reason is worth keeping. 349// wc_num was a hand-rolled number printer: two sys_mmap(28) per call, page-granular, so every number 350// this organ ever logged cost 8 KB. My first fix hoisted both buffers to lazy statics. That WORKED but 351// was still the wrong shape -- it left a SECOND implementation of a primitive the estate already owns. 352// nx_itoa_lib.nx (runtime/ layer) has been the canonical emitter since 2026-07-31 and names itself 353// 'the drop-in for every hand-rolled putn/gn/wn/pn clone'; this organ is simply one of the ~1792 clones. 354// Also drops 2 of the nx_mmapbal findings the hoist introduced (row 3795): a delegation has no 355// unmatched sys_mmap at all, so it stops triggering the scanner rather than arguing with it. 356func wc_num(v: i64) -> i64 { nxi_out(v); return 0 } 357func wc_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n } 358// resolve a feed href against the page url: absolute (http...) kept; "/path" -> scheme://host/path. 359func wc_resolve(base: *u8, href: *u8, out: *u8) -> i64 { 360 if href[0]==(104 as u8) { if href[1]==(116 as u8) { if href[2]==(116 as u8) { if href[3]==(112 as u8) { 361 var a: i64=0; while href[a]!=(0 as u8) { out[a]=href[a]; a=a+1 } out[a]=0 as u8; return 0 362 } } } } 363 var sp: i64 = 0 - 1; var i: i64 = 0; var df: i64 = 0 364 while df == 0 { if base[i]==(0 as u8) { df=1 } else { if base[i]==(58 as u8) { if base[i+1]==(47 as u8) { if base[i+2]==(47 as u8) { sp=i; df=1 } } } if df==0 { i=i+1 } } } 365 if sp < 0 { var k: i64=0; while href[k]!=(0 as u8) { out[k]=href[k]; k=k+1 } out[k]=0 as u8; return 0 } 366 var he: i64 = sp + 3; var dn: i64 = 0 367 while dn == 0 { if base[he]==(0 as u8) { dn=1 } else { if base[he]==(47 as u8) { dn=1 } else { he=he+1 } } } 368 var o: i64 = 0; while o < he { out[o]=base[o]; o=o+1 } 369 if href[0]==(47 as u8) { var j: i64=0; while href[j]!=(0 as u8) { out[o]=href[j]; o=o+1; j=j+1 } } 370 else { out[o]=47 as u8; o=o+1; var j: i64=0; while href[j]!=(0 as u8) { out[o]=href[j]; o=o+1; j=j+1 } } 371 out[o]=0 as u8; return 0 372} 373// host (between "://" and next '/' or ':') -> out; flag-based (never index past the string). returns len. 374// HOST SCRATCH BOUND (2026-08-15). Every wc_host call site allocates sys_mmap(64); the scan below 375// had NO cap on `o` and stopped only at NUL, '/' or ':'. MEASURED LIVE, twice, on consecutive runs: 376// ARENA-OVERRUN prev_alloc_size=64 own: www.asianude4u.net.k.du.-q4fz.googlevideo.com.2F 377// A url that percent-encodes its slashes (%2F) presents NO literal '/', so the host scan walks the 378// whole PATH into a 64-byte buffer. olh_urlok_q rejects '?' and '#' but not %2F, so such urls are 379// admitted -- i.e. this is reachable from crawled third-party page content, not just our own seeds. 380// A "host" longer than the buffer is never a long host; it is a parse that ran off the end of one. 381// REFUSE (return 0 = no host), never truncate: callers all guard `if hl > 0`, so a refusal skips the 382// row, which is the wrong-in-the-direction-of-doing-nothing failure a guard is required to have. 383const WC_HOSTBUF: i64 = 64 // == the sys_mmap(64) every caller passes 384const WC_HOSTMAX: i64 = WC_HOSTBUF - 1 // DERIVED, authored once: longest host + its NUL 385func wc_host(url: *u8, out: *u8) -> i64 { 386 var sp: i64 = 0 - 1; var i: i64 = 0; var d: i64 = 0 387 while d == 0 { if url[i]==(0 as u8) { d=1 } else { if url[i]==(58 as u8) { if url[i+1]==(47 as u8) { if url[i+2]==(47 as u8) { sp=i; d=1 } } } if d==0 { i=i+1 } } } 388 var s: i64 = 0; if sp >= 0 { s = sp + 3 } 389 var o: i64 = 0; var d2: i64 = 0 390 while d2 == 0 { let c: i64 = url[s] as i64; if c==0 { d2=1 } else { if c==47 { d2=1 } else { if c==58 { d2=1 } else { if o >= WC_HOSTMAX { out[0] = 0 as u8; return 0 } out[o]=url[s]; o=o+1; s=s+1 } } } } 391 out[o] = 0 as u8; return o 392} 393func wc_atoi(s: *u8) -> i64 { var v: i64 = 0; var i: i64 = 0; while s[i] != (0 as u8) { let c: i64 = s[i] as i64; if c >= 48 { if c <= 57 { v = v * 10 + (c - 48) } } i = i + 1 } return v } 394func wc_hassub(u: *u8, ul: i64, lit: *u8) -> i64 { 395 let ll: i64 = wc_len(lit) 396 if ll == 0 { return 0 } 397 var i: i64 = 0 398 while i + ll <= ul { 399 var m: i64 = 1 400 var k: i64 = 0 401 while k < ll { if u[i + k] != lit[k] { m = 0; k = ll } else { k = k + 1 } } 402 if m == 1 { return 1 } 403 i = i + 1 404 } 405 return 0 406} 407func wc_endswith(u: *u8, ul: i64, lit: *u8) -> i64 { 408 let ll: i64 = wc_len(lit) 409 if ul < ll { return 0 } 410 var k: i64 = 0 411 while k < ll { if u[ul - ll + k] != lit[k] { return 0 } k = k + 1 } 412 return 1 413} 414// CONTENT-URL FILTER: 1 = worth crawling. Delegates to the CANONICAL olh_urlok (verbatim extraction of the 415// rules that lived here; shared with nx_cc_ingest) so frontier urls and link-graph nodes follow ONE policy 416// (OLH_MAXURL == the measured 1500 junk threshold that was WC_MAXURL). 417// wc_urlok MOVED below the R4 canon helpers (it now consults them) -- see wc_urlok after wc_host_is_canon. 418// frontier key for a URL: "fr:<decimal cid of the url bytes>" 419static wc_frkey_t_g: *u8 420func wc_frkey(u: *u8, ul: i64, out: *u8) -> i64 { 421 let cid: i64 = ci_hash(u, ul) 422 out[0] = 102 as u8 // f 423 out[1] = 114 as u8 // r 424 out[2] = 58 as u8 // : 425 var m: i64 = cid 426 // HOISTED 2026-08-06 (see wc_frstate above): 1 of the 5 pages leaked per candidate href. Reached 427 // per-href via wc_frstate and the harvest new-url path. `t` holds decimal digits that are copied 428 // into `out` before return -- nothing retains it, so a single reused scratch is safe. 429 if (wc_frkey_t_g as i64) == 0 { wc_frkey_t_g = sys_mmap(28) } 430 let t: *u8 = wc_frkey_t_g 431 var k: i64 = 0 432 if m == 0 { t[0] = 48 as u8; k = 1 } 433 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 } 434 var i: i64 = 0 435 while i < k { out[3 + i] = t[k - 1 - i]; i = i + 1 } 436 out[3 + k] = 0 as u8 437 return 3 + k 438} 439// out:<cid(page_url)> = the LINK GRAPH edge list for PageRank (P1): value = packed i64 target cid(url)s. 440// Node identity = cid(url) (SAME hash as the frontier's fr:<cid(url)>), so an outlink resolves to a graph 441// node with zero extra index; serve-time authority = content_cid -> url:<cid> -> ci_hash(url) -> pr:<cid>. 442// Key shape lives in the SHARED lib (one producer policy with nx_cc_ingest). 443func wc_outkey(cid: i64, out: *u8) -> i64 { return olh_outkey(cid, out) } 444// frontier state for a URL: 0 = unknown (no row), 1 = pending, 2 = done 445// ---- HOISTED 2026-08-06: THIS FUNCTION WAS 3 OF THE 5 PAGES LEAKED PER CANDIDATE HREF ---- 446// MEASURED: ./nx_web_crawl_step.elf 400 reached 2662 MB RSS in ~11 min, ~6.6 MB retained PER PAGE, and 447// drove the box to 828 permille swap and I/O-bound (procs_blocked 6 > running 5). sys_mmap is PAGE 448// GRANULAR here -- this file says so itself at lines 283-284, "a per-row sys_mmap would burn a page per 449// frontier row" -- so these three 16-64 byte requests cost 3 x 4096 B, per href, forever. 450// WHY IT WENT UNSEEN: the crawl loop body (1440+) really is clean; its "no mmap in the crawl loop" 451// comment is TRUE. Every leaked byte entered through a CALLEE. A loop is not the extent of a loop. 452// WHY THE GUARD DID NOT SAVE US: the WC_MAXDISC test at the harvest site gates on discbox[0], which only 453// increments on a NEW bank. Once discovered_new decays to 0 (this file's own R5 note: 121 -> 35 -> 0) the 454// guard NEVER trips, so every page allocates at full rate. It bounds the DISCOVERY budget, not the 455// ALLOCATION -- the allocation happens before the discovery test. The leak is WORST in the steady cron regime. 456// WHY HOIST AND NOT munmap: there are four post-allocation return paths below; a munmap pairing needs all 457// of them and one miss is a silent partial leak. Hoisting is one edit with no exit-path discipline. 458// CORRECTNESS, checked not assumed: `key` is consumed by ss_hget IN-CALL; pp/pl are out-boxes whose 459// pp[0] points into the MAPPED SHARD, not into pp. Nothing retains a pointer past the call. The proof is 460// in this organ's own hot path -- main already passes a SINGLE reused `key` buffer to ss_hget/ss_add for 461// every page, so if those retained rather than copied, the live crawler would already corrupt every row. 462// Lazy init: statics default 0, so gate fixtures and unit harnesses need no ordering rule. 463static wc_frst_key_g: *u8 464static wc_frst_pp_g: *i64 465static wc_frst_pl_g: *i64 466func wc_frstate(h: *i64, u: *u8, ul: i64) -> i64 { 467 if (h as i64) == 0 { return 0 } 468 if (wc_frst_key_g as i64) == 0 { wc_frst_key_g = sys_mmap(64) } 469 if (wc_frst_pp_g as i64) == 0 { wc_frst_pp_g = sys_mmap(16) as *i64 } 470 if (wc_frst_pl_g as i64) == 0 { wc_frst_pl_g = sys_mmap(16) as *i64 } 471 let key: *u8 = wc_frst_key_g 472 wc_frkey(u, ul, key) 473 let pp: *i64 = wc_frst_pp_g 474 let pl: *i64 = wc_frst_pl_g 475 if ss_hget(h, key, pp, pl) != 1 { return 0 } 476 if pl[0] < 1 { return 0 } 477 let vp: *u8 = pp[0] as *u8 478 if vp[0] == (80 as u8) { return 1 } // P 479 if vp[0] == (68 as u8) { return 2 } // D 480 return 0 481} 482// bank a frontier state row: value = <P|D><url> (append-only; ss_hget last-wins = the state flip) 483// ---- HOISTED 2026-08-08: THE REMAINING 2 OF THE 5 PAGES LEAKED PER CANDIDATE HREF ---- 484// The 2026-08-06 pass above hoisted wc_frstate (3 pages) and wc_frkey (1). wc_frmark's OWN two 485// allocations were left, so the per-href cost fell 5 pages -> 2, not to 0. A partial fix on a 486// per-item path is still an unbounded leak; it only changes the slope. 487// MEASURED 2026-08-08: nx_web_crawl_step pid 19016 was the estate's #1 grower at 22-40 MB/s, grew_in 488// 5/5 in 2 of 3 nx_memvel samples and 5-6x the next process, while the box sat at 853 permil swap. 489// CORRECTNESS, checked not assumed: `key` is consumed by ss_add IN-CALL, exactly as wc_frstate's key is 490// consumed by ss_hget in-call, and main already passes a SINGLE reused key buffer to ss_add for every 491// page -- a retaining ss_add would already corrupt every row today. wc_frkey writes at most 3+20+1 492// bytes, so 64 is bounded by construction. 493// `val` is variable-length, so the hoist is GUARDED rather than assumed: WC_MAXURL (1500) bounds every 494// url that reaches the frontier, so the 2048 static covers the whole real population -- but the 495// oversize branch is KEPT and falls back to a per-call mmap instead of truncating. Behaviour is 496// therefore byte-identical for every input and the hoist can only remove an allocation, never change a 497// row. Fail-safe in the direction of doing nothing. 498static wc_frm_key_g: *u8 499static wc_frm_val_g: *u8 500// HOISTED 2026-08-08 (harvest + walk paths). wc_ubuf_g IS THE 5TH OF THE 5 PAGES leaked per candidate 501// href: the 2026-08-06 pass hoisted wc_frstate (3) and wc_frkey (1) and stopped one allocation short. 502// It is paid BEFORE wc_urlok judges the url, so every REJECTED href was charged for it too. 503// wc_kbuf_g is worse per run: one page per fr: row per frontier walk, and this shard has held >1040 504// distinct fr: keys, so a single walk burned megabytes before reading a byte of it. 505// CORRECTNESS: every one of these is consumed IN-CALL and nothing retains a pointer past the call -- 506// nx_url_canon WRITES into ubuf, wc_urlok/wc_frstate only READ it, wc_runseen COPIES frk9 into its own 507// 32B slots, ss_hget reads kbuf, and ss_add2 BYTE-COPIES both key and val into the writer buffer 508// (nx_seg_store.nx:193, read not assumed). So a single reused buffer cannot alias a live row. 509static wc_ubuf_g: *u8 510static wc_frk9_g: *u8 511static wc_kbuf_g: *u8 512static wc_okey_g: *u8 513// HOISTED 2026-08-08 (host-health + seed-cursor paths). Same shape, lower rate than the per-href set 514// above: these run per HOST and per SEED rather than per href, which is why they survived two prior 515// passes -- but a per-host allocation is still unbounded in a crawl that keeps discovering hosts, and 516// leaving them is what lets this whole class regrow quietly. 13 pages across 5 functions. 517// CORRECTNESS is the SAME argument already proven for the set above, not a new one: key is consumed 518// in-call by ss_hget/ss_add (ss_add2 byte-copies), pp/pl are out-boxes whose pp[0] points INTO the 519// mapped shard and never into pp itself, val is fixed-width, and each `t` is a decimal-digit scratch 520// fully copied into `out`/`val` before return. Nothing retains a pointer past the call. 521// The three digit-scratch statics that used to sit here (wc_hdk_t_g / wc_hdp_t_g / wc_srp_t_g) are GONE: 522// their functions now call ccz_cat_num, which writes into the caller's buffer and allocates nothing, so 523// there is no scratch to hoist. What remains below is genuinely per-call state that must persist across 524// the call (keys, out-boxes, value buffers) and is correctly a lazy static. 525static wc_hds_key_g: *u8 526static wc_hds_pp_g: *i64 527static wc_hds_pl_g: *i64 528static wc_hdp_key_g: *u8 529static wc_hdp_val_g: *u8 530static wc_src_key_g: *u8 531static wc_src_pp_g: *i64 532static wc_src_pl_g: *i64 533static wc_srp_key_g: *u8 534static wc_srp_val_g: *u8 535// per-PAGE (not per-href): olh_scan WRITES the edge list into it and ss_add2 byte-copies it straight 536// out into the writer, so it is consumed in-call like every buffer above. OLH_MAXEDGE*8 every harvest. 537static wc_edgebuf_g: *i64 538func wc_frmark(w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, u: *u8, ul: i64, state: i64) -> i64 { 539 if (wc_frm_key_g as i64) == 0 { wc_frm_key_g = sys_mmap(64) } 540 let key: *u8 = wc_frm_key_g 541 wc_frkey(u, ul, key) 542 var val: *u8 = 0 as *u8 543 if ul + 2 > WC_MAGIC_2048 { val = sys_mmap(ul + 2) } else { 544 if (wc_frm_val_g as i64) == 0 { wc_frm_val_g = sys_mmap(WC_MAGIC_2048) } 545 val = wc_frm_val_g 546 } 547 if state == 1 { val[0] = 80 as u8 } else { val[0] = 68 as u8 } 548 var i: i64 = 0 549 while i < ul { val[1 + i] = u[i]; i = i + 1 } 550 if ss_add(w, 1, key, val, ul + 1) < 0 { 551 // writer full: commit, then REUSE the buffer (w[0]=buf w[1]=len w[2]=cap; reset len = fresh writer) 552 if ss_commit(prefix, w, segidbox[0]) == 0 { segsbox[0] = segsbox[0] + 1 } 553 segidbox[0] = segidbox[0] + 1 554 w[1] = 0 555 ss_add(w, 1, key, val, ul + 1) 556 } 557 return 0 558} 559// R8 (2026-08-05): the canonical host table, module-level so the shared URL policy can consult it 560// WITHOUT threading it through wc_harvest's already-wide arity. Filled ONCE in main after the seed 561// file parses; zero until then, so every gate fixture and any run without a seed file sees exactly 562// the pre-R8 policy (statics default 0 -> allowq=0 -> byte-identical). 563static wc_canh_g: *u8 564static wc_ncan_g: i64 565static wc_qscratch_g: *u8 566 567// ---- R4 CANONICAL LANE (2026-08-05) ----------------------------------------------------------- 568// MEASURED that day: the corpus held ~89,000 docs but only 64 from en.wikipedia.org, 0 from who.int, 569// 1 from python.org and 2 from irs.gov -- while ALL of those hosts sat in canonical_seeds.txt and 570// answered HTTP 200. They were never blocked, never walled and never retired. They were STARVED. 571// wc_pending walks segments in stable append order and stops at WC_MAXPEND, so once the frontier held 572// more than 2048 P rows the canonical rows (banked LATER, so living in NEWER segments) never entered 573// the window at all, and the shared WC_MAXDISC budget was spent by whichever junk host happened to be 574// processed first. Ranking work could not have fixed this: the documents did not exist to rank. 575// ★A FLAT WALK OVER A FRONTIER THAT GREW JUNK-FIRST IS A PERMANENT BLOCKADE OF ITS OWN HEAD. 576// The lane is ADDITIVE (rule 25 -- build intelligence, never strip): nothing is filtered out and no 577// row is dropped. Canonical rows are pulled in a SECOND host-filtered pass and placed at the FRONT of 578// the work list, so they are fetched first and get first claim on the discovery budget; every other 579// row keeps its original order behind the reserve. 580 581// byte-equal NUL-terminated strings 582func wc_streq(a: *u8, b: *u8) -> i64 { 583 var i: i64 = 0 584 while a[i] == b[i] { if a[i] == (0 as u8) { return 1 } i = i + 1 } 585 return 0 586} 587// the seed file's DISTINCT hosts with a leading "www." stripped, so a seed of https://www.who.int/ 588// covers who.int, www.who.int AND data.who.int -- the same dot-suffix contract the search engine's 589// site: filter already uses. 64-byte slots. Returns the host count. 590func wc_canon_hosts(seedstore: *u8, nseeds: i64, out: *u8) -> i64 { 591 var n: i64 = 0 592 var i: i64 = 0 593 let hb: *u8 = sys_mmap(64) 594 while i < nseeds { 595 let sp: *u8 = (seedstore as i64 + i * WC_MAGIC_2048) as *u8 596 let hl: i64 = wc_host(sp, hb) 597 if hl > 0 { if hl < 60 { 598 hb[hl] = 0 as u8 599 var st: i64 = 0 600 if hl > 4 { if hb[0] == (119 as u8) { if hb[1] == (119 as u8) { if hb[2] == (119 as u8) { if hb[3] == (46 as u8) { st = 4 } } } } } 601 let hp: *u8 = (hb as i64 + st) as *u8 602 var dup: i64 = 0 603 var d: i64 = 0 604 while d < n { 605 if wc_streq((out as i64 + d * 64) as *u8, hp) == 1 { dup = 1; d = n } else { d = d + 1 } 606 } 607 if dup == 0 { if n < WC_MAXCANH { 608 let dst: *u8 = (out as i64 + n * 64) as *u8 609 var c: i64 = 0 610 while hp[c] != (0 as u8) { dst[c] = hp[c]; c = c + 1 } 611 dst[c] = 0 as u8 612 n = n + 1 613 } } 614 } } 615 i = i + 1 616 } 617 return n 618} 619// 1 iff u's host IS a canonical host or a dot-suffixed subdomain of one ("en.wikipedia.org" matches a 620// canonical "wikipedia.org"; "notwikipedia.org" does NOT, because the boundary dot is required). 621// hb = caller-owned 64B scratch, allocated ONCE per walk -- a per-row sys_mmap would burn a page per 622// frontier row. 623// HOST-STRING form of the canonical test. The fetch loop already holds the page's host (phost/phl), 624// so re-parsing a url there would be waste -- and duplicating the match would be two copies of one 625// rule (rule 15). This is the ONE implementation; wc_host_is_canon is the url-taking wrapper. 626func wc_host_str_is_canon(canh: *u8, ncan: i64, hb: *u8, hl: i64) -> i64 { 627 if ncan <= 0 { return 0 } 628 if hl <= 0 { return 0 } 629 var i: i64 = 0 630 while i < ncan { 631 let cp: *u8 = (canh as i64 + i * 64) as *u8 632 var cl: i64 = 0 633 while cp[cl] != (0 as u8) { cl = cl + 1 } 634 if cl > 0 { if cl <= hl { 635 var eq: i64 = 1 636 var x: i64 = 0 637 while x < cl { if hb[hl - cl + x] != cp[x] { eq = 0; x = cl } else { x = x + 1 } } 638 if eq == 1 { 639 if hl == cl { return 1 } 640 if hb[hl - cl - 1] == (46 as u8) { return 1 } 641 } 642 } } 643 i = i + 1 644 } 645 return 0 646} 647func wc_host_is_canon(canh: *u8, ncan: i64, u: *u8, hb: *u8) -> i64 { 648 if ncan <= 0 { return 0 } 649 let hl: i64 = wc_host(u, hb) 650 if hl <= 0 { return 0 } 651 if hl >= 60 { return 0 } 652 hb[hl] = 0 as u8 653 return wc_host_str_is_canon(canh, ncan, hb, hl) 654} 655// R8: the ONE content-URL policy, now trust-aware. A curated seed host may carry its CONTENT on 656// query strings (measured: graphis.ne.jp galleries are model.php?ID=... -- the blanket '?' reject 657// made that entire vertical structurally unreachable, so seeding the hub could never yield a page). 658// Everything else keeps the blanket reject, which is what stops tracking junk and pagination traps 659// on uncurated hosts. Same shape as the R4b depth cap: the LIMIT is right, its SUBJECT's trust is 660// what changed. wc_ncan_g==0 (gates, seedless runs) -> allowq=0 -> the exact pre-R8 contract. 661// ---- R10 URL POLICY: JUDGE THE SHAPE, NOT THE HOST ------------------------------------------------ 662// R8 admitted query strings ONLY on curated seed hosts. That is an ALLOWLIST, and an allowlist is the 663// wrong shape for a search engine: most of the web's content rides query strings, so "uncurated => 664// unreachable" meant this crawler could only ever see hosts a human had already typed in. That scales 665// to a DIRECTORY, never to an INDEX -- and it is why a real gravure result like 666// www.v2ph.com/actor/JULIA?page=3&hl=en was refused before a single byte was fetched. 667// Operator 2026-08-06: "that's a dumb way to have a search engine setup." Correct. 668// R10 judges the QUERY ITSELF, host-independently: 669// - a TRACKING param (utm_*, fbclid, session ids) means this URL is a DUPLICATE of the clean one 670// -> reject, because indexing it twice is worse than not indexing it 671// - too many params, or an overlong query, is a facet/calendar TRAP -> reject 672// - anything else is CONTENT -> admit, curated or not 673// Curation survives as a BOOST (a curated host gets a bigger param budget), never as the gate. 674// The lists are DATA in knowledge/store/urlpolicy-, loaded ONCE per run: adding a tracking param must 675// never require a recompile. ★AN ALLOWLIST ANSWERS "WHO DO WE TRUST"; AN INDEX MUST ANSWER "IS THIS A PAGE". 676const WC_URLPOL_PLANE: *u8 = "knowledge/store/urlpolicy-" 677const WC_QSLOT: i64 = 32 678const WC_QMAXDENY: i64 = 192 679const WC_QPLANECAP: i64 = 65536 680const WC_QDEF_MAXP: i64 = 4 // default param budget for an UNCURATED host 681const WC_QDEF_MAXL: i64 = 120 // default max query length in bytes 682const WC_QCAN_BOOST: i64 = 4 // curated hosts get this many EXTRA params (trust boost, not a gate) 683static wc_qdeny_g: *u8 684static wc_nqdeny_g: i64 685static wc_qmaxp_g: i64 686static wc_qmaxlen_g: i64 687func wc_qeq(a: *u8, b: *u8, bl: i64) -> i64 { 688 var k: i64 = 0 689 while k < bl { if a[k] != b[k] { return 0 } k = k + 1 } 690 if a[bl] != (0 as u8) { return 0 } 691 return 1 692} 693func wc_qnum(b: *u8, s: i64, e: i64) -> i64 { 694 var v: i64 = 0 695 var any: i64 = 0 696 var i: i64 = s 697 while i < e { let c: i64 = b[i] as i64; if c >= 48 { if c <= 57 { v = v * 10 + (c - 48); any = 1 } } i = i + 1 } 698 if any == 0 { return 0 - 1 } 699 return v 700} 701// Load the policy plane. ABSENT plane = the built-in defaults, NEVER a closed gate: a missing data file 702// must not silently return the crawler to allowlist behaviour, which is the failure nobody would notice. 703func wc_load_urlpolicy() -> i64 { 704 wc_qdeny_g = sys_mmap(WC_QSLOT * WC_QMAXDENY) 705 wc_nqdeny_g = 0 706 wc_qmaxp_g = WC_QDEF_MAXP 707 wc_qmaxlen_g = WC_QDEF_MAXL 708 let b: *u8 = sys_mmap(WC_QPLANECAP) 709 let n: i64 = sts_load(WC_URLPOL_PLANE, b, WC_QPLANECAP - 16) 710 if n <= 0 { return 0 } 711 var i: i64 = 0 712 while i < n { 713 let ls: i64 = i 714 var le: i64 = ls 715 var s: i64 = 1 716 while s == 1 { if le >= n { s = 0 } else { if b[le] == (10 as u8) { s = 0 } else { le = le + 1 } } } 717 i = le + 1 718 if le - ls > 2 { if b[ls] != (35 as u8) { 719 var t: i64 = 0 - 1 720 var j: i64 = ls 721 while j < le { if b[j] == (9 as u8) { t = j; j = le } else { j = j + 1 } } 722 if t > ls { 723 let nl: i64 = t - ls 724 // config knobs are named __maxparams / __maxqlen so they cannot collide with a param name 725 if wc_qeq("__maxparams\x00" as *u8, (b as i64 + ls) as *u8, nl) == 1 { 726 let v: i64 = wc_qnum(b, t + 1, le); if v > 0 { wc_qmaxp_g = v } 727 } 728 else { if wc_qeq("__maxqlen\x00" as *u8, (b as i64 + ls) as *u8, nl) == 1 { 729 let v2: i64 = wc_qnum(b, t + 1, le); if v2 > 0 { wc_qmaxlen_g = v2 } 730 } 731 else { 732 if nl < WC_QSLOT - 1 { if wc_nqdeny_g < WC_QMAXDENY { 733 let dst: *u8 = (wc_qdeny_g as i64 + wc_nqdeny_g * WC_QSLOT) as *u8 734 var c: i64 = 0 735 while c < nl { dst[c] = b[ls + c]; c = c + 1 } 736 dst[nl] = 0 as u8 737 wc_nqdeny_g = wc_nqdeny_g + 1 738 } } 739 } } 740 } 741 } } 742 } 743 return wc_nqdeny_g 744} 745// is this param name on the deny list? (exact match, so "sid" never kills "sid_of_article") 746func wc_qparam_denied(u: *u8, s: i64, e: i64) -> i64 { 747 if wc_nqdeny_g <= 0 { return 0 } 748 if (wc_qdeny_g as i64) == 0 { return 0 } 749 let nl: i64 = e - s 750 if nl <= 0 { return 0 } 751 var i: i64 = 0 752 while i < wc_nqdeny_g { 753 let cp: *u8 = (wc_qdeny_g as i64 + i * WC_QSLOT) as *u8 754 var cl: i64 = 0 755 while cp[cl] != (0 as u8) { cl = cl + 1 } 756 if cl == nl { 757 var eq: i64 = 1 758 var k: i64 = 0 759 while k < nl { if u[s + k] != cp[k] { eq = 0; k = nl } else { k = k + 1 } } 760 if eq == 1 { return 1 } 761 } 762 i = i + 1 763 } 764 return 0 765} 766// THE QUERY JUDGE. 1 = a content URL worth fetching. Host-independent by design. 767func wc_query_ok(u: *u8, ul: i64, canon: i64) -> i64 { 768 var q: i64 = 0 - 1 769 var i: i64 = 0 770 while i < ul { if u[i] == (63 as u8) { q = i; i = ul } else { i = i + 1 } } 771 if q < 0 { return 1 } // no query at all -- nothing to judge 772 let qlen: i64 = ul - q - 1 773 if qlen <= 0 { return 0 } // a bare trailing '?' is the clean URL wearing a costume 774 // SELF-DEFAULTING, NOT LOADER-DEPENDENT. These budgets live in statics that only wc_load_urlpolicy 775 // sets, so ANY entry path that skips the loader (the gate, a future caller, a unit harness) would 776 // otherwise read 0 and reject EVERY query url -- a silent regression to something STRICTER than the 777 // allowlist this replaced. The gate caught exactly that. Defaults belong where the value is READ. 778 // ★A LIMIT THAT READS 0 WHEN UNINITIALISED IS A CLOSED GATE WEARING AN OPEN GATE'S NAME. 779 var maxlen: i64 = wc_qmaxlen_g 780 if maxlen <= 0 { maxlen = WC_QDEF_MAXL } 781 if qlen > maxlen { return 0 } 782 var budget: i64 = wc_qmaxp_g 783 if budget <= 0 { budget = WC_QDEF_MAXP } 784 if canon == 1 { budget = budget + WC_QCAN_BOOST } 785 var nparam: i64 = 0 786 var ps: i64 = q + 1 787 while ps < ul { 788 // end of this param = the next '&', or the end of the url 789 var end: i64 = ps 790 var f: i64 = 0 791 while f == 0 { if end >= ul { f = 1 } else { if u[end] == (38 as u8) { f = 1 } else { end = end + 1 } } } 792 // the NAME runs to the first '=' inside this param (a valueless param is all name) 793 var nameend: i64 = ps 794 var g: i64 = 0 795 while g == 0 { if nameend >= end { g = 1 } else { if u[nameend] == (61 as u8) { g = 1 } else { nameend = nameend + 1 } } } 796 if nameend > ps { if wc_qparam_denied(u, ps, nameend) == 1 { return 0 } } 797 nparam = nparam + 1 798 if nparam > budget { return 0 } 799 ps = end + 1 800 } 801 return 1 802} 803func wc_urlok(u: *u8, ul: i64) -> i64 { 804 var canon: i64 = 0 805 if wc_ncan_g > 0 { if (wc_canh_g as i64) != 0 { if (wc_qscratch_g as i64) != 0 { 806 if wc_host_is_canon(wc_canh_g, wc_ncan_g, u, wc_qscratch_g) == 1 { canon = 1 } 807 } } } 808 // allowq=1 ALWAYS: the shared filter's blanket '?' kill is retired here in favour of the shape judge 809 // below. Everything else it enforces (scheme, fragments, wiki namespaces, binary extensions) still runs. 810 if olh_urlok_q(u, ul, 1) == 0 { return 0 } 811 return wc_query_ok(u, ul, canon) 812} 813// enumerate PENDING frontier urls: walk every segment's key index for "fr:"-prefixed keys (dedup by key), 814// ss_hget each key's LATEST value, keep the P ones. urlsout = flat 2KB slots. Returns count. 815// R4: canh/ncan/canonly extend the walk WITHOUT duplicating it (rule 15 -- one walk, one contract). 816// canonly=0 ignores canh/ncan entirely and is byte-identical to the pre-R4 behaviour, which is what 817// the original full-frontier call site passes and what gate T2 keeps pinned. 818// ==== G0 HOST-BALANCED PULL WINDOW (2026-09-03, /compare/search rung G0, contract wc_frontier_fair) ==== 819// MEASURED, not designed. Two windows of knowledge/status/crawl_outcomes.log 41 minutes apart (epochs 820// 1788448047 and 1788450535) were BOTH 100 percent hostcap-defer and every url in both was 821// en.wikipedia.org; nx_crawlaudit independently re-fetched a newest-first sample and returned 6 of 8 822// disagreements reading "we DROPPED it, independent re-check finds REAL PROSE" -- Kierkegaard at 182147 823// chars, Ricoeur at 42727. The R7 rotation above fixes POSITIONAL starvation (rows beyond the window 824// never getting a turn) and it CANNOT fix HOST starvation, because a rotated slice of a 825// Wikipedia-dominated frontier is still Wikipedia. wc_hostcap_defers' own note names the regime we are 826// in: past ONE pull window a deferred row can be shadowed by rows ahead of it, and the pending 827// population is far past one window. 828// THE QUOTA IS DERIVED AND NO CAP IS LOOSENED. The fetch stage indexes at most WC_HOSTCAP_CANON pages 829// from one host in a step, so admitting more than that many rows for the SAME host is provably wasted: 830// the surplus can only become hostcap-defer while consuming a slot another host could have used. The 831// window quota is therefore the LARGEST per-host index cap in the system, applied one stage earlier -- 832// which means a row the fetch stage COULD have indexed is never excluded, so this can only remove rows 833// that were guaranteed to be deferred. The max (canonical) cap is used rather than the per-host cap on 834// purpose: the main-lane caller passes ncan=0, so canonicality is not knowable here, and guessing it in 835// the TIGHT direction would under-fill a canonical host the fetch stage could take 40 pages from. 836// WC_HOSTCAP and WC_HOSTCAP_CANON are UNTOUCHED -- a run that indexed more pages by raising them would 837// have failed rung G0, not passed it. At a full WC_MAXPEND window this guarantees at least 51 distinct 838// hosts are offered whenever that many are pending. 839const WC_WINQ: i64 = WC_HOSTCAP_CANON 840// Distinct hosts are LINEAR-SCANNED, not hashed, and that is deliberate twice over: wc_hosthash is 841// defined BELOW wc_pending so calling it here would be a forward reference, and writing a second host 842// hash beside it would be the duplicate-ruler defect. The scan is bounded by DISTINCT hosts, at most 843// maxout/WC_WINQ, so it is a smaller order than the dedup scan wc_pending already performs per row. 844// Table rows are WC_HOSTBUF bytes of NUL-terminated host, parallel to one i64 count. 845func wc_winq_admit(tblh: *u8, tblc: *i64, nh: *i64, slots: i64, hb: *u8, hl: i64, quota: i64) -> i64 { 846 var d: i64 = 0 847 while d < nh[0] { 848 let sp: *u8 = (tblh as i64 + d * WC_HOSTBUF) as *u8 849 var eq: i64 = 1 850 var x: i64 = 0 851 while x <= hl { if sp[x] != hb[x] { eq = 0; x = hl + 1 } else { x = x + 1 } } 852 if eq == 1 { 853 if tblc[d] >= quota { return 0 } 854 tblc[d] = tblc[d] + 1 855 return 1 856 } 857 d = d + 1 858 } 859 // TABLE FULL: ADMIT rather than refuse. A sizing error must be wrong in the direction of doing 860 // nothing -- refusing here would starve the crawl on a bookkeeping limit, which is a worse defect 861 // than the imbalance this function exists to correct. 862 if nh[0] >= slots { return 1 } 863 let dst: *u8 = (tblh as i64 + nh[0] * WC_HOSTBUF) as *u8 864 var c: i64 = 0 865 while c <= hl { dst[c] = hb[c]; c = c + 1 } 866 tblc[nh[0]] = 1 867 nh[0] = nh[0] + 1 868 return 1 869} 870// THE CONTRACT SYMBOL for /compare/search rung G0. A matrix watch row measures that this symbol EXISTS, 871// so it must BE the invariant rather than a decoration named after one: it answers "is this window 872// host-balanced?" for a (rows, distinct-hosts) pair, which is exactly the pair the G1 census reports. 873// A window is fair when no host could have exceeded its quota, i.e. rows <= hosts * WC_WINQ. Kept PURE 874// and separate from wc_winq_admit on purpose -- admit ENFORCES per row, this JUDGES a finished window, 875// and a census that re-derived the arithmetic itself would be a second ruler that can drift from the 876// enforcer. One definition, two readers. 877func wc_frontier_fair(rows: i64, hosts: i64) -> i64 { 878 if rows <= 0 { return 1 } // an empty window is vacuously balanced, and saying so beats dividing by zero 879 if hosts <= 0 { return 0 } // rows with NO host attributed is not a fair window, it is an unmeasured one 880 if rows <= hosts * WC_WINQ { return 1 } 881 return 0 882} 883func wc_pending(h: *i64, prefix: *u8, urlsout: *u8, maxout: i64, canh: *u8, ncan: i64, canonly: i64) -> i64 { 884 if (h as i64) == 0 { return 0 } 885 let ns: i64 = h[0] 886 wc_pc_seen_g = 0 // R7: this walk's pending population, counted by wc_pc_take 887 let hbuf4: *u8 = sys_mmap(64) // R4 host scratch, allocated ONCE per walk 888 let seenk: *u8 = sys_mmap(64 * (WC_MAXPEND * 4 + 16)) 889 // G0: the host-balance table for THIS window. At most `maxout` DISTINCT hosts can occupy a 890 // maxout-row window, so that is the exact slot count -- nothing to guess and no way to overflow. 891 let wqh: *u8 = sys_mmap(WC_HOSTBUF * (maxout + 1)) 892 let wqc: *i64 = sys_mmap(8 * (maxout + 1)) as *i64 893 let wqn: *i64 = sys_mmap(16) as *i64 894 wqn[0] = 0 895 var nseen: i64 = 0 896 var npend: i64 = 0 897 let pp: *i64 = sys_mmap(16) as *i64 898 let pl: *i64 = sys_mmap(16) as *i64 899 var s: i64 = 0 900 while s < ns { 901 let kb: *u8 = h[1 + 8 * s] as *u8 902 if h[2 + 8 * s] >= 8 { 903 let m9: i64 = ss_r32(kb, 4) 904 var e9: i64 = 0 905 while e9 < m9 { 906 let eo: i64 = 8 + 4 * m9 + ss_r32(kb, 8 + 4 * e9) 907 if (kb[eo] as i64) == 1 { 908 let kl9: i64 = ss_r32(kb, eo + 1) 909 if kl9 >= 4 { if kl9 < 60 { 910 if kb[eo + 5] == (102 as u8) { if kb[eo + 6] == (114 as u8) { if kb[eo + 7] == (58 as u8) { 911 // copy the key NUL-terminated (bounded by the kl9 < 60 guard above, so 64 912 // holds key+NUL; hoisted 2026-08-08 -- this ran once PER fr: ROW PER WALK) 913 if (wc_kbuf_g as i64) == 0 { wc_kbuf_g = sys_mmap(64) } 914 let kbuf: *u8 = wc_kbuf_g 915 var c: i64 = 0 916 while c < kl9 { kbuf[c] = kb[eo + 5 + c]; c = c + 1 } 917 kbuf[kl9] = 0 as u8 918 // STATE FIRST, then dedup among COLLECTED urls only (seq1288, measured 919 // 2026-07-29): the old pre-state seen-list capped at WC_MAXPEND*4+16 and 920 // SATURATED once the shard held >~1040 distinct fr: keys -- every later key 921 // was never state-checked, so a compounding crawl read pending=0 while 922 // holding hundreds of P rows (pending decayed 256 -> 2 -> 0 in one day; the 923 // coverage leg silently self-wedged). ss_hget is last-wins across segments, 924 // so re-checking a key already walked in another segment is idempotent; the 925 // only dedup that matters is among the <=maxout COLLECTED urls, which is 926 // bounded by construction and cannot saturate. 927 if ss_hget(h, kbuf, pp, pl) == 1 { if pl[0] > 1 { 928 let vp: *u8 = pp[0] as *u8 929 if vp[0] == (80 as u8) { if wc_pc_take(npend, maxout) == 1 { if pl[0] < WC_MAGIC_2000 { 930 var dup: i64 = 0 931 var d: i64 = 0 932 while d < npend { 933 let sp: *u8 = (seenk as i64 + d * 64) as *u8 934 var eq: i64 = 1 935 var x: i64 = 0 936 while x <= kl9 { if sp[x] != kbuf[x] { eq = 0; x = kl9 + 1 } else { x = x + 1 } } 937 if eq == 1 { dup = 1; d = npend } else { d = d + 1 } 938 } 939 if dup == 0 { 940 let dst: *u8 = (seenk as i64 + npend * 64) as *u8 941 var c2: i64 = 0 942 while c2 <= kl9 { dst[c2] = kbuf[c2]; c2 = c2 + 1 } 943 let uo: *u8 = (urlsout as i64 + npend * WC_MAGIC_2048) as *u8 944 var y: i64 = 0 945 while y < pl[0] - 1 { uo[y] = vp[1 + y]; y = y + 1 } 946 uo[pl[0] - 1] = 0 as u8 947 // R4 CANONICAL LANE: canonly=1 keeps ONLY canonical-seed hosts. 948 // The slot is written BEFORE the test and simply not committed 949 // when it fails (npend not incremented), so the next candidate 950 // overwrites it -- no second buffer, no shifting. 951 var keep4: i64 = 1 952 if canonly == 1 { if wc_host_is_canon(canh, ncan, uo, hbuf4) == 0 { keep4 = 0 } } 953 // G0 HOST-BALANCED WINDOW: refuse a row whose host already holds 954 // WC_WINQ slots in THIS window (see wc_winq_admit above). The 955 // surplus was guaranteed to become hostcap-defer at the fetch 956 // stage, so no indexable row is lost -- only a wasted slot. 957 if keep4 == 1 { 958 let hl5: i64 = wc_host(uo, hbuf4) 959 if hl5 > 0 { if wc_winq_admit(wqh, wqc, wqn, maxout, hbuf4, hl5, WC_WINQ) == 0 { keep4 = 0 } } 960 } 961 if keep4 == 1 { npend = npend + 1 } 962 } 963 } } } 964 } } 965 } } } 966 } } 967 } 968 e9 = e9 + 1 969 } 970 } 971 s = s + 1 972 } 973 return npend 974} 975// has this frontier key been banked THIS run? (the open handle can't see the uncommitted writer, so 976// without this a url found on two pages double-banks). runseen = 32-byte frkey slots. 977func wc_runseen(runseen: *u8, nrs: *i64, frk: *u8) -> i64 { 978 var d: i64 = 0 979 while d < nrs[0] { 980 let sp: *u8 = (runseen as i64 + d * 32) as *u8 981 var eq: i64 = 1 982 var x: i64 = 0 983 var go: i64 = 1 984 while go == 1 { 985 if sp[x] != frk[x] { eq = 0; go = 0 } else { 986 if frk[x] == (0 as u8) { go = 0 } else { x = x + 1 } 987 } 988 } 989 if eq == 1 { return 1 } 990 d = d + 1 991 } 992 if nrs[0] < WC_DISC_CEIL + 7 { 993 let dst: *u8 = (runseen as i64 + nrs[0] * 32) as *u8 994 var c: i64 = 0 995 var go2: i64 = 1 996 while go2 == 1 { if c >= 31 { go2 = 0 } else { dst[c] = frk[c]; if frk[c] == (0 as u8) { go2 = 0 } else { c = c + 1 } } } 997 dst[31] = 0 as u8 998 nrs[0] = nrs[0] + 1 999 } 1000 return 0 1001} 1002// link extraction (the nx_web_ingest mechanics, feeding the FILTER + frontier instead of a BFS queue). 1003// Returns how many NEW pending rows were banked. 1004// ENTRY SCRATCH -> LAZY STATIC (2026-08-19 lane F, the 8 KiB/page mapping the debt row named -- the one 1005// three hoist passes stepped over; measured class-mate of the nx_feed_extract 272 kB/call fix, same 1006// nx_crawl_callee_probe campaign). Rewritten per call; fetch children fork their own copy. 1007static wc_hscr_g: *u8 1008func wc_harvest(hh: *u8, hlen: i64, base: *u8, blen: i64, h: *i64, w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, discbox: *i64, runseen: *u8, nrs: *i64) -> i64 { 1009 var banked: i64 = 0 1010 if (wc_hscr_g as i64) == 0 { wc_hscr_g = sys_mmap(WC_MAGIC_8192) } 1011 let scratch: *u8 = wc_hscr_g 1012 var i: i64 = 0 1013 while i < hlen { 1014 var step: i64 = 1 1015 // href= (case-insensitive) 1016 var ish: i64 = 0 1017 if i + 5 < hlen { 1018 var c0: i64 = hh[i] as i64 1019 if c0 >= 0x41 { if c0 <= 0x5a { c0 = c0 + 0x20 } } 1020 if c0 == 0x68 { 1021 var c1: i64 = hh[i+1] as i64 1022 if c1 >= 0x41 { if c1 <= 0x5a { c1 = c1 + 0x20 } } 1023 var c2: i64 = hh[i+2] as i64 1024 if c2 >= 0x41 { if c2 <= 0x5a { c2 = c2 + 0x20 } } 1025 var c3: i64 = hh[i+3] as i64 1026 if c3 >= 0x41 { if c3 <= 0x5a { c3 = c3 + 0x20 } } 1027 if c1 == 0x72 { if c2 == 0x65 { if c3 == 0x66 { if (hh[i+4] as i64) == 0x3d { ish = 1 } } } } 1028 } 1029 } 1030 if ish == 1 { 1031 let q: i64 = hh[i+5] as i64 1032 var quoted: i64 = 0 1033 if q == 0x22 { quoted = 1 } 1034 if q == 0x27 { quoted = 1 } 1035 if quoted == 1 { 1036 let cs: i64 = i + 6 1037 var e: i64 = cs 1038 var run: i64 = 1 1039 while run == 1 { run = 0; if e < hlen { if (hh[e] as i64) != q { e = e + 1; run = 1 } } } 1040 let clen: i64 = e - cs 1041 if clen > 0 { if discbox[0] < wc_disc_cap() { 1042 let hp: *u8 = ((hh as i64) + cs) as *u8 1043 var cand: *u8 = 0 as *u8 1044 var candlen: i64 = 0 1045 var abs: i64 = 0 1046 if clen >= 8 { if wc_hassub(hp, 8, "https://" as *u8) == 1 { abs = 1 } } 1047 if abs == 0 { if clen >= 7 { if wc_hassub(hp, 7, "http://" as *u8) == 1 { abs = 1 } } } 1048 if abs == 1 { 1049 cand = hp 1050 candlen = clen 1051 } else { 1052 if (hp[0] as i64) == 0x2f { 1053 var protorel: i64 = 0 1054 if clen >= 2 { if (hp[1] as i64) == 0x2f { protorel = 1 } } 1055 if protorel == 0 { 1056 // host-based via the SHARED resolver (was base+path concat, which banked phantom 1057 // urls like ".../Trust_law/about" -> 404 fetches + rank leaked to fake nodes) 1058 let rl: i64 = olh_resolve_root(base, hp, clen, scratch) 1059 if rl > 0 { 1060 cand = scratch 1061 candlen = rl 1062 } 1063 } 1064 } 1065 } 1066 if (cand as i64) != 0 { 1067 // CANONICALISE FIRST (debt 1786031857): an href is HTML *text*, so `&amp;` must 1068 // become `&` before wc_urlok judges it, before the fr: key is hashed, and long 1069 // before the fetcher sees it -- otherwise every discovered MULTI-PARAM url is 1070 // malformed, the request 404s or 403s, and the HOST gets blamed (dead-host streak) 1071 // for OUR parsing bug. The SHARED nx_url_canon is the same pass the link-graph 1072 // harvest below runs, so a frontier row and its graph node cannot disagree about 1073 // what the url IS. 3x buffer for worst-case percent-encoding expansion; canon 1074 // returns 0 (skip the link) rather than truncate -- a truncated url is a WRONG 1075 // url, not a shorter one. 1076 var ubuf: *u8 = 0 as *u8 1077 if candlen * 3 + 8 > WC_MAGIC_8192 { ubuf = sys_mmap(candlen * 3 + 8) } else { 1078 if (wc_ubuf_g as i64) == 0 { wc_ubuf_g = sys_mmap(WC_MAGIC_8192) } 1079 ubuf = wc_ubuf_g 1080 } 1081 let cnl: i64 = nx_url_canon(cand, candlen, ubuf, candlen * 3 + 4) 1082 if cnl > 0 { 1083 if wc_urlok(ubuf, cnl) == 1 { 1084 if wc_frstate(h, ubuf, cnl) == 0 { 1085 if (wc_frk9_g as i64) == 0 { wc_frk9_g = sys_mmap(64) } 1086 let frk9: *u8 = wc_frk9_g 1087 wc_frkey(ubuf, cnl, frk9) 1088 if wc_runseen(runseen, nrs, frk9) == 0 { 1089 wc_frmark(w, prefix, segidbox, segsbox, ubuf, cnl, 1) 1090 discbox[0] = discbox[0] + 1 1091 banked = banked + 1 1092 } 1093 } 1094 } 1095 } 1096 } 1097 } } 1098 step = (e - i) + 1 1099 } 1100 } 1101 i = i + step 1102 } 1103 // P1 LINK GRAPH: canonical edge harvest via the SHARED olh_scan (the same pass nx_cc_ingest runs -> 1104 // identical node identity + resolution), stored as ONE out:<cid(base)> row (writer-full handling mirrors 1105 // wc_frmark). Decoupled from the frontier walk above: edges keep flowing after WC_MAXDISC trips. 1106 if (wc_edgebuf_g as i64) == 0 { wc_edgebuf_g = sys_mmap(OLH_MAXEDGE * 8) as *i64 } 1107 let edgebuf: *i64 = wc_edgebuf_g 1108 let nedge: i64 = olh_scan(hh, hlen, base, blen, edgebuf, OLH_MAXEDGE) 1109 if nedge > 0 { if (w as i64) != 0 { 1110 if (wc_okey_g as i64) == 0 { wc_okey_g = sys_mmap(64) } 1111 let okey: *u8 = wc_okey_g 1112 wc_outkey(ci_hash(base, blen), okey) 1113 if ss_add(w, 1, okey, edgebuf as *u8, nedge * 8) < 0 { 1114 if ss_commit(prefix, w, segidbox[0]) == 0 { segsbox[0] = segsbox[0] + 1 } 1115 segidbox[0] = segidbox[0] + 1 1116 w[1] = 0 1117 ss_add(w, 1, okey, edgebuf as *u8, nedge * 8) 1118 } 1119 } } 1120 return banked 1121} 1122 1123// SEED INJECTION (factored 2026-07-26 so one URL and a whole seed FILE share ONE code path -- the 1124// single-seed logic was inline and a second copy would have drifted). Banks the url as a PENDING fr: row 1125// and prepends it to THIS run's work list (the store write is not visible to the already-open handle). 1126// No-op if the frontier already knows the url, so re-seeding the same list is idempotent. 1127func wc_seed_add(h: *i64, w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, urls: *u8, npendbox: *i64, seed: *u8, sl: i64) -> i64 { 1128 if sl <= 0 { return 0 } 1129 if wc_frstate(h, seed, sl) != 0 { return 0 } 1130 var npend: i64 = npendbox[0] 1131 var sh: i64 = npend 1132 if sh > WC_MAXPEND - 1 { sh = WC_MAXPEND - 1 } 1133 while sh > 0 { 1134 let dst9: *u8 = (urls as i64 + sh * WC_MAGIC_2048) as *u8 1135 let src9: *u8 = (urls as i64 + (sh - 1) * WC_MAGIC_2048) as *u8 1136 var y9: i64 = 0 1137 var go9: i64 = 1 1138 while go9 == 1 { dst9[y9] = src9[y9]; if src9[y9] == (0 as u8) { go9 = 0 } y9 = y9 + 1 } 1139 sh = sh - 1 1140 } 1141 let u0: *u8 = urls 1142 var z9: i64 = 0 1143 while z9 < sl { u0[z9] = seed[z9]; z9 = z9 + 1 } 1144 u0[sl] = 0 as u8 1145 if npend < WC_MAXPEND { npendbox[0] = npend + 1 } 1146 wc_frmark(w, prefix, segidbox, segsbox, seed, sl, 1) 1147 return 1 1148} 1149 1150// ---- R15 (2026-08-25): THE DOMAIN-MAP ADOPTION -- reconnaissance findings reach the frontier ------- 1151// nx_domain_map is the /compare/webscraping R8 organ. It discovers a domain's urls from robots.txt 1152// Sitemap: lines, from sitemap trees (RECURSING sitemapindex and inflating .gz), from feeds, from the 1153// Wayback CDX and from crt.sh -- and it fingerprints the host's not-found page so a 200-that-is-really 1154// -a-404 gets labelled SOFT404 instead of being crawled. 1155// MEASURED 2026-08-25: nx_catalog reports it SOURCE+BUILT+PROMOTED+REGISTERED+INVOKED with its own 1156// gate, and an importer census over buildroot/runtime (coverage_complete=1 corpus_complete=1) returns 1157// SEVEN matches of which EVERY ONE is the organ itself or its own gate. Nothing in the crawl path 1158// consumed it. So sitemap recursion has existed, complete and proven, one organ away from a frontier 1159// sitting flat at ~1,880 urls. 1160// A CAPABILITY THAT IS BUILT, GATED, AND UNREACHABLE FROM THE THING THAT NEEDS IT IS INDISTINGUISHABLE 1161// FROM ONE NOBODY EVER WROTE -- and it is more expensive, because it also looks done. 1162// 1163// WHY A READER OF ITS ARTIFACT RATHER THAN AN IN-PROCESS CALL TO dm_scan: nx_domain_map is 1164// RECONNAISSANCE. Its own header says "one map is a reconnaissance, not a crawl" and it spends up to 1165// DM_DEF_MAX_FETCHES (64) fetches per domain. Calling it in-process would put a 64-fetch census inside 1166// a crawl step, per host, every run -- and this crawler's entire in-flight budget is WC_POOL_N = 8. 1167// The two organs compose correctly at the ARTIFACT: the map writes rows on its own cadence, the 1168// crawler reads them. No fetch is duplicated, no sitemap parser is re-implemented (rule 15), and 1169// neither organ has to learn anything about the other's internals. 1170// 1171// WHY BY CANONICAL HOST RATHER THAN BY DIRECTORY WALK: iterating the canonical host table costs at 1172// most WC_MAXCANH opens of a KNOWN path. Enumerating DM_OUTDIR would need a getdents64 LOOP, and ONE 1173// getdents64 CALL IS NOT A DIRECTORY LISTING -- a partial read there would silently seed a PREFIX of 1174// the corpus and report success, which is the silent-cap defect with a frontier attached. 1175const WC_DM_DIR: *u8 = "knowledge/status/domain_map" 1176const WC_DM_HOSTSLOT: i64 = 64 // == wc_canon_hosts' stride; DERIVED from that writer, not guessed 1177const WC_DM_MAX_ROWS: i64 = 4096 // rows read per host map. The map APPENDS one run per invocation, 1178 // so a long-lived map is mostly repeats -- and wc_seed_add dedupes 1179 // by frontier key, which makes a re-read idempotent rather than 1180 // duplicative. The cap bounds the read, it does not bound coverage. 1181 1182// PURE, so the seeding RULE is gate-testable with no filesystem: does this (kind, verdict) pair name a 1183// url we are entitled to put in the frontier? 1184// ADMITTED: 1185// url + LISTED -- an absolute url a sitemap or a feed DECLARED. This is the sitemap payload, and it 1186// is the row class that makes this rung worth building. 1187// path + LIVE -- a probe path the host actually SERVED, and which the map's own soft-404 1188// fingerprint did not classify as the not-found page. 1189// REFUSED -- each for its own stated reason, never by omission: 1190// MISSING / UNREACHABLE / BUDGET-REFUSED -- the map found no page there. Seeding these would spend 1191// crawl budget rediscovering an absence the reconnaissance has already paid for once. 1192// SOFT404 -- the map PROVED this is the not-found page wearing a 200. Seeding it is precisely the 1193// waste the fingerprint exists to prevent. 1194// ROBOTS-REFUSED -- THE COMPLIANCE TOOTH. The map recorded that the host's robots.txt refused this 1195// url. A seeder that admitted it would launder a refusal into a fetch through a side door, 1196// defeating wc_rb_allowed without ever calling it. 1197// FINGERPRINT -- not a url at all; it is the "status=.. len=.. title_hash=.." record itself. 1198// DECLARED -- a robots.txt Allow/Disallow PATH. A path named in a rule is not a page that exists; 1199// DECLARED means "this string appeared in a directive", never "this resource was served". 1200// CERT / ARCHIVED -- crt.sh hostnames and Wayback rows. Out of scope for this rung, and named here 1201// so the next reader knows they were considered and declined rather than forgotten. 1202func wc_dm_seedable(kind: *u8, verdict: *u8) -> i64 { 1203 if wc_streq(verdict, "LISTED" as *u8) == 1 { if wc_streq(kind, "url" as *u8) == 1 { return 1 } } 1204 if wc_streq(verdict, "LIVE" as *u8) == 1 { if wc_streq(kind, "path" as *u8) == 1 { return 1 } } 1205 return 0 1206} 1207 1208// Read ONE host's map and bank its seedable rows. Returns urls NEWLY added to the frontier. 1209// foundbox[0] is incremented when a map FILE existed, which is what separates "this host has no map" 1210// from "this host's map held nothing new" -- two states with completely different remedies that a 1211// single added-count would render identical. 1212func wc_dm_seed_host(h: *i64, w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, 1213 urls: *u8, npendbox: *i64, host: *u8, foundbox: *i64) -> i64 { 1214 let p: *u8 = sys_mmap(WC_MAGIC_1024) 1215 var o: i64 = wc_outcat(p, 0, WC_DM_DIR) 1216 o = wc_outcat(p, o, "/" as *u8) 1217 var hk: i64 = 0 1218 while host[hk] != (0 as u8) { p[o] = host[hk]; o = o + 1; hk = hk + 1 } 1219 o = wc_outcat(p, o, ".tsv" as *u8) 1220 p[o] = 0 as u8 1221 let szb: *i64 = sys_mmap(16) as *i64 1222 szb[0] = 0 1223 let fb: *u8 = ss_loadfile(p, szb, 0) 1224 if (fb as i64) == 0 { return 0 } 1225 if szb[0] <= 0 { return 0 } 1226 foundbox[0] = foundbox[0] + 1 1227 // Row: <epoch> TAB <source> TAB <kind> TAB <verdict> TAB <value>. Only fields 2,3,4 are read. 1228 // ALL FOUR BUFFERS ARE HOISTED OUT OF THE ROW LOOP -- a per-row sys_mmap here would burn a page 1229 // per row across up to WC_MAXCANH maps (NEVER ALLOCATE IN A HOT LOOP). 1230 let kind: *u8 = sys_mmap(WC_DM_HOSTSLOT) 1231 let verd: *u8 = sys_mmap(WC_DM_HOSTSLOT) 1232 let val: *u8 = sys_mmap(WC_MAGIC_2048) 1233 let su: *u8 = sys_mmap(WC_MAGIC_2048) 1234 var added: i64 = 0 1235 var rows: i64 = 0 1236 var i: i64 = 0 1237 var fld: i64 = 0 1238 var kl: i64 = 0 1239 var vl: i64 = 0 1240 var al: i64 = 0 1241 while i <= szb[0] { 1242 var ch: i64 = 10 1243 if i < szb[0] { ch = fb[i] as i64 } 1244 if ch == 10 { 1245 if fld >= 4 { if rows < WC_DM_MAX_ROWS { 1246 kind[kl] = 0 as u8 1247 verd[vl] = 0 as u8 1248 val[al] = 0 as u8 1249 rows = rows + 1 1250 if al > 0 { if wc_dm_seedable(kind, verd) == 1 { 1251 var so: i64 = 0 1252 // a LISTED url is absolute; a LIVE path is relative to the host whose map this is 1253 if wc_streq(kind, "path" as *u8) == 1 { 1254 so = wc_outcat(su, 0, "https://" as *u8) 1255 var hh2: i64 = 0 1256 while host[hh2] != (0 as u8) { su[so] = host[hh2]; so = so + 1; hh2 = hh2 + 1 } 1257 } 1258 var vv: i64 = 0 1259 while vv < al { su[so] = val[vv]; so = so + 1; vv = vv + 1 } 1260 su[so] = 0 as u8 1261 // COMPOSE THE INCUMBENT FILTER, never a second one: everything the ordinary crawl 1262 // refuses (tracking params, tag farms, binary extensions, oversize urls) must be 1263 // refused on this path too, or the map becomes a bypass around wc_urlok. 1264 if wc_urlok(su, so) == 1 { 1265 added = added + wc_seed_add(h, w, prefix, segidbox, segsbox, urls, npendbox, su, so) 1266 } 1267 } } 1268 } } 1269 fld = 0; kl = 0; vl = 0; al = 0 1270 } else { 1271 if ch == 9 { fld = fld + 1 } 1272 else { if ch != 13 { 1273 if fld == 2 { if kl < WC_DM_HOSTSLOT - 1 { kind[kl] = ch as u8; kl = kl + 1 } } 1274 if fld == 3 { if vl < WC_DM_HOSTSLOT - 1 { verd[vl] = ch as u8; vl = vl + 1 } } 1275 if fld == 4 { if al < WC_MAGIC_2048 - 2 { val[al] = ch as u8; al = al + 1 } } 1276 } } 1277 } 1278 i = i + 1 1279 } 1280 return added 1281} 1282// djb2 host hash (factored: the per-host index-cap map and the pool's politeness/dead-host lists must 1283// agree on host identity, so ONE hash function). 1284// ---- R11 CRAWL OUTCOME LOG: every fetch decision becomes MINEABLE --------------------------------- 1285// Operator 2026-08-06: "make sure success isnt accidental and that we have an error log we are mining 1286// and improving our crawler on." Both halves were real. MEASURED the same morning: the SAME v2ph url 1287// that ingested 2,176 chars overnight returned a 403 Cloudflare "Just a moment" interstitial on EVERY 1288// leg -- so that win was WEATHER, not capability, and nothing recorded the difference. 1289// The interstitial guard below already REFUSED to index it (correct), but it refused SILENTLY: no line, 1290// no row, nothing to count. A crawler that discards a page without saying why cannot be improved -- and 1291// worse, the host-health streak then retires the host, laundering OUR capability gap into permanent 1292// coverage loss. This writes ONE tab-separated row per terminal decision so every loss becomes DATA. 1293// class: ok | botwall | http | fetchfail | timeout | thin | dup | hostcap 1294// A CRAWLER THAT CANNOT SAY WHY IT DROPPED A PAGE CANNOT BE IMPROVED, ONLY GUESSED AT. 1295const WC_OUTLOG: *u8 = "knowledge/status/crawl_outcomes.log" 1296static wc_outfd_g: i64 1297func wc_outopen() -> i64 { 1298 // O_WRONLY|O_CREAT|O_APPEND (1089) -- APPEND is load-bearing: this is an event stream spanning runs, 1299 // and a truncating open would erase exactly the history the miner exists to read. 1300 wc_outfd_g = __syscall(56, 0 - 100, WC_OUTLOG as i64, WC_MAGIC_1089, 420, 0, 0) 1301 return wc_outfd_g 1302} 1303func wc_outn(b: *u8, at: i64, v: i64) -> i64 { 1304 var a: i64 = at 1305 var x: i64 = v 1306 if x < 0 { b[a] = 45 as u8; a = a + 1; x = 0 - x } 1307 let t: *u8 = sys_mmap(24) 1308 var k: i64 = 0 1309 if x == 0 { t[0] = 48 as u8; k = 1 } 1310 while x > 0 { t[k] = (48 + x % 10) as u8; x = x / 10; k = k + 1 } 1311 var j: i64 = 0 1312 while j < k { b[a] = t[k - 1 - j]; a = a + 1; j = j + 1 } 1313 return a 1314} 1315func wc_outcat(b: *u8, at: i64, s: *u8) -> i64 { 1316 var a: i64 = at 1317 var i: i64 = 0 1318 while s[i] != (0 as u8) { b[a] = s[i]; a = a + 1; i = i + 1 } 1319 return a 1320} 1321func wc_outcome(cls: *u8, status: i64, bytes: i64, url: *u8, ul: i64) -> i64 { 1322 if wc_outfd_g <= 0 { return 0 } 1323 let b: *u8 = sys_mmap(WC_MAGIC_4096) 1324 var o: i64 = 0 1325 o = wc_outn(b, o, sys_now_realtime_sec()) 1326 b[o] = 9 as u8; o = o + 1 1327 o = wc_outcat(b, o, cls) 1328 b[o] = 9 as u8; o = o + 1 1329 o = wc_outn(b, o, status) 1330 b[o] = 9 as u8; o = o + 1 1331 o = wc_outn(b, o, bytes) 1332 b[o] = 9 as u8; o = o + 1 1333 var i: i64 = 0 1334 while i < ul { if o < WC_MAGIC_4000 { b[o] = url[i]; o = o + 1 } i = i + 1 } 1335 b[o] = 10 as u8; o = o + 1 1336 sys_write(wc_outfd_g, b, o) 1337 sys_munmap(b, WC_MAGIC_4096) 1338 return 1 1339} 1340func wc_hosthash(hostp: *u8, hl: i64) -> i64 { 1341 var hh: i64 = WC_MAGIC_5381 1342 var i: i64 = 0 1343 while i < hl { hh = (((hh << 5) + hh) + (hostp[i] as i64)) & WC_HASH_MASK; i = i + 1 } 1344 return hh 1345} 1346func wc_deadhost_has(dh: *i64, ndh: i64, hh: i64) -> i64 { 1347 var i: i64 = 0 1348 while i < ndh { if dh[i] == hh { return 1 } i = i + 1 } 1349 return 0 1350} 1351func wc_deadhost_add(dh: *i64, ndhbox: *i64, hh: i64) -> i64 { 1352 if wc_deadhost_has(dh, ndhbox[0], hh) == 1 { return 0 } 1353 if ndhbox[0] < WC_DEADHOSTS { dh[ndhbox[0]] = hh; ndhbox[0] = ndhbox[0] + 1; return 1 } 1354 return 0 1355} 1356// ==== R3 HOST-HEALTH (2026-08-04, debt 1785892352): persistent per-host dead-run streak as 1357// hd:<hosthash> rows in the SAME shard (the frontier's store is the crawler's only state). MEASURED 1358// need: post-widening, 1983 of the 2048-row pull window were dead-host rows (97% rot) recurring in 1359// stable walk order -- skip-without-retire made the rot immortal. Streak++ once per run the host is 1360// found dead; reset on any completed HTTP response; at WC_HD_RETIRE the batch selector RETIRES the 1361// rows it skips (P->D, same last-wins flip as a fetched url). A retired url is re-addable the moment 1362// any live page links to it again -- rows are cheap, seeds are not (see the exemption). 1363func wc_hdkey(hh: i64, out: *u8) -> i64 { 1364 out[0] = 104 as u8; out[1] = 100 as u8; out[2] = 58 as u8 // "hd:" 1365 var o: i64 = 3 1366 // DELEGATED 2026-08-08 -- this was a hand-rolled integer->ascii clone, so it is one of the ~1792. 1367 // ccz_cat_num (nx_itoa_lib) writes MSB-first INTO THE CALLER'S buffer and allocates NOTHING, which 1368 // is strictly better than the lazy static I put here this morning: no scratch buffer exists at all, 1369 // so there is nothing to hoist and nothing for nx_mmapbal to convict. 1370 // BEHAVIOUR IDENTITY CHECKED, NOT ASSUMED: for hh==0 the old code wrote '0' then NUL at offset 3 and 1371 // returned 4; ccz_cat_num writes '0', NUL and returns the NUL offset = 4. For hh>0 both emit the same 1372 // MSB-first digits, NUL-terminate, and return the offset OF the NUL. Same bytes, same return, no alloc. 1373 return ccz_cat_num(out, o, hh) 1374} 1375// the host's persisted dead-run streak (0 = healthy/unknown). Reads the RUN-START handle: retire 1376// decisions deliberately see the streak as of run start, so death #N retires on run N+1, never same-run. 1377func wc_hd_streak(h: *i64, hh: i64) -> i64 { 1378 if (h as i64) == 0 { return 0 } 1379 if (wc_hds_key_g as i64) == 0 { wc_hds_key_g = sys_mmap(64) } 1380 let key: *u8 = wc_hds_key_g 1381 wc_hdkey(hh, key) 1382 if (wc_hds_pp_g as i64) == 0 { wc_hds_pp_g = sys_mmap(16) as *i64 } 1383 if (wc_hds_pl_g as i64) == 0 { wc_hds_pl_g = sys_mmap(16) as *i64 } 1384 let pp: *i64 = wc_hds_pp_g 1385 let pl: *i64 = wc_hds_pl_g 1386 if ss_hget(h, key, pp, pl) != 1 { return 0 } 1387 let vp: *u8 = pp[0] as *u8 1388 var v: i64 = 0 1389 var i: i64 = 0 1390 while i < pl[0] { 1391 var c: i64 = vp[i] as i64 1392 if c >= 48 { if c <= 57 { v = v * 10 + (c - 48) } } 1393 i = i + 1 1394 } 1395 return v 1396} 1397// bank the streak (append-only, last-wins; mirrors wc_frmark's writer-full discipline) 1398func wc_hd_put(w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, hh: i64, v: i64) -> i64 { 1399 if (wc_hdp_key_g as i64) == 0 { wc_hdp_key_g = sys_mmap(64) } 1400 let key: *u8 = wc_hdp_key_g 1401 wc_hdkey(hh, key) 1402 if (wc_hdp_val_g as i64) == 0 { wc_hdp_val_g = sys_mmap(28) } 1403 let val: *u8 = wc_hdp_val_g 1404 // DELEGATED 2026-08-08 (same reasoning as wc_hdkey): ccz_cat_num writes MSB-first straight into 1405 // val and allocates NOTHING, so the scratch buffer and its hoisted static both disappear. 1406 // It returns the offset OF the NUL, which is exactly the digit count this code computed as vl -- 1407 // v==0 -> 1, v>0 -> k. The extra NUL it writes at val[vl] is in bounds (val is 28 B) and harmless 1408 // because ss_add is given vl explicitly and never reads past it. 1409 let vl: i64 = ccz_cat_num(val, 0, v) 1410 if ss_add(w, 1, key, val, vl) < 0 { 1411 if ss_commit(prefix, w, segidbox[0]) == 0 { segsbox[0] = segsbox[0] + 1 } 1412 segidbox[0] = segidbox[0] + 1 1413 w[1] = 0 1414 ss_add(w, 1, key, val, vl) 1415 } 1416 return 0 1417} 1418// ---- R5 SEED FRESHNESS RE-QUEUE (2026-08-05, debt 1785937865) --------------------------------- 1419// MEASURED across 4 consecutive runs: frontier_remaining 2047->1963->1769->1748 while 1420// discovered_new fell 121->35->0 -- run 3 fetched 300 pages and banked ZERO new frontier rows. 1421// wc_harvest only banks a NEVER-SEEN url, so once a host's known links are exhausted its depth 1422// stops compounding. ★A CRAWLER IS A CLOSED LOOP: IF CONSUMPTION EXCEEDS DISCOVERY, THE FRONTIER 1423// IS A BURN-DOWN AND COVERAGE PLATEAUS NO MATTER HOW FAIRLY THE BUDGET IS SCHEDULED. The single 1424// biggest replenishment blocker is done-is-forever: a seed hub fetched once is D and can never 1425// rediscover links -- but hub pages (portals, news indexes, topic lists) CHANGE, which is exactly 1426// why they are seeds. This re-queue walks the seed list with a PERSISTED cursor (sr:cursor row, 1427// same ASCII-digit encoding as the hd: streak rows) and re-marks up to WC_SEED_REFRESH already-D 1428// seed urls back to P each run. Bounded by construction; P and unknown rows are left alone. 1429func wc_srkey(out: *u8) -> i64 { 1430 out[0] = 115 as u8; out[1] = 114 as u8; out[2] = 58 as u8 1431 out[3] = 99 as u8; out[4] = 117 as u8; out[5] = 114 as u8 1432 out[6] = 0 as u8 1433 return 6 1434} 1435func wc_sr_cursor(h: *i64) -> i64 { 1436 if (h as i64) == 0 { return 0 } 1437 if (wc_src_key_g as i64) == 0 { wc_src_key_g = sys_mmap(64) } 1438 let key: *u8 = wc_src_key_g 1439 wc_srkey(key) 1440 if (wc_src_pp_g as i64) == 0 { wc_src_pp_g = sys_mmap(16) as *i64 } 1441 if (wc_src_pl_g as i64) == 0 { wc_src_pl_g = sys_mmap(16) as *i64 } 1442 let pp: *i64 = wc_src_pp_g 1443 let pl: *i64 = wc_src_pl_g 1444 if ss_hget(h, key, pp, pl) != 1 { return 0 } 1445 let vp: *u8 = pp[0] as *u8 1446 var v: i64 = 0 1447 var i: i64 = 0 1448 while i < pl[0] { 1449 var c: i64 = vp[i] as i64 1450 if c >= 48 { if c <= 57 { v = v * 10 + (c - 48) } } 1451 i = i + 1 1452 } 1453 return v 1454} 1455func wc_sr_put(w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, v: i64) -> i64 { 1456 if (wc_srp_key_g as i64) == 0 { wc_srp_key_g = sys_mmap(64) } 1457 let key: *u8 = wc_srp_key_g 1458 wc_srkey(key) 1459 if (wc_srp_val_g as i64) == 0 { wc_srp_val_g = sys_mmap(28) } 1460 let val: *u8 = wc_srp_val_g 1461 // DELEGATED 2026-08-08 -- identical case to wc_hd_put above; see that comment for the identity check. 1462 let vl: i64 = ccz_cat_num(val, 0, v) 1463 if ss_add(w, 1, key, val, vl) < 0 { 1464 if ss_commit(prefix, w, segidbox[0]) == 0 { segsbox[0] = segsbox[0] + 1 } 1465 segidbox[0] = segidbox[0] + 1 1466 w[1] = 0 1467 ss_add(w, 1, key, val, vl) 1468 } 1469 return 0 1470} 1471// re-mark up to WC_SEED_REFRESH done seeds as pending, copying each into dst (2KB slots) so the 1472// caller's canonical block fetches them THIS run. dstbox[0]=count in/out, dstbox[1]=slot capacity 1473// (2-slot box keeps the arity at 9 -- the tree's proven ceiling). Examines each seed at most once 1474// per run (lim <= nseeds) so a short list cannot be requeued twice through the uncommitted writer. 1475func wc_seed_refresh(h: *i64, w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, seedstore: *u8, nseeds: i64, dst: *u8, dstbox: *i64) -> i64 { 1476 if nseeds <= 0 { return 0 } 1477 var cur: i64 = wc_sr_cursor(h) 1478 if cur < 0 { cur = 0 } 1479 cur = cur % nseeds 1480 var lim: i64 = WC_SEED_REFRESH 1481 if lim > nseeds { lim = nseeds } 1482 var rq: i64 = 0 1483 var j: i64 = 0 1484 while j < lim { 1485 let ix: i64 = (cur + j) % nseeds 1486 let sp: *u8 = (seedstore as i64 + ix * WC_MAGIC_2048) as *u8 1487 var sl: i64 = 0 1488 while sp[sl] != (0 as u8) { sl = sl + 1 } 1489 if sl > 0 { if wc_frstate(h, sp, sl) == 2 { 1490 wc_frmark(w, prefix, segidbox, segsbox, sp, sl, 1) 1491 if dstbox[0] < dstbox[1] { 1492 let dp: *u8 = (dst as i64 + dstbox[0] * WC_MAGIC_2048) as *u8 1493 var c: i64 = 0 1494 while c <= sl { dp[c] = sp[c]; c = c + 1 } 1495 dstbox[0] = dstbox[0] + 1 1496 } 1497 rq = rq + 1 1498 } } 1499 j = j + 1 1500 } 1501 wc_sr_put(w, prefix, segidbox, segsbox, (cur + lim) % nseeds) 1502 return rq 1503} 1504// seed-file membership (exact url match) -- seed rows are NEVER retired 1505func wc_is_seed(seedstore: *u8, nseeds: i64, u: *u8, ul: i64) -> i64 { 1506 var s: i64 = 0 1507 while s < nseeds { 1508 let sp: *u8 = (seedstore as i64 + s * WC_MAGIC_2048) as *u8 1509 var eq: i64 = 1 1510 var i: i64 = 0 1511 while i < ul { if sp[i] != u[i] { eq = 0; i = ul } else { i = i + 1 } } 1512 if eq == 1 { if sp[ul] == (0 as u8) { return 1 } } 1513 s = s + 1 1514 } 1515 return 0 1516} 1517// the retire decision + act, factored so the gate can bite it network-free. Returns 1 = row retired. 1518// SEED EXEMPTION LIVES AT THE CALL SITE (wc_is_seed guard) -- 8 params, within the tree's proven arity. 1519func wc_retire_if_rotten(h: *i64, w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, u: *u8, ul: i64, hh: i64) -> i64 { 1520 if wc_hd_streak(h, hh) < WC_HD_RETIRE { return 0 } 1521 wc_frmark(w, prefix, segidbox, segsbox, u, ul, 2) 1522 return 1 1523} 1524// spool path for a worker slot: /tmp/nx_wcpool_<slot>.bin. Slot-indexed names cannot collide: the 1525// searchcrawl sub's liveness guard enforces ONE crawler instance, and a slot is reused only after its 1526// child was reaped and its spool consumed. 1527func wc_spool_path(slot: i64, out: *u8) -> i64 { 1528 let lit: *u8 = "/tmp/nx_wcpool_" as *u8 1529 var o: i64 = 0 1530 while lit[o] != (0 as u8) { out[o] = lit[o]; o = o + 1 } 1531 let t: *u8 = sys_mmap(28) 1532 var m: i64 = slot 1533 var k: i64 = 0 1534 if m == 0 { t[0] = 48 as u8; k = 1 } 1535 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 } 1536 var i: i64 = 0 1537 while i < k { out[o] = t[k - 1 - i]; o = o + 1; i = i + 1 } 1538 let ext: *u8 = ".bin" as *u8 1539 var j: i64 = 0 1540 while ext[j] != (0 as u8) { out[o] = ext[j]; o = o + 1; j = j + 1 } 1541 out[o] = 0 as u8 1542 return o 1543} 1544// CHILD: one fetch -> spool file [status][len][body] -> exit. All child memory is freed by the kernel 1545// on exit (the windowed-fork idiom); the PARENT owns the wall-clock budget, so the child needs no timer 1546// of its own. exit 0 = body spooled; 1 = fetch returned nothing; 2 = spool unwritable. 1547// ==================== R11: ROBOTS.TXT -- THE HOST'S RULES OVER THE HOST'S CONTENT ==================== 1548// (2026-08-06) MEASURED GAP, not a suspected one: this crawler contained ZERO references to robots. 1549// `nx_robots.nx` -- a complete RFC 9309 parser (most-specific user-agent group, longest-match 1550// Allow/Disallow with Allow winning ties, Crawl-delay extraction), tested by nx_robots_test -- has 1551// existed the whole time and was imported by exactly ONE organ (nx_media_import). Its own header 1552// declares `wired_status: FULLY_WIRED`. And `pace_set_crawl_delay`, the slot the pacer keeps for 1553// precisely this number, was called from NOTHING but its own gate. So the crawler paced every host on 1554// a fixed 1s floor and never once asked a host what pace IT wanted. 1555// **A SELF-DECLARED wired_status IS A COMMENT, NOT AN ADOPTION MEASUREMENT.** 1556// **POLITENESS THAT IS BUILT BUT NOT WIRED IS RUDENESS WITH GOOD INTENTIONS.** 1557// 1558// This closes a loop the estate had already reasoned its way to: on 2026-08-06 loc.gov served two 1559// pages and 403'd the rest, and the independent auditor proved that was RATE LIMITING rather than a 1560// bot wall. The lever named that day was PACING -- and the host had been publishing its preferred 1561// pace in a file we never fetched. 1562// 1563// CONTRACT (deliberately conservative: a robots failure must never become a coverage failure) 1564// - THE THREE-WAY RFC 9309 s2.3.1 SPLIT (corrected 2026-08-25; this bullet USED to read "absent / 1565// 404 / unreachable / empty -> ALLOW", which CONFLATED two cases the RFC separates, and the 1566// conflation was the ALLOWING one): 1567// * 2xx -> s2.3.1.1 "Successful Access": follow the parseable rules. A 2xx with an EMPTY body 1568// is a successful fetch of a file that states no rules, so it allows -- by the RULES 1569// path, not by a failure path. 1570// * 4xx -> s2.3.1.3 "Unavailable": VERBATIM, "If a server status code indicates that the 1571// robots.txt file is unavailable to the crawler, then the crawler MAY access any 1572// resources on the server." ALLOW is correct and is what we already did. 1573// * 5xx or a transport failure with NO status line (connect / TLS / read) -> s2.3.1.4 1574// "Unreachable": VERBATIM, "If the robots.txt file is unreachable due to server or 1575// network errors, this means the robots.txt file is undefined and the crawler MUST 1576// assume complete disallow." We now DISALLOW THE WHOLE HOST for the run. 1577// WHAT WAS BROKEN: a host answering 500 on /robots.txt was crawled exactly like a host answering 1578// 404, because both landed in the same len-0 cache slot and wc_rb_allowed read len<=0 as ALLOW. 1579// The old comment's reasoning -- "our own inability to fetch a file is not a rule" -- is TRUE of 1580// 4xx and FALSE of 5xx: the RFC deliberately makes the server's own failure a DISALLOW, because 1581// a host that cannot serve its rules cannot be assumed to have none. 1582// - NO DEVIATION IS TAKEN. Google's published parser retries 5xx and, after a long outage, falls 1583// back to treating the host as unavailable; RFC 9309 s2.3.1.4 itself permits that after "a 1584// reasonably long period of time (for example, 30 days)". We do NOT implement it, because a 1585// per-host 30-day outage memory is state we do not keep, and INVENTING the fallback without the 1586// memory would just be the old bug with a citation attached. WC_RB_DEVIATION_30D names the 1587// choice so it is reviewable rather than accidental. 1588// - a disallowed url is marked DONE and logged `robots`, and is NEVER charged to the host's 1589// dead-host streak. Refusing to fetch is OUR decision; laundering it into "the host is dead" 1590// would retire the host permanently (wc_seed_add refuses any existing row) -- the exact harm the 1591// R9 retire exemption was written to prevent. 1592// - Crawl-delay feeds pace_set_crawl_delay CLAMPED, so one hostile or fat-fingered robots.txt 1593// cannot wedge a whole run; the clamp is LOGGED, because a silent clamp is a lie about what we 1594// actually honoured. 1595// - the robots.txt fetch is ITSELF paced and is attempted AT MOST ONCE PER HOST PER RUN -- a 1596// politeness probe that hammers is not politeness. 1597// - WC_ROBOTS_ON is a named switch, so turning this off is a deliberate, reviewable one-line act. 1598// UA NOTE: we match the group `nishibot` (the NishiBot/1.0 product token already used elsewhere in 1599// the fetch stack), falling back to `*` per RFC 9309. No site publishes a nishibot group today, so in 1600// practice `*` governs -- which is the conservative reading, and is honest about what we are. 1601const WC_ROBOTS_ON: i64 = 1 1602const WC_ROBOTS_HOSTS: i64 = 256 // distinct hosts whose robots.txt we cache per run 1603const WC_ROBOTS_ARENA: i64 = 1048576 // ONE packed arena for all cached bodies (256 x 16KB would be 4MB of holes) 1604const WC_ROBOTS_MAX: i64 = 16384 // per-host body cap 1605const WC_ROBOTS_FCAP: i64 = 262144 // fetch buffer for robots.txt 1606const WC_RB_UALEN: i64 = 8 // len("nishibot") 1607 1608// ---- RFC 9309 s2.3.1 ACCESS-RESULT STATES (2026-08-25) ---------------------------------------- 1609// These are ordinals naming the RFC's three outcomes. They are NOT thresholds and nothing compares 1610// them with < or >; a reader who sees 0/1/2 should read the RFC section, not an ordering. 1611const WC_RB_ST_RULES: i64 = 0 // s2.3.1.1 Successful Access -- a body was parsed, follow it 1612const WC_RB_ST_ALLOW_ALL: i64 = 1 // s2.3.1.3 Unavailable -- "MAY access any resources" 1613const WC_RB_ST_DISALLOW_ALL: i64 = 2 // s2.3.1.4 Unreachable -- "MUST assume complete disallow" 1614 1615// HTTP status CLASS bounds, from RFC 9110 s15 (Successful 2xx / Redirection 3xx / Client Error 4xx / 1616// Server Error 5xx). Written as named bounds rather than bare 200/400/500 so the mapping onto the 1617// RFC 9309 states above is legible at the comparison site instead of being folklore. 1618const WC_HTTP_2XX_LO: i64 = 200 1619const WC_HTTP_2XX_HI: i64 = 299 1620const WC_HTTP_3XX_LO: i64 = 300 1621const WC_HTTP_3XX_HI: i64 = 399 1622const WC_HTTP_4XX_LO: i64 = 400 1623const WC_HTTP_4XX_HI: i64 = 499 1624const WC_HTTP_5XX_LO: i64 = 500 1625const WC_HTTP_5XX_HI: i64 = 599 1626// ff_core_x sets out_status[0] = 0 before the loop and only overwrites it once a response LINE has 1627// been parsed. So 0 is not a status -- it is the transport never having produced one (bad url, 1628// connect refused, TLS handshake failure, read failure). RFC 9309 s2.3.1.4 names exactly this class 1629// alongside 5xx: "unreachable due to server OR NETWORK errors". 1630const WC_HTTP_NO_STATUS: i64 = 0 1631 1632// RFC 9309 s2.3.1.2 Redirects: "The crawlers SHOULD follow at least five consecutive redirects". 1633// MEASURED DEFECT 2026-08-25: this call passed 3, which is below that SHOULD -- a robots.txt behind 1634// 4 hops read as unreachable and (post-fix) would now DISALLOW the whole host. Raising it to the 1635// RFC's floor is therefore not cosmetic: under the corrected 5xx rule an under-budgeted redirect 1636// chain converts into lost coverage. 1637const WC_ROBOTS_REDIRECTS: i64 = 5 1638// s2.3.1.2 also says: "If there are more than five consecutive redirects, crawlers MAY assume that 1639// the robots.txt file is unavailable." So a chain still unresolved AFTER the budget is ALLOW_ALL by 1640// explicit permission -- it is not folded into the unreachable class. 1641 1642// ---- CRAWL-DELAY: WE NO LONGER CLAMP THE NUMBER DOWN (2026-08-25) ----------------------------- 1643// WC_ROBOTS_CDMAX used to rewrite any published Crawl-delay above 30 s DOWN to 30 s. That is a 1644// COMPLIANCE VIOLATION, not a safety valve: a host publishing "Crawl-delay: 120" was then crawled 1645// every 30 s, i.e. four times faster than it asked. Crawl-delay is absent from RFC 9309, but it is 1646// load-bearing for verified-crawler status -- Cloudflare's verified-bot policy names "an AI Crawler 1647// that does not respect the crawl-delay directive in robots.txt" as a removal-triggering breach. 1648// THE FIX IS NOT A BIGGER CLAMP, IT IS DEFERRAL: the published number is now adopted EXACTLY, and a 1649// host whose interval has not elapsed is left PENDING by pace_should_defer_tbl and picked up on a 1650// later run. Deferral costs nothing (the frontier row is already persisted) and is the only 1651// behaviour that both honours the host and keeps the coverage. 1652// The one remaining bound is a HORIZON, not a rate clamp, and when it binds we crawl the host LESS, 1653// never more. Its value is DERIVED from RFC 9309 s2.4: "Crawlers SHOULD NOT use the cached version 1654// for more than 24 hours" -- a Crawl-delay longer than the maximum lifetime of the very file that 1655// published it cannot be honoured coherently within one rule-lifetime, so we decline the host for 1656// the run and say so, rather than invent a faster pace it never agreed to. 1657const WC_ROBOTS_CD_HORIZON_S: i64 = 86400 // 24 h, per RFC 9309 s2.4 cache lifetime 1658const WC_ROBOTS_CD_MS_PER_S: i64 = 1000 // seconds -> ms for pace_set_crawl_delay 1659 1660// The 30-day "treat a long-unreachable robots.txt as unavailable" fallback that RFC 9309 s2.3.1.4 1661// PERMITS ("crawlers MAY assume that the robots.txt file is unavailable") and that Google's parser 1662// implements. 0 = NOT TAKEN. Implementing it needs a persisted per-host first-unreachable timestamp 1663// that this crawler does not keep, and a fallback without that memory is indistinguishable from the 1664// bug this whole block exists to fix. Named so the choice is reviewable, never accidental. 1665const WC_RB_DEVIATION_30D: i64 = 0 1666 1667static wc_rb_arena_g: *u8 1668static wc_rb_used_g: i64 1669static wc_rb_hash_g: *i64 1670static wc_rb_off_g: *i64 1671static wc_rb_len_g: *i64 1672static wc_rb_n_g: i64 1673static wc_rb_skip_g: i64 1674static wc_rb_cd_g: i64 1675// The RFC 9309 s2.3.1 access-result state per cached host. THIS ARRAY IS THE FIX: before it existed 1676// the only per-host fact carried out of wc_rb_load was the BODY LENGTH, and a 404 and a 500 both 1677// produce length 0 -- so the two cases the RFC most carefully separates were byte-identical to 1678// every reader downstream. A length cannot express a policy. 1679static wc_rb_state_g: *i64 1680static wc_rb_deny_g: i64 // hosts this run put into DISALLOW_ALL (observability for the new rule) 1681 1682func wc_rb_ua() -> *u8 { return "nishibot" as *u8 } 1683 1684func wc_rb_init() -> i64 { 1685 wc_rb_arena_g = sys_mmap(WC_ROBOTS_ARENA) 1686 wc_rb_hash_g = sys_mmap(WC_ROBOTS_HOSTS * 8) as *i64 1687 wc_rb_off_g = sys_mmap(WC_ROBOTS_HOSTS * 8) as *i64 1688 wc_rb_len_g = sys_mmap(WC_ROBOTS_HOSTS * 8) as *i64 1689 wc_rb_state_g = sys_mmap(WC_ROBOTS_HOSTS * 8) as *i64 1690 wc_rb_used_g = 0 1691 wc_rb_n_g = 0 1692 wc_rb_skip_g = 0 1693 wc_rb_cd_g = 0 1694 wc_rb_deny_g = 0 1695 return 0 1696} 1697 1698// THE CLASSIFIER, EXTRACTED AS A PURE FUNCTION ON PURPOSE (2026-08-25). 1699// It takes only (status, body_bytes) and returns one of the three WC_RB_ST_* states, so the RFC 9309 1700// s2.3.1 rule can be gate-proven with NO network, NO TrustStore and NO cache seeding -- the whole 1701// reason the old bug survived is that the decision was three scattered lines inside a function that 1702// cannot run without a socket, and a rule you cannot call is a rule you cannot test. 1703// status 2xx, n > 0 -> RULES (s2.3.1.1, follow the parseable rules) 1704// status 2xx, n == 0 -> ALLOW_ALL (a successful fetch of a file that states no rules allows 1705// everything -- reached by the RULES path, never by a failure path) 1706// status 2xx, n < 0 -> DISALLOW_ALL. The response line said 200 but the fetch machinery could 1707// not deliver a body (dechunk / inflate / truncation). We did NOT read the 1708// rules, and "the server said the rules exist and we failed to obtain 1709// them" is exactly s2.3.1.4's undefined case. Note this check lives INSIDE 1710// the 2xx arm on purpose: a 4xx with an undecodable error page is still a 1711// definitive "not there", so a blanket n<0 test ahead of the status would 1712// wrongly disallow every 404 whose error page failed to decode. 1713// status 4xx -> ALLOW_ALL (s2.3.1.3 "unavailable": "MAY access any resources") 1714// status 5xx -> DISALLOW_ALL(s2.3.1.4 "unreachable": "MUST assume complete disallow") 1715// no status line at all-> DISALLOW_ALL(s2.3.1.4 names "server OR NETWORK errors") 1716// status 3xx left over -> ALLOW_ALL (s2.3.1.2: past the redirect budget a crawler "MAY assume 1717// that the robots.txt file is unavailable") 1718// anything else -> DISALLOW_ALL, and DELIBERATELY NOT FOLDED INTO THE 5xx BUCKET. The RFC 1719// defines no outcome for a 1xx or a malformed status, and "we could not 1720// determine the rules" is the unreachable case, not the unavailable one. 1721// It gets its OWN outcome label at the call site so the class is countable: 1722// if this ever fires in volume it is a parser bug wearing a policy hat, and 1723// a shared counter would hide that. 1724func wc_rb_classify(status: i64, nbytes: i64) -> i64 { 1725 if status >= WC_HTTP_2XX_LO { if status <= WC_HTTP_2XX_HI { 1726 if nbytes > 0 { return WC_RB_ST_RULES } 1727 if nbytes == 0 { return WC_RB_ST_ALLOW_ALL } 1728 return WC_RB_ST_DISALLOW_ALL 1729 } } 1730 if status >= WC_HTTP_4XX_LO { if status <= WC_HTTP_4XX_HI { return WC_RB_ST_ALLOW_ALL } } 1731 if status >= WC_HTTP_5XX_LO { if status <= WC_HTTP_5XX_HI { return WC_RB_ST_DISALLOW_ALL } } 1732 if status >= WC_HTTP_3XX_LO { if status <= WC_HTTP_3XX_HI { return WC_RB_ST_ALLOW_ALL } } 1733 if status == WC_HTTP_NO_STATUS { return WC_RB_ST_DISALLOW_ALL } 1734 return WC_RB_ST_DISALLOW_ALL 1735} 1736 1737// The Crawl-delay HORIZON rule, as a pure predicate for the same reason the classifier is one: a rule 1738// that can only be exercised by a live robots.txt fetch is a rule with no test. 1739// 0 = adopt the published delay EXACTLY. 1 = decline the host for this run. 1740// There is deliberately no third answer, because the third answer people reach for -- "adopt a smaller 1741// number" -- is the violation this replaced. 1742func wc_rb_cd_policy(cd_s: i64) -> i64 { 1743 if cd_s > WC_ROBOTS_CD_HORIZON_S { return 1 } 1744 return 0 1745} 1746 1747// Did the classifier reach DISALLOW_ALL via the UNDEFINED-status door rather than a defined one? 1748// Split out so the caller can label the outcome row distinctly without re-deriving the class. 1749func wc_rb_status_undefined(status: i64) -> i64 { 1750 if status == WC_HTTP_NO_STATUS { return 0 } 1751 if status >= WC_HTTP_2XX_LO { if status <= WC_HTTP_5XX_HI { return 0 } } 1752 return 1 1753} 1754 1755func wc_rb_find(hh: i64) -> i64 { 1756 var i: i64 = 0 1757 while i < wc_rb_n_g { if wc_rb_hash_g[i] == hh { return i } i = i + 1 } 1758 return 0 - 1 1759} 1760 1761// path (+query) of a url, always at least "/". robots matching is defined over this, not the host. 1762func wc_rb_path(url: *u8, ul: i64, out: *u8, outcap: i64) -> i64 { 1763 var s: i64 = 0 1764 var i: i64 = 0 1765 var d: i64 = 0 1766 while d == 0 { 1767 if i + 2 >= ul { d = 1 } 1768 else { 1769 if url[i] == (58 as u8) { if url[i+1] == (47 as u8) { if url[i+2] == (47 as u8) { s = i + 3; d = 1 } } } 1770 if d == 0 { i = i + 1 } 1771 } 1772 } 1773 var p: i64 = s 1774 var f: i64 = 0 - 1 1775 while p < ul { if url[p] == (47 as u8) { f = p; p = ul } else { p = p + 1 } } 1776 if f < 0 { out[0] = 47 as u8; out[1] = 0 as u8; return 1 } 1777 var o: i64 = 0 1778 var k: i64 = f 1779 while k < ul { if o + 1 < outcap { out[o] = url[k]; o = o + 1 } k = k + 1 } 1780 out[o] = 0 as u8 1781 return o 1782} 1783 1784// Fetch + cache this host's robots.txt for the run. Returns the cache index. The slot is recorded 1785// EVEN ON FAILURE, which is what bounds us to one probe per host per run -- otherwise a host with no 1786// robots.txt would be re-probed on every single url and we would be the impolite one. 1787func wc_rb_load(host: *u8, hl: i64, hh: i64, store: *TrustStore) -> i64 { 1788 if (wc_rb_arena_g as i64) == 0 { return 0 - 1 } 1789 if wc_rb_n_g >= WC_ROBOTS_HOSTS { return 0 - 1 } 1790 let ru: *u8 = sys_mmap(WC_MAGIC_1024) 1791 var o: i64 = 0 1792 o = wc_outcat(ru, o, "https://" as *u8) 1793 var k: i64 = 0 1794 while k < hl { ru[o] = host[k]; o = o + 1; k = k + 1 } 1795 o = wc_outcat(ru, o, "/robots.txt" as *u8) 1796 ru[o] = 0 as u8 1797 1798 let buf: *u8 = sys_mmap(WC_ROBOTS_FCAP) 1799 let st: *i64 = sys_mmap(16) as *i64 1800 st[0] = 0 1801 pace_before(host, hl) 1802 let n: i64 = nx_https_fetch_follow_best(ru, store, buf, WC_ROBOTS_FCAP, WC_ROBOTS_REDIRECTS, st) 1803 pace_after(host, hl, st[0], 0) 1804 1805 let idx: i64 = wc_rb_n_g 1806 wc_rb_hash_g[idx] = hh 1807 wc_rb_off_g[idx] = 0 1808 wc_rb_len_g[idx] = 0 1809 // RFC 9309 s2.3.1: the fetch RESULT decides the host's access policy. This used to be three lines 1810 // collapsing every non-200 into one len-0 slot; it is now one call to a pure, gate-testable rule. 1811 let stt: i64 = wc_rb_classify(st[0], n) 1812 wc_rb_state_g[idx] = stt 1813 wc_rb_n_g = idx + 1 1814 1815 if stt == WC_RB_ST_DISALLOW_ALL { 1816 wc_rb_deny_g = wc_rb_deny_g + 1 1817 // TWO LABELS, ONE POLICY. The undefined-status door is counted separately because it is the 1818 // only one that would indicate a defect in US rather than in the host: if it ever appears in 1819 // volume, that is a response parser bug, and a shared counter with the honest 5xx case would 1820 // hide it behind traffic we expect to see. 1821 if wc_rb_status_undefined(st[0]) == 1 { wc_outcome("robots-undefined-status" as *u8, st[0], n, ru, o) } 1822 else { wc_outcome("robots-unreachable-disallow-all" as *u8, st[0], n, ru, o) } 1823 sys_munmap(buf, WC_ROBOTS_FCAP) 1824 return idx 1825 } 1826 if stt == WC_RB_ST_ALLOW_ALL { 1827 // label kept as "robots-none" DELIBERATELY: nx_crawlmine and the outcome log have counted this 1828 // string since R11, and renaming it would silently reset a denominator that spans months. 1829 wc_outcome("robots-none" as *u8, st[0], n, ru, o) 1830 sys_munmap(buf, WC_ROBOTS_FCAP) 1831 return idx 1832 } 1833 var rl: i64 = n 1834 if rl > WC_ROBOTS_MAX { rl = WC_ROBOTS_MAX } 1835 if wc_rb_used_g + rl > WC_ROBOTS_ARENA { 1836 wc_outcome("robots-arena-full" as *u8, 0, rl, ru, o) 1837 sys_munmap(buf, WC_ROBOTS_FCAP) 1838 return idx 1839 } 1840 var z: i64 = 0 1841 while z < rl { wc_rb_arena_g[wc_rb_used_g + z] = buf[z]; z = z + 1 } 1842 wc_rb_off_g[idx] = wc_rb_used_g 1843 wc_rb_len_g[idx] = rl 1844 wc_rb_used_g = wc_rb_used_g + rl 1845 sys_munmap(buf, WC_ROBOTS_FCAP) 1846 1847 let rp: *u8 = (wc_rb_arena_g as i64 + wc_rb_off_g[idx]) as *u8 1848 var cd: i64 = nx_robots_crawl_delay(rp, rl, wc_rb_ua(), WC_RB_UALEN) 1849 if cd > 0 { 1850 if wc_rb_cd_policy(cd) == 1 { 1851 // BEYOND THE HORIZON WE CRAWL LESS, NEVER MORE. The branch this replaces rewrote cd DOWN 1852 // to 30 s and carried on -- it answered "this host wants to be crawled slowly" by crawling 1853 // it fast. Declining costs one host for one run; clamping costs our standing as a crawler, 1854 // and it is the specific behaviour a CDN verified-bot policy names as a removal breach. 1855 wc_outcome("robots-cd-refused-host" as *u8, cd, WC_ROBOTS_CD_HORIZON_S, ru, o) 1856 wc_rb_state_g[idx] = WC_RB_ST_DISALLOW_ALL 1857 wc_rb_deny_g = wc_rb_deny_g + 1 1858 return idx 1859 } 1860 // ADOPTED EXACTLY AS PUBLISHED -- no clamp. pace_should_defer_tbl leaves the row PENDING until 1861 // the host's own interval has elapsed, so honouring even a very long delay costs coverage on no 1862 // run at all: the frontier row is already persisted and simply returns on a later step. 1863 pace_set_crawl_delay(host, hl, cd * WC_ROBOTS_CD_MS_PER_S) 1864 wc_rb_cd_g = wc_rb_cd_g + 1 1865 wc_outcome("robots-cd" as *u8, cd, rl, ru, o) 1866 } 1867 return idx 1868} 1869 1870// 1 = we may fetch this url, 0 = we may not. 1871// HEADER CORRECTED 2026-08-25. It used to read "EVERY failure path returns 1 (allow) by design" -- 1872// the defect stated as an intention. RFC 9309 s2.3.1.4 makes the SERVER's failure a DISALLOW, so 1873// "every failure allows" was never the design the RFC describes. 1874// The allow-on-failure paths that REMAIN are OUR OWN failures, and they stay ALLOW deliberately: 1875// robots switched off, arena unallocated, per-run cache full. Our inability to run the check is not 1876// a rule about the host -- which is exactly the distinction the old one-line rule collapsed, since a 1877// len-0 slot meant both "the host said nothing" and "the host could not answer". 1878func wc_rb_allowed(host: *u8, hl: i64, hh: i64, url: *u8, ul: i64, store: *TrustStore) -> i64 { 1879 if WC_ROBOTS_ON == 0 { return 1 } 1880 if (wc_rb_arena_g as i64) == 0 { return 1 } 1881 var idx: i64 = wc_rb_find(hh) 1882 if idx < 0 { idx = wc_rb_load(host, hl, hh, store) } 1883 // idx < 0 here means the per-run cache is FULL (WC_ROBOTS_HOSTS hosts already probed). That is 1884 // OUR budget, not the host's answer, and it is the one remaining case where we fetch without 1885 // having asked. Left as ALLOW to match the pre-existing contract rather than silently introducing 1886 // a coverage cliff at host 257, but named here so it is a known bound and not an assumption. 1887 if idx < 0 { return 1 } 1888 // ---- THE RFC 9309 s2.3.1 STATES. Order matters: DISALLOW_ALL is tested FIRST, because a host in 1889 // that state has a len-0 body and would otherwise fall straight into the len<=0 allow below -- 1890 // which is precisely how the 5xx case escaped for as long as it did. 1891 if wc_rb_state_g[idx] == WC_RB_ST_DISALLOW_ALL { return 0 } 1892 if wc_rb_state_g[idx] == WC_RB_ST_ALLOW_ALL { return 1 } 1893 if wc_rb_len_g[idx] <= 0 { return 1 } 1894 let pbuf: *u8 = sys_mmap(WC_MAGIC_2048) 1895 let pl: i64 = wc_rb_path(url, ul, pbuf, WC_MAGIC_2048) 1896 let rp: *u8 = (wc_rb_arena_g as i64 + wc_rb_off_g[idx]) as *u8 1897 let v: i64 = nx_robots_allowed(rp, wc_rb_len_g[idx], wc_rb_ua(), WC_RB_UALEN, pbuf, pl) 1898 sys_munmap(pbuf, WC_MAGIC_2048) 1899 return v 1900} 1901 1902// R17: THE PER-RUN DISCOVERY BUDGET, DERIVED RATHER THAN FIXED. 1903// MEASURED 2026-08-25: discovered_new read exactly 300 in 58 of 60 consecutive runs -- pinned at its 1904// ceiling every time, which means the harvest had MORE SUPPLY than the budget would accept, while the 1905// frontier had drained to 313 against a 2048-row pull window. A starved frontier and a pinned discovery 1906// cap in the same run is the cap being the binding constraint, not the link supply. 1907// Fill toward one pull window; never bank LESS than the old fixed budget, so this can only ever widen 1908// and a full frontier still behaves exactly as before. PURE, so it is gateable without a crawl. 1909func wc_disc_budget(npend_now: i64) -> i64 { 1910 var b: i64 = WC_DISC_CEIL - npend_now 1911 if b < WC_MAXDISC { b = WC_MAXDISC } 1912 if b > WC_DISC_CEIL { b = WC_DISC_CEIL } 1913 return b 1914} 1915// Set once per run from the measured pending count. The getter FLOORS to the old fixed budget when it is 1916// unset, so any path that runs before main computes it keeps the previous behaviour byte for byte -- 1917// an uninitialised budget must never mean "discover nothing". 1918static wc_disc_budget_g: i64 1919func wc_disc_cap() -> i64 { 1920 if wc_disc_budget_g <= 0 { return WC_MAXDISC } 1921 return wc_disc_budget_g 1922} 1923 1924// R16: does a host-capped row get DEFERRED (left pending) or RETIRED (marked done)? 1925// PURE, so it is testable without a store, a socket or a crawl -- the compliance lane learned the same 1926// lesson the hard way: a rule that can only be reached through a fetch is a rule nobody can gate. 1927// While the whole frontier fits inside one pull window, every pending row is pulled again next run, so a 1928// deferral is guaranteed to come back and the page is merely postponed. Past that width a deferred row 1929// can be shadowed by rows ahead of it, which is the blockade that made retirement the right call 1930// originally -- so past that width we still retire. No new constant: the bound IS the pull window. 1931func wc_hostcap_defers(npend_now: i64) -> i64 { 1932 if npend_now <= WC_MAXPEND { return 1 } 1933 return 0 1934} 1935 1936func wc_fetch_child(url: *u8, store: *TrustStore, spath: *u8) -> i64 { 1937 // CRAWL AS OURSELVES (2026-08-25). Until this line the bulk crawler sent a hardcoded Chrome 1938 // User-Agent -- MEASURED, not inferred: this child calls nx_https_fetch_follow_best, which reaches 1939 // nx_http_client_build_request_cookie_xhdr, which emitted the browser identity for every request. 1940 // Two things were wrong with that beyond the forgery. We MATCH the robots.txt group `nishibot` while 1941 // SENDING Chrome, so a site operator can write a rule for us, watch us obey it, and never be able to 1942 // tell it was us. And the identity mechanism added later appended a SECOND User-Agent header, which 1943 // RFC 9110 5.3 forbids and which receivers resolve by taking the FIRST -- so the honest identity was 1944 // never on the wire at all. 1945 // Set HERE, in the fetch child, rather than once in main: this is the exact process that performs a 1946 // crawl fetch, so the identity cannot be left set by one caller and inherited by an unrelated one. 1947 // The browser lane's Chrome UA is untouched and still default -- it is load-bearing for JA3 matching. 1948 hc_set_identity(HC_UA_BOT) 1949 let cap: i64 = WC_MAGIC_8388608 1950 let out: *u8 = sys_mmap(cap) 1951 let status: *i64 = sys_mmap(8) as *i64 1952 let n: i64 = nx_https_fetch_follow_best(url, store, out, cap, 6, status) 1953 let fd: i64 = sys_openat_wr(spath, WC_MODE_RW) 1954 if fd < 0 { sys_exit_group(2) } 1955 let hdr: *i64 = sys_mmap(WC_SPOOL_HDRB) as *i64 1956 hdr[0] = status[0] 1957 var bn: i64 = 0 1958 if n > 0 { bn = n } 1959 hdr[1] = bn 1960 // Redirect depth of THIS child's fetch, read immediately after it returns -- the static is 1961 // last-call-wins and this process performed exactly one fetch, so the read is unambiguous here 1962 // and would be meaningless in the parent. 1963 hdr[2] = nx_https_last_hops() 1964 sys_write(fd, hdr as *u8, WC_SPOOL_HDRB) 1965 if bn > 0 { sys_write(fd, out, bn) } 1966 sys_close(fd) 1967 if bn > 0 { sys_exit_group(0) } 1968 sys_exit_group(1) 1969 return 0 1970} 1971func main(argc: i64, argv: *i64) -> i64 { 1972 if argc < 2 { wc_puts("usage: nx_web_crawl_step <max_pages> [seed-url | seed-file | -] [query words]\n" as *u8); return 1 } 1973 let max_pages: i64 = wc_atoi(argv[1] as *u8) 1974 // HEAVY-I/O BOUND (2026-09-03): a crawl step fans out fetch children and seg-store writes, so it asks the 1975 // estate-wide concurrency bound BEFORE the 4 MB certdata load. DEFER exits 4 (the estate's refused-admission 1976 // code): the crawl clock re-fires the step. UNOBSERVABLE proceeds, announced. 1977 let hio: *i64 = sys_mmap(32) as *i64 1978 let hv: i64 = hio_admit(hio) 1979 hio_announce(1, hv, hio) 1980 if hv == HIO_DEFER { wc_puts("CRAWL-STEP DEFERRED: heavy-I/O bound reached; the clock re-fires\n" as *u8); return 4 } 1981 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, WC_MAGIC_4194304) 1982 if r <= 0 { wc_puts("certdata load failed\n" as *u8); return 2 } 1983 let store: *TrustStore = r as *TrustStore 1984 wc_puts("=== nx_web_crawl_step: frontier-persisted crawl -> dp-web-pub- ===\n" as *u8) 1985 wc_rb_init() // R11: per-run robots.txt cache 1986 1987 let prefix: *u8 = sys_mmap(512) 1988 dss_prefix("web" as *u8, prefix) 1989 // seq1730 ADOPTION (2026-07-31): the digit-parse above was CORRECT and still deadlocked this 1990 // organ. It took max+1 over EVERY id in the manifest -- and 64 of this plane's 65 entries are 1991 // already pointer-shaped poison from before the guard existed -- so it computed 1992 // 140712850411540, which ss_commit's own guard then correctly REFUSED on every crawl step. 1993 // Five consecutive steps logged `segments=0` while still printing CRAWL-STEP GREEN: fetched, 1994 // parsed and discarded. The fix was right, the guard was right, the DATA was poisoned, and the 1995 // leaf never adopted the lib that already knew that. ss_next_segid skips pointer-band ids 1996 // (nx_seg_store.nx:1403) so a corrupted manifest HEALS on the next write, while the poisoned 1997 // segments stay on disk and readable (rule 13, additive-only -- nothing is deleted). 1998 // ★★★MIGRATE THE CHOKEPOINT LIB, NOT THE LEAF. 10+ organs already called ss_next_segid; this 1999 // crawler hand-rolled its own scan, and being the holdout cost it every commit it attempted. 2000 // ★★★A GREEN VERDICT ON A STEP THAT PERSISTED NOTHING IS THE LOUDEST KIND OF SILENT FAILURE. 2001 let segidbox: *i64 = sys_mmap(16) as *i64 2002 segidbox[0] = ss_next_segid(prefix) 2003 if segidbox[0] <= 0 { segidbox[0] = 1 } 2004 // MMAP-OPEN (2026-07-25, seq628 root cause): read-all ss_open pulls the ENTIRE shard into anonymous 2005 // RAM (the sev-9 nx_seg_store keystone) -- at ~2GB across 94 segments that is what made this crawler 2006 // hang after a single fetch on the NAS. nx_pagerank_build and nx_web_shard_compact were both moved to 2007 // ss_open2(prefix,1) earlier this month (measured -45% peak RSS); THIS ORGAN WAS MISSED IN THAT SWEEP. 2008 // Same primitive, same call shape: pages fault in on demand and stay evictable in the page cache. 2009 let h: *i64 = ss_open2(prefix, 1) 2010 let segsbox: *i64 = sys_mmap(16) as *i64; segsbox[0] = 0 2011 let discbox: *i64 = sys_mmap(16) as *i64; discbox[0] = 0 2012 2013 // pending frontier (persisted). A seed bootstraps or extends it. 2014 let urls: *u8 = sys_mmap(WC_MAGIC_2048 * (WC_MAXPEND + 2)) 2015 let canh: *u8 = sys_mmap(64 * WC_MAXCANH) // R4 canonical-seed hosts (filled once the seed file loads) 2016 // R7 ROTATION (2026-08-24): phase 0 skips the persisted cursor; phase 1 wraps to the head when the window is 2017 // not full; the cursor then advances by the rows taken and wraps at the pending population -- every pending 2018 // row is offered in turn instead of the same head of the walk order every run. 2019 let pc0: i64 = wc_pc_cursor(h) 2020 wc_pc_skip_g = pc0 2021 wc_pc_stop_g = 0 2022 var npend: i64 = wc_pending(h, prefix, urls, WC_MAXPEND, canh, 0, 0) 2023 let pcseen: i64 = wc_pc_seen_g 2024 var npend1: i64 = 0 2025 if npend < WC_MAXPEND { if pc0 > 0 { 2026 wc_pc_skip_g = 0 2027 wc_pc_stop_g = pc0 2028 npend1 = wc_pending(h, prefix, (urls as i64 + npend * WC_MAGIC_2048) as *u8, WC_MAXPEND - npend, canh, 0, 0) 2029 npend = npend + npend1 2030 } } 2031 wc_pc_skip_g = 0 2032 wc_pc_stop_g = 0 2033 let pcnext: i64 = wc_pc_next(pc0, npend, pcseen) 2034 wc_puts(" rotation cursor=" as *u8); wc_num(pc0); wc_puts(" -> " as *u8); wc_num(pcnext); wc_puts(" pending_seen=" as *u8); wc_num(pcseen); wc_puts(" wrapped=" as *u8); wc_num(npend1); wc_puts("\n" as *u8) 2035 let w: *i64 = ss_begin() 2036 let npendbox: *i64 = sys_mmap(16) as *i64 2037 npendbox[0] = npend 2038 wc_pc_put(w, prefix, segidbox, segsbox, pcnext) 2039 // R3: seed urls banked for the retire exemption (filled by the seed-injection block below) 2040 let seedstore: *u8 = sys_mmap(WC_MAGIC_2048 * WC_MAXSEEDS) 2041 var nseeds: i64 = 0 2042 // R5 (2026-08-24): argv[2] = "-" means NO seed (so a query can be given as argv[3] without inventing one) 2043 if argc >= 3 { if wc_streq(argv[2] as *u8, "-" as *u8) == 0 { 2044 let sarg: *u8 = argv[2] as *u8 2045 // SEED FILE vs SEED URL (2026-07-26): if argv[2] names a readable non-empty file, EVERY non-empty 2046 // non-# line is a seed url -- that is how a curated canonical-site list gets aimed at the frontier 2047 // in one call. Otherwise argv[2] is a single url (original behaviour, byte-identical path). 2048 let szb: *i64 = sys_mmap(16) as *i64 2049 szb[0] = 0 2050 let fb: *u8 = ss_loadfile(sarg, szb, 0) 2051 var isfile: i64 = 0 2052 if (fb as i64) != 0 { if szb[0] > 0 { isfile = 1 } } 2053 if isfile == 1 { 2054 let line: *u8 = sys_mmap(WC_MAGIC_2048) 2055 var added: i64 = 0 2056 var sdropped: i64 = 0 // R5: seeds beyond WC_MAXSEEDS are REFUSED LOUDLY, never silently 2057 var i9: i64 = 0 2058 var ll: i64 = 0 2059 while i9 <= szb[0] { 2060 var ch: i64 = 10 2061 if i9 < szb[0] { ch = fb[i9] as i64 } 2062 if ch == 10 { if ll > 0 { 2063 line[ll] = 0 as u8 2064 if line[0] != (35 as u8) { 2065 added = added + wc_seed_add(h, w, prefix, segidbox, segsbox, urls, npendbox, line, ll) 2066 // R3: bank EVERY seed url (not just newly-added) for the retire exemption 2067 if nseeds < WC_MAXSEEDS { 2068 let sd9: *u8 = (seedstore as i64 + nseeds * WC_MAGIC_2048) as *u8 2069 var sc9: i64 = 0 2070 while sc9 < ll { sd9[sc9] = line[sc9]; sc9 = sc9 + 1 } 2071 sd9[ll] = 0 as u8 2072 nseeds = nseeds + 1 2073 } else { sdropped = sdropped + 1 } 2074 } 2075 ll = 0 2076 } } else { 2077 if ch != 13 { if ll < WC_MAGIC_2048 - 2 { line[ll] = ch as u8; ll = ll + 1 } } 2078 } 2079 i9 = i9 + 1 2080 } 2081 wc_puts(" seed-file " as *u8); wc_puts(sarg); wc_puts(" -> new pending seeds=" as *u8); wc_num(added); wc_puts("\n" as *u8) 2082 if sdropped > 0 { wc_puts(" [seed-cap] REFUSED " as *u8); wc_num(sdropped); wc_puts(" seed rows beyond WC_MAXSEEDS -- raise the const or trim the file; a dropped seed silently loses its retire exemption AND its canonical-lane membership\n" as *u8) } 2083 } else { 2084 wc_seed_add(h, w, prefix, segidbox, segsbox, urls, npendbox, sarg, wc_len(sarg)) 2085 let sl9: i64 = wc_len(sarg) 2086 if sl9 < WC_MAGIC_2048 - 1 { 2087 var sc8: i64 = 0 2088 while sc8 < sl9 { seedstore[sc8] = sarg[sc8]; sc8 = sc8 + 1 } 2089 seedstore[sl9] = 0 as u8 2090 nseeds = 1 2091 } 2092 } 2093 } } 2094 npend = npendbox[0] 2095 wc_puts(" frontier pending=" as *u8); wc_num(npend); wc_puts("\n" as *u8) 2096 // R17: set the per-run discovery budget from the frontier we just measured, and PRINT it. A budget 2097 // that is derived but invisible is indistinguishable from the old fixed one in the log, and this is 2098 // the number that decides whether the frontier grows -- so it goes on the record every run. 2099 wc_disc_budget_g = wc_disc_budget(npend) 2100 wc_puts(" discovery budget=" as *u8); wc_num(wc_disc_budget_g) 2101 wc_puts(" (floor " as *u8); wc_num(WC_MAXDISC) 2102 wc_puts(", ceiling " as *u8); wc_num(WC_DISC_CEIL); wc_puts(" = one pull window)\n" as *u8) 2103 // R5 QUERY-SCOPED CRAWL (2026-08-24): argv[3] = query words. The general window is ranked by query hits in the 2104 // url (then shallow-first), and the run STOPS EARLY once the sufficiency judge clears its bar. Without a query 2105 // the crawl is the breadth crawl it was, ranked shallow-first only. 2106 if argc >= 4 { 2107 let qarg: *u8 = argv[3] as *u8 2108 let qn9: i64 = wc_query_set(qarg) 2109 wc_cs_g = cs_new(qarg, wc_len(qarg)) 2110 wc_puts(" query-scoped crawl terms=" as *u8); wc_num(qn9); wc_puts(" confidence_bar=" as *u8); wc_num(wc_cs_g[CS_S_CONF_STOP]); wc_puts(" saturation_bar=" as *u8); wc_num(wc_cs_g[CS_S_SAT_STOP]); wc_puts("\n" as *u8) 2111 } 2112 wc_rank_window(urls, npend) 2113 // ---- R4 CANONICAL LANE: reserve the HEAD of the work list for canonical-seed hosts ----------- 2114 // Runs here because it needs the seed file, which is parsed above; the first wc_pending call 2115 // (canonly=0) has already filled `urls` with the ordinary window. This second pass walks the SAME 2116 // frontier host-filtered, then rebuilds the work list as [canonical rows] ++ [everything else, 2117 // original order, byte-deduped]. ncan==0 (no seed file) -> the whole block is skipped and the run 2118 // is byte-identical to pre-R4. 2119 var k4ncan: i64 = wc_canon_hosts(seedstore, nseeds, canh) 2120 // R8: publish the table to the shared URL policy (see wc_urlok). AFTER the seed parse, BEFORE 2121 // any harvest -- so this run's admissions are already trust-aware. 2122 wc_canh_g = canh 2123 wc_ncan_g = k4ncan 2124 wc_qscratch_g = sys_mmap(64) 2125 // R10: load the URL-shape policy (tracking-param denylist + budgets) BEFORE any harvest, so this 2126 // run's admissions are judged by shape rather than by host membership. 2127 let npol: i64 = wc_load_urlpolicy() 2128 wc_puts(" url-policy deny_params=" as *u8); wc_num(npol) 2129 wc_puts(" max_params=" as *u8); wc_num(wc_qmaxp_g) 2130 wc_puts(" max_qlen=" as *u8); wc_num(wc_qmaxlen_g) 2131 // R11 PATH AXIS ANNOUNCE: a feature that prints nothing is indistinguishable from one that was 2132 // never compiled in. 5 = the built-in 2026-08-04 defaults are firing (plane holds no path rules); 2133 // any other count means the plane is AUTHORITATIVE and policy came from DATA. 2134 wc_puts(" path_rules=" as *u8); wc_num(olh_load_pathpolicy()) 2135 if npol == 0 { wc_puts(" (plane ABSENT -- built-in defaults, query URLs still admitted by shape)" as *u8) } 2136 // R15 DOMAIN-MAP ADOPTION. Placed HERE deliberately: AFTER wc_canh_g/wc_ncan_g and AFTER 2137 // wc_load_urlpolicy(), because wc_dm_seed_host filters every candidate through wc_urlok and that 2138 // filter reads both. Seeding before the policy loaded would admit urls the ordinary crawl refuses 2139 // -- the map would become a bypass around our own url filter rather than a feeder into it. 2140 let dmfound: *i64 = sys_mmap(16) as *i64 2141 dmfound[0] = 0 2142 var dmseed: i64 = 0 2143 var dmi: i64 = 0 2144 while dmi < k4ncan { 2145 let dmh: *u8 = (canh as i64 + dmi * WC_DM_HOSTSLOT) as *u8 2146 dmseed = dmseed + wc_dm_seed_host(h, w, prefix, segidbox, segsbox, urls, npendbox, dmh, dmfound) 2147 dmi = dmi + 1 2148 } 2149 wc_puts("\n domain-map adoption: maps_found=" as *u8); wc_num(dmfound[0]) 2150 wc_puts(" of canonical_hosts=" as *u8); wc_num(k4ncan) 2151 wc_puts(" new_pending_seeds=" as *u8); wc_num(dmseed) 2152 // BOTH NUMBERS ARE PRINTED BECAUSE ONE CANNOT BE READ WITHOUT THE OTHER. maps_found=0 means the 2153 // reconnaissance has never been run for these hosts (remedy: run nx_domain_map). maps_found>0 with 2154 // new_pending_seeds=0 means every discovered url was ALREADY in the frontier (dedupe working as 2155 // designed -- not a failure). A single counter would render those two identical, and they have 2156 // opposite remedies. 2157 if dmfound[0] == 0 { wc_puts(" (no map for any canonical host -- run nx_domain_map <domain> to feed this)" as *u8) } 2158 wc_puts("\n" as *u8) 2159 wc_puts("\n" as *u8) 2160 wc_outopen() 2161 if wc_outfd_g <= 0 { wc_puts(" [outcome-log] UNAVAILABLE -- this run's drops will not be mineable\n" as *u8) } 2162 if k4ncan > 0 { 2163 let k4urls: *u8 = sys_mmap(WC_MAGIC_2048 * (WC_CANON_RESERVE + WC_SEED_REFRESH + 2)) 2164 // The reserve is a FLOOR DERIVED FROM THE RUN'S OWN BUDGET, not a fixed ceiling: half of 2165 // max_pages goes to canonical hosts and half stays with general discovery, so the lane can 2166 // never become a takeover of the crawl (rule 25 -- nothing is stripped). WC_CANON_RESERVE is 2167 // only the upper CAP on that half. MEASURED 2026-08-05 on the first live run: reserved=384 2168 // SATURATED, i.e. at least 384 canonical rows were already sitting pending and starved. 2169 var k4res: i64 = max_pages / 2 2170 if k4res > WC_CANON_RESERVE { k4res = WC_CANON_RESERVE } 2171 if k4res < 1 { k4res = 1 } 2172 let k4n0: i64 = wc_pending(h, prefix, k4urls, k4res, canh, k4ncan, 1) 2173 // R5: re-queue stale seed hubs INTO the canonical block. The R4c round-robin pass below 2174 // interleaves them by host, so a refreshed hub is fetched early without displacing any 2175 // other canonical host's first row. 2176 let k4box5: *i64 = sys_mmap(16) as *i64 2177 k4box5[0] = k4n0 2178 k4box5[1] = WC_CANON_RESERVE + WC_SEED_REFRESH 2179 let k4rf: i64 = wc_seed_refresh(h, w, prefix, segidbox, segsbox, seedstore, nseeds, k4urls, k4box5) 2180 let k4n: i64 = k4box5[0] 2181 if k4rf > 0 { wc_puts(" seed-refresh requeued=" as *u8); wc_num(k4rf); wc_puts("\n" as *u8) } 2182 if k4n > 0 { 2183 let k4merged: *u8 = sys_mmap(WC_MAGIC_2048 * (WC_MAXPEND + 2)) 2184 // ---- R4c ROUND-ROBIN THE CANONICAL BLOCK BY HOST ---------------------------------- 2185 // MEASURED 2026-08-05, and it is the R4 bug repeating one level down: with the block in 2186 // stable walk order, whichever canonical host happens to hold the most pending rows eats 2187 // the whole reserve. Observed old.reddit.com taking 18 of 64 ingests in a single run 2188 // while who.int and en.wikipedia.org -- the hosts the reserve EXISTS for -- waited. 2189 // ★★A FAIRNESS FIX APPLIED TO A SET, NOT TO THE MEMBERS WITHIN IT, JUST MOVES THE 2190 // STARVATION INSIDE THE SET. rank = how many EARLIER canonical rows share this host; 2191 // emitting all rank-0 rows, then all rank-1, gives EVERY canonical host its first row 2192 // before ANY host takes a second. Host identity via the same djb2 the dead-host list and 2193 // the host cap use, so "same host" means one thing everywhere in this organ. 2194 let k4hh: *i64 = sys_mmap(8 * (WC_CANON_RESERVE + WC_SEED_REFRESH + 4)) as *i64 2195 let k4rk: *i64 = sys_mmap(8 * (WC_CANON_RESERVE + WC_SEED_REFRESH + 4)) as *i64 2196 let k4hb: *u8 = sys_mmap(64) 2197 var k4z: i64 = 0 2198 while k4z < k4n { 2199 let k4rp: *u8 = (k4urls as i64 + k4z * WC_MAGIC_2048) as *u8 2200 let k4hl: i64 = wc_host(k4rp, k4hb) 2201 k4hh[k4z] = wc_hosthash(k4hb, k4hl) 2202 var k4r: i64 = 0 2203 var k4y: i64 = 0 2204 while k4y < k4z { if k4hh[k4y] == k4hh[k4z] { k4r = k4r + 1 } k4y = k4y + 1 } 2205 k4rk[k4z] = k4r 2206 k4z = k4z + 1 2207 } 2208 var k4mn: i64 = 0 2209 var k4p: i64 = 0 2210 while k4p < k4n { 2211 var k4q2: i64 = 0 2212 while k4q2 < k4n { 2213 if k4rk[k4q2] == k4p { 2214 let k4s: *u8 = (k4urls as i64 + k4q2 * WC_MAGIC_2048) as *u8 2215 let k4d: *u8 = (k4merged as i64 + k4mn * WC_MAGIC_2048) as *u8 2216 var k4c: i64 = 0 2217 while k4s[k4c] != (0 as u8) { k4d[k4c] = k4s[k4c]; k4c = k4c + 1 } 2218 k4d[k4c] = 0 as u8 2219 k4mn = k4mn + 1 2220 } 2221 k4q2 = k4q2 + 1 2222 } 2223 k4p = k4p + 1 2224 } 2225 var k4oi: i64 = 0 2226 while k4oi < npend { 2227 let k4op: *u8 = (urls as i64 + k4oi * WC_MAGIC_2048) as *u8 2228 var k4dup: i64 = 0 2229 var k4q: i64 = 0 2230 while k4q < k4n { 2231 if wc_streq(k4op, (k4urls as i64 + k4q * WC_MAGIC_2048) as *u8) == 1 { k4dup = 1; k4q = k4n } else { k4q = k4q + 1 } 2232 } 2233 if k4dup == 0 { if k4mn < WC_MAXPEND { 2234 let k4dp: *u8 = (k4merged as i64 + k4mn * WC_MAGIC_2048) as *u8 2235 var k4c2: i64 = 0 2236 while k4op[k4c2] != (0 as u8) { k4dp[k4c2] = k4op[k4c2]; k4c2 = k4c2 + 1 } 2237 k4dp[k4c2] = 0 as u8 2238 k4mn = k4mn + 1 2239 } } 2240 k4oi = k4oi + 1 2241 } 2242 var k4cb: i64 = 0 2243 while k4cb < k4mn { 2244 let k4sp: *u8 = (k4merged as i64 + k4cb * WC_MAGIC_2048) as *u8 2245 let k4dp2: *u8 = (urls as i64 + k4cb * WC_MAGIC_2048) as *u8 2246 var k4c3: i64 = 0 2247 while k4sp[k4c3] != (0 as u8) { k4dp2[k4c3] = k4sp[k4c3]; k4c3 = k4c3 + 1 } 2248 k4dp2[k4c3] = 0 as u8 2249 k4cb = k4cb + 1 2250 } 2251 npend = k4mn 2252 npendbox[0] = k4mn 2253 } 2254 wc_puts(" canon-lane hosts=" as *u8); wc_num(k4ncan); wc_puts(" reserved=" as *u8); wc_num(k4n); wc_puts(" worklist=" as *u8); wc_num(npend); wc_puts("\n" as *u8) 2255 } 2256 if npend == 0 { 2257 wc_puts("CRAWL-STEP GREEN (frontier empty -- give a seed url to bootstrap)\n" as *u8) 2258 return 0 2259 } 2260 2261 let cap: i64 = WC_MAGIC_8388608 2262 let out: *u8 = sys_mmap(cap) 2263 let hydrbuf: *u8 = sys_mmap(cap) // R-JS-HEADLESS: serialized hydrated DOM for inline-script pages 2264 let tcap: i64 = WC_MAGIC_1048576 2265 let text: *u8 = sys_mmap(tcap) 2266 let status: *i64 = sys_mmap(8) as *i64 2267 let keptfp: *i64 = sys_mmap(128 * 8) as *i64 2268 let key: *u8 = sys_mmap(64) 2269 let ukey: *u8 = sys_mmap(64) 2270 let pbox: *i64 = sys_mmap(16) as *i64 2271 let lbox: *i64 = sys_mmap(16) as *i64 2272 let base: *u8 = sys_mmap(WC_MAGIC_2048) 2273 // Sized from the CEILING, not the floor: the budget is now dynamic, so a buffer dimensioned off the 2274 // floor would be overrun the moment the frontier had headroom. 32 * (2048 + 8) is ~66 KB, once. 2275 let runseen: *u8 = sys_mmap(32 * (WC_DISC_CEIL + 8)) 2276 let nrs: *i64 = sys_mmap(16) as *i64; nrs[0] = 0 2277 2278 // feed-seed scratch (allocated ONCE, reused per page -- no mmap in the crawl loop) 2279 let feedhref: *u8 = sys_mmap(WC_MAGIC_4096) 2280 let feedabs: *u8 = sys_mmap(WC_MAGIC_4096) 2281 let fhost: *u8 = sys_mmap(512) 2282 let fxml: *u8 = sys_mmap(cap) 2283 let fstat: *i64 = sys_mmap(16) as *i64 2284 let fit: *u8 = sys_mmap(WC_MAGIC_2048) 2285 let fil: *u8 = sys_mmap(WC_MAGIC_2048) 2286 let fisum: *u8 = sys_mmap(WC_MAGIC_8192) 2287 let ffk: *u8 = sys_mmap(64) 2288 // per-host INDEX-count map for THIS run (host_hash -> count), linear-scan (few hosts/run). Diversity cap. 2289 let hc_hash: *i64 = sys_mmap(512*8) as *i64 2290 let hc_cnt: *i64 = sys_mmap(512*8) as *i64 2291 var hc_n: i64 = 0 2292 // ---- seq628 batched parallel fetch; single-writer ingest stays in the parent ---- 2293 let dh: *i64 = sys_mmap(WC_DEADHOSTS * 8) as *i64 2294 let ndhbox: *i64 = sys_mmap(16) as *i64 2295 ndhbox[0] = 0 2296 let bidx: *i64 = sys_mmap(WC_POOL_N * 8) as *i64 2297 let bpid: *i64 = sys_mmap(WC_POOL_N * 8) as *i64 2298 let bhh: *i64 = sys_mmap(WC_POOL_N * 8) as *i64 2299 let brc: *i64 = sys_mmap(WC_POOL_N * 8) as *i64 2300 let bstart: *i64 = sys_mmap(WC_POOL_N * 8) as *i64 2301 let spaths: *u8 = sys_mmap(WC_SPOOLPATH_B * WC_POOL_N) 2302 var sp0: i64 = 0 2303 while sp0 < WC_POOL_N { wc_spool_path(sp0, (spaths as i64 + sp0 * WC_SPOOLPATH_B) as *u8); sp0 = sp0 + 1 } 2304 let phost: *u8 = sys_mmap(512) 2305 let stbox: *i64 = sys_mmap(16) as *i64 2306 let shdr: *i64 = sys_mmap(WC_SPOOL_HDRB) as *i64 2307 var fetched: i64 = 0 2308 var ingested: i64 = 0 2309 var present: i64 = 0 2310 var skipdead: i64 = 0 2311 var retired: i64 = 0 // R3: dead-host rows retired (P->D) this run 2312 var deferred: i64 = 0 // R13: rows LEFT PENDING because their host is in throttle backoff (2026-08-25) 2313 // R13: rows retired P->D by the SELECTION-TIME host cap. This outflow existed since R12 and was the 2314 // ONLY one of the four with no counter, so frontier_remaining~= below overstated the frontier by 2315 // ~176 rows EVERY RUN -- and the resulting flat-at-~1900 reading was the GAUGE, not the frontier. 2316 // A COUNT THAT IS NEVER TAKEN CANNOT BE RECONCILED, AND AN UNRECONCILED PARTITION IS A LEAK. 2317 var hostcapped: i64 = 0 2318 // R16: rows the host cap SKIPPED THIS ROUND but LEFT PENDING (they index on a later run when the cap 2319 // resets). Counted separately from hostcapped because they are the opposite outcome: one is a page 2320 // deferred, the other is a page discarded, and a single counter would hide which one we did. 2321 var hostcap_deferred: i64 = 0 2322 // R13b: ONE pacing-table buffer for the WHOLE RUN, refilled once per BATCH (see pace_should_defer_tbl). 2323 // The per-row 0-arg form would mmap and read 160 KB per candidate row -- WC_MAXPEND = 2048 rows, i.e. 2324 // ~335 MB of leaked anon VMA and ~335 MB of read I/O per run, in the parent. A CHANGE THAT HAMMERS THE 2325 // BOX IS A BUG EVEN WHEN ITS FEATURE WORKS. 2326 let pacetbl: *u8 = sys_mmap(PACE_TBL_BYTES) 2327 var qi: i64 = 0 2328 while qi < npend { 2329 if fetched >= max_pages { qi = npend } else { 2330 // BATCH SELECT: up to WC_POOL_N urls, one DISTINCT host per slot. A url whose host is already 2331 // in this batch CLOSES the batch (it leads the next one) so same-host depth is never starved; 2332 // a url whose host went DEAD this run is skipped (stays PENDING for a future run). 2333 var nb: i64 = 0 2334 var scan: i64 = 1 2335 // Refill the pacing snapshot for THIS batch. pace_after rewrote the table during the reap of 2336 // the previous batch, so this is exactly as fresh as a per-row load would have been. 2337 pace_load_tbl_into(pacetbl) 2338 if wc_cs_stop_g == 1 { qi = npend } // R5: the sufficiency judge cleared its bar -- no new batch 2339 while scan == 1 { 2340 if qi >= npend { scan = 0 } else { 2341 if nb >= WC_POOL_N { scan = 0 } else { 2342 if fetched + nb >= max_pages { scan = 0 } else { 2343 let us1: *u8 = (urls as i64 + qi * WC_MAGIC_2048) as *u8 2344 let pl1: i64 = wc_host(us1, phost) 2345 let hh9: i64 = wc_hosthash(phost, pl1) 2346 if wc_deadhost_has(dh, ndhbox[0], hh9) == 1 { 2347 skipdead = skipdead + 1 2348 // R3: the host's PERSISTED streak has hit the bar -> retire this row (P->D) so 2349 // the rot stops blockading the pull window. Seeds exempt; rediscovery re-adds. 2350 // R9 (2026-08-05): a CURATED HOST IS NEVER RETIRED. wc_is_seed exempts only 2351 // the exact seed URLs, so a curated host's DISCOVERED rows were still being 2352 // retired after 3 dead runs -- and we cannot tell "the host is down" from 2353 // "we cannot speak its TLS". MEASURED that day: our client advertises 0x1302 2354 // (nginx's default) and cannot decrypt it, so such hosts fail EVERY run, 2355 // hit the streak bar, and are retired PERMANENTLY (wc_seed_add refuses any 2356 // existing row). ★A TRANSPORT-LAYER CAPABILITY GAP MUST NOT BE LAUNDERED 2357 // INTO PERMANENT COVERAGE LOSS -- RETIRE ON EVIDENCE THE HOST IS DEAD, 2358 // NEVER ON EVIDENCE OF OUR OWN INABILITY. We curated these hosts, so their 2359 // absence is OUR bug to fix; keeping the row P means they return for free 2360 // the moment the client learns the cipher. Uncurated rot still retires. 2361 var r9keep: i64 = 0 2362 if wc_is_seed(seedstore, nseeds, us1, wc_len(us1)) == 1 { r9keep = 1 } 2363 if wc_ncan_g > 0 { if (wc_qscratch_g as i64) != 0 { 2364 if wc_host_is_canon(wc_canh_g, wc_ncan_g, us1, wc_qscratch_g) == 1 { r9keep = 1 } 2365 } } 2366 if r9keep == 0 { 2367 retired = retired + wc_retire_if_rotten(h, w, prefix, segidbox, segsbox, us1, wc_len(us1), hh9) 2368 } 2369 qi = qi + 1 2370 } else { 2371 // R11 ROBOTS: ask the host BEFORE spending a fetch on it. Checked HERE, at 2372 // batch SELECTION, and deliberately not after the fork: a url skipped after 2373 // launch has no spool file, which the reaper reads as a fetch failure, which 2374 // bumps the dead-host streak, which retires the host permanently. Filtering 2375 // before selection means our own policy can never be mistaken for the host 2376 // being dead. 2377 var rbok: i64 = 1 2378 if WC_ROBOTS_ON == 1 { rbok = wc_rb_allowed(phost, pl1, hh9, us1, wc_len(us1), store) } 2379 if rbok == 0 { 2380 wc_outcome("robots" as *u8, 0, 0, us1, wc_len(us1)) 2381 wc_frmark(w, prefix, segidbox, segsbox, us1, wc_len(us1), 0) 2382 wc_rb_skip_g = wc_rb_skip_g + 1 2383 qi = qi + 1 2384 } else { 2385 // R12 HOSTCAP AT SELECTION (2026-08-12): the per-run host diversity cap used to be 2386 // enforced only AFTER fetch+parse+dedup (the index gate below), so every page of an 2387 // already-capped host still paid a FULL fetch. MEASURED (nx_crawlmine, 6.16 days): 2388 // hostcap = 16,001 of 50,427 rows = 32% of the whole fetch budget spent on pages that 2389 // could never index this run (13,992/18,115 en.wikipedia rows). Checked HERE like 2390 // robots (R11): our own policy must never cost a fetch. SAME TERMINAL SEMANTICS as the 2391 // post-fetch gate (frmark done + a hostcap outcome row, so the miner keeps its 2392 // denominator and the frontier never wedges) -- but with http=0, which is the 2393 // PRE-DECLARED accept signature: post-fix, hostcap rows with http=200 (paid) must 2394 // collapse toward zero and the ok/day rate must rise; referee = nx_crawlmine over the 2395 // post-fix window. Deferral (keep the row pending so the page indexes on a FUTURE run 2396 // when the cap resets) is the v2 once the pull window can rotate capped rows without 2397 // blockading -- today's behavior discards these pages anyway, so this is strictly 2398 // better: same index outcome, fetch refunded. The post-fetch gate below STAYS: it 2399 // catches hosts that fill their cap while a batch is already in flight. 2400 var hcok: i64 = 1 2401 // R13 THROTTLE DEFERRAL (2026-08-25). A host in throttle backoff is skipped HERE and 2402 // its row LEFT PENDING, instead of being selected and then slept on. 2403 // WHY AT SELECTION AND NOT IN THE LAUNCH LOOP: every slot that enters the batch is 2404 // marked DONE by the process loop ("done FIRST -- a dead url must not wedge the 2405 // frontier"), so a url skipped after selection would be silently retired UNFETCHED. 2406 // Deferral has to happen before bidx[] is written, or it becomes data loss. 2407 // WHAT IT FIXES, MEASURED 2026-08-25: pace_before sleeps IN THE PARENT inside the 2408 // launch loop, so ONE throttled host stalls the whole WC_POOL_N-way pool -- 2409 // books.google.com was fetched at ~62 s intervals 41 consecutive times while crawl 2410 // throughput fell 280.7 -> 72.3 docs/hour. 2411 // AND IT SERVES THE BACKOFF FOR THE FIRST TIME: PACE_WAIT_CAP_MS caps the WAIT at 2412 // 60 s, so a 300 s backoff was never actually honoured -- the host was re-hit at 60 s, 2413 // returned another 429, and consec could never decay (pace_decay needs a success). 2414 // Deferring is the only thing that lets a backoff elapse. 2415 // NOT A THRESHOLD CHANGE: pace_should_defer keys on consec_throttle, so ordinary 2416 // politeness gaps are still slept through and only ACTIVE REFUSALS are deferred. 2417 if pace_should_defer_tbl(pacetbl, phost, pl1) == 1 { 2418 wc_outcome("pacedefer" as *u8, 0, 0, us1, wc_len(us1)) 2419 wc_puts(" [pace] DEFER host in throttle backoff, row stays PENDING " as *u8); wc_puts(us1); wc_puts("\n" as *u8) 2420 deferred = deferred + 1 2421 hcok = 0 2422 qi = qi + 1 2423 } 2424 var hci9: i64 = 0 - 1 2425 var hj9: i64 = 0 2426 while hj9 < hc_n { if hc_hash[hj9] == hh9 { hci9 = hj9; hj9 = hc_n } else { hj9 = hj9 + 1 } } 2427 // GUARDED: a row already DEFERRED above must not also be host-capped. Without this 2428 // it would advance qi twice AND wc_frmark the row DONE -- turning a deferral into a 2429 // silent retirement of a url that was never fetched. 2430 if hcok == 1 { if hci9 >= 0 { 2431 var hcap9: i64 = WC_HOSTCAP 2432 if wc_host_str_is_canon(canh, k4ncan, phost, pl1) == 1 { hcap9 = WC_HOSTCAP_CANON } 2433 if hc_cnt[hci9] >= hcap9 { 2434 // R16 DEFER-INSTEAD-OF-RETIRE (2026-08-25). The host cap is a PER-ROUND 2435 // diversity measure, yet it marked the row DONE -- permanently. MEASURED on 2436 // the live beat the day the counter for it first existed: hostcap_retired=227 2437 // in ONE run, i.e. 227 pages discarded and never indexed, while the frontier 2438 // had drained to 313. The cap exists to stop one host dominating a PAGE, not 2439 // to rule that a page is worthless. 2440 // THE PRECONDITION THE ORIGINAL COMMENT DEMANDED IS NOW TESTED, NOT ASSUMED. 2441 // It said deferral was the v2 "once the pull window can rotate capped rows 2442 // without blockading". That is exactly decidable: while the whole frontier 2443 // fits inside one pull window, every pending row IS pulled next run, so a 2444 // deferred row cannot be shadowed. Past that it could be, which is the 2445 // blockade the original reasoning cited -- so past that we still retire. 2446 // Derived from WC_MAXPEND; it self-corrects as the frontier grows, and no 2447 // new constant is introduced to be guessed wrong. 2448 let hc_defer: i64 = wc_hostcap_defers(npend) 2449 if hc_defer == 1 { 2450 wc_outcome("hostcap-defer" as *u8, 0, hc_cnt[hci9], us1, wc_len(us1)) 2451 hostcap_deferred = hostcap_deferred + 1 2452 } else { 2453 wc_outcome("hostcap" as *u8, 0, hc_cnt[hci9], us1, wc_len(us1)) 2454 wc_frmark(w, prefix, segidbox, segsbox, us1, wc_len(us1), 2) 2455 hostcapped = hostcapped + 1 2456 } 2457 wc_puts(" [host-cap] skip FETCH (cap hit pre-selection) " as *u8); wc_puts(us1); wc_puts("\n" as *u8) 2458 hcok = 0 2459 qi = qi + 1 2460 } 2461 } } 2462 if hcok == 1 { 2463 var infl: i64 = 0 2464 var b9: i64 = 0 2465 while b9 < nb { if bhh[b9] == hh9 { infl = 1; b9 = nb } else { b9 = b9 + 1 } } 2466 if infl == 1 { scan = 0 } else { 2467 bidx[nb] = qi 2468 bhh[nb] = hh9 2469 nb = nb + 1 2470 qi = qi + 1 2471 } 2472 } 2473 } 2474 } 2475 } } } 2476 } 2477 if nb == 0 { qi = npend } else { 2478 // LAUNCH: pace in the PARENT (the persisted pacing state keeps one writer), then fork the fetch. 2479 var b: i64 = 0 2480 while b < nb { 2481 let us2: *u8 = (urls as i64 + bidx[b] * WC_MAGIC_2048) as *u8 2482 let pl2: i64 = wc_host(us2, phost) 2483 pace_before(phost, pl2) 2484 let spb: *u8 = (spaths as i64 + b * WC_SPOOLPATH_B) as *u8 2485 sys_unlinkat(spb) 2486 let pid: i64 = sys_fork() 2487 if pid == 0 { wc_fetch_child(us2, store, spb) } 2488 if pid > 0 { bpid[b] = pid } else { bpid[b] = 0; brc[b] = 0 - 3 } 2489 if pid > 0 { brc[b] = 0 - 2 } 2490 bstart[b] = sys_clock_now_us() 2491 b = b + 1 2492 } 2493 // REAP with the wall-clock budget: WNOHANG poll; past budget -> SIGKILL + blocking reap. 2494 var live: i64 = 0 2495 var bl: i64 = 0 2496 while bl < nb { if bpid[bl] > 0 { live = live + 1 } bl = bl + 1 } 2497 while live > 0 { 2498 var b2: i64 = 0 2499 while b2 < nb { 2500 if bpid[b2] > 0 { 2501 let rp: i64 = sys_wait4(bpid[b2], stbox, WC_WNOHANG) 2502 if rp == bpid[b2] { 2503 brc[b2] = wait_exit_code(stbox[0]) 2504 bpid[b2] = 0 2505 live = live - 1 2506 } else { 2507 if sys_clock_now_us() - bstart[b2] > WC_FETCH_BUDGET_MS * WC_US_PER_MS { 2508 nx_kill(bpid[b2], WC_SIGKILL) 2509 sys_wait4(bpid[b2], stbox, 0) 2510 brc[b2] = 0 - 1 2511 bpid[b2] = 0 2512 live = live - 1 2513 } 2514 } 2515 } 2516 b2 = b2 + 1 2517 } 2518 if live > 0 { sys_sleep_ms(WC_POLL_MS) } 2519 } 2520 // PROCESS sequentially (single seg-store writer): spool -> the unchanged ingest/harvest path. 2521 var b3: i64 = 0 2522 while b3 < nb { 2523 let uslot: *u8 = (urls as i64 + bidx[b3] * WC_MAGIC_2048) as *u8 2524 let ul: i64 = wc_len(uslot) 2525 let phl: i64 = wc_host(uslot, phost) 2526 fetched = fetched + 1 2527 // done FIRST (even on failure -- a dead url must not wedge the frontier) 2528 wc_frmark(w, prefix, segidbox, segsbox, uslot, ul, 2) 2529 var n: i64 = 0 2530 // -1 = UNSPOOLED: no header read, so no depth reported. Distinct from 0, which means the 2531 // child fetched with no redirect. One value for both would report "never measured" as 2532 // "measured zero" -- the exact defect this signal exists to expose. 2533 var fhops: i64 = 0 - 1 2534 status[0] = 0 2535 if brc[b3] == 0 { 2536 let spb3: *u8 = (spaths as i64 + b3 * WC_SPOOLPATH_B) as *u8 2537 let fd3: i64 = sys_openat_rd(spb3) 2538 if fd3 >= 0 { 2539 if sys_read(fd3, shdr as *u8, WC_SPOOL_HDRB) == WC_SPOOL_HDRB { 2540 status[0] = shdr[0] 2541 fhops = shdr[2] 2542 var want: i64 = shdr[1] 2543 if want > cap { want = cap } 2544 var got: i64 = 0 2545 var rr: i64 = 1 2546 while rr == 1 { 2547 if got >= want { rr = 0 } else { 2548 let g: i64 = sys_read(fd3, (out as i64 + got) as *u8, want - got) 2549 if g <= 0 { rr = 0 } else { got = got + g } 2550 } 2551 } 2552 n = got 2553 } 2554 sys_close(fd3) 2555 sys_unlinkat(spb3) 2556 } 2557 } 2558 pace_after(phost, phl, status[0], 0) 2559 // R3 host-health: any completed response heals the streak; a failure bumps it ONCE per run 2560 // (wc_deadhost_add returns 1 only on the host's FIRST death this run). 2561 let hhx: i64 = wc_hosthash(phost, phl) 2562 if brc[b3] == 0 { if status[0] > 0 { 2563 if wc_hd_streak(h, hhx) > 0 { wc_hd_put(w, prefix, segidbox, segsbox, hhx, 0) } 2564 } } 2565 if brc[b3] != 0 { 2566 // timeout (-1), fetch-fail (1/2) or fork-fail (-3): defer the host for the rest of this run 2567 if wc_deadhost_add(dh, ndhbox, hhx) == 1 { 2568 wc_hd_put(w, prefix, segidbox, segsbox, hhx, wc_hd_streak(h, hhx) + 1) 2569 } 2570 if brc[b3] == (0 - 1) { 2571 wc_outcome("timeout" as *u8, status[0], n, uslot, ul) 2572 wc_puts(" [fetch-timeout] " as *u8); wc_puts(uslot); wc_puts(" (budget " as *u8); wc_num(WC_FETCH_BUDGET_MS); wc_puts("ms) -> host deferred this run\n" as *u8) 2573 } else { 2574 wc_outcome("fetchfail" as *u8, status[0], n, uslot, ul) 2575 wc_puts(" [fetch-fail] " as *u8); wc_puts(uslot); wc_puts(" -> host deferred this run\n" as *u8) 2576 } 2577 } 2578 // QUALITY: don't ingest a bot-interstitial (verification/challenge wall) as content 2579 // R4 (2026-08-24): the three literals that lived here are ROWS in knowledge/antibot_markers.conf now, and the 2580 // classifier names the vendor. RATE-LIMIT and STATUS-BLOCK keep their existing http / pacer paths below. 2581 var intr: i64 = 0 2582 var abtc: i64 = 0 2583 if n > 0 { abtc = abt_classify(status[0], out, n); if abtc == ABT_VENDOR_WALL { intr = 1 } if abtc == ABT_GENERIC_WALL { intr = 1 } } 2584 // R11: SAY IT OUT LOUD. Both of these used to end the page's life in silence -- the 2585 // interstitial branch simply fell through the `intr == 0` guard with no print and no row, 2586 // so a Cloudflare wall and a genuinely empty host were INDISTINGUISHABLE downstream. 2587 if intr == 1 { 2588 wc_outcome("botwall" as *u8, status[0], n, uslot, ul) 2589 wc_puts(" [botwall " as *u8); sys_write(1, abt_last_vendor_g, abt_last_vendor_len_g); wc_puts("] " as *u8); wc_puts(uslot); wc_puts(" -> " as *u8); sys_write(1, abt_last_reason_g, abt_last_reason_len_g); wc_puts(", NOT indexed (logged for mining)\n" as *u8) 2590 } 2591 else { if n > 0 { if status[0] != 200 { 2592 wc_outcome("http" as *u8, status[0], n, uslot, ul) 2593 wc_puts(" [http-" as *u8); wc_num(status[0]); wc_puts("] " as *u8); wc_puts(uslot); wc_puts(" -> not indexed (logged)\n" as *u8) 2594 } } } 2595 if intr == 0 { if n > 0 { if status[0] == 200 { 2596 // R-JS-HEADLESS hydration -- DEFAULT-OFF since 2026-07-25 (seq628 root cause, seq1002). 2597 // js_render_page executes ARBITRARY page JavaScript in an interpreter that has NO execution 2598 // budget (measured: zero fuel/step-cap/deadline/watchdog anywhere in nx_js_eval.nx), so ONE 2599 // page with a long or non-terminating script hangs this crawler FOREVER -- which is exactly 2600 // the stall: fetch 1 completes, hydration never returns, the log freezes, the process lives. 2601 // A crawler that indexes raw HTML is strictly better than one that wedges on page 1, so the 2602 // enhancement yields to the invariant. Re-enable ONLY once the interpreter is budgeted; the 2603 // link harvest and text extraction below never needed hydration to work. 2604 var phtml: *u8 = out; var phn: i64 = n 2605 if WC_JS_HYDRATE == 1 { if js_has_inline_script(out, n) == 1 { let hyr: i64 = js_render_page(out, n, hydrbuf, cap); if hyr > 0 { phtml = hydrbuf; phn = hyr } } } 2606 // R6 (2026-08-24): index the density-FIT text -- nav, footer, sidebar and link farms dropped by the 2607 // published Kohlschuetter rule; the lib falls back to the full render when nothing survives, and says so. 2608 var tlen: i64 = 0 2609 if WC_FIT_TEXT == 1 { 2610 tlen = bd_fit_text(phtml, phn, text, tcap) 2611 wc_fit_pages_g = wc_fit_pages_g + 1 2612 wc_fit_saved_g = wc_fit_saved_g + (phn - bd_last_fit_html_g) 2613 if bd_last_fallback_g == 1 { wc_fit_fallback_g = wc_fit_fallback_g + 1 } 2614 } else { tlen = nx_html_to_text(phtml, phn, text, tcap) } 2615 // INDEX only real content -- measured bimodal gap (junk<=56 chars, real>=1251, nothing between). 2616 // The link HARVEST below is SEPARATE and runs for ALL pages, so filtering the index never 2617 // starves the frontier (a thin tag/index page is junk to index but a fine link source). 2618 if tlen < WC_MININDEX { wc_outcome("thin" as *u8, status[0], tlen, uslot, ul) } 2619 if tlen >= WC_MININDEX { 2620 let fp: i64 = nx_simhash_fingerprint(text, tlen) 2621 var dup: i64 = 0 2622 var k: i64 = 0 2623 // Past the window NOTHING is compared, so "unique" beyond it is UNCHECKED, not proven. 2624 if fetched > WC_DEDUP_WINDOW { if wc_dwarn_g == 0 { wc_dwarn_g = 1 2625 wc_puts(" [dedup-window] compared " as *u8); wc_num(WC_DEDUP_WINDOW) 2626 wc_puts(" of " as *u8); wc_num(fetched); wc_puts(" fingerprints -- beyond this UNCHECKED\n" as *u8) } } 2627 while k < fetched { if k < WC_DEDUP_WINDOW { if nx_simhash_hamming(fp, keptfp[k]) <= WC_SIMHASH_HAM { dup = 1 } } k = k + 1 } 2628 if dup == 0 { 2629 if fetched < 128 { keptfp[fetched] = fp } 2630 // per-host diversity cap lookup (the factored djb2 -- one host-identity hash) 2631 let hh: i64 = wc_hosthash(phost, phl) 2632 var hci: i64 = 0 - 1; var hj: i64 = 0 2633 while hj < hc_n { if hc_hash[hj] == hh { hci = hj; hj = hc_n } else { hj = hj + 1 } } 2634 var hcnt: i64 = 0 2635 if hci >= 0 { hcnt = hc_cnt[hci] } 2636 // R4b: a CURATED host earns the depth cap; everything else keeps the 2637 // diversity cap. The seed list is the only thing that grants this, so the 2638 // privilege is data-driven and revocable by editing one file. 2639 var hcap: i64 = WC_HOSTCAP 2640 if wc_host_str_is_canon(canh, k4ncan, phost, phl) == 1 { hcap = WC_HOSTCAP_CANON } 2641 if hcnt >= hcap { 2642 wc_outcome("hostcap" as *u8, status[0], hcnt, uslot, ul) 2643 wc_puts(" [host-cap] skip index (>=" as *u8); wc_num(hcap); wc_puts("/host this run) " as *u8); wc_puts(uslot); wc_puts("\n" as *u8) 2644 } else { 2645 var tn: i64 = tlen 2646 if tn > CI_DOCCAP { tn = CI_DOCCAP } 2647 let cid: i64 = ci_hash(text, tn) 2648 dss_mkkey(cid, key) 2649 var already: i64 = 0 2650 if (h as i64) != 0 { if ss_hget(h, key, pbox, lbox) == 1 { already = 1 } } 2651 // R11b: `already_present` is a TERMINAL decision and was unlogged, so a 2652 // steady-state run (every page already indexed) produced an EMPTY outcome 2653 // log and the miner had no denominator. Measured immediately: fetched=2 2654 // ingested=0 already_present=2 -> 0 rows. An instrument blind to the 2655 // COMMON case only ever sees crises. 2656 if already == 1 { present = present + 1; wc_outcome("dup" as *u8, status[0], tn, uslot, ul) } else { 2657 if ss_add(w, 1, key, text, tn) < 0 { 2658 if ss_commit(prefix, w, segidbox[0]) == 0 { segsbox[0] = segsbox[0] + 1 } 2659 segidbox[0] = segidbox[0] + 1 2660 w[1] = 0 2661 ss_add(w, 1, key, text, tn) 2662 } 2663 ci_mkurlkey(cid, ukey) 2664 if ss_add(w, 1, ukey, uslot, ul) < 0 { 2665 if ss_commit(prefix, w, segidbox[0]) == 0 { segsbox[0] = segsbox[0] + 1 } 2666 segidbox[0] = segidbox[0] + 1 2667 w[1] = 0 2668 ss_add(w, 1, ukey, uslot, ul) 2669 } 2670 // PERSIST the fingerprint this run ALREADY computed. It was used to 2671 // dedupe within the run and then discarded, so intra-host near-duplication 2672 // -- the real doorway signature -- could only be asked by re-fetching the 2673 // whole corpus. Now it is a query. Additive row: absent fp: simply means 2674 // "ingested before this shipped", never a wrong answer. 2675 if (wc_fpk_g as i64) == 0 { wc_fpk_g = sys_mmap(64) } 2676 if (wc_fpv_g as i64) == 0 { wc_fpv_g = sys_mmap(32) } 2677 ci_mkfpkey(cid, wc_fpk_g) 2678 let fpn: i64 = ccz_cat_num(wc_fpv_g, 0, fp) 2679 if ss_add(w, 1, wc_fpk_g, wc_fpv_g, fpn) < 0 { 2680 if ss_commit(prefix, w, segidbox[0]) == 0 { segsbox[0] = segsbox[0] + 1 } 2681 segidbox[0] = segidbox[0] + 1 2682 w[1] = 0 2683 ss_add(w, 1, wc_fpk_g, wc_fpv_g, fpn) 2684 } 2685 // S12 (2026-09-17): the FETCH EPOCH beside the doc (fe:<cid>), the same commit dance as the fp row, 2686 // so every result can say how old its bytes are; an absent row (older ingests) reads as unobserved. 2687 if (wc_fek_g as i64) == 0 { wc_fek_g = sys_mmap(64) } 2688 if (wc_fev_g as i64) == 0 { wc_fev_g = sys_mmap(32) } 2689 ci_mkfekey(cid, wc_fek_g) 2690 let fen: i64 = ccz_cat_num(wc_fev_g, 0, sys_now_realtime_sec()) 2691 if ss_add(w, 1, wc_fek_g, wc_fev_g, fen) < 0 { 2692 if ss_commit(prefix, w, segidbox[0]) == 0 { segsbox[0] = segsbox[0] + 1 } 2693 segidbox[0] = segidbox[0] + 1 2694 w[1] = 0 2695 ss_add(w, 1, wc_fek_g, wc_fev_g, fen) 2696 } 2697 ingested = ingested + 1 2698 // R5: feed the sufficiency judge the FIT text just indexed; its verdict ends the run at the next batch 2699 if (wc_cs_g as i64) != 0 { cs_add_doc(wc_cs_g, text, tn); if cs_should_stop(wc_cs_g) != CS_CONTINUE { wc_cs_stop_g = 1 } } 2700 if hci < 0 { if hc_n < 512 { hc_hash[hc_n] = hh; hc_cnt[hc_n] = 1; hc_n = hc_n + 1 } } else { hc_cnt[hci] = hc_cnt[hci] + 1 } 2701 wc_outcome("ok" as *u8, status[0], tn, uslot, ul) 2702 wc_puts(" ingested #" as *u8); wc_num(ingested); wc_puts(" " as *u8); wc_puts(uslot); wc_puts(" (" as *u8); wc_num(tn); wc_puts(" chars hops=" as *u8); wc_num(fhops); wc_puts(")\n" as *u8) 2703 } 2704 } 2705 } 2706 } 2707 // HARVEST links from EVERY 200 page (NOT index-gated): thin index/tag pages are good link sources. 2708 var blen: i64 = 0 2709 var bi: i64 = 0 2710 var hit: i64 = 0 2711 while hit == 0 { 2712 if bi + 2 >= ul { hit = 1 } else { 2713 if uslot[bi] == (58 as u8) { if uslot[bi+1] == (47 as u8) { if uslot[bi+2] == (47 as u8) { 2714 var e2: i64 = bi + 3 2715 var go2: i64 = 1 2716 while go2 == 1 { if e2 >= ul { go2 = 0 } else { if uslot[e2] == (47 as u8) { go2 = 0 } else { e2 = e2 + 1 } } } 2717 var k2: i64 = 0 2718 while k2 < e2 { base[k2] = uslot[k2]; k2 = k2 + 1 } 2719 base[e2] = 0 as u8 2720 blen = e2 2721 hit = 1 2722 } } } 2723 if hit == 0 { bi = bi + 1 } 2724 } 2725 } 2726 wc_harvest(out, n, base, blen, h, w, prefix, segidbox, segsbox, discbox, runseen, nrs) 2727 // FEED FRONTIER SEED: discover the page's RSS/Atom feed, fetch it (paced), enqueue each 2728 // ARTICLE url into the frontier -- clean links > <a> harvest; coverage COMPOUNDS via feeds. 2729 if nx_feed_discover(out, n, feedhref, WC_MAGIC_4096) == 1 { 2730 wc_resolve(uslot, feedhref, feedabs) 2731 let fhl: i64 = wc_host(feedabs, fhost) 2732 pace_before(fhost, fhl) 2733 let fnn: i64 = nx_https_fetch_follow_best(feedabs, store, fxml, cap, 6, fstat) 2734 pace_after(fhost, fhl, fstat[0], 0) 2735 if fstat[0] == 200 { if fnn > 0 { 2736 var fidx: i64 = 0; var fseed: i64 = 0; var floop: i64 = 1 2737 while floop == 1 { 2738 if fidx >= 100 { floop = 0 } else { 2739 if nx_feed_item_at(fxml, fnn, fidx, fit, WC_MAGIC_2048, fil, WC_MAGIC_2048, fisum, WC_MAGIC_8192) == 1 { 2740 let fill: i64 = wc_len(fil) 2741 if fill >= 8 { if discbox[0] < wc_disc_cap() { 2742 if wc_urlok(fil, fill) == 1 { if wc_frstate(h, fil, fill) == 0 { 2743 wc_frkey(fil, fill, ffk) 2744 if wc_runseen(runseen, nrs, ffk) == 0 { 2745 wc_frmark(w, prefix, segidbox, segsbox, fil, fill, 1) 2746 discbox[0] = discbox[0] + 1; fseed = fseed + 1 2747 } 2748 } } 2749 } } 2750 fidx = fidx + 1 2751 } else { floop = 0 } 2752 } 2753 } 2754 if fseed > 0 { wc_puts(" feed-seeded " as *u8); wc_num(fseed); wc_puts(" article URLs -> frontier (" as *u8); wc_puts(feedabs); wc_puts(")\n" as *u8) } 2755 } } 2756 } 2757 } } } 2758 b3 = b3 + 1 2759 } 2760 } 2761 } 2762 } 2763 if w[1] > 0 { if ss_commit(prefix, w, segidbox[0]) == 0 { segsbox[0] = segsbox[0] + 1 } } 2764 wc_puts("CRAWL-STEP done: fetched=" as *u8); wc_num(fetched) 2765 wc_puts(" ingested=" as *u8); wc_num(ingested) 2766 wc_puts(" already_present=" as *u8); wc_num(present) 2767 wc_puts(" discovered_new=" as *u8); wc_num(discbox[0]) 2768 wc_puts(" skipped_deadhost=" as *u8); wc_num(skipdead) 2769 wc_puts(" retired_deadrows=" as *u8); wc_num(retired) 2770 wc_puts(" robots_hosts=" as *u8); wc_num(wc_rb_n_g) 2771 wc_puts(" robots_cd_adopted=" as *u8); wc_num(wc_rb_cd_g) 2772 wc_puts(" robots_skipped=" as *u8); wc_num(wc_rb_skip_g) 2773 wc_puts(" hostcap_retired=" as *u8); wc_num(hostcapped) 2774 // DEFERRALS ARE PRINTED BESIDE THE RETIREMENTS ON PURPOSE: the pair is the whole point. A run showing 2775 // hostcap_retired=227 discarded 227 pages; a run showing hostcap_deferred=227 kept all 227 for a later 2776 // pass. One number alone cannot tell those apart, and they are opposite outcomes for the crawl. 2777 wc_puts(" hostcap_deferred=" as *u8); wc_num(hostcap_deferred) 2778 wc_puts(" pace_deferred=" as *u8); wc_num(deferred) 2779 // FRONTIER GAUGE, CORRECTED 2026-08-25. It used to read `npend - fetched + disc`, subtracting ONE of 2780 // the FOUR ways a row leaves the frontier. Measured over six consecutive runs, the residual it hid was 2781 // 138/171/189/219/164 rows -- mean 176/run -- all of it the selection-time host cap, which marks rows 2782 // D with no counter. The gauge therefore reported ~2000-2060 while true pending was 1871-1883, and an 2783 // investigation into "why is the frontier flat at ~1900" was reading this number, not the frontier. 2784 // pace_deferred is deliberately NOT subtracted: those rows stay PENDING by design, which is the whole 2785 // point of deferring them. A PARTITION IS A CLAIM: THESE PARTS NOW SUM. 2786 wc_puts(" frontier_remaining~=" as *u8); wc_num(wc_frontier_remaining(npend, fetched, retired, wc_rb_skip_g, hostcapped, discbox[0])) 2787 wc_puts(" segments=" as *u8); wc_num(segsbox[0]); wc_puts("\n" as *u8) 2788 // R6 / R5 announce: a feature that prints nothing is indistinguishable from one that was never compiled in 2789 wc_puts(" fit_text pages=" as *u8); wc_num(wc_fit_pages_g); wc_puts(" html_bytes_removed=" as *u8); wc_num(wc_fit_saved_g); wc_puts(" fallback=" as *u8); wc_num(wc_fit_fallback_g); wc_puts("\n" as *u8) 2790 if (wc_cs_g as i64) != 0 { 2791 let csr: *u8 = sys_mmap(WC_MAGIC_4096) 2792 let csn: i64 = cs_report(wc_cs_g, csr, WC_MAGIC_4096) 2793 wc_puts(" " as *u8); sys_write(1, csr, csn) 2794 if wc_cs_stop_g == 1 { wc_puts(" [sufficiency] STOPPED EARLY: the judge cleared its bar before the page budget\n" as *u8) } 2795 } 2796 if fetched >= 1 { wc_puts("CRAWL-STEP GREEN\n" as *u8); return 0 } 2797 wc_puts("CRAWL-STEP RED (nothing fetched)\n" as *u8) 2798 return 3 2799}