code wiki / _hdl_build / nx_web_crawl_step.nx
nx_web_crawl_step.nx source
↩ module page · 1478 lines · 82847 B
1// nx_web_crawl_step.nx -- CRAWL AT SCALE: the frontier-PERSISTED successor to nx_web_ingest's one-shot BFS.
2// The frontier lives IN the web shard itself (seg_store-native, no side files): key fr:<cid(url)> whose
3// value is "P<url>" (pending) or "D<url>" (done) -- append-only, ss_hget last-wins flips state. Each run:
4// 1. enumerate pending frontier rows (keys-index walk, dedup, latest state wins)
5// 2. fetch up to <max_pages> of them (sovereign TLS; seq628 pool: WC_POOL_N forked workers on
6// DISTINCT hosts, parent-enforced WC_FETCH_BUDGET_MS wall clock, dead-host deferral)
7// 3. ingest kept pages (doc:<cid> + url:<cid>, the nx_web_ingest mechanics)
8// 4. bank every NEWLY-DISCOVERED content link as a pending fr: row (filtered: no queries/fragments/
9// binaries/wiki-meta) and mark fetched urls done (even failures -- a dead url never wedges the loop)
10// So the crawl RESUMES where it stopped: run it any number of times and the index only grows. A seed url
11// is only needed to BOOTSTRAP an empty frontier (or inject a new start).
12// usage: nx_web_crawl_step <max_pages> [seed-url]
13// license_tier: ORIGINAL
14import "nx_corpus_ingest.nx" // ci_hash / ci_mkurlkey / dss_prefix / dss_mkkey / seg_store / nx_html_to_text
15import "nx_x509_trust_store.nx"
16import "nx_trust_store_load_from_certdata.nx"
17import "nx_https_fetch_follow.nx"
18import "nx_simhash.nx"
19import "nx_crawl_pace.nx" // adaptive per-host pacing (replaces the fixed 350ms; 429 backoff at scale)
20import "nx_feed_extract.nx" // nx_feed_discover + nx_feed_item_at -- seed the frontier with clean article URLs
21import "nx_js_eval.nx" // R-JS-HEADLESS: js_render_page + js_has_inline_script (hydrate JS pages for indexing)
22import "nx_outlink_harvest.nx"
23import "nx_store_seed_lib.nx" // sts_load -> the urlpolicy- plane (R10: the denylist is DATA, not code) // olh_urlok / olh_outkey / olh_resolve_root / olh_scan -- CANONICAL url filter +
24// seq1002: JS hydration is an ENHANCEMENT gated behind an UNBUDGETED interpreter. 0 = off (safe default,
25// crawler always terminates); 1 = on (only once nx_js_eval enforces an execution budget). Named const so
26// re-enabling is a deliberate one-line decision with this comment attached, not a silent edit.
27const WC_JS_HYDRATE: i64 = 0
28const WC_MAGIC_2000: i64 = 2000
29const WC_MAGIC_2048: i64 = 2048
30const WC_MAGIC_8192: i64 = 8192
31const WC_MAGIC_4194304: i64 = 4194304
32const WC_MAGIC_4096: i64 = 4096
33const WC_MAGIC_8388608: i64 = 8388608
34const WC_MAGIC_1048576: i64 = 1048576
35const WC_MAGIC_5381: i64 = 5381
36 // link-graph edge semantics, SHARED with nx_cc_ingest (one node-identity policy)
37
38const WC_MAXPEND: i64 = 2048 // pending urls pulled per run. MEASURED 2026-08-04: at 256 the pull
39 // window sat 93% full of skipped-dead-host rows (~240 recur every run
40 // in stable walk order, deliberately left pending) -> fetched=13-48 of a
41 // 400-page budget, ~300-600 pages/day vs the ~7-8K the beat intends. 2048
42 // leaves ~1800 live-host candidates after the rot blockade; all buffers
43 // (urls mmap, seenk, seed shift) scale off this const. The rot itself
44 // still needs a persistent host-health retire policy (debt filed).
45const WC_HD_RETIRE: i64 = 3 // R3 host-health: retire (P->D) a skipped row once its host's PERSISTED
46 // dead-run streak reaches this (streak increments once per dead run,
47 // resets on any completed response; retire fires on run streak+1).
48const WC_MAXSEEDS: i64 = 256 // seed-file urls held for the retire EXEMPTION + the R4 canonical lane +
49 // the R5 re-queue. Was 96 while the seed file sat at 95 rows -- ONE row
50 // from a SILENT cliff: the 97th line would have been dropped from the
51 // retire exemption with no log line, and a retired seed is PERMANENT
52 // host loss (wc_seed_add refuses ANY existing fr: row). Raised
53 // 2026-08-05 (debt 1785936869); the drop is now LOUD (seed parse below).
54const WC_SEED_REFRESH: i64 = 8 // R5: already-done seed urls re-marked pending per run. The persisted
55 // sr:cursor rotates the whole seed list, so every hub is re-fetched
56 // roughly every nseeds/8 runs and re-harvests its outlinks -- the
57 // frontier REPLENISHES instead of burning down.
58const WC_MAXDISC: i64 = 300 // new frontier rows banked per run (bounds frontier growth per step)
59const WC_CANON_RESERVE: i64 = 384 // R4 CANONICAL LANE (2026-08-05): head slots of each run's work list
60 // reserved for canonical-seed hosts. MEASURED that day: an 89,000-doc
61 // corpus held 64 en.wikipedia.org / 0 who.int / 1 python.org / 2 irs.gov
62 // docs -- every one of them seeded and answering HTTP 200, none blocked
63 // and none retired. They were STARVED, not refused.
64const WC_MAXCANH: i64 = 96 // distinct canonical HOSTS parsed from the seed file (mirrors WC_MAXSEEDS)
65const WC_MAXURL: i64 = 1500 // urls longer than this are skipped (junk/tracking monsters)
66const WC_MININDEX: i64 = 500 // min extracted chars to INDEX a page (not to harvest its links). Measured
67 // bimodal gap 2026-07-05: junk<=56 chars, real content>=1251, nothing between
68 // -> 500 filters thin junk (person/tag stubs) with huge margin. (>CI_MINDOC=20.)
69const WC_HOSTCAP_CANON: i64 = 40 // R4b DEPTH CAP FOR CURATED HOSTS (2026-08-05). WC_HOSTCAP below exists to
70 // stop ONE junk host exploding into N near-duplicate variants -- that is a
71 // defence against UNCURATED breadth, and it is exactly backwards for a
72 // canonical seed host, where DEPTH is the whole objective. MEASURED: at 6
73 // pages/host/run en.wikipedia.org gains at most 432 docs/day, so climbing
74 // out of 64 docs would take about three weeks; at 40 it is a few days.
75 // Still bounded, and still spread across ~60 canonical hosts.
76const WC_HOSTCAP: i64 = 6 // max pages INDEXED per host per crawl step (diversity). Measured 2026-07-06:
77 // one CC0 deed page exploded into 16 language variants (deed.fr/de/fi/...) that
78 // simhash misses (translated = different fingerprint). A per-host cap defends
79 // against that + faceted-nav/calendar/pagination traps. Harvest is NOT capped.
80
81// ---- seq628 PARALLEL FETCH POOL (2026-07-29) ----
82// The crawl was SEQUENTIAL: one unbounded fetch at a time, so a frontier of mostly-dead hosts burned
83// minutes per url on connect timeouts (measured 2026-07-23: a 30-page crawl stalled 10+ min). Fix shape:
84// fetch in BATCHES of WC_POOL_N forked children (network wait overlaps), each child hard-bounded by a
85// PARENT-enforced wall clock (stronger than a connect timeout: it bounds connect+TLS+read+redirects),
86// one DISTINCT host per in-flight slot (politeness: never two concurrent fetches to one host), and a
87// host that times out or hard-fails is DEFERRED for the rest of the run (its urls stay PENDING, not
88// done -- recency/priority, retried next run at one-probe cost). Ingest/harvest stay single-process in
89// the parent: the seg-store writer keeps ONE writer by construction.
90const WC_POOL_N: i64 = 8 // in-flight fetch children per batch. Bounded by spool RAM (8 x 8MiB
91 // fetch caps = 64MiB peak) and by typical distinct-host diversity in
92 // a 256-url frontier pull; speedup ~ min(N, distinct hosts).
93const WC_FETCH_BUDGET_MS: i64 = 20000 // per-fetch wall clock. Healthy fetches measure <1s recv with
94 // worst-case ~5s TLS handshake flakes; 20s = ~4x the worst GOOD
95 // fetch observed, where a dead host used to cost minutes.
96const WC_POLL_MS: i64 = 100 // parent reap-poll quantum while children are in flight
97const WC_US_PER_MS: i64 = 1000
98const WC_WNOHANG: i64 = 1
99const WC_SIGKILL: i64 = 9
100const WC_SPOOL_HDRB: i64 = 16 // spool header: [i64 http-status][i64 body-bytes]
101const WC_SPOOLPATH_B: i64 = 128
102const WC_MODE_RW: i64 = 420 // 0644 spool file mode
103const WC_DEADHOSTS: i64 = 128 // dead-host slots per run (frontier pull is 256 urls; hosts <= urls)
104const WC_HASH_MASK: i64 = 0x7fffffffffffffff
105
106func wc_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
107func wc_num(v: i64) -> i64 {
108 let bb: *u8 = sys_mmap(28); var m: i64 = v
109 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
110 let t: *u8 = sys_mmap(28); var k: i64 = 0
111 if m == 0 { t[0] = 48 as u8; k = 1 }
112 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
113 var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 }
114 sys_write(1, bb, k); return 0
115}
116func wc_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
117// resolve a feed href against the page url: absolute (http...) kept; "/path" -> scheme://host/path.
118func wc_resolve(base: *u8, href: *u8, out: *u8) -> i64 {
119 if href[0]==(104 as u8) { if href[1]==(116 as u8) { if href[2]==(116 as u8) { if href[3]==(112 as u8) {
120 var a: i64=0; while href[a]!=(0 as u8) { out[a]=href[a]; a=a+1 } out[a]=0 as u8; return 0
121 } } } }
122 var sp: i64 = 0 - 1; var i: i64 = 0; var df: i64 = 0
123 while df == 0 { if base[i]==(0 as u8) { df=1 } else { if base[i]==(58 as u8) { if base[i+1]==(47 as u8) { if base[i+2]==(47 as u8) { sp=i; df=1 } } } if df==0 { i=i+1 } } }
124 if sp < 0 { var k: i64=0; while href[k]!=(0 as u8) { out[k]=href[k]; k=k+1 } out[k]=0 as u8; return 0 }
125 var he: i64 = sp + 3; var dn: i64 = 0
126 while dn == 0 { if base[he]==(0 as u8) { dn=1 } else { if base[he]==(47 as u8) { dn=1 } else { he=he+1 } } }
127 var o: i64 = 0; while o < he { out[o]=base[o]; o=o+1 }
128 if href[0]==(47 as u8) { var j: i64=0; while href[j]!=(0 as u8) { out[o]=href[j]; o=o+1; j=j+1 } }
129 else { out[o]=47 as u8; o=o+1; var j: i64=0; while href[j]!=(0 as u8) { out[o]=href[j]; o=o+1; j=j+1 } }
130 out[o]=0 as u8; return 0
131}
132// host (between "://" and next '/' or ':') -> out; flag-based (never index past the string). returns len.
133func wc_host(url: *u8, out: *u8) -> i64 {
134 var sp: i64 = 0 - 1; var i: i64 = 0; var d: i64 = 0
135 while d == 0 { if url[i]==(0 as u8) { d=1 } else { if url[i]==(58 as u8) { if url[i+1]==(47 as u8) { if url[i+2]==(47 as u8) { sp=i; d=1 } } } if d==0 { i=i+1 } } }
136 var s: i64 = 0; if sp >= 0 { s = sp + 3 }
137 var o: i64 = 0; var d2: i64 = 0
138 while d2 == 0 { let c: i64 = url[s] as i64; if c==0 { d2=1 } else { if c==47 { d2=1 } else { if c==58 { d2=1 } else { out[o]=url[s]; o=o+1; s=s+1 } } } }
139 out[o] = 0 as u8; return o
140}
141func wc_atoi(s: *u8) -> i64 { var v: i64 = 0; var i: i64 = 0; while s[i] != (0 as u8) { let c: i64 = s[i] as i64; if c >= 48 { if c <= 57 { v = v * 10 + (c - 48) } } i = i + 1 } return v }
142func wc_hassub(u: *u8, ul: i64, lit: *u8) -> i64 {
143 let ll: i64 = wc_len(lit)
144 if ll == 0 { return 0 }
145 var i: i64 = 0
146 while i + ll <= ul {
147 var m: i64 = 1
148 var k: i64 = 0
149 while k < ll { if u[i + k] != lit[k] { m = 0; k = ll } else { k = k + 1 } }
150 if m == 1 { return 1 }
151 i = i + 1
152 }
153 return 0
154}
155func wc_endswith(u: *u8, ul: i64, lit: *u8) -> i64 {
156 let ll: i64 = wc_len(lit)
157 if ul < ll { return 0 }
158 var k: i64 = 0
159 while k < ll { if u[ul - ll + k] != lit[k] { return 0 } k = k + 1 }
160 return 1
161}
162// CONTENT-URL FILTER: 1 = worth crawling. Delegates to the CANONICAL olh_urlok (verbatim extraction of the
163// rules that lived here; shared with nx_cc_ingest) so frontier urls and link-graph nodes follow ONE policy
164// (OLH_MAXURL == the measured 1500 junk threshold that was WC_MAXURL).
165// wc_urlok MOVED below the R4 canon helpers (it now consults them) -- see wc_urlok after wc_host_is_canon.
166// frontier key for a URL: "fr:<decimal cid of the url bytes>"
167func wc_frkey(u: *u8, ul: i64, out: *u8) -> i64 {
168 let cid: i64 = ci_hash(u, ul)
169 out[0] = 102 as u8 // f
170 out[1] = 114 as u8 // r
171 out[2] = 58 as u8 // :
172 var m: i64 = cid
173 let t: *u8 = sys_mmap(28)
174 var k: i64 = 0
175 if m == 0 { t[0] = 48 as u8; k = 1 }
176 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
177 var i: i64 = 0
178 while i < k { out[3 + i] = t[k - 1 - i]; i = i + 1 }
179 out[3 + k] = 0 as u8
180 return 3 + k
181}
182// out:<cid(page_url)> = the LINK GRAPH edge list for PageRank (P1): value = packed i64 target cid(url)s.
183// Node identity = cid(url) (SAME hash as the frontier's fr:<cid(url)>), so an outlink resolves to a graph
184// node with zero extra index; serve-time authority = content_cid -> url:<cid> -> ci_hash(url) -> pr:<cid>.
185// Key shape lives in the SHARED lib (one producer policy with nx_cc_ingest).
186func wc_outkey(cid: i64, out: *u8) -> i64 { return olh_outkey(cid, out) }
187// frontier state for a URL: 0 = unknown (no row), 1 = pending, 2 = done
188func wc_frstate(h: *i64, u: *u8, ul: i64) -> i64 {
189 if (h as i64) == 0 { return 0 }
190 let key: *u8 = sys_mmap(64)
191 wc_frkey(u, ul, key)
192 let pp: *i64 = sys_mmap(16) as *i64
193 let pl: *i64 = sys_mmap(16) as *i64
194 if ss_hget(h, key, pp, pl) != 1 { return 0 }
195 if pl[0] < 1 { return 0 }
196 let vp: *u8 = pp[0] as *u8
197 if vp[0] == (80 as u8) { return 1 } // P
198 if vp[0] == (68 as u8) { return 2 } // D
199 return 0
200}
201// bank a frontier state row: value = <P|D><url> (append-only; ss_hget last-wins = the state flip)
202func wc_frmark(w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, u: *u8, ul: i64, state: i64) -> i64 {
203 let key: *u8 = sys_mmap(64)
204 wc_frkey(u, ul, key)
205 let val: *u8 = sys_mmap(ul + 2)
206 if state == 1 { val[0] = 80 as u8 } else { val[0] = 68 as u8 }
207 var i: i64 = 0
208 while i < ul { val[1 + i] = u[i]; i = i + 1 }
209 if ss_add(w, 1, key, val, ul + 1) < 0 {
210 // writer full: commit, then REUSE the buffer (w[0]=buf w[1]=len w[2]=cap; reset len = fresh writer)
211 if ss_commit(prefix, w, segidbox[0]) == 0 { segsbox[0] = segsbox[0] + 1 }
212 segidbox[0] = segidbox[0] + 1
213 w[1] = 0
214 ss_add(w, 1, key, val, ul + 1)
215 }
216 return 0
217}
218// R8 (2026-08-05): the canonical host table, module-level so the shared URL policy can consult it
219// WITHOUT threading it through wc_harvest's already-wide arity. Filled ONCE in main after the seed
220// file parses; zero until then, so every gate fixture and any run without a seed file sees exactly
221// the pre-R8 policy (statics default 0 -> allowq=0 -> byte-identical).
222static wc_canh_g: *u8
223static wc_ncan_g: i64
224static wc_qscratch_g: *u8
225
226// ---- R4 CANONICAL LANE (2026-08-05) -----------------------------------------------------------
227// MEASURED that day: the corpus held ~89,000 docs but only 64 from en.wikipedia.org, 0 from who.int,
228// 1 from python.org and 2 from irs.gov -- while ALL of those hosts sat in canonical_seeds.txt and
229// answered HTTP 200. They were never blocked, never walled and never retired. They were STARVED.
230// wc_pending walks segments in stable append order and stops at WC_MAXPEND, so once the frontier held
231// more than 2048 P rows the canonical rows (banked LATER, so living in NEWER segments) never entered
232// the window at all, and the shared WC_MAXDISC budget was spent by whichever junk host happened to be
233// processed first. Ranking work could not have fixed this: the documents did not exist to rank.
234// ★A FLAT WALK OVER A FRONTIER THAT GREW JUNK-FIRST IS A PERMANENT BLOCKADE OF ITS OWN HEAD.
235// The lane is ADDITIVE (rule 25 -- build intelligence, never strip): nothing is filtered out and no
236// row is dropped. Canonical rows are pulled in a SECOND host-filtered pass and placed at the FRONT of
237// the work list, so they are fetched first and get first claim on the discovery budget; every other
238// row keeps its original order behind the reserve.
239
240// byte-equal NUL-terminated strings
241func wc_streq(a: *u8, b: *u8) -> i64 {
242 var i: i64 = 0
243 while a[i] == b[i] { if a[i] == (0 as u8) { return 1 } i = i + 1 }
244 return 0
245}
246// the seed file's DISTINCT hosts with a leading "www." stripped, so a seed of https://www.who.int/
247// covers who.int, www.who.int AND data.who.int -- the same dot-suffix contract the search engine's
248// site: filter already uses. 64-byte slots. Returns the host count.
249func wc_canon_hosts(seedstore: *u8, nseeds: i64, out: *u8) -> i64 {
250 var n: i64 = 0
251 var i: i64 = 0
252 let hb: *u8 = sys_mmap(64)
253 while i < nseeds {
254 let sp: *u8 = (seedstore as i64 + i * WC_MAGIC_2048) as *u8
255 let hl: i64 = wc_host(sp, hb)
256 if hl > 0 { if hl < 60 {
257 hb[hl] = 0 as u8
258 var st: i64 = 0
259 if hl > 4 { if hb[0] == (119 as u8) { if hb[1] == (119 as u8) { if hb[2] == (119 as u8) { if hb[3] == (46 as u8) { st = 4 } } } } }
260 let hp: *u8 = (hb as i64 + st) as *u8
261 var dup: i64 = 0
262 var d: i64 = 0
263 while d < n {
264 if wc_streq((out as i64 + d * 64) as *u8, hp) == 1 { dup = 1; d = n } else { d = d + 1 }
265 }
266 if dup == 0 { if n < WC_MAXCANH {
267 let dst: *u8 = (out as i64 + n * 64) as *u8
268 var c: i64 = 0
269 while hp[c] != (0 as u8) { dst[c] = hp[c]; c = c + 1 }
270 dst[c] = 0 as u8
271 n = n + 1
272 } }
273 } }
274 i = i + 1
275 }
276 return n
277}
278// 1 iff u's host IS a canonical host or a dot-suffixed subdomain of one ("en.wikipedia.org" matches a
279// canonical "wikipedia.org"; "notwikipedia.org" does NOT, because the boundary dot is required).
280// hb = caller-owned 64B scratch, allocated ONCE per walk -- a per-row sys_mmap would burn a page per
281// frontier row.
282// HOST-STRING form of the canonical test. The fetch loop already holds the page's host (phost/phl),
283// so re-parsing a url there would be waste -- and duplicating the match would be two copies of one
284// rule (rule 15). This is the ONE implementation; wc_host_is_canon is the url-taking wrapper.
285func wc_host_str_is_canon(canh: *u8, ncan: i64, hb: *u8, hl: i64) -> i64 {
286 if ncan <= 0 { return 0 }
287 if hl <= 0 { return 0 }
288 var i: i64 = 0
289 while i < ncan {
290 let cp: *u8 = (canh as i64 + i * 64) as *u8
291 var cl: i64 = 0
292 while cp[cl] != (0 as u8) { cl = cl + 1 }
293 if cl > 0 { if cl <= hl {
294 var eq: i64 = 1
295 var x: i64 = 0
296 while x < cl { if hb[hl - cl + x] != cp[x] { eq = 0; x = cl } else { x = x + 1 } }
297 if eq == 1 {
298 if hl == cl { return 1 }
299 if hb[hl - cl - 1] == (46 as u8) { return 1 }
300 }
301 } }
302 i = i + 1
303 }
304 return 0
305}
306func wc_host_is_canon(canh: *u8, ncan: i64, u: *u8, hb: *u8) -> i64 {
307 if ncan <= 0 { return 0 }
308 let hl: i64 = wc_host(u, hb)
309 if hl <= 0 { return 0 }
310 if hl >= 60 { return 0 }
311 hb[hl] = 0 as u8
312 return wc_host_str_is_canon(canh, ncan, hb, hl)
313}
314// R8: the ONE content-URL policy, now trust-aware. A curated seed host may carry its CONTENT on
315// query strings (measured: graphis.ne.jp galleries are model.php?ID=... -- the blanket '?' reject
316// made that entire vertical structurally unreachable, so seeding the hub could never yield a page).
317// Everything else keeps the blanket reject, which is what stops tracking junk and pagination traps
318// on uncurated hosts. Same shape as the R4b depth cap: the LIMIT is right, its SUBJECT's trust is
319// what changed. wc_ncan_g==0 (gates, seedless runs) -> allowq=0 -> the exact pre-R8 contract.
320// ---- R10 URL POLICY: JUDGE THE SHAPE, NOT THE HOST ------------------------------------------------
321// R8 admitted query strings ONLY on curated seed hosts. That is an ALLOWLIST, and an allowlist is the
322// wrong shape for a search engine: most of the web's content rides query strings, so "uncurated =>
323// unreachable" meant this crawler could only ever see hosts a human had already typed in. That scales
324// to a DIRECTORY, never to an INDEX -- and it is why a real gravure result like
325// www.v2ph.com/actor/JULIA?page=3&hl=en was refused before a single byte was fetched.
326// Operator 2026-08-06: "that's a dumb way to have a search engine setup." Correct.
327// R10 judges the QUERY ITSELF, host-independently:
328// - a TRACKING param (utm_*, fbclid, session ids) means this URL is a DUPLICATE of the clean one
329// -> reject, because indexing it twice is worse than not indexing it
330// - too many params, or an overlong query, is a facet/calendar TRAP -> reject
331// - anything else is CONTENT -> admit, curated or not
332// Curation survives as a BOOST (a curated host gets a bigger param budget), never as the gate.
333// The lists are DATA in knowledge/store/urlpolicy-, loaded ONCE per run: adding a tracking param must
334// never require a recompile. ★AN ALLOWLIST ANSWERS "WHO DO WE TRUST"; AN INDEX MUST ANSWER "IS THIS A PAGE".
335const WC_URLPOL_PLANE: *u8 = "knowledge/store/urlpolicy-"
336const WC_QSLOT: i64 = 32
337const WC_QMAXDENY: i64 = 192
338const WC_QPLANECAP: i64 = 65536
339const WC_QDEF_MAXP: i64 = 4 // default param budget for an UNCURATED host
340const WC_QDEF_MAXL: i64 = 120 // default max query length in bytes
341const WC_QCAN_BOOST: i64 = 4 // curated hosts get this many EXTRA params (trust boost, not a gate)
342static wc_qdeny_g: *u8
343static wc_nqdeny_g: i64
344static wc_qmaxp_g: i64
345static wc_qmaxlen_g: i64
346func wc_qeq(a: *u8, b: *u8, bl: i64) -> i64 {
347 var k: i64 = 0
348 while k < bl { if a[k] != b[k] { return 0 } k = k + 1 }
349 if a[bl] != (0 as u8) { return 0 }
350 return 1
351}
352func wc_qnum(b: *u8, s: i64, e: i64) -> i64 {
353 var v: i64 = 0
354 var any: i64 = 0
355 var i: i64 = s
356 while i < e { let c: i64 = b[i] as i64; if c >= 48 { if c <= 57 { v = v * 10 + (c - 48); any = 1 } } i = i + 1 }
357 if any == 0 { return 0 - 1 }
358 return v
359}
360// Load the policy plane. ABSENT plane = the built-in defaults, NEVER a closed gate: a missing data file
361// must not silently return the crawler to allowlist behaviour, which is the failure nobody would notice.
362func wc_load_urlpolicy() -> i64 {
363 wc_qdeny_g = sys_mmap(WC_QSLOT * WC_QMAXDENY)
364 wc_nqdeny_g = 0
365 wc_qmaxp_g = WC_QDEF_MAXP
366 wc_qmaxlen_g = WC_QDEF_MAXL
367 let b: *u8 = sys_mmap(WC_QPLANECAP)
368 let n: i64 = sts_load(WC_URLPOL_PLANE, b, WC_QPLANECAP - 16)
369 if n <= 0 { return 0 }
370 var i: i64 = 0
371 while i < n {
372 let ls: i64 = i
373 var le: i64 = ls
374 var s: i64 = 1
375 while s == 1 { if le >= n { s = 0 } else { if b[le] == (10 as u8) { s = 0 } else { le = le + 1 } } }
376 i = le + 1
377 if le - ls > 2 { if b[ls] != (35 as u8) {
378 var t: i64 = 0 - 1
379 var j: i64 = ls
380 while j < le { if b[j] == (9 as u8) { t = j; j = le } else { j = j + 1 } }
381 if t > ls {
382 let nl: i64 = t - ls
383 // config knobs are named __maxparams / __maxqlen so they cannot collide with a param name
384 if wc_qeq("__maxparams\x00" as *u8, (b as i64 + ls) as *u8, nl) == 1 {
385 let v: i64 = wc_qnum(b, t + 1, le); if v > 0 { wc_qmaxp_g = v }
386 }
387 else { if wc_qeq("__maxqlen\x00" as *u8, (b as i64 + ls) as *u8, nl) == 1 {
388 let v2: i64 = wc_qnum(b, t + 1, le); if v2 > 0 { wc_qmaxlen_g = v2 }
389 }
390 else {
391 if nl < WC_QSLOT - 1 { if wc_nqdeny_g < WC_QMAXDENY {
392 let dst: *u8 = (wc_qdeny_g as i64 + wc_nqdeny_g * WC_QSLOT) as *u8
393 var c: i64 = 0
394 while c < nl { dst[c] = b[ls + c]; c = c + 1 }
395 dst[nl] = 0 as u8
396 wc_nqdeny_g = wc_nqdeny_g + 1
397 } }
398 } }
399 }
400 } }
401 }
402 return wc_nqdeny_g
403}
404// is this param name on the deny list? (exact match, so "sid" never kills "sid_of_article")
405func wc_qparam_denied(u: *u8, s: i64, e: i64) -> i64 {
406 if wc_nqdeny_g <= 0 { return 0 }
407 if (wc_qdeny_g as i64) == 0 { return 0 }
408 let nl: i64 = e - s
409 if nl <= 0 { return 0 }
410 var i: i64 = 0
411 while i < wc_nqdeny_g {
412 let cp: *u8 = (wc_qdeny_g as i64 + i * WC_QSLOT) as *u8
413 var cl: i64 = 0
414 while cp[cl] != (0 as u8) { cl = cl + 1 }
415 if cl == nl {
416 var eq: i64 = 1
417 var k: i64 = 0
418 while k < nl { if u[s + k] != cp[k] { eq = 0; k = nl } else { k = k + 1 } }
419 if eq == 1 { return 1 }
420 }
421 i = i + 1
422 }
423 return 0
424}
425// THE QUERY JUDGE. 1 = a content URL worth fetching. Host-independent by design.
426func wc_query_ok(u: *u8, ul: i64, canon: i64) -> i64 {
427 var q: i64 = 0 - 1
428 var i: i64 = 0
429 while i < ul { if u[i] == (63 as u8) { q = i; i = ul } else { i = i + 1 } }
430 if q < 0 { return 1 } // no query at all -- nothing to judge
431 let qlen: i64 = ul - q - 1
432 if qlen <= 0 { return 0 } // a bare trailing '?' is the clean URL wearing a costume
433 // SELF-DEFAULTING, NOT LOADER-DEPENDENT. These budgets live in statics that only wc_load_urlpolicy
434 // sets, so ANY entry path that skips the loader (the gate, a future caller, a unit harness) would
435 // otherwise read 0 and reject EVERY query url -- a silent regression to something STRICTER than the
436 // allowlist this replaced. The gate caught exactly that. Defaults belong where the value is READ.
437 // ★A LIMIT THAT READS 0 WHEN UNINITIALISED IS A CLOSED GATE WEARING AN OPEN GATE'S NAME.
438 var maxlen: i64 = wc_qmaxlen_g
439 if maxlen <= 0 { maxlen = WC_QDEF_MAXL }
440 if qlen > maxlen { return 0 }
441 var budget: i64 = wc_qmaxp_g
442 if budget <= 0 { budget = WC_QDEF_MAXP }
443 if canon == 1 { budget = budget + WC_QCAN_BOOST }
444 var nparam: i64 = 0
445 var ps: i64 = q + 1
446 while ps < ul {
447 // end of this param = the next '&', or the end of the url
448 var end: i64 = ps
449 var f: i64 = 0
450 while f == 0 { if end >= ul { f = 1 } else { if u[end] == (38 as u8) { f = 1 } else { end = end + 1 } } }
451 // the NAME runs to the first '=' inside this param (a valueless param is all name)
452 var nameend: i64 = ps
453 var g: i64 = 0
454 while g == 0 { if nameend >= end { g = 1 } else { if u[nameend] == (61 as u8) { g = 1 } else { nameend = nameend + 1 } } }
455 if nameend > ps { if wc_qparam_denied(u, ps, nameend) == 1 { return 0 } }
456 nparam = nparam + 1
457 if nparam > budget { return 0 }
458 ps = end + 1
459 }
460 return 1
461}
462func wc_urlok(u: *u8, ul: i64) -> i64 {
463 var canon: i64 = 0
464 if wc_ncan_g > 0 { if (wc_canh_g as i64) != 0 { if (wc_qscratch_g as i64) != 0 {
465 if wc_host_is_canon(wc_canh_g, wc_ncan_g, u, wc_qscratch_g) == 1 { canon = 1 }
466 } } }
467 // allowq=1 ALWAYS: the shared filter's blanket '?' kill is retired here in favour of the shape judge
468 // below. Everything else it enforces (scheme, fragments, wiki namespaces, binary extensions) still runs.
469 if olh_urlok_q(u, ul, 1) == 0 { return 0 }
470 return wc_query_ok(u, ul, canon)
471}
472// enumerate PENDING frontier urls: walk every segment's key index for "fr:"-prefixed keys (dedup by key),
473// ss_hget each key's LATEST value, keep the P ones. urlsout = flat 2KB slots. Returns count.
474// R4: canh/ncan/canonly extend the walk WITHOUT duplicating it (rule 15 -- one walk, one contract).
475// canonly=0 ignores canh/ncan entirely and is byte-identical to the pre-R4 behaviour, which is what
476// the original full-frontier call site passes and what gate T2 keeps pinned.
477func wc_pending(h: *i64, prefix: *u8, urlsout: *u8, maxout: i64, canh: *u8, ncan: i64, canonly: i64) -> i64 {
478 if (h as i64) == 0 { return 0 }
479 let ns: i64 = h[0]
480 let hbuf4: *u8 = sys_mmap(64) // R4 host scratch, allocated ONCE per walk
481 let seenk: *u8 = sys_mmap(64 * (WC_MAXPEND * 4 + 16))
482 var nseen: i64 = 0
483 var npend: i64 = 0
484 let pp: *i64 = sys_mmap(16) as *i64
485 let pl: *i64 = sys_mmap(16) as *i64
486 var s: i64 = 0
487 while s < ns {
488 let kb: *u8 = h[1 + 8 * s] as *u8
489 if h[2 + 8 * s] >= 8 {
490 let m9: i64 = ss_r32(kb, 4)
491 var e9: i64 = 0
492 while e9 < m9 {
493 let eo: i64 = 8 + 4 * m9 + ss_r32(kb, 8 + 4 * e9)
494 if (kb[eo] as i64) == 1 {
495 let kl9: i64 = ss_r32(kb, eo + 1)
496 if kl9 >= 4 { if kl9 < 60 {
497 if kb[eo + 5] == (102 as u8) { if kb[eo + 6] == (114 as u8) { if kb[eo + 7] == (58 as u8) {
498 // copy the key NUL-terminated
499 let kbuf: *u8 = sys_mmap(64)
500 var c: i64 = 0
501 while c < kl9 { kbuf[c] = kb[eo + 5 + c]; c = c + 1 }
502 kbuf[kl9] = 0 as u8
503 // STATE FIRST, then dedup among COLLECTED urls only (seq1288, measured
504 // 2026-07-29): the old pre-state seen-list capped at WC_MAXPEND*4+16 and
505 // SATURATED once the shard held >~1040 distinct fr: keys -- every later key
506 // was never state-checked, so a compounding crawl read pending=0 while
507 // holding hundreds of P rows (pending decayed 256 -> 2 -> 0 in one day; the
508 // coverage leg silently self-wedged). ss_hget is last-wins across segments,
509 // so re-checking a key already walked in another segment is idempotent; the
510 // only dedup that matters is among the <=maxout COLLECTED urls, which is
511 // bounded by construction and cannot saturate.
512 if ss_hget(h, kbuf, pp, pl) == 1 { if pl[0] > 1 {
513 let vp: *u8 = pp[0] as *u8
514 if vp[0] == (80 as u8) { if npend < maxout { if pl[0] < WC_MAGIC_2000 {
515 var dup: i64 = 0
516 var d: i64 = 0
517 while d < npend {
518 let sp: *u8 = (seenk as i64 + d * 64) as *u8
519 var eq: i64 = 1
520 var x: i64 = 0
521 while x <= kl9 { if sp[x] != kbuf[x] { eq = 0; x = kl9 + 1 } else { x = x + 1 } }
522 if eq == 1 { dup = 1; d = npend } else { d = d + 1 }
523 }
524 if dup == 0 {
525 let dst: *u8 = (seenk as i64 + npend * 64) as *u8
526 var c2: i64 = 0
527 while c2 <= kl9 { dst[c2] = kbuf[c2]; c2 = c2 + 1 }
528 let uo: *u8 = (urlsout as i64 + npend * WC_MAGIC_2048) as *u8
529 var y: i64 = 0
530 while y < pl[0] - 1 { uo[y] = vp[1 + y]; y = y + 1 }
531 uo[pl[0] - 1] = 0 as u8
532 // R4 CANONICAL LANE: canonly=1 keeps ONLY canonical-seed hosts.
533 // The slot is written BEFORE the test and simply not committed
534 // when it fails (npend not incremented), so the next candidate
535 // overwrites it -- no second buffer, no shifting.
536 var keep4: i64 = 1
537 if canonly == 1 { if wc_host_is_canon(canh, ncan, uo, hbuf4) == 0 { keep4 = 0 } }
538 if keep4 == 1 { npend = npend + 1 }
539 }
540 } } }
541 } }
542 } } }
543 } }
544 }
545 e9 = e9 + 1
546 }
547 }
548 s = s + 1
549 }
550 return npend
551}
552// has this frontier key been banked THIS run? (the open handle can't see the uncommitted writer, so
553// without this a url found on two pages double-banks). runseen = 32-byte frkey slots.
554func wc_runseen(runseen: *u8, nrs: *i64, frk: *u8) -> i64 {
555 var d: i64 = 0
556 while d < nrs[0] {
557 let sp: *u8 = (runseen as i64 + d * 32) as *u8
558 var eq: i64 = 1
559 var x: i64 = 0
560 var go: i64 = 1
561 while go == 1 {
562 if sp[x] != frk[x] { eq = 0; go = 0 } else {
563 if frk[x] == (0 as u8) { go = 0 } else { x = x + 1 }
564 }
565 }
566 if eq == 1 { return 1 }
567 d = d + 1
568 }
569 if nrs[0] < WC_MAXDISC + 7 {
570 let dst: *u8 = (runseen as i64 + nrs[0] * 32) as *u8
571 var c: i64 = 0
572 var go2: i64 = 1
573 while go2 == 1 { if c >= 31 { go2 = 0 } else { dst[c] = frk[c]; if frk[c] == (0 as u8) { go2 = 0 } else { c = c + 1 } } }
574 dst[31] = 0 as u8
575 nrs[0] = nrs[0] + 1
576 }
577 return 0
578}
579// link extraction (the nx_web_ingest mechanics, feeding the FILTER + frontier instead of a BFS queue).
580// Returns how many NEW pending rows were banked.
581func wc_harvest(hh: *u8, hlen: i64, base: *u8, blen: i64, h: *i64, w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, discbox: *i64, runseen: *u8, nrs: *i64) -> i64 {
582 var banked: i64 = 0
583 let scratch: *u8 = sys_mmap(WC_MAGIC_8192)
584 var i: i64 = 0
585 while i < hlen {
586 var step: i64 = 1
587 // href= (case-insensitive)
588 var ish: i64 = 0
589 if i + 5 < hlen {
590 var c0: i64 = hh[i] as i64
591 if c0 >= 0x41 { if c0 <= 0x5a { c0 = c0 + 0x20 } }
592 if c0 == 0x68 {
593 var c1: i64 = hh[i+1] as i64
594 if c1 >= 0x41 { if c1 <= 0x5a { c1 = c1 + 0x20 } }
595 var c2: i64 = hh[i+2] as i64
596 if c2 >= 0x41 { if c2 <= 0x5a { c2 = c2 + 0x20 } }
597 var c3: i64 = hh[i+3] as i64
598 if c3 >= 0x41 { if c3 <= 0x5a { c3 = c3 + 0x20 } }
599 if c1 == 0x72 { if c2 == 0x65 { if c3 == 0x66 { if (hh[i+4] as i64) == 0x3d { ish = 1 } } } }
600 }
601 }
602 if ish == 1 {
603 let q: i64 = hh[i+5] as i64
604 var quoted: i64 = 0
605 if q == 0x22 { quoted = 1 }
606 if q == 0x27 { quoted = 1 }
607 if quoted == 1 {
608 let cs: i64 = i + 6
609 var e: i64 = cs
610 var run: i64 = 1
611 while run == 1 { run = 0; if e < hlen { if (hh[e] as i64) != q { e = e + 1; run = 1 } } }
612 let clen: i64 = e - cs
613 if clen > 0 { if discbox[0] < WC_MAXDISC {
614 let hp: *u8 = ((hh as i64) + cs) as *u8
615 var cand: *u8 = 0 as *u8
616 var candlen: i64 = 0
617 var abs: i64 = 0
618 if clen >= 8 { if wc_hassub(hp, 8, "https://" as *u8) == 1 { abs = 1 } }
619 if abs == 0 { if clen >= 7 { if wc_hassub(hp, 7, "http://" as *u8) == 1 { abs = 1 } } }
620 if abs == 1 {
621 cand = hp
622 candlen = clen
623 } else {
624 if (hp[0] as i64) == 0x2f {
625 var protorel: i64 = 0
626 if clen >= 2 { if (hp[1] as i64) == 0x2f { protorel = 1 } }
627 if protorel == 0 {
628 // host-based via the SHARED resolver (was base+path concat, which banked phantom
629 // urls like ".../Trust_law/about" -> 404 fetches + rank leaked to fake nodes)
630 let rl: i64 = olh_resolve_root(base, hp, clen, scratch)
631 if rl > 0 {
632 cand = scratch
633 candlen = rl
634 }
635 }
636 }
637 }
638 if (cand as i64) != 0 {
639 // NUL-terminate a copy (cand may be a raw html slice)
640 let ubuf: *u8 = sys_mmap(candlen + 1)
641 var z: i64 = 0
642 while z < candlen { ubuf[z] = cand[z]; z = z + 1 }
643 ubuf[candlen] = 0 as u8
644 if wc_urlok(ubuf, candlen) == 1 {
645 if wc_frstate(h, ubuf, candlen) == 0 {
646 let frk9: *u8 = sys_mmap(64)
647 wc_frkey(ubuf, candlen, frk9)
648 if wc_runseen(runseen, nrs, frk9) == 0 {
649 wc_frmark(w, prefix, segidbox, segsbox, ubuf, candlen, 1)
650 discbox[0] = discbox[0] + 1
651 banked = banked + 1
652 }
653 }
654 }
655 }
656 } }
657 step = (e - i) + 1
658 }
659 }
660 i = i + step
661 }
662 // P1 LINK GRAPH: canonical edge harvest via the SHARED olh_scan (the same pass nx_cc_ingest runs ->
663 // identical node identity + resolution), stored as ONE out:<cid(base)> row (writer-full handling mirrors
664 // wc_frmark). Decoupled from the frontier walk above: edges keep flowing after WC_MAXDISC trips.
665 let edgebuf: *i64 = sys_mmap(OLH_MAXEDGE * 8) as *i64
666 let nedge: i64 = olh_scan(hh, hlen, base, blen, edgebuf, OLH_MAXEDGE)
667 if nedge > 0 { if (w as i64) != 0 {
668 let okey: *u8 = sys_mmap(64)
669 wc_outkey(ci_hash(base, blen), okey)
670 if ss_add(w, 1, okey, edgebuf as *u8, nedge * 8) < 0 {
671 if ss_commit(prefix, w, segidbox[0]) == 0 { segsbox[0] = segsbox[0] + 1 }
672 segidbox[0] = segidbox[0] + 1
673 w[1] = 0
674 ss_add(w, 1, okey, edgebuf as *u8, nedge * 8)
675 }
676 } }
677 return banked
678}
679
680// SEED INJECTION (factored 2026-07-26 so one URL and a whole seed FILE share ONE code path -- the
681// single-seed logic was inline and a second copy would have drifted). Banks the url as a PENDING fr: row
682// and prepends it to THIS run's work list (the store write is not visible to the already-open handle).
683// No-op if the frontier already knows the url, so re-seeding the same list is idempotent.
684func wc_seed_add(h: *i64, w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, urls: *u8, npendbox: *i64, seed: *u8, sl: i64) -> i64 {
685 if sl <= 0 { return 0 }
686 if wc_frstate(h, seed, sl) != 0 { return 0 }
687 var npend: i64 = npendbox[0]
688 var sh: i64 = npend
689 if sh > WC_MAXPEND - 1 { sh = WC_MAXPEND - 1 }
690 while sh > 0 {
691 let dst9: *u8 = (urls as i64 + sh * WC_MAGIC_2048) as *u8
692 let src9: *u8 = (urls as i64 + (sh - 1) * WC_MAGIC_2048) as *u8
693 var y9: i64 = 0
694 var go9: i64 = 1
695 while go9 == 1 { dst9[y9] = src9[y9]; if src9[y9] == (0 as u8) { go9 = 0 } y9 = y9 + 1 }
696 sh = sh - 1
697 }
698 let u0: *u8 = urls
699 var z9: i64 = 0
700 while z9 < sl { u0[z9] = seed[z9]; z9 = z9 + 1 }
701 u0[sl] = 0 as u8
702 if npend < WC_MAXPEND { npendbox[0] = npend + 1 }
703 wc_frmark(w, prefix, segidbox, segsbox, seed, sl, 1)
704 return 1
705}
706// djb2 host hash (factored: the per-host index-cap map and the pool's politeness/dead-host lists must
707// agree on host identity, so ONE hash function).
708func wc_hosthash(hostp: *u8, hl: i64) -> i64 {
709 var hh: i64 = WC_MAGIC_5381
710 var i: i64 = 0
711 while i < hl { hh = (((hh << 5) + hh) + (hostp[i] as i64)) & WC_HASH_MASK; i = i + 1 }
712 return hh
713}
714func wc_deadhost_has(dh: *i64, ndh: i64, hh: i64) -> i64 {
715 var i: i64 = 0
716 while i < ndh { if dh[i] == hh { return 1 } i = i + 1 }
717 return 0
718}
719func wc_deadhost_add(dh: *i64, ndhbox: *i64, hh: i64) -> i64 {
720 if wc_deadhost_has(dh, ndhbox[0], hh) == 1 { return 0 }
721 if ndhbox[0] < WC_DEADHOSTS { dh[ndhbox[0]] = hh; ndhbox[0] = ndhbox[0] + 1; return 1 }
722 return 0
723}
724// ==== R3 HOST-HEALTH (2026-08-04, debt 1785892352): persistent per-host dead-run streak as
725// hd:<hosthash> rows in the SAME shard (the frontier's store is the crawler's only state). MEASURED
726// need: post-widening, 1983 of the 2048-row pull window were dead-host rows (97% rot) recurring in
727// stable walk order -- skip-without-retire made the rot immortal. Streak++ once per run the host is
728// found dead; reset on any completed HTTP response; at WC_HD_RETIRE the batch selector RETIRES the
729// rows it skips (P->D, same last-wins flip as a fetched url). A retired url is re-addable the moment
730// any live page links to it again -- rows are cheap, seeds are not (see the exemption).
731func wc_hdkey(hh: i64, out: *u8) -> i64 {
732 out[0] = 104 as u8; out[1] = 100 as u8; out[2] = 58 as u8 // "hd:"
733 var o: i64 = 3
734 if hh == 0 { out[o] = 48 as u8; out[o + 1] = 0 as u8; return o + 1 }
735 let t: *u8 = sys_mmap(28)
736 var m: i64 = hh
737 var k: i64 = 0
738 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
739 var i: i64 = 0
740 while i < k { out[o] = t[k - 1 - i]; o = o + 1; i = i + 1 }
741 out[o] = 0 as u8
742 return o
743}
744// the host's persisted dead-run streak (0 = healthy/unknown). Reads the RUN-START handle: retire
745// decisions deliberately see the streak as of run start, so death #N retires on run N+1, never same-run.
746func wc_hd_streak(h: *i64, hh: i64) -> i64 {
747 if (h as i64) == 0 { return 0 }
748 let key: *u8 = sys_mmap(64)
749 wc_hdkey(hh, key)
750 let pp: *i64 = sys_mmap(16) as *i64
751 let pl: *i64 = sys_mmap(16) as *i64
752 if ss_hget(h, key, pp, pl) != 1 { return 0 }
753 let vp: *u8 = pp[0] as *u8
754 var v: i64 = 0
755 var i: i64 = 0
756 while i < pl[0] {
757 var c: i64 = vp[i] as i64
758 if c >= 48 { if c <= 57 { v = v * 10 + (c - 48) } }
759 i = i + 1
760 }
761 return v
762}
763// bank the streak (append-only, last-wins; mirrors wc_frmark's writer-full discipline)
764func wc_hd_put(w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, hh: i64, v: i64) -> i64 {
765 let key: *u8 = sys_mmap(64)
766 wc_hdkey(hh, key)
767 let val: *u8 = sys_mmap(28)
768 var vl: i64 = 0
769 if v == 0 { val[0] = 48 as u8; vl = 1 } else {
770 let t: *u8 = sys_mmap(28)
771 var m: i64 = v
772 var k: i64 = 0
773 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
774 while vl < k { val[vl] = t[k - 1 - vl]; vl = vl + 1 }
775 }
776 if ss_add(w, 1, key, val, vl) < 0 {
777 if ss_commit(prefix, w, segidbox[0]) == 0 { segsbox[0] = segsbox[0] + 1 }
778 segidbox[0] = segidbox[0] + 1
779 w[1] = 0
780 ss_add(w, 1, key, val, vl)
781 }
782 return 0
783}
784// ---- R5 SEED FRESHNESS RE-QUEUE (2026-08-05, debt 1785937865) ---------------------------------
785// MEASURED across 4 consecutive runs: frontier_remaining 2047->1963->1769->1748 while
786// discovered_new fell 121->35->0 -- run 3 fetched 300 pages and banked ZERO new frontier rows.
787// wc_harvest only banks a NEVER-SEEN url, so once a host's known links are exhausted its depth
788// stops compounding. ★A CRAWLER IS A CLOSED LOOP: IF CONSUMPTION EXCEEDS DISCOVERY, THE FRONTIER
789// IS A BURN-DOWN AND COVERAGE PLATEAUS NO MATTER HOW FAIRLY THE BUDGET IS SCHEDULED. The single
790// biggest replenishment blocker is done-is-forever: a seed hub fetched once is D and can never
791// rediscover links -- but hub pages (portals, news indexes, topic lists) CHANGE, which is exactly
792// why they are seeds. This re-queue walks the seed list with a PERSISTED cursor (sr:cursor row,
793// same ASCII-digit encoding as the hd: streak rows) and re-marks up to WC_SEED_REFRESH already-D
794// seed urls back to P each run. Bounded by construction; P and unknown rows are left alone.
795func wc_srkey(out: *u8) -> i64 {
796 out[0] = 115 as u8; out[1] = 114 as u8; out[2] = 58 as u8
797 out[3] = 99 as u8; out[4] = 117 as u8; out[5] = 114 as u8
798 out[6] = 0 as u8
799 return 6
800}
801func wc_sr_cursor(h: *i64) -> i64 {
802 if (h as i64) == 0 { return 0 }
803 let key: *u8 = sys_mmap(64)
804 wc_srkey(key)
805 let pp: *i64 = sys_mmap(16) as *i64
806 let pl: *i64 = sys_mmap(16) as *i64
807 if ss_hget(h, key, pp, pl) != 1 { return 0 }
808 let vp: *u8 = pp[0] as *u8
809 var v: i64 = 0
810 var i: i64 = 0
811 while i < pl[0] {
812 var c: i64 = vp[i] as i64
813 if c >= 48 { if c <= 57 { v = v * 10 + (c - 48) } }
814 i = i + 1
815 }
816 return v
817}
818func wc_sr_put(w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, v: i64) -> i64 {
819 let key: *u8 = sys_mmap(64)
820 wc_srkey(key)
821 let val: *u8 = sys_mmap(28)
822 var vl: i64 = 0
823 if v == 0 { val[0] = 48 as u8; vl = 1 } else {
824 let t: *u8 = sys_mmap(28)
825 var m: i64 = v
826 var k: i64 = 0
827 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
828 while vl < k { val[vl] = t[k - 1 - vl]; vl = vl + 1 }
829 }
830 if ss_add(w, 1, key, val, vl) < 0 {
831 if ss_commit(prefix, w, segidbox[0]) == 0 { segsbox[0] = segsbox[0] + 1 }
832 segidbox[0] = segidbox[0] + 1
833 w[1] = 0
834 ss_add(w, 1, key, val, vl)
835 }
836 return 0
837}
838// re-mark up to WC_SEED_REFRESH done seeds as pending, copying each into dst (2KB slots) so the
839// caller's canonical block fetches them THIS run. dstbox[0]=count in/out, dstbox[1]=slot capacity
840// (2-slot box keeps the arity at 9 -- the tree's proven ceiling). Examines each seed at most once
841// per run (lim <= nseeds) so a short list cannot be requeued twice through the uncommitted writer.
842func wc_seed_refresh(h: *i64, w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, seedstore: *u8, nseeds: i64, dst: *u8, dstbox: *i64) -> i64 {
843 if nseeds <= 0 { return 0 }
844 var cur: i64 = wc_sr_cursor(h)
845 if cur < 0 { cur = 0 }
846 cur = cur % nseeds
847 var lim: i64 = WC_SEED_REFRESH
848 if lim > nseeds { lim = nseeds }
849 var rq: i64 = 0
850 var j: i64 = 0
851 while j < lim {
852 let ix: i64 = (cur + j) % nseeds
853 let sp: *u8 = (seedstore as i64 + ix * WC_MAGIC_2048) as *u8
854 var sl: i64 = 0
855 while sp[sl] != (0 as u8) { sl = sl + 1 }
856 if sl > 0 { if wc_frstate(h, sp, sl) == 2 {
857 wc_frmark(w, prefix, segidbox, segsbox, sp, sl, 1)
858 if dstbox[0] < dstbox[1] {
859 let dp: *u8 = (dst as i64 + dstbox[0] * WC_MAGIC_2048) as *u8
860 var c: i64 = 0
861 while c <= sl { dp[c] = sp[c]; c = c + 1 }
862 dstbox[0] = dstbox[0] + 1
863 }
864 rq = rq + 1
865 } }
866 j = j + 1
867 }
868 wc_sr_put(w, prefix, segidbox, segsbox, (cur + lim) % nseeds)
869 return rq
870}
871// seed-file membership (exact url match) -- seed rows are NEVER retired
872func wc_is_seed(seedstore: *u8, nseeds: i64, u: *u8, ul: i64) -> i64 {
873 var s: i64 = 0
874 while s < nseeds {
875 let sp: *u8 = (seedstore as i64 + s * WC_MAGIC_2048) as *u8
876 var eq: i64 = 1
877 var i: i64 = 0
878 while i < ul { if sp[i] != u[i] { eq = 0; i = ul } else { i = i + 1 } }
879 if eq == 1 { if sp[ul] == (0 as u8) { return 1 } }
880 s = s + 1
881 }
882 return 0
883}
884// the retire decision + act, factored so the gate can bite it network-free. Returns 1 = row retired.
885// SEED EXEMPTION LIVES AT THE CALL SITE (wc_is_seed guard) -- 8 params, within the tree's proven arity.
886func wc_retire_if_rotten(h: *i64, w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, u: *u8, ul: i64, hh: i64) -> i64 {
887 if wc_hd_streak(h, hh) < WC_HD_RETIRE { return 0 }
888 wc_frmark(w, prefix, segidbox, segsbox, u, ul, 2)
889 return 1
890}
891// spool path for a worker slot: /tmp/nx_wcpool_<slot>.bin. Slot-indexed names cannot collide: the
892// searchcrawl sub's liveness guard enforces ONE crawler instance, and a slot is reused only after its
893// child was reaped and its spool consumed.
894func wc_spool_path(slot: i64, out: *u8) -> i64 {
895 let lit: *u8 = "/tmp/nx_wcpool_" as *u8
896 var o: i64 = 0
897 while lit[o] != (0 as u8) { out[o] = lit[o]; o = o + 1 }
898 let t: *u8 = sys_mmap(28)
899 var m: i64 = slot
900 var k: i64 = 0
901 if m == 0 { t[0] = 48 as u8; k = 1 }
902 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
903 var i: i64 = 0
904 while i < k { out[o] = t[k - 1 - i]; o = o + 1; i = i + 1 }
905 let ext: *u8 = ".bin" as *u8
906 var j: i64 = 0
907 while ext[j] != (0 as u8) { out[o] = ext[j]; o = o + 1; j = j + 1 }
908 out[o] = 0 as u8
909 return o
910}
911// CHILD: one fetch -> spool file [status][len][body] -> exit. All child memory is freed by the kernel
912// on exit (the windowed-fork idiom); the PARENT owns the wall-clock budget, so the child needs no timer
913// of its own. exit 0 = body spooled; 1 = fetch returned nothing; 2 = spool unwritable.
914func wc_fetch_child(url: *u8, store: *TrustStore, spath: *u8) -> i64 {
915 let cap: i64 = WC_MAGIC_8388608
916 let out: *u8 = sys_mmap(cap)
917 let status: *i64 = sys_mmap(8) as *i64
918 let n: i64 = nx_https_fetch_follow_best(url, store, out, cap, 6, status)
919 let fd: i64 = sys_openat_wr(spath, WC_MODE_RW)
920 if fd < 0 { sys_exit_group(2) }
921 let hdr: *i64 = sys_mmap(WC_SPOOL_HDRB) as *i64
922 hdr[0] = status[0]
923 var bn: i64 = 0
924 if n > 0 { bn = n }
925 hdr[1] = bn
926 sys_write(fd, hdr as *u8, WC_SPOOL_HDRB)
927 if bn > 0 { sys_write(fd, out, bn) }
928 sys_close(fd)
929 if bn > 0 { sys_exit_group(0) }
930 sys_exit_group(1)
931 return 0
932}
933func main(argc: i64, argv: *i64) -> i64 {
934 if argc < 2 { wc_puts("usage: nx_web_crawl_step <max_pages> [seed-url | seed-file]\n" as *u8); return 1 }
935 let max_pages: i64 = wc_atoi(argv[1] as *u8)
936 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, WC_MAGIC_4194304)
937 if r <= 0 { wc_puts("certdata load failed\n" as *u8); return 2 }
938 let store: *TrustStore = r as *TrustStore
939 wc_puts("=== nx_web_crawl_step: frontier-persisted crawl -> dp-web-pub- ===\n" as *u8)
940
941 let prefix: *u8 = sys_mmap(512)
942 dss_prefix("web" as *u8, prefix)
943 // seq1730 ADOPTION (2026-07-31): the digit-parse above was CORRECT and still deadlocked this
944 // organ. It took max+1 over EVERY id in the manifest -- and 64 of this plane's 65 entries are
945 // already pointer-shaped poison from before the guard existed -- so it computed
946 // 140712850411540, which ss_commit's own guard then correctly REFUSED on every crawl step.
947 // Five consecutive steps logged `segments=0` while still printing CRAWL-STEP GREEN: fetched,
948 // parsed and discarded. The fix was right, the guard was right, the DATA was poisoned, and the
949 // leaf never adopted the lib that already knew that. ss_next_segid skips pointer-band ids
950 // (nx_seg_store.nx:1403) so a corrupted manifest HEALS on the next write, while the poisoned
951 // segments stay on disk and readable (rule 13, additive-only -- nothing is deleted).
952 // ★★★MIGRATE THE CHOKEPOINT LIB, NOT THE LEAF. 10+ organs already called ss_next_segid; this
953 // crawler hand-rolled its own scan, and being the holdout cost it every commit it attempted.
954 // ★★★A GREEN VERDICT ON A STEP THAT PERSISTED NOTHING IS THE LOUDEST KIND OF SILENT FAILURE.
955 let segidbox: *i64 = sys_mmap(16) as *i64
956 segidbox[0] = ss_next_segid(prefix)
957 if segidbox[0] <= 0 { segidbox[0] = 1 }
958 // MMAP-OPEN (2026-07-25, seq628 root cause): read-all ss_open pulls the ENTIRE shard into anonymous
959 // RAM (the sev-9 nx_seg_store keystone) -- at ~2GB across 94 segments that is what made this crawler
960 // hang after a single fetch on the NAS. nx_pagerank_build and nx_web_shard_compact were both moved to
961 // ss_open2(prefix,1) earlier this month (measured -45% peak RSS); THIS ORGAN WAS MISSED IN THAT SWEEP.
962 // Same primitive, same call shape: pages fault in on demand and stay evictable in the page cache.
963 let h: *i64 = ss_open2(prefix, 1)
964 let segsbox: *i64 = sys_mmap(16) as *i64; segsbox[0] = 0
965 let discbox: *i64 = sys_mmap(16) as *i64; discbox[0] = 0
966
967 // pending frontier (persisted). A seed bootstraps or extends it.
968 let urls: *u8 = sys_mmap(WC_MAGIC_2048 * (WC_MAXPEND + 2))
969 let canh: *u8 = sys_mmap(64 * WC_MAXCANH) // R4 canonical-seed hosts (filled once the seed file loads)
970 var npend: i64 = wc_pending(h, prefix, urls, WC_MAXPEND, canh, 0, 0)
971 let w: *i64 = ss_begin()
972 let npendbox: *i64 = sys_mmap(16) as *i64
973 npendbox[0] = npend
974 // R3: seed urls banked for the retire exemption (filled by the seed-injection block below)
975 let seedstore: *u8 = sys_mmap(WC_MAGIC_2048 * WC_MAXSEEDS)
976 var nseeds: i64 = 0
977 if argc >= 3 {
978 let sarg: *u8 = argv[2] as *u8
979 // SEED FILE vs SEED URL (2026-07-26): if argv[2] names a readable non-empty file, EVERY non-empty
980 // non-# line is a seed url -- that is how a curated canonical-site list gets aimed at the frontier
981 // in one call. Otherwise argv[2] is a single url (original behaviour, byte-identical path).
982 let szb: *i64 = sys_mmap(16) as *i64
983 szb[0] = 0
984 let fb: *u8 = ss_loadfile(sarg, szb, 0)
985 var isfile: i64 = 0
986 if (fb as i64) != 0 { if szb[0] > 0 { isfile = 1 } }
987 if isfile == 1 {
988 let line: *u8 = sys_mmap(WC_MAGIC_2048)
989 var added: i64 = 0
990 var sdropped: i64 = 0 // R5: seeds beyond WC_MAXSEEDS are REFUSED LOUDLY, never silently
991 var i9: i64 = 0
992 var ll: i64 = 0
993 while i9 <= szb[0] {
994 var ch: i64 = 10
995 if i9 < szb[0] { ch = fb[i9] as i64 }
996 if ch == 10 { if ll > 0 {
997 line[ll] = 0 as u8
998 if line[0] != (35 as u8) {
999 added = added + wc_seed_add(h, w, prefix, segidbox, segsbox, urls, npendbox, line, ll)
1000 // R3: bank EVERY seed url (not just newly-added) for the retire exemption
1001 if nseeds < WC_MAXSEEDS {
1002 let sd9: *u8 = (seedstore as i64 + nseeds * WC_MAGIC_2048) as *u8
1003 var sc9: i64 = 0
1004 while sc9 < ll { sd9[sc9] = line[sc9]; sc9 = sc9 + 1 }
1005 sd9[ll] = 0 as u8
1006 nseeds = nseeds + 1
1007 } else { sdropped = sdropped + 1 }
1008 }
1009 ll = 0
1010 } } else {
1011 if ch != 13 { if ll < WC_MAGIC_2048 - 2 { line[ll] = ch as u8; ll = ll + 1 } }
1012 }
1013 i9 = i9 + 1
1014 }
1015 wc_puts(" seed-file " as *u8); wc_puts(sarg); wc_puts(" -> new pending seeds=" as *u8); wc_num(added); wc_puts("\n" as *u8)
1016 if sdropped > 0 { wc_puts(" [seed-cap] REFUSED " as *u8); wc_num(sdropped); wc_puts(" seed rows beyond WC_MAXSEEDS -- raise the const or trim the file; a dropped seed silently loses its retire exemption AND its canonical-lane membership\n" as *u8) }
1017 } else {
1018 wc_seed_add(h, w, prefix, segidbox, segsbox, urls, npendbox, sarg, wc_len(sarg))
1019 let sl9: i64 = wc_len(sarg)
1020 if sl9 < WC_MAGIC_2048 - 1 {
1021 var sc8: i64 = 0
1022 while sc8 < sl9 { seedstore[sc8] = sarg[sc8]; sc8 = sc8 + 1 }
1023 seedstore[sl9] = 0 as u8
1024 nseeds = 1
1025 }
1026 }
1027 }
1028 npend = npendbox[0]
1029 wc_puts(" frontier pending=" as *u8); wc_num(npend); wc_puts("\n" as *u8)
1030 // ---- R4 CANONICAL LANE: reserve the HEAD of the work list for canonical-seed hosts -----------
1031 // Runs here because it needs the seed file, which is parsed above; the first wc_pending call
1032 // (canonly=0) has already filled `urls` with the ordinary window. This second pass walks the SAME
1033 // frontier host-filtered, then rebuilds the work list as [canonical rows] ++ [everything else,
1034 // original order, byte-deduped]. ncan==0 (no seed file) -> the whole block is skipped and the run
1035 // is byte-identical to pre-R4.
1036 var k4ncan: i64 = wc_canon_hosts(seedstore, nseeds, canh)
1037 // R8: publish the table to the shared URL policy (see wc_urlok). AFTER the seed parse, BEFORE
1038 // any harvest -- so this run's admissions are already trust-aware.
1039 wc_canh_g = canh
1040 wc_ncan_g = k4ncan
1041 wc_qscratch_g = sys_mmap(64)
1042 // R10: load the URL-shape policy (tracking-param denylist + budgets) BEFORE any harvest, so this
1043 // run's admissions are judged by shape rather than by host membership.
1044 let npol: i64 = wc_load_urlpolicy()
1045 wc_puts(" url-policy deny_params=" as *u8); wc_num(npol)
1046 wc_puts(" max_params=" as *u8); wc_num(wc_qmaxp_g)
1047 wc_puts(" max_qlen=" as *u8); wc_num(wc_qmaxlen_g)
1048 if npol == 0 { wc_puts(" (plane ABSENT -- built-in defaults, query URLs still admitted by shape)" as *u8) }
1049 wc_puts("\n" as *u8)
1050 if k4ncan > 0 {
1051 let k4urls: *u8 = sys_mmap(WC_MAGIC_2048 * (WC_CANON_RESERVE + WC_SEED_REFRESH + 2))
1052 // The reserve is a FLOOR DERIVED FROM THE RUN'S OWN BUDGET, not a fixed ceiling: half of
1053 // max_pages goes to canonical hosts and half stays with general discovery, so the lane can
1054 // never become a takeover of the crawl (rule 25 -- nothing is stripped). WC_CANON_RESERVE is
1055 // only the upper CAP on that half. MEASURED 2026-08-05 on the first live run: reserved=384
1056 // SATURATED, i.e. at least 384 canonical rows were already sitting pending and starved.
1057 var k4res: i64 = max_pages / 2
1058 if k4res > WC_CANON_RESERVE { k4res = WC_CANON_RESERVE }
1059 if k4res < 1 { k4res = 1 }
1060 let k4n0: i64 = wc_pending(h, prefix, k4urls, k4res, canh, k4ncan, 1)
1061 // R5: re-queue stale seed hubs INTO the canonical block. The R4c round-robin pass below
1062 // interleaves them by host, so a refreshed hub is fetched early without displacing any
1063 // other canonical host's first row.
1064 let k4box5: *i64 = sys_mmap(16) as *i64
1065 k4box5[0] = k4n0
1066 k4box5[1] = WC_CANON_RESERVE + WC_SEED_REFRESH
1067 let k4rf: i64 = wc_seed_refresh(h, w, prefix, segidbox, segsbox, seedstore, nseeds, k4urls, k4box5)
1068 let k4n: i64 = k4box5[0]
1069 if k4rf > 0 { wc_puts(" seed-refresh requeued=" as *u8); wc_num(k4rf); wc_puts("\n" as *u8) }
1070 if k4n > 0 {
1071 let k4merged: *u8 = sys_mmap(WC_MAGIC_2048 * (WC_MAXPEND + 2))
1072 // ---- R4c ROUND-ROBIN THE CANONICAL BLOCK BY HOST ----------------------------------
1073 // MEASURED 2026-08-05, and it is the R4 bug repeating one level down: with the block in
1074 // stable walk order, whichever canonical host happens to hold the most pending rows eats
1075 // the whole reserve. Observed old.reddit.com taking 18 of 64 ingests in a single run
1076 // while who.int and en.wikipedia.org -- the hosts the reserve EXISTS for -- waited.
1077 // ★★A FAIRNESS FIX APPLIED TO A SET, NOT TO THE MEMBERS WITHIN IT, JUST MOVES THE
1078 // STARVATION INSIDE THE SET. rank = how many EARLIER canonical rows share this host;
1079 // emitting all rank-0 rows, then all rank-1, gives EVERY canonical host its first row
1080 // before ANY host takes a second. Host identity via the same djb2 the dead-host list and
1081 // the host cap use, so "same host" means one thing everywhere in this organ.
1082 let k4hh: *i64 = sys_mmap(8 * (WC_CANON_RESERVE + WC_SEED_REFRESH + 4)) as *i64
1083 let k4rk: *i64 = sys_mmap(8 * (WC_CANON_RESERVE + WC_SEED_REFRESH + 4)) as *i64
1084 let k4hb: *u8 = sys_mmap(64)
1085 var k4z: i64 = 0
1086 while k4z < k4n {
1087 let k4rp: *u8 = (k4urls as i64 + k4z * WC_MAGIC_2048) as *u8
1088 let k4hl: i64 = wc_host(k4rp, k4hb)
1089 k4hh[k4z] = wc_hosthash(k4hb, k4hl)
1090 var k4r: i64 = 0
1091 var k4y: i64 = 0
1092 while k4y < k4z { if k4hh[k4y] == k4hh[k4z] { k4r = k4r + 1 } k4y = k4y + 1 }
1093 k4rk[k4z] = k4r
1094 k4z = k4z + 1
1095 }
1096 var k4mn: i64 = 0
1097 var k4p: i64 = 0
1098 while k4p < k4n {
1099 var k4q2: i64 = 0
1100 while k4q2 < k4n {
1101 if k4rk[k4q2] == k4p {
1102 let k4s: *u8 = (k4urls as i64 + k4q2 * WC_MAGIC_2048) as *u8
1103 let k4d: *u8 = (k4merged as i64 + k4mn * WC_MAGIC_2048) as *u8
1104 var k4c: i64 = 0
1105 while k4s[k4c] != (0 as u8) { k4d[k4c] = k4s[k4c]; k4c = k4c + 1 }
1106 k4d[k4c] = 0 as u8
1107 k4mn = k4mn + 1
1108 }
1109 k4q2 = k4q2 + 1
1110 }
1111 k4p = k4p + 1
1112 }
1113 var k4oi: i64 = 0
1114 while k4oi < npend {
1115 let k4op: *u8 = (urls as i64 + k4oi * WC_MAGIC_2048) as *u8
1116 var k4dup: i64 = 0
1117 var k4q: i64 = 0
1118 while k4q < k4n {
1119 if wc_streq(k4op, (k4urls as i64 + k4q * WC_MAGIC_2048) as *u8) == 1 { k4dup = 1; k4q = k4n } else { k4q = k4q + 1 }
1120 }
1121 if k4dup == 0 { if k4mn < WC_MAXPEND {
1122 let k4dp: *u8 = (k4merged as i64 + k4mn * WC_MAGIC_2048) as *u8
1123 var k4c2: i64 = 0
1124 while k4op[k4c2] != (0 as u8) { k4dp[k4c2] = k4op[k4c2]; k4c2 = k4c2 + 1 }
1125 k4dp[k4c2] = 0 as u8
1126 k4mn = k4mn + 1
1127 } }
1128 k4oi = k4oi + 1
1129 }
1130 var k4cb: i64 = 0
1131 while k4cb < k4mn {
1132 let k4sp: *u8 = (k4merged as i64 + k4cb * WC_MAGIC_2048) as *u8
1133 let k4dp2: *u8 = (urls as i64 + k4cb * WC_MAGIC_2048) as *u8
1134 var k4c3: i64 = 0
1135 while k4sp[k4c3] != (0 as u8) { k4dp2[k4c3] = k4sp[k4c3]; k4c3 = k4c3 + 1 }
1136 k4dp2[k4c3] = 0 as u8
1137 k4cb = k4cb + 1
1138 }
1139 npend = k4mn
1140 npendbox[0] = k4mn
1141 }
1142 wc_puts(" canon-lane hosts=" as *u8); wc_num(k4ncan); wc_puts(" reserved=" as *u8); wc_num(k4n); wc_puts(" worklist=" as *u8); wc_num(npend); wc_puts("\n" as *u8)
1143 }
1144 if npend == 0 {
1145 wc_puts("CRAWL-STEP GREEN (frontier empty -- give a seed url to bootstrap)\n" as *u8)
1146 return 0
1147 }
1148
1149 let cap: i64 = WC_MAGIC_8388608
1150 let out: *u8 = sys_mmap(cap)
1151 let hydrbuf: *u8 = sys_mmap(cap) // R-JS-HEADLESS: serialized hydrated DOM for inline-script pages
1152 let tcap: i64 = WC_MAGIC_1048576
1153 let text: *u8 = sys_mmap(tcap)
1154 let status: *i64 = sys_mmap(8) as *i64
1155 let keptfp: *i64 = sys_mmap(128 * 8) as *i64
1156 let key: *u8 = sys_mmap(64)
1157 let ukey: *u8 = sys_mmap(64)
1158 let pbox: *i64 = sys_mmap(16) as *i64
1159 let lbox: *i64 = sys_mmap(16) as *i64
1160 let base: *u8 = sys_mmap(WC_MAGIC_2048)
1161 let runseen: *u8 = sys_mmap(32 * (WC_MAXDISC + 8))
1162 let nrs: *i64 = sys_mmap(16) as *i64; nrs[0] = 0
1163
1164 // feed-seed scratch (allocated ONCE, reused per page -- no mmap in the crawl loop)
1165 let feedhref: *u8 = sys_mmap(WC_MAGIC_4096)
1166 let feedabs: *u8 = sys_mmap(WC_MAGIC_4096)
1167 let fhost: *u8 = sys_mmap(512)
1168 let fxml: *u8 = sys_mmap(cap)
1169 let fstat: *i64 = sys_mmap(16) as *i64
1170 let fit: *u8 = sys_mmap(WC_MAGIC_2048)
1171 let fil: *u8 = sys_mmap(WC_MAGIC_2048)
1172 let fisum: *u8 = sys_mmap(WC_MAGIC_8192)
1173 let ffk: *u8 = sys_mmap(64)
1174 // per-host INDEX-count map for THIS run (host_hash -> count), linear-scan (few hosts/run). Diversity cap.
1175 let hc_hash: *i64 = sys_mmap(512*8) as *i64
1176 let hc_cnt: *i64 = sys_mmap(512*8) as *i64
1177 var hc_n: i64 = 0
1178 // ---- seq628 batched parallel fetch; single-writer ingest stays in the parent ----
1179 let dh: *i64 = sys_mmap(WC_DEADHOSTS * 8) as *i64
1180 let ndhbox: *i64 = sys_mmap(16) as *i64
1181 ndhbox[0] = 0
1182 let bidx: *i64 = sys_mmap(WC_POOL_N * 8) as *i64
1183 let bpid: *i64 = sys_mmap(WC_POOL_N * 8) as *i64
1184 let bhh: *i64 = sys_mmap(WC_POOL_N * 8) as *i64
1185 let brc: *i64 = sys_mmap(WC_POOL_N * 8) as *i64
1186 let bstart: *i64 = sys_mmap(WC_POOL_N * 8) as *i64
1187 let spaths: *u8 = sys_mmap(WC_SPOOLPATH_B * WC_POOL_N)
1188 var sp0: i64 = 0
1189 while sp0 < WC_POOL_N { wc_spool_path(sp0, (spaths as i64 + sp0 * WC_SPOOLPATH_B) as *u8); sp0 = sp0 + 1 }
1190 let phost: *u8 = sys_mmap(512)
1191 let stbox: *i64 = sys_mmap(16) as *i64
1192 let shdr: *i64 = sys_mmap(WC_SPOOL_HDRB) as *i64
1193 var fetched: i64 = 0
1194 var ingested: i64 = 0
1195 var present: i64 = 0
1196 var skipdead: i64 = 0
1197 var retired: i64 = 0 // R3: dead-host rows retired (P->D) this run
1198 var qi: i64 = 0
1199 while qi < npend {
1200 if fetched >= max_pages { qi = npend } else {
1201 // BATCH SELECT: up to WC_POOL_N urls, one DISTINCT host per slot. A url whose host is already
1202 // in this batch CLOSES the batch (it leads the next one) so same-host depth is never starved;
1203 // a url whose host went DEAD this run is skipped (stays PENDING for a future run).
1204 var nb: i64 = 0
1205 var scan: i64 = 1
1206 while scan == 1 {
1207 if qi >= npend { scan = 0 } else {
1208 if nb >= WC_POOL_N { scan = 0 } else {
1209 if fetched + nb >= max_pages { scan = 0 } else {
1210 let us1: *u8 = (urls as i64 + qi * WC_MAGIC_2048) as *u8
1211 let pl1: i64 = wc_host(us1, phost)
1212 let hh9: i64 = wc_hosthash(phost, pl1)
1213 if wc_deadhost_has(dh, ndhbox[0], hh9) == 1 {
1214 skipdead = skipdead + 1
1215 // R3: the host's PERSISTED streak has hit the bar -> retire this row (P->D) so
1216 // the rot stops blockading the pull window. Seeds exempt; rediscovery re-adds.
1217 // R9 (2026-08-05): a CURATED HOST IS NEVER RETIRED. wc_is_seed exempts only
1218 // the exact seed URLs, so a curated host's DISCOVERED rows were still being
1219 // retired after 3 dead runs -- and we cannot tell "the host is down" from
1220 // "we cannot speak its TLS". MEASURED that day: our client advertises 0x1302
1221 // (nginx's default) and cannot decrypt it, so such hosts fail EVERY run,
1222 // hit the streak bar, and are retired PERMANENTLY (wc_seed_add refuses any
1223 // existing row). ★A TRANSPORT-LAYER CAPABILITY GAP MUST NOT BE LAUNDERED
1224 // INTO PERMANENT COVERAGE LOSS -- RETIRE ON EVIDENCE THE HOST IS DEAD,
1225 // NEVER ON EVIDENCE OF OUR OWN INABILITY. We curated these hosts, so their
1226 // absence is OUR bug to fix; keeping the row P means they return for free
1227 // the moment the client learns the cipher. Uncurated rot still retires.
1228 var r9keep: i64 = 0
1229 if wc_is_seed(seedstore, nseeds, us1, wc_len(us1)) == 1 { r9keep = 1 }
1230 if wc_ncan_g > 0 { if (wc_qscratch_g as i64) != 0 {
1231 if wc_host_is_canon(wc_canh_g, wc_ncan_g, us1, wc_qscratch_g) == 1 { r9keep = 1 }
1232 } }
1233 if r9keep == 0 {
1234 retired = retired + wc_retire_if_rotten(h, w, prefix, segidbox, segsbox, us1, wc_len(us1), hh9)
1235 }
1236 qi = qi + 1
1237 } else {
1238 var infl: i64 = 0
1239 var b9: i64 = 0
1240 while b9 < nb { if bhh[b9] == hh9 { infl = 1; b9 = nb } else { b9 = b9 + 1 } }
1241 if infl == 1 { scan = 0 } else {
1242 bidx[nb] = qi
1243 bhh[nb] = hh9
1244 nb = nb + 1
1245 qi = qi + 1
1246 }
1247 }
1248 } } }
1249 }
1250 if nb == 0 { qi = npend } else {
1251 // LAUNCH: pace in the PARENT (the persisted pacing state keeps one writer), then fork the fetch.
1252 var b: i64 = 0
1253 while b < nb {
1254 let us2: *u8 = (urls as i64 + bidx[b] * WC_MAGIC_2048) as *u8
1255 let pl2: i64 = wc_host(us2, phost)
1256 pace_before(phost, pl2)
1257 let spb: *u8 = (spaths as i64 + b * WC_SPOOLPATH_B) as *u8
1258 sys_unlinkat(spb)
1259 let pid: i64 = sys_fork()
1260 if pid == 0 { wc_fetch_child(us2, store, spb) }
1261 if pid > 0 { bpid[b] = pid } else { bpid[b] = 0; brc[b] = 0 - 3 }
1262 if pid > 0 { brc[b] = 0 - 2 }
1263 bstart[b] = sys_clock_now_us()
1264 b = b + 1
1265 }
1266 // REAP with the wall-clock budget: WNOHANG poll; past budget -> SIGKILL + blocking reap.
1267 var live: i64 = 0
1268 var bl: i64 = 0
1269 while bl < nb { if bpid[bl] > 0 { live = live + 1 } bl = bl + 1 }
1270 while live > 0 {
1271 var b2: i64 = 0
1272 while b2 < nb {
1273 if bpid[b2] > 0 {
1274 let rp: i64 = sys_wait4(bpid[b2], stbox, WC_WNOHANG)
1275 if rp == bpid[b2] {
1276 brc[b2] = wait_exit_code(stbox[0])
1277 bpid[b2] = 0
1278 live = live - 1
1279 } else {
1280 if sys_clock_now_us() - bstart[b2] > WC_FETCH_BUDGET_MS * WC_US_PER_MS {
1281 nx_kill(bpid[b2], WC_SIGKILL)
1282 sys_wait4(bpid[b2], stbox, 0)
1283 brc[b2] = 0 - 1
1284 bpid[b2] = 0
1285 live = live - 1
1286 }
1287 }
1288 }
1289 b2 = b2 + 1
1290 }
1291 if live > 0 { sys_sleep_ms(WC_POLL_MS) }
1292 }
1293 // PROCESS sequentially (single seg-store writer): spool -> the unchanged ingest/harvest path.
1294 var b3: i64 = 0
1295 while b3 < nb {
1296 let uslot: *u8 = (urls as i64 + bidx[b3] * WC_MAGIC_2048) as *u8
1297 let ul: i64 = wc_len(uslot)
1298 let phl: i64 = wc_host(uslot, phost)
1299 fetched = fetched + 1
1300 // done FIRST (even on failure -- a dead url must not wedge the frontier)
1301 wc_frmark(w, prefix, segidbox, segsbox, uslot, ul, 2)
1302 var n: i64 = 0
1303 status[0] = 0
1304 if brc[b3] == 0 {
1305 let spb3: *u8 = (spaths as i64 + b3 * WC_SPOOLPATH_B) as *u8
1306 let fd3: i64 = sys_openat_rd(spb3)
1307 if fd3 >= 0 {
1308 if sys_read(fd3, shdr as *u8, WC_SPOOL_HDRB) == WC_SPOOL_HDRB {
1309 status[0] = shdr[0]
1310 var want: i64 = shdr[1]
1311 if want > cap { want = cap }
1312 var got: i64 = 0
1313 var rr: i64 = 1
1314 while rr == 1 {
1315 if got >= want { rr = 0 } else {
1316 let g: i64 = sys_read(fd3, (out as i64 + got) as *u8, want - got)
1317 if g <= 0 { rr = 0 } else { got = got + g }
1318 }
1319 }
1320 n = got
1321 }
1322 sys_close(fd3)
1323 sys_unlinkat(spb3)
1324 }
1325 }
1326 pace_after(phost, phl, status[0], 0)
1327 // R3 host-health: any completed response heals the streak; a failure bumps it ONCE per run
1328 // (wc_deadhost_add returns 1 only on the host's FIRST death this run).
1329 let hhx: i64 = wc_hosthash(phost, phl)
1330 if brc[b3] == 0 { if status[0] > 0 {
1331 if wc_hd_streak(h, hhx) > 0 { wc_hd_put(w, prefix, segidbox, segsbox, hhx, 0) }
1332 } }
1333 if brc[b3] != 0 {
1334 // timeout (-1), fetch-fail (1/2) or fork-fail (-3): defer the host for the rest of this run
1335 if wc_deadhost_add(dh, ndhbox, hhx) == 1 {
1336 wc_hd_put(w, prefix, segidbox, segsbox, hhx, wc_hd_streak(h, hhx) + 1)
1337 }
1338 if brc[b3] == (0 - 1) {
1339 wc_puts(" [fetch-timeout] " as *u8); wc_puts(uslot); wc_puts(" (budget " as *u8); wc_num(WC_FETCH_BUDGET_MS); wc_puts("ms) -> host deferred this run\n" as *u8)
1340 } else {
1341 wc_puts(" [fetch-fail] " as *u8); wc_puts(uslot); wc_puts(" -> host deferred this run\n" as *u8)
1342 }
1343 }
1344 // QUALITY: don't ingest a bot-interstitial (verification/challenge wall) as content
1345 var intr: i64 = 0
1346 if n > 0 { if wc_hassub(out, n, "Please wait for verification" as *u8) == 1 { intr = 1 } if wc_hassub(out, n, "Just a moment" as *u8) == 1 { intr = 1 } if wc_hassub(out, n, "Attention Required! | Cloudflare" as *u8) == 1 { intr = 1 } }
1347 if intr == 0 { if n > 0 { if status[0] == 200 {
1348 // R-JS-HEADLESS hydration -- DEFAULT-OFF since 2026-07-25 (seq628 root cause, seq1002).
1349 // js_render_page executes ARBITRARY page JavaScript in an interpreter that has NO execution
1350 // budget (measured: zero fuel/step-cap/deadline/watchdog anywhere in nx_js_eval.nx), so ONE
1351 // page with a long or non-terminating script hangs this crawler FOREVER -- which is exactly
1352 // the stall: fetch 1 completes, hydration never returns, the log freezes, the process lives.
1353 // A crawler that indexes raw HTML is strictly better than one that wedges on page 1, so the
1354 // enhancement yields to the invariant. Re-enable ONLY once the interpreter is budgeted; the
1355 // link harvest and text extraction below never needed hydration to work.
1356 var phtml: *u8 = out; var phn: i64 = n
1357 if WC_JS_HYDRATE == 1 { if js_has_inline_script(out, n) == 1 { let hyr: i64 = js_render_page(out, n, hydrbuf, cap); if hyr > 0 { phtml = hydrbuf; phn = hyr } } }
1358 let tlen: i64 = nx_html_to_text(phtml, phn, text, tcap)
1359 // INDEX only real content -- measured bimodal gap (junk<=56 chars, real>=1251, nothing between).
1360 // The link HARVEST below is SEPARATE and runs for ALL pages, so filtering the index never
1361 // starves the frontier (a thin tag/index page is junk to index but a fine link source).
1362 if tlen >= WC_MININDEX {
1363 let fp: i64 = nx_simhash_fingerprint(text, tlen)
1364 var dup: i64 = 0
1365 var k: i64 = 0
1366 while k < fetched { if k < 128 { if nx_simhash_hamming(fp, keptfp[k]) <= 4 { dup = 1 } } k = k + 1 }
1367 if dup == 0 {
1368 if fetched < 128 { keptfp[fetched] = fp }
1369 // per-host diversity cap lookup (the factored djb2 -- one host-identity hash)
1370 let hh: i64 = wc_hosthash(phost, phl)
1371 var hci: i64 = 0 - 1; var hj: i64 = 0
1372 while hj < hc_n { if hc_hash[hj] == hh { hci = hj; hj = hc_n } else { hj = hj + 1 } }
1373 var hcnt: i64 = 0
1374 if hci >= 0 { hcnt = hc_cnt[hci] }
1375 // R4b: a CURATED host earns the depth cap; everything else keeps the
1376 // diversity cap. The seed list is the only thing that grants this, so the
1377 // privilege is data-driven and revocable by editing one file.
1378 var hcap: i64 = WC_HOSTCAP
1379 if wc_host_str_is_canon(canh, k4ncan, phost, phl) == 1 { hcap = WC_HOSTCAP_CANON }
1380 if hcnt >= hcap {
1381 wc_puts(" [host-cap] skip index (>=" as *u8); wc_num(hcap); wc_puts("/host this run) " as *u8); wc_puts(uslot); wc_puts("\n" as *u8)
1382 } else {
1383 var tn: i64 = tlen
1384 if tn > CI_DOCCAP { tn = CI_DOCCAP }
1385 let cid: i64 = ci_hash(text, tn)
1386 dss_mkkey(cid, key)
1387 var already: i64 = 0
1388 if (h as i64) != 0 { if ss_hget(h, key, pbox, lbox) == 1 { already = 1 } }
1389 if already == 1 { present = present + 1 } else {
1390 if ss_add(w, 1, key, text, tn) < 0 {
1391 if ss_commit(prefix, w, segidbox[0]) == 0 { segsbox[0] = segsbox[0] + 1 }
1392 segidbox[0] = segidbox[0] + 1
1393 w[1] = 0
1394 ss_add(w, 1, key, text, tn)
1395 }
1396 ci_mkurlkey(cid, ukey)
1397 if ss_add(w, 1, ukey, uslot, ul) < 0 {
1398 if ss_commit(prefix, w, segidbox[0]) == 0 { segsbox[0] = segsbox[0] + 1 }
1399 segidbox[0] = segidbox[0] + 1
1400 w[1] = 0
1401 ss_add(w, 1, ukey, uslot, ul)
1402 }
1403 ingested = ingested + 1
1404 if hci < 0 { if hc_n < 512 { hc_hash[hc_n] = hh; hc_cnt[hc_n] = 1; hc_n = hc_n + 1 } } else { hc_cnt[hci] = hc_cnt[hci] + 1 }
1405 wc_puts(" ingested #" as *u8); wc_num(ingested); wc_puts(" " as *u8); wc_puts(uslot); wc_puts(" (" as *u8); wc_num(tn); wc_puts(" chars)\n" as *u8)
1406 }
1407 }
1408 }
1409 }
1410 // HARVEST links from EVERY 200 page (NOT index-gated): thin index/tag pages are good link sources.
1411 var blen: i64 = 0
1412 var bi: i64 = 0
1413 var hit: i64 = 0
1414 while hit == 0 {
1415 if bi + 2 >= ul { hit = 1 } else {
1416 if uslot[bi] == (58 as u8) { if uslot[bi+1] == (47 as u8) { if uslot[bi+2] == (47 as u8) {
1417 var e2: i64 = bi + 3
1418 var go2: i64 = 1
1419 while go2 == 1 { if e2 >= ul { go2 = 0 } else { if uslot[e2] == (47 as u8) { go2 = 0 } else { e2 = e2 + 1 } } }
1420 var k2: i64 = 0
1421 while k2 < e2 { base[k2] = uslot[k2]; k2 = k2 + 1 }
1422 base[e2] = 0 as u8
1423 blen = e2
1424 hit = 1
1425 } } }
1426 if hit == 0 { bi = bi + 1 }
1427 }
1428 }
1429 wc_harvest(out, n, base, blen, h, w, prefix, segidbox, segsbox, discbox, runseen, nrs)
1430 // FEED FRONTIER SEED: discover the page's RSS/Atom feed, fetch it (paced), enqueue each
1431 // ARTICLE url into the frontier -- clean links > <a> harvest; coverage COMPOUNDS via feeds.
1432 if nx_feed_discover(out, n, feedhref, WC_MAGIC_4096) == 1 {
1433 wc_resolve(uslot, feedhref, feedabs)
1434 let fhl: i64 = wc_host(feedabs, fhost)
1435 pace_before(fhost, fhl)
1436 let fnn: i64 = nx_https_fetch_follow_best(feedabs, store, fxml, cap, 6, fstat)
1437 pace_after(fhost, fhl, fstat[0], 0)
1438 if fstat[0] == 200 { if fnn > 0 {
1439 var fidx: i64 = 0; var fseed: i64 = 0; var floop: i64 = 1
1440 while floop == 1 {
1441 if fidx >= 100 { floop = 0 } else {
1442 if nx_feed_item_at(fxml, fnn, fidx, fit, WC_MAGIC_2048, fil, WC_MAGIC_2048, fisum, WC_MAGIC_8192) == 1 {
1443 let fill: i64 = wc_len(fil)
1444 if fill >= 8 { if discbox[0] < WC_MAXDISC {
1445 if wc_urlok(fil, fill) == 1 { if wc_frstate(h, fil, fill) == 0 {
1446 wc_frkey(fil, fill, ffk)
1447 if wc_runseen(runseen, nrs, ffk) == 0 {
1448 wc_frmark(w, prefix, segidbox, segsbox, fil, fill, 1)
1449 discbox[0] = discbox[0] + 1; fseed = fseed + 1
1450 }
1451 } }
1452 } }
1453 fidx = fidx + 1
1454 } else { floop = 0 }
1455 }
1456 }
1457 if fseed > 0 { wc_puts(" feed-seeded " as *u8); wc_num(fseed); wc_puts(" article URLs -> frontier (" as *u8); wc_puts(feedabs); wc_puts(")\n" as *u8) }
1458 } }
1459 }
1460 } } }
1461 b3 = b3 + 1
1462 }
1463 }
1464 }
1465 }
1466 if w[1] > 0 { if ss_commit(prefix, w, segidbox[0]) == 0 { segsbox[0] = segsbox[0] + 1 } }
1467 wc_puts("CRAWL-STEP done: fetched=" as *u8); wc_num(fetched)
1468 wc_puts(" ingested=" as *u8); wc_num(ingested)
1469 wc_puts(" already_present=" as *u8); wc_num(present)
1470 wc_puts(" discovered_new=" as *u8); wc_num(discbox[0])
1471 wc_puts(" skipped_deadhost=" as *u8); wc_num(skipdead)
1472 wc_puts(" retired_deadrows=" as *u8); wc_num(retired)
1473 wc_puts(" frontier_remaining~=" as *u8); wc_num(npend - fetched + discbox[0])
1474 wc_puts(" segments=" as *u8); wc_num(segsbox[0]); wc_puts("\n" as *u8)
1475 if fetched >= 1 { wc_puts("CRAWL-STEP GREEN\n" as *u8); return 0 }
1476 wc_puts("CRAWL-STEP RED (nothing fetched)\n" as *u8)
1477 return 3
1478}