code wiki / _hdl_build / nx_web_crawl_step.nx
nx_web_crawl_step.nx
buildroot/runtime/_hdl_build/nx_web_crawl_step.nx
about
nx_web_crawl_step.nx -- CRAWL AT SCALE: the frontier-PERSISTED successor to nx_web_ingest's one-shot BFS.
The frontier lives IN the web shard itself (seg_store-native, no side files): key fr:<cid(url)> whose
value is "P<url>" (pending) or "D<url>" (done) -- append-only, ss_hget last-wins flips state. Each run:
1. enumerate pending frontier rows (keys-index walk, dedup, latest state wins)
2. fetch up to <max_pages> of them (sovereign TLS; seq628 pool: WC_POOL_N forked workers on
DISTINCT hosts, parent-enforced WC_FETCH_BUDGET_MS wall clock, dead-host deferral)
3. ingest kept pages (doc:<cid> + url:<cid>, the nx_web_ingest mechanics)
4. bank every NEWLY-DISCOVERED content link as a pending fr: row (filtered: no queries/fragments/
binaries/wiki-meta) and mark fetched urls done (even failures -- a dead url never wedges the loop)
So the crawl RESUMES where it stopped: run it any number of times and the index only grows. A seed url
is only needed to BOOTSTRAP an empty frontier (or inject a new start).
usage: nx_web_crawl_step <max_pages> [seed-url]
license_tier: ORIGINAL
dependencies 10 imports · 1 importers
imports: nx_corpus_ingest.nxnx_x509_trust_store.nxnx_trust_store_load_from_certdata.nxnx_https_fetch_follow.nxnx_simhash.nxnx_crawl_pace.nxnx_feed_extract.nxnx_js_eval.nxnx_outlink_harvest.nxnx_store_seed_lib.nx
imported by: nx_web_crawl_step_gate.nx
call flow from main pre-order; caps 40 nodes / depth 6 declared; ↻ = already shown
structs
| none |
consts
| 27 | const WC_JS_HYDRATE: i64 = 0 |
| 28 | const WC_MAGIC_2000: i64 = 2000 |
| 29 | const WC_MAGIC_2048: i64 = 2048 |
| 30 | const WC_MAGIC_8192: i64 = 8192 |
| 31 | const WC_MAGIC_4194304: i64 = 4194304 |
| 32 | const WC_MAGIC_4096: i64 = 4096 |
| 33 | const WC_MAGIC_8388608: i64 = 8388608 |
| 34 | const WC_MAGIC_1048576: i64 = 1048576 |
| 35 | const WC_MAGIC_5381: i64 = 5381 |
| 38 | const WC_MAXPEND: i64 = 2048 // pending urls pulled per run. MEASURED 2026-08-04: at 256 the pull |
| 45 | const WC_HD_RETIRE: i64 = 3 // R3 host-health: retire (P->D) a skipped row once its host's PERSISTED |
| 48 | const WC_MAXSEEDS: i64 = 256 // seed-file urls held for the retire EXEMPTION + the R4 canonical lane + |
| 54 | const WC_SEED_REFRESH: i64 = 8 // R5: already-done seed urls re-marked pending per run. The persisted |
| 58 | const WC_MAXDISC: i64 = 300 // new frontier rows banked per run (bounds frontier growth per step) |
| 59 | const WC_CANON_RESERVE: i64 = 384 // R4 CANONICAL LANE (2026-08-05): head slots of each run's work list |
| 64 | const WC_MAXCANH: i64 = 96 // distinct canonical HOSTS parsed from the seed file (mirrors WC_MAXSEEDS) |
| 65 | const WC_MAXURL: i64 = 1500 // urls longer than this are skipped (junk/tracking monsters) |
| 66 | const WC_MININDEX: i64 = 500 // min extracted chars to INDEX a page (not to harvest its links). Measured |
| 69 | const WC_HOSTCAP_CANON: i64 = 40 // R4b DEPTH CAP FOR CURATED HOSTS (2026-08-05). WC_HOSTCAP below exists to |
| 76 | const WC_HOSTCAP: i64 = 6 // max pages INDEXED per host per crawl step (diversity). Measured 2026-07-06: |
| 90 | const WC_POOL_N: i64 = 8 // in-flight fetch children per batch. Bounded by spool RAM (8 x 8MiB |
| 93 | const WC_FETCH_BUDGET_MS: i64 = 20000 // per-fetch wall clock. Healthy fetches measure <1s recv with |
| 96 | const WC_POLL_MS: i64 = 100 // parent reap-poll quantum while children are in flight |
| 97 | const WC_US_PER_MS: i64 = 1000 |
| 98 | const WC_WNOHANG: i64 = 1 |
| 99 | const WC_SIGKILL: i64 = 9 |
| 100 | const WC_SPOOL_HDRB: i64 = 16 // spool header: [i64 http-status][i64 body-bytes] |
| 101 | const WC_SPOOLPATH_B: i64 = 128 |
| 102 | const WC_MODE_RW: i64 = 420 // 0644 spool file mode |
| 103 | const WC_DEADHOSTS: i64 = 128 // dead-host slots per run (frontier pull is 256 urls; hosts <= urls) |
| 104 | const WC_HASH_MASK: i64 = 0x7fffffffffffffff |
| 335 | const WC_URLPOL_PLANE: *u8 = "knowledge/store/urlpolicy-" |
| 336 | const WC_QSLOT: i64 = 32 |
| 337 | const WC_QMAXDENY: i64 = 192 |
| 338 | const WC_QPLANECAP: i64 = 65536 |
| 339 | const WC_QDEF_MAXP: i64 = 4 // default param budget for an UNCURATED host |
| 340 | const WC_QDEF_MAXL: i64 = 120 // default max query length in bytes |
| 341 | const WC_QCAN_BOOST: i64 = 4 // curated hosts get this many EXTRA params (trust boost, not a gate) |
functions
| 106 | func wc_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 } called by 1: main |
| 107 | func wc_num(v: i64) -> i64 called by 1: main |
| 116 | func wc_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n } |
| 118 | func wc_resolve(base: *u8, href: *u8, out: *u8) -> i64 called by 1: main |
| 133 | func wc_host(url: *u8, out: *u8) -> i64 |
| 141 | func wc_atoi(s: *u8) -> i64 { var v: i64 = 0; var i: i64 = 0; while s[i] != (0 as u8) { let c: i64 = s[i] as i64; if c >= 48 { if c <= 57 { v = v * 10 + (c - 48) } } i = i + 1 } return v } called by 1: main |
| 142 | func wc_hassub(u: *u8, ul: i64, lit: *u8) -> i64 |
| 155 | func wc_endswith(u: *u8, ul: i64, lit: *u8) -> i64 calls 1: wc_len |
| 167 | func wc_frkey(u: *u8, ul: i64, out: *u8) -> i64 |
| 186 | func wc_outkey(cid: i64, out: *u8) -> i64 { return olh_outkey(cid, out) } called by 1: wc_harvest |
| 188 | func wc_frstate(h: *i64, u: *u8, ul: i64) -> i64 |
| 202 | func wc_frmark(w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, u: *u8, ul: i64, state: i64) -> i64 called by 6: wc_harvestwc_seed_addwc_seed_refreshwc_retire_if_rottenmainmain calls 3: wc_frkeyss_addss_commit |
| 241 | func wc_streq(a: *u8, b: *u8) -> i64 |
| 249 | func wc_canon_hosts(seedstore: *u8, nseeds: i64, out: *u8) -> i64 |
| 285 | func wc_host_str_is_canon(canh: *u8, ncan: i64, hb: *u8, hl: i64) -> i64 |
| 306 | func wc_host_is_canon(canh: *u8, ncan: i64, u: *u8, hb: *u8) -> i64 |
| 346 | func wc_qeq(a: *u8, b: *u8, bl: i64) -> i64 called by 1: wc_load_urlpolicy |
| 352 | func wc_qnum(b: *u8, s: i64, e: i64) -> i64 called by 1: wc_load_urlpolicy |
| 362 | func wc_load_urlpolicy() -> i64 |
| 405 | func wc_qparam_denied(u: *u8, s: i64, e: i64) -> i64 called by 1: wc_query_ok |
| 426 | func wc_query_ok(u: *u8, ul: i64, canon: i64) -> i64 |
| 462 | func wc_urlok(u: *u8, ul: i64) -> i64 |
| 477 | func wc_pending(h: *i64, prefix: *u8, urlsout: *u8, maxout: i64, canh: *u8, ncan: i64, canonly: i64) -> i64 |
| 554 | func wc_runseen(runseen: *u8, nrs: *i64, frk: *u8) -> i64 |
| 581 | func wc_harvest(hh: *u8, hlen: i64, base: *u8, blen: i64, h: *i64, w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, discbox: *i64, runseen: *u8, nrs: *i64) -> i64 |
| 684 | func wc_seed_add(h: *i64, w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, urls: *u8, npendbox: *i64, seed: *u8, sl: i64) -> i64 |
| 708 | func wc_hosthash(hostp: *u8, hl: i64) -> i64 |
| 714 | func wc_deadhost_has(dh: *i64, ndh: i64, hh: i64) -> i64 |
| 719 | func wc_deadhost_add(dh: *i64, ndhbox: *i64, hh: i64) -> i64 |
| 731 | func wc_hdkey(hh: i64, out: *u8) -> i64 |
| 746 | func wc_hd_streak(h: *i64, hh: i64) -> i64 |
| 764 | func wc_hd_put(w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, hh: i64, v: i64) -> i64 |
| 795 | func wc_srkey(out: *u8) -> i64 |
| 801 | func wc_sr_cursor(h: *i64) -> i64 |
| 818 | func wc_sr_put(w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, v: i64) -> i64 |
| 842 | func wc_seed_refresh(h: *i64, w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, seedstore: *u8, nseeds: i64, dst: *u8, dstbox: *i64) -> i64 |
| 872 | func wc_is_seed(seedstore: *u8, nseeds: i64, u: *u8, ul: i64) -> i64 |
| 886 | func wc_retire_if_rotten(h: *i64, w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, u: *u8, ul: i64, hh: i64) -> i64 |
| 894 | func wc_spool_path(slot: i64, out: *u8) -> i64 |
| 914 | func wc_fetch_child(url: *u8, store: *TrustStore, spath: *u8) -> i64 |
| 933 | func main(argc: i64, argv: *i64) -> i64 |