code wiki / _hdl_build / nx_web_crawl_step.nx
nx_web_crawl_step.nx
buildroot/runtime/_hdl_build/nx_web_crawl_step.nx
about
nx_web_crawl_step.nx -- CRAWL AT SCALE: the frontier-PERSISTED successor to nx_web_ingest's one-shot BFS.
The frontier lives IN the web shard itself (seg_store-native, no side files): key fr:<cid(url)> whose
value is "P<url>" (pending) or "D<url>" (done) -- append-only, ss_hget last-wins flips state. Each run:
1. enumerate pending frontier rows (keys-index walk, dedup, latest state wins)
2. fetch up to <max_pages> of them (sovereign TLS; seq628 pool: WC_POOL_N forked workers on
DISTINCT hosts, parent-enforced WC_FETCH_BUDGET_MS wall clock, dead-host deferral)
3. ingest kept pages (doc:<cid> + url:<cid>, the nx_web_ingest mechanics)
4. bank every NEWLY-DISCOVERED content link as a pending fr: row (filtered: no queries/fragments/
binaries/wiki-meta) and mark fetched urls done (even failures -- a dead url never wedges the loop)
So the crawl RESUMES where it stopped: run it any number of times and the index only grows. A seed url
is only needed to BOOTSTRAP an empty frontier (or inject a new start).
usage: nx_web_crawl_step <max_pages> [seed-url]
license_tier: ORIGINAL
dependencies 16 imports · 2 importers
diagram shows first 10 each side; +6 more imports, +0 more importers in the complete lists below.
imports: nx_heavyio_lib.nxnx_corpus_ingest.nxnx_x509_trust_store.nxnx_trust_store_load_from_certdata.nxnx_https_fetch_follow.nxnx_simhash.nxnx_itoa_lib.nxnx_crawl_pace.nxnx_robots.nxnx_feed_extract.nxnx_js_eval.nxnx_outlink_harvest.nxnx_store_seed_lib.nxnx_block_density.nxnx_crawl_sufficiency.nxnx_antibot.nx
imported by: nx_search_census.nxnx_web_crawl_step_gate.nx
structs
| none |
consts
| 30 | const WC_MAGIC_1024: i64 = 1024 // NX-UNNAMED: mechanical, collision-free placeholder -- this name RESTATES the value and so carries no meaning. Replace it with a domain term that says WHY, or move the number to a config row. |
| 31 | const WC_MAGIC_1089: i64 = 1089 |
| 32 | const WC_MAGIC_4000: i64 = 4000 |
| 36 | const WC_JS_HYDRATE: i64 = 0 |
| 37 | const WC_MAGIC_2000: i64 = 2000 |
| 38 | const WC_MAGIC_2048: i64 = 2048 |
| 39 | const WC_MAGIC_8192: i64 = 8192 |
| 40 | const WC_MAGIC_4194304: i64 = 4194304 |
| 41 | const WC_MAGIC_4096: i64 = 4096 |
| 42 | const WC_MAGIC_8388608: i64 = 8388608 |
| 43 | const WC_MAGIC_1048576: i64 = 1048576 |
| 44 | const WC_MAGIC_5381: i64 = 5381 |
| 47 | const WC_MAXPEND: i64 = 2048 // pending urls pulled per run. MEASURED 2026-08-04: at 256 the pull |
| 65 | const WC_HD_RETIRE: i64 = 3 // R3 host-health: retire (P->D) a skipped row once its host's PERSISTED |
| 68 | const WC_MAXSEEDS: i64 = 256 // seed-file urls held for the retire EXEMPTION + the R4 canonical lane + |
| 74 | const WC_SEED_REFRESH: i64 = 8 // R5: already-done seed urls re-marked pending per run. The persisted |
| 78 | const WC_MAXDISC: i64 = 300 // FLOOR for the per-run discovery budget -- see wc_disc_budget below. |
| 87 | const WC_DISC_CEIL: i64 = WC_MAXPEND |
| 88 | const WC_CANON_RESERVE: i64 = 384 // R4 CANONICAL LANE (2026-08-05): head slots of each run's work list |
| 93 | const WC_MAXCANH: i64 = 96 // distinct canonical HOSTS parsed from the seed file (mirrors WC_MAXSEEDS) |
| 94 | const WC_MAXURL: i64 = 1500 // urls longer than this are skipped (junk/tracking monsters) |
| 95 | const WC_MININDEX: i64 = 500 // min extracted chars to INDEX a page (not to harvest its links). Measured |
| 98 | const WC_HOSTCAP_CANON: i64 = 40 // R4b DEPTH CAP FOR CURATED HOSTS (2026-08-05). WC_HOSTCAP below exists to |
| 105 | const WC_HOSTCAP: i64 = 6 // max pages INDEXED per host per crawl step (diversity). Measured 2026-07-06: |
| 119 | const WC_POOL_N: i64 = 8 // in-flight fetch children per batch. Bounded by spool RAM (8 x 8MiB |
| 122 | const WC_FETCH_BUDGET_MS: i64 = 20000 // per-fetch wall clock. Healthy fetches measure <1s recv with |
| 125 | const WC_POLL_MS: i64 = 100 // parent reap-poll quantum while children are in flight |
| 126 | const WC_US_PER_MS: i64 = 1000 |
| 127 | const WC_WNOHANG: i64 = 1 |
| 128 | const WC_SIGKILL: i64 = 9 |
| 141 | const WC_SPOOL_HDRB: i64 = 24 |
| 153 | const WC_DEDUP_WINDOW: i64 = 128 |
| 158 | const WC_SIMHASH_HAM: i64 = 4 |
| 160 | const WC_SPOOLPATH_B: i64 = 128 |
| 161 | const WC_MODE_RW: i64 = 420 // 0644 spool file mode |
| 162 | const WC_DEADHOSTS: i64 = 128 // dead-host slots per run (frontier pull is 256 urls; hosts <= urls) |
| 163 | const WC_HASH_MASK: i64 = 0x7fffffffffffffff |
| 192 | const WC_FIT_TEXT: i64 = 1 // R6: index the density-fit text (nx_block_density); 0 = the pre-R6 full render |
| 383 | const WC_HOSTBUF: i64 = 64 // == the sys_mmap(64) every caller passes |
| 384 | const WC_HOSTMAX: i64 = WC_HOSTBUF - 1 // DERIVED, authored once: longest host + its NUL |
| 676 | const WC_URLPOL_PLANE: *u8 = "knowledge/store/urlpolicy-" |
| 677 | const WC_QSLOT: i64 = 32 |
| 678 | const WC_QMAXDENY: i64 = 192 |
| 679 | const WC_QPLANECAP: i64 = 65536 |
| 680 | const WC_QDEF_MAXP: i64 = 4 // default param budget for an UNCURATED host |
| 681 | const WC_QDEF_MAXL: i64 = 120 // default max query length in bytes |
| 682 | const WC_QCAN_BOOST: i64 = 4 // curated hosts get this many EXTRA params (trust boost, not a gate) |
| 839 | const WC_WINQ: i64 = WC_HOSTCAP_CANON |
| 1175 | const WC_DM_DIR: *u8 = "knowledge/status/domain_map" |
| 1176 | const WC_DM_HOSTSLOT: i64 = 64 // == wc_canon_hosts' stride; DERIVED from that writer, not guessed |
| 1177 | const WC_DM_MAX_ROWS: i64 = 4096 // rows read per host map. The map APPENDS one run per invocation, |
| 1295 | const WC_OUTLOG: *u8 = "knowledge/status/crawl_outcomes.log" |
| 1601 | const WC_ROBOTS_ON: i64 = 1 |
| 1602 | const WC_ROBOTS_HOSTS: i64 = 256 // distinct hosts whose robots.txt we cache per run |
| 1603 | const WC_ROBOTS_ARENA: i64 = 1048576 // ONE packed arena for all cached bodies (256 x 16KB would be 4MB of holes) |
| 1604 | const WC_ROBOTS_MAX: i64 = 16384 // per-host body cap |
| 1605 | const WC_ROBOTS_FCAP: i64 = 262144 // fetch buffer for robots.txt |
| 1606 | const WC_RB_UALEN: i64 = 8 // len("nishibot") |
| 1611 | const WC_RB_ST_RULES: i64 = 0 // s2.3.1.1 Successful Access -- a body was parsed, follow it |
| 1612 | const WC_RB_ST_ALLOW_ALL: i64 = 1 // s2.3.1.3 Unavailable -- "MAY access any resources" |
| 1613 | const WC_RB_ST_DISALLOW_ALL: i64 = 2 // s2.3.1.4 Unreachable -- "MUST assume complete disallow" |
| 1618 | const WC_HTTP_2XX_LO: i64 = 200 |
| 1619 | const WC_HTTP_2XX_HI: i64 = 299 |
| 1620 | const WC_HTTP_3XX_LO: i64 = 300 |
| 1621 | const WC_HTTP_3XX_HI: i64 = 399 |
| 1622 | const WC_HTTP_4XX_LO: i64 = 400 |
| 1623 | const WC_HTTP_4XX_HI: i64 = 499 |
| 1624 | const WC_HTTP_5XX_LO: i64 = 500 |
| 1625 | const WC_HTTP_5XX_HI: i64 = 599 |
| 1630 | const WC_HTTP_NO_STATUS: i64 = 0 |
| 1637 | const WC_ROBOTS_REDIRECTS: i64 = 5 |
| 1657 | const WC_ROBOTS_CD_HORIZON_S: i64 = 86400 // 24 h, per RFC 9309 s2.4 cache lifetime |
| 1658 | const WC_ROBOTS_CD_MS_PER_S: i64 = 1000 // seconds -> ms for pace_set_crawl_delay |
| 1665 | const WC_RB_DEVIATION_30D: i64 = 0 |
functions
| 193 | func wc_query_set(q: *u8) -> i64 |
| 202 | func wc_url_depth(u: *u8, ul: i64) -> i64 |
| 219 | func wc_url_qhits(u: *u8, ul: i64) -> i64 |
| 234 | func wc_priority(u: *u8, ul: i64) -> i64 { return wc_url_qhits(u, ul) * WC_MAXURL - wc_url_depth(u, ul) } |
| 236 | func wc_rank_window(urls: *u8, n: i64) -> i64 |
| 275 | func wc_pckey(out: *u8) -> i64 |
| 281 | func wc_pc_cursor(h: *i64) -> i64 |
| 294 | func wc_pc_put(w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, v: i64) -> i64 |
| 308 | func wc_pc_next(cur: i64, took: i64, seen: i64) -> i64 |
| 325 | func wc_frontier_remaining(npend: i64, fetched: i64, retired: i64, rbskip: i64, hostcapped: i64, disc: i64) -> i64 |
| 331 | func wc_frontier_left(fetched: i64, retired: i64, rbskip: i64, hostcapped: i64) -> i64 |
| 335 | func wc_pc_take(npend: i64, maxout: i64) -> i64 |
| 343 | func wc_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 } |
| 356 | func wc_num(v: i64) -> i64 { nxi_out(v); return 0 } |
| 357 | func wc_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n } |
| 359 | func wc_resolve(base: *u8, href: *u8, out: *u8) -> i64 |
| 385 | func wc_host(url: *u8, out: *u8) -> i64 |
| 393 | func wc_atoi(s: *u8) -> i64 { var v: i64 = 0; var i: i64 = 0; while s[i] != (0 as u8) { let c: i64 = s[i] as i64; if c >= 48 { if c <= 57 { v = v * 10 + (c - 48) } } i = i + 1 } return v } |
| 394 | func wc_hassub(u: *u8, ul: i64, lit: *u8) -> i64 |
| 407 | func wc_endswith(u: *u8, ul: i64, lit: *u8) -> i64 |
| 420 | func wc_frkey(u: *u8, ul: i64, out: *u8) -> i64 |
| 443 | func wc_outkey(cid: i64, out: *u8) -> i64 { return olh_outkey(cid, out) } |
| 466 | func wc_frstate(h: *i64, u: *u8, ul: i64) -> i64 |
| 538 | func wc_frmark(w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, u: *u8, ul: i64, state: i64) -> i64 |
| 582 | func wc_streq(a: *u8, b: *u8) -> i64 |
| 590 | func wc_canon_hosts(seedstore: *u8, nseeds: i64, out: *u8) -> i64 |
| 626 | func wc_host_str_is_canon(canh: *u8, ncan: i64, hb: *u8, hl: i64) -> i64 |
| 647 | func wc_host_is_canon(canh: *u8, ncan: i64, u: *u8, hb: *u8) -> i64 |
| 687 | func wc_qeq(a: *u8, b: *u8, bl: i64) -> i64 |
| 693 | func wc_qnum(b: *u8, s: i64, e: i64) -> i64 |
| 703 | func wc_load_urlpolicy() -> i64 |
| 746 | func wc_qparam_denied(u: *u8, s: i64, e: i64) -> i64 |
| 767 | func wc_query_ok(u: *u8, ul: i64, canon: i64) -> i64 |
| 803 | func wc_urlok(u: *u8, ul: i64) -> i64 |
| 845 | func wc_winq_admit(tblh: *u8, tblc: *i64, nh: *i64, slots: i64, hb: *u8, hl: i64, quota: i64) -> i64 |
| 877 | func wc_frontier_fair(rows: i64, hosts: i64) -> i64 |
| 883 | func wc_pending(h: *i64, prefix: *u8, urlsout: *u8, maxout: i64, canh: *u8, ncan: i64, canonly: i64) -> i64 |
| 977 | func wc_runseen(runseen: *u8, nrs: *i64, frk: *u8) -> i64 |
| 1008 | func wc_harvest(hh: *u8, hlen: i64, base: *u8, blen: i64, h: *i64, w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, discbox: *i64, runseen: *u8, nrs: *i64) -> i64 |
| 1127 | func wc_seed_add(h: *i64, w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, urls: *u8, npendbox: *i64, seed: *u8, sl: i64) -> i64 |
| 1202 | func wc_dm_seedable(kind: *u8, verdict: *u8) -> i64 |
| 1212 | func wc_dm_seed_host(h: *i64, w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, |
| 1297 | func wc_outopen() -> i64 |
| 1303 | func wc_outn(b: *u8, at: i64, v: i64) -> i64 |
| 1315 | func wc_outcat(b: *u8, at: i64, s: *u8) -> i64 |
| 1321 | func wc_outcome(cls: *u8, status: i64, bytes: i64, url: *u8, ul: i64) -> i64 |
| 1340 | func wc_hosthash(hostp: *u8, hl: i64) -> i64 |
| 1346 | func wc_deadhost_has(dh: *i64, ndh: i64, hh: i64) -> i64 |
| 1351 | func wc_deadhost_add(dh: *i64, ndhbox: *i64, hh: i64) -> i64 |
| 1363 | func wc_hdkey(hh: i64, out: *u8) -> i64 |
| 1377 | func wc_hd_streak(h: *i64, hh: i64) -> i64 |
| 1398 | func wc_hd_put(w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, hh: i64, v: i64) -> i64 |
| 1429 | func wc_srkey(out: *u8) -> i64 |
| 1435 | func wc_sr_cursor(h: *i64) -> i64 |
| 1455 | func wc_sr_put(w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, v: i64) -> i64 |
| 1475 | func wc_seed_refresh(h: *i64, w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, seedstore: *u8, nseeds: i64, dst: *u8, dstbox: *i64) -> i64 |
| 1505 | func wc_is_seed(seedstore: *u8, nseeds: i64, u: *u8, ul: i64) -> i64 |
| 1519 | func wc_retire_if_rotten(h: *i64, w: *i64, prefix: *u8, segidbox: *i64, segsbox: *i64, u: *u8, ul: i64, hh: i64) -> i64 |
| 1527 | func wc_spool_path(slot: i64, out: *u8) -> i64 |
| 1682 | func wc_rb_ua() -> *u8 { return "nishibot" as *u8 } |
| 1684 | func wc_rb_init() -> i64 |
| 1724 | func wc_rb_classify(status: i64, nbytes: i64) -> i64 |
| 1742 | func wc_rb_cd_policy(cd_s: i64) -> i64 |
| 1749 | func wc_rb_status_undefined(status: i64) -> i64 |
| 1755 | func wc_rb_find(hh: i64) -> i64 |
| 1762 | func wc_rb_path(url: *u8, ul: i64, out: *u8, outcap: i64) -> i64 |
| 1787 | func wc_rb_load(host: *u8, hl: i64, hh: i64, store: *TrustStore) -> i64 |
| 1878 | func wc_rb_allowed(host: *u8, hl: i64, hh: i64, url: *u8, ul: i64, store: *TrustStore) -> i64 |
| 1909 | func wc_disc_budget(npend_now: i64) -> i64 |
| 1919 | func wc_disc_cap() -> i64 |
| 1931 | func wc_hostcap_defers(npend_now: i64) -> i64 |
| 1936 | func wc_fetch_child(url: *u8, store: *TrustStore, spath: *u8) -> i64 |
| 1971 | func main(argc: i64, argv: *i64) -> i64 |