code wiki / _hdl_build / nx_crawl_pace.nx

nx_crawl_pace.nx

buildroot/runtime/_hdl_build/nx_crawl_pace.nx

8568 B176 linesdepth 2pulls 2 transitivereach 5 importersview sourcekind tooltopic crawl

docs dependencies structs consts functions

about

nx_crawl_pace.nx -- SOTA polite crawler pacing (operator 2026-07-05: "pacing is a great callout on all the systems... get to state of the art"). Per-host adaptive rate limiting: a min interval between hits to the same host, HONOR Retry-After on 429/503, EXPONENTIAL backoff on repeated throttles (reset on success), and robots crawl-delay override. State persists across invocations (each nx_url_index run is a fresh process) in a fixed hash-table file, so serial crawls stay polite and don't get IP-banned (a ban = LOST reach, not just rudeness -- reddit 429'd us on rapid feed hits). A shared organ any fetcher can call: pace_before(host) before a request, pace_after(host,status,retry_after) after. license_tier: ORIGINAL

dependencies 1 imports · 4 importers

imports: nx_syscalls.nx

imported by: nx_bulk_index.nxnx_crawl_pace_gate.nxnx_url_index.nxnx_web_crawl_step.nx

structs

none

consts

9	const PACE_MAGIC_5381: i64 = 5381
10	const PACE_MAGIC_1024: i64 = 1024
11	const PACE_MAGIC_1000000: i64 = 1000000
14	const PACE_BASE_MS: i64 = 1000 // polite default: >= 1s between requests to the same host
15	const PACE_MAX_BACKOFF_MS: i64 = 300000 // cap any single backoff at 5 min
16	const PACE_WAIT_CAP_MS: i64 = 60000 // never block ONE fetch > 60s (caller may defer beyond)
17	const PACE_SLOTS: i64 = 4096
18	const PACE_REC: i64 = 32 // host_hash(8) \| next_allowed_ms(8) \| consec_throttle(8) \| crawl_delay_ms(8)
19	const PACE_TBL_BYTES: i64 = 131072 // PACE_SLOTS * PACE_REC
20	const PACE_TBL_MODE: i64 = 0x1A4 // 0o644 file MODE for sys_openat_wr (flags O_CREAT\|O_WRONLY\|O_TRUNC

functions

25	func pace_ld(tbl: u8, off: i64) -> i64 { var v: i64=0; var i: i64=0; while i<8 { v = v \| ((tbl[off+i] as i64) << (i8)); i=i+1 } return v } called by 5: pace_slot pace_before pace_after pace_set_crawl_delay main
26	func pace_st(tbl: u8, off: i64, val: i64) -> i64 { var i: i64=0; while i<8 { tbl[off+i] = ((val >> (i8)) & 0xff) as u8; i=i+1 } return 0 } called by 2: pace_after pace_set_crawl_delay
29	func pace_hash(h: *u8, n: i64) -> i64 called by 4: pace_before pace_after pace_set_crawl_delay main
38	func pace_load_tbl() -> *u8 called by 4: pace_before pace_after pace_set_crawl_delay main calls 2: sys_mmap sys_read_file
50	func pace_save_tbl(tbl: *u8) -> i64 called by 2: pace_after pace_set_crawl_delay calls 3: sys_openat_wr sys_write sys_close
63	func pace_slot(tbl: *u8, hh: i64) -> i64 called by 4: pace_before pace_after pace_set_crawl_delay main calls 1: pace_ld
78	func pace_delay_after(base_ms: i64, crawl_delay_ms: i64, consec_in: i64, called by 2: pace_after main
102	func pace_wait_ms(now_ms: i64, next_allowed_ms: i64) -> i64 called by 2: pace_before main
110	func pace_before(host: *u8, host_len: i64) -> i64 called by 3: main main main calls 7: pace_load_tbl pace_hash pace_slot pace_ld pace_wait_ms sys_now_realtime_ms+1
122	func pace_after(host: *u8, host_len: i64, status: i64, retry_after_s: i64) -> i64 called by 4: main main main main calls 9: pace_load_tbl pace_hash pace_slot pace_ld sys_mmap pace_delay_after+3
141	func pace_set_crawl_delay(host: *u8, host_len: i64, cd_ms: i64) -> i64 called by 1: main calls 6: pace_load_tbl pace_hash pace_slot pace_st pace_ld pace_save_tbl
154	func pace_retry_after(resp: *u8, n: i64) -> i64 called by 1: main
176	func main() -> i64 { return 0 }