code wiki / _hdl_build / nx_crawl_pace.nx

nx_crawl_pace.nx

buildroot/runtime/_hdl_build/nx_crawl_pace.nx

27045 B440 linesdepth 2pulls 2 transitivereach 22 importersview sourcekind tooltopic crawl

docs dependencies structs consts functions

about

nx_crawl_pace.nx -- SOTA polite crawler pacing (operator 2026-07-05: "pacing is a great callout on all the systems... get to state of the art"). Per-host adaptive rate limiting: a min interval between hits to the same host, HONOR Retry-After on 429/503, EXPONENTIAL backoff on repeated throttles (R9 2026-08-24: a success HALVES the backoff instead of erasing it -- pace_decay -- so one lucky 200 cannot re-trip a host's limit), and robots crawl-delay override. State persists across invocations (each nx_url_index run is a fresh process) in a fixed hash-table file, so serial crawls stay polite and don't get IP-banned (a ban = LOST reach, not just rudeness -- reddit 429'd us on rapid feed hits). A shared organ any fetcher can call: pace_before(host) before a request, pace_after(host,status,retry_after) after. license_tier: ORIGINAL

dependencies 1 imports · 9 importers

imports: nx_syscalls.nx

imported by: nx_4chan.nxnx_bulk_index.nxnx_crawl_pace_gate.nxnx_domain_map.nxnx_paced_fetch.nxnx_paced_fetch_gate.nxnx_pacetbl_probe.nxnx_url_index.nxnx_web_crawl_step.nx

structs

none

consts

10	const PACE_MAGIC_5381: i64 = 5381
11	const PACE_MAGIC_1024: i64 = 1024
12	const PACE_MAGIC_1000000: i64 = 1000000
15	const PACE_BASE_MS: i64 = 1000 // polite default: >= 1s between requests to the same host
16	const PACE_MAX_BACKOFF_MS: i64 = 300000 // cap any single backoff at 5 min
17	const PACE_WAIT_CAP_MS: i64 = 60000 // never block ONE fetch > 60s (caller may defer beyond)
18	const PACE_SLOTS: i64 = 4096
19	const PACE_REC: i64 = 40 // host_hash(8) \| next_allowed_ms(8) \| consec_throttle(8) \| crawl_delay_ms(8) \| flags(8)
20	const PACE_TBL_BYTES: i64 = 163840 // PACE_SLOTS * PACE_REC
21	const PACE_REC_V1: i64 = 32 // the pre-R12 record; on-disk tables in that layout are MIGRATED, never discarded
22	const PACE_TBL_BYTES_V1: i64 = 131072
23	const PACE_F_OKSEEN: i64 = 1 // flags bit 0: this host has returned a 2xx at least once
28	const PACE_OFF_HASH: i64 = 0
29	const PACE_OFF_NEXT_ALLOWED: i64 = 8
30	const PACE_OFF_CONSEC: i64 = 16
31	const PACE_OFF_CRAWL_DELAY: i64 = 24
32	const PACE_OFF_FLAGS: i64 = 32
33	const PACE_TBL_MODE: i64 = 0x1A4 // 0o644 file MODE for sys_openat_wr (flags O_CREAT\|O_WRONLY\|O_TRUNC

functions

38	func pace_ld(tbl: u8, off: i64) -> i64 { var v: i64=0; var i: i64=0; while i<8 { v = v \| ((tbl[off+i] as i64) << (i8)); i=i+1 } return v } called by 12: pace_slot pace_slot_or_evict pace_reset_host pace_before_tbl pace_should_defer_tbl pace_after_tbl+6
39	func pace_st(tbl: u8, off: i64, val: i64) -> i64 { var i: i64=0; while i<8 { tbl[off+i] = ((val >> (i8)) & 0xff) as u8; i=i+1 } return 0 } called by 4: pace_reset_host pace_after_tbl pace_set_crawl_delay main
42	func pace_hash(h: *u8, n: i64) -> i64 called by 7: pace_reset_host pace_before_tbl pace_should_defer_tbl pace_after_tbl pace_set_crawl_delay main+1
62	func pace_load_tbl_into(dst: *u8) -> i64 called by 3: pace_load_tbl main pf_tbl calls 2: sys_mmap sys_read_file
99	func pace_load_tbl() -> *u8 called by 8: pace_reset_host pace_before pace_should_defer pace_after pace_set_crawl_delay main+2 calls 2: sys_mmap pace_load_tbl_into
109	func pace_ew(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(2, s, n); return 0 } called by 2: pace_en pace_save_tbl calls 1: sys_write
110	func pace_en(v: i64) -> i64 called by 1: pace_save_tbl calls 3: sys_mmap pace_ew sys_write
144	func pace_save_tbl(tbl: *u8) -> i64 called by 4: pace_reset_host pace_after_tbl pace_set_crawl_delay main calls 5: sys_openat_wr pace_ew pace_en sys_write sys_close
172	func pace_slot(tbl: *u8, hh: i64) -> i64 called by 6: pace_slot_or_evict pace_reset_host pace_before_tbl pace_should_defer_tbl main pp_report calls 1: pace_ld
194	func pace_slot_or_evict(tbl: *u8, hh: i64) -> i64 called by 2: pace_after_tbl pace_set_crawl_delay calls 2: pace_slot pace_ld
213	func pace_delay_after(base_ms: i64, crawl_delay_ms: i64, consec_in: i64, called by 1: main calls 1: pace_delay_after2
219	func pace_delay_after2(base_ms: i64, crawl_delay_ms: i64, consec_in: i64, called by 3: pace_delay_after pace_after_tbl main calls 1: pace_decay
258	func pace_decay(consec_in: i64) -> i64 { if consec_in <= 0 { return 0 } return consec_in - 1 } called by 2: pace_delay_after2 main
261	func pace_reset_host(host: *u8, host_len: i64) -> i64 called by 2: main pf_reset calls 6: pace_load_tbl pace_hash pace_slot pace_ld pace_st pace_save_tbl
275	func pace_wait_ms(now_ms: i64, next_allowed_ms: i64) -> i64 called by 3: pace_before_tbl pace_should_defer_tbl main
283	func pace_before(host: *u8, host_len: i64) -> i64 called by 4: fc_fetch main dm_fetch main calls 2: pace_before_tbl pace_load_tbl
291	func pace_before_tbl(tbl: u8, host: u8, host_len: i64) -> i64 called by 3: pace_before pf_before pf_before_burst calls 6: pace_hash pace_slot pace_ld pace_wait_ms sys_now_realtime_ms sys_sleep_ms
330	func pace_should_defer_tbl(tbl: u8, host: u8, host_len: i64) -> i64 called by 4: pace_should_defer main pf_before_burst pf_should_defer calls 5: pace_hash pace_slot pace_ld pace_wait_ms sys_now_realtime_ms
364	func pace_should_defer(host: *u8, host_len: i64) -> i64 called by 1: main calls 2: pace_should_defer_tbl pace_load_tbl
369	func pace_after(host: *u8, host_len: i64, status: i64, retry_after_s: i64) -> i64 called by 4: main main dm_fetch main calls 2: pace_after_tbl pace_load_tbl
374	func pace_after_tbl(tbl: u8, host: u8, host_len: i64, status: i64, retry_after_s: i64) -> i64 called by 2: pace_after pf_after calls 8: pace_hash pace_slot_or_evict pace_ld sys_mmap pace_delay_after2 pace_st+2
405	func pace_set_crawl_delay(host: *u8, host_len: i64, cd_ms: i64) -> i64 called by 1: main calls 6: pace_load_tbl pace_hash pace_slot_or_evict pace_st pace_ld pace_save_tbl
418	func pace_retry_after(resp: *u8, n: i64) -> i64 called by 1: main
440	func main() -> i64 { return 0 }