nx_crawl_pace.nx source

1// nx_crawl_pace.nx -- SOTA polite crawler pacing (operator 2026-07-05: "pacing is a great callout on all the 2// systems... get to state of the art"). Per-host adaptive rate limiting: a min interval between hits to the 3// same host, HONOR Retry-After on 429/503, EXPONENTIAL backoff on repeated throttles (reset on success), and 4// robots crawl-delay override. State persists across invocations (each nx_url_index run is a fresh process) in 5// a fixed hash-table file, so serial crawls stay polite and don't get IP-banned (a ban = LOST reach, not just 6// rudeness -- reddit 429'd us on rapid feed hits). A shared organ any fetcher can call: pace_before(host) 7// before a request, pace_after(host,status,retry_after) after. license_tier: ORIGINAL 8import "nx_syscalls.nx" 9const PACE_MAGIC_5381: i64 = 5381 10const PACE_MAGIC_1024: i64 = 1024 11const PACE_MAGIC_1000000: i64 = 1000000 12 13// --- policy config (v1 surface; a later rung reads svc-config per Cardinal 11) --- 14const PACE_BASE_MS: i64 = 1000 // polite default: >= 1s between requests to the same host 15const PACE_MAX_BACKOFF_MS: i64 = 300000 // cap any single backoff at 5 min 16const PACE_WAIT_CAP_MS: i64 = 60000 // never block ONE fetch > 60s (caller may defer beyond) 17const PACE_SLOTS: i64 = 4096 18const PACE_REC: i64 = 32 // host_hash(8) | next_allowed_ms(8) | consec_throttle(8) | crawl_delay_ms(8) 19const PACE_TBL_BYTES: i64 = 131072 // PACE_SLOTS * PACE_REC 20const PACE_TBL_MODE: i64 = 0x1A4 // 0o644 file MODE for sys_openat_wr (flags O_CREAT|O_WRONLY|O_TRUNC 21 // are hardcoded inside it). Passing flags here made the file 0o1101 22 // (no owner-write) -> 2nd open EACCES; a real mode fixes re-writes. 23 24// --- little-endian i64 load/store into the table buffer --- 25func pace_ld(tbl: *u8, off: i64) -> i64 { var v: i64=0; var i: i64=0; while i<8 { v = v | ((tbl[off+i] as i64) << (i*8)); i=i+1 } return v } 26func pace_st(tbl: *u8, off: i64, val: i64) -> i64 { var i: i64=0; while i<8 { tbl[off+i] = ((val >> (i*8)) & 0xff) as u8; i=i+1 } return 0 } 27 28// stable positive host hash (djb2, masked) 29func pace_hash(h: *u8, n: i64) -> i64 { 30 var x: i64 = PACE_MAGIC_5381 31 var i: i64 = 0 32 while i < n { x = (((x << 5) + x) + (h[i] as i64)) & 0x7fffffffffffffff; i = i + 1 } 33 if x == 0 { x = 1 } // 0 reserved for "empty slot" 34 return x 35} 36 37// --- persistence: read-modify-write a fixed table file (serial-crawler safe) --- 38func pace_load_tbl() -> *u8 { 39 // ALWAYS return a PRIVATE anon buffer (never the file's own mapping): sys_read_file hands back a shared 40 // file mapping, and pace_save_tbl's O_TRUNC would then wipe the buffer mid-write. Copy in, decouple. 41 let dst: *u8 = sys_mmap(PACE_TBL_BYTES) // anon, zero-filled 42 let lp: *i64 = sys_mmap(16) as *i64 43 let src: *u8 = sys_read_file("knowledge/status/pace_crawl.tbl" as *u8, lp) 44 if (src as i64) != 0 { if lp[0] >= PACE_TBL_BYTES { 45 var i: i64 = 0 46 while i < PACE_TBL_BYTES { dst[i] = src[i]; i = i + 1 } 47 } } 48 return dst 49} 50func pace_save_tbl(tbl: *u8) -> i64 { 51 let fd: i64 = sys_openat_wr("knowledge/status/pace_crawl.tbl" as *u8, PACE_TBL_MODE) 52 if fd < 0 { return 0 - 1 } 53 var w: i64 = 0 // loop: a single write() need not flush 128KB in one go 54 while w < PACE_TBL_BYTES { 55 let k: i64 = sys_write(fd, (tbl as i64 + w) as *u8, PACE_TBL_BYTES - w) 56 if k <= 0 { sys_close(fd); return 0 - 1 } 57 w = w + k 58 } 59 sys_close(fd) 60 return 0 61} 62// find the slot for hh (existing match or first empty), linear probe. -1 if full. 63func pace_slot(tbl: *u8, hh: i64) -> i64 { 64 var s: i64 = hh % PACE_SLOTS 65 var tries: i64 = 0 66 while tries < PACE_SLOTS { 67 let stored: i64 = pace_ld(tbl, s * PACE_REC) 68 if stored == 0 { return s } 69 if stored == hh { return s } 70 s = (s + 1) % PACE_SLOTS 71 tries = tries + 1 72 } 73 return 0 - 1 74} 75 76// --- PURE POLICY (deterministic, gate-tested): post-fetch delay before the host may be hit again --- 77// status 429/503 -> honor Retry-After (s), else exponential base*2^consec (capped); success -> reset to base. 78func pace_delay_after(base_ms: i64, crawl_delay_ms: i64, consec_in: i64, 79 status: i64, retry_after_s: i64, out_consec: *i64) -> i64 { 80 var eff: i64 = base_ms 81 if crawl_delay_ms > eff { eff = crawl_delay_ms } 82 var throttle: i64 = 0 83 if status == 429 { throttle = 1 } 84 if status == 503 { throttle = 1 } 85 if throttle == 1 { 86 out_consec[0] = consec_in + 1 87 var delay: i64 = 0 88 if retry_after_s > 0 { delay = retry_after_s * 1000 } 89 else { 90 var mult: i64 = 1; var k: i64 = 0 91 while k < consec_in { mult = mult * 2; if mult >= PACE_MAGIC_1024 { mult = PACE_MAGIC_1024; k = consec_in } else { k = k + 1 } } 92 delay = eff * mult 93 } 94 if delay < eff { delay = eff } 95 if delay > PACE_MAX_BACKOFF_MS { delay = PACE_MAX_BACKOFF_MS } 96 return delay 97 } 98 out_consec[0] = 0 99 return eff 100} 101// wait (ms) required before hitting the host now, given its next-allowed time. capped. 102func pace_wait_ms(now_ms: i64, next_allowed_ms: i64) -> i64 { 103 var w: i64 = next_allowed_ms - now_ms 104 if w <= 0 { return 0 } 105 if w > PACE_WAIT_CAP_MS { return PACE_WAIT_CAP_MS } 106 return w 107} 108 109// --- top-level: call BEFORE a request to a host (loads state, sleeps the polite interval) --- 110func pace_before(host: *u8, host_len: i64) -> i64 { 111 let tbl: *u8 = pace_load_tbl() 112 let hh: i64 = pace_hash(host, host_len) 113 let s: i64 = pace_slot(tbl, hh) 114 if s < 0 { return 0 } 115 let off: i64 = s * PACE_REC 116 if pace_ld(tbl, off) != hh { return 0 } // no prior state -> no wait 117 let w: i64 = pace_wait_ms(sys_now_realtime_ms(), pace_ld(tbl, off + 8)) 118 if w > 0 { sys_sleep_ms(w) } 119 return w 120} 121// call AFTER the response: update next-allowed + backoff. returns the delay applied (ms). 122func pace_after(host: *u8, host_len: i64, status: i64, retry_after_s: i64) -> i64 { 123 let tbl: *u8 = pace_load_tbl() 124 let hh: i64 = pace_hash(host, host_len) 125 let s: i64 = pace_slot(tbl, hh) 126 if s < 0 { return 0 - 1 } 127 let off: i64 = s * PACE_REC 128 var consec_in: i64 = 0 129 var crawl_delay: i64 = 0 130 if pace_ld(tbl, off) == hh { consec_in = pace_ld(tbl, off + 16); crawl_delay = pace_ld(tbl, off + 24) } 131 let oc: *i64 = sys_mmap(16) as *i64 132 let delay: i64 = pace_delay_after(PACE_BASE_MS, crawl_delay, consec_in, status, retry_after_s, oc) 133 pace_st(tbl, off, hh) 134 pace_st(tbl, off + 8, sys_now_realtime_ms() + delay) 135 pace_st(tbl, off + 16, oc[0]) 136 pace_st(tbl, off + 24, crawl_delay) 137 pace_save_tbl(tbl) 138 return delay 139} 140// robots.txt crawl-delay for a host (ms): recorded so pace_delay_after uses max(base, crawl_delay). 141func pace_set_crawl_delay(host: *u8, host_len: i64, cd_ms: i64) -> i64 { 142 let tbl: *u8 = pace_load_tbl() 143 let hh: i64 = pace_hash(host, host_len) 144 let s: i64 = pace_slot(tbl, hh) 145 if s < 0 { return 0 - 1 } 146 let off: i64 = s * PACE_REC 147 pace_st(tbl, off, hh) 148 if pace_ld(tbl, off + 8) == 0 { pace_st(tbl, off + 8, 0) } 149 pace_st(tbl, off + 24, cd_ms) 150 pace_save_tbl(tbl) 151 return 0 152} 153// parse "Retry-After: <seconds>" from response headers (integer-seconds form). 0 if absent/date-form. 154func pace_retry_after(resp: *u8, n: i64) -> i64 { 155 let key: *u8 = "retry-after:" as *u8 156 var i: i64 = 0 157 while i + 12 < n { 158 var m: i64 = 1; var j: i64 = 0 159 while j < 12 { let c: i64 = resp[i+j] as i64; var lc: i64 = c; if c >= 65 { if c <= 90 { lc = c + 32 } } if lc != (key[j] as i64) { m = 0; j = 12 } else { j = j + 1 } } 160 if m == 1 { 161 var p: i64 = i + 12 162 while p < n { let c: i64 = resp[p] as i64; if c == 32 { p = p + 1 } else { if c == 9 { p = p + 1 } else { p = n + PACE_MAGIC_1000000 } } } 163 var q: i64 = i + 12 164 var st: i64 = 0 165 while st == 0 { if q >= n { st = 1 } else { let c: i64 = resp[q] as i64; if c == 32 { q = q + 1 } else { if c == 9 { q = q + 1 } else { st = 1 } } } } 166 var v: i64 = 0; var got: i64 = 0 167 while q < n { let d: i64 = resp[q] as i64; if d >= 48 { if d <= 57 { v = v*10 + (d - 48); got = 1; q = q + 1 } else { q = n } } else { q = n } } 168 if got == 1 { return v } 169 return 0 170 } 171 i = i + 1 172 } 173 return 0 174} 175 176func main() -> i64 { return 0 }