code wiki / _hdl_build / nx_crawl_pace_gate.nx
nx_crawl_pace_gate.nx source
↩ module page · 84 lines · 5120 B
1// nx_crawl_pace_gate.nx -- deterministic gate for the pacing POLICY (no network): honor-Retry-After,
2// exponential backoff, reset-on-success, crawl-delay override, wait clamp, Retry-After parse, persistence.
3import "nx_syscalls.nx"
4import "nx_crawl_pace.nx"
5import "nx_gate_verdict.nx"
6
7func gw(s: *u8) -> i64 { var k: i64=0; while s[k]!=(0 as u8){k=k+1} sys_write(1,s,k); return 0 }
8func gn(v: i64) -> i64 { if v==0 { sys_write(1,"0" as *u8,1); return 0 } var m: i64=v; if m<0 { sys_write(1,"-" as *u8,1); m=0-m } let t: *u8=sys_mmap(24); var k: i64=0; while m>0 { t[k]=(48+(m%10)) as u8; m=m/10; k=k+1 } let o: *u8=sys_mmap(24); var q: i64=k-1; var x: i64=0; while q>=0 { o[x]=t[q]; x=x+1; q=q-1 } sys_write(1,o,x); return 0 }
9func chk(name: *u8, got: i64, want: i64, p: *i64, t: *i64) -> i64 {
10 t[0]=t[0]+1
11 if got == want { p[0]=p[0]+1; gw(" ok " as *u8) } else { gw(" FAIL " as *u8) }
12 gw(name); gw(" got=" as *u8); gn(got); gw(" want=" as *u8); gn(want); gw("\n" as *u8)
13 return 0
14}
15
16func main() -> i64 {
17 gw("=== NX-CRAWL-PACE policy gate (SOTA semantics) ===\n" as *u8)
18 let p: *i64=sys_mmap(16) as *i64; p[0]=0
19 let t: *i64=sys_mmap(16) as *i64; t[0]=0
20 let oc: *i64=sys_mmap(16) as *i64
21
22 // success -> base interval, consec reset
23 chk("200 success -> base" as *u8, pace_delay_after(1000, 0, 0, 200, 0, oc), 1000, p, t)
24 chk("200 resets consec" as *u8, oc[0], 0, p, t)
25 // 429 no Retry-After -> exponential base*2^consec
26 chk("429 consec0 -> base*1" as *u8, pace_delay_after(1000, 0, 0, 429, 0, oc), 1000, p, t)
27 chk("429 consec0 -> consec=1" as *u8, oc[0], 1, p, t)
28 chk("429 consec1 -> base*2" as *u8, pace_delay_after(1000, 0, 1, 429, 0, oc), 2000, p, t)
29 chk("429 consec3 -> base*8" as *u8, pace_delay_after(1000, 0, 3, 429, 0, oc), 8000, p, t)
30 // 429 with Retry-After honored (seconds -> ms)
31 chk("429 Retry-After=120 -> 120000" as *u8, pace_delay_after(1000, 0, 0, 429, 120, oc), 120000, p, t)
32 // backoff cap
33 chk("429 consec20 -> capped 300000" as *u8, pace_delay_after(1000, 0, 20, 429, 0, oc), 300000, p, t)
34 // 503 also throttles
35 chk("503 consec0 -> base*1" as *u8, pace_delay_after(1000, 0, 0, 503, 0, oc), 1000, p, t)
36 // crawl-delay override (max of base, crawl_delay)
37 chk("crawl_delay 5000 on success" as *u8, pace_delay_after(1000, 5000, 0, 200, 0, oc), 5000, p, t)
38 chk("crawl_delay 5000 on 429" as *u8, pace_delay_after(1000, 5000, 0, 429, 0, oc), 5000, p, t)
39 // wait clamp
40 chk("wait now1000 next3000 -> 2000" as *u8, pace_wait_ms(1000, 3000), 2000, p, t)
41 chk("wait past -> 0" as *u8, pace_wait_ms(5000, 3000), 0, p, t)
42 chk("wait huge -> capped 60000" as *u8, pace_wait_ms(0, 999999999), 60000, p, t)
43 // Retry-After header parse
44 let hdr: *u8 = "HTTP/1.1 429 Too Many Requests\r\nRetry-After: 42\r\nServer: x\r\n\r\n" as *u8
45 var hl: i64=0; while hdr[hl]!=(0 as u8){hl=hl+1}
46 chk("parse Retry-After: 42" as *u8, pace_retry_after(hdr, hl), 42, p, t)
47 let hdr2: *u8 = "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n" as *u8
48 var hl2: i64=0; while hdr2[hl2]!=(0 as u8){hl2=hl2+1}
49 chk("parse no Retry-After -> 0" as *u8, pace_retry_after(hdr2, hl2), 0, p, t)
50
51 // persistence round-trip: set crawl-delay for a host, reload, verify it stuck
52 let host: *u8 = "gate.example.test" as *u8; var hnl: i64=0; while host[hnl]!=(0 as u8){hnl=hnl+1}
53 pace_set_crawl_delay(host, hnl, 7000)
54 let tbl: *u8 = pace_load_tbl()
55 let hh: i64 = pace_hash(host, hnl)
56 let s: i64 = pace_slot(tbl, hh)
57 var cdval: i64 = 0 - 1
58 if s >= 0 { cdval = pace_ld(tbl, s*32 + 24) }
59 chk("persist crawl_delay round-trip" as *u8, cdval, 7000, p, t)
60 // SELF-ISOLATION (2026-08-02): this gate MUTATES a PERSISTENT host table, so without a reset the
61 // consec counter accumulates across runs and the gate passes only on a machine that never ran it
62 // (measured: got=3 want=1 on the 3rd run). A re-runnable gate must establish its own precondition:
63 // a 200 resets consec to 0 by policy, so the following 429 must land on exactly 1 every time.
64 pace_after(host, hnl, 200, 0)
65 // pace_after 429 persists incremented consec (2nd file-write in the process -> guards the EACCES bug)
66 pace_after(host, hnl, 429, 0)
67 let tbl2: *u8 = pace_load_tbl()
68 let s2: i64 = pace_slot(tbl2, hh)
69 var consecval: i64 = 0 - 1
70 if s2 >= 0 { consecval = pace_ld(tbl2, s2*32 + 16) }
71 chk("persist 429 consec=1 (2nd write)" as *u8, consecval, 1, p, t)
72 // and next-allowed got stamped (nonzero)
73 var nextval: i64 = 0
74 if s2 >= 0 { nextval = pace_ld(tbl2, s2*32 + 8) }
75 var nz: i64 = 0; if nextval > 0 { nz = 1 }
76 chk("persist next-allowed stamped" as *u8, nz, 1, p, t)
77
78 gw("=== NX-CRAWL-PACE " as *u8); gn(p[0]); gw("/" as *u8); gn(t[0])
79 if p[0] == t[0] { gw(" GREEN\n" as *u8) } else { gw(" RED\n" as *u8) }
80 let ctr: *i64 = gv_ctr()
81 ctr[0] = p[0]
82 ctr[1] = t[0]
83 return gv_verdict("CRAWL-PACE-GATE" as *u8, ctr, "politeness policy: Retry-After honored, exponential backoff, reset-on-success, crawl-delay persisted" as *u8)
84}