nx_polite_crawl_test.nx source

1// nx_polite_crawl_test.nx -- ACCEPTANCE GATE for the polite crawl governor. Exact-integer KATs + 2// a SIMULATED crawl proving the synergy: spreading across many hosts is BOTH broad AND polite, and 3// a host that pushes back (429) gets backed off, not hammered. 4import "nx_polite_crawl.nx" 5import "nx_syscalls.nx" 6 7func ct_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 8func ct_putn(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)} let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1} sys_write(1,bb,k); return 0 } 9 10func main() -> i64 { 11 var pass: i64 = 0 12 var total: i64 = 0 13 14 // T1: backoff doubles and caps -- base 1000, cap 30000: consec 0..6 -> 1000,2000,4000,8000,16000,30000,30000 15 total = total + 1 16 var t1: i64 = 1 17 if pc_backoff_ms(0, 1000, 30000) != 1000 { t1 = 0 } 18 if pc_backoff_ms(1, 1000, 30000) != 2000 { t1 = 0 } 19 if pc_backoff_ms(3, 1000, 30000) != 8000 { t1 = 0 } 20 if pc_backoff_ms(5, 1000, 30000) != 30000 { t1 = 0 } // 32000 capped to 30000 21 if pc_backoff_ms(9, 1000, 30000) != 30000 { t1 = 0 } 22 if t1 == 1 { pass = pass + 1 } else { ct_puts("T1 FAIL backoff " as *u8); ct_putn(pc_backoff_ms(5,1000,30000)); ct_puts("\n" as *u8) } 23 24 // T2: may_fetch respects min interval AND robots crawl-delay (max of the two) 25 total = total + 1 26 var t2: i64 = 1 27 // last=1000, interval=500, crawl=0 : at now=1400 NO (1000+500=1500), at 1500 YES 28 if pc_may_fetch(1000, 0, 1400, 500, 0) != 0 { t2 = 0 } 29 if pc_may_fetch(1000, 0, 1500, 500, 0) != 1 { t2 = 0 } 30 // crawl-delay 2000 overrides interval 500: at now=1500 NO (need 1000+2000=3000) 31 if pc_may_fetch(1000, 0, 1500, 500, 2000) != 0 { t2 = 0 } 32 if pc_may_fetch(1000, 0, 3000, 500, 2000) != 1 { t2 = 0 } 33 if t2 == 1 { pass = pass + 1 } else { ct_puts("T2 FAIL mayfetch\n" as *u8) } 34 35 // T3: may_fetch blocked while backoff active 36 total = total + 1 37 var t3: i64 = 1 38 if pc_may_fetch(1000, 5000, 4000, 500, 0) != 0 { t3 = 0 } // backoff until 5000, now 4000 -> no 39 if pc_may_fetch(1000, 5000, 5000, 500, 0) != 1 { t3 = 0 } // now 5000 -> ok 40 if t3 == 1 { pass = pass + 1 } else { ct_puts("T3 FAIL backoff-gate\n" as *u8) } 41 42 // T4: on_response -- 429 sets exponential backoff + increments consec; 200 resets 43 total = total + 1 44 let bu: *i64 = sys_mmap(64) as *i64 45 let cs: *i64 = sys_mmap(64) as *i64 46 bu[0] = 0; cs[0] = 0 47 pc_on_response(bu, cs, 0, PC_RETRY, 10000, 1000, 30000) // 1st 429: consec1 -> backoff 2000 -> until 12000 48 var t4: i64 = 1 49 if cs[0] != 1 { t4 = 0 } 50 if bu[0] != 12000 { t4 = 0 } 51 pc_on_response(bu, cs, 0, PC_RETRY, 12000, 1000, 30000) // 2nd 429: consec2 -> backoff 4000 -> until 16000 52 if cs[0] != 2 { t4 = 0 } 53 if bu[0] != 16000 { t4 = 0 } 54 pc_on_response(bu, cs, 0, PC_OK, 16000, 1000, 30000) // 200: reset 55 if cs[0] != 0 { t4 = 0 } 56 if bu[0] != 0 { t4 = 0 } 57 if t4 == 1 { pass = pass + 1 } else { ct_puts("T4 FAIL onresp bu=" as *u8); ct_putn(bu[0]); ct_puts(" cs=" as *u8); ct_putn(cs[0]); ct_puts("\n" as *u8) } 58 59 // T5: 403 hard-block -> effectively-forever backoff (stop hitting it) 60 total = total + 1 61 let bu2: *i64 = sys_mmap(32) as *i64; let cs2: *i64 = sys_mmap(32) as *i64; bu2[0]=0; cs2[0]=0 62 pc_on_response(bu2, cs2, 0, PC_BLOCKED, 1000, 1000, 30000) 63 if pc_may_fetch(1000, bu2[0], 999999, 500, 0) == 0 { pass = pass + 1 } else { ct_puts("T5 FAIL block\n" as *u8) } 64 65 // T6: scheduler picks the LONGEST-IDLE eligible host (spreads load) 66 total = total + 1 67 let last: *i64 = sys_mmap(64) as *i64 68 let bk: *i64 = sys_mmap(64) as *i64 69 let cd: *i64 = sys_mmap(64) as *i64 70 // 4 hosts; last-times 5000,1000,3000,2000 ; all backoff 0, no crawl delay ; interval 500, now 10000 71 last[0]=5000; last[1]=1000; last[2]=3000; last[3]=2000 72 bk[0]=0; bk[1]=0; bk[2]=0; bk[3]=0; cd[0]=0; cd[1]=0; cd[2]=0; cd[3]=0 73 // host 1 idle longest (last=1000) -> picked 74 if pc_next_host(last, bk, cd, 4, 10000, 500) == 1 { pass = pass + 1 } else { ct_puts("T6 FAIL sched=" as *u8); ct_putn(pc_next_host(last,bk,cd,4,10000,500)); ct_puts("\n" as *u8) } 75 76 // T7: scheduler returns -1 when NO host is eligible (all just fetched) -> caller waits, never busy-hits 77 total = total + 1 78 last[0]=9900; last[1]=9900; last[2]=9900; last[3]=9900 // all fetched at 9900, interval 500, now 10000 79 if pc_next_host(last, bk, cd, 4, 10000, 500) == (0 - 1) { pass = pass + 1 } else { ct_puts("T7 FAIL noneligible\n" as *u8) } 80 81 // T8: breadth metric -- 100 distinct hosts over 120 requests = 833 permil (broad+polite) 82 total = total + 1 83 var t8: i64 = 1 84 if pc_breadth_permil(100, 120) != 833 { t8 = 0 } 85 if pc_breadth_permil(2, 200) != 10 { t8 = 0 } // 2 hosts 200 reqs = hammering = low breadth 86 if t8 == 1 { pass = pass + 1 } else { ct_puts("T8 FAIL breadth\n" as *u8) } 87 88 // T9: THE RESEARCH GATE -- broad (100 hosts), polite (<=2 each), high breadth -> PASS; 89 // small-sampled OR hammering -> FAIL 90 total = total + 1 91 let rph: *i64 = sys_mmap(1024) as *i64 92 var i: i64 = 0 93 while i < 100 { rph[i] = 1; i = i + 1 } // 100 hosts, 1 request each = ideal 94 var t9: i64 = 1 95 // broad+polite: 100 hosts, 100 reqs, floor 2, min 50 hosts, min breadth 500 permil -> PASS 96 if pc_research_gate(100, 100, rph, 100, 50, 2, 500) != 1 { t9 = 0 } 97 // make host 0 hammered (50 reqs) -> not polite -> FAIL 98 rph[0] = 50 99 if pc_research_gate(100, 149, rph, 100, 50, 2, 500) != 0 { t9 = 0 } 100 // small-sampled: only 5 hosts -> not broad -> FAIL 101 rph[0] = 1 102 if pc_research_gate(5, 5, rph, 5, 50, 2, 500) != 0 { t9 = 0 } 103 if t9 == 1 { pass = pass + 1 } else { ct_puts("T9 FAIL gate\n" as *u8) } 104 105 // T10: SIMULATED CRAWL -- 60 hosts, round-robin, each fetched exactly twice, paced; prove 106 // it stays polite (max 2/host) AND broad (60 hosts) over the run. 107 total = total + 1 108 let L: *i64 = sys_mmap(8192) as *i64 109 let B: *i64 = sys_mmap(8192) as *i64 110 let C: *i64 = sys_mmap(8192) as *i64 111 let R: *i64 = sys_mmap(8192) as *i64 112 let NH: i64 = 60 113 i = 0 114 while i < NH { L[i] = 0 - 100000; B[i] = 0; C[i] = 0; R[i] = 0; i = i + 1 } 115 var now: i64 = 0 116 var issued: i64 = 0 117 var guard: i64 = 0 118 while issued < 120 { 119 if guard > 100000 { issued = 120 } // safety: never spin 120 let h: i64 = pc_next_host(L, B, C, NH, now, 1000) // 1s min interval per host 121 if h < 0 { now = now + 250 } // nobody ready -> advance the clock (wait) 122 else { 123 pc_record(L, h, now) 124 R[h] = R[h] + 1 125 pc_on_response(B, C, h, PC_OK, now, 1000, 30000) 126 issued = issued + 1 127 now = now + 50 // 50ms between any two requests globally 128 } 129 guard = guard + 1 130 } 131 var t10: i64 = 1 132 if pc_is_polite(R, NH, 2) != 1 { t10 = 0 } // no host hit > 2x 133 if pc_is_broad(NH, 50) != 1 { t10 = 0 } // 60 >= 50 hosts = broad 134 if issued != 120 { t10 = 0 } 135 if t10 == 1 { pass = pass + 1 } else { ct_puts("T10 FAIL crawl polite=" as *u8); ct_putn(pc_is_polite(R,NH,2)); ct_puts(" issued=" as *u8); ct_putn(issued); ct_puts("\n" as *u8) } 136 137 ct_puts("POLITE-CRAWL " as *u8); ct_putn(pass); ct_puts("/" as *u8); ct_putn(total); ct_puts("\n" as *u8) 138 if pass == total { ct_puts("POLITE-CRAWL ALL-PASS\n" as *u8); sys_exit(0) } 139 sys_exit(1) 140 return 1 141}