nx_polite_crawl.nx source

1// nx_polite_crawl.nx -- the POLITE CRAWL GOVERNOR (operator: researcher must "not be small sampled 2// but doesnt trigger bot blowback"). The key synergy this encodes: BROAD coverage and POLITENESS are 3// the SAME move -- you reach hundreds of sources by spreading THIN across many hosts (1-2 hits each, 4// paced), never hammering one. The anti-pattern that triggers bot-blocking is small-sample-but- 5// aggressive (one host 100x). The governor enforces broad-and-gentle: 6// * per-host MIN INTERVAL (>= N ms between requests to the SAME host) + robots crawl-delay 7// * EXPONENTIAL BACKOFF on 429/503 (capped) -- back off the moment a host pushes back 8// * round-robin LONGEST-IDLE scheduler -- always fetch the host idle longest = spreads load 9// * BREADTH metric (distinct hosts / total requests) -- high breadth = both large-sample AND polite 10// Per-host state in parallel i64 arrays (struct-free): last_ms[], backoff_until_ms[], consec_429[]. 11// All integer; deterministic given a clock. Composes nx_research_synth (corroborate what's fetched) 12// + nx_resource_governor (host CPU politeness). LAWS: struct-free, integer-only. license_tier: ORIGINAL 13import "nx_syscalls.nx" 14const PC_MAGIC_1000000000: i64 = 1000000000 15 16const PC_OK: i64 = 200 17const PC_RETRY: i64 = 429 // rate-limited (or 503) -> back off 18const PC_BLOCKED: i64 = 403 // hard block -> stop hitting this host entirely 19 20// exponential backoff in ms: base * 2^consec, capped. consec=0 -> base, 1 -> 2*base, ... 21func pc_backoff_ms(consec: i64, base_ms: i64, cap_ms: i64) -> i64 { 22 var b: i64 = base_ms 23 var i: i64 = 0 24 while i < consec { b = b * 2; if b >= cap_ms { b = cap_ms; i = consec } i = i + 1 } 25 if b > cap_ms { b = cap_ms } 26 return b 27} 28 29// may we fetch this host now? requires BOTH: min-interval since last request, AND past any backoff. 30// crawl_delay_ms is the host's robots.txt Crawl-delay (0 if none) -- the effective interval is the max. 31func pc_may_fetch(last_ms: i64, backoff_until_ms: i64, now_ms: i64, min_interval_ms: i64, crawl_delay_ms: i64) -> i64 { 32 var iv: i64 = min_interval_ms 33 if crawl_delay_ms > iv { iv = crawl_delay_ms } 34 if now_ms < last_ms + iv { return 0 } 35 if now_ms < backoff_until_ms { return 0 } 36 return 1 37} 38 39// record a fetch was issued to host idx (update its last-time) 40func pc_record(last_ms: *i64, idx: i64, now_ms: i64) -> i64 { last_ms[idx] = now_ms; return 0 } 41 42// apply a response: 200 resets backoff; 429/503 -> exponential backoff; 403 -> permanent (huge backoff). 43func pc_on_response(backoff_until_ms: *i64, consec_429: *i64, idx: i64, status: i64, now_ms: i64, base_ms: i64, cap_ms: i64) -> i64 { 44 if status == PC_OK { consec_429[idx] = 0; backoff_until_ms[idx] = 0; return PC_OK } 45 if status == PC_BLOCKED { backoff_until_ms[idx] = now_ms + PC_MAGIC_1000000000; return PC_BLOCKED } // ~stop forever 46 // PC_RETRY / 503 / anything else transient 47 consec_429[idx] = consec_429[idx] + 1 48 backoff_until_ms[idx] = now_ms + pc_backoff_ms(consec_429[idx], base_ms, cap_ms) 49 return PC_RETRY 50} 51 52// the SCHEDULER: among all eligible hosts (may_fetch==1), return the index idle LONGEST (smallest 53// last_ms) so load spreads evenly = broad AND gentle. Returns -1 if NONE is eligible right now 54// (caller should wait/sleep, never busy-hit). crawl[] = per-host robots crawl-delay (0 if none). 55func pc_next_host(last_ms: *i64, backoff_until_ms: *i64, crawl: *i64, n: i64, now_ms: i64, min_interval_ms: i64) -> i64 { 56 var best: i64 = 0 - 1 57 var best_last: i64 = 0 58 var i: i64 = 0 59 while i < n { 60 if pc_may_fetch(last_ms[i], backoff_until_ms[i], now_ms, min_interval_ms, crawl[i]) == 1 { 61 if best < 0 { best = i; best_last = last_ms[i] } 62 else { if last_ms[i] < best_last { best = i; best_last = last_ms[i] } } 63 } 64 i = i + 1 65 } 66 return best 67} 68 69// coverage BREADTH in per-mil: distinct hosts reached / total requests. HIGH = large-sample-and-polite 70// (each host hit few times). LOW = hammering few hosts = blowback risk. 71func pc_breadth_permil(distinct_hosts: i64, total_requests: i64) -> i64 { 72 if total_requests <= 0 { return 0 } 73 return distinct_hosts * 1000 / total_requests 74} 75 76// politeness verdict: every host stayed at or under the per-host request floor (no host hammered). 77// returns 1 if polite, 0 if any host exceeded the floor. 78func pc_is_polite(req_per_host: *i64, n: i64, floor_per_host: i64) -> i64 { 79 var i: i64 = 0 80 while i < n { if req_per_host[i] > floor_per_host { return 0 } i = i + 1 } 81 return 1 82} 83 84// is this sampling BROAD ENOUGH (not small-sampled)? distinct hosts >= a minimum breadth target. 85func pc_is_broad(distinct_hosts: i64, min_hosts: i64) -> i64 { 86 if distinct_hosts >= min_hosts { return 1 } 87 return 0 88} 89 90// the combined gate the researcher must pass: BROAD (many distinct hosts) AND POLITE (no host 91// hammered) AND high BREADTH ratio. Returns 1 only if all three hold. 92func pc_research_gate(distinct_hosts: i64, total_requests: i64, req_per_host: *i64, n: i64, 93 min_hosts: i64, floor_per_host: i64, min_breadth_permil: i64) -> i64 { 94 if pc_is_broad(distinct_hosts, min_hosts) == 0 { return 0 } 95 if pc_is_polite(req_per_host, n, floor_per_host) == 0 { return 0 } 96 if pc_breadth_permil(distinct_hosts, total_requests) < min_breadth_permil { return 0 } 97 return 1 98}