code wiki / _hdl_build / nx_web_crawl_step_gate.nx

nx_web_crawl_step_gate.nx source

↩ module page · 916 lines · 62323 B

1// nx_web_crawl_step_gate.nx -- network-free KATs for the frontier-persisted crawler's STORE mechanics: 2// the url filter, fr: row write/enumerate, the P->D state flip (append-only last-wins), cross-segment 3// dedupe, and unknown-state. The fetch/ingest spine is nx_web_ingest's, already proven on the real web. 4// Runs against a /tmp shard (wiped per boot). license_tier: ORIGINAL 5import "nx_web_crawl_step.nx" 6import "nx_gate_verdict.nx" 7 8func gw_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 } 9func gw_num(v: i64) -> i64 { 10 let bb: *u8 = sys_mmap(28); var m: i64 = v 11 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m } 12 let t: *u8 = sys_mmap(28); var k: i64 = 0 13 if m == 0 { t[0] = 48 as u8; k = 1 } 14 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 } 15 var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 } 16 sys_write(1, bb, k); return 0 17} 18// Scratch for the T22 identity teeth: one rendered identity block (User-Agent + Accept-Language). 19// Named for its PURPOSE, not its value -- the ratchet refused the bare 1024 and was right to. 20const WCID_UABUF: i64 = 1024 21// The frontier population this gate's partition fixture is built on -- see its use below. 22// This is the frontier as measured BEFORE the R16/R17 work, recovered from six consecutive runs. 23const WCID_FIXTURE_FRONTIER_PENDING: i64 = 1873 24// The frontier as measured AFTER it drained, on the run whose hostcap_retired=227 motivated R16. 25// TWO DIFFERENT MEASUREMENTS AT TWO DIFFERENT TIMES, each named for which one it is. Conflating them 26// is exactly the error T24a/T24b caught: I wrote those teeth against 313 and passed them the 1873 27// constant, and 2048-1873=175 correctly floors to 300, so the teeth failed on a sound function. 28// A FIXTURE NAMED FOR ITS ROLE RATHER THAN ITS MOMENT INVITES THAT SUBSTITUTION. 29const WCID_FIXTURE_FRONTIER_DRAINED: i64 = 313 30 31// substring helpers for the T22 identity teeth. Named wcid_* so they cannot collide with a sibling. 32func wcid_count(hay: *u8, hn: i64, ndl: *u8) -> i64 { 33 var nl: i64 = 0 34 while ndl[nl] != (0 as u8) { nl = nl + 1 } 35 if nl == 0 { return 0 } 36 var c: i64 = 0 37 var i: i64 = 0 38 while i + nl <= hn { 39 var m: i64 = 1 40 var j: i64 = 0 41 while j < nl { if hay[i + j] != ndl[j] { m = 0; j = nl } else { j = j + 1 } } 42 if m == 1 { c = c + 1 } 43 i = i + 1 44 } 45 return c 46} 47func wcid_has(hay: *u8, hn: i64, ndl: *u8) -> i64 { if wcid_count(hay, hn, ndl) > 0 { return 1 } return 0 } 48 49func gw_check(name: *u8, cond: i64, pass: *i64, total: *i64) -> i64 { 50 total[0] = total[0] + 1 51 gw_puts(name) 52 if cond == 1 { pass[0] = pass[0] + 1; gw_puts(" PASS\n" as *u8) } else { gw_puts(" FAIL\n" as *u8) } 53 return 0 54} 55func gw_ok(u: *u8) -> i64 { return wc_urlok(u, wc_len(u)) } 56 57// --- R11 robots teeth: seed the per-run cache directly, so verdicts need NO network + NO TrustStore. 58func t_rb_seed_state(hh: i64, body: *u8, bl: i64, st: i64) -> i64 { 59 let idx: i64 = wc_rb_n_g 60 var z: i64 = 0 61 while z < bl { wc_rb_arena_g[wc_rb_used_g + z] = body[z]; z = z + 1 } 62 wc_rb_hash_g[idx] = hh 63 wc_rb_off_g[idx] = wc_rb_used_g 64 wc_rb_len_g[idx] = bl 65 wc_rb_state_g[idx] = st 66 wc_rb_used_g = wc_rb_used_g + bl 67 wc_rb_n_g = idx + 1 68 return idx 69} 70// The pre-existing seeder, preserved for every tooth already written against it. Its state is DERIVED 71// by asking the SHIPPING classifier what a successful fetch of this body would yield, rather than by 72// hand-writing a state here: a fixture that restates the rule under test can never disagree with it, 73// and a fixture that cannot disagree is not evidence. 74func t_rb_seed(hh: i64, body: *u8, bl: i64) -> i64 { 75 return t_rb_seed_state(hh, body, bl, wc_rb_classify(WC_HTTP_2XX_LO, bl)) 76} 77func t_rb_streq(a: *u8, b: *u8) -> i64 { 78 var i: i64 = 0 79 while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 } 80 if b[i] != (0 as u8) { return 0 } 81 return 1 82} 83 84func main() -> i64 { 85 gw_puts("=== nx_web_crawl_step gate (frontier store mechanics, network-free) ===\n" as *u8) 86 let pass: *i64 = sys_mmap(16) as *i64; pass[0] = 0 87 let total: *i64 = sys_mmap(16) as *i64; total[0] = 0 88 89 // R10: establish the query policy ONCE, up front, so EVERY tooth below runs under a known, 90 // self-contained deny list rather than whatever the live urlpolicy- plane happens to hold. 91 // 'action' earns its place next to 'utm_source': ?action=edit|raw|history is a MediaWiki ACTION 92 // ENDPOINT, not a document. R8 killed it for free via the blanket '?' reject; once queries are 93 // judged by shape, that free kill has to be paid for explicitly -- and T1 is what noticed. 94 // ★REMOVING A BLANKET RULE BILLS YOU FOR EVERY CASE IT WAS SILENTLY COVERING. 95 let gqd: *u8 = sys_mmap(32 * 8) 96 var gqi: i64 = 0 97 let gqd0: *u8 = "utm_source" as *u8 98 while gqd0[gqi] != (0 as u8) { gqd[gqi] = gqd0[gqi]; gqi = gqi + 1 } 99 gqd[gqi] = 0 as u8 100 let gqd1: *u8 = "action" as *u8 101 var gqj: i64 = 0 102 while gqd1[gqj] != (0 as u8) { gqd[32 + gqj] = gqd1[gqj]; gqj = gqj + 1 } 103 gqd[32 + gqj] = 0 as u8 104 wc_qdeny_g = gqd 105 wc_nqdeny_g = 2 106 107 // T1 url filter: keeps article urls, kills queries/fragments/meta/binaries 108 var f1: i64 = 1 109 if gw_ok("https://en.wikipedia.org/wiki/Chess" as *u8) != 1 { f1 = 0 } 110 if gw_ok("https://example.org/a/deep/page" as *u8) != 1 { f1 = 0 } 111 if gw_ok("https://en.wikipedia.org/w/index.php?action=edit" as *u8) != 0 { f1 = 0 } 112 if gw_ok("https://en.wikipedia.org/wiki/Chess#History" as *u8) != 0 { f1 = 0 } 113 if gw_ok("https://en.wikipedia.org/wiki/Special:Random" as *u8) != 0 { f1 = 0 } 114 if gw_ok("https://en.wikipedia.org/wiki/File:Board.png" as *u8) != 0 { f1 = 0 } 115 if gw_ok("https://x.org/logo.png" as *u8) != 0 { f1 = 0 } 116 if gw_ok("https://x.org/style.css" as *u8) != 0 { f1 = 0 } 117 if gw_ok("mailto:someone@example.org" as *u8) != 0 { f1 = 0 } 118 if gw_ok("/relative/only" as *u8) != 0 { f1 = 0 } 119 if gw_ok("https://ro.wikipedia.org/wiki/Literatura" as *u8) != 0 { f1 = 0 } // interwiki lang edition 120 if gw_ok("https://en.wikipedia.org/wiki/Chess_opening" as *u8) != 1 { f1 = 0 } // en stays 121 if gw_ok("https://commons.wikimedia.org/wiki/X" as *u8) != 0 { f1 = 0 } 122 // API / feed endpoints = real bytes but not content -> rejected 123 if gw_ok("https://creativecommons.org/wp-json/" as *u8) != 0 { f1 = 0 } 124 if gw_ok("https://site.com/api/data.json" as *u8) != 0 { f1 = 0 } 125 if gw_ok("https://blog.com/feed" as *u8) != 0 { f1 = 0 } 126 if gw_ok("https://blog.com/comments/feed/" as *u8) != 0 { f1 = 0 } 127 if gw_ok("https://site.com/articles/a-real-story" as *u8) != 1 { f1 = 0 } // content still passes 128 // anti-slop listing classes (2026-08-04): tag/category link farms refused at admission 129 if gw_ok("https://healthquerys.com/tag/puriva-france/" as *u8) != 0 { f1 = 0 } 130 if gw_ok("https://blog.com/category/wellness/" as *u8) != 0 { f1 = 0 } 131 if gw_ok("https://blog.com/posts/2024/tagged-story" as *u8) != 1 { f1 = 0 } // 'tag' inside a word still passes 132 gw_check("T1 url filter (21 KATs incl. interwiki + API/feed + tag/category farms)" as *u8, f1, pass, total) 133 134 // FIXTURE SELF-CLEAN (2026-08-04, debt 1785882035): /tmp persists across runs on the NAS, so a 135 // previous run's segments made the count teeth (T2-T4) go 4/7 RED on any second invocation -- a 136 // gate that is only green on a fresh boot is a boot-witness, not a gate. Unlink the whole fixture. 137 sys_unlinkat("/tmp/crawlgate-manifest.txt" as *u8) 138 let cleanbuf: *u8 = sys_mmap(96) 139 let cdig: *u8 = sys_mmap(28) 140 var ci0: i64 = 1 141 while ci0 <= 32 { 142 var ce: i64 = 0 143 while ce < 5 { 144 var ext: *u8 = ".docs" as *u8 145 if ce == 1 { ext = ".idx" as *u8 } 146 if ce == 2 { ext = ".pos" as *u8 } 147 if ce == 3 { ext = ".imp" as *u8 } 148 if ce == 4 { ext = ".terms" as *u8 } 149 var co: i64 = 0 150 let cpre: *u8 = "/tmp/crawlgate-seg-" as *u8 151 while cpre[co] != (0 as u8) { cleanbuf[co] = cpre[co]; co = co + 1 } 152 var cm: i64 = ci0 153 var ck: i64 = 0 154 while cm > 0 { cdig[ck] = (48 + (cm % 10)) as u8; cm = cm / 10; ck = ck + 1 } 155 var cj: i64 = 0 156 while cj < ck { cleanbuf[co] = cdig[ck - 1 - cj]; co = co + 1; cj = cj + 1 } 157 var cx: i64 = 0 158 while ext[cx] != (0 as u8) { cleanbuf[co] = ext[cx]; co = co + 1; cx = cx + 1 } 159 cleanbuf[co] = 0 as u8 160 sys_unlinkat(cleanbuf) 161 ce = ce + 1 162 } 163 ci0 = ci0 + 1 164 } 165 166 // build a tiny /tmp frontier shard 167 let prefix: *u8 = "/tmp/crawlgate-" as *u8 168 let segidbox: *i64 = sys_mmap(16) as *i64; segidbox[0] = 1 169 let segsbox: *i64 = sys_mmap(16) as *i64; segsbox[0] = 0 170 let w: *i64 = ss_begin() 171 let u1: *u8 = "https://a.example/one" as *u8 172 let u2: *u8 = "https://a.example/two" as *u8 173 let u3: *u8 = "https://a.example/three" as *u8 174 wc_frmark(w, prefix, segidbox, segsbox, u1, wc_len(u1), 1) 175 wc_frmark(w, prefix, segidbox, segsbox, u2, wc_len(u2), 1) 176 wc_frmark(w, prefix, segidbox, segsbox, u3, wc_len(u3), 2) // three = already done 177 ss_commit(prefix, w, segidbox[0]) 178 179 // T2 enumerate: exactly the two P urls 180 let h1: *i64 = ss_open(prefix) 181 let urls: *u8 = sys_mmap(WC_MAGIC_2048 * 16) 182 let canh0: *u8 = sys_mmap(64 * 8) // R4: never read when canonly=0 -- T2/T3/T4 keep the pre-R4 contract 183 let n2: i64 = wc_pending(h1, prefix, urls, 16, canh0, 0, 0) 184 gw_check("T2 pending enumeration -> 2 (P rows only)" as *u8, (n2 == 2) as i64, pass, total) 185 186 // T3 state flip: mark one P->D in a NEW segment; enumeration drops to 1 (last-wins across segments) 187 segidbox[0] = segidbox[0] + 1 188 let w2: *i64 = ss_begin() 189 wc_frmark(w2, prefix, segidbox, segsbox, u1, wc_len(u1), 2) 190 ss_commit(prefix, w2, segidbox[0]) 191 let h2: *i64 = ss_open(prefix) 192 let n3: i64 = wc_pending(h2, prefix, urls, 16, canh0, 0, 0) 193 var t3: i64 = 0 194 if n3 == 1 { 195 // the survivor must be u2 196 var eq: i64 = 1 197 var x: i64 = 0 198 var go: i64 = 1 199 while go == 1 { if urls[x] != u2[x] { eq = 0; go = 0 } else { if u2[x] == (0 as u8) { go = 0 } else { x = x + 1 } } } 200 if eq == 1 { t3 = 1 } 201 } 202 gw_check("T3 P->D flip (append-only, last-wins) -> 1 pending = /two" as *u8, t3, pass, total) 203 204 // T4 duplicate P rows across segments dedupe to one pending 205 segidbox[0] = segidbox[0] + 1 206 let w3: *i64 = ss_begin() 207 wc_frmark(w3, prefix, segidbox, segsbox, u2, wc_len(u2), 1) // re-bank the SAME pending url 208 ss_commit(prefix, w3, segidbox[0]) 209 let h3: *i64 = ss_open(prefix) 210 let n4: i64 = wc_pending(h3, prefix, urls, 16, canh0, 0, 0) 211 gw_check("T4 dup P rows across segments dedupe -> still 1" as *u8, (n4 == 1) as i64, pass, total) 212 213 // T5 unknown state for a never-seen url 214 let s5: i64 = wc_frstate(h3, "https://a.example/never" as *u8, 23) 215 gw_check("T5 unknown url -> state 0" as *u8, (s5 == 0) as i64, pass, total) 216 // T5b known states read back 217 var t5b: i64 = 0 218 if wc_frstate(h3, u2, wc_len(u2)) == 1 { if wc_frstate(h3, u1, wc_len(u1)) == 2 { t5b = 1 } } 219 gw_check("T5b states read back (two=P, one=D)" as *u8, t5b, pass, total) 220 221 // T6 (P1 LINK GRAPH): wc_harvest captures ALL of a page's outlinks as out:<cid(base)> = packed i64 target 222 // cid(url)s -- the edge list PageRank runs on. Distinguishing by construction: before this change the 223 // crawler stored NO edges, so out:<cid> is absent and t6=0. 224 let t1u: *u8 = "https://tgt1.example/story-alpha" as *u8 225 let t2u: *u8 = "https://tgt2.example/story-beta" as *u8 226 let hbase: *u8 = "https://src.example/page-x" as *u8 227 let html6: *u8 = "<html><body><a href='https://tgt1.example/story-alpha'>A</a> <a href='https://tgt2.example/story-beta'>B</a></body></html>" as *u8 228 let hdisc: *i64 = sys_mmap(16) as *i64; hdisc[0] = 0 229 let hrs: *u8 = sys_mmap(256 * 64); let hnrs: *i64 = sys_mmap(16) as *i64; hnrs[0] = 0 230 segidbox[0] = segidbox[0] + 1 231 let w6: *i64 = ss_begin() 232 wc_harvest(html6, wc_len(html6), hbase, wc_len(hbase), 0 as *i64, w6, prefix, segidbox, segsbox, hdisc, hrs, hnrs) 233 ss_commit(prefix, w6, segidbox[0]) 234 let h6: *i64 = ss_open(prefix) 235 let okey6: *u8 = sys_mmap(64) 236 wc_outkey(ci_hash(hbase, wc_len(hbase)), okey6) 237 let op6: *i64 = sys_mmap(16) as *i64 238 let ol6: *i64 = sys_mmap(16) as *i64 239 var t6: i64 = 0 240 if ss_hget(h6, okey6, op6, ol6) == 1 { if ol6[0] == 16 { 241 let ev: *i64 = op6[0] as *i64 242 if ev[0] == ci_hash(t1u, wc_len(t1u)) { if ev[1] == ci_hash(t2u, wc_len(t2u)) { t6 = 1 } } 243 } } 244 gw_check("T6 P1 outlink capture (out:<cid(base)> = [cid(tgt1),cid(tgt2)], ordered)" as *u8, t6, pass, total) 245 246 // T7 (R3 host-health): hd:<hosthash> streak round-trip; an unknown host reads 0 247 segidbox[0] = segidbox[0] + 1 248 let w7: *i64 = ss_begin() 249 let hhA: i64 = wc_hosthash("dead.example" as *u8, 12) 250 wc_hd_put(w7, prefix, segidbox, segsbox, hhA, 3) 251 ss_commit(prefix, w7, segidbox[0]) 252 let h7: *i64 = ss_open(prefix) 253 var t7: i64 = 0 254 if wc_hd_streak(h7, hhA) == 3 { if wc_hd_streak(h7, wc_hosthash("live.example" as *u8, 12)) == 0 { t7 = 1 } } 255 gw_check("T7 hd: streak round-trip (3) + unknown-host 0" as *u8, t7, pass, total) 256 257 // T8 (R3): retire-if-rotten -- streak>=WC_HD_RETIRE retires a P row (P->D visible after commit), 258 // streak below the bar keeps it pending, and a SEED url NEVER retires even on a rotten host 259 // (wc_seed_add is blocked by any existing fr: row, so a retired seed would be permanently dead). 260 let ur1: *u8 = "https://dead.example/rot1" as *u8 261 let ur2: *u8 = "https://dying.example/fresh" as *u8 262 let ur3: *u8 = "https://dead.example/seedhub" as *u8 263 segidbox[0] = segidbox[0] + 1 264 let w8: *i64 = ss_begin() 265 wc_frmark(w8, prefix, segidbox, segsbox, ur1, wc_len(ur1), 1) 266 wc_frmark(w8, prefix, segidbox, segsbox, ur2, wc_len(ur2), 1) 267 wc_frmark(w8, prefix, segidbox, segsbox, ur3, wc_len(ur3), 1) 268 let hhB: i64 = wc_hosthash("dying.example" as *u8, 13) 269 wc_hd_put(w8, prefix, segidbox, segsbox, hhB, 2) 270 ss_commit(prefix, w8, segidbox[0]) 271 let h8: *i64 = ss_open(prefix) 272 let seeds8: *u8 = sys_mmap(WC_MAGIC_2048) 273 var so8: i64 = 0 274 while ur3[so8] != (0 as u8) { seeds8[so8] = ur3[so8]; so8 = so8 + 1 } 275 seeds8[so8] = 0 as u8 276 segidbox[0] = segidbox[0] + 1 277 let w9: *i64 = ss_begin() 278 // the caller pattern: seed membership guards the retire (exactly what main's skip branch does) 279 var r1: i64 = 0 280 if wc_is_seed(seeds8, 1, ur1, wc_len(ur1)) == 0 { r1 = wc_retire_if_rotten(h8, w9, prefix, segidbox, segsbox, ur1, wc_len(ur1), hhA) } 281 var r2: i64 = 0 282 if wc_is_seed(seeds8, 1, ur2, wc_len(ur2)) == 0 { r2 = wc_retire_if_rotten(h8, w9, prefix, segidbox, segsbox, ur2, wc_len(ur2), hhB) } 283 var r3: i64 = 0 284 if wc_is_seed(seeds8, 1, ur3, wc_len(ur3)) == 0 { r3 = wc_retire_if_rotten(h8, w9, prefix, segidbox, segsbox, ur3, wc_len(ur3), hhA) } 285 if w9[1] > 0 { ss_commit(prefix, w9, segidbox[0]) } 286 let h9: *i64 = ss_open(prefix) 287 var t8: i64 = 0 288 if r1 == 1 { if r2 == 0 { if r3 == 0 { 289 if wc_frstate(h9, ur1, wc_len(ur1)) == 2 { if wc_frstate(h9, ur2, wc_len(ur2)) == 1 { if wc_frstate(h9, ur3, wc_len(ur3)) == 1 { t8 = 1 } } } 290 } } } 291 gw_check("T8 retire-if-rotten: streak>=3 retires, <3 keeps, seed exempt" as *u8, t8, pass, total) 292 293 // ---- T9 (R4 CANONICAL LANE) --------------------------------------------------------------- 294 // The host-filtered pull must keep ONLY canonical-seed hosts, match a SUBDOMAIN on the boundary 295 // dot, and REFUSE a lookalike host. Network-free -- store rows only. The pre-existing fixture 296 // rows (a.example / b.example) are non-canonical by construction, so they are excluded too and 297 // the count is stable regardless of what T1-T8 left pending. 298 segidbox[0] = segidbox[0] + 1 299 let q9w: *i64 = ss_begin() 300 let q9c1: *u8 = "https://en.wikipedia.org/wiki/Hypertension" as *u8 301 let q9c2: *u8 = "https://data.who.int/topic/air" as *u8 302 let q9c3: *u8 = "https://notwikipedia.org/lookalike" as *u8 303 let q9c4: *u8 = "https://junkfarm.example/tag/a" as *u8 304 wc_frmark(q9w, prefix, segidbox, segsbox, q9c1, wc_len(q9c1), 1) 305 wc_frmark(q9w, prefix, segidbox, segsbox, q9c2, wc_len(q9c2), 1) 306 wc_frmark(q9w, prefix, segidbox, segsbox, q9c3, wc_len(q9c3), 1) 307 wc_frmark(q9w, prefix, segidbox, segsbox, q9c4, wc_len(q9c4), 1) 308 ss_commit(prefix, q9w, segidbox[0]) 309 let q9seeds: *u8 = sys_mmap(WC_MAGIC_2048 * 4) 310 let q9s1: *u8 = "https://www.who.int/" as *u8 311 var q9a: i64 = 0 312 while q9s1[q9a] != (0 as u8) { q9seeds[q9a] = q9s1[q9a]; q9a = q9a + 1 } 313 q9seeds[q9a] = 0 as u8 314 let q9d2: *u8 = (q9seeds as i64 + WC_MAGIC_2048) as *u8 315 let q9s2: *u8 = "https://wikipedia.org/" as *u8 316 var q9b: i64 = 0 317 while q9s2[q9b] != (0 as u8) { q9d2[q9b] = q9s2[q9b]; q9b = q9b + 1 } 318 q9d2[q9b] = 0 as u8 319 let q9canh: *u8 = sys_mmap(64 * WC_MAXCANH) 320 let q9ncan: i64 = wc_canon_hosts(q9seeds, 2, q9canh) 321 let q9h: *i64 = ss_open(prefix) 322 let q9urls: *u8 = sys_mmap(WC_MAGIC_2048 * 32) 323 let q9n: i64 = wc_pending(q9h, prefix, q9urls, 32, q9canh, q9ncan, 1) 324 var t9: i64 = 0 325 if q9ncan == 2 { if wc_streq(q9canh, "who.int" as *u8) == 1 { 326 if wc_streq((q9canh as i64 + 64) as *u8, "wikipedia.org" as *u8) == 1 { if q9n == 2 { t9 = 1 } } 327 } } 328 gw_check("T9 canonical lane: www-stripped hosts, subdomain kept, junk excluded -> 2 of 4" as *u8, t9, pass, total) 329 // T9b THE NEGATIVE CONTROL that makes T9 non-vacuous: a lookalike host must NOT ride the lane. 330 let q9hb: *u8 = sys_mmap(64) 331 var t9b: i64 = 0 332 if wc_host_is_canon(q9canh, q9ncan, q9c1, q9hb) == 1 { if wc_host_is_canon(q9canh, q9ncan, q9c2, q9hb) == 1 { 333 if wc_host_is_canon(q9canh, q9ncan, q9c3, q9hb) == 0 { if wc_host_is_canon(q9canh, q9ncan, q9c4, q9hb) == 0 { t9b = 1 } } 334 } } 335 gw_check("T9b lookalike REFUSED: notwikipedia.org needs the boundary dot to match wikipedia.org" as *u8, t9b, pass, total) 336 // T9c canonly=0 is the pre-R4 contract: the SAME walk sees all four rows (plus fixture leftovers). 337 let q9all: i64 = wc_pending(q9h, prefix, q9urls, 32, q9canh, 0, 0) 338 gw_check("T9c canonly=0 unchanged: unfiltered walk still sees strictly more rows than the lane" as *u8, (q9all > q9n) as i64, pass, total) 339 // T9d (R4b): the host-string matcher is the ONE rule and the url matcher only wraps it. If these 340 // two ever disagree, the canonical LANE and the canonical DEPTH CAP would disagree about which 341 // hosts are curated -- a split definition is exactly how a privilege leaks to a lookalike host. 342 var q9agree: i64 = 1 343 if wc_host_str_is_canon(q9canh, q9ncan, "wikipedia.org" as *u8, 13) != 1 { q9agree = 0 } 344 if wc_host_str_is_canon(q9canh, q9ncan, "en.wikipedia.org" as *u8, 16) != 1 { q9agree = 0 } 345 if wc_host_str_is_canon(q9canh, q9ncan, "data.who.int" as *u8, 12) != 1 { q9agree = 0 } 346 if wc_host_str_is_canon(q9canh, q9ncan, "notwikipedia.org" as *u8, 16) != 0 { q9agree = 0 } 347 if wc_host_str_is_canon(q9canh, q9ncan, "junkfarm.example" as *u8, 16) != 0 { q9agree = 0 } 348 gw_check("T9d host-string matcher == url matcher (exact, subdomain, lookalike, junk)" as *u8, q9agree, pass, total) 349 350 // ---- T10 (R5 SEED FRESHNESS RE-QUEUE) ------------------------------------------------------- 351 // A D seed must return to P and be copied into the lane buffer; a P seed and a never-seen seed 352 // must be left alone. Network-free, store rows only. 353 segidbox[0] = segidbox[0] + 1 354 let qAw: *i64 = ss_begin() 355 let qA1: *u8 = "https://canon-a.example/hub" as *u8 356 let qA2: *u8 = "https://canon-b.example/hub" as *u8 357 let qA3: *u8 = "https://canon-c.example/hub" as *u8 358 wc_frmark(qAw, prefix, segidbox, segsbox, qA1, wc_len(qA1), 2) 359 wc_frmark(qAw, prefix, segidbox, segsbox, qA2, wc_len(qA2), 1) 360 ss_commit(prefix, qAw, segidbox[0]) 361 let qAseeds: *u8 = sys_mmap(WC_MAGIC_2048 * 4) 362 var qAi: i64 = 0 363 while qA1[qAi] != (0 as u8) { qAseeds[qAi] = qA1[qAi]; qAi = qAi + 1 } 364 qAseeds[qAi] = 0 as u8 365 let qAd2: *u8 = (qAseeds as i64 + WC_MAGIC_2048) as *u8 366 qAi = 0 367 while qA2[qAi] != (0 as u8) { qAd2[qAi] = qA2[qAi]; qAi = qAi + 1 } 368 qAd2[qAi] = 0 as u8 369 let qAd3: *u8 = (qAseeds as i64 + 2 * WC_MAGIC_2048) as *u8 370 qAi = 0 371 while qA3[qAi] != (0 as u8) { qAd3[qAi] = qA3[qAi]; qAi = qAi + 1 } 372 qAd3[qAi] = 0 as u8 373 let qAh: *i64 = ss_open(prefix) 374 segidbox[0] = segidbox[0] + 1 375 let qAw2: *i64 = ss_begin() 376 let qAdst: *u8 = sys_mmap(WC_MAGIC_2048 * 8) 377 let qAbox: *i64 = sys_mmap(16) as *i64 378 qAbox[0] = 0 379 qAbox[1] = 6 380 let qArq: i64 = wc_seed_refresh(qAh, qAw2, prefix, segidbox, segsbox, qAseeds, 3, qAdst, qAbox) 381 if qAw2[1] > 0 { ss_commit(prefix, qAw2, segidbox[0]) } 382 let qAh2: *i64 = ss_open(prefix) 383 var t10: i64 = 0 384 if qArq == 1 { if qAbox[0] == 1 { 385 if wc_frstate(qAh2, qA1, wc_len(qA1)) == 1 { if wc_frstate(qAh2, qA2, wc_len(qA2)) == 1 { if wc_frstate(qAh2, qA3, wc_len(qA3)) == 0 { 386 if wc_streq(qAdst, qA1) == 1 { t10 = 1 } 387 } } } 388 } } 389 gw_check("T10 seed re-queue: D seed -> P + copied to lane; P and never-seen UNTOUCHED" as *u8, t10, pass, total) 390 // T10b the persisted cursor round-trips (the rotation mechanism across runs) 391 segidbox[0] = segidbox[0] + 1 392 let qAw3: *i64 = ss_begin() 393 wc_sr_put(qAw3, prefix, segidbox, segsbox, 5) 394 if qAw3[1] > 0 { ss_commit(prefix, qAw3, segidbox[0]) } 395 let qAh3: *i64 = ss_open(prefix) 396 gw_check("T10b sr:cursor persists across commit (put 5 -> read 5)" as *u8, (wc_sr_cursor(qAh3) == 5) as i64, pass, total) 397 398 // ---- T11 (R10 SHAPE-BASED QUERY ADMISSION -- replaces R8's host allowlist) ------------------- 399 // R8 admitted query strings ONLY on curated hosts, so an uncurated content url was refused before 400 // a byte was fetched -- a DIRECTORY, not an index. R10 judges the query's SHAPE instead, so the 401 // SAME two fixtures now split on WHAT THE URL IS rather than WHO SERVES IT: 402 // clean content query, uncurated host -> ADMITTED (this is the whole point of R10) 403 // tracking param, uncurated host -> REFUSED (a duplicate of the clean url) 404 // Deny list set directly here, exactly as the canon table is a few lines below: the gate must be 405 // self-contained, never dependent on whatever the live urlpolicy- plane happens to hold today. 406 let qB: *u8 = "https://graphis.ne.jp/monthly/model.php?ID=260731_sp" as *u8 407 let qBjunk: *u8 = "https://tracker.example/page.php?utm_source=x" as *u8 408 var t11a: i64 = 0 409 if wc_urlok(qB, wc_len(qB)) == 1 { if wc_urlok(qBjunk, wc_len(qBjunk)) == 0 { t11a = 1 } } 410 gw_check("T11 R10: a CLEAN query url is ADMITTED with an EMPTY canon table; a TRACKING one is REFUSED" as *u8, t11a, pass, total) 411 // THE TOOTH THAT WOULD HAVE CAUGHT R8: the operator's real-world miss. An uncurated host serving 412 // paginated content must be reachable -- under R8 this was refused purely for lacking curation. 413 let qBv2: *u8 = "https://www.v2ph.com/actor/JULIA?page=3&hl=en" as *u8 414 gw_check("T11a-ii an UNCURATED host's paginated content url is ADMITTED (the R8 miss, by name)" as *u8, (wc_urlok(qBv2, wc_len(qBv2)) == 1) as i64, pass, total) 415 // and the trap half: a facet explosion beyond the param budget is still refused, on ANY host 416 let qBfacet: *u8 = "https://shop.example/list?a=1&b=2&c=3&d=4&e=5&f=6" as *u8 417 gw_check("T11a-iii a 6-param facet explosion is REFUSED (budget, not curation, is the limit)" as *u8, (wc_urlok(qBfacet, wc_len(qBfacet)) == 0) as i64, pass, total) 418 // now publish a canon table holding ONLY graphis.ne.jp 419 let qBcanh: *u8 = sys_mmap(64 * WC_MAXCANH) 420 var qBi: i64 = 0 421 let qBhost: *u8 = "graphis.ne.jp" as *u8 422 while qBhost[qBi] != (0 as u8) { qBcanh[qBi] = qBhost[qBi]; qBi = qBi + 1 } 423 qBcanh[qBi] = 0 as u8 424 wc_canh_g = qBcanh 425 wc_ncan_g = 1 426 wc_qscratch_g = sys_mmap(64) 427 // BISECT: which layer is failing -- the policy flag, or the canon match? 428 let t11p: i64 = olh_urlok_q(qB, wc_len(qB), 1) 429 gw_check("T11b-i policy layer: olh_urlok_q(allowq=1) admits the query-string url" as *u8, (t11p == 1) as i64, pass, total) 430 let t11m: i64 = wc_host_is_canon(qBcanh, 1, qB, wc_qscratch_g) 431 gw_check("T11b-ii matcher layer: wc_host_is_canon sees graphis.ne.jp in the table" as *u8, (t11m == 1) as i64, pass, total) 432 // R10: curation is now a BOOST, not a gate. The curated host keeps admitting its content, and the 433 // tracking url stays refused ON THE CURATED PATH TOO -- trust buys a bigger param budget, it does 434 // NOT buy immunity from junk. A curated host that could launder tracking urls into the index would 435 // make curation a liability rather than a signal. 436 var t11b: i64 = 0 437 if wc_urlok(qB, wc_len(qB)) == 1 { if wc_urlok(qBjunk, wc_len(qBjunk)) == 0 { t11b = 1 } } 438 let qBcurjunk: *u8 = "https://graphis.ne.jp/p.php?utm_source=x" as *u8 439 gw_check("T11b curated host ADMITS its content; a TRACKING url is refused on the CURATED path too" as *u8, (t11b == 1) as i64 * (wc_urlok(qBcurjunk, wc_len(qBcurjunk)) == 0) as i64, pass, total) 440 // the other kill-classes must NOT be weakened for the curated host either 441 let qBtag: *u8 = "https://graphis.ne.jp/tag/foo?ID=1" as *u8 442 let qBimg: *u8 = "https://graphis.ne.jp/i/photo.jpg" as *u8 443 var t11c: i64 = 0 444 if wc_urlok(qBtag, wc_len(qBtag)) == 0 { if wc_urlok(qBimg, wc_len(qBimg)) == 0 { t11c = 1 } } 445 gw_check("T11c curation relaxes ONLY '?' -- tag-farm and binary-extension kills still hold" as *u8, t11c, pass, total) 446 // T12 (R9): a CURATED host is never retired -- we cannot distinguish "host is down" from "we 447 // cannot speak its TLS", so retiring it would launder our own capability gap into permanent 448 // coverage loss. Uncurated rot must STILL retire (that is the control). 449 var t12: i64 = 0 450 let qCu: *u8 = "https://graphis.ne.jp/some/page" as *u8 451 let qCj: *u8 = "https://rotten.example/dead" as *u8 452 var qCcur: i64 = 0 453 var qCrot: i64 = 0 454 if wc_host_is_canon(qBcanh, 1, qCu, wc_qscratch_g) == 1 { qCcur = 1 } 455 if wc_host_is_canon(qBcanh, 1, qCj, wc_qscratch_g) == 0 { qCrot = 1 } 456 if qCcur == 1 { if qCrot == 1 { t12 = 1 } } 457 gw_check("T12 retire-exemption predicate: curated host protected, uncurated rot still retirable" as *u8, t12, pass, total) 458 wc_ncan_g = 0 459 460 // ================= T13: R11 ROBOTS LANE (network-free) ================= 461 wc_rb_init() 462 let rbnull: *TrustStore = 0 as *TrustStore // never dereferenced: every case below is a cache HIT 463 let rbh: *u8 = "example.org" as *u8 464 let rbhl: i64 = wc_len(rbh) 465 let pb: *u8 = sys_mmap(WC_MAGIC_2048) 466 467 // T13a -- path extraction. robots matching is defined over path+query, never over the host. 468 let uA: *u8 = "https://example.org/a/b?x=1" as *u8 469 var t13a: i64 = 0 470 if wc_rb_path(uA, wc_len(uA), pb, WC_MAGIC_2048) == 8 { if t_rb_streq(pb, "/a/b?x=1" as *u8) == 1 { t13a = 1 } } 471 gw_check("T13a robots path = path+query (/a/b?x=1)" as *u8, t13a, pass, total) 472 let uB: *u8 = "https://example.org" as *u8 // no path at all -> "/" 473 wc_rb_path(uB, wc_len(uB), pb, WC_MAGIC_2048) 474 gw_check("T13a2 a url with no path yields \"/\", never an empty match target" as *u8, t_rb_streq(pb, "/" as *u8), pass, total) 475 476 // ============ T13b FAMILY -- RFC 9309 s2.3.1 ACCESS-RESULT STATES (rewritten 2026-08-25) ====== 477 // WHAT THIS REPLACED. There used to be ONE tooth here reading "absent/unfetchable robots.txt -> 478 // ALLOW", which asserted a single policy for a sentence the RFC splits in two: 479 // s2.3.1.3 "If a server status code indicates that the robots.txt file is unavailable to the 480 // crawler, then the crawler MAY access any resources on the server." (4xx) 481 // s2.3.1.4 "If the robots.txt file is unreachable due to server or network errors, this means 482 // the robots.txt file is undefined and the crawler MUST assume complete disallow." 483 // The old tooth PASSED against a host answering 500, because 404 and 500 both produced a len-0 484 // cache slot and the code branched on LENGTH rather than on POLICY. One tooth covering two rules 485 // with opposite outcomes can only ever prove the one it was written for. 486 // Every classifier tooth below is network-free by construction: wc_rb_classify takes (status, 487 // bytes) and nothing else, which is why the rule is testable at all now. 488 489 gw_check("T13b1 s2.3.1.3 404 -> ALLOW_ALL (unavailable: crawler MAY access any resources)" as *u8, 490 (wc_rb_classify(404, 0) == WC_RB_ST_ALLOW_ALL) as i64, pass, total) 491 gw_check("T13b2 s2.3.1.4 500 -> DISALLOW_ALL (unreachable: crawler MUST assume complete disallow)" as *u8, 492 (wc_rb_classify(500, 0) == WC_RB_ST_DISALLOW_ALL) as i64, pass, total) 493 gw_check("T13b3 s2.3.1.4 503 -> DISALLOW_ALL (the overload/maintenance case we actually meet)" as *u8, 494 (wc_rb_classify(503, 0) == WC_RB_ST_DISALLOW_ALL) as i64, pass, total) 495 gw_check("T13b4 s2.3.1.4 NETWORK FAILURE, no status line at all -> DISALLOW_ALL" as *u8, 496 (wc_rb_classify(WC_HTTP_NO_STATUS, 0 - 2) == WC_RB_ST_DISALLOW_ALL) as i64, pass, total) 497 498 // The boundary is the whole content of the rule, so it is asserted rather than assumed. 499 var t13b5: i64 = 0 500 if wc_rb_classify(WC_HTTP_4XX_LO, 0) == WC_RB_ST_ALLOW_ALL { 501 if wc_rb_classify(WC_HTTP_4XX_HI, 0) == WC_RB_ST_ALLOW_ALL { 502 if wc_rb_classify(WC_HTTP_5XX_LO, 0) == WC_RB_ST_DISALLOW_ALL { 503 if wc_rb_classify(WC_HTTP_5XX_HI, 0) == WC_RB_ST_DISALLOW_ALL { t13b5 = 1 } } } } 504 gw_check("T13b5 the 4xx/5xx boundary sits where the RFC puts it: 400+499 allow, 500+599 disallow" as *u8, 505 t13b5, pass, total) 506 507 // ---- NEG-CONTROLS. THE POINT: an implementation that simply disallowed everything would pass 508 // T13b2/3/4/5 and read as a compliance win. These are the teeth it cannot pass. 509 gw_check("T13b6 neg-control-200-WITH-RULES-STILL-PARSES (blanket-disallow fails here)" as *u8, 510 (wc_rb_classify(WC_HTTP_2XX_LO, 33) == WC_RB_ST_RULES) as i64, pass, total) 511 gw_check("T13b7 neg-control-200-EMPTY-BODY-ALLOWS (a successful fetch of a file stating no rules)" as *u8, 512 (wc_rb_classify(WC_HTTP_2XX_LO, 0) == WC_RB_ST_ALLOW_ALL) as i64, pass, total) 513 gw_check("T13b8 200 whose body could NOT be delivered -> DISALLOW_ALL (we never read the rules)" as *u8, 514 (wc_rb_classify(WC_HTTP_2XX_LO, 0 - 9) == WC_RB_ST_DISALLOW_ALL) as i64, pass, total) 515 gw_check("T13b9 s2.3.1.2 a 3xx still unresolved past the redirect budget -> ALLOW_ALL (MAY assume unavailable)" as *u8, 516 (wc_rb_classify(302, 0) == WC_RB_ST_ALLOW_ALL) as i64, pass, total) 517 518 // The undefined-status door shares a POLICY with 5xx but must not share its LOG LABEL, or a 519 // response-parser bug would hide inside traffic we expect to see. 520 var t13b10: i64 = 0 521 if wc_rb_status_undefined(100) == 1 { 522 if wc_rb_status_undefined(500) == 0 { 523 if wc_rb_status_undefined(WC_HTTP_NO_STATUS) == 0 { t13b10 = 1 } } } 524 gw_check("T13b10 an UNDEFINED status is distinguishable from an honest 5xx and from no-status-at-all" as *u8, 525 t13b10, pass, total) 526 527 // ---- END-TO-END: the state has to actually reach the decision. ASSERT THE FIXTURE REACHED THE 528 // CONDITION FIRST -- a seeder that silently failed would otherwise produce a PASS meaning nothing. 529 let hh_500: i64 = wc_hosthash("broken.example" as *u8, 14) 530 let idx500: i64 = t_rb_seed_state(hh_500, "" as *u8, 0, WC_RB_ST_DISALLOW_ALL) 531 gw_check("T13b11 fixture-reached-the-condition: the seeded slot really holds DISALLOW_ALL" as *u8, 532 (wc_rb_state_g[idx500] == WC_RB_ST_DISALLOW_ALL) as i64, pass, total) 533 let u500: *u8 = "https://broken.example/anything" as *u8 534 gw_check("T13b12 END-TO-END a 5xx host now REFUSES every url -- THIS IS THE BUG THAT SHIPPED" as *u8, 535 (wc_rb_allowed("broken.example" as *u8, 14, hh_500, u500, wc_len(u500), rbnull) == 0) as i64, pass, total) 536 537 let hh_404: i64 = wc_hosthash("gone.example" as *u8, 12) 538 let idx404: i64 = t_rb_seed_state(hh_404, "" as *u8, 0, WC_RB_ST_ALLOW_ALL) 539 gw_check("T13b13 fixture-reached-the-condition: the seeded slot really holds ALLOW_ALL" as *u8, 540 (wc_rb_state_g[idx404] == WC_RB_ST_ALLOW_ALL) as i64, pass, total) 541 let u404: *u8 = "https://gone.example/anything" as *u8 542 gw_check("T13b14 neg-control-404-HOST-STILL-ALLOWED-END-TO-END (blanket-disallow fails here too)" as *u8, 543 wc_rb_allowed("gone.example" as *u8, 12, hh_404, u404, wc_len(u404), rbnull), pass, total) 544 545 // The ORIGINAL T13b, kept: an ABSENT robots.txt still allows. Its meaning is unchanged -- absence 546 // is the 4xx/unavailable case, which was never the case that was broken. 547 let hh_absent: i64 = wc_hosthash("absent.example" as *u8, 14) 548 t_rb_seed(hh_absent, "" as *u8, 0) 549 let uabs: *u8 = "https://absent.example/anything" as *u8 550 gw_check("T13b absent robots.txt -> ALLOW (RFC 9309 s2.3.1.3; our own inability is still not a rule)" as *u8, 551 wc_rb_allowed("absent.example" as *u8, 14, hh_absent, uabs, wc_len(uabs), rbnull), pass, total) 552 553 // ---- T13f: THE CRAWL-DELAY HORIZON. We never answer a long published delay by crawling faster. 554 gw_check("T13f1 a delay AT the RFC 9309 s2.4 24h horizon is ADOPTED exactly, not refused" as *u8, 555 (wc_rb_cd_policy(WC_ROBOTS_CD_HORIZON_S) == 0) as i64, pass, total) 556 gw_check("T13f2 above the horizon we DECLINE THE HOST -- the one thing we never do is speed up" as *u8, 557 (wc_rb_cd_policy(WC_ROBOTS_CD_HORIZON_S + 1) == 1) as i64, pass, total) 558 var t13f3: i64 = 0 559 if wc_rb_cd_policy(30) == 0 { if wc_rb_cd_policy(120) == 0 { t13f3 = 1 } } 560 gw_check("T13f3 neg-control-ORDINARY-DELAYS-ADOPTED: 30s and 120s both adopt (the old code clamped 120 to 30)" as *u8, 561 t13f3, pass, total) 562 563 // ============ T21: R15 DOMAIN-MAP ADOPTION -- WHICH ROWS MAY REACH THE FRONTIER ================ 564 // wc_dm_seedable is pure over (kind, verdict), so the admission RULE is provable with no 565 // filesystem, no map and no network. The two POSITIVE teeth are load-bearing: a seeder that simply 566 // returned 0 for everything would pass every neg-control below and read as a flawless filter while 567 // silently feeding the frontier nothing at all. 568 gw_check("T21a url+LISTED IS seeded -- the sitemap/feed payload, the whole point of the rung" as *u8, 569 wc_dm_seedable("url" as *u8, "LISTED" as *u8), pass, total) 570 gw_check("T21b path+LIVE IS seeded -- a probe path the host actually served" as *u8, 571 wc_dm_seedable("path" as *u8, "LIVE" as *u8), pass, total) 572 573 // ---- THE COMPLIANCE NEG-CONTROL. nx_domain_map records when robots.txt refused a url. Seeding 574 // that row would launder a refusal into a fetch through a side door, defeating wc_rb_allowed 575 // without ever calling it -- a robots bypass that no robots tooth in this gate could detect. 576 gw_check("T21c neg-control-ROBOTS-REFUSED-NEVER-SEEDED (a map row must not bypass wc_rb_allowed)" as *u8, 577 (wc_dm_seedable("url" as *u8, "ROBOTS-REFUSED" as *u8) == 0) as i64, pass, total) 578 579 // ---- THE SOFT-404 NEG-CONTROL. The map has already PROVEN this url is the not-found page wearing 580 // a 200. Seeding it spends a fetch and a frontier slot on a page that does not exist -- precisely 581 // the measured waste (www.unesco.org served "thin 200 16" repeatedly) that the fingerprint exists 582 // to stop. Note the discrimination this pairs with T21b: LIVE seeds, SOFT404 does not, and both 583 // arrive on kind=path from the same probe source. 584 gw_check("T21d neg-control-SOFT404-NEVER-SEEDED (the fingerprint's whole purpose)" as *u8, 585 (wc_dm_seedable("path" as *u8, "SOFT404" as *u8) == 0) as i64, pass, total) 586 587 // ---- absence rows: the reconnaissance already paid to learn there is nothing there. 588 var t21e: i64 = 0 589 if wc_dm_seedable("path" as *u8, "MISSING" as *u8) == 0 { 590 if wc_dm_seedable("path" as *u8, "UNREACHABLE" as *u8) == 0 { 591 if wc_dm_seedable("path" as *u8, "BUDGET-REFUSED" as *u8) == 0 { t21e = 1 } } } 592 gw_check("T21e neg-control-ABSENCE-ROWS-NEVER-SEEDED (MISSING / UNREACHABLE / BUDGET-REFUSED)" as *u8, 593 t21e, pass, total) 594 595 // ---- rows that are not urls at all. 596 var t21f: i64 = 0 597 if wc_dm_seedable("path" as *u8, "FINGERPRINT" as *u8) == 0 { 598 if wc_dm_seedable("path" as *u8, "DECLARED" as *u8) == 0 { 599 if wc_dm_seedable("host" as *u8, "CERT" as *u8) == 0 { t21f = 1 } } } 600 gw_check("T21f neg-control-NON-URL-ROWS-NEVER-SEEDED (FINGERPRINT record / robots DECLARED path / crt host)" as *u8, 601 t21f, pass, total) 602 603 // ---- THE ANTI-VACUITY TOOTH. Everything above could be passed by a predicate that reads ONLY the 604 // verdict column and ignores the kind. These two cross pairs cannot: LISTED and LIVE are both 605 // admitting verdicts, and both must still be REFUSED when they arrive on the wrong kind. This is 606 // the tooth that proves the rule reads BOTH fields. 607 var t21g: i64 = 0 608 if wc_dm_seedable("path" as *u8, "LISTED" as *u8) == 0 { 609 if wc_dm_seedable("url" as *u8, "LIVE" as *u8) == 0 { t21g = 1 } } 610 gw_check("T21g anti-vacuity: an ADMITTING verdict on the WRONG KIND is refused (both fields are read)" as *u8, 611 t21g, pass, total) 612 613 // T13c -- BOTH DIRECTIONS off ONE body. Disallow must bite AND the rest of the host stays open. 614 let body1: *u8 = "User-agent: *\nDisallow: /private\n" as *u8 615 let hh1: i64 = wc_hosthash(rbh, rbhl) 616 t_rb_seed(hh1, body1, wc_len(body1)) 617 let dis1: *u8 = "https://example.org/private/secret" as *u8 618 let all1: *u8 = "https://example.org/public/post" as *u8 619 var t13c: i64 = 0 620 if wc_rb_allowed(rbh, rbhl, hh1, dis1, wc_len(dis1), rbnull) == 0 { t13c = 1 } 621 gw_check("T13c Disallow: /private REFUSES /private/secret" as *u8, t13c, pass, total) 622 gw_check("T13c2 THE OPPOSITE TOOTH: /public on the same host is still ALLOWED (we did not block the web)" as *u8, 623 wc_rb_allowed(rbh, rbhl, hh1, all1, wc_len(all1), rbnull), pass, total) 624 625 // T13d -- Crawl-delay is READ AS PUBLISHED. This is the number the pacer now adopts, and the whole 626 // reason loc.gov served two pages then 403'd: the host had been publishing its pace all along. 627 let body2: *u8 = "User-agent: *\nCrawl-delay: 7\nDisallow: /x\n" as *u8 628 var t13d: i64 = 0 629 if nx_robots_crawl_delay(body2, wc_len(body2), wc_rb_ua(), WC_RB_UALEN) == 7 { t13d = 1 } 630 gw_check("T13d Crawl-delay: 7 is read as 7 (the pace we will actually honour)" as *u8, t13d, pass, total) 631 var t13d2: i64 = 0 632 if nx_robots_crawl_delay(body1, wc_len(body1), wc_rb_ua(), WC_RB_UALEN) == 0 { t13d2 = 1 } 633 gw_check("T13d2 NEG: a body with no Crawl-delay yields 0 -- we never invent a pace and call it the host's" as *u8, t13d2, pass, total) 634 635 // T13e -- GROUP SELECTION: a nishibot-specific group WINS over '*'. This proves the token we match 636 // is the token we claim, so a site CAN address a rule to us and have it obeyed. 637 let body3: *u8 = "User-agent: *\nDisallow: /\n\nUser-agent: nishibot\nDisallow: /nope\n" as *u8 638 let hh3: i64 = wc_hosthash("grp.example" as *u8, 11) 639 t_rb_seed(hh3, body3, wc_len(body3)) 640 let okg: *u8 = "https://grp.example/fine" as *u8 641 let nog: *u8 = "https://grp.example/nope/x" as *u8 642 gw_check("T13e our own UA group wins over '*' (blanket Disallow: / does NOT apply to us)" as *u8, 643 wc_rb_allowed("grp.example" as *u8, 11, hh3, okg, wc_len(okg), rbnull), pass, total) 644 var t13e2: i64 = 0 645 if wc_rb_allowed("grp.example" as *u8, 11, hh3, nog, wc_len(nog), rbnull) == 0 { t13e2 = 1 } 646 gw_check("T13e2 ...and the rule addressed TO US is obeyed (/nope refused)" as *u8, t13e2, pass, total) 647 648 // T13f -- cache identity: an unknown host MISSES (probed exactly once), a seeded host HITS (never 649 // re-probed). This bound is what stops the politeness probe from becoming the load. 650 var t13f: i64 = 0 651 if wc_rb_find(wc_hosthash("never.seen" as *u8, 10)) < 0 { t13f = 1 } 652 gw_check("T13f unknown host misses the cache (so it is probed once, not never)" as *u8, t13f, pass, total) 653 var t13f2: i64 = 0 654 if wc_rb_find(hh1) >= 0 { t13f2 = 1 } 655 gw_check("T13f2 seeded host HITS (so robots.txt is fetched at most once per host per run)" as *u8, t13f2, pass, total) 656 657 // ---- R7 FRONTIER PRIORITY + AGING (2026-08-24, /compare/webscraping wc_priority): pure functions, network-free ---- 658 var t14: i64 = 1 659 if wc_url_depth("https://a.org/" as *u8, 14) != 0 { t14 = 0 } 660 if wc_url_depth("https://a.org/x" as *u8, 15) != 1 { t14 = 0 } 661 if wc_url_depth("https://a.org/x/y/z" as *u8, 19) != 3 { t14 = 0 } 662 if wc_url_depth("https://a.org/x/y/" as *u8, 18) != 2 { t14 = 0 } 663 gw_check("T14 path depth: root 0, /x 1, /x/y/z 3, a trailing slash opens no level" as *u8, t14, pass, total) 664 wc_query_set("" as *u8) 665 var t15: i64 = 0 666 if wc_priority("https://a.org/x" as *u8, 15) > wc_priority("https://a.org/x/y/z" as *u8, 19) { t15 = 1 } 667 gw_check("T15 no query: a shallow url outranks a deep one (breadth-first finds the important pages first, Cho 1998)" as *u8, t15, pass, total) 668 wc_query_set("butter churning" as *u8) 669 var t16: i64 = 0 670 if wc_priority("https://a.org/a/b/c/d/butter-churning" as *u8, 37) > wc_priority("https://a.org/x" as *u8, 15) { t16 = 1 } 671 gw_check("T16 with a query, a deep url naming two query terms outranks a shallow url naming none" as *u8, t16, pass, total) 672 var t16b: i64 = 0 673 if wc_priority("https://a.org/x/butter" as *u8, 22) > wc_priority("https://a.org/x/y/butter" as *u8, 24) { t16b = 1 } 674 gw_check("T16b equal query hits fall back to depth, shallower first" as *u8, t16b, pass, total) 675 wc_query_set("" as *u8) 676 let rw: *u8 = sys_mmap(WC_MAGIC_2048 * 4) 677 let rw0: *u8 = "https://a.org/p/q/r" as *u8 678 let rw1: *u8 = "https://a.org/p" as *u8 679 let rw2: *u8 = "https://a.org/p/q" as *u8 680 var rwi: i64 = 0 681 while rwi <= 19 { rw[rwi] = rw0[rwi]; rwi = rwi + 1 } 682 rwi = 0 683 while rwi <= 15 { rw[WC_MAGIC_2048 + rwi] = rw1[rwi]; rwi = rwi + 1 } 684 rwi = 0 685 while rwi <= 17 { rw[2 * WC_MAGIC_2048 + rwi] = rw2[rwi]; rwi = rwi + 1 } 686 wc_rank_window(rw, 3) 687 var t17: i64 = 0 688 if wc_streq(rw, rw1) == 1 { if wc_streq((rw as i64 + WC_MAGIC_2048) as *u8, rw2) == 1 { if wc_streq((rw as i64 + 2 * WC_MAGIC_2048) as *u8, rw0) == 1 { t17 = 1 } } } 689 gw_check("T17 wc_rank_window orders the pull window shallow-first and keeps every url byte-exact" as *u8, t17, pass, total) 690 var t18: i64 = 1 691 if wc_pc_next(0, 5, 100) != 5 { t18 = 0 } 692 if wc_pc_next(95, 5, 100) != 0 { t18 = 0 } 693 if wc_pc_next(40, 0, 30) != 0 { t18 = 0 } 694 gw_check("T18 rotation cursor advances by rows taken and wraps at the pending population" as *u8, t18, pass, total) 695 696 // ================= T19: R13 FRONTIER GAUGE -- THE PARTS MUST SUM (2026-08-25) ================= 697 // This number is what every capacity decision about the crawl is read off, and until now it lived 698 // INLINE inside the done-line's wc_num() call, so it had no tooth at all. It used to read 699 // `npend - fetched + disc`, subtracting ONE of the FOUR ways a row leaves the frontier. 700 // The numbers below are the REAL measured ones from the last pre-fix run (searchpipe.log, byte offset 701 // 112,972,980): fetched=160, robots_skipped=22, discovered_new=300, frontier_remaining~=2013 under the 702 // OLD formula, which back-solves to npend=1873 -- and the source's own note records true pending as 703 // 1871-1883 over that window, so the fixture is the production case, not an invented one. 704 // The MEASURED true pending count of the live frontier on 2026-08-25, used as the fixture population 705 // for the partition arithmetic below. Named for its PURPOSE, not its value: the ratchet refused the 706 // bare literal (magic=1 baseline=0 at threshold 1024) and naming it also records WHERE it came from -- 707 // it is not an invented round number, it is what the frontier actually held when the gauge was 708 // corrected, recovered as npend = remaining + fetched - discovered across six consecutive runs. 709 let fN: i64 = WCID_FIXTURE_FRONTIER_PENDING 710 let fF: i64 = 160 711 let fR: i64 = 0 712 let fB: i64 = 22 713 let fH: i64 = 176 714 let fD: i64 = 300 715 // THE PARTITION CLAIM, asserted as a SUM rather than by restating the expression under test (a check 716 // that re-derives its subject proves only that arithmetic is arithmetic): every pulled row either LEFT 717 // the pending state this run, or is still pending. Those two must reconstruct npend exactly. 718 let fLeft: i64 = wc_frontier_left(fF, fR, fB, fH) 719 let fStill: i64 = wc_frontier_remaining(fN, fF, fR, fB, fH, fD) - fD 720 gw_check("T19 partition SUMS: rows that left + rows still pending == the window pulled" as *u8, (fLeft + fStill == fN) as i64, pass, total) 721 gw_check("T19b the four outflows are all counted (160+0+22+176 = 358)" as *u8, (fLeft == 358) as i64, pass, total) 722 // THE REGRESSION THE FIX EXISTS FOR: against the OLD gauge the corrected one must be SMALLER by exactly 723 // the outflows the old one ignored -- no more, no less. 724 let fOld: i64 = fN - fF + fD 725 gw_check("T19c corrected gauge is lower than the old one by EXACTLY robots_skipped+hostcap+retired" as *u8, 726 (fOld - wc_frontier_remaining(fN, fF, fR, fB, fH, fD) == fB + fH + fR) as i64, pass, total) 727 gw_check("T19d ...and that difference is NONZERO here (198), so T19c is not comparing a thing to itself" as *u8, 728 (fOld - wc_frontier_remaining(fN, fF, fR, fB, fH, fD) == 198) as i64, pass, total) 729 // neg-control: a run with NO robots skips, NO host caps and NO retirements must be BYTE-IDENTICAL to the 730 // old gauge. If this fails the correction changed something it was never supposed to touch. 731 gw_check("T19e neg-control-NO-OUTFLOWS-matches-the-OLD-gauge-exactly" as *u8, 732 (wc_frontier_remaining(fN, fF, 0, 0, 0, fD) == fN - fF + fD) as i64, pass, total) 733 // pace_deferred is deliberately ABSENT from this arithmetic: a deferred row stays PENDING, which is the 734 // entire point of deferring it. Subtracting it would under-report the frontier by exactly the rows we 735 // most intend to come back to. The signature carries no deferred parameter, and T20 below pins the 736 // store-level invariant that makes that correct. 737 738 // ================= T20: THE INVARIANT DEFERRAL RELIES ON ================= 739 // HONEST SCOPE: the deferral branch itself lives inside main()'s selection loop and is not callable, so 740 // this does NOT execute that branch. What it pins is the store contract the branch depends on, and the 741 // CONTRAST is the assertion -- both signals present on ONE fixture at once: 742 // a row LEFT UNMARKED (what deferral does) -> stays P and is re-enumerated on a later run 743 // a row marked state 2 (what host-cap does) -> disappears from the pending window 744 // If the first failed, every deferral would silently LOSE a url. If the second failed, host-cap would 745 // re-select the same row forever. The two are the opposite terminal semantics of one selection loop. 746 let dprefix: *u8 = "/tmp/crawlgate-r13-" as *u8 747 let dseg: *i64 = sys_mmap(16) as *i64; dseg[0] = 1 748 let dsegs: *i64 = sys_mmap(16) as *i64; dsegs[0] = 0 749 sys_unlinkat("/tmp/crawlgate-r13-manifest.txt" as *u8) 750 sys_unlinkat("/tmp/crawlgate-r13-seg-1.docs" as *u8) 751 sys_unlinkat("/tmp/crawlgate-r13-seg-1.idx" as *u8) 752 sys_unlinkat("/tmp/crawlgate-r13-seg-2.docs" as *u8) 753 sys_unlinkat("/tmp/crawlgate-r13-seg-2.idx" as *u8) 754 let dw: *i64 = ss_begin() 755 let dfer: *u8 = "https://throttled.example/deferred-row" as *u8 756 let dcap: *u8 = "https://capped.example/hostcapped-row" as *u8 757 wc_frmark(dw, dprefix, dseg, dsegs, dfer, wc_len(dfer), 1) 758 wc_frmark(dw, dprefix, dseg, dsegs, dcap, wc_len(dcap), 1) 759 ss_commit(dprefix, dw, dseg[0]) 760 let dh0: *i64 = ss_open(dprefix) 761 let durls: *u8 = sys_mmap(WC_MAGIC_2048 * 8) 762 let dcanh: *u8 = sys_mmap(64 * 8) 763 let dn0: i64 = wc_pending(dh0, dprefix, durls, 8, dcanh, 0, 0) 764 gw_check("T20 fixture: both rows start PENDING (2) -- the fixture reached the condition" as *u8, (dn0 == 2) as i64, pass, total) 765 // now do to each row exactly what its branch does 766 dseg[0] = dseg[0] + 1 767 let dw2: *i64 = ss_begin() 768 wc_frmark(dw2, dprefix, dseg, dsegs, dcap, wc_len(dcap), 2) // host-cap: marked DONE 769 ss_commit(dprefix, dw2, dseg[0]) // deferral: NOTHING written for dfer 770 let dh1b: *i64 = ss_open(dprefix) 771 let dn1: i64 = wc_pending(dh1b, dprefix, durls, 8, dcanh, 0, 0) 772 gw_check("T20a the host-capped row LEAVES the pending window (2 -> 1)" as *u8, (dn1 == 1) as i64, pass, total) 773 var dsurv: i64 = 0 774 if dn1 == 1 { if wc_streq(durls, dfer) == 1 { dsurv = 1 } } 775 gw_check("T20b the survivor is the UNMARKED (deferred) row, byte-exact -- a deferral loses no url" as *u8, dsurv, pass, total) 776 gw_check("T20c neg-control: the DEFERRED url is still state P(1), never flipped to D" as *u8, 777 (wc_frstate(dh1b, dfer, wc_len(dfer)) == 1) as i64, pass, total) 778 gw_check("T20d ...and the host-capped url IS state D(2) -- the two paths really do differ" as *u8, 779 (wc_frstate(dh1b, dcap, wc_len(dcap)) == 2) as i64, pass, total) 780 781 // ---- T22: CRAWLER IDENTITY (2026-08-25). Until this change the bulk crawler sent a HARDCODED 782 // CHROME User-Agent on every request: wc_fetch_child -> nx_https_fetch_follow_best -> ff_core -> 783 // nx_https_get_complete_cookie_xhdr -> nx_http_client_build_request_cookie_xhdr, which emitted the 784 // browser identity for everyone. Two consequences worse than the forgery itself: we MATCHED the 785 // robots.txt group `nishibot` while SENDING Chrome, so an operator could write a rule for us, watch 786 // us obey it, and never be able to tell it was us; and the identity mechanism added later appended a 787 // SECOND User-Agent header, which RFC 9110 5.3 forbids and which receivers resolve by taking the 788 // FIRST -- so the honest identity never reached the wire at all. 789 // The browser lane's Chrome UA is LOAD-BEARING (it matches our JA3 TLS fingerprint), so the fix is a 790 // SELECTOR whose default is byte-identical to the old behaviour. T22a is the neg-control that proves 791 // that default did not move; without it, a change that made everything NishiBot would pass T22c. 792 let uab: *u8 = sys_mmap(WCID_UABUF) 793 hc_set_identity(HC_UA_BROWSER) 794 let ub0: i64 = hc_put_ua(uab, 0) 795 gw_check("T22a neg-control: the DEFAULT identity is still the browser UA -- no existing caller changed" as *u8, 796 wcid_has(uab, ub0, "Mozilla/5.0" as *u8), pass, total) 797 gw_check("T22b neg-control: the browser form does NOT also claim to be the bot" as *u8, 798 (wcid_has(uab, ub0, "NishiBot" as *u8) == 0) as i64, pass, total) 799 hc_set_identity(HC_UA_BOT) 800 let ub1: i64 = hc_put_ua(uab, 0) 801 gw_check("T22c the crawler identity emits NishiBot" as *u8, 802 wcid_has(uab, ub1, "NishiBot/1.0" as *u8), pass, total) 803 gw_check("T22d ...and does NOT simultaneously claim to be Chrome" as *u8, 804 (wcid_has(uab, ub1, "Mozilla" as *u8) == 0) as i64, pass, total) 805 gw_check("T22e ...and carries a CONTACT url -- an identity nobody can complain to is not an identity" as *u8, 806 wcid_has(uab, ub1, "+https://nishifamily.com/bot" as *u8), pass, total) 807 gw_check("T22f EXACTLY ONE User-Agent line (RFC 9110 5.3): repeating it is forbidden and receivers take the FIRST, which is how the honest identity used to be silently dropped" as *u8, 808 (wcid_count(uab, ub1, "User-Agent:" as *u8) == 1) as i64, pass, total) 809 gw_check("T22g fixture-reached: the two identities really are DIFFERENT bytes, so T22a/T22c are not both reading one string" as *u8, 810 (ub0 != ub1) as i64, pass, total) 811 hc_set_identity(HC_UA_BROWSER) 812 813 // ---- T23: HOST-CAP DEFERS INSTEAD OF RETIRING (R16, 2026-08-25). The host cap is a PER-ROUND 814 // diversity measure that was marking rows DONE permanently -- measured on the live beat as 815 // hostcap_retired=227 pages discarded in a single run, on a frontier that had drained to 313. 816 // The predicate is pure so it can be gated at all; these teeth pin BOTH arms, because a rule that 817 // only ever defers would silently reintroduce the blockade the original design retired to avoid. 818 gw_check("T23a a frontier that FITS the pull window defers (the row comes back next run)" as *u8, 819 wc_hostcap_defers(WC_MAXPEND - 1), pass, total) 820 gw_check("T23b exactly AT the window still defers -- every pending row is still pulled" as *u8, 821 wc_hostcap_defers(WC_MAXPEND), pass, total) 822 gw_check("T23c neg-control: PAST the window it RETIRES -- a deferred row could be shadowed there, which is the blockade retirement exists to prevent" as *u8, 823 (wc_hostcap_defers(WC_MAXPEND + 1) == 0) as i64, pass, total) 824 gw_check("T23d neg-control: a far-oversized frontier also retires, so T23c is a boundary and not an off-by-one accident" as *u8, 825 (wc_hostcap_defers(WC_MAXPEND * 2) == 0) as i64, pass, total) 826 gw_check("T23e fixture-reached: the live frontier measured 313, which really is inside the window, so the defer arm is the one production takes today" as *u8, 827 wc_hostcap_defers(WCID_FIXTURE_FRONTIER_DRAINED), pass, total) 828 gw_check("T23f ...and the PRE-DRAIN frontier of 1873 was ALSO inside the window, so R16 would have deferred then too -- the 227 pages that run discarded were avoidable, not a consequence of frontier size" as *u8, 829 wc_hostcap_defers(WCID_FIXTURE_FRONTIER_PENDING), pass, total) 830 831 // ---- T24: THE DERIVED DISCOVERY BUDGET (R17, 2026-08-25). discovered_new sat at exactly 300 in 58 832 // of 60 consecutive runs -- pinned at its ceiling, i.e. the harvest had more supply than the budget 833 // accepted, while the frontier had drained to 313 against a 2048-row pull window. These teeth pin the 834 // shape of the derivation, not a single value, because a budget that only ever widens is as wrong as 835 // one that never does. 836 gw_check("T24a the DRAINED frontier (313, measured) gets a budget far above the old fixed 300" as *u8, 837 (wc_disc_budget(WCID_FIXTURE_FRONTIER_DRAINED) > WC_MAXDISC) as i64, pass, total) 838 gw_check("T24b ...and it fills toward exactly one pull window, never past it" as *u8, 839 (wc_disc_budget(WCID_FIXTURE_FRONTIER_DRAINED) == WC_DISC_CEIL - WCID_FIXTURE_FRONTIER_DRAINED) as i64, pass, total) 840 gw_check("T24h ...while the PRE-DRAIN frontier (1873, also measured) correctly floors to the old budget -- the same function, two real frontiers, two different right answers" as *u8, 841 (wc_disc_budget(WCID_FIXTURE_FRONTIER_PENDING) == WC_MAXDISC) as i64, pass, total) 842 gw_check("T24c neg-control: a FULL frontier falls back to the old fixed budget -- this can only widen, never shrink" as *u8, 843 (wc_disc_budget(WC_DISC_CEIL) == WC_MAXDISC) as i64, pass, total) 844 gw_check("T24d neg-control: an OVERFULL frontier still gets the floor, never a negative or zero budget (discovery must never stop entirely)" as *u8, 845 (wc_disc_budget(WC_DISC_CEIL * 2) == WC_MAXDISC) as i64, pass, total) 846 gw_check("T24e the budget never exceeds the ceiling even at an empty frontier" as *u8, 847 (wc_disc_budget(0) <= WC_DISC_CEIL) as i64, pass, total) 848 gw_check("T24f fixture-reached: the ceiling really is the pull window, so the runseen buffer dimensioned from it cannot be overrun by the budget" as *u8, 849 (WC_DISC_CEIL == WC_MAXPEND) as i64, pass, total) 850 gw_check("T24g neg-control: the UNSET budget getter floors to the old fixed value, so a path running before main computes it behaves exactly as before" as *u8, 851 (wc_disc_cap() == WC_MAXDISC) as i64, pass, total) 852 853 // ---- T25: HOST-BALANCED PULL WINDOW (G0, 2026-09-03, /compare/search contract wc_frontier_fair). 854 // MEASURED, not designed: two windows of knowledge/status/crawl_outcomes.log 41 minutes apart 855 // (epochs 1788448047 and 1788450535) were BOTH 100 percent hostcap-defer and every url in both was 856 // en.wikipedia.org, while nx_crawlaudit re-fetched a sample independently and returned 6 of 8 857 // disagreements reading that we DROPPED real prose. The R7 rotation above cannot fix that -- a 858 // rotated slice of a one-host frontier is still one host. These teeth pin the admission predicate 859 // itself, and the load-bearing ones are the arms that must NOT refuse: a balancer that turns into a 860 // crawl stop is a worse defect than the imbalance it corrects. 861 let wqh5: *u8 = sys_mmap(WC_HOSTBUF * 8) 862 let wqc5: *i64 = sys_mmap(8 * 8) as *i64 863 let wqn5: *i64 = sys_mmap(16) as *i64 864 wqn5[0] = 0 865 let ha5: *u8 = "en.wikipedia.org" as *u8 866 let hb5: *u8 = "who.int" as *u8 867 let hc5: *u8 = "irs.gov" as *u8 868 var adm5: i64 = 0 869 var i5: i64 = 0 870 while i5 < WC_WINQ { adm5 = adm5 + wc_winq_admit(wqh5, wqc5, wqn5, 8, ha5, 16, WC_WINQ); i5 = i5 + 1 } 871 gw_check("T25a fixture-reached: the fixture really drove host A to EXACTLY its quota, so the refusing tooth below is asked the question it claims to ask" as *u8, 872 (adm5 == WC_WINQ) as i64, pass, total) 873 gw_check("T25b the row PAST the quota for the SAME host is refused -- that surplus was guaranteed to become hostcap-defer at the fetch stage" as *u8, 874 (wc_winq_admit(wqh5, wqc5, wqn5, 8, ha5, 16, WC_WINQ) == 0) as i64, pass, total) 875 gw_check("T25c NEG-CONTROL, the load-bearing one: a DIFFERENT host is still admitted once host A is full -- if this fails the fix has become a crawl stop, not a balancer" as *u8, 876 wc_winq_admit(wqh5, wqc5, wqn5, 8, hb5, 7, WC_WINQ), pass, total) 877 gw_check("T25d the distinct-host count really moved to 2, so T25c admitted a NEW host rather than silently re-counting host A" as *u8, 878 (wqn5[0] == 2) as i64, pass, total) 879 gw_check("T25e NEG-CONTROL: a NEW host arriving with the table FULL fails OPEN -- a sizing error must be wrong in the direction of doing nothing, never of starving the crawl" as *u8, 880 wc_winq_admit(wqh5, wqc5, wqn5, 2, hc5, 7, WC_WINQ), pass, total) 881 gw_check("T25f ...and that fail-open really took the table-full branch: the distinct-host count is STILL 2, so it admitted without recording" as *u8, 882 (wqn5[0] == 2) as i64, pass, total) 883 gw_check("T25g the quota is DERIVED from the fetch stage's own LARGEST per-host cap, so no row the fetcher could have indexed is ever excluded from the window" as *u8, 884 (WC_WINQ == WC_HOSTCAP_CANON) as i64, pass, total) 885 gw_check("T25h a FULL pull window therefore offers at least 51 distinct hosts whenever that many are pending -- the arithmetic the whole diversity claim rests on" as *u8, 886 (WC_MAXPEND / WC_WINQ >= 51) as i64, pass, total) 887 gw_check("T25i NEG-CONTROL: the per-host caps themselves are UNTOUCHED -- a run that indexed more pages by RAISING them would have failed rung G0, not passed it" as *u8, 888 ((WC_HOSTCAP == 6) as i64) * ((WC_HOSTCAP_CANON == 40) as i64), pass, total) 889 890 // ---- T26: THE G0 CONTRACT PREDICATE (2026-09-03). wc_frontier_fair is the exact symbol the 891 // /compare/search watch row measures, so it has to BE the invariant rather than a name shaped like 892 // one -- a completion signal that keys on a name rewards writing the name. These teeth pin the 893 // boundary in BOTH directions and pin the two shapes that actually matter: the window we measured 894 // in production, and the window the fix is supposed to produce. 895 gw_check("T26a a window exactly at quota-times-hosts is FAIR" as *u8, 896 wc_frontier_fair(WC_WINQ * 4, 4), pass, total) 897 gw_check("T26b NEG-CONTROL: ONE row past that is NOT fair -- the boundary is exact, not an approximation" as *u8, 898 (wc_frontier_fair(WC_WINQ * 4 + 1, 4) == 0) as i64, pass, total) 899 gw_check("T26c NEG-CONTROL, the MEASURED case this rung exists for: a full WC_MAXPEND window drawn from ONE host is NOT fair -- that is what crawl_outcomes.log showed on 2026-08-11 and again on 2026-09-03" as *u8, 900 (wc_frontier_fair(WC_MAXPEND, 1) == 0) as i64, pass, total) 901 gw_check("T26d ...and the shape the fix produces IS fair: a full window spread over the 51-plus hosts WC_WINQ guarantees" as *u8, 902 wc_frontier_fair(WC_MAXPEND, WC_MAXPEND / WC_WINQ + 1), pass, total) 903 gw_check("T26e NEG-CONTROL: rows present with ZERO hosts attributed reads UNMEASURED, not balanced -- an abstention must never score as a pass" as *u8, 904 (wc_frontier_fair(WC_MAXPEND, 0) == 0) as i64, pass, total) 905 gw_check("T26f an EMPTY window is vacuously fair and says so, instead of dividing by zero" as *u8, 906 wc_frontier_fair(0, 0), pass, total) 907 gw_check("T26g the JUDGE agrees with the ENFORCER: wc_winq_admit can never let one host past WC_WINQ, so any window it built is fair by construction" as *u8, 908 wc_frontier_fair(WC_WINQ, 1), pass, total) 909 910 gw_puts("=== crawl-step gate: " as *u8); gw_num(pass[0]); gw_puts("/" as *u8); gw_num(total[0]); gw_puts(" " as *u8) 911 if pass[0] == total[0] { gw_puts("GREEN\n" as *u8) } else { gw_puts("RED\n" as *u8) } 912 let ctr: *i64 = gv_ctr() 913 ctr[0] = pass[0] 914 ctr[1] = total[0] 915 return gv_verdict("WEB-CRAWL-STEP-GATE" as *u8, ctr, "frontier store mechanics: url filter, P-to-D state flip, cross-segment dedupe, outlink capture" as *u8) 916}