code wiki / _hdl_build / nx_web_crawl_step_gate.nx
nx_web_crawl_step_gate.nx source
↩ module page · 916 lines · 62323 B
1// nx_web_crawl_step_gate.nx -- network-free KATs for the frontier-persisted crawler's STORE mechanics:
2// the url filter, fr: row write/enumerate, the P->D state flip (append-only last-wins), cross-segment
3// dedupe, and unknown-state. The fetch/ingest spine is nx_web_ingest's, already proven on the real web.
4// Runs against a /tmp shard (wiped per boot). license_tier: ORIGINAL
5import "nx_web_crawl_step.nx"
6import "nx_gate_verdict.nx"
7
8func gw_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
9func gw_num(v: i64) -> i64 {
10 let bb: *u8 = sys_mmap(28); var m: i64 = v
11 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
12 let t: *u8 = sys_mmap(28); var k: i64 = 0
13 if m == 0 { t[0] = 48 as u8; k = 1 }
14 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
15 var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 }
16 sys_write(1, bb, k); return 0
17}
18// Scratch for the T22 identity teeth: one rendered identity block (User-Agent + Accept-Language).
19// Named for its PURPOSE, not its value -- the ratchet refused the bare 1024 and was right to.
20const WCID_UABUF: i64 = 1024
21// The frontier population this gate's partition fixture is built on -- see its use below.
22// This is the frontier as measured BEFORE the R16/R17 work, recovered from six consecutive runs.
23const WCID_FIXTURE_FRONTIER_PENDING: i64 = 1873
24// The frontier as measured AFTER it drained, on the run whose hostcap_retired=227 motivated R16.
25// TWO DIFFERENT MEASUREMENTS AT TWO DIFFERENT TIMES, each named for which one it is. Conflating them
26// is exactly the error T24a/T24b caught: I wrote those teeth against 313 and passed them the 1873
27// constant, and 2048-1873=175 correctly floors to 300, so the teeth failed on a sound function.
28// A FIXTURE NAMED FOR ITS ROLE RATHER THAN ITS MOMENT INVITES THAT SUBSTITUTION.
29const WCID_FIXTURE_FRONTIER_DRAINED: i64 = 313
30
31// substring helpers for the T22 identity teeth. Named wcid_* so they cannot collide with a sibling.
32func wcid_count(hay: *u8, hn: i64, ndl: *u8) -> i64 {
33 var nl: i64 = 0
34 while ndl[nl] != (0 as u8) { nl = nl + 1 }
35 if nl == 0 { return 0 }
36 var c: i64 = 0
37 var i: i64 = 0
38 while i + nl <= hn {
39 var m: i64 = 1
40 var j: i64 = 0
41 while j < nl { if hay[i + j] != ndl[j] { m = 0; j = nl } else { j = j + 1 } }
42 if m == 1 { c = c + 1 }
43 i = i + 1
44 }
45 return c
46}
47func wcid_has(hay: *u8, hn: i64, ndl: *u8) -> i64 { if wcid_count(hay, hn, ndl) > 0 { return 1 } return 0 }
48
49func gw_check(name: *u8, cond: i64, pass: *i64, total: *i64) -> i64 {
50 total[0] = total[0] + 1
51 gw_puts(name)
52 if cond == 1 { pass[0] = pass[0] + 1; gw_puts(" PASS\n" as *u8) } else { gw_puts(" FAIL\n" as *u8) }
53 return 0
54}
55func gw_ok(u: *u8) -> i64 { return wc_urlok(u, wc_len(u)) }
56
57// --- R11 robots teeth: seed the per-run cache directly, so verdicts need NO network + NO TrustStore.
58func t_rb_seed_state(hh: i64, body: *u8, bl: i64, st: i64) -> i64 {
59 let idx: i64 = wc_rb_n_g
60 var z: i64 = 0
61 while z < bl { wc_rb_arena_g[wc_rb_used_g + z] = body[z]; z = z + 1 }
62 wc_rb_hash_g[idx] = hh
63 wc_rb_off_g[idx] = wc_rb_used_g
64 wc_rb_len_g[idx] = bl
65 wc_rb_state_g[idx] = st
66 wc_rb_used_g = wc_rb_used_g + bl
67 wc_rb_n_g = idx + 1
68 return idx
69}
70// The pre-existing seeder, preserved for every tooth already written against it. Its state is DERIVED
71// by asking the SHIPPING classifier what a successful fetch of this body would yield, rather than by
72// hand-writing a state here: a fixture that restates the rule under test can never disagree with it,
73// and a fixture that cannot disagree is not evidence.
74func t_rb_seed(hh: i64, body: *u8, bl: i64) -> i64 {
75 return t_rb_seed_state(hh, body, bl, wc_rb_classify(WC_HTTP_2XX_LO, bl))
76}
77func t_rb_streq(a: *u8, b: *u8) -> i64 {
78 var i: i64 = 0
79 while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 }
80 if b[i] != (0 as u8) { return 0 }
81 return 1
82}
83
84func main() -> i64 {
85 gw_puts("=== nx_web_crawl_step gate (frontier store mechanics, network-free) ===\n" as *u8)
86 let pass: *i64 = sys_mmap(16) as *i64; pass[0] = 0
87 let total: *i64 = sys_mmap(16) as *i64; total[0] = 0
88
89 // R10: establish the query policy ONCE, up front, so EVERY tooth below runs under a known,
90 // self-contained deny list rather than whatever the live urlpolicy- plane happens to hold.
91 // 'action' earns its place next to 'utm_source': ?action=edit|raw|history is a MediaWiki ACTION
92 // ENDPOINT, not a document. R8 killed it for free via the blanket '?' reject; once queries are
93 // judged by shape, that free kill has to be paid for explicitly -- and T1 is what noticed.
94 // ★REMOVING A BLANKET RULE BILLS YOU FOR EVERY CASE IT WAS SILENTLY COVERING.
95 let gqd: *u8 = sys_mmap(32 * 8)
96 var gqi: i64 = 0
97 let gqd0: *u8 = "utm_source" as *u8
98 while gqd0[gqi] != (0 as u8) { gqd[gqi] = gqd0[gqi]; gqi = gqi + 1 }
99 gqd[gqi] = 0 as u8
100 let gqd1: *u8 = "action" as *u8
101 var gqj: i64 = 0
102 while gqd1[gqj] != (0 as u8) { gqd[32 + gqj] = gqd1[gqj]; gqj = gqj + 1 }
103 gqd[32 + gqj] = 0 as u8
104 wc_qdeny_g = gqd
105 wc_nqdeny_g = 2
106
107 // T1 url filter: keeps article urls, kills queries/fragments/meta/binaries
108 var f1: i64 = 1
109 if gw_ok("https://en.wikipedia.org/wiki/Chess" as *u8) != 1 { f1 = 0 }
110 if gw_ok("https://example.org/a/deep/page" as *u8) != 1 { f1 = 0 }
111 if gw_ok("https://en.wikipedia.org/w/index.php?action=edit" as *u8) != 0 { f1 = 0 }
112 if gw_ok("https://en.wikipedia.org/wiki/Chess#History" as *u8) != 0 { f1 = 0 }
113 if gw_ok("https://en.wikipedia.org/wiki/Special:Random" as *u8) != 0 { f1 = 0 }
114 if gw_ok("https://en.wikipedia.org/wiki/File:Board.png" as *u8) != 0 { f1 = 0 }
115 if gw_ok("https://x.org/logo.png" as *u8) != 0 { f1 = 0 }
116 if gw_ok("https://x.org/style.css" as *u8) != 0 { f1 = 0 }
117 if gw_ok("mailto:someone@example.org" as *u8) != 0 { f1 = 0 }
118 if gw_ok("/relative/only" as *u8) != 0 { f1 = 0 }
119 if gw_ok("https://ro.wikipedia.org/wiki/Literatura" as *u8) != 0 { f1 = 0 } // interwiki lang edition
120 if gw_ok("https://en.wikipedia.org/wiki/Chess_opening" as *u8) != 1 { f1 = 0 } // en stays
121 if gw_ok("https://commons.wikimedia.org/wiki/X" as *u8) != 0 { f1 = 0 }
122 // API / feed endpoints = real bytes but not content -> rejected
123 if gw_ok("https://creativecommons.org/wp-json/" as *u8) != 0 { f1 = 0 }
124 if gw_ok("https://site.com/api/data.json" as *u8) != 0 { f1 = 0 }
125 if gw_ok("https://blog.com/feed" as *u8) != 0 { f1 = 0 }
126 if gw_ok("https://blog.com/comments/feed/" as *u8) != 0 { f1 = 0 }
127 if gw_ok("https://site.com/articles/a-real-story" as *u8) != 1 { f1 = 0 } // content still passes
128 // anti-slop listing classes (2026-08-04): tag/category link farms refused at admission
129 if gw_ok("https://healthquerys.com/tag/puriva-france/" as *u8) != 0 { f1 = 0 }
130 if gw_ok("https://blog.com/category/wellness/" as *u8) != 0 { f1 = 0 }
131 if gw_ok("https://blog.com/posts/2024/tagged-story" as *u8) != 1 { f1 = 0 } // 'tag' inside a word still passes
132 gw_check("T1 url filter (21 KATs incl. interwiki + API/feed + tag/category farms)" as *u8, f1, pass, total)
133
134 // FIXTURE SELF-CLEAN (2026-08-04, debt 1785882035): /tmp persists across runs on the NAS, so a
135 // previous run's segments made the count teeth (T2-T4) go 4/7 RED on any second invocation -- a
136 // gate that is only green on a fresh boot is a boot-witness, not a gate. Unlink the whole fixture.
137 sys_unlinkat("/tmp/crawlgate-manifest.txt" as *u8)
138 let cleanbuf: *u8 = sys_mmap(96)
139 let cdig: *u8 = sys_mmap(28)
140 var ci0: i64 = 1
141 while ci0 <= 32 {
142 var ce: i64 = 0
143 while ce < 5 {
144 var ext: *u8 = ".docs" as *u8
145 if ce == 1 { ext = ".idx" as *u8 }
146 if ce == 2 { ext = ".pos" as *u8 }
147 if ce == 3 { ext = ".imp" as *u8 }
148 if ce == 4 { ext = ".terms" as *u8 }
149 var co: i64 = 0
150 let cpre: *u8 = "/tmp/crawlgate-seg-" as *u8
151 while cpre[co] != (0 as u8) { cleanbuf[co] = cpre[co]; co = co + 1 }
152 var cm: i64 = ci0
153 var ck: i64 = 0
154 while cm > 0 { cdig[ck] = (48 + (cm % 10)) as u8; cm = cm / 10; ck = ck + 1 }
155 var cj: i64 = 0
156 while cj < ck { cleanbuf[co] = cdig[ck - 1 - cj]; co = co + 1; cj = cj + 1 }
157 var cx: i64 = 0
158 while ext[cx] != (0 as u8) { cleanbuf[co] = ext[cx]; co = co + 1; cx = cx + 1 }
159 cleanbuf[co] = 0 as u8
160 sys_unlinkat(cleanbuf)
161 ce = ce + 1
162 }
163 ci0 = ci0 + 1
164 }
165
166 // build a tiny /tmp frontier shard
167 let prefix: *u8 = "/tmp/crawlgate-" as *u8
168 let segidbox: *i64 = sys_mmap(16) as *i64; segidbox[0] = 1
169 let segsbox: *i64 = sys_mmap(16) as *i64; segsbox[0] = 0
170 let w: *i64 = ss_begin()
171 let u1: *u8 = "https://a.example/one" as *u8
172 let u2: *u8 = "https://a.example/two" as *u8
173 let u3: *u8 = "https://a.example/three" as *u8
174 wc_frmark(w, prefix, segidbox, segsbox, u1, wc_len(u1), 1)
175 wc_frmark(w, prefix, segidbox, segsbox, u2, wc_len(u2), 1)
176 wc_frmark(w, prefix, segidbox, segsbox, u3, wc_len(u3), 2) // three = already done
177 ss_commit(prefix, w, segidbox[0])
178
179 // T2 enumerate: exactly the two P urls
180 let h1: *i64 = ss_open(prefix)
181 let urls: *u8 = sys_mmap(WC_MAGIC_2048 * 16)
182 let canh0: *u8 = sys_mmap(64 * 8) // R4: never read when canonly=0 -- T2/T3/T4 keep the pre-R4 contract
183 let n2: i64 = wc_pending(h1, prefix, urls, 16, canh0, 0, 0)
184 gw_check("T2 pending enumeration -> 2 (P rows only)" as *u8, (n2 == 2) as i64, pass, total)
185
186 // T3 state flip: mark one P->D in a NEW segment; enumeration drops to 1 (last-wins across segments)
187 segidbox[0] = segidbox[0] + 1
188 let w2: *i64 = ss_begin()
189 wc_frmark(w2, prefix, segidbox, segsbox, u1, wc_len(u1), 2)
190 ss_commit(prefix, w2, segidbox[0])
191 let h2: *i64 = ss_open(prefix)
192 let n3: i64 = wc_pending(h2, prefix, urls, 16, canh0, 0, 0)
193 var t3: i64 = 0
194 if n3 == 1 {
195 // the survivor must be u2
196 var eq: i64 = 1
197 var x: i64 = 0
198 var go: i64 = 1
199 while go == 1 { if urls[x] != u2[x] { eq = 0; go = 0 } else { if u2[x] == (0 as u8) { go = 0 } else { x = x + 1 } } }
200 if eq == 1 { t3 = 1 }
201 }
202 gw_check("T3 P->D flip (append-only, last-wins) -> 1 pending = /two" as *u8, t3, pass, total)
203
204 // T4 duplicate P rows across segments dedupe to one pending
205 segidbox[0] = segidbox[0] + 1
206 let w3: *i64 = ss_begin()
207 wc_frmark(w3, prefix, segidbox, segsbox, u2, wc_len(u2), 1) // re-bank the SAME pending url
208 ss_commit(prefix, w3, segidbox[0])
209 let h3: *i64 = ss_open(prefix)
210 let n4: i64 = wc_pending(h3, prefix, urls, 16, canh0, 0, 0)
211 gw_check("T4 dup P rows across segments dedupe -> still 1" as *u8, (n4 == 1) as i64, pass, total)
212
213 // T5 unknown state for a never-seen url
214 let s5: i64 = wc_frstate(h3, "https://a.example/never" as *u8, 23)
215 gw_check("T5 unknown url -> state 0" as *u8, (s5 == 0) as i64, pass, total)
216 // T5b known states read back
217 var t5b: i64 = 0
218 if wc_frstate(h3, u2, wc_len(u2)) == 1 { if wc_frstate(h3, u1, wc_len(u1)) == 2 { t5b = 1 } }
219 gw_check("T5b states read back (two=P, one=D)" as *u8, t5b, pass, total)
220
221 // T6 (P1 LINK GRAPH): wc_harvest captures ALL of a page's outlinks as out:<cid(base)> = packed i64 target
222 // cid(url)s -- the edge list PageRank runs on. Distinguishing by construction: before this change the
223 // crawler stored NO edges, so out:<cid> is absent and t6=0.
224 let t1u: *u8 = "https://tgt1.example/story-alpha" as *u8
225 let t2u: *u8 = "https://tgt2.example/story-beta" as *u8
226 let hbase: *u8 = "https://src.example/page-x" as *u8
227 let html6: *u8 = "<html><body><a href='https://tgt1.example/story-alpha'>A</a> <a href='https://tgt2.example/story-beta'>B</a></body></html>" as *u8
228 let hdisc: *i64 = sys_mmap(16) as *i64; hdisc[0] = 0
229 let hrs: *u8 = sys_mmap(256 * 64); let hnrs: *i64 = sys_mmap(16) as *i64; hnrs[0] = 0
230 segidbox[0] = segidbox[0] + 1
231 let w6: *i64 = ss_begin()
232 wc_harvest(html6, wc_len(html6), hbase, wc_len(hbase), 0 as *i64, w6, prefix, segidbox, segsbox, hdisc, hrs, hnrs)
233 ss_commit(prefix, w6, segidbox[0])
234 let h6: *i64 = ss_open(prefix)
235 let okey6: *u8 = sys_mmap(64)
236 wc_outkey(ci_hash(hbase, wc_len(hbase)), okey6)
237 let op6: *i64 = sys_mmap(16) as *i64
238 let ol6: *i64 = sys_mmap(16) as *i64
239 var t6: i64 = 0
240 if ss_hget(h6, okey6, op6, ol6) == 1 { if ol6[0] == 16 {
241 let ev: *i64 = op6[0] as *i64
242 if ev[0] == ci_hash(t1u, wc_len(t1u)) { if ev[1] == ci_hash(t2u, wc_len(t2u)) { t6 = 1 } }
243 } }
244 gw_check("T6 P1 outlink capture (out:<cid(base)> = [cid(tgt1),cid(tgt2)], ordered)" as *u8, t6, pass, total)
245
246 // T7 (R3 host-health): hd:<hosthash> streak round-trip; an unknown host reads 0
247 segidbox[0] = segidbox[0] + 1
248 let w7: *i64 = ss_begin()
249 let hhA: i64 = wc_hosthash("dead.example" as *u8, 12)
250 wc_hd_put(w7, prefix, segidbox, segsbox, hhA, 3)
251 ss_commit(prefix, w7, segidbox[0])
252 let h7: *i64 = ss_open(prefix)
253 var t7: i64 = 0
254 if wc_hd_streak(h7, hhA) == 3 { if wc_hd_streak(h7, wc_hosthash("live.example" as *u8, 12)) == 0 { t7 = 1 } }
255 gw_check("T7 hd: streak round-trip (3) + unknown-host 0" as *u8, t7, pass, total)
256
257 // T8 (R3): retire-if-rotten -- streak>=WC_HD_RETIRE retires a P row (P->D visible after commit),
258 // streak below the bar keeps it pending, and a SEED url NEVER retires even on a rotten host
259 // (wc_seed_add is blocked by any existing fr: row, so a retired seed would be permanently dead).
260 let ur1: *u8 = "https://dead.example/rot1" as *u8
261 let ur2: *u8 = "https://dying.example/fresh" as *u8
262 let ur3: *u8 = "https://dead.example/seedhub" as *u8
263 segidbox[0] = segidbox[0] + 1
264 let w8: *i64 = ss_begin()
265 wc_frmark(w8, prefix, segidbox, segsbox, ur1, wc_len(ur1), 1)
266 wc_frmark(w8, prefix, segidbox, segsbox, ur2, wc_len(ur2), 1)
267 wc_frmark(w8, prefix, segidbox, segsbox, ur3, wc_len(ur3), 1)
268 let hhB: i64 = wc_hosthash("dying.example" as *u8, 13)
269 wc_hd_put(w8, prefix, segidbox, segsbox, hhB, 2)
270 ss_commit(prefix, w8, segidbox[0])
271 let h8: *i64 = ss_open(prefix)
272 let seeds8: *u8 = sys_mmap(WC_MAGIC_2048)
273 var so8: i64 = 0
274 while ur3[so8] != (0 as u8) { seeds8[so8] = ur3[so8]; so8 = so8 + 1 }
275 seeds8[so8] = 0 as u8
276 segidbox[0] = segidbox[0] + 1
277 let w9: *i64 = ss_begin()
278 // the caller pattern: seed membership guards the retire (exactly what main's skip branch does)
279 var r1: i64 = 0
280 if wc_is_seed(seeds8, 1, ur1, wc_len(ur1)) == 0 { r1 = wc_retire_if_rotten(h8, w9, prefix, segidbox, segsbox, ur1, wc_len(ur1), hhA) }
281 var r2: i64 = 0
282 if wc_is_seed(seeds8, 1, ur2, wc_len(ur2)) == 0 { r2 = wc_retire_if_rotten(h8, w9, prefix, segidbox, segsbox, ur2, wc_len(ur2), hhB) }
283 var r3: i64 = 0
284 if wc_is_seed(seeds8, 1, ur3, wc_len(ur3)) == 0 { r3 = wc_retire_if_rotten(h8, w9, prefix, segidbox, segsbox, ur3, wc_len(ur3), hhA) }
285 if w9[1] > 0 { ss_commit(prefix, w9, segidbox[0]) }
286 let h9: *i64 = ss_open(prefix)
287 var t8: i64 = 0
288 if r1 == 1 { if r2 == 0 { if r3 == 0 {
289 if wc_frstate(h9, ur1, wc_len(ur1)) == 2 { if wc_frstate(h9, ur2, wc_len(ur2)) == 1 { if wc_frstate(h9, ur3, wc_len(ur3)) == 1 { t8 = 1 } } }
290 } } }
291 gw_check("T8 retire-if-rotten: streak>=3 retires, <3 keeps, seed exempt" as *u8, t8, pass, total)
292
293 // ---- T9 (R4 CANONICAL LANE) ---------------------------------------------------------------
294 // The host-filtered pull must keep ONLY canonical-seed hosts, match a SUBDOMAIN on the boundary
295 // dot, and REFUSE a lookalike host. Network-free -- store rows only. The pre-existing fixture
296 // rows (a.example / b.example) are non-canonical by construction, so they are excluded too and
297 // the count is stable regardless of what T1-T8 left pending.
298 segidbox[0] = segidbox[0] + 1
299 let q9w: *i64 = ss_begin()
300 let q9c1: *u8 = "https://en.wikipedia.org/wiki/Hypertension" as *u8
301 let q9c2: *u8 = "https://data.who.int/topic/air" as *u8
302 let q9c3: *u8 = "https://notwikipedia.org/lookalike" as *u8
303 let q9c4: *u8 = "https://junkfarm.example/tag/a" as *u8
304 wc_frmark(q9w, prefix, segidbox, segsbox, q9c1, wc_len(q9c1), 1)
305 wc_frmark(q9w, prefix, segidbox, segsbox, q9c2, wc_len(q9c2), 1)
306 wc_frmark(q9w, prefix, segidbox, segsbox, q9c3, wc_len(q9c3), 1)
307 wc_frmark(q9w, prefix, segidbox, segsbox, q9c4, wc_len(q9c4), 1)
308 ss_commit(prefix, q9w, segidbox[0])
309 let q9seeds: *u8 = sys_mmap(WC_MAGIC_2048 * 4)
310 let q9s1: *u8 = "https://www.who.int/" as *u8
311 var q9a: i64 = 0
312 while q9s1[q9a] != (0 as u8) { q9seeds[q9a] = q9s1[q9a]; q9a = q9a + 1 }
313 q9seeds[q9a] = 0 as u8
314 let q9d2: *u8 = (q9seeds as i64 + WC_MAGIC_2048) as *u8
315 let q9s2: *u8 = "https://wikipedia.org/" as *u8
316 var q9b: i64 = 0
317 while q9s2[q9b] != (0 as u8) { q9d2[q9b] = q9s2[q9b]; q9b = q9b + 1 }
318 q9d2[q9b] = 0 as u8
319 let q9canh: *u8 = sys_mmap(64 * WC_MAXCANH)
320 let q9ncan: i64 = wc_canon_hosts(q9seeds, 2, q9canh)
321 let q9h: *i64 = ss_open(prefix)
322 let q9urls: *u8 = sys_mmap(WC_MAGIC_2048 * 32)
323 let q9n: i64 = wc_pending(q9h, prefix, q9urls, 32, q9canh, q9ncan, 1)
324 var t9: i64 = 0
325 if q9ncan == 2 { if wc_streq(q9canh, "who.int" as *u8) == 1 {
326 if wc_streq((q9canh as i64 + 64) as *u8, "wikipedia.org" as *u8) == 1 { if q9n == 2 { t9 = 1 } }
327 } }
328 gw_check("T9 canonical lane: www-stripped hosts, subdomain kept, junk excluded -> 2 of 4" as *u8, t9, pass, total)
329 // T9b THE NEGATIVE CONTROL that makes T9 non-vacuous: a lookalike host must NOT ride the lane.
330 let q9hb: *u8 = sys_mmap(64)
331 var t9b: i64 = 0
332 if wc_host_is_canon(q9canh, q9ncan, q9c1, q9hb) == 1 { if wc_host_is_canon(q9canh, q9ncan, q9c2, q9hb) == 1 {
333 if wc_host_is_canon(q9canh, q9ncan, q9c3, q9hb) == 0 { if wc_host_is_canon(q9canh, q9ncan, q9c4, q9hb) == 0 { t9b = 1 } }
334 } }
335 gw_check("T9b lookalike REFUSED: notwikipedia.org needs the boundary dot to match wikipedia.org" as *u8, t9b, pass, total)
336 // T9c canonly=0 is the pre-R4 contract: the SAME walk sees all four rows (plus fixture leftovers).
337 let q9all: i64 = wc_pending(q9h, prefix, q9urls, 32, q9canh, 0, 0)
338 gw_check("T9c canonly=0 unchanged: unfiltered walk still sees strictly more rows than the lane" as *u8, (q9all > q9n) as i64, pass, total)
339 // T9d (R4b): the host-string matcher is the ONE rule and the url matcher only wraps it. If these
340 // two ever disagree, the canonical LANE and the canonical DEPTH CAP would disagree about which
341 // hosts are curated -- a split definition is exactly how a privilege leaks to a lookalike host.
342 var q9agree: i64 = 1
343 if wc_host_str_is_canon(q9canh, q9ncan, "wikipedia.org" as *u8, 13) != 1 { q9agree = 0 }
344 if wc_host_str_is_canon(q9canh, q9ncan, "en.wikipedia.org" as *u8, 16) != 1 { q9agree = 0 }
345 if wc_host_str_is_canon(q9canh, q9ncan, "data.who.int" as *u8, 12) != 1 { q9agree = 0 }
346 if wc_host_str_is_canon(q9canh, q9ncan, "notwikipedia.org" as *u8, 16) != 0 { q9agree = 0 }
347 if wc_host_str_is_canon(q9canh, q9ncan, "junkfarm.example" as *u8, 16) != 0 { q9agree = 0 }
348 gw_check("T9d host-string matcher == url matcher (exact, subdomain, lookalike, junk)" as *u8, q9agree, pass, total)
349
350 // ---- T10 (R5 SEED FRESHNESS RE-QUEUE) -------------------------------------------------------
351 // A D seed must return to P and be copied into the lane buffer; a P seed and a never-seen seed
352 // must be left alone. Network-free, store rows only.
353 segidbox[0] = segidbox[0] + 1
354 let qAw: *i64 = ss_begin()
355 let qA1: *u8 = "https://canon-a.example/hub" as *u8
356 let qA2: *u8 = "https://canon-b.example/hub" as *u8
357 let qA3: *u8 = "https://canon-c.example/hub" as *u8
358 wc_frmark(qAw, prefix, segidbox, segsbox, qA1, wc_len(qA1), 2)
359 wc_frmark(qAw, prefix, segidbox, segsbox, qA2, wc_len(qA2), 1)
360 ss_commit(prefix, qAw, segidbox[0])
361 let qAseeds: *u8 = sys_mmap(WC_MAGIC_2048 * 4)
362 var qAi: i64 = 0
363 while qA1[qAi] != (0 as u8) { qAseeds[qAi] = qA1[qAi]; qAi = qAi + 1 }
364 qAseeds[qAi] = 0 as u8
365 let qAd2: *u8 = (qAseeds as i64 + WC_MAGIC_2048) as *u8
366 qAi = 0
367 while qA2[qAi] != (0 as u8) { qAd2[qAi] = qA2[qAi]; qAi = qAi + 1 }
368 qAd2[qAi] = 0 as u8
369 let qAd3: *u8 = (qAseeds as i64 + 2 * WC_MAGIC_2048) as *u8
370 qAi = 0
371 while qA3[qAi] != (0 as u8) { qAd3[qAi] = qA3[qAi]; qAi = qAi + 1 }
372 qAd3[qAi] = 0 as u8
373 let qAh: *i64 = ss_open(prefix)
374 segidbox[0] = segidbox[0] + 1
375 let qAw2: *i64 = ss_begin()
376 let qAdst: *u8 = sys_mmap(WC_MAGIC_2048 * 8)
377 let qAbox: *i64 = sys_mmap(16) as *i64
378 qAbox[0] = 0
379 qAbox[1] = 6
380 let qArq: i64 = wc_seed_refresh(qAh, qAw2, prefix, segidbox, segsbox, qAseeds, 3, qAdst, qAbox)
381 if qAw2[1] > 0 { ss_commit(prefix, qAw2, segidbox[0]) }
382 let qAh2: *i64 = ss_open(prefix)
383 var t10: i64 = 0
384 if qArq == 1 { if qAbox[0] == 1 {
385 if wc_frstate(qAh2, qA1, wc_len(qA1)) == 1 { if wc_frstate(qAh2, qA2, wc_len(qA2)) == 1 { if wc_frstate(qAh2, qA3, wc_len(qA3)) == 0 {
386 if wc_streq(qAdst, qA1) == 1 { t10 = 1 }
387 } } }
388 } }
389 gw_check("T10 seed re-queue: D seed -> P + copied to lane; P and never-seen UNTOUCHED" as *u8, t10, pass, total)
390 // T10b the persisted cursor round-trips (the rotation mechanism across runs)
391 segidbox[0] = segidbox[0] + 1
392 let qAw3: *i64 = ss_begin()
393 wc_sr_put(qAw3, prefix, segidbox, segsbox, 5)
394 if qAw3[1] > 0 { ss_commit(prefix, qAw3, segidbox[0]) }
395 let qAh3: *i64 = ss_open(prefix)
396 gw_check("T10b sr:cursor persists across commit (put 5 -> read 5)" as *u8, (wc_sr_cursor(qAh3) == 5) as i64, pass, total)
397
398 // ---- T11 (R10 SHAPE-BASED QUERY ADMISSION -- replaces R8's host allowlist) -------------------
399 // R8 admitted query strings ONLY on curated hosts, so an uncurated content url was refused before
400 // a byte was fetched -- a DIRECTORY, not an index. R10 judges the query's SHAPE instead, so the
401 // SAME two fixtures now split on WHAT THE URL IS rather than WHO SERVES IT:
402 // clean content query, uncurated host -> ADMITTED (this is the whole point of R10)
403 // tracking param, uncurated host -> REFUSED (a duplicate of the clean url)
404 // Deny list set directly here, exactly as the canon table is a few lines below: the gate must be
405 // self-contained, never dependent on whatever the live urlpolicy- plane happens to hold today.
406 let qB: *u8 = "https://graphis.ne.jp/monthly/model.php?ID=260731_sp" as *u8
407 let qBjunk: *u8 = "https://tracker.example/page.php?utm_source=x" as *u8
408 var t11a: i64 = 0
409 if wc_urlok(qB, wc_len(qB)) == 1 { if wc_urlok(qBjunk, wc_len(qBjunk)) == 0 { t11a = 1 } }
410 gw_check("T11 R10: a CLEAN query url is ADMITTED with an EMPTY canon table; a TRACKING one is REFUSED" as *u8, t11a, pass, total)
411 // THE TOOTH THAT WOULD HAVE CAUGHT R8: the operator's real-world miss. An uncurated host serving
412 // paginated content must be reachable -- under R8 this was refused purely for lacking curation.
413 let qBv2: *u8 = "https://www.v2ph.com/actor/JULIA?page=3&hl=en" as *u8
414 gw_check("T11a-ii an UNCURATED host's paginated content url is ADMITTED (the R8 miss, by name)" as *u8, (wc_urlok(qBv2, wc_len(qBv2)) == 1) as i64, pass, total)
415 // and the trap half: a facet explosion beyond the param budget is still refused, on ANY host
416 let qBfacet: *u8 = "https://shop.example/list?a=1&b=2&c=3&d=4&e=5&f=6" as *u8
417 gw_check("T11a-iii a 6-param facet explosion is REFUSED (budget, not curation, is the limit)" as *u8, (wc_urlok(qBfacet, wc_len(qBfacet)) == 0) as i64, pass, total)
418 // now publish a canon table holding ONLY graphis.ne.jp
419 let qBcanh: *u8 = sys_mmap(64 * WC_MAXCANH)
420 var qBi: i64 = 0
421 let qBhost: *u8 = "graphis.ne.jp" as *u8
422 while qBhost[qBi] != (0 as u8) { qBcanh[qBi] = qBhost[qBi]; qBi = qBi + 1 }
423 qBcanh[qBi] = 0 as u8
424 wc_canh_g = qBcanh
425 wc_ncan_g = 1
426 wc_qscratch_g = sys_mmap(64)
427 // BISECT: which layer is failing -- the policy flag, or the canon match?
428 let t11p: i64 = olh_urlok_q(qB, wc_len(qB), 1)
429 gw_check("T11b-i policy layer: olh_urlok_q(allowq=1) admits the query-string url" as *u8, (t11p == 1) as i64, pass, total)
430 let t11m: i64 = wc_host_is_canon(qBcanh, 1, qB, wc_qscratch_g)
431 gw_check("T11b-ii matcher layer: wc_host_is_canon sees graphis.ne.jp in the table" as *u8, (t11m == 1) as i64, pass, total)
432 // R10: curation is now a BOOST, not a gate. The curated host keeps admitting its content, and the
433 // tracking url stays refused ON THE CURATED PATH TOO -- trust buys a bigger param budget, it does
434 // NOT buy immunity from junk. A curated host that could launder tracking urls into the index would
435 // make curation a liability rather than a signal.
436 var t11b: i64 = 0
437 if wc_urlok(qB, wc_len(qB)) == 1 { if wc_urlok(qBjunk, wc_len(qBjunk)) == 0 { t11b = 1 } }
438 let qBcurjunk: *u8 = "https://graphis.ne.jp/p.php?utm_source=x" as *u8
439 gw_check("T11b curated host ADMITS its content; a TRACKING url is refused on the CURATED path too" as *u8, (t11b == 1) as i64 * (wc_urlok(qBcurjunk, wc_len(qBcurjunk)) == 0) as i64, pass, total)
440 // the other kill-classes must NOT be weakened for the curated host either
441 let qBtag: *u8 = "https://graphis.ne.jp/tag/foo?ID=1" as *u8
442 let qBimg: *u8 = "https://graphis.ne.jp/i/photo.jpg" as *u8
443 var t11c: i64 = 0
444 if wc_urlok(qBtag, wc_len(qBtag)) == 0 { if wc_urlok(qBimg, wc_len(qBimg)) == 0 { t11c = 1 } }
445 gw_check("T11c curation relaxes ONLY '?' -- tag-farm and binary-extension kills still hold" as *u8, t11c, pass, total)
446 // T12 (R9): a CURATED host is never retired -- we cannot distinguish "host is down" from "we
447 // cannot speak its TLS", so retiring it would launder our own capability gap into permanent
448 // coverage loss. Uncurated rot must STILL retire (that is the control).
449 var t12: i64 = 0
450 let qCu: *u8 = "https://graphis.ne.jp/some/page" as *u8
451 let qCj: *u8 = "https://rotten.example/dead" as *u8
452 var qCcur: i64 = 0
453 var qCrot: i64 = 0
454 if wc_host_is_canon(qBcanh, 1, qCu, wc_qscratch_g) == 1 { qCcur = 1 }
455 if wc_host_is_canon(qBcanh, 1, qCj, wc_qscratch_g) == 0 { qCrot = 1 }
456 if qCcur == 1 { if qCrot == 1 { t12 = 1 } }
457 gw_check("T12 retire-exemption predicate: curated host protected, uncurated rot still retirable" as *u8, t12, pass, total)
458 wc_ncan_g = 0
459
460 // ================= T13: R11 ROBOTS LANE (network-free) =================
461 wc_rb_init()
462 let rbnull: *TrustStore = 0 as *TrustStore // never dereferenced: every case below is a cache HIT
463 let rbh: *u8 = "example.org" as *u8
464 let rbhl: i64 = wc_len(rbh)
465 let pb: *u8 = sys_mmap(WC_MAGIC_2048)
466
467 // T13a -- path extraction. robots matching is defined over path+query, never over the host.
468 let uA: *u8 = "https://example.org/a/b?x=1" as *u8
469 var t13a: i64 = 0
470 if wc_rb_path(uA, wc_len(uA), pb, WC_MAGIC_2048) == 8 { if t_rb_streq(pb, "/a/b?x=1" as *u8) == 1 { t13a = 1 } }
471 gw_check("T13a robots path = path+query (/a/b?x=1)" as *u8, t13a, pass, total)
472 let uB: *u8 = "https://example.org" as *u8 // no path at all -> "/"
473 wc_rb_path(uB, wc_len(uB), pb, WC_MAGIC_2048)
474 gw_check("T13a2 a url with no path yields \"/\", never an empty match target" as *u8, t_rb_streq(pb, "/" as *u8), pass, total)
475
476 // ============ T13b FAMILY -- RFC 9309 s2.3.1 ACCESS-RESULT STATES (rewritten 2026-08-25) ======
477 // WHAT THIS REPLACED. There used to be ONE tooth here reading "absent/unfetchable robots.txt ->
478 // ALLOW", which asserted a single policy for a sentence the RFC splits in two:
479 // s2.3.1.3 "If a server status code indicates that the robots.txt file is unavailable to the
480 // crawler, then the crawler MAY access any resources on the server." (4xx)
481 // s2.3.1.4 "If the robots.txt file is unreachable due to server or network errors, this means
482 // the robots.txt file is undefined and the crawler MUST assume complete disallow."
483 // The old tooth PASSED against a host answering 500, because 404 and 500 both produced a len-0
484 // cache slot and the code branched on LENGTH rather than on POLICY. One tooth covering two rules
485 // with opposite outcomes can only ever prove the one it was written for.
486 // Every classifier tooth below is network-free by construction: wc_rb_classify takes (status,
487 // bytes) and nothing else, which is why the rule is testable at all now.
488
489 gw_check("T13b1 s2.3.1.3 404 -> ALLOW_ALL (unavailable: crawler MAY access any resources)" as *u8,
490 (wc_rb_classify(404, 0) == WC_RB_ST_ALLOW_ALL) as i64, pass, total)
491 gw_check("T13b2 s2.3.1.4 500 -> DISALLOW_ALL (unreachable: crawler MUST assume complete disallow)" as *u8,
492 (wc_rb_classify(500, 0) == WC_RB_ST_DISALLOW_ALL) as i64, pass, total)
493 gw_check("T13b3 s2.3.1.4 503 -> DISALLOW_ALL (the overload/maintenance case we actually meet)" as *u8,
494 (wc_rb_classify(503, 0) == WC_RB_ST_DISALLOW_ALL) as i64, pass, total)
495 gw_check("T13b4 s2.3.1.4 NETWORK FAILURE, no status line at all -> DISALLOW_ALL" as *u8,
496 (wc_rb_classify(WC_HTTP_NO_STATUS, 0 - 2) == WC_RB_ST_DISALLOW_ALL) as i64, pass, total)
497
498 // The boundary is the whole content of the rule, so it is asserted rather than assumed.
499 var t13b5: i64 = 0
500 if wc_rb_classify(WC_HTTP_4XX_LO, 0) == WC_RB_ST_ALLOW_ALL {
501 if wc_rb_classify(WC_HTTP_4XX_HI, 0) == WC_RB_ST_ALLOW_ALL {
502 if wc_rb_classify(WC_HTTP_5XX_LO, 0) == WC_RB_ST_DISALLOW_ALL {
503 if wc_rb_classify(WC_HTTP_5XX_HI, 0) == WC_RB_ST_DISALLOW_ALL { t13b5 = 1 } } } }
504 gw_check("T13b5 the 4xx/5xx boundary sits where the RFC puts it: 400+499 allow, 500+599 disallow" as *u8,
505 t13b5, pass, total)
506
507 // ---- NEG-CONTROLS. THE POINT: an implementation that simply disallowed everything would pass
508 // T13b2/3/4/5 and read as a compliance win. These are the teeth it cannot pass.
509 gw_check("T13b6 neg-control-200-WITH-RULES-STILL-PARSES (blanket-disallow fails here)" as *u8,
510 (wc_rb_classify(WC_HTTP_2XX_LO, 33) == WC_RB_ST_RULES) as i64, pass, total)
511 gw_check("T13b7 neg-control-200-EMPTY-BODY-ALLOWS (a successful fetch of a file stating no rules)" as *u8,
512 (wc_rb_classify(WC_HTTP_2XX_LO, 0) == WC_RB_ST_ALLOW_ALL) as i64, pass, total)
513 gw_check("T13b8 200 whose body could NOT be delivered -> DISALLOW_ALL (we never read the rules)" as *u8,
514 (wc_rb_classify(WC_HTTP_2XX_LO, 0 - 9) == WC_RB_ST_DISALLOW_ALL) as i64, pass, total)
515 gw_check("T13b9 s2.3.1.2 a 3xx still unresolved past the redirect budget -> ALLOW_ALL (MAY assume unavailable)" as *u8,
516 (wc_rb_classify(302, 0) == WC_RB_ST_ALLOW_ALL) as i64, pass, total)
517
518 // The undefined-status door shares a POLICY with 5xx but must not share its LOG LABEL, or a
519 // response-parser bug would hide inside traffic we expect to see.
520 var t13b10: i64 = 0
521 if wc_rb_status_undefined(100) == 1 {
522 if wc_rb_status_undefined(500) == 0 {
523 if wc_rb_status_undefined(WC_HTTP_NO_STATUS) == 0 { t13b10 = 1 } } }
524 gw_check("T13b10 an UNDEFINED status is distinguishable from an honest 5xx and from no-status-at-all" as *u8,
525 t13b10, pass, total)
526
527 // ---- END-TO-END: the state has to actually reach the decision. ASSERT THE FIXTURE REACHED THE
528 // CONDITION FIRST -- a seeder that silently failed would otherwise produce a PASS meaning nothing.
529 let hh_500: i64 = wc_hosthash("broken.example" as *u8, 14)
530 let idx500: i64 = t_rb_seed_state(hh_500, "" as *u8, 0, WC_RB_ST_DISALLOW_ALL)
531 gw_check("T13b11 fixture-reached-the-condition: the seeded slot really holds DISALLOW_ALL" as *u8,
532 (wc_rb_state_g[idx500] == WC_RB_ST_DISALLOW_ALL) as i64, pass, total)
533 let u500: *u8 = "https://broken.example/anything" as *u8
534 gw_check("T13b12 END-TO-END a 5xx host now REFUSES every url -- THIS IS THE BUG THAT SHIPPED" as *u8,
535 (wc_rb_allowed("broken.example" as *u8, 14, hh_500, u500, wc_len(u500), rbnull) == 0) as i64, pass, total)
536
537 let hh_404: i64 = wc_hosthash("gone.example" as *u8, 12)
538 let idx404: i64 = t_rb_seed_state(hh_404, "" as *u8, 0, WC_RB_ST_ALLOW_ALL)
539 gw_check("T13b13 fixture-reached-the-condition: the seeded slot really holds ALLOW_ALL" as *u8,
540 (wc_rb_state_g[idx404] == WC_RB_ST_ALLOW_ALL) as i64, pass, total)
541 let u404: *u8 = "https://gone.example/anything" as *u8
542 gw_check("T13b14 neg-control-404-HOST-STILL-ALLOWED-END-TO-END (blanket-disallow fails here too)" as *u8,
543 wc_rb_allowed("gone.example" as *u8, 12, hh_404, u404, wc_len(u404), rbnull), pass, total)
544
545 // The ORIGINAL T13b, kept: an ABSENT robots.txt still allows. Its meaning is unchanged -- absence
546 // is the 4xx/unavailable case, which was never the case that was broken.
547 let hh_absent: i64 = wc_hosthash("absent.example" as *u8, 14)
548 t_rb_seed(hh_absent, "" as *u8, 0)
549 let uabs: *u8 = "https://absent.example/anything" as *u8
550 gw_check("T13b absent robots.txt -> ALLOW (RFC 9309 s2.3.1.3; our own inability is still not a rule)" as *u8,
551 wc_rb_allowed("absent.example" as *u8, 14, hh_absent, uabs, wc_len(uabs), rbnull), pass, total)
552
553 // ---- T13f: THE CRAWL-DELAY HORIZON. We never answer a long published delay by crawling faster.
554 gw_check("T13f1 a delay AT the RFC 9309 s2.4 24h horizon is ADOPTED exactly, not refused" as *u8,
555 (wc_rb_cd_policy(WC_ROBOTS_CD_HORIZON_S) == 0) as i64, pass, total)
556 gw_check("T13f2 above the horizon we DECLINE THE HOST -- the one thing we never do is speed up" as *u8,
557 (wc_rb_cd_policy(WC_ROBOTS_CD_HORIZON_S + 1) == 1) as i64, pass, total)
558 var t13f3: i64 = 0
559 if wc_rb_cd_policy(30) == 0 { if wc_rb_cd_policy(120) == 0 { t13f3 = 1 } }
560 gw_check("T13f3 neg-control-ORDINARY-DELAYS-ADOPTED: 30s and 120s both adopt (the old code clamped 120 to 30)" as *u8,
561 t13f3, pass, total)
562
563 // ============ T21: R15 DOMAIN-MAP ADOPTION -- WHICH ROWS MAY REACH THE FRONTIER ================
564 // wc_dm_seedable is pure over (kind, verdict), so the admission RULE is provable with no
565 // filesystem, no map and no network. The two POSITIVE teeth are load-bearing: a seeder that simply
566 // returned 0 for everything would pass every neg-control below and read as a flawless filter while
567 // silently feeding the frontier nothing at all.
568 gw_check("T21a url+LISTED IS seeded -- the sitemap/feed payload, the whole point of the rung" as *u8,
569 wc_dm_seedable("url" as *u8, "LISTED" as *u8), pass, total)
570 gw_check("T21b path+LIVE IS seeded -- a probe path the host actually served" as *u8,
571 wc_dm_seedable("path" as *u8, "LIVE" as *u8), pass, total)
572
573 // ---- THE COMPLIANCE NEG-CONTROL. nx_domain_map records when robots.txt refused a url. Seeding
574 // that row would launder a refusal into a fetch through a side door, defeating wc_rb_allowed
575 // without ever calling it -- a robots bypass that no robots tooth in this gate could detect.
576 gw_check("T21c neg-control-ROBOTS-REFUSED-NEVER-SEEDED (a map row must not bypass wc_rb_allowed)" as *u8,
577 (wc_dm_seedable("url" as *u8, "ROBOTS-REFUSED" as *u8) == 0) as i64, pass, total)
578
579 // ---- THE SOFT-404 NEG-CONTROL. The map has already PROVEN this url is the not-found page wearing
580 // a 200. Seeding it spends a fetch and a frontier slot on a page that does not exist -- precisely
581 // the measured waste (www.unesco.org served "thin 200 16" repeatedly) that the fingerprint exists
582 // to stop. Note the discrimination this pairs with T21b: LIVE seeds, SOFT404 does not, and both
583 // arrive on kind=path from the same probe source.
584 gw_check("T21d neg-control-SOFT404-NEVER-SEEDED (the fingerprint's whole purpose)" as *u8,
585 (wc_dm_seedable("path" as *u8, "SOFT404" as *u8) == 0) as i64, pass, total)
586
587 // ---- absence rows: the reconnaissance already paid to learn there is nothing there.
588 var t21e: i64 = 0
589 if wc_dm_seedable("path" as *u8, "MISSING" as *u8) == 0 {
590 if wc_dm_seedable("path" as *u8, "UNREACHABLE" as *u8) == 0 {
591 if wc_dm_seedable("path" as *u8, "BUDGET-REFUSED" as *u8) == 0 { t21e = 1 } } }
592 gw_check("T21e neg-control-ABSENCE-ROWS-NEVER-SEEDED (MISSING / UNREACHABLE / BUDGET-REFUSED)" as *u8,
593 t21e, pass, total)
594
595 // ---- rows that are not urls at all.
596 var t21f: i64 = 0
597 if wc_dm_seedable("path" as *u8, "FINGERPRINT" as *u8) == 0 {
598 if wc_dm_seedable("path" as *u8, "DECLARED" as *u8) == 0 {
599 if wc_dm_seedable("host" as *u8, "CERT" as *u8) == 0 { t21f = 1 } } }
600 gw_check("T21f neg-control-NON-URL-ROWS-NEVER-SEEDED (FINGERPRINT record / robots DECLARED path / crt host)" as *u8,
601 t21f, pass, total)
602
603 // ---- THE ANTI-VACUITY TOOTH. Everything above could be passed by a predicate that reads ONLY the
604 // verdict column and ignores the kind. These two cross pairs cannot: LISTED and LIVE are both
605 // admitting verdicts, and both must still be REFUSED when they arrive on the wrong kind. This is
606 // the tooth that proves the rule reads BOTH fields.
607 var t21g: i64 = 0
608 if wc_dm_seedable("path" as *u8, "LISTED" as *u8) == 0 {
609 if wc_dm_seedable("url" as *u8, "LIVE" as *u8) == 0 { t21g = 1 } }
610 gw_check("T21g anti-vacuity: an ADMITTING verdict on the WRONG KIND is refused (both fields are read)" as *u8,
611 t21g, pass, total)
612
613 // T13c -- BOTH DIRECTIONS off ONE body. Disallow must bite AND the rest of the host stays open.
614 let body1: *u8 = "User-agent: *\nDisallow: /private\n" as *u8
615 let hh1: i64 = wc_hosthash(rbh, rbhl)
616 t_rb_seed(hh1, body1, wc_len(body1))
617 let dis1: *u8 = "https://example.org/private/secret" as *u8
618 let all1: *u8 = "https://example.org/public/post" as *u8
619 var t13c: i64 = 0
620 if wc_rb_allowed(rbh, rbhl, hh1, dis1, wc_len(dis1), rbnull) == 0 { t13c = 1 }
621 gw_check("T13c Disallow: /private REFUSES /private/secret" as *u8, t13c, pass, total)
622 gw_check("T13c2 THE OPPOSITE TOOTH: /public on the same host is still ALLOWED (we did not block the web)" as *u8,
623 wc_rb_allowed(rbh, rbhl, hh1, all1, wc_len(all1), rbnull), pass, total)
624
625 // T13d -- Crawl-delay is READ AS PUBLISHED. This is the number the pacer now adopts, and the whole
626 // reason loc.gov served two pages then 403'd: the host had been publishing its pace all along.
627 let body2: *u8 = "User-agent: *\nCrawl-delay: 7\nDisallow: /x\n" as *u8
628 var t13d: i64 = 0
629 if nx_robots_crawl_delay(body2, wc_len(body2), wc_rb_ua(), WC_RB_UALEN) == 7 { t13d = 1 }
630 gw_check("T13d Crawl-delay: 7 is read as 7 (the pace we will actually honour)" as *u8, t13d, pass, total)
631 var t13d2: i64 = 0
632 if nx_robots_crawl_delay(body1, wc_len(body1), wc_rb_ua(), WC_RB_UALEN) == 0 { t13d2 = 1 }
633 gw_check("T13d2 NEG: a body with no Crawl-delay yields 0 -- we never invent a pace and call it the host's" as *u8, t13d2, pass, total)
634
635 // T13e -- GROUP SELECTION: a nishibot-specific group WINS over '*'. This proves the token we match
636 // is the token we claim, so a site CAN address a rule to us and have it obeyed.
637 let body3: *u8 = "User-agent: *\nDisallow: /\n\nUser-agent: nishibot\nDisallow: /nope\n" as *u8
638 let hh3: i64 = wc_hosthash("grp.example" as *u8, 11)
639 t_rb_seed(hh3, body3, wc_len(body3))
640 let okg: *u8 = "https://grp.example/fine" as *u8
641 let nog: *u8 = "https://grp.example/nope/x" as *u8
642 gw_check("T13e our own UA group wins over '*' (blanket Disallow: / does NOT apply to us)" as *u8,
643 wc_rb_allowed("grp.example" as *u8, 11, hh3, okg, wc_len(okg), rbnull), pass, total)
644 var t13e2: i64 = 0
645 if wc_rb_allowed("grp.example" as *u8, 11, hh3, nog, wc_len(nog), rbnull) == 0 { t13e2 = 1 }
646 gw_check("T13e2 ...and the rule addressed TO US is obeyed (/nope refused)" as *u8, t13e2, pass, total)
647
648 // T13f -- cache identity: an unknown host MISSES (probed exactly once), a seeded host HITS (never
649 // re-probed). This bound is what stops the politeness probe from becoming the load.
650 var t13f: i64 = 0
651 if wc_rb_find(wc_hosthash("never.seen" as *u8, 10)) < 0 { t13f = 1 }
652 gw_check("T13f unknown host misses the cache (so it is probed once, not never)" as *u8, t13f, pass, total)
653 var t13f2: i64 = 0
654 if wc_rb_find(hh1) >= 0 { t13f2 = 1 }
655 gw_check("T13f2 seeded host HITS (so robots.txt is fetched at most once per host per run)" as *u8, t13f2, pass, total)
656
657 // ---- R7 FRONTIER PRIORITY + AGING (2026-08-24, /compare/webscraping wc_priority): pure functions, network-free ----
658 var t14: i64 = 1
659 if wc_url_depth("https://a.org/" as *u8, 14) != 0 { t14 = 0 }
660 if wc_url_depth("https://a.org/x" as *u8, 15) != 1 { t14 = 0 }
661 if wc_url_depth("https://a.org/x/y/z" as *u8, 19) != 3 { t14 = 0 }
662 if wc_url_depth("https://a.org/x/y/" as *u8, 18) != 2 { t14 = 0 }
663 gw_check("T14 path depth: root 0, /x 1, /x/y/z 3, a trailing slash opens no level" as *u8, t14, pass, total)
664 wc_query_set("" as *u8)
665 var t15: i64 = 0
666 if wc_priority("https://a.org/x" as *u8, 15) > wc_priority("https://a.org/x/y/z" as *u8, 19) { t15 = 1 }
667 gw_check("T15 no query: a shallow url outranks a deep one (breadth-first finds the important pages first, Cho 1998)" as *u8, t15, pass, total)
668 wc_query_set("butter churning" as *u8)
669 var t16: i64 = 0
670 if wc_priority("https://a.org/a/b/c/d/butter-churning" as *u8, 37) > wc_priority("https://a.org/x" as *u8, 15) { t16 = 1 }
671 gw_check("T16 with a query, a deep url naming two query terms outranks a shallow url naming none" as *u8, t16, pass, total)
672 var t16b: i64 = 0
673 if wc_priority("https://a.org/x/butter" as *u8, 22) > wc_priority("https://a.org/x/y/butter" as *u8, 24) { t16b = 1 }
674 gw_check("T16b equal query hits fall back to depth, shallower first" as *u8, t16b, pass, total)
675 wc_query_set("" as *u8)
676 let rw: *u8 = sys_mmap(WC_MAGIC_2048 * 4)
677 let rw0: *u8 = "https://a.org/p/q/r" as *u8
678 let rw1: *u8 = "https://a.org/p" as *u8
679 let rw2: *u8 = "https://a.org/p/q" as *u8
680 var rwi: i64 = 0
681 while rwi <= 19 { rw[rwi] = rw0[rwi]; rwi = rwi + 1 }
682 rwi = 0
683 while rwi <= 15 { rw[WC_MAGIC_2048 + rwi] = rw1[rwi]; rwi = rwi + 1 }
684 rwi = 0
685 while rwi <= 17 { rw[2 * WC_MAGIC_2048 + rwi] = rw2[rwi]; rwi = rwi + 1 }
686 wc_rank_window(rw, 3)
687 var t17: i64 = 0
688 if wc_streq(rw, rw1) == 1 { if wc_streq((rw as i64 + WC_MAGIC_2048) as *u8, rw2) == 1 { if wc_streq((rw as i64 + 2 * WC_MAGIC_2048) as *u8, rw0) == 1 { t17 = 1 } } }
689 gw_check("T17 wc_rank_window orders the pull window shallow-first and keeps every url byte-exact" as *u8, t17, pass, total)
690 var t18: i64 = 1
691 if wc_pc_next(0, 5, 100) != 5 { t18 = 0 }
692 if wc_pc_next(95, 5, 100) != 0 { t18 = 0 }
693 if wc_pc_next(40, 0, 30) != 0 { t18 = 0 }
694 gw_check("T18 rotation cursor advances by rows taken and wraps at the pending population" as *u8, t18, pass, total)
695
696 // ================= T19: R13 FRONTIER GAUGE -- THE PARTS MUST SUM (2026-08-25) =================
697 // This number is what every capacity decision about the crawl is read off, and until now it lived
698 // INLINE inside the done-line's wc_num() call, so it had no tooth at all. It used to read
699 // `npend - fetched + disc`, subtracting ONE of the FOUR ways a row leaves the frontier.
700 // The numbers below are the REAL measured ones from the last pre-fix run (searchpipe.log, byte offset
701 // 112,972,980): fetched=160, robots_skipped=22, discovered_new=300, frontier_remaining~=2013 under the
702 // OLD formula, which back-solves to npend=1873 -- and the source's own note records true pending as
703 // 1871-1883 over that window, so the fixture is the production case, not an invented one.
704 // The MEASURED true pending count of the live frontier on 2026-08-25, used as the fixture population
705 // for the partition arithmetic below. Named for its PURPOSE, not its value: the ratchet refused the
706 // bare literal (magic=1 baseline=0 at threshold 1024) and naming it also records WHERE it came from --
707 // it is not an invented round number, it is what the frontier actually held when the gauge was
708 // corrected, recovered as npend = remaining + fetched - discovered across six consecutive runs.
709 let fN: i64 = WCID_FIXTURE_FRONTIER_PENDING
710 let fF: i64 = 160
711 let fR: i64 = 0
712 let fB: i64 = 22
713 let fH: i64 = 176
714 let fD: i64 = 300
715 // THE PARTITION CLAIM, asserted as a SUM rather than by restating the expression under test (a check
716 // that re-derives its subject proves only that arithmetic is arithmetic): every pulled row either LEFT
717 // the pending state this run, or is still pending. Those two must reconstruct npend exactly.
718 let fLeft: i64 = wc_frontier_left(fF, fR, fB, fH)
719 let fStill: i64 = wc_frontier_remaining(fN, fF, fR, fB, fH, fD) - fD
720 gw_check("T19 partition SUMS: rows that left + rows still pending == the window pulled" as *u8, (fLeft + fStill == fN) as i64, pass, total)
721 gw_check("T19b the four outflows are all counted (160+0+22+176 = 358)" as *u8, (fLeft == 358) as i64, pass, total)
722 // THE REGRESSION THE FIX EXISTS FOR: against the OLD gauge the corrected one must be SMALLER by exactly
723 // the outflows the old one ignored -- no more, no less.
724 let fOld: i64 = fN - fF + fD
725 gw_check("T19c corrected gauge is lower than the old one by EXACTLY robots_skipped+hostcap+retired" as *u8,
726 (fOld - wc_frontier_remaining(fN, fF, fR, fB, fH, fD) == fB + fH + fR) as i64, pass, total)
727 gw_check("T19d ...and that difference is NONZERO here (198), so T19c is not comparing a thing to itself" as *u8,
728 (fOld - wc_frontier_remaining(fN, fF, fR, fB, fH, fD) == 198) as i64, pass, total)
729 // neg-control: a run with NO robots skips, NO host caps and NO retirements must be BYTE-IDENTICAL to the
730 // old gauge. If this fails the correction changed something it was never supposed to touch.
731 gw_check("T19e neg-control-NO-OUTFLOWS-matches-the-OLD-gauge-exactly" as *u8,
732 (wc_frontier_remaining(fN, fF, 0, 0, 0, fD) == fN - fF + fD) as i64, pass, total)
733 // pace_deferred is deliberately ABSENT from this arithmetic: a deferred row stays PENDING, which is the
734 // entire point of deferring it. Subtracting it would under-report the frontier by exactly the rows we
735 // most intend to come back to. The signature carries no deferred parameter, and T20 below pins the
736 // store-level invariant that makes that correct.
737
738 // ================= T20: THE INVARIANT DEFERRAL RELIES ON =================
739 // HONEST SCOPE: the deferral branch itself lives inside main()'s selection loop and is not callable, so
740 // this does NOT execute that branch. What it pins is the store contract the branch depends on, and the
741 // CONTRAST is the assertion -- both signals present on ONE fixture at once:
742 // a row LEFT UNMARKED (what deferral does) -> stays P and is re-enumerated on a later run
743 // a row marked state 2 (what host-cap does) -> disappears from the pending window
744 // If the first failed, every deferral would silently LOSE a url. If the second failed, host-cap would
745 // re-select the same row forever. The two are the opposite terminal semantics of one selection loop.
746 let dprefix: *u8 = "/tmp/crawlgate-r13-" as *u8
747 let dseg: *i64 = sys_mmap(16) as *i64; dseg[0] = 1
748 let dsegs: *i64 = sys_mmap(16) as *i64; dsegs[0] = 0
749 sys_unlinkat("/tmp/crawlgate-r13-manifest.txt" as *u8)
750 sys_unlinkat("/tmp/crawlgate-r13-seg-1.docs" as *u8)
751 sys_unlinkat("/tmp/crawlgate-r13-seg-1.idx" as *u8)
752 sys_unlinkat("/tmp/crawlgate-r13-seg-2.docs" as *u8)
753 sys_unlinkat("/tmp/crawlgate-r13-seg-2.idx" as *u8)
754 let dw: *i64 = ss_begin()
755 let dfer: *u8 = "https://throttled.example/deferred-row" as *u8
756 let dcap: *u8 = "https://capped.example/hostcapped-row" as *u8
757 wc_frmark(dw, dprefix, dseg, dsegs, dfer, wc_len(dfer), 1)
758 wc_frmark(dw, dprefix, dseg, dsegs, dcap, wc_len(dcap), 1)
759 ss_commit(dprefix, dw, dseg[0])
760 let dh0: *i64 = ss_open(dprefix)
761 let durls: *u8 = sys_mmap(WC_MAGIC_2048 * 8)
762 let dcanh: *u8 = sys_mmap(64 * 8)
763 let dn0: i64 = wc_pending(dh0, dprefix, durls, 8, dcanh, 0, 0)
764 gw_check("T20 fixture: both rows start PENDING (2) -- the fixture reached the condition" as *u8, (dn0 == 2) as i64, pass, total)
765 // now do to each row exactly what its branch does
766 dseg[0] = dseg[0] + 1
767 let dw2: *i64 = ss_begin()
768 wc_frmark(dw2, dprefix, dseg, dsegs, dcap, wc_len(dcap), 2) // host-cap: marked DONE
769 ss_commit(dprefix, dw2, dseg[0]) // deferral: NOTHING written for dfer
770 let dh1b: *i64 = ss_open(dprefix)
771 let dn1: i64 = wc_pending(dh1b, dprefix, durls, 8, dcanh, 0, 0)
772 gw_check("T20a the host-capped row LEAVES the pending window (2 -> 1)" as *u8, (dn1 == 1) as i64, pass, total)
773 var dsurv: i64 = 0
774 if dn1 == 1 { if wc_streq(durls, dfer) == 1 { dsurv = 1 } }
775 gw_check("T20b the survivor is the UNMARKED (deferred) row, byte-exact -- a deferral loses no url" as *u8, dsurv, pass, total)
776 gw_check("T20c neg-control: the DEFERRED url is still state P(1), never flipped to D" as *u8,
777 (wc_frstate(dh1b, dfer, wc_len(dfer)) == 1) as i64, pass, total)
778 gw_check("T20d ...and the host-capped url IS state D(2) -- the two paths really do differ" as *u8,
779 (wc_frstate(dh1b, dcap, wc_len(dcap)) == 2) as i64, pass, total)
780
781 // ---- T22: CRAWLER IDENTITY (2026-08-25). Until this change the bulk crawler sent a HARDCODED
782 // CHROME User-Agent on every request: wc_fetch_child -> nx_https_fetch_follow_best -> ff_core ->
783 // nx_https_get_complete_cookie_xhdr -> nx_http_client_build_request_cookie_xhdr, which emitted the
784 // browser identity for everyone. Two consequences worse than the forgery itself: we MATCHED the
785 // robots.txt group `nishibot` while SENDING Chrome, so an operator could write a rule for us, watch
786 // us obey it, and never be able to tell it was us; and the identity mechanism added later appended a
787 // SECOND User-Agent header, which RFC 9110 5.3 forbids and which receivers resolve by taking the
788 // FIRST -- so the honest identity never reached the wire at all.
789 // The browser lane's Chrome UA is LOAD-BEARING (it matches our JA3 TLS fingerprint), so the fix is a
790 // SELECTOR whose default is byte-identical to the old behaviour. T22a is the neg-control that proves
791 // that default did not move; without it, a change that made everything NishiBot would pass T22c.
792 let uab: *u8 = sys_mmap(WCID_UABUF)
793 hc_set_identity(HC_UA_BROWSER)
794 let ub0: i64 = hc_put_ua(uab, 0)
795 gw_check("T22a neg-control: the DEFAULT identity is still the browser UA -- no existing caller changed" as *u8,
796 wcid_has(uab, ub0, "Mozilla/5.0" as *u8), pass, total)
797 gw_check("T22b neg-control: the browser form does NOT also claim to be the bot" as *u8,
798 (wcid_has(uab, ub0, "NishiBot" as *u8) == 0) as i64, pass, total)
799 hc_set_identity(HC_UA_BOT)
800 let ub1: i64 = hc_put_ua(uab, 0)
801 gw_check("T22c the crawler identity emits NishiBot" as *u8,
802 wcid_has(uab, ub1, "NishiBot/1.0" as *u8), pass, total)
803 gw_check("T22d ...and does NOT simultaneously claim to be Chrome" as *u8,
804 (wcid_has(uab, ub1, "Mozilla" as *u8) == 0) as i64, pass, total)
805 gw_check("T22e ...and carries a CONTACT url -- an identity nobody can complain to is not an identity" as *u8,
806 wcid_has(uab, ub1, "+https://nishifamily.com/bot" as *u8), pass, total)
807 gw_check("T22f EXACTLY ONE User-Agent line (RFC 9110 5.3): repeating it is forbidden and receivers take the FIRST, which is how the honest identity used to be silently dropped" as *u8,
808 (wcid_count(uab, ub1, "User-Agent:" as *u8) == 1) as i64, pass, total)
809 gw_check("T22g fixture-reached: the two identities really are DIFFERENT bytes, so T22a/T22c are not both reading one string" as *u8,
810 (ub0 != ub1) as i64, pass, total)
811 hc_set_identity(HC_UA_BROWSER)
812
813 // ---- T23: HOST-CAP DEFERS INSTEAD OF RETIRING (R16, 2026-08-25). The host cap is a PER-ROUND
814 // diversity measure that was marking rows DONE permanently -- measured on the live beat as
815 // hostcap_retired=227 pages discarded in a single run, on a frontier that had drained to 313.
816 // The predicate is pure so it can be gated at all; these teeth pin BOTH arms, because a rule that
817 // only ever defers would silently reintroduce the blockade the original design retired to avoid.
818 gw_check("T23a a frontier that FITS the pull window defers (the row comes back next run)" as *u8,
819 wc_hostcap_defers(WC_MAXPEND - 1), pass, total)
820 gw_check("T23b exactly AT the window still defers -- every pending row is still pulled" as *u8,
821 wc_hostcap_defers(WC_MAXPEND), pass, total)
822 gw_check("T23c neg-control: PAST the window it RETIRES -- a deferred row could be shadowed there, which is the blockade retirement exists to prevent" as *u8,
823 (wc_hostcap_defers(WC_MAXPEND + 1) == 0) as i64, pass, total)
824 gw_check("T23d neg-control: a far-oversized frontier also retires, so T23c is a boundary and not an off-by-one accident" as *u8,
825 (wc_hostcap_defers(WC_MAXPEND * 2) == 0) as i64, pass, total)
826 gw_check("T23e fixture-reached: the live frontier measured 313, which really is inside the window, so the defer arm is the one production takes today" as *u8,
827 wc_hostcap_defers(WCID_FIXTURE_FRONTIER_DRAINED), pass, total)
828 gw_check("T23f ...and the PRE-DRAIN frontier of 1873 was ALSO inside the window, so R16 would have deferred then too -- the 227 pages that run discarded were avoidable, not a consequence of frontier size" as *u8,
829 wc_hostcap_defers(WCID_FIXTURE_FRONTIER_PENDING), pass, total)
830
831 // ---- T24: THE DERIVED DISCOVERY BUDGET (R17, 2026-08-25). discovered_new sat at exactly 300 in 58
832 // of 60 consecutive runs -- pinned at its ceiling, i.e. the harvest had more supply than the budget
833 // accepted, while the frontier had drained to 313 against a 2048-row pull window. These teeth pin the
834 // shape of the derivation, not a single value, because a budget that only ever widens is as wrong as
835 // one that never does.
836 gw_check("T24a the DRAINED frontier (313, measured) gets a budget far above the old fixed 300" as *u8,
837 (wc_disc_budget(WCID_FIXTURE_FRONTIER_DRAINED) > WC_MAXDISC) as i64, pass, total)
838 gw_check("T24b ...and it fills toward exactly one pull window, never past it" as *u8,
839 (wc_disc_budget(WCID_FIXTURE_FRONTIER_DRAINED) == WC_DISC_CEIL - WCID_FIXTURE_FRONTIER_DRAINED) as i64, pass, total)
840 gw_check("T24h ...while the PRE-DRAIN frontier (1873, also measured) correctly floors to the old budget -- the same function, two real frontiers, two different right answers" as *u8,
841 (wc_disc_budget(WCID_FIXTURE_FRONTIER_PENDING) == WC_MAXDISC) as i64, pass, total)
842 gw_check("T24c neg-control: a FULL frontier falls back to the old fixed budget -- this can only widen, never shrink" as *u8,
843 (wc_disc_budget(WC_DISC_CEIL) == WC_MAXDISC) as i64, pass, total)
844 gw_check("T24d neg-control: an OVERFULL frontier still gets the floor, never a negative or zero budget (discovery must never stop entirely)" as *u8,
845 (wc_disc_budget(WC_DISC_CEIL * 2) == WC_MAXDISC) as i64, pass, total)
846 gw_check("T24e the budget never exceeds the ceiling even at an empty frontier" as *u8,
847 (wc_disc_budget(0) <= WC_DISC_CEIL) as i64, pass, total)
848 gw_check("T24f fixture-reached: the ceiling really is the pull window, so the runseen buffer dimensioned from it cannot be overrun by the budget" as *u8,
849 (WC_DISC_CEIL == WC_MAXPEND) as i64, pass, total)
850 gw_check("T24g neg-control: the UNSET budget getter floors to the old fixed value, so a path running before main computes it behaves exactly as before" as *u8,
851 (wc_disc_cap() == WC_MAXDISC) as i64, pass, total)
852
853 // ---- T25: HOST-BALANCED PULL WINDOW (G0, 2026-09-03, /compare/search contract wc_frontier_fair).
854 // MEASURED, not designed: two windows of knowledge/status/crawl_outcomes.log 41 minutes apart
855 // (epochs 1788448047 and 1788450535) were BOTH 100 percent hostcap-defer and every url in both was
856 // en.wikipedia.org, while nx_crawlaudit re-fetched a sample independently and returned 6 of 8
857 // disagreements reading that we DROPPED real prose. The R7 rotation above cannot fix that -- a
858 // rotated slice of a one-host frontier is still one host. These teeth pin the admission predicate
859 // itself, and the load-bearing ones are the arms that must NOT refuse: a balancer that turns into a
860 // crawl stop is a worse defect than the imbalance it corrects.
861 let wqh5: *u8 = sys_mmap(WC_HOSTBUF * 8)
862 let wqc5: *i64 = sys_mmap(8 * 8) as *i64
863 let wqn5: *i64 = sys_mmap(16) as *i64
864 wqn5[0] = 0
865 let ha5: *u8 = "en.wikipedia.org" as *u8
866 let hb5: *u8 = "who.int" as *u8
867 let hc5: *u8 = "irs.gov" as *u8
868 var adm5: i64 = 0
869 var i5: i64 = 0
870 while i5 < WC_WINQ { adm5 = adm5 + wc_winq_admit(wqh5, wqc5, wqn5, 8, ha5, 16, WC_WINQ); i5 = i5 + 1 }
871 gw_check("T25a fixture-reached: the fixture really drove host A to EXACTLY its quota, so the refusing tooth below is asked the question it claims to ask" as *u8,
872 (adm5 == WC_WINQ) as i64, pass, total)
873 gw_check("T25b the row PAST the quota for the SAME host is refused -- that surplus was guaranteed to become hostcap-defer at the fetch stage" as *u8,
874 (wc_winq_admit(wqh5, wqc5, wqn5, 8, ha5, 16, WC_WINQ) == 0) as i64, pass, total)
875 gw_check("T25c NEG-CONTROL, the load-bearing one: a DIFFERENT host is still admitted once host A is full -- if this fails the fix has become a crawl stop, not a balancer" as *u8,
876 wc_winq_admit(wqh5, wqc5, wqn5, 8, hb5, 7, WC_WINQ), pass, total)
877 gw_check("T25d the distinct-host count really moved to 2, so T25c admitted a NEW host rather than silently re-counting host A" as *u8,
878 (wqn5[0] == 2) as i64, pass, total)
879 gw_check("T25e NEG-CONTROL: a NEW host arriving with the table FULL fails OPEN -- a sizing error must be wrong in the direction of doing nothing, never of starving the crawl" as *u8,
880 wc_winq_admit(wqh5, wqc5, wqn5, 2, hc5, 7, WC_WINQ), pass, total)
881 gw_check("T25f ...and that fail-open really took the table-full branch: the distinct-host count is STILL 2, so it admitted without recording" as *u8,
882 (wqn5[0] == 2) as i64, pass, total)
883 gw_check("T25g the quota is DERIVED from the fetch stage's own LARGEST per-host cap, so no row the fetcher could have indexed is ever excluded from the window" as *u8,
884 (WC_WINQ == WC_HOSTCAP_CANON) as i64, pass, total)
885 gw_check("T25h a FULL pull window therefore offers at least 51 distinct hosts whenever that many are pending -- the arithmetic the whole diversity claim rests on" as *u8,
886 (WC_MAXPEND / WC_WINQ >= 51) as i64, pass, total)
887 gw_check("T25i NEG-CONTROL: the per-host caps themselves are UNTOUCHED -- a run that indexed more pages by RAISING them would have failed rung G0, not passed it" as *u8,
888 ((WC_HOSTCAP == 6) as i64) * ((WC_HOSTCAP_CANON == 40) as i64), pass, total)
889
890 // ---- T26: THE G0 CONTRACT PREDICATE (2026-09-03). wc_frontier_fair is the exact symbol the
891 // /compare/search watch row measures, so it has to BE the invariant rather than a name shaped like
892 // one -- a completion signal that keys on a name rewards writing the name. These teeth pin the
893 // boundary in BOTH directions and pin the two shapes that actually matter: the window we measured
894 // in production, and the window the fix is supposed to produce.
895 gw_check("T26a a window exactly at quota-times-hosts is FAIR" as *u8,
896 wc_frontier_fair(WC_WINQ * 4, 4), pass, total)
897 gw_check("T26b NEG-CONTROL: ONE row past that is NOT fair -- the boundary is exact, not an approximation" as *u8,
898 (wc_frontier_fair(WC_WINQ * 4 + 1, 4) == 0) as i64, pass, total)
899 gw_check("T26c NEG-CONTROL, the MEASURED case this rung exists for: a full WC_MAXPEND window drawn from ONE host is NOT fair -- that is what crawl_outcomes.log showed on 2026-08-11 and again on 2026-09-03" as *u8,
900 (wc_frontier_fair(WC_MAXPEND, 1) == 0) as i64, pass, total)
901 gw_check("T26d ...and the shape the fix produces IS fair: a full window spread over the 51-plus hosts WC_WINQ guarantees" as *u8,
902 wc_frontier_fair(WC_MAXPEND, WC_MAXPEND / WC_WINQ + 1), pass, total)
903 gw_check("T26e NEG-CONTROL: rows present with ZERO hosts attributed reads UNMEASURED, not balanced -- an abstention must never score as a pass" as *u8,
904 (wc_frontier_fair(WC_MAXPEND, 0) == 0) as i64, pass, total)
905 gw_check("T26f an EMPTY window is vacuously fair and says so, instead of dividing by zero" as *u8,
906 wc_frontier_fair(0, 0), pass, total)
907 gw_check("T26g the JUDGE agrees with the ENFORCER: wc_winq_admit can never let one host past WC_WINQ, so any window it built is fair by construction" as *u8,
908 wc_frontier_fair(WC_WINQ, 1), pass, total)
909
910 gw_puts("=== crawl-step gate: " as *u8); gw_num(pass[0]); gw_puts("/" as *u8); gw_num(total[0]); gw_puts(" " as *u8)
911 if pass[0] == total[0] { gw_puts("GREEN\n" as *u8) } else { gw_puts("RED\n" as *u8) }
912 let ctr: *i64 = gv_ctr()
913 ctr[0] = pass[0]
914 ctr[1] = total[0]
915 return gv_verdict("WEB-CRAWL-STEP-GATE" as *u8, ctr, "frontier store mechanics: url filter, P-to-D state flip, cross-segment dedupe, outlink capture" as *u8)
916}