code wiki / _hdl_build / nx_web_crawl_step_gate.nx
`nx_web_crawl_step_gate.nx` source

↩ module page · 408 lines · 24415 B
1// nx_web_crawl_step_gate.nx -- network-free KATs for the frontier-persisted crawler's STORE mechanics:
2// the url filter, fr: row write/enumerate, the P->D state flip (append-only last-wins), cross-segment
3// dedupe, and unknown-state. The fetch/ingest spine is nx_web_ingest's, already proven on the real web.
4// Runs against a /tmp shard (wiped per boot). license_tier: ORIGINAL
5import "nx_web_crawl_step.nx"
6import "nx_gate_verdict.nx"
7
8func gw_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
9func gw_num(v: i64) -> i64 {
  let bb: *u8 = sys_mmap(28); var m: i64 = v
  if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
  let t: *u8 = sys_mmap(28); var k: i64 = 0
  if m == 0 { t[0] = 48 as u8; k = 1 }
  while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
  var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 }
  sys_write(1, bb, k); return 0
17}
18func gw_check(name: *u8, cond: i64, pass: *i64, total: *i64) -> i64 {
  total[0] = total[0] + 1
  gw_puts(name)
  if cond == 1 { pass[0] = pass[0] + 1; gw_puts(" PASS\n" as *u8) } else { gw_puts(" FAIL\n" as *u8) }
  return 0
23}
24func gw_ok(u: *u8) -> i64 { return wc_urlok(u, wc_len(u)) }
25
26func main() -> i64 {
  gw_puts("=== nx_web_crawl_step gate (frontier store mechanics, network-free) ===\n" as *u8)
  let pass: *i64 = sys_mmap(16) as *i64; pass[0] = 0
  let total: *i64 = sys_mmap(16) as *i64; total[0] = 0
30
  // R10: establish the query policy ONCE, up front, so EVERY tooth below runs under a known,
  // self-contained deny list rather than whatever the live urlpolicy- plane happens to hold.
  // 'action' earns its place next to 'utm_source': ?action=edit|raw|history is a MediaWiki ACTION
  // ENDPOINT, not a document. R8 killed it for free via the blanket '?' reject; once queries are
  // judged by shape, that free kill has to be paid for explicitly -- and T1 is what noticed.
  // ★REMOVING A BLANKET RULE BILLS YOU FOR EVERY CASE IT WAS SILENTLY COVERING.
  let gqd: *u8 = sys_mmap(32 * 8)
  var gqi: i64 = 0
  let gqd0: *u8 = "utm_source" as *u8
  while gqd0[gqi] != (0 as u8) { gqd[gqi] = gqd0[gqi]; gqi = gqi + 1 }
  gqd[gqi] = 0 as u8
  let gqd1: *u8 = "action" as *u8
  var gqj: i64 = 0
  while gqd1[gqj] != (0 as u8) { gqd[32 + gqj] = gqd1[gqj]; gqj = gqj + 1 }
  gqd[32 + gqj] = 0 as u8
  wc_qdeny_g = gqd
  wc_nqdeny_g = 2
48
  // T1 url filter: keeps article urls, kills queries/fragments/meta/binaries
  var f1: i64 = 1
  if gw_ok("https://en.wikipedia.org/wiki/Chess" as *u8) != 1 { f1 = 0 }
  if gw_ok("https://example.org/a/deep/page" as *u8) != 1 { f1 = 0 }
  if gw_ok("https://en.wikipedia.org/w/index.php?action=edit" as *u8) != 0 { f1 = 0 }
  if gw_ok("https://en.wikipedia.org/wiki/Chess#History" as *u8) != 0 { f1 = 0 }
  if gw_ok("https://en.wikipedia.org/wiki/Special:Random" as *u8) != 0 { f1 = 0 }
  if gw_ok("https://en.wikipedia.org/wiki/File:Board.png" as *u8) != 0 { f1 = 0 }
  if gw_ok("https://x.org/logo.png" as *u8) != 0 { f1 = 0 }
  if gw_ok("https://x.org/style.css" as *u8) != 0 { f1 = 0 }
  if gw_ok("mailto:someone@example.org" as *u8) != 0 { f1 = 0 }
  if gw_ok("/relative/only" as *u8) != 0 { f1 = 0 }
  if gw_ok("https://ro.wikipedia.org/wiki/Literatura" as *u8) != 0 { f1 = 0 }   // interwiki lang edition
  if gw_ok("https://en.wikipedia.org/wiki/Chess_opening" as *u8) != 1 { f1 = 0 } // en stays
  if gw_ok("https://commons.wikimedia.org/wiki/X" as *u8) != 0 { f1 = 0 }
  // API / feed endpoints = real bytes but not content -> rejected
  if gw_ok("https://creativecommons.org/wp-json/" as *u8) != 0 { f1 = 0 }
  if gw_ok("https://site.com/api/data.json" as *u8) != 0 { f1 = 0 }
  if gw_ok("https://blog.com/feed" as *u8) != 0 { f1 = 0 }
  if gw_ok("https://blog.com/comments/feed/" as *u8) != 0 { f1 = 0 }
  if gw_ok("https://site.com/articles/a-real-story" as *u8) != 1 { f1 = 0 }   // content still passes
  // anti-slop listing classes (2026-08-04): tag/category link farms refused at admission
  if gw_ok("https://healthquerys.com/tag/puriva-france/" as *u8) != 0 { f1 = 0 }
  if gw_ok("https://blog.com/category/wellness/" as *u8) != 0 { f1 = 0 }
  if gw_ok("https://blog.com/posts/2024/tagged-story" as *u8) != 1 { f1 = 0 }   // 'tag' inside a word still passes
  gw_check("T1 url filter (21 KATs incl. interwiki + API/feed + tag/category farms)" as *u8, f1, pass, total)
75
  // FIXTURE SELF-CLEAN (2026-08-04, debt 1785882035): /tmp persists across runs on the NAS, so a
  // previous run's segments made the count teeth (T2-T4) go 4/7 RED on any second invocation -- a
  // gate that is only green on a fresh boot is a boot-witness, not a gate. Unlink the whole fixture.
  sys_unlinkat("/tmp/crawlgate-manifest.txt" as *u8)
  let cleanbuf: *u8 = sys_mmap(96)
  let cdig: *u8 = sys_mmap(28)
  var ci0: i64 = 1
  while ci0 <= 32 {
      var ce: i64 = 0
      while ce < 5 {
          var ext: *u8 = ".docs" as *u8
          if ce == 1 { ext = ".idx" as *u8 }
          if ce == 2 { ext = ".pos" as *u8 }
          if ce == 3 { ext = ".imp" as *u8 }
          if ce == 4 { ext = ".terms" as *u8 }
          var co: i64 = 0
          let cpre: *u8 = "/tmp/crawlgate-seg-" as *u8
          while cpre[co] != (0 as u8) { cleanbuf[co] = cpre[co]; co = co + 1 }
          var cm: i64 = ci0
          var ck: i64 = 0
          while cm > 0 { cdig[ck] = (48 + (cm % 10)) as u8; cm = cm / 10; ck = ck + 1 }
          var cj: i64 = 0
          while cj < ck { cleanbuf[co] = cdig[ck - 1 - cj]; co = co + 1; cj = cj + 1 }
          var cx: i64 = 0
          while ext[cx] != (0 as u8) { cleanbuf[co] = ext[cx]; co = co + 1; cx = cx + 1 }
          cleanbuf[co] = 0 as u8
          sys_unlinkat(cleanbuf)
          ce = ce + 1
      }
      ci0 = ci0 + 1
  }
107
  // build a tiny /tmp frontier shard
  let prefix: *u8 = "/tmp/crawlgate-" as *u8
  let segidbox: *i64 = sys_mmap(16) as *i64; segidbox[0] = 1
  let segsbox: *i64 = sys_mmap(16) as *i64; segsbox[0] = 0
  let w: *i64 = ss_begin()
  let u1: *u8 = "https://a.example/one" as *u8
  let u2: *u8 = "https://a.example/two" as *u8
  let u3: *u8 = "https://a.example/three" as *u8
  wc_frmark(w, prefix, segidbox, segsbox, u1, wc_len(u1), 1)
  wc_frmark(w, prefix, segidbox, segsbox, u2, wc_len(u2), 1)
  wc_frmark(w, prefix, segidbox, segsbox, u3, wc_len(u3), 2)   // three = already done
  ss_commit(prefix, w, segidbox[0])
120
  // T2 enumerate: exactly the two P urls
  let h1: *i64 = ss_open(prefix)
  let urls: *u8 = sys_mmap(2048 * 16)
  let canh0: *u8 = sys_mmap(64 * 8)   // R4: never read when canonly=0 -- T2/T3/T4 keep the pre-R4 contract
  let n2: i64 = wc_pending(h1, prefix, urls, 16, canh0, 0, 0)
  gw_check("T2 pending enumeration -> 2 (P rows only)" as *u8, (n2 == 2) as i64, pass, total)
127
  // T3 state flip: mark one P->D in a NEW segment; enumeration drops to 1 (last-wins across segments)
  segidbox[0] = segidbox[0] + 1
  let w2: *i64 = ss_begin()
  wc_frmark(w2, prefix, segidbox, segsbox, u1, wc_len(u1), 2)
  ss_commit(prefix, w2, segidbox[0])
  let h2: *i64 = ss_open(prefix)
  let n3: i64 = wc_pending(h2, prefix, urls, 16, canh0, 0, 0)
  var t3: i64 = 0
  if n3 == 1 {
      // the survivor must be u2
      var eq: i64 = 1
      var x: i64 = 0
      var go: i64 = 1
      while go == 1 { if urls[x] != u2[x] { eq = 0; go = 0 } else { if u2[x] == (0 as u8) { go = 0 } else { x = x + 1 } } }
      if eq == 1 { t3 = 1 }
  }
  gw_check("T3 P->D flip (append-only, last-wins) -> 1 pending = /two" as *u8, t3, pass, total)
145
  // T4 duplicate P rows across segments dedupe to one pending
  segidbox[0] = segidbox[0] + 1
  let w3: *i64 = ss_begin()
  wc_frmark(w3, prefix, segidbox, segsbox, u2, wc_len(u2), 1)   // re-bank the SAME pending url
  ss_commit(prefix, w3, segidbox[0])
  let h3: *i64 = ss_open(prefix)
  let n4: i64 = wc_pending(h3, prefix, urls, 16, canh0, 0, 0)
  gw_check("T4 dup P rows across segments dedupe -> still 1" as *u8, (n4 == 1) as i64, pass, total)
154
  // T5 unknown state for a never-seen url
  let s5: i64 = wc_frstate(h3, "https://a.example/never" as *u8, 23)
  gw_check("T5 unknown url -> state 0" as *u8, (s5 == 0) as i64, pass, total)
  // T5b known states read back
  var t5b: i64 = 0
  if wc_frstate(h3, u2, wc_len(u2)) == 1 { if wc_frstate(h3, u1, wc_len(u1)) == 2 { t5b = 1 } }
  gw_check("T5b states read back (two=P, one=D)" as *u8, t5b, pass, total)
162
  // T6 (P1 LINK GRAPH): wc_harvest captures ALL of a page's outlinks as out:<cid(base)> = packed i64 target
  // cid(url)s -- the edge list PageRank runs on. Distinguishing by construction: before this change the
  // crawler stored NO edges, so out:<cid> is absent and t6=0.
  let t1u: *u8 = "https://tgt1.example/story-alpha" as *u8
  let t2u: *u8 = "https://tgt2.example/story-beta" as *u8
  let hbase: *u8 = "https://src.example/page-x" as *u8
  let html6: *u8 = "<html><body><a href='https://tgt1.example/story-alpha'>A</a> <a href='https://tgt2.example/story-beta'>B</a></body></html>" as *u8
  let hdisc: *i64 = sys_mmap(16) as *i64; hdisc[0] = 0
  let hrs: *u8 = sys_mmap(256 * 64); let hnrs: *i64 = sys_mmap(16) as *i64; hnrs[0] = 0
  segidbox[0] = segidbox[0] + 1
  let w6: *i64 = ss_begin()
  wc_harvest(html6, wc_len(html6), hbase, wc_len(hbase), 0 as *i64, w6, prefix, segidbox, segsbox, hdisc, hrs, hnrs)
  ss_commit(prefix, w6, segidbox[0])
  let h6: *i64 = ss_open(prefix)
  let okey6: *u8 = sys_mmap(64)
  wc_outkey(ci_hash(hbase, wc_len(hbase)), okey6)
  let op6: *i64 = sys_mmap(16) as *i64
  let ol6: *i64 = sys_mmap(16) as *i64
  var t6: i64 = 0
  if ss_hget(h6, okey6, op6, ol6) == 1 { if ol6[0] == 16 {
      let ev: *i64 = op6[0] as *i64
      if ev[0] == ci_hash(t1u, wc_len(t1u)) { if ev[1] == ci_hash(t2u, wc_len(t2u)) { t6 = 1 } }
  } }
  gw_check("T6 P1 outlink capture (out:<cid(base)> = [cid(tgt1),cid(tgt2)], ordered)" as *u8, t6, pass, total)
187
  // T7 (R3 host-health): hd:<hosthash> streak round-trip; an unknown host reads 0
  segidbox[0] = segidbox[0] + 1
  let w7: *i64 = ss_begin()
  let hhA: i64 = wc_hosthash("dead.example" as *u8, 12)
  wc_hd_put(w7, prefix, segidbox, segsbox, hhA, 3)
  ss_commit(prefix, w7, segidbox[0])
  let h7: *i64 = ss_open(prefix)
  var t7: i64 = 0
  if wc_hd_streak(h7, hhA) == 3 { if wc_hd_streak(h7, wc_hosthash("live.example" as *u8, 12)) == 0 { t7 = 1 } }
  gw_check("T7 hd: streak round-trip (3) + unknown-host 0" as *u8, t7, pass, total)
198
  // T8 (R3): retire-if-rotten -- streak>=WC_HD_RETIRE retires a P row (P->D visible after commit),
  // streak below the bar keeps it pending, and a SEED url NEVER retires even on a rotten host
  // (wc_seed_add is blocked by any existing fr: row, so a retired seed would be permanently dead).
  let ur1: *u8 = "https://dead.example/rot1" as *u8
  let ur2: *u8 = "https://dying.example/fresh" as *u8
  let ur3: *u8 = "https://dead.example/seedhub" as *u8
  segidbox[0] = segidbox[0] + 1
  let w8: *i64 = ss_begin()
  wc_frmark(w8, prefix, segidbox, segsbox, ur1, wc_len(ur1), 1)
  wc_frmark(w8, prefix, segidbox, segsbox, ur2, wc_len(ur2), 1)
  wc_frmark(w8, prefix, segidbox, segsbox, ur3, wc_len(ur3), 1)
  let hhB: i64 = wc_hosthash("dying.example" as *u8, 13)
  wc_hd_put(w8, prefix, segidbox, segsbox, hhB, 2)
  ss_commit(prefix, w8, segidbox[0])
  let h8: *i64 = ss_open(prefix)
  let seeds8: *u8 = sys_mmap(WC_MAGIC_2048)
  var so8: i64 = 0
  while ur3[so8] != (0 as u8) { seeds8[so8] = ur3[so8]; so8 = so8 + 1 }
  seeds8[so8] = 0 as u8
  segidbox[0] = segidbox[0] + 1
  let w9: *i64 = ss_begin()
  // the caller pattern: seed membership guards the retire (exactly what main's skip branch does)
  var r1: i64 = 0
  if wc_is_seed(seeds8, 1, ur1, wc_len(ur1)) == 0 { r1 = wc_retire_if_rotten(h8, w9, prefix, segidbox, segsbox, ur1, wc_len(ur1), hhA) }
  var r2: i64 = 0
  if wc_is_seed(seeds8, 1, ur2, wc_len(ur2)) == 0 { r2 = wc_retire_if_rotten(h8, w9, prefix, segidbox, segsbox, ur2, wc_len(ur2), hhB) }
  var r3: i64 = 0
  if wc_is_seed(seeds8, 1, ur3, wc_len(ur3)) == 0 { r3 = wc_retire_if_rotten(h8, w9, prefix, segidbox, segsbox, ur3, wc_len(ur3), hhA) }
  if w9[1] > 0 { ss_commit(prefix, w9, segidbox[0]) }
  let h9: *i64 = ss_open(prefix)
  var t8: i64 = 0
  if r1 == 1 { if r2 == 0 { if r3 == 0 {
      if wc_frstate(h9, ur1, wc_len(ur1)) == 2 { if wc_frstate(h9, ur2, wc_len(ur2)) == 1 { if wc_frstate(h9, ur3, wc_len(ur3)) == 1 { t8 = 1 } } }
  } } }
  gw_check("T8 retire-if-rotten: streak>=3 retires, <3 keeps, seed exempt" as *u8, t8, pass, total)
234
  // ---- T9 (R4 CANONICAL LANE) ---------------------------------------------------------------
  // The host-filtered pull must keep ONLY canonical-seed hosts, match a SUBDOMAIN on the boundary
  // dot, and REFUSE a lookalike host. Network-free -- store rows only. The pre-existing fixture
  // rows (a.example / b.example) are non-canonical by construction, so they are excluded too and
  // the count is stable regardless of what T1-T8 left pending.
  segidbox[0] = segidbox[0] + 1
  let q9w: *i64 = ss_begin()
  let q9c1: *u8 = "https://en.wikipedia.org/wiki/Hypertension" as *u8
  let q9c2: *u8 = "https://data.who.int/topic/air" as *u8
  let q9c3: *u8 = "https://notwikipedia.org/lookalike" as *u8
  let q9c4: *u8 = "https://junkfarm.example/tag/a" as *u8
  wc_frmark(q9w, prefix, segidbox, segsbox, q9c1, wc_len(q9c1), 1)
  wc_frmark(q9w, prefix, segidbox, segsbox, q9c2, wc_len(q9c2), 1)
  wc_frmark(q9w, prefix, segidbox, segsbox, q9c3, wc_len(q9c3), 1)
  wc_frmark(q9w, prefix, segidbox, segsbox, q9c4, wc_len(q9c4), 1)
  ss_commit(prefix, q9w, segidbox[0])
  let q9seeds: *u8 = sys_mmap(WC_MAGIC_2048 * 4)
  let q9s1: *u8 = "https://www.who.int/" as *u8
  var q9a: i64 = 0
  while q9s1[q9a] != (0 as u8) { q9seeds[q9a] = q9s1[q9a]; q9a = q9a + 1 }
  q9seeds[q9a] = 0 as u8
  let q9d2: *u8 = (q9seeds as i64 + WC_MAGIC_2048) as *u8
  let q9s2: *u8 = "https://wikipedia.org/" as *u8
  var q9b: i64 = 0
  while q9s2[q9b] != (0 as u8) { q9d2[q9b] = q9s2[q9b]; q9b = q9b + 1 }
  q9d2[q9b] = 0 as u8
  let q9canh: *u8 = sys_mmap(64 * WC_MAXCANH)
  let q9ncan: i64 = wc_canon_hosts(q9seeds, 2, q9canh)
  let q9h: *i64 = ss_open(prefix)
  let q9urls: *u8 = sys_mmap(WC_MAGIC_2048 * 32)
  let q9n: i64 = wc_pending(q9h, prefix, q9urls, 32, q9canh, q9ncan, 1)
  var t9: i64 = 0
  if q9ncan == 2 { if wc_streq(q9canh, "who.int" as *u8) == 1 {
      if wc_streq((q9canh as i64 + 64) as *u8, "wikipedia.org" as *u8) == 1 { if q9n == 2 { t9 = 1 } }
  } }
  gw_check("T9 canonical lane: www-stripped hosts, subdomain kept, junk excluded -> 2 of 4" as *u8, t9, pass, total)
  // T9b THE NEGATIVE CONTROL that makes T9 non-vacuous: a lookalike host must NOT ride the lane.
  let q9hb: *u8 = sys_mmap(64)
  var t9b: i64 = 0
  if wc_host_is_canon(q9canh, q9ncan, q9c1, q9hb) == 1 { if wc_host_is_canon(q9canh, q9ncan, q9c2, q9hb) == 1 {
      if wc_host_is_canon(q9canh, q9ncan, q9c3, q9hb) == 0 { if wc_host_is_canon(q9canh, q9ncan, q9c4, q9hb) == 0 { t9b = 1 } }
  } }
  gw_check("T9b lookalike REFUSED: notwikipedia.org needs the boundary dot to match wikipedia.org" as *u8, t9b, pass, total)
  // T9c canonly=0 is the pre-R4 contract: the SAME walk sees all four rows (plus fixture leftovers).
  let q9all: i64 = wc_pending(q9h, prefix, q9urls, 32, q9canh, 0, 0)
  gw_check("T9c canonly=0 unchanged: unfiltered walk still sees strictly more rows than the lane" as *u8, (q9all > q9n) as i64, pass, total)
  // T9d (R4b): the host-string matcher is the ONE rule and the url matcher only wraps it. If these
  // two ever disagree, the canonical LANE and the canonical DEPTH CAP would disagree about which
  // hosts are curated -- a split definition is exactly how a privilege leaks to a lookalike host.
  var q9agree: i64 = 1
  if wc_host_str_is_canon(q9canh, q9ncan, "wikipedia.org" as *u8, 13) != 1 { q9agree = 0 }
  if wc_host_str_is_canon(q9canh, q9ncan, "en.wikipedia.org" as *u8, 16) != 1 { q9agree = 0 }
  if wc_host_str_is_canon(q9canh, q9ncan, "data.who.int" as *u8, 12) != 1 { q9agree = 0 }
  if wc_host_str_is_canon(q9canh, q9ncan, "notwikipedia.org" as *u8, 16) != 0 { q9agree = 0 }
  if wc_host_str_is_canon(q9canh, q9ncan, "junkfarm.example" as *u8, 16) != 0 { q9agree = 0 }
  gw_check("T9d host-string matcher == url matcher (exact, subdomain, lookalike, junk)" as *u8, q9agree, pass, total)
291
  // ---- T10 (R5 SEED FRESHNESS RE-QUEUE) -------------------------------------------------------
  // A D seed must return to P and be copied into the lane buffer; a P seed and a never-seen seed
  // must be left alone. Network-free, store rows only.
  segidbox[0] = segidbox[0] + 1
  let qAw: *i64 = ss_begin()
  let qA1: *u8 = "https://canon-a.example/hub" as *u8
  let qA2: *u8 = "https://canon-b.example/hub" as *u8
  let qA3: *u8 = "https://canon-c.example/hub" as *u8
  wc_frmark(qAw, prefix, segidbox, segsbox, qA1, wc_len(qA1), 2)
  wc_frmark(qAw, prefix, segidbox, segsbox, qA2, wc_len(qA2), 1)
  ss_commit(prefix, qAw, segidbox[0])
  let qAseeds: *u8 = sys_mmap(WC_MAGIC_2048 * 4)
  var qAi: i64 = 0
  while qA1[qAi] != (0 as u8) { qAseeds[qAi] = qA1[qAi]; qAi = qAi + 1 }
  qAseeds[qAi] = 0 as u8
  let qAd2: *u8 = (qAseeds as i64 + WC_MAGIC_2048) as *u8
  qAi = 0
  while qA2[qAi] != (0 as u8) { qAd2[qAi] = qA2[qAi]; qAi = qAi + 1 }
  qAd2[qAi] = 0 as u8
  let qAd3: *u8 = (qAseeds as i64 + 2 * WC_MAGIC_2048) as *u8
  qAi = 0
  while qA3[qAi] != (0 as u8) { qAd3[qAi] = qA3[qAi]; qAi = qAi + 1 }
  qAd3[qAi] = 0 as u8
  let qAh: *i64 = ss_open(prefix)
  segidbox[0] = segidbox[0] + 1
  let qAw2: *i64 = ss_begin()
  let qAdst: *u8 = sys_mmap(WC_MAGIC_2048 * 8)
  let qAbox: *i64 = sys_mmap(16) as *i64
  qAbox[0] = 0
  qAbox[1] = 6
  let qArq: i64 = wc_seed_refresh(qAh, qAw2, prefix, segidbox, segsbox, qAseeds, 3, qAdst, qAbox)
  if qAw2[1] > 0 { ss_commit(prefix, qAw2, segidbox[0]) }
  let qAh2: *i64 = ss_open(prefix)
  var t10: i64 = 0
  if qArq == 1 { if qAbox[0] == 1 {
      if wc_frstate(qAh2, qA1, wc_len(qA1)) == 1 { if wc_frstate(qAh2, qA2, wc_len(qA2)) == 1 { if wc_frstate(qAh2, qA3, wc_len(qA3)) == 0 {
          if wc_streq(qAdst, qA1) == 1 { t10 = 1 }
      } } }
  } }
  gw_check("T10 seed re-queue: D seed -> P + copied to lane; P and never-seen UNTOUCHED" as *u8, t10, pass, total)
  // T10b the persisted cursor round-trips (the rotation mechanism across runs)
  segidbox[0] = segidbox[0] + 1
  let qAw3: *i64 = ss_begin()
  wc_sr_put(qAw3, prefix, segidbox, segsbox, 5)
  if qAw3[1] > 0 { ss_commit(prefix, qAw3, segidbox[0]) }
  let qAh3: *i64 = ss_open(prefix)
  gw_check("T10b sr:cursor persists across commit (put 5 -> read 5)" as *u8, (wc_sr_cursor(qAh3) == 5) as i64, pass, total)
339
  // ---- T11 (R10 SHAPE-BASED QUERY ADMISSION -- replaces R8's host allowlist) -------------------
  // R8 admitted query strings ONLY on curated hosts, so an uncurated content url was refused before
  // a byte was fetched -- a DIRECTORY, not an index. R10 judges the query's SHAPE instead, so the
  // SAME two fixtures now split on WHAT THE URL IS rather than WHO SERVES IT:
  //   clean content query, uncurated host -> ADMITTED  (this is the whole point of R10)
  //   tracking param, uncurated host      -> REFUSED   (a duplicate of the clean url)
  // Deny list set directly here, exactly as the canon table is a few lines below: the gate must be
  // self-contained, never dependent on whatever the live urlpolicy- plane happens to hold today.
  let qB: *u8 = "https://graphis.ne.jp/monthly/model.php?ID=260731_sp" as *u8
  let qBjunk: *u8 = "https://tracker.example/page.php?utm_source=x" as *u8
  var t11a: i64 = 0
  if wc_urlok(qB, wc_len(qB)) == 1 { if wc_urlok(qBjunk, wc_len(qBjunk)) == 0 { t11a = 1 } }
  gw_check("T11 R10: a CLEAN query url is ADMITTED with an EMPTY canon table; a TRACKING one is REFUSED" as *u8, t11a, pass, total)
  // THE TOOTH THAT WOULD HAVE CAUGHT R8: the operator's real-world miss. An uncurated host serving
  // paginated content must be reachable -- under R8 this was refused purely for lacking curation.
  let qBv2: *u8 = "https://www.v2ph.com/actor/JULIA?page=3&hl=en" as *u8
  gw_check("T11a-ii an UNCURATED host's paginated content url is ADMITTED (the R8 miss, by name)" as *u8, (wc_urlok(qBv2, wc_len(qBv2)) == 1) as i64, pass, total)
  // and the trap half: a facet explosion beyond the param budget is still refused, on ANY host
  let qBfacet: *u8 = "https://shop.example/list?a=1&b=2&c=3&d=4&e=5&f=6" as *u8
  gw_check("T11a-iii a 6-param facet explosion is REFUSED (budget, not curation, is the limit)" as *u8, (wc_urlok(qBfacet, wc_len(qBfacet)) == 0) as i64, pass, total)
  // now publish a canon table holding ONLY graphis.ne.jp
  let qBcanh: *u8 = sys_mmap(64 * WC_MAXCANH)
  var qBi: i64 = 0
  let qBhost: *u8 = "graphis.ne.jp" as *u8
  while qBhost[qBi] != (0 as u8) { qBcanh[qBi] = qBhost[qBi]; qBi = qBi + 1 }
  qBcanh[qBi] = 0 as u8
  wc_canh_g = qBcanh
  wc_ncan_g = 1
  wc_qscratch_g = sys_mmap(64)
  // BISECT: which layer is failing -- the policy flag, or the canon match?
  let t11p: i64 = olh_urlok_q(qB, wc_len(qB), 1)
  gw_check("T11b-i policy layer: olh_urlok_q(allowq=1) admits the query-string url" as *u8, (t11p == 1) as i64, pass, total)
  let t11m: i64 = wc_host_is_canon(qBcanh, 1, qB, wc_qscratch_g)
  gw_check("T11b-ii matcher layer: wc_host_is_canon sees graphis.ne.jp in the table" as *u8, (t11m == 1) as i64, pass, total)
  // R10: curation is now a BOOST, not a gate. The curated host keeps admitting its content, and the
  // tracking url stays refused ON THE CURATED PATH TOO -- trust buys a bigger param budget, it does
  // NOT buy immunity from junk. A curated host that could launder tracking urls into the index would
  // make curation a liability rather than a signal.
  var t11b: i64 = 0
  if wc_urlok(qB, wc_len(qB)) == 1 { if wc_urlok(qBjunk, wc_len(qBjunk)) == 0 { t11b = 1 } }
  let qBcurjunk: *u8 = "https://graphis.ne.jp/p.php?utm_source=x" as *u8
  gw_check("T11b curated host ADMITS its content; a TRACKING url is refused on the CURATED path too" as *u8, (t11b == 1) as i64 * (wc_urlok(qBcurjunk, wc_len(qBcurjunk)) == 0) as i64, pass, total)
  // the other kill-classes must NOT be weakened for the curated host either
  let qBtag: *u8 = "https://graphis.ne.jp/tag/foo?ID=1" as *u8
  let qBimg: *u8 = "https://graphis.ne.jp/i/photo.jpg" as *u8
  var t11c: i64 = 0
  if wc_urlok(qBtag, wc_len(qBtag)) == 0 { if wc_urlok(qBimg, wc_len(qBimg)) == 0 { t11c = 1 } }
  gw_check("T11c curation relaxes ONLY '?' -- tag-farm and binary-extension kills still hold" as *u8, t11c, pass, total)
  // T12 (R9): a CURATED host is never retired -- we cannot distinguish "host is down" from "we
  // cannot speak its TLS", so retiring it would launder our own capability gap into permanent
  // coverage loss. Uncurated rot must STILL retire (that is the control).
  var t12: i64 = 0
  let qCu: *u8 = "https://graphis.ne.jp/some/page" as *u8
  let qCj: *u8 = "https://rotten.example/dead" as *u8
  var qCcur: i64 = 0
  var qCrot: i64 = 0
  if wc_host_is_canon(qBcanh, 1, qCu, wc_qscratch_g) == 1 { qCcur = 1 }
  if wc_host_is_canon(qBcanh, 1, qCj, wc_qscratch_g) == 0 { qCrot = 1 }
  if qCcur == 1 { if qCrot == 1 { t12 = 1 } }
  gw_check("T12 retire-exemption predicate: curated host protected, uncurated rot still retirable" as *u8, t12, pass, total)
  wc_ncan_g = 0
401
  gw_puts("=== crawl-step gate: " as *u8); gw_num(pass[0]); gw_puts("/" as *u8); gw_num(total[0]); gw_puts(" " as *u8)
  if pass[0] == total[0] { gw_puts("GREEN\n" as *u8) } else { gw_puts("RED\n" as *u8) }
  let ctr: *i64 = gv_ctr()
  ctr[0] = pass[0]
  ctr[1] = total[0]
  return gv_verdict("WEB-CRAWL-STEP-GATE" as *u8, ctr, "frontier store mechanics: url filter, P-to-D state flip, cross-segment dedupe, outlink capture" as *u8)
408}
nx_web_crawl_step_gate.nx source

`nx_web_crawl_step_gate.nx` source
