code wiki / (root) / nx_crawl_sufficiency_gate.nx
`nx_crawl_sufficiency_gate.nx` source

↩ module page · 162 lines · 9815 B
1// nx_crawl_sufficiency_gate.nx -- GATE for nx_crawl_sufficiency (the /compare/webscraping R5 contract
2// cs_confidence). Network-free, fixtures assembled at runtime. The accept rule pre-declared on the board: a
3// crawl that has covered half the query MUST NOT stop; near-duplicate pages MUST NOT raise confidence; a
4// crawl that has covered the query and stopped discovering vocabulary MUST stop and say which metric moved;
5// off-topic pages can never reach CONFIDENT; a full term set makes saturation UNOBSERVABLE, never 0-as-fact.
6// license_tier: ORIGINAL
7import "nx_syscalls.nx"
8import "nx_crawl_sufficiency.nx"
9import "nx_gate_verdict.nx"
10
11const SG_CAP: i64 = 8192
12
13func sg_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
14func sg_has(hay: *u8, n: i64, needle: *u8) -> i64 {
  let nl: i64 = sg_len(needle)
  if nl == 0 { return 1 }
  var i: i64 = 0
  while i + nl <= n {
      var k: i64 = 0
      var m: i64 = 1
      while k < nl { if hay[i + k] != needle[k] { m = 0; k = nl } else { k = k + 1 } }
      if m == 1 { return 1 }
      i = i + 1
  }
  return 0
26}
27func sg_add(st: *i64, s: *u8) -> i64 { return cs_add_doc(st, s, sg_len(s)) }
28
29func main() -> i64 {
  gv_head("=== nx_crawl_sufficiency gate (adaptive stop: coverage, consistency, saturation; network-free) ===" as *u8)
  let ctr: *i64 = gv_ctr()
  let q: *u8 = "butter churning temperature texture" as *u8
  let d1: *u8 = "Butter churned at a low temperature gives a firm result; the churning temperature decides how the cream separates and how the fat globules pack together afterwards." as *u8
  let d1dup: *u8 = "BUTTER churned at a low temperature   gives a firm result -- the churning temperature decides how the cream separates, and how the fat globules pack together afterwards" as *u8
  let d2: *u8 = "The rheology study measured texture after the cream was aged twelve hours, and butter firmness rose steadily with lower texture readings on the penetrometer." as *u8
  let d5: *u8 = "Churning temperature and texture: butter churned low gives firm texture, the cream aged twelve hours, firmness rose, the fat globules pack, the churning temperature decides the butter." as *u8
  let off1: *u8 = "Quarterly earnings rose as the company expanded its logistics fleet across three regions and hired regional managers." as *u8
  let off2: *u8 = "Quarterly earnings rose again as the company expanded its logistics fleet across four regions and hired more regional managers." as *u8
  let off3: *u8 = "Earnings rose as the company expanded its fleet across regions and hired managers for logistics." as *u8
  let rep: *u8 = sys_mmap(SG_CAP)
41
  // ---- T1 query terms ----
  let st: *i64 = cs_new(q, sg_len(q))
  gv_puts("  conf_src=" as *u8); if st[CS_S_CONF_SRC] == 1 { gv_puts("file" as *u8) } else { gv_puts("defaults" as *u8) }
  gv_puts(" w=" as *u8); gv_num(st[CS_S_W_COV]); gv_puts("/" as *u8); gv_num(st[CS_S_W_CON]); gv_puts("/" as *u8); gv_num(st[CS_S_W_SAT])
  gv_puts(" bar=" as *u8); gv_num(st[CS_S_CONF_STOP]); gv_puts(" sat_bar=" as *u8); gv_num(st[CS_S_SAT_STOP]); gv_puts("\n" as *u8)
  var t1: i64 = 0
  if st[CS_S_QN] == 4 { if st[CS_S_DROPPED_TERMS] == 0 { t1 = 1 } }
  gv_check("T1 the query tokenizes to 4 distinct terms with none dropped" as *u8, t1, ctr)
50
  // ---- T2 empty state: nothing is known, nothing stops ----
  var t2: i64 = 0
  if cs_confidence(st) == 0 { if st[CS_S_STOP] == CS_CONTINUE { t2 = 1 } }
  gv_check("T2 with no documents confidence is 0 and the verdict is CONTINUE" as *u8, t2, ctr)
55
  // ---- T3 half the query covered -> MUST NOT stop ----
  let a1: i64 = sg_add(st, d1)          // butter, churning, temperature (3 of 4 terms; no 'texture')
  let c1: i64 = cs_confidence(st)
  let stop1: i64 = st[CS_S_STOP]
  cs_report(st, rep, SG_CAP); sys_write(1, "  after d1: " as *u8, 12); sys_write(1, rep, cs_report(st, rep, SG_CAP))
  var t3: i64 = 0
  if a1 == 1 { if stop1 == CS_CONTINUE { if c1 > 0 { if c1 < st[CS_S_CONF_STOP] { t3 = 1 } } } }
  gv_check("T3 one on-topic page covering 3 of 4 terms is counted, raises confidence, and does NOT stop" as *u8, t3, ctr)
  var t3b: i64 = 0
  if st[CS_S_SAT_OBSERVED] == 0 { t3b = 1 }
  gv_check("T3b saturation is UNOBSERVED with one page (needs two windows), not reported as 0-as-fact" as *u8, t3b, ctr)
  // T3c: a SINGLE page holding every query term reaches the bar arithmetically (the source stops here) -- ours abstains
  let st1: *i64 = cs_new(q, sg_len(q))
  sg_add(st1, d5)
  let c1c: i64 = cs_confidence(st1)
  var t3c: i64 = 0
  if c1c >= st1[CS_S_CONF_STOP] { if st1[CS_S_STOP] == CS_CONTINUE { if st1[CS_S_SAT_OBSERVED] == 0 { t3c = 1 } } }
  gv_check("T3c one page holding ALL terms hits the bar but cannot STOP while saturation is unobserved (abstain, never acquit)" as *u8, t3c, ctr)
  cs_free(st1)
75
  // ---- T4 a near-duplicate does not move anything (the exceed over Jaccard consistency) ----
  let a2: i64 = sg_add(st, d1dup)
  let c2: i64 = cs_confidence(st)
  var t4: i64 = 0
  if a2 == 0 { if st[CS_S_DUPS] == 1 { if st[CS_S_DOCS] == 1 { if c2 == c1 { t4 = 1 } } } }
  gv_check("T4 a near-duplicate page is tallied as a dup, not counted, and confidence is byte-unchanged" as *u8, t4, ctr)
82
  // ---- T5 the fourth term arrives, vocabulary still growing -> still CONTINUE ----
  sg_add(st, d2)
  let c3: i64 = cs_confidence(st)
  sys_write(1, "  after d2: " as *u8, 12); sys_write(1, rep, cs_report(st, rep, SG_CAP))
  var t5: i64 = 0
  if c3 > c1 { if st[CS_S_COVERAGE] > 500 { if st[CS_S_SAT_OBSERVED] == 1 { t5 = 1 } } }
  gv_check("T5 the page carrying the missing term raises coverage above half and saturation becomes observable" as *u8, t5, ctr)
90
  // ---- T6 a page that repeats known vocabulary -> saturation and confidence cross the bar -> STOP ----
  sg_add(st, d5)
  let c4: i64 = cs_confidence(st)
  sys_write(1, "  after d5: " as *u8, 12); sys_write(1, rep, cs_report(st, rep, SG_CAP))
  var t6: i64 = 0
  if st[CS_S_STOP] != CS_CONTINUE { if c4 >= st[CS_S_CONF_STOP] { if st[CS_S_LAST_NEW] < st[CS_S_FIRST_NEW] { t6 = 1 } } }
  gv_check("T6 a page that adds no new vocabulary drives saturation up and the verdict STOPS at the bar" as *u8, t6, ctr)
  var t6b: i64 = 0
  if st[CS_S_DOCS] + st[CS_S_DUPS] == 4 { t6b = 1 }
  gv_check("T6b partition sums: counted + dups == pages added (4)" as *u8, t6b, ctr)
  let rn: i64 = cs_report(st, rep, SG_CAP)
  var t6c: i64 = 0
  if sg_has(rep, rn, "verdict=CONFIDENT-STOP" as *u8) == 1 { t6c = 1 }
  if sg_has(rep, rn, "verdict=SATURATED-STOP" as *u8) == 1 { t6c = 1 }
  gv_check("T6c the report names the stop verdict" as *u8, t6c, ctr)
  var fired_bad: i64 = 0
  if st[CS_S_STOP] != CS_CONTINUE { fired_bad = 1 }
  var fired_good: i64 = 0
  if stop1 != CS_CONTINUE { fired_good = 1 }
  gv_bite("T7 BITE: the stop fires on the covered-and-saturated crawl, stays silent on the half-covered one" as *u8, fired_bad, fired_good, ctr)
  cs_free(st)
112
  // ---- T8 NEG-CONTROL: off-topic pages can never reach CONFIDENT ----
  let st2: *i64 = cs_new(q, sg_len(q))
  sg_add(st2, off1); sg_add(st2, off2); sg_add(st2, off3)
  let c8: i64 = cs_confidence(st2)
  sys_write(1, "  off-topic: " as *u8, 13); sys_write(1, rep, cs_report(st2, rep, SG_CAP))
  var t8: i64 = 0
  if st2[CS_S_COVERAGE] == 0 { if st2[CS_S_CONSISTENCY] == 0 { if st2[CS_S_STOP] != CS_CONFIDENT { if c8 < st2[CS_S_CONF_STOP] { t8 = 1 } } } }
  gv_check("T8 NEG-CONTROL: three off-topic pages -> coverage 0, consistency 0, never CONFIDENT" as *u8, t8, ctr)
  cs_free(st2)
122
  // ---- T9 a full term set makes saturation UNOBSERVABLE (announced), never a number ----
  let st3: *i64 = cs_new_t(q, sg_len(q), 16, 64)     // 16 slots, full at 12 distinct terms
  sg_add(st3, d1); sg_add(st3, d2); sg_add(st3, d5)
  cs_confidence(st3)
  let r9: i64 = cs_report(st3, rep, SG_CAP)
  var t9: i64 = 0
  if st3[CS_S_TERMSET_FULL] == 1 { if st3[CS_S_SAT_OBSERVED] == 0 { if st3[CS_S_STOP] != CS_SATURATED { if sg_has(rep, r9, "UNOBSERVABLE" as *u8) == 1 { t9 = 1 } } } }
  gv_check("T9 term set full -> saturation UNOBSERVABLE, cannot produce SATURATED-STOP, and the report says so" as *u8, t9, ctr)
  cs_free(st3)
132
  // ---- T10 history full refuses further pages loudly ----
  let st4: *i64 = cs_new_t(q, sg_len(q), 65536, 2)
  let h1: i64 = sg_add(st4, d1)
  let h2: i64 = sg_add(st4, d2)
  let h3: i64 = sg_add(st4, d5)
  var t10: i64 = 0
  if h1 == 1 { if h2 == 1 { if h3 == 0 - 1 { if st4[CS_S_HIST_FULL] == 1 { t10 = 1 } } } }
  gv_check("T10 the page history cap REFUSES the page past it (-1) and flags itself, never drops silently" as *u8, t10, ctr)
  cs_free(st4)
142
  // ---- T11 query-term cap refuses loudly ----
  let big: *u8 = sys_mmap(SG_CAP)
  var bo: i64 = 0
  var wi: i64 = 0
  while wi < 70 {
      big[bo] = 113 as u8; bo = bo + 1                       // 'q'
      big[bo] = (97 + (wi % 26)) as u8; bo = bo + 1
      big[bo] = (97 + ((wi / 26) % 26)) as u8; bo = bo + 1
      big[bo] = 120 as u8; bo = bo + 1                       // 'x'
      big[bo] = 32 as u8; bo = bo + 1
      wi = wi + 1
  }
  let st5: *i64 = cs_new(big, bo)
  var t11: i64 = 0
  if st5[CS_S_QN] == CS_QMAX { if st5[CS_S_DROPPED_TERMS] == 70 - CS_QMAX { t11 = 1 } }
  gv_check("T11 a 70-term query keeps 64 and REPORTS 6 dropped -- the cap is loud" as *u8, t11, ctr)
  cs_free(st5)
160
  return gv_verdict("CRAWL-SUFFICIENCY-GATE" as *u8, ctr, "adaptive stop in permil: half-covered crawl continues, duplicates cannot inflate, saturated crawl stops and names the metric, off-topic never confident, full term set abstains" as *u8)
162}
nx_crawl_sufficiency_gate.nx source

`nx_crawl_sufficiency_gate.nx` source
