code wiki / _hdl_build / nx_docportal_search_seg_gate.nx
`nx_docportal_search_seg_gate.nx` source

↩ module page · 547 lines · 34726 B
1// nx_docportal_search_seg_gate.nx -- GATE for the SOVEREIGN seg_store-native onsite search (nx_docportal_search_seg).
2// Writes 3 PUBLIC docs into a test shard through the RAW sovereign store (doc:<cid> keys, text values so
3// ss_write_seg builds .terms), then drives dss_search end-to-end over the REAL seg_store -- proving the sovereign
4// path (ss_open -> ss_term postings -> ss_hget -> rank) with NO tsv anywhere. Negative controls: absent term -> 0,
5// absent shard -> 0 (isolation, not a crash). license_tier: ORIGINAL
6import "nx_docportal_search_seg.nx"
7
8// evidence-line builders (local, so this gate stays self-contained)
9func dsg_cat(b: *u8, off: i64, s: *u8) -> i64 { var o: i64 = off; var i: i64 = 0; while s[i] != (0 as u8) { b[o] = s[i]; o = o + 1; i = i + 1 } return o }
10func dsg_catn(b: *u8, off: i64, v: i64) -> i64 {
  var m: i64 = v
  if m < 0 { m = 0 }
  let t: *u8 = sys_mmap(28)
  var k: i64 = 0
  if m == 0 { t[0] = 48 as u8; k = 1 }
  while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
  var o: i64 = off
  var i: i64 = 0
  while i < k { b[o] = t[k - 1 - i]; o = o + 1; i = i + 1 }
  return o
21}
22
23func g_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
24func g_num(v: i64) -> i64 {
  let bb: *u8 = sys_mmap(28); var m: i64 = v
  if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
  let t: *u8 = sys_mmap(28); var k: i64 = 0
  if m == 0 { t[0] = 48 as u8; k = 1 }
  while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
  var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 }
  sys_write(1, bb, k); return 0
32}
33func g_strlen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
34
35// one check: got n results with cids_out[0]==want_top (want_top<0 = don't care about top); expect_n<0 = don't care
36func g_check(name: *u8, got_n: i64, cids: *i64, expect_n: i64, want_top: i64, pass: *i64, total: *i64) -> i64 {
  total[0] = total[0] + 1
  var ok: i64 = 1
  if expect_n >= 0 { if got_n != expect_n { ok = 0 } }
  if want_top >= 0 { if got_n < 1 { ok = 0 } else { if cids[0] != want_top { ok = 0 } } }
  g_puts(name); g_puts(" n="); g_num(got_n)
  if ok == 1 { pass[0] = pass[0] + 1; g_puts(" PASS\n") } else { g_puts(" FAIL\n") }
  return 0
44}
45
46func main(argc: i64, argv: *i64) -> i64 {
  g_puts("=== nx_docportal_search_seg gate (SOVEREIGN seg_store search, no tsv) ===\n")
  // --- write a test PUBLIC shard through the raw sovereign store ---
  let prefix: *u8 = "knowledge/store/dp-ssgatetest-pub-" as *u8
  let d1: *u8 = "Estate Planning wills living trusts probate inheritance executor" as *u8
  let d2: *u8 = "Family Law divorce child custody support alimony parenting plans" as *u8
  let d3: *u8 = "Business Law llc corporation formation contracts operating agreement" as *u8
  // consent rows (same segment, deterministic re-runs): 104 opted OUT of search (pol=2 = AI_BLOG only),
  // 105 opted IN (pol=1). 101-103 stay pol-FREE = the absent-pol default-searchable path.
  let d4: *u8 = "Adoption confidential retainer matters" as *u8
  let d5: *u8 = "Municipal annexation ordinance guidance" as *u8
  let w: *i64 = ss_begin()
  ss_add(w, 1, "doc:101" as *u8, d1, g_strlen(d1))
  ss_add(w, 1, "doc:102" as *u8, d2, g_strlen(d2))
  ss_add(w, 1, "doc:103" as *u8, d3, g_strlen(d3))
  // url rows for the site: field-filter rows (103 deliberately has NO url = not site-attributable)
  ss_add(w, 1, "url:101" as *u8, "https://en.wikipedia.org/wiki/Trust_law" as *u8, 39)
  ss_add(w, 1, "url:102" as *u8, "https://law.example.com/family" as *u8, 30)
  ss_add(w, 1, "doc:104" as *u8, d4, g_strlen(d4))
  ss_add(w, 1, "pol:104" as *u8, "2" as *u8, 1)
  ss_add(w, 1, "doc:105" as *u8, d5, g_strlen(d5))
  ss_add(w, 1, "pol:105" as *u8, "1" as *u8, 1)
  // IDF fixture (T9): idfrare lives in ONE doc (106); idfcommon in FOUR (107-110). Equal tf. The OLD
  // distinct+tf scorer TIED 106 vs 107 (first candidate won = 107, postings order); only IDF ranking
  // puts the rare-term doc on top -- this row fails on the old scorer BY CONSTRUCTION.
  let d6: *u8 = "idfrare idfrare notice" as *u8
  let d7: *u8 = "idfcommon idfcommon memo" as *u8
  let d8: *u8 = "idfcommon lorem" as *u8
  let d9: *u8 = "idfcommon ipsum" as *u8
  let d10: *u8 = "idfcommon dolor" as *u8
  ss_add(w, 1, "doc:106" as *u8, d6, g_strlen(d6))
  ss_add(w, 1, "doc:107" as *u8, d7, g_strlen(d7))
  ss_add(w, 1, "doc:108" as *u8, d8, g_strlen(d8))
  ss_add(w, 1, "doc:109" as *u8, d9, g_strlen(d9))
  ss_add(w, 1, "doc:110" as *u8, d10, g_strlen(d10))
  // length-norm fixture (T14): the LONG doc (126) is inserted FIRST -- under b=0 equal tf tied and
  // candidate order won (126); full BM25's verbosity correction must put the SHORT doc (127) on top.
  let dlong: *u8 = sys_mmap(4096)
  var dlo: i64 = 0
  var lr: i64 = 0
  while lr < 80 { let wf: *u8 = "verbose filler prose words continue onward " as *u8; var wj: i64 = 0; while wf[wj] != (0 as u8) { dlong[dlo] = wf[wj]; dlo = dlo + 1; wj = wj + 1 } lr = lr + 1 }
  let lm: *u8 = "lennorm appears once here" as *u8
  var lj: i64 = 0
  while lm[lj] != (0 as u8) { dlong[dlo] = lm[lj]; dlo = dlo + 1; lj = lj + 1 }
  let dshort: *u8 = "lennorm short focused note" as *u8
  ss_add(w, 1, "doc:126" as *u8, dlong, dlo)
  ss_add(w, 1, "doc:127" as *u8, dshort, g_strlen(dshort))
  // stemming fixture (T18): a doc that only says the INFLECTED form -- query stem-expansion must recall it
  let dstem: *u8 = "stemtoken indexing crawlers were ranking pages" as *u8
  ss_add(w, 1, "doc:128" as *u8, dstem, g_strlen(dstem))
  // pagination fixture (T10): 15 pgseg docs -> offset/total behavior is provable (page sizes 10/5/0)
  let dpg: *u8 = "pgseg pagination fixture entry" as *u8
  let pgkey: *u8 = sys_mmap(24)
  var pgi: i64 = 111
  while pgi <= 125 {
      var ko: i64 = 0
      let kp: *u8 = "doc:" as *u8
      while kp[ko] != (0 as u8) { pgkey[ko] = kp[ko]; ko = ko + 1 }
      pgkey[ko] = (48 + (pgi / 100)) as u8
      pgkey[ko + 1] = (48 + ((pgi / 10) % 10)) as u8
      pgkey[ko + 2] = (48 + (pgi % 10)) as u8
      pgkey[ko + 3] = 0 as u8
      ss_add(w, 1, pgkey, dpg, g_strlen(dpg))
      pgi = pgi + 1
  }
  // host-diversity fixture (T22/T23): 5 docs on crowd.example.com (tf=2) + 2 on other.example.com (tf=1),
  // all matching "diversitytest". WITHOUT the cap (webdiv=0) the 5 tf=2 crowd docs take the top 5 slots
  // (cids[3]=304); WITH the cap (webdiv=1, DSS_HOSTCAP=3) an OTHER-host doc is promoted into slot 3
  // (cids[3]=306) once crowd hits the cap -- distinguishing by construction (the old scorer had no cap).
  let dcrowd: *u8 = "diversitytest diversitytest note" as *u8
  let dother: *u8 = "diversitytest memo" as *u8
  ss_add(w, 1, "doc:301" as *u8, dcrowd, g_strlen(dcrowd))
  ss_add(w, 1, "url:301" as *u8, "https://crowd.example.com/1" as *u8, 27)
  ss_add(w, 1, "doc:302" as *u8, dcrowd, g_strlen(dcrowd))
  ss_add(w, 1, "url:302" as *u8, "https://crowd.example.com/2" as *u8, 27)
  ss_add(w, 1, "doc:303" as *u8, dcrowd, g_strlen(dcrowd))
  ss_add(w, 1, "url:303" as *u8, "https://crowd.example.com/3" as *u8, 27)
  ss_add(w, 1, "doc:304" as *u8, dcrowd, g_strlen(dcrowd))
  ss_add(w, 1, "url:304" as *u8, "https://crowd.example.com/4" as *u8, 27)
  ss_add(w, 1, "doc:305" as *u8, dcrowd, g_strlen(dcrowd))
  ss_add(w, 1, "url:305" as *u8, "https://crowd.example.com/5" as *u8, 27)
  ss_add(w, 1, "doc:306" as *u8, dother, g_strlen(dother))
  ss_add(w, 1, "url:306" as *u8, "https://other.example.com/6" as *u8, 27)
  ss_add(w, 1, "doc:307" as *u8, dother, g_strlen(dother))
  ss_add(w, 1, "url:307" as *u8, "https://other.example.com/7" as *u8, 27)
  // authority fusion fixture (T28/T29): two docs identical content (equal BM25), different urls; doc 601's url
  // carries a high pr: prior, doc 600's has none. webdiv=0 -> tie broken by candidate order (600 first);
  // webdiv=1 -> the PageRank prior lifts 601 over 600 (distinguishing: pre-fuse, webdiv=1 also gave 600).
  let dae: *u8 = "authorityterm equalcontent alpha" as *u8
  let u600: *u8 = "https://plain.example/b" as *u8
  let u601: *u8 = "https://authority.example/a" as *u8
  ss_add(w, 1, "doc:600" as *u8, dae, g_strlen(dae))
  ss_add(w, 1, "url:600" as *u8, u600, g_strlen(u600))
  ss_add(w, 1, "doc:601" as *u8, dae, g_strlen(dae))
  ss_add(w, 1, "url:601" as *u8, u601, g_strlen(u601))
  let prk: *u8 = sys_mmap(48)
  dss_prkey(dss_urlcid(u601, g_strlen(u601)), prk)
  let prv: *i64 = sys_mmap(16) as *i64
  prv[0] = 500000000
  ss_add(w, 1, prk, prv as *u8, 8)
  // url-dedup fixture (T30/T31): docs 700+701 = two crawl SNAPSHOTS of the SAME url (identical content ->
  // equal BM25; distinct content cids in prod, distinct doc keys here). webdiv=1 must serve ONE.
  let ddup: *u8 = "zqduppage snapshot content for url dedup" as *u8
  let udup: *u8 = "https://dup.example.com/page" as *u8
  ss_add(w, 1, "doc:700" as *u8, ddup, g_strlen(ddup))
  ss_add(w, 1, "url:700" as *u8, udup, g_strlen(udup))
  ss_add(w, 1, "doc:701" as *u8, ddup, g_strlen(ddup))
  ss_add(w, 1, "url:701" as *u8, udup, g_strlen(udup))
  // proximity fixture (T-prox, R1b): 801 (scattered) + 800 (clustered) have the SAME length + SAME tf
  // -> IDENTICAL BM25; ONLY the proximity boost distinguishes them. 801 is indexed FIRST (so a naive
  // tie would pick 801); the CLUSTERED 800 must win via proximity. proxterma/proxtermb/zfil* are unique.
  let dpxs: *u8 = "proxterma zfila zfilb zfilc zfild zfile zfilf proxtermb" as *u8
  let dpxc: *u8 = "zfilg zfilh zfili zfilj proxterma proxtermb zfilk zfill" as *u8
  ss_add(w, 1, "doc:801" as *u8, dpxs, g_strlen(dpxs))
  ss_add(w, 1, "doc:800" as *u8, dpxc, g_strlen(dpxc))
  // title-field fixture (T-title, R1c): 901 (body) + 900 (title) have SAME length + SAME tf -> IDENTICAL
  // BM25; only the title-field boost distinguishes. 901 indexed FIRST; the TITLE-match 900 must win.
  let dttl: *u8 = "titletok aaa1 aaa2 aaa3 aaa4 aaa5 aaa6 aaa7" as *u8
  let dbod: *u8 = "bbb1 bbb2 bbb3 bbb4 bbb5 bbb6 titletok bbb7" as *u8
  ss_add(w, 1, "doc:901" as *u8, dbod, g_strlen(dbod))
  ss_add(w, 1, "doc:900" as *u8, dttl, g_strlen(dttl))
  // multilingual fixture (2026-07-23, T33-T36): Russian (Cyrillic word tokens, case-folded),
  // Japanese (Han bigrams), Korean (Hangul bigrams) -- end-to-end through the FULL dss path
  // (query tokenize -> postings -> BM25 tf-scan re-tokenizing the doc). Unique terms, no fixture overlap.
  let dru: *u8 = "\xD1\x81\xD1\x83\xD0\xB4 \xD0\xBF\xD0\xBE \xD0\xBD\xD0\xB0\xD1\x81\xD0\xBB\xD0\xB5\xD0\xB4\xD1\x81\xD1\x82\xD0\xB2\xD1\x83 \xD0\xB2 \xD0\xBC\xD0\xBE\xD1\x81\xD0\xBA\xD0\xB2\xD0\xB5 \xD1\x80\xD0\xB0\xD1\x81\xD1\x81\xD0\xBC\xD0\xBE\xD1\x82\xD1\x80\xD0\xB5\xD0\xBB \xD0\xB4\xD0\xB5\xD0\xBB\xD0\xBE" as *u8
  let dja: *u8 = "\xE6\x9D\xB1\xE4\xBA\xAC\xE5\xA4\xA7\xE5\xAD\xA6 \xE7\xA0\x94\xE7\xA9\xB6" as *u8
  let dko: *u8 = "\xED\x95\x9C\xEA\xB5\xAD \xEA\xB2\x80\xEC\x83\x89 \xEC\x97\x94\xEC\xA7\x84 \xEA\xB0\x9C\xEB\xB0\x9C" as *u8
  ss_add(w, 1, "doc:950" as *u8, dru, g_strlen(dru))
  ss_add(w, 1, "doc:951" as *u8, dja, g_strlen(dja))
  ss_add(w, 1, "doc:952" as *u8, dko, g_strlen(dko))
  // R2b url-scope fixture (T37-T40, 2026-08-04): four docs share the term "subgadget"; only 960 lives
  // under /r/gadgetlab on reddit. 963 = the SEGMENT-ANCHOR decoy (/r/gadgetlabmemes must NOT satisfy
  // r/gadgetlab). 962 = the off-reddit control for inurl: (its url carries no "comments").
  let dsr1: *u8 = "subgadget review thread alpha" as *u8
  let dsr2: *u8 = "subgadget review thread beta" as *u8
  let dsr3: *u8 = "subgadget review blogpost" as *u8
  let dsr4: *u8 = "subgadget meme collection" as *u8
  ss_add(w, 1, "doc:960" as *u8, dsr1, g_strlen(dsr1))
  let u960: *u8 = "https://old.reddit.com/r/gadgetlab/comments/ab1/thread/" as *u8
  ss_add(w, 1, "url:960" as *u8, u960, g_strlen(u960))
  ss_add(w, 1, "doc:961" as *u8, dsr2, g_strlen(dsr2))
  let u961: *u8 = "https://old.reddit.com/r/cooking/comments/cd2/thread/" as *u8
  ss_add(w, 1, "url:961" as *u8, u961, g_strlen(u961))
  ss_add(w, 1, "doc:962" as *u8, dsr3, g_strlen(dsr3))
  let u962: *u8 = "https://techblog.example.com/subgadget-review" as *u8
  ss_add(w, 1, "url:962" as *u8, u962, g_strlen(u962))
  ss_add(w, 1, "doc:963" as *u8, dsr4, g_strlen(dsr4))
  let u963: *u8 = "https://old.reddit.com/r/gadgetlabmemes/xyz" as *u8
  ss_add(w, 1, "url:963" as *u8, u963, g_strlen(u963))
  ss_commit(prefix, w, 1)
  g_puts("wrote 40 docs (+2 pol, +7 host-crowding, +2 authority +1 pr:, +2 url-dup, +4 url-scope) -> dp-ssgatetest-pub-\n")
197
  let pass: *i64 = sys_mmap(16) as *i64; pass[0] = 0
  let total: *i64 = sys_mmap(16) as *i64; total[0] = 0
  let cids: *i64 = sys_mmap(64 * 8) as *i64
  let scores: *i64 = sys_mmap(64 * 8) as *i64
  let dom: *u8 = "ssgatetest" as *u8
203
  // T1: "probate" -> exactly doc 101 (only estate mentions probate)
  let q1: *u8 = "probate" as *u8
  let n1: i64 = dss_search(dom, q1, g_strlen(q1), cids, scores, 10)
  g_check("T1 probate->101" as *u8, n1, cids, 1, 101, pass, total)
208
  // T2: "divorce custody" -> doc 102 top (matches BOTH terms)
  let q2: *u8 = "divorce custody" as *u8
  let n2: i64 = dss_search(dom, q2, g_strlen(q2), cids, scores, 10)
  g_check("T2 divorce+custody->102" as *u8, n2, cids, 1, 102, pass, total)
213
  // T3: "law contracts" -> doc 103 TOP (matches law+contracts=2) above 102 (law=1). 2 candidates.
  let q3: *u8 = "law contracts" as *u8
  let n3: i64 = dss_search(dom, q3, g_strlen(q3), cids, scores, 10)
  g_check("T3 law+contracts->103 top" as *u8, n3, cids, 2, 103, pass, total)
218
  // T4: "law" -> both 102 (Family Law) and 103 (Business Law); 101 has no 'law'
  let q4: *u8 = "law" as *u8
  let n4: i64 = dss_search(dom, q4, g_strlen(q4), cids, scores, 10)
  g_check("T4 law->2 docs" as *u8, n4, cids, 2, 0 - 1, pass, total)
223
  // T5 NEG: absent term -> 0 results (no false hit)
  let q5: *u8 = "zzznotpresentanywhere" as *u8
  let n5: i64 = dss_search(dom, q5, g_strlen(q5), cids, scores, 10)
  g_check("T5 NEG absent-term->0" as *u8, n5, cids, 0, 0 - 1, pass, total)
228
  // T6 NEG (isolation): a domain with NO shard -> 0, not -2, not a crash
  let q6: *u8 = "probate" as *u8
  let n6: i64 = dss_search("nosuchdomain" as *u8, q6, g_strlen(q6), cids, scores, 10)
  g_check("T6 NEG absent-shard->0" as *u8, n6, cids, 0, 0 - 1, pass, total)
233
  // T7 NEG (owner consent): doc 104 exists in the pub shard but its pol row LACKS the search bit
  // (pol=2 = AI_BLOG only) -> query-time consent filter EXCLUDES it (the retired tsv emitter's
  // emission-time enforcement, now live per hit)
  let q7: *u8 = "adoption" as *u8
  let n7: i64 = dss_search(dom, q7, g_strlen(q7), cids, scores, 10)
  g_check("T7 NEG consent opt-out excluded" as *u8, n7, cids, 0, 0 - 1, pass, total)
240
  // T8 (owner consent): doc 105's pol row HAS the search bit (pol=1) -> included
  let q8: *u8 = "annexation" as *u8
  let n8: i64 = dss_search(dom, q8, g_strlen(q8), cids, scores, 10)
  g_check("T8 consent opt-in included" as *u8, n8, cids, 1, 105, pass, total)
245
  // T9 (IDF ranking): query "idfcommon idfrare" -- 5 candidates; the RARE-term doc (106) must be TOP.
  // idf(idfrare)=log2(N/1) >> idf(idfcommon)=log2(N/4); tf equal (2 each for 106/107). The retired
  // distinct+tf scorer tied them and candidate order picked 107 -- this row is the ranking-rung proof.
  let q9: *u8 = "idfcommon idfrare" as *u8
  let n9: i64 = dss_search(dom, q9, g_strlen(q9), cids, scores, 10)
  g_check("T9 IDF: rare-term doc outranks common-term doc" as *u8, n9, cids, 5, 106, pass, total)
252
  // T10 (pagination): 15 pgseg docs -- offset windows must return 10/5/0 with total=15 and disjoint pages
  let qp: *u8 = "pgseg" as *u8
  let tot: *i64 = sys_mmap(16) as *i64
  let np0: i64 = dss_search_off(dom, qp, g_strlen(qp), cids, scores, 10, 0, tot)
  let p0top: i64 = cids[0]
  let np1: i64 = dss_search_off(dom, qp, g_strlen(qp), cids, scores, 10, 10, tot)
  let p1top: i64 = cids[0]
  let np2: i64 = dss_search_off(dom, qp, g_strlen(qp), cids, scores, 10, 20, tot)
  var t10: i64 = 0
  if np0 == 10 { if np1 == 5 { if np2 == 0 { if tot[0] == 15 { if p0top != p1top { t10 = 1 } } } } }
  g_puts("T10 pagination offsets 10/5/0 total=15 disjoint")
  total[0] = total[0] + 1
  if t10 == 1 { pass[0] = pass[0] + 1; g_puts(" PASS\n") } else { g_puts(" FAIL\n") }
266
  // T11 (+term AND): "law +contracts" keeps ONLY the doc that carries contracts (plain OR returned 2)
  let q11: *u8 = "law +contracts" as *u8
  let n11: i64 = dss_search(dom, q11, g_strlen(q11), cids, scores, 10)
  g_check("T11 +term AND filter" as *u8, n11, cids, 1, 103, pass, total)
271
  // T12 did-you-mean: an unknown term corrects to the closest dictionary term (probete -> probate)
  let fix: *u8 = sys_mmap(256)
  let fl: i64 = dss_correct(dom, "probete" as *u8, 7, fix, 255)
  var t12: i64 = 0
  if fl == 7 {
      let want: *u8 = "probate" as *u8
      var eq12: i64 = 1
      var z12: i64 = 0
      while z12 < 7 { if fix[z12] != want[z12] { eq12 = 0; z12 = 7 } else { z12 = z12 + 1 } }
      t12 = eq12
  }
  g_puts("T12 did-you-mean probete->probate")
  total[0] = total[0] + 1
  if t12 == 1 { pass[0] = pass[0] + 1; g_puts(" PASS\n") } else { g_puts(" FAIL\n") }
286
  // T13 suggest: the dictionary completes "prob" (top completion = probate)
  let sg: *u8 = sys_mmap(8 * 64)
  let n13: i64 = dss_suggest(dom, "prob" as *u8, 4, sg, 8)
  var t13: i64 = 0
  if n13 >= 1 {
      let w13: *u8 = "probate" as *u8
      var eq13: i64 = 1
      var z13: i64 = 0
      while z13 < 8 { if sg[z13] != w13[z13] { eq13 = 0; z13 = 8 } else { z13 = z13 + 1 } }
      t13 = eq13
  }
  g_puts("T13 suggest prob->probate")
  total[0] = total[0] + 1
  if t13 == 1 { pass[0] = pass[0] + 1; g_puts(" PASS\n") } else { g_puts(" FAIL\n") }
301
  // T14 (BM25 length norm): equal tf, wildly different lengths -- the SHORT doc must outrank the LONG
  // one (b=0 tied them and insertion order picked the long 126; b=0.75 must pick 127)
  let q14: *u8 = "lennorm" as *u8
  let n14: i64 = dss_search(dom, q14, g_strlen(q14), cids, scores, 10)
  g_check("T14 length-norm: short doc outranks verbose doc" as *u8, n14, cids, 2, 127, pass, total)
307
  // T15/T16 PHRASE through dss: d1 carries "... living trusts ..." adjacent -- the quoted phrase
  // matches; the REVERSED phrase must not (order-sensitive adjacency via the NXQ1 sidecar)
  let q15: *u8 = "\"living trusts\"" as *u8
  let n15: i64 = dss_search(dom, q15, g_strlen(q15), cids, scores, 10)
  g_check("T15 phrase [living trusts] -> doc 101" as *u8, n15, cids, 1, 101, pass, total)
  let q16: *u8 = "\"trusts living\"" as *u8
  let n16: i64 = dss_search(dom, q16, g_strlen(q16), cids, scores, 10)
  g_check("T16 NEG phrase [trusts living] -> 0" as *u8, n16, cids, 0, 0 - 1, pass, total)
316
  // T17 (-term NOT): "law -contracts" -- OR gave both law docs; the exclusion drops the contracts one
  let q17: *u8 = "law -contracts" as *u8
  let n17: i64 = dss_search(dom, q17, g_strlen(q17), cids, scores, 10)
  g_check("T17 -term exclusion (law -contracts -> 102 only)" as *u8, n17, cids, 1, 102, pass, total)
321
  // T18b (NO ZERO-SCORE NOISE, operator-reported "julia kyoka" bug): doc 128 has "ranking" (matched via
  // stem) but NOT "zznomatch". A 2-term query "ranking zznomatch" must return ONLY positively-scored docs;
  // no doc may appear at score 0. Assert every returned score is > 0.
  let qzs: *u8 = "ranking zznomatchword" as *u8
  let nzs: i64 = dss_search(dom, qzs, g_strlen(qzs), cids, scores, 20)
  var allpos: i64 = 1
  var zi: i64 = 0
  while zi < nzs { if scores[zi] <= 0 { allpos = 0 } zi = zi + 1 }
  g_puts("T18b no zero-score results (n="); g_num(nzs); g_puts(")")
  total[0] = total[0] + 1
  if allpos == 1 { if nzs >= 1 { pass[0] = pass[0] + 1; g_puts(" PASS\n") } else { g_puts(" FAIL(none)\n") } } else { g_puts(" FAIL(zero-score shown)\n") }
333
  // T18 (STEMMING recall): the doc says "indexing"/"crawlers"/"ranking"; queries for the STEM-shared
  // forms "index"/"crawler"/"rank" must recall it via query-time dictionary stem-expansion (no index change)
  let q18a: *u8 = "index" as *u8
  let n18a: i64 = dss_search(dom, q18a, g_strlen(q18a), cids, scores, 10)
  var t18: i64 = 0
  if n18a >= 1 {
      // 128 must be among the results (its only 'index'-family token is 'indexing')
      var f18: i64 = 0
      var z18: i64 = 0
      while z18 < n18a { if cids[z18] == 128 { f18 = 1 } z18 = z18 + 1 }
      // and a query for "crawler" must recall 128 (which only says "crawlers")
      let n18b: i64 = dss_search(dom, "crawler" as *u8, 7, cids, scores, 10)
      var f18b: i64 = 0
      var z18b: i64 = 0
      while z18b < n18b { if cids[z18b] == 128 { f18b = 1 } z18b = z18b + 1 }
      if f18 == 1 { if f18b == 1 { t18 = 1 } }
  }
  g_puts("T18 stemming recall (index->indexing, crawler->crawlers)")
  total[0] = total[0] + 1
  if t18 == 1 { pass[0] = pass[0] + 1; g_puts(" PASS\n") } else { g_puts(" FAIL\n") }
354
  // T19-T21 (site: FIELD FILTER, the faceted rung): "law" matches 102 (law.example.com) + 103 (no url).
  // site:example.com keeps only 102 (dot-suffix host match; url-less 103 dropped as not site-attributable)
  let q19: *u8 = "law site:example.com" as *u8
  let n19: i64 = dss_search(dom, q19, g_strlen(q19), cids, scores, 10)
  g_check("T19 site: filter (law site:example.com -> 102 only)" as *u8, n19, cids, 1, 102, pass, total)
  // cross-host NEG: no "law" doc lives on wikipedia.org
  let q20: *u8 = "law site:wikipedia.org" as *u8
  let n20: i64 = dss_search(dom, q20, g_strlen(q20), cids, scores, 10)
  g_check("T20 NEG site: cross-host (law site:wikipedia.org -> 0)" as *u8, n20, cids, 0, 0 - 1, pass, total)
  // suffix-boundary NEG: kipedia.org must NOT match en.wikipedia.org (the '.' boundary rule)
  let q21: *u8 = "trusts site:kipedia.org" as *u8
  let n21: i64 = dss_search(dom, q21, g_strlen(q21), cids, scores, 10)
  var t21b: i64 = 0
  if n21 == 0 {
      let q21b: *u8 = "trusts site:wikipedia.org" as *u8
      let n21b: i64 = dss_search(dom, q21b, g_strlen(q21b), cids, scores, 10)
      if n21b == 1 { if cids[0] == 101 { t21b = 1 } }
  }
  g_puts("T21 suffix boundary (kipedia.org->0, wikipedia.org->101)")
  total[0] = total[0] + 1
  if t21b == 1 { pass[0] = pass[0] + 1; g_puts(" PASS\n") } else { g_puts(" FAIL\n") }
376
  // T22 (host-diversity CONTROL, webdiv=0): no cap -> the 5 tf=2 crowd-host docs fill the top 5, so the
  // 4th result (cids[3]) is a crowd-host doc (304). Non-web scopes (site/trusted) always use this path.
  let qd: *u8 = "diversitytest" as *u8
  let td: *i64 = sys_mmap(16) as *i64
  let nd0: i64 = dss_search_off_div(dom, qd, g_strlen(qd), cids, scores, 10, 0, td, 0)
  var t22: i64 = 0
  if nd0 == 7 { if cids[3] == 304 { t22 = 1 } }
  g_puts("T22 host-cap CONTROL (webdiv=0: crowd fills top, cids[3]=304)")
  total[0] = total[0] + 1
  if t22 == 1 { pass[0] = pass[0] + 1; g_puts(" PASS\n") } else { g_puts(" FAIL cids[3]="); g_num(cids[3]); g_puts(" n="); g_num(nd0); g_puts("\n") }
387
  // T23 (host-diversity CAP, webdiv=1): crowd host caps at DSS_HOSTCAP=3, so an OTHER-host doc (306) is
  // promoted into slot 3 -- and ALL 7 still return (over-cap crowd docs overflow via phase B, never
  // deleted). Fails on the old code by construction (no cap -> cids[3]=304).
  let nd1: i64 = dss_search_off_div(dom, qd, g_strlen(qd), cids, scores, 10, 0, td, 1)
  var t23: i64 = 0
  if nd1 == 7 { if cids[3] == 306 { t23 = 1 } }
  g_puts("T23 host-cap FIRES (webdiv=1: cids[3]=306, all 7 still returned)")
  total[0] = total[0] + 1
  if t23 == 1 { pass[0] = pass[0] + 1; g_puts(" PASS\n") } else { g_puts(" FAIL cids[3]="); g_num(cids[3]); g_puts(" n="); g_num(nd1); g_puts("\n") }
397
  // T24 (dss_is_web predicate): only the exact "web" shard triggers the cap; site domains + "trusted" +
  // a "web"-prefixed decoy do NOT (so a customer site named web*.com is never wrongly capped).
  var t24: i64 = 0
  if dss_is_web("web" as *u8) == 1 { if dss_is_web("trusted" as *u8) == 0 { if dss_is_web("andelinwest.com" as *u8) == 0 { if dss_is_web("weblog.com" as *u8) == 0 { t24 = 1 } } } }
  g_puts("T24 dss_is_web exact-match (web=1; trusted/site/web*=0)")
  total[0] = total[0] + 1
  if t24 == 1 { pass[0] = pass[0] + 1; g_puts(" PASS\n") } else { g_puts(" FAIL\n") }
405
  // T25 (dss_hosthash): same host -> same fingerprint; different host -> different; no url host -> 0.
  let hscr2: *u8 = sys_mmap(256)
  let hh_a: i64 = dss_hosthash("https://crowd.example.com/x" as *u8, 27, hscr2)
  let hh_b: i64 = dss_hosthash("https://crowd.example.com/y" as *u8, 27, hscr2)
  let hh_c: i64 = dss_hosthash("https://other.example.com/z" as *u8, 27, hscr2)
  let hh_d: i64 = dss_hosthash("relative/path/nohost" as *u8, 20, hscr2)
  var t25: i64 = 0
  if hh_a == hh_b { if hh_a != hh_c { if hh_a != 0 { if hh_d == 0 { t25 = 1 } } } }
  g_puts("T25 dss_hosthash (same-host eq, diff-host neq, no-host=0)")
  total[0] = total[0] + 1
  if t25 == 1 { pass[0] = pass[0] + 1; g_puts(" PASS\n") } else { g_puts(" FAIL\n") }
417
  // T28 (P1 authority CONTROL, webdiv=0): equal-BM25 tie -> candidate order -> doc 600 first (no pr fusion)
  let qa: *u8 = "authorityterm" as *u8
  let ta: *i64 = sys_mmap(16) as *i64
  let na0: i64 = dss_search_off_div(dom, qa, g_strlen(qa), cids, scores, 10, 0, ta, 0)
  g_check("T28 authority CONTROL (webdiv=0: tie -> 600)" as *u8, na0, cids, 2, 600, pass, total)
  // T29 (P1 authority FUSION, webdiv=1): doc 601's PageRank prior lifts it above the equal-BM25 doc 600
  let na1: i64 = dss_search_off_div(dom, qa, g_strlen(qa), cids, scores, 10, 0, ta, 1)
  g_check("T29 authority FUSION (webdiv=1: pr: prior -> 601 first)" as *u8, na1, cids, 2, 601, pass, total)
426
  // T30 URL-DEDUP CONTROL (webdiv=0): both snapshots of one url serve; site/trusted stay byte-identical
  let qd: *u8 = "zqduppage" as *u8
  let nd0: i64 = dss_search_off_div(dom, qd, g_strlen(qd), cids, scores, 10, 0, ta, 0)
  g_check("T30 url-dedup CONTROL (webdiv=0: both snapshots serve)" as *u8, nd0, cids, 2, 700, pass, total)
  // T31 URL-DEDUP (webdiv=1): ONE result per url (tie -> first candidate) and totals = DISTINCT pages
  let nd1: i64 = dss_search_off_div(dom, qd, g_strlen(qd), cids, scores, 10, 0, ta, 1)
  var t31: i64 = 0
  if nd1 == 1 { if cids[0] == 700 { if ta[0] == 1 { t31 = 1 } } }
  g_puts("T31 url-dedup (webdiv=1: 1 result, cid 700, total=1)")
  total[0] = total[0] + 1
  if t31 == 1 { pass[0] = pass[0] + 1; g_puts(" PASS\n") } else { g_puts(" FAIL\n") }
438
  // T-prox (R1b PROXIMITY): 800 (clustered) and 801 (scattered) carry IDENTICAL BM25 (same len+tf);
  // the proximity boost must rank the CLUSTERED doc 800 first even though 801 was indexed first. This
  // row FAILS by construction without the boost (tie -> indexed-first 801) and PASSES with it.
  let qpx: *u8 = "proxterma proxtermb" as *u8
  let npx: i64 = dss_search(dom, qpx, g_strlen(qpx), cids, scores, 10)
  g_check("T-prox clustered 800 outranks scattered 801" as *u8, npx, cids, 2, 800, pass, total)
445
  // T-title (R1c): 900 (query term in the TITLE) vs 901 (in the BODY), IDENTICAL BM25; the title-field
  // boost must rank the TITLE-match doc 900 first even though 901 was indexed first.
  let qtt: *u8 = "titletok" as *u8
  let ntt: i64 = dss_search(dom, qtt, g_strlen(qtt), cids, scores, 10)
  g_check("T-title: title-field match 900 outranks body match 901" as *u8, ntt, cids, 2, 900, pass, total)
451
  // T33 RUSSIAN end-to-end: Cyrillic query term -> doc 950 (word tokens through the whole dss path)
  let q33: *u8 = "\xD0\xBD\xD0\xB0\xD1\x81\xD0\xBB\xD0\xB5\xD0\xB4\xD1\x81\xD1\x82\xD0\xB2\xD1\x83" as *u8
  let n33: i64 = dss_search(dom, q33, g_strlen(q33), cids, scores, 10)
  g_check("T33 russian (nasledstvu)->950" as *u8, n33, cids, 1, 950, pass, total)
  // T34 JAPANESE end-to-end: 4-char Han query -> overlapping bigrams -> doc 951
  let q34: *u8 = "\xE6\x9D\xB1\xE4\xBA\xAC\xE5\xA4\xA7\xE5\xAD\xA6" as *u8
  let n34: i64 = dss_search(dom, q34, g_strlen(q34), cids, scores, 10)
  g_check("T34 japanese (toukyoudaigaku)->951" as *u8, n34, cids, 1, 951, pass, total)
  // T35 KOREAN end-to-end: Hangul bigram query -> doc 952
  let q35: *u8 = "\xEA\xB2\x80\xEC\x83\x89" as *u8
  let n35: i64 = dss_search(dom, q35, g_strlen(q35), cids, scores, 10)
  g_check("T35 korean (geomsaek)->952" as *u8, n35, cids, 1, 952, pass, total)
  // T36 CYRILLIC CASE-FOLD: uppercase query MOSKVE matches the lowercase doc
  let q36: *u8 = "\xD0\x9C\xD0\x9E\xD0\xA1\xD0\x9A\xD0\x92\xD0\x95" as *u8
  let n36: i64 = dss_search(dom, q36, g_strlen(q36), cids, scores, 10)
  g_check("T36 cyrillic case-fold (MOSKVE)->950" as *u8, n36, cids, 1, 950, pass, total)
468
  // T37 SUBREDDIT idiom (R2b, 2026-08-04): "r/GadgetLab subgadget" implies site:reddit.com + a case-
  // folded, segment-anchored /r/gadgetlab path -> ONLY 960. 961 = other sub, 962 = off-reddit, 963 =
  // /r/gadgetlabmemes (the anchor decoy). The pre-rewrite engine returned all four ("r" died at the
  // tokenizer's 2-char floor) -- this tooth fails without the fix BY CONSTRUCTION.
  let q37: *u8 = "r/GadgetLab subgadget" as *u8
  let n37: i64 = dss_search(dom, q37, g_strlen(q37), cids, scores, 10)
  g_check("T37 r/<sub> scopes to the subreddit (960 only)" as *u8, n37, cids, 1, 960, pass, total)
  // T38 inurl: general URL filter, ANY site: docs whose url carries "comments" = 960+961 exactly
  let q38: *u8 = "inurl:comments subgadget" as *u8
  let n38: i64 = dss_search(dom, q38, g_strlen(q38), cids, scores, 10)
  g_check("T38 inurl:comments -> the two thread docs" as *u8, n38, cids, 2, 0 - 1, pass, total)
  // T39 CONTROL: the bare term returns ALL FOUR docs -- queries without a url-scope clause are
  // byte-identical to the pre-R2b engine (the ruler-safety claim, positively controlled)
  let q39: *u8 = "subgadget" as *u8
  let n39: i64 = dss_search(dom, q39, g_strlen(q39), cids, scores, 10)
  g_check("T39 CONTROL bare term -> all 4 docs" as *u8, n39, cids, 4, 0 - 1, pass, total)
  // T40 NEG: an unknown subreddit -> honest 0, never a silent fallback to unscoped results
  let q40: *u8 = "r/nosuchsub subgadget" as *u8
  let n40: i64 = dss_search(dom, q40, g_strlen(q40), cids, scores, 10)
  g_check("T40 NEG r/nosuchsub -> 0" as *u8, n40, cids, 0, 0 - 1, pass, total)
489
  // T41 (R2d stuffing penalty): KATs on the factored decision, pinned to the MEASURED live docs
  // (nx_doc_lexstat 2026-08-04). bign=150000, common dcnt=40000 (blood/pressure class),
  // rare dcnt=120 (entity name). Spam blogspot tf34/825 -> 512; real paper tf14/1226 -> 1024;
  // a RARE name at profile density tf40/800 -> 1024 (the julia exemption); tiny doc + small
  // corpus -> inert 1024 (why every fixture tooth above stays byte-identical).
  var t41: i64 = 1
  if dss_stuff_factor(34, 825, 40000, 150000) != 512 { t41 = 0 }
  if dss_stuff_factor(14, 1226, 40000, 150000) != 1024 { t41 = 0 }
  if dss_stuff_factor(40, 800, 120, 150000) != 1024 { t41 = 0 }
  if dss_stuff_factor(30, 40, 40000, 150000) != 1024 { t41 = 0 }
  if dss_stuff_factor(34, 825, 40000, 500) != 1024 { t41 = 0 }
  total[0] = total[0] + 1
  g_puts("T41 stuffing factor: spam 512 / real+rare+tiny+small-corpus 1024")
  if t41 == 1 { pass[0] = pass[0] + 1; g_puts(" PASS\n") } else { g_puts(" FAIL\n") }
504
  // T42 (R2d brand-label exemption): a dense term that IS a host label of the doc's url is the
  // term's owner, not a stuffer -- and a substring without a boundary earns NOTHING (the fix for
  // the 368->297 navigational regression; the measured spam domain must stay penalized).
  let t42scr: *u8 = sys_mmap(256)
  var t42: i64 = 1
  let u42a: *u8 = "https://github.com/features" as *u8
  let u42b: *u8 = "https://highbloodpressure67.blogspot.com/" as *u8
  let u42c: *u8 = "https://www.rust-lang.org/learn" as *u8
  if dss_host_label_match(u42a, g_strlen(u42a), "github" as *u8, t42scr) != 1 { t42 = 0 }
  if dss_host_label_match(u42b, g_strlen(u42b), "blood" as *u8, t42scr) != 0 { t42 = 0 }
  if dss_host_label_match(u42b, g_strlen(u42b), "pressure" as *u8, t42scr) != 0 { t42 = 0 }
  if dss_host_label_match(u42c, g_strlen(u42c), "rust" as *u8, t42scr) != 1 { t42 = 0 }
  if dss_host_label_match(u42a, g_strlen(u42a), "hub" as *u8, t42scr) != 0 { t42 = 0 }
  total[0] = total[0] + 1
  g_puts("T42 brand-label: github/rust exempt, blood/pressure/hub substrings not")
  if t42 == 1 { pass[0] = pass[0] + 1; g_puts(" PASS\n") } else { g_puts(" FAIL\n") }
521
  g_puts("=== dss gate: "); g_num(pass[0]); g_puts("/"); g_num(total[0]); g_puts(" ")
523
  // EVIDENCE (2026-07-30): this gate is the SEARCH lane's only NAS-runnable witness -- 36 teeth over
  // BM25 ranking, proximity, title-field boost and Cyrillic/CJK case-folding -- and it wrote its
  // verdict to STDOUT ONLY, so nx_gatereg_derive could not see the lane and the maturity rollup could
  // not grade it. Same defect that hid lab-science and the browser renderer.
  // ★It is SAFE to publish: every tooth runs against SYNTHETIC fixture docs (800/801/900/901/950-952),
  // never sampled production content, so no dp-*-secret / dp-*-prv material can reach this log.
  // ★Plain verdict=GREEN|RED -- gv_is_pass accepts GREEN|PASS|VALID only, and this ecosystem has
  // already invented four different pass-words. Do not add a fifth.
  let lfd: i64 = sys_openat_append("knowledge/status/search_seg_gate.log\x00" as *u8, 420)
  if lfd >= 0 {
      let lb: *u8 = sys_mmap(256)
      var lo: i64 = 0
      lo = dsg_cat(lb, lo, "DSS-GATE rows=" as *u8)
      lo = dsg_catn(lb, lo, total[0])
      lo = dsg_cat(lb, lo, " passed=" as *u8)
      lo = dsg_catn(lb, lo, pass[0])
      if pass[0] == total[0] { lo = dsg_cat(lb, lo, " verdict=GREEN\n" as *u8) } else { lo = dsg_cat(lb, lo, " verdict=RED\n" as *u8) }
      sys_write(lfd, lb, lo)
      sys_close(lfd)
  }
544
  if pass[0] == total[0] { g_puts("GREEN\n"); return 0 }
  g_puts("RED\n"); return 1
547}
nx_docportal_search_seg_gate.nx source

`nx_docportal_search_seg_gate.nx` source
