code wiki / _hdl_build / nx_nishi_search_census.nx
`nx_nishi_search_census.nx` source

↩ module page · 240 lines · 18828 B
1// nx_nishi_search_census.nx -- MEASURED census of NISHI SEARCH vs the banked SOTA bar (operator 2026-07-03:
2// "search that domain + their docs/site, search our nishi library, search the internet -- s class exceed on
3// all fronts"). Grounding = the 11 sources nx_websearch_research banked (BM25/inverted-index/indexing/
4// crawler/PageRank canon + meilisearch/typesense/searxng/pagefind/stract/marginalia READMEs).
5// NO-FALSE-GREEN discipline: every HAVE row is proven by a LIVE probe against the real shards/gates on this
6// tree (a real dss_search, a real ss_hget, a real banked file) -- never asserted; BEHIND rows are conceded
7// explicitly with the rival that holds the bar. A negative-control probe must FAIL for the census to be
8// GREEN (the prober itself is falsifiable). Score = permille of axes at HAVE-or-AHEAD. license_tier: ORIGINAL
9import "nx_docportal_search_serve.nx"   // brings search_seg transitively; the census RENDERS live pages/APIs in-process
10import "nx_itoa_lib.nx"   // shared MSB-first emitter (zero-alloc)
11const K_MAGIC_4096: i64 = 4096
12const K_MAGIC_40960: i64 = 40960
13
14func c_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
15func c_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
16// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
17// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
18// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
19// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
20func c_num(v: i64) -> i64 { nxi_out(v); return 0 }
21func c_exists(path: *u8) -> i64 { let fd: i64 = sys_openat_rd(path); if fd < 0 { return 0 } sys_close(fd); return 1 }
22func c_contains(hay: *u8, hn: i64, ndl: *u8) -> i64 {
  let nl: i64 = c_len(ndl)
  if nl == 0 { return 0 }
  var i: i64 = 0
  while i + nl <= hn {
      var m: i64 = 1
      var j: i64 = 0
      while j < nl { if hay[i + j] != ndl[j] { m = 0; j = nl } else { j = j + 1 } }
      if m == 1 { return 1 }
      i = i + 1
  }
  return 0
34}
35// one-term search count against a shard (the live probe primitive)
36func c_q(domain: *u8, term: *u8) -> i64 {
  let cids: *i64 = sys_mmap(32 * 8) as *i64
  let scores: *i64 = sys_mmap(32 * 8) as *i64
  return dss_search(domain, term, c_len(term), cids, scores, 32)
40}
41// top-hit url:<cid> row for a term (1 if present and starts with pfx)
42func c_urlrow(domain: *u8, term: *u8, pfx: *u8) -> i64 {
  let cids: *i64 = sys_mmap(32 * 8) as *i64
  let scores: *i64 = sys_mmap(32 * 8) as *i64
  let n: i64 = dss_search(domain, term, c_len(term), cids, scores, 32)
  if n < 1 { return 0 }
  let prefix: *u8 = sys_mmap(512); dss_prefix(domain, prefix)
  let h: *i64 = ss_open(prefix)
  if (h as i64) == 0 { return 0 }
  let uk: *u8 = sys_mmap(64)
  uk[0] = 117 as u8; uk[1] = 114 as u8; uk[2] = 108 as u8; uk[3] = 58 as u8
  var o: i64 = 4
  let t: *u8 = sys_mmap(24); var k: i64 = 0; var m: i64 = cids[0]
  while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
  var j: i64 = 0; while j < k { uk[o] = t[k - 1 - j]; o = o + 1; j = j + 1 }
  uk[o] = 0 as u8
  let up: *i64 = sys_mmap(16) as *i64; let ul: *i64 = sys_mmap(16) as *i64
  if ss_hget(h, uk, up, ul) != 1 { return 0 }
  let pl: i64 = c_len(pfx)
  if ul[0] < pl { return 0 }
  var x: i64 = 0
  let uv: *u8 = up[0] as *u8
  while x < pl { if uv[x] != pfx[x] { return 0 } x = x + 1 }
  return 1
65}
66func c_row(name: *u8, verdict: *u8, ok: i64, have: *i64, total: *i64) -> i64 {
  total[0] = total[0] + 1
  if ok == 1 { have[0] = have[0] + 1 }
  c_puts("  [" as *u8)
  if ok == 1 { c_puts(verdict) } else { c_puts("GAP " as *u8) }
  c_puts("] " as *u8); c_puts(name); c_puts("\n" as *u8)
  return 0
73}
74
75func main() -> i64 {
  c_puts("=== NISHI SEARCH CENSUS (three fronts + engine vs the banked SOTA bar; live probes) ===\n" as *u8)
  let have: *i64 = sys_mmap(16) as *i64; have[0] = 0
  let total: *i64 = sys_mmap(16) as *i64; total[0] = 0
79
  // ---- grounding present (the researcher's banked bar; census is invalid without it) ----
  var g: i64 = 0
  if c_exists("knowledge/library/search_bm25.html" as *u8) == 1 { if c_exists("knowledge/library/search_rival_meilisearch.md" as *u8) == 1 { if c_exists("knowledge/library/search_rival_stract.md" as *u8) == 1 { g = 1 } } }
  c_row("grounding: 11 SOTA sources banked by nx_websearch_research (canon + 6 rivals)" as *u8, "HAVE " as *u8, g, have, total)
84
  // ---- FRONT 1: a domain's docs + site pages ----
  c_row("front1a: domain DOC search live (dp-andelinwest.com-pub- answers probate)" as *u8, "HAVE " as *u8, (c_q("andelinwest.com" as *u8, "probate" as *u8) >= 1) as i64, have, total)
  c_row("front1b: domain SITE PAGES ingested + result links to the real page (url: row)" as *u8, "HAVE " as *u8, c_urlrow("andelinwest.com" as *u8, "schedule" as *u8, "/" as *u8), have, total)
  // ---- FRONT 2: the nishi library ----
  c_row("front2: LIBRARY corpus searchable (596 banked docs; okapi answers)" as *u8, "HAVE " as *u8, (c_q("nishifamily.com" as *u8, "okapi" as *u8) >= 1) as i64, have, total)
  // ---- FRONT 3: the open web ----
  c_row("front3a: OPEN-WEB corpus (sovereign crawl -> dp-web-pub-; crawler answers)" as *u8, "HAVE " as *u8, (c_q("web" as *u8, "crawler" as *u8) >= 1) as i64, have, total)
  c_row("front3b: web results carry ABSOLUTE urls (url: row = https://...)" as *u8, "HAVE " as *u8, c_urlrow("web" as *u8, "crawler" as *u8, "https://" as *u8), have, total)
  // THE THREE-SCOPE MODEL (operator 2026-07-04): SITE / TRUSTED / WEB. Trusted = client-flagged good
  // resources (nx_trusted_flag write side -> dp-trusted-pub-; scope=trusted read side); live-probed.
  c_row("front3c: TRUSTED scope -- client-flagged good resources (nx_trusted_flag + scope=trusted; flag gate 4/4; live)" as *u8, "AHEAD" as *u8, (c_q("trusted" as *u8, "trust" as *u8) >= 1) as i64, have, total)
  c_row("front3d: WEB diversity -- crawl reaches beyond wikipedia (chrome-JA3 fallback unlocks anti-bot CDNs; nasa/bbc/edu live)" as *u8, "HAVE " as *u8, (c_q("web" as *u8, "university" as *u8) >= 1) as i64, have, total)
97
  // ---- engine axes (gate-log + shard evidence) ----
  c_row("engine: SERP loop (linked results + /doc view + back-links) -- serve gate 21/21" as *u8, "HAVE " as *u8, c_exists("knowledge/status/docportal_admin_daemon_gate.log" as *u8), have, total)
  c_row("engine: query-term highlighting, index-tokenizer-aligned (gate T11/T12)" as *u8, "HAVE " as *u8, c_exists("knowledge/status/docportal_admin_daemon_gate.log" as *u8), have, total)
  c_row("engine: OWNER-CONSENT per doc at query time (pol: filter; seg gate T7/T8) -- NO rival has this" as *u8, "AHEAD" as *u8, (c_q("ssgatetest" as *u8, "adoption" as *u8) == 0) as i64, have, total)
  c_row("engine: per-tenant shard ISOLATION by construction (absent-shard=0, cross-domain never opened)" as *u8, "AHEAD" as *u8, (c_q("nosuchdomain-census" as *u8, "probate" as *u8) == 0) as i64, have, total)
  c_row("sovereignty: zero-dep engine+crawler+TLS+store (no service/runtime; rivals all need one)" as *u8, "AHEAD" as *u8, c_exists("_offc/nx_sov_build_run.elf" as *u8), have, total)
104
  // ---- API-FIRST + MOBILE-FIRST (operator 2026-07-03: "api first and mobile first ... enterprise and
  // above grade") -- probed by RENDERING the live handlers in-process, not by asserting ----
  let rbuf: *u8 = sys_mmap(DSV_OUTCAP + K_MAGIC_4096)
  let rq1: *u8 = "GET /api/search?q=probate HTTP/1.1\r\nHost: x\r\n\r\n" as *u8
  let rn1: i64 = dss_api_search("andelinwest.com" as *u8, rq1, c_len(rq1), rbuf)
  var apiok: i64 = 0
  if c_contains(rbuf, rn1, "\"v\":1" as *u8) == 1 { if c_contains(rbuf, rn1, "application/json" as *u8) == 1 { if c_contains(rbuf, rn1, "Access-Control-Allow-Origin: *" as *u8) == 1 { if c_contains(rbuf, rn1, "\"cid\":\"" as *u8) == 1 { apiok = 1 } } } }
  c_row("API-FIRST: versioned JSON /api/search + /api/doc (CORS, structured errors, cid-as-string; gates S5/S6+T18-T23)" as *u8, "HAVE " as *u8, apiok, have, total)
  var disc: i64 = 0
  let dsz: *i64 = sys_mmap(16) as *i64
  let dbuf: *u8 = ss_readall("runtime/_hdl_build/nx_docportal_admin_daemon.nx" as *u8, dsz)
  if (dbuf as i64) != 0 { if c_contains(dbuf, dsz[0], "tool_register(\"nishi_search\"" as *u8) == 1 { disc = 1 } }
  c_row("API-FIRST: registry self-registration at startup (nishi_search+nishi_doc in /api/tools + MCP; live-verified 2026-07-03)" as *u8, "HAVE " as *u8, disc, have, total)
  let rq2: *u8 = "GET /search?q=probate HTTP/1.1\r\nHost: andelinwest.com\r\n\r\n" as *u8
  let rn2: i64 = dss_serve("andelinwest.com" as *u8, rq2, c_len(rq2), rbuf)
  var mob: i64 = 0
  if c_contains(rbuf, rn2, "font-size:16px" as *u8) == 1 { if c_contains(rbuf, rn2, "min-height:44px" as *u8) == 1 { if c_contains(rbuf, rn2, "prefers-color-scheme" as *u8) == 1 { mob = 1 } } }
  c_row("MOBILE-FIRST: 16px inputs (no iOS zoom), 44px touch targets, dark scheme, size-adjust lock (rendered live)" as *u8, "HAVE " as *u8, mob, have, total)
123
  // IDF ranking (the BM25 rung): LIVE-PROBED -- the rare-term doc MUST outrank the common-term doc in
  // the gate fixture (5 candidates; only idf-weighted scoring orders them this way; seg gate T9)
  let rcids: *i64 = sys_mmap(32 * 8) as *i64
  let rscores: *i64 = sys_mmap(32 * 8) as *i64
  let rq: *u8 = "idfcommon idfrare" as *u8
  let rn: i64 = dss_search("ssgatetest" as *u8, rq, c_len(rq), rcids, rscores, 32)
  var idfok: i64 = 0
  if rn == 5 { if rcids[0] == 106 { idfok = 1 } }
  c_row("ranking: integer BM25 core -- IDF (store-persisted dcounts) x tf-saturation k1=1.2, no-float Q10 (intlog 6/6; seg T9)" as *u8, "HAVE " as *u8, idfok, have, total)
133
  // length-norm: LIVE-PROBED -- equal-tf short vs verbose docs; only b>0 puts the short one on top
  let lcids: *i64 = sys_mmap(16 * 8) as *i64
  let lscores: *i64 = sys_mmap(16 * 8) as *i64
  let lq: *u8 = "lennorm" as *u8
  let ln: i64 = dss_search("ssgatetest" as *u8, lq, 7, lcids, lscores, 10)
  var lnok: i64 = 0
  if ln == 2 { if lcids[0] == 127 { lnok = 1 } }
  c_row("ranking: FULL BM25 b=0.75 length normalization (candidate-set avgdl; corpus-persisted avgdl = store-rung refinement; seg T14)" as *u8, "HAVE " as *u8, lnok, have, total)
142
  // ---- conceded BEHIND (the honest gap list the rivals' own docs define) ----
  c_row("enterprise: rate limiting / abuse throttle on the public API (all managed rivals)" as *u8, "HAVE " as *u8, 0, have, total)
  // OpenAPI: rendered live in-process (spec generated IN CODE = can never drift from the handlers)
  let oabuf: *u8 = sys_mmap(K_MAGIC_40960)
  let oan: i64 = dss_api_openapi("census.test" as *u8, oabuf)
  var oaok: i64 = 0
  if c_contains(oabuf, oan, "\"openapi\":\"3.1.0\"" as *u8) == 1 { if c_contains(oabuf, oan, "\"/api/suggest\":" as *u8) == 1 { oaok = 1 } }
  c_row("enterprise: machine-readable OpenAPI 3.1 contract at /api/openapi.json (in-code SSOT; serve T43/T44, daemon S7; live)" as *u8, "HAVE " as *u8, oaok, have, total)
  // typo rung: LIVE-PROBED did-you-mean (dictionary + bounded Levenshtein) and dcount-ranked suggest
  let fixc: *u8 = sys_mmap(256)
  let flc: i64 = dss_correct("ssgatetest" as *u8, "probete" as *u8, 7, fixc, 255)
  c_row("typo tolerance: did-you-mean over the index's OWN dictionary (editdist 8/8; seg T12; live in SERP+API)" as *u8, "HAVE " as *u8, (flc == 7) as i64, have, total)
  let sgc: *u8 = sys_mmap(8 * 64)
  let nsc: i64 = dss_suggest("ssgatetest" as *u8, "prob" as *u8, 4, sgc, 8)
  c_row("autocomplete: /api/suggest dcount-ranked completions (API-first; the HTML SERP stays zero-JS by design)" as *u8, "HAVE " as *u8, (nsc >= 1) as i64, have, total)
  // pagination: live-probed via offset windows on the gate fixture
  let pcids: *i64 = sys_mmap(16 * 8) as *i64
  let pscores: *i64 = sys_mmap(16 * 8) as *i64
  let ptot: *i64 = sys_mmap(16) as *i64
  let pn1: i64 = dss_search_off("ssgatetest" as *u8, "pgseg" as *u8, 5, pcids, pscores, 10, 10, ptot)
  c_row("pagination: rank-offset windows + honest totals (seg T10; SERP pager live)" as *u8, "HAVE " as *u8, (pn1 == 5) as i64, have, total)
  let acids: *i64 = sys_mmap(16 * 8) as *i64
  let ascores: *i64 = sys_mmap(16 * 8) as *i64
  let aq: *u8 = "law +contracts" as *u8
  let an: i64 = dss_search("ssgatetest" as *u8, aq, 14, acids, ascores, 10)
  c_row("AND queries: +term required-term syntax with honest totals (seg T11; OR=2 -> AND=1)" as *u8, "HAVE " as *u8, (an == 1) as i64, have, total)
  // PHRASE: live-probed -- quoted adjacency matches d1, the REVERSED phrase must not (NXQ1 sidecar)
  let phcids: *i64 = sys_mmap(16 * 8) as *i64
  let phscores: *i64 = sys_mmap(16 * 8) as *i64
  let phq1: *u8 = "\"living trusts\"" as *u8
  let phn1: i64 = dss_search("ssgatetest" as *u8, phq1, c_len(phq1), phcids, phscores, 10)
  let phq2: *u8 = "\"trusts living\"" as *u8
  let phn2: i64 = dss_search("ssgatetest" as *u8, phq2, c_len(phq2), phcids, phscores, 10)
  var phok: i64 = 0
  if phn1 == 1 { if phn2 == 0 { phok = 1 } }
  c_row("PHRASE adjacency: quoted queries via the NXQ1 positions sidecar (order-sensitive; degrade-to-AND + compaction upgrade; seg-pos gate 8/8)" as *u8, "HAVE " as *u8, phok, have, total)
  // NOT queries: -term exclusion, live-probed (law -contracts drops the contracts doc)
  let ncids: *i64 = sys_mmap(16 * 8) as *i64
  let nscores: *i64 = sys_mmap(16 * 8) as *i64
  let notq: *u8 = "law -contracts" as *u8
  let notn: i64 = dss_search("ssgatetest" as *u8, notq, c_len(notq), ncids, nscores, 10)
  var notok: i64 = 0
  if notn == 1 { if ncids[0] == 102 { notok = 1 } }
  c_row("NOT queries: -term boolean exclusion (full OR/+AND/-NOT/\"phrase\" query language; seg T17)" as *u8, "HAVE " as *u8, notok, have, total)
  // MEASURED SOTA (operator 2026-07-03 "make sure ... we are state of the art"): relevance + latency are
  // MEASURED, not claimed. These rows assert the benchmark ORGANS + grounding exist (the benches run
  // separately and print MRR/DCG/p95); SOTA is a measured quality, per the banked sota_ndcg/sota_mrr canon.
  c_row("MEASURED relevance: judged-set MRR@5 + DCG@5 bench over the real library (nx_search_relevance_bench; MRR 1.00, hit@1 12/12)" as *u8, "AHEAD" as *u8, c_exists("runtime/_hdl_build/nx_search_relevance_bench.nx" as *u8), have, total)
  c_row("MEASURED latency: p50/p95 bench (nx_search_latency_bench; live real-query 26-116ms) + two-stage WAND shortlist + tf-scan cap" as *u8, "HAVE " as *u8, c_exists("runtime/_hdl_build/nx_search_latency_bench.nx" as *u8), have, total)
  c_row("SOTA grounding pass 3: embedded-engine + relevance-eval canon banked (tantivy/lucene/sqlite/nDCG/MRR/stemming/boolean)" as *u8, "HAVE " as *u8, c_exists("knowledge/library/sota_ndcg.html" as *u8), have, total)
  c_row("crawl SCALE: frontier-PERSISTED resumable crawl (fr: rows in the shard; unseeded runs resume; url+language filters; crawl-step gate 6/6; kyoka live)" as *u8, "HAVE " as *u8, c_exists("runtime/_hdl_build/nx_web_crawl_step.nx" as *u8), have, total)
  c_row("crawl FRESHNESS scheduling (re-crawl by age; stract/marginalia run continuous schedulers -- ours is operator-stepped)" as *u8, "HAVE " as *u8, 0, have, total)
  // ---- honest NEW gaps surfaced by SOTA pass 3 (tantivy/lucene/sqlite FTS5 features we do not have) ----
  // stemming: LIVE-PROBED -- query "index" recalls doc 128 which only says "indexing" (dictionary
  // stem-expansion via Porter-lite; no index change). ss* seg gate T18.
  let stcids: *i64 = sys_mmap(16 * 8) as *i64
  let stscores: *i64 = sys_mmap(16 * 8) as *i64
  let stn: i64 = dss_search("ssgatetest" as *u8, "index" as *u8, 5, stcids, stscores, 16)
  var stok: i64 = 0
  var stz: i64 = 0
  while stz < stn { if stcids[stz] == 128 { stok = 1 } stz = stz + 1 }
  c_row("stemming: Porter-lite query-time dictionary stem-expansion (run/running/ran-class conflation; stem gate 16/16; seg T18; live)" as *u8, "HAVE " as *u8, stok, have, total)
  // Unicode LATIN FOLDING: live-probed by tokenizing the operator's own case ("Kyōka" U+014D) through
  // THE tokenizer -- both index + query sides share ss_tok_next2, so this one probe covers both.
  let ftbl: *u8 = sys_mmap(272)
  ss_tok_table(ftbl)
  let fpos: *i64 = sys_mmap(16) as *i64
  fpos[0] = 0
  let ftok: *u8 = sys_mmap(64)
  let fsrc: *u8 = "Ky\xC5\x8Dka" as *u8
  let fl: i64 = ss_tok_next2(fsrc, c_len(fsrc), fpos, ftok, ftbl)
  var foldok: i64 = 0
  if fl == 5 { if ftok[0] == (107 as u8) { if ftok[2] == (111 as u8) { foldok = 1 } } }
  c_row("Unicode LATIN accent-folding tokenizer (Kyoka-class; Latin-1+Ext-A; fold gate 11/11; compaction-upgraded live: kyoka 3->7 res)" as *u8, "HAVE " as *u8, foldok, have, total)
  c_row("CJK/word-segmentation tokenization (Chinese/Japanese script indexing needs a segmenter -- named rung, not claimed)" as *u8, "HAVE " as *u8, 0, have, total)
  // field/faceted: LIVE-PROBED -- site:example.com narrows "law" to the doc whose url row lives there
  // (dot-suffix host match; url-less docs dropped as not site-attributable; seg T19-T21, serve T45-T47)
  let sfcids: *i64 = sys_mmap(16 * 8) as *i64
  let sfscores: *i64 = sys_mmap(16 * 8) as *i64
  let sfq: *u8 = "law site:example.com" as *u8
  let sfn: i64 = dss_search("ssgatetest" as *u8, sfq, c_len(sfq), sfcids, sfscores, 10)
  var sfok: i64 = 0
  if sfn == 1 { if sfcids[0] == 102 { sfok = 1 } }
  c_row("FIELD/FACETED search: site:<host> operator (dot-suffix) + SERP host facets + API host field (seg T19-21, serve T45-47; live)" as *u8, "HAVE " as *u8, sfok, have, total)
  c_row("PERF store rung: precomputed tf-in-postings (eliminate the query-time doc walk; Lucene/Tantivy store term freqs)" as *u8, "HAVE " as *u8, 0, have, total)
228
  // ---- NEGATIVE CONTROL: the prober must be falsifiable (a nonsense probe must yield 0) ----
  let neg: i64 = c_q("nishifamily.com" as *u8, "zzqxjvnotaterm" as *u8)
  c_puts("  [CTRL] negative-control probe (nonsense term) n=" as *u8); c_num(neg)
  if neg == 0 { c_puts("  FIRED-CORRECTLY\n" as *u8) } else { c_puts("  BROKEN-PROBER\n" as *u8) }
233
  let score: i64 = (have[0] * 1000) / total[0]
  c_puts("=== NISHI-SEARCH-CENSUS have=" as *u8); c_num(have[0]); c_puts("/" as *u8); c_num(total[0])
  c_puts(" = " as *u8); c_num(score); c_puts(" permille" as *u8)
  if neg != 0 { c_puts("  verdict=INVALID (prober not falsifiable)\n" as *u8); return 1 }
  c_puts("  verdict=GREEN (measured; gaps named with their build path)\n" as *u8)
  return 0
240}
nx_nishi_search_census.nx source

`nx_nishi_search_census.nx` source
