code wiki / _hdl_build / nx_docportal_search_seg.nx
`nx_docportal_search_seg.nx` source

↩ module page · 1862 lines · 99759 B
1// nx_docportal_search_seg.nx -- SOVEREIGN seg_store-native onsite search.
2// Operator 2026-07-02: "we dont want to use tsv we want to use nishi ecosystem from the hardware rung up each rung."
3// This RETIRES the derived flat-file bridge (knowledge/index/<domain>_src.tsv -> nx_onsite_index -> separate .idx)
4// for onsite search. It queries the domain's PUBLIC seg_store shard's OWN term postings DIRECTLY:
5//   ss_open(shard) -> per query-term ss_term (current-state postings, tombstone-shadowed) -> ss_hget doc text ->
6//   rank by (# distinct query terms matched, then total term frequency).
7// The store IS the index by construction: ss_write_seg builds .terms for EVERY segment (nx_seg_store:690), so no
8// derived artifact and no flat file appear anywhere in the path. Cross-shard isolation is inherited (a public
9// query ss_open's ONLY the -pub- shard; the -prv- shard's files are never opened). license_tier: ORIGINAL
10//
11// Shard/key format MUST match nx_docportal_lib (dp_prefix/dp_key): prefix "knowledge/store/dp-<domain>-pub-",
12// key "doc:<cid>". Kept inlined (not imported) so this stays a leaf on nx_seg_store alone -> gate-able in isolation.
13import "nx_seg_store.nx"
14import "nx_intlog.nx"   // integer Q10 log2 / idf / tf-saturation -- the BM25 ranking rung's math floor
15import "nx_editdist.nx" // bounded Levenshtein -- the typo/did-you-mean rung's math floor
16import "nx_stem.nx"     // Porter-lite stemmer -- query-time dictionary stem-expansion (recall; no index change)
17
18// Digit consts for the zero-alloc MSB-first key builders. MUST sit above their first reader
19// (dss_mkkey) -- nx_cc refuses a const used before declaration because it would silently read 0.
20const DSC_PREFIXBUF: i64 = 512
21const DSC_TOKBUF: i64 = 64
22const DSC_POSBUF: i64 = 16
23const DSC_ASCII_0: i64 = 48
24const DSC_DEC: i64 = 10
25
26const DSS_MAGIC_1125899906842597: i64 = 1125899906842597
27const DSS_MAGIC_1024: i64 = 1024
28
29const DSS_MAXTERMS: i64 = 16
30const DSS_MAXHITS:  i64 = 2048        // per-term candidacy cap. FIVE-CELL MEASUREMENT 2026-07-25 --
                                    // candidacy depth, stage-1 quality and shortlist width are ONE
                                    // joint decision, never three independent knobs (MRR@10):
                                    //   512 /tf-blind    /short-128  = 705
                                    //   2048/tf-blind    /short-128  = 534   depth ALONE hurts
                                    //   2048/impact-aware/short-128  = 648   stage-1 recovers 2/3
                                    //   512 /impact-aware/short-512  = 705   width ALONE is a no-op
                                    //   2048/impact-aware/short-2048 = 721   SHIPPED, new ratchet
                                    // Raising any ONE of the three measured neutral or negative;
                                    // all three together beat the previous best. Change them AS A SET.
40const DSS_MAXCAND:  i64 = 2048        // unique candidate docs cap (same five-cell measurement)
41const DSS_PHRASEHITS: i64 = 512       // phrase candidacy cap -- ss_phrase's internal rank buffers are
                                    // sized for 512+8; the phrase path keeps its own proven bound
43const DSS_POL_SEARCH:  i64 = 1         // owner-consent search bit -- MUST equal nx_docportal_lib DP_USE_PUB_SEARCH
44const DSS_TFSCAN:      i64 = 8192      // per-doc tf-scan cap (perf, 2026-07-24 32768->8192): the TWO per-candidate walks (tf + proximity) dominate WEB latency on large HTML->text docs; tf saturates ~5 so 8KB of leading content keeps ranking (|d| now comes from byte-length, cap-independent), and the rare-term floor + postings-truth keep deep matches findable. 4x less scanning = the p95 win until precomputed-tf-in-postings lands.
45const DSS_SHORT_FLOOR: i64 = 2048      // seq877: min candidates handed to full BM25 (was a bare 128).
                                    // Equal to DSS_MAXCAND, so stage-1 is a pure RANKER and never a
                                    // lossy gate: every candidate gets full BM25. Worth +16 MRR only
                                    // in combination with the deeper pool -- see the five-cell note.
49const DSS_S1_TFK: i64 = 2            // seq871 stage-1 tf saturation constant: weight = idf*tf/(tf+K).
                                     // tf=1 -> 0.33x, tf=2 -> 0.50x, tf=5 -> 0.71x, tf=100 -> 0.98x --
                                     // a keyword-stuffed page gains ~3x over a single mention, NEVER 100x.
                                     // Saturation is the whole point: round 2 proved RAW tf is a spam
                                     // amplifier, so tf informs SELECTION here, never the BM25 score.
54const DSS_RARE_DCOUNT: i64 = 4       // a POSTED term this rare (<=4 docs corpus-wide) gets a tf=1 floor when the scan cap hid it: near-unique terms (names, ids, gate markers) stay findable even at a huge doc's tail, while COMMON terms hidden by the cap stay dropped (deep-boilerplate noise -- the julia-kyoka complaint). Postings are full-doc truth; the floor only restores what they assert.
55const DSS_HOSTCAP:    i64 = 3          // web scope ONLY: max results per host per page (host-crowding cap). Over-cap hits still appear via phase-B overflow -- down-rank, never delete. Site/trusted shards are never capped.
56const DSS_PR_SCALE:   i64 = 1000000000 // == nx_pagerank PR_SCALE; pr: values are ppb ranks in [0, DSS_PR_SCALE]
57const DSS_PR_BOOST:   i64 = 3072       // web authority: score *= (1024 + rank*BOOST/SCALE)/1024 -- a top page (~5e8) ~2.5x
58const DSS_PR_MAXFAC:  i64 = 4096       // cap the authority factor at 4x (no single super-authority page dominates)
59const DSS_TRUST_BOOST: i64 = 2048      // R2c TRUST PRIOR (2026-08-04, debt 1785895889): bounded 2x for a host
                                     // in canonical_seeds.txt -- DATA-DRIVEN from the SAME curated file that
                                     // aims the crawler, never a hardcoded list; dot-suffix matched; gated by
                                     // the SAME R1f salient-idf floor as the PR boost (a trusted host matching
                                     // only function words earns nothing). Ruler-gated ship.
64const DSS_TRUST_MAXH:  i64 = 96        // seed hosts held per query (the file is ~60 rows)
65// CORPUS-DERIVED STOPWORD DISCOUNT (2026-07-23, the toward-Yandex ranking rung): pure BM25 let function
66// words ("how","to","of","the") dominate -- "how to lower blood pressure" returned "How To Guide - NASA"
67// over the 199 hypertension pages that exist. A term appearing in > DSS_STOP_DFPCT% of the corpus is a
68// stopword and its idf (hence its whole ranking contribution, stage-1 AND full BM25) is divided by
69// DSS_STOP_DISCOUNT -- kept as a weak tiebreaker, never zeroed. DERIVED FROM DOCUMENT FREQUENCY, not a
70// hardcoded English list, so it works for every language in the multilingual index. Only active once the
71// corpus is big enough for df to be meaningful (DSS_STOP_MINCORPUS) -- small onsite/gate shards are untouched.
72const DSS_STOP_DFPCT:     i64 = 12     // term in >12% of docs = stopword (Zipf: function words cluster here)
73const DSS_STOP_DISCOUNT:  i64 = 12     // stopword idf /= this (weak tiebreaker, not deleted)
74const DSS_STOP_MINCORPUS: i64 = 1000   // stopword detection only above this doc count (protects tiny corpora)
75
76// public shard prefix: knowledge/store/dp-<domain>-pub-  (public search NEVER touches the -prv- shard)
77func dss_prefix(domain: *u8, out: *u8) -> i64 {
  var o: i64 = 0
  let a: *u8 = "knowledge/store/dp-" as *u8
  var i: i64 = 0
  while a[i] != (0 as u8) { out[o] = a[i]; o = o + 1; i = i + 1 }
  i = 0
  while domain[i] != (0 as u8) { out[o] = domain[i]; o = o + 1; i = i + 1 }
  let b: *u8 = "-pub-" as *u8
  i = 0
  while b[i] != (0 as u8) { out[o] = b[i]; o = o + 1; i = i + 1 }
  out[o] = 0 as u8
  return o
89}
90
91// build the content-addressed key "doc:<cid>" (null-terminated). Returns length.
92func dss_mkkey(cid: i64, out: *u8) -> i64 {
  out[0] = 100 as u8; out[1] = 111 as u8; out[2] = 99 as u8; out[3] = 58 as u8   // "doc:"
  var o: i64 = 4
  if cid == 0 { out[o] = 48 as u8; o = o + 1; out[o] = 0 as u8; return o }
  // MSB-FIRST: no scratch buffer, no allocation (2026-07-31 leak fix, debt 1785516350). The old body
  // built digits least-significant-first, which comes out backwards and needed a sys_mmap(24) scratch
  // to reverse through -- never freed, once per key built, on every search request. Output identical.
  var pw: i64 = 1
  var m: i64 = cid
  while m / pw >= DSC_DEC { pw = pw * DSC_DEC }
  while pw > 0 { out[o] = (DSC_ASCII_0 + ((m / pw) % DSC_DEC)) as u8; o = o + 1; pw = pw / DSC_DEC }
  out[o] = 0 as u8
  return o
105}
106
107// ============ WEB-SHARD CACHED OPEN (serve-scale rung, 2026-07-23) ============
108// ss_open cost grows with total keys x segments (the live-doc map's shadow probes) -- measured ~0.7s
109// on the bulk-CC web shard, paid PER QUERY because every serve path opened fresh. The web shard is
110// the only scale shard, so its handle is cached process-wide, invalidated by the manifest's
111// (size, mtime-sec) signature. Site shards keep per-request opens (small by design; a single-slot
112// cache would thrash between Host domains). The docportal daemon is FORK-PER-REQUEST: the PARENT
113// pre-warms this cache at startup + refreshes it in the accept loop (dss_web_cache_refresh), so every
114// child inherits the built handle COW and the per-request open cost is ZERO. On reopen the old
115// handle's mappings are not unmapped -- a bounded, rare event (only when new segments ship).
116static dsc_handle: *i64      // cached web-shard handle (0 = not opened yet)
117static dsc_sig:    *i64      // [0]=manifest st_size  [1]=manifest st_mtime-sec
118func dsc_web_prefix_is(prefix: *u8) -> i64 {
  let wp: *u8 = "knowledge/store/dp-web-pub-" as *u8
  var i: i64 = 0
  while wp[i] != (0 as u8) { if prefix[i] != wp[i] { return 0 } i = i + 1 }
  if prefix[i] == (0 as u8) { return 1 }
  return 0
124}
125// manifest signature -> sig[0]=st_size sig[1]=st_mtime-sec ((0,0) when absent)
126// Buffer sizes promoted out of the call sites (rule 11), 2026-07-31 seg-store handle-leak fix.
127const DSC_PATHBUF: i64 = 560
128const DSC_STATBUF: i64 = 160
129const DSC_SIGBUF: i64 = 32
130func dsc_manifest_sig(prefix: *u8, sig: *i64) -> i64 {
  let mp: *u8 = sys_mmap(DSC_PATHBUF)
  var o: i64 = 0
  o = ss_cat(mp, o, prefix)
  o = ss_cat(mp, o, "manifest.txt" as *u8)
  mp[o] = 0 as u8
  let stb: *u8 = sys_mmap(DSC_STATBUF)
  sig[0] = 0
  sig[1] = 0
  if sys_fstatat(mp, stb) == 0 {
      let szp: *i64 = ((stb as i64) + 48) as *i64    // st_size @ +48 (x86_64 struct stat)
      let mtp: *i64 = ((stb as i64) + 88) as *i64    // st_mtime sec @ +88
      sig[0] = szp[0]
      sig[1] = mtp[0]
  }
  sys_munmap(mp, DSC_PATHBUF)
  sys_munmap(stb, DSC_STATBUF)
  return 0
148}
149// the ONE open the serve paths use: cached for the web shard, plain ss_open for every other prefix
150func dss_open_maybe_cached(prefix: *u8) -> *i64 {
  if dsc_web_prefix_is(prefix) == 0 { return ss_open(prefix) }
  if (dsc_sig as i64) == 0 { dsc_sig = sys_mmap(32) as *i64 }
  let cur: *i64 = sys_mmap(DSC_SIGBUF) as *i64
  dsc_manifest_sig(prefix, cur)
  if (dsc_handle as i64) != 0 { if cur[0] == dsc_sig[0] { if cur[1] == dsc_sig[1] { sys_munmap(cur as *u8, DSC_SIGBUF); return dsc_handle } } }
  // RELEASE THE SUPERSEDED HANDLE BEFORE REPLACING IT. Measured 2026-07-31 (debt 1785520503): every
  // manifest change -- i.e. every segguard compaction -- re-opened the web shard and ABANDONED the
  // previous handle, so nx_docportal_ad accumulated 343 anonymous rwx maps totalling 398 GB plus 4275
  // file maps totalling 277 GB (VmSize == VmPeak == 698 GB) and drove box Committed_AS to 12.4x the
  // CommitLimit. ss_close has existed since 2026-07-25; this call site simply never adopted it.
  // Safe across the per-request fork: children forked earlier own separate address spaces, and the
  // refresh runs parent-side (dss_web_cache_refresh) outside the fork.
  if (dsc_handle as i64) != 0 { ss_close(dsc_handle) }
  // MMAP-SERVE: the web shard is the only large shard -> file-backed maps (usemmap=1) so serving is
  // disk-bound not RAM-bound, shared across forked request-children, and page-cache-warm across restarts.
  dsc_handle = ss_open2(prefix, 1)
  dsc_sig[0] = cur[0]
  dsc_sig[1] = cur[1]
  sys_munmap(cur as *u8, DSC_SIGBUF)
  return dsc_handle
171}
172// parent-side pre-warm/refresh: call OUTSIDE the per-request fork. Returns 1 = handle ready.
173func dss_web_cache_refresh() -> i64 {
  let prefix: *u8 = sys_mmap(DSC_PREFIXBUF)
  dss_prefix("web" as *u8, prefix)
  let h: *i64 = dss_open_maybe_cached(prefix)
  // safe to free: ss_open2 only ss_cat-COPIES prefix into path buffers, it never retains the pointer.
  sys_munmap(prefix, DSC_PREFIXBUF)
  if (h as i64) == 0 { return 0 }
  return 1
181}
182
183// build the policy key "pol:<cid>" (null-terminated) -- MUST match nx_docportal_lib dp_polkey. Returns length.
184func dss_mkpolkey(cid: i64, out: *u8) -> i64 {
  out[0] = 112 as u8; out[1] = 111 as u8; out[2] = 108 as u8; out[3] = 58 as u8   // "pol:"
  var o: i64 = 4
  if cid == 0 { out[o] = 48 as u8; o = o + 1; out[o] = 0 as u8; return o }
  // MSB-FIRST: no scratch buffer, no allocation (2026-07-31 leak fix, debt 1785516350). The old body
  // built digits least-significant-first, which comes out backwards and needed a sys_mmap(24) scratch
  // to reverse through -- never freed, once per key built, on every search request. Output identical.
  var pw: i64 = 1
  var m: i64 = cid
  while m / pw >= DSC_DEC { pw = pw * DSC_DEC }
  while pw > 0 { out[o] = (DSC_ASCII_0 + ((m / pw) % DSC_DEC)) as u8; o = o + 1; pw = pw / DSC_DEC }
  out[o] = 0 as u8
  return o
197}
198
199// build the source-url key "url:<cid>" (must byte-match the serve layer's dsv_mkurlkey / corpus ci_mkurlkey)
200func dss_mkurlkey(cid: i64, out: *u8) -> i64 {
  out[0] = 117 as u8; out[1] = 114 as u8; out[2] = 108 as u8; out[3] = 58 as u8   // "url:"
  var o: i64 = 4
  if cid == 0 { out[o] = 48 as u8; o = o + 1; out[o] = 0 as u8; return o }
  // MSB-FIRST: no scratch buffer, no allocation (2026-07-31 leak fix, debt 1785516350). The old body
  // built digits least-significant-first, which comes out backwards and needed a sys_mmap(24) scratch
  // to reverse through -- never freed, once per key built, on every search request. Output identical.
  var pw: i64 = 1
  var m: i64 = cid
  while m / pw >= DSC_DEC { pw = pw * DSC_DEC }
  while pw > 0 { out[o] = (DSC_ASCII_0 + ((m / pw) % DSC_DEC)) as u8; o = o + 1; pw = pw / DSC_DEC }
  out[o] = 0 as u8
  return o
213}
214// extract the url's HOST (bytes after "://" up to '/' or ':', lowercased) into out. Returns host length, 0 = none.
215func dss_url_host(u: *u8, ul: i64, out: *u8) -> i64 {
  var hs: i64 = 0 - 1
  var i: i64 = 0
  while i + 2 < ul {
      if u[i] == (58 as u8) { if u[i+1] == (47 as u8) { if u[i+2] == (47 as u8) { hs = i + 3; i = ul } } }
      i = i + 1
  }
  if hs < 0 { out[0] = 0 as u8; return 0 }
  var o: i64 = 0
  var he: i64 = hs
  var go: i64 = 1
  while go == 1 {
      if he >= ul { go = 0 } else {
          if u[he] == (47 as u8) { go = 0 } else { if u[he] == (58 as u8) { go = 0 } else {
              var c: i64 = u[he] as i64
              if c >= 65 { if c <= 90 { c = c + 32 } }
              if o < 250 { out[o] = c as u8; o = o + 1 }
              he = he + 1
          } }
      }
  }
  out[o] = 0 as u8
  return o
238}
239// is this the broad open-web shard? The host-crowding diversity cap applies ONLY to "web": site + trusted
240// shards are single-origin / curated, where capping per host would wrongly hide the owner's own pages.
241func dss_is_web(domain: *u8) -> i64 {
  if domain[0] == (119 as u8) { if domain[1] == (101 as u8) { if domain[2] == (98 as u8) { if domain[3] == (0 as u8) { return 1 } } } }
  return 0
244}
245// cheap host fingerprint for the per-page diversity cap: extract the url's host (reusing dss_url_host) into
246// `scratch`, then a 131-base rolling hash (wrapping i64; collisions across one query's <=512 candidates are
247// negligible). Returns 0 when the doc has no url host (library/relative docs) -> never capped.
248func dss_hosthash(u: *u8, ul: i64, scratch: *u8) -> i64 {
  let hl: i64 = dss_url_host(u, ul, scratch)
  if hl <= 0 { return 0 }
  var hsh: i64 = 0
  var i: i64 = 0
  while i < hl { hsh = hsh * 131 + (scratch[i] as i64); i = i + 1 }
  if hsh == 0 { hsh = 1 }
  return hsh
256}
257// url-cid = the crawler's ci_hash (nx_corpus_ingest) -- the LINK-GRAPH node id (out:/pr: keys). MUST match
258// ci_hash byte-for-byte so serve-time pr:<ci_hash(url)> resolves the authority nx_pagerank_build stored.
259func dss_urlcid(s: *u8, n: i64) -> i64 {
  var h: i64 = DSS_MAGIC_1125899906842597
  var i: i64 = 0
  while i < n { h = (h * 131) + (s[i] as i64); i = i + 1 }
  if h < 0 { h = 0 - h }
  return h & 0x7fffffffffffffff
265}
266// build "pr:<cid>" authority-prior key (MUST match nx_pagerank_build pb_prkey).
267func dss_prkey(cid: i64, out: *u8) -> i64 {
  out[0] = 112 as u8   // p
  out[1] = 114 as u8   // r
  out[2] = 58 as u8    // :
  // MSB-FIRST: zero allocation (2026-07-31, debt 1785516350). Byte-identical incl. the cid==0 case,
  // which the digit walk emits naturally as a single 0 rather than needing a special branch.
  var m: i64 = cid
  var o: i64 = 3
  var pw: i64 = 1
  while m / pw >= DSC_DEC { pw = pw * DSC_DEC }
  while pw > 0 { out[o] = (DSC_ASCII_0 + ((m / pw) % DSC_DEC)) as u8; o = o + 1; pw = pw / DSC_DEC }
  out[o] = 0 as u8
  return o
280}
281// does the url's HOST match `host` (exact or dot-suffix: site:wikipedia.org covers en.wikipedia.org)?
282// Host = url bytes after "://" up to '/' or ':' (lowercased). Returns 1 match / 0 no.
283func dss_url_host_match(u: *u8, ul: i64, host: *u8) -> i64 {
  if ul < 4 { return 0 }
  let hl: i64 = dss_tlen(host)
  if hl == 0 { return 0 }
  // find "://" (absent -> host starts at 0: relative urls have no host -> no match unless ul starts w/ host? treat as no-host, no match)
  var hs: i64 = 0 - 1
  var i: i64 = 0
  while i + 2 < ul {
      if u[i] == (58 as u8) { if u[i+1] == (47 as u8) { if u[i+2] == (47 as u8) { hs = i + 3; i = ul } } }
      i = i + 1
  }
  if hs < 0 { return 0 }
  var he: i64 = hs
  var go: i64 = 1
  while go == 1 {
      if he >= ul { go = 0 } else {
          if u[he] == (47 as u8) { go = 0 } else { if u[he] == (58 as u8) { go = 0 } else { he = he + 1 } }
      }
  }
  let hn: i64 = he - hs
  if hn < hl { return 0 }
  // compare the TAIL of the host with `host` (lowercase both sides)
  var x: i64 = 0
  while x < hl {
      var ca: i64 = u[hs + hn - hl + x] as i64
      if ca >= 65 { if ca <= 90 { ca = ca + 32 } }
      if ca != (host[x] as i64) { return 0 }
      x = x + 1
  }
  if hn == hl { return 1 }
  // longer actual host: the char just before the suffix must be '.' (en.wikipedia.org vs notwikipedia.org)
  if u[hs + hn - hl - 1] == (46 as u8) { return 1 }
  return 0
316}
317// R2b URL-PATH FILTER (2026-08-04, the subreddit/inurl rung): does the url CONTAIN `pat` (caller
318// pre-lowercased) case-insensitively? seganchor=1 additionally requires the byte after the match to be
319// a segment boundary ('/', '?', '#', or end-of-url) so "/r/game" never claims "/r/gamedev". Shares the
320// site: contract: a candidate with no url row is not path-attributable and the caller drops it.
321func dss_url_path_has(u: *u8, ul: i64, pat: *u8, seganchor: i64) -> i64 {
  let pl: i64 = dss_tlen(pat)
  if pl == 0 { return 0 }
  if ul < pl { return 0 }
  var i: i64 = 0
  while i + pl <= ul {
      var m: i64 = 1
      var k: i64 = 0
      while k < pl {
          var ca: i64 = u[i + k] as i64
          if ca >= 65 { if ca <= 90 { ca = ca + 32 } }
          if ca != (pat[k] as i64) { m = 0; k = pl } else { k = k + 1 }
      }
      if m == 1 {
          if seganchor == 0 { return 1 }
          if i + pl >= ul { return 1 }
          let cb: i64 = u[i + pl] as i64
          if cb == 47 { return 1 }
          if cb == 63 { return 1 }
          if cb == 35 { return 1 }
      }
      i = i + 1
  }
  return 0
345}
346// parse the cid out of a "doc:<cid>" key (kp,kl). Returns cid, or -1 if not a doc key.
347func dss_key_cid(kp: *u8, kl: i64) -> i64 {
  if kl < 5 { return 0 - 1 }
  if kp[0] != (100 as u8) { return 0 - 1 }   // 'd'
  if kp[1] != (111 as u8) { return 0 - 1 }   // 'o'
  if kp[2] != (99 as u8)  { return 0 - 1 }   // 'c'
  if kp[3] != (58 as u8)  { return 0 - 1 }   // ':'
  var v: i64 = 0
  var i: i64 = 4
  while i < kl {
      if kp[i] >= (48 as u8) { if kp[i] <= (57 as u8) { v = v * 10 + ((kp[i] - (48 as u8)) as i64) } }
      i = i + 1
  }
  return v
360}
361
362// exact match of a normalized token against a null-terminated term
363func dss_streq(tok: *u8, term: *u8) -> i64 {
  var i: i64 = 0
  while tok[i] != (0 as u8) {
      if tok[i] != term[i] { return 0 }
      i = i + 1
  }
  if term[i] != (0 as u8) { return 0 }
  return 1
371}
372
373// term frequency of `term` in doc[0..dn) using the SAME tokenizer/table the index was built with (tbl prebuilt)
374func dss_tf(doc: *u8, dn: i64, term: *u8, tbl: *u8) -> i64 {
  let tbuf: *u8 = sys_mmap(DSC_TOKBUF)
  let pos: *i64 = sys_mmap(DSC_POSBUF) as *i64
  pos[0] = 0
  var tf: i64 = 0
  var go: i64 = 1
  while go == 1 {
      let l: i64 = ss_tok_next2(doc, dn, pos, tbuf, tbl)
      if l < 0 { go = 0 } else {
          if dss_streq(tbuf, term) == 1 { tf = tf + 1 }
      }
  }
  sys_munmap(tbuf, DSC_TOKBUF)
  sys_munmap(pos as *u8, DSC_POSBUF)
  return tf
389}
390// ONE walk, ALL the statistics: per-term tf into tf_out[0..nterms) AND the doc's total token count |d|
391// (the BM25 length-norm needs it; also kills the old nterms-walks-per-candidate cost). Returns |d|.
392func dss_tf_all(doc: *u8, dn: i64, termptrs: *i64, nterms: i64, tbl: *u8, tf_out: *i64) -> i64 {
  let tbuf: *u8 = sys_mmap(DSC_TOKBUF)
  let pos: *i64 = sys_mmap(DSC_POSBUF) as *i64
  pos[0] = 0
  // first-byte discriminator: this loop runs per TOKEN x per TERM over 32KB x 128 candidates -- a
  // one-byte gate before the full compare kills ~96% of the dss_streq calls (the p95 hot path)
  let fb: *u8 = sys_mmap(DSS_MAXTERMS + 8)
  var t0: i64 = 0
  while t0 < nterms {
      tf_out[t0] = 0
      let tp0: *u8 = termptrs[t0] as *u8
      fb[t0] = tp0[0]
      t0 = t0 + 1
  }
  var dl: i64 = 0
  var go: i64 = 1
  while go == 1 {
      let l: i64 = ss_tok_next2(doc, dn, pos, tbuf, tbl)
      if l < 0 { go = 0 } else {
          dl = dl + 1
          let b0: u8 = tbuf[0]
          var t: i64 = 0
          while t < nterms {
              if fb[t] == b0 {
                  if dss_streq(tbuf, termptrs[t] as *u8) == 1 { tf_out[t] = tf_out[t] + 1 }
              }
              t = t + 1
          }
      }
  }
  sys_munmap(tbuf, DSC_TOKBUF)
  sys_munmap(pos as *u8, DSC_POSBUF)
  sys_munmap(fb, DSS_MAXTERMS + 8)
  return dl
426}
427
428// ---- serve-time TERM-PROXIMITY (R1b): query terms that CLUSTER in a doc = more relevant (Buttcher-
429// Clarke-Cormack 2006). Token-ALIGNED: clones dss_tf_all's ss_tok_next2 loop so it matches EXACTLY the
430// tokens BM25 tf matched (a raw-substring version would mis-fire on case/stem/substring-in-word and
431// REGRESS ranking -- the anti-cheat). Records the token INDEX of each query-term hit -> smallest token
432// window containing all PRESENT terms -> strength. <2 present terms -> 0 (ranking unchanged = safety).
433// Does NOT touch the tf/BM25 path; applied as a bounded PASS-2 multiply, mirroring the authority fusion.
434const DSS_PROX_SCALE: i64 = 1024
435const DSS_PROX_WREF:  i64 = 8       // reference window in TOKENS; span << WREF -> near-full strength
436const DSS_PROX_BOOST: i64 = 1024    // max boost: prox=SCALE -> factor 2048/1024 = 2x (bounded, capped)
437const DSS_PROX_OCAP:  i64 = 512     // max query-term occurrences tracked (bounded serve cost)
438const DSS_TITLE_TOKENS: i64 = 6     // R1c: leading tokens treated as the TITLE / lead field
439const DSS_TITLE_BONUS:  i64 = 384   // R1c: per-distinct-term title hit (BM25F-lite), summed then capped at SCALE
440// R1d COORDINATION (2026-07-24): reward docs that match MORE distinct query terms. A doc covering `cov` of
441// `nterms` terms gets factor DSS_COV_FLOOR + (1024-FLOOR)*(cov-1)/(nterms-1): cov==nterms -> 1024 (byte-identical,
442// so single-term queries + full matches are unchanged), cov=1-of-2 -> 256 (0.25x). Multi-term NON-phrase only.
443// This is Lucene-coord's principle and fulfils this module's stated "rank by # distinct terms matched" intent --
444// it stops a rare single-term hit (idf high, tf high) from out-SUMMING a genuine two-term match (e.g. "julia
445// kyoka" no longer ranks Izumi-Kyoka literature pages, and "diora baird" demotes baird-only law-firm pages).
446const DSS_COV_FLOOR:    i64 = 256   // coord factor for a doc matching only the FIRST of >=2 distinct query terms
447func dss_popcount(m: i64) -> i64 { var c: i64 = 0; var x: i64 = m; while x != 0 { c = c + (x & 1); x = x >> 1 } return c }
448// R1e SEARCH-PAGE DE-RANK (2026-07-24): a URL that is itself a SEARCH / QUERY-ECHO page (/search, ?q=, ?s=,
449// ?query=, ?search=) is a query box echoing the query, not content ABOUT it (e.g. jav.guru/?s=julia+kyoka
450// "You searched for julia kyoka"). Half-weight it so real content outranks it WHEN content exists -- but it is
451// NEVER buried: an aggregator that is the best 2-term match available still surfaces (bounded multiply, not a drop).
452const DSS_SEARCHPAGE_PEN: i64 = 512   // 512/1024 = 0.5x on a detected search/query-echo URL (web scope only)
453// R1f SALIENT-IDF AUTHORITY FLOOR (2026-07-24): a high-PageRank domain earns its authority boost only if it
454// matched >= this fraction (Q10) of the query's CONTENT idf-mass -- not just any peripheral term. Stops
455// science.nasa.gov (matches "pressure" but not the "blood pressure" concept + "how to") riding authority to #1.
456const DSS_AUTH_IDF_MIN: i64 = 512   // 512/1024 = 50% of the query content idf-mass required to earn the PR boost
457func dss_sub_at(u: *u8, n: i64, i: i64, pat: *u8, plen: i64) -> i64 {
  if i + plen > n { return 0 }
  var j: i64 = 0
  while j < plen { if u[i + j] != pat[j] { return 0 } j = j + 1 }
  return 1
462}
463func dss_is_search_url(u: *u8, n: i64) -> i64 {
  var i: i64 = 0
  while i < n {
      if dss_sub_at(u, n, i, "/search" as *u8, 7) == 1 { return 1 }
      if dss_sub_at(u, n, i, "?q=" as *u8, 3) == 1 { return 1 }
      if dss_sub_at(u, n, i, "&q=" as *u8, 3) == 1 { return 1 }
      if dss_sub_at(u, n, i, "?s=" as *u8, 3) == 1 { return 1 }
      if dss_sub_at(u, n, i, "&s=" as *u8, 3) == 1 { return 1 }
      if dss_sub_at(u, n, i, "?query=" as *u8, 7) == 1 { return 1 }
      if dss_sub_at(u, n, i, "?search=" as *u8, 8) == 1 { return 1 }
      i = i + 1
  }
  return 0
476}
477// R1g URL-SLUG ENTITY PRIOR (2026-07-24): a CONTENT query term that IS a whole path segment of the URL
478// (/idols/julia/) -- or a segment prefix ending at a wordbreak (_ - .) as in /wiki/Julia_(novel) -- marks a
479// page ABOUT the entity, not a mere mention. BM25 length-norm buries long authority profile pages under short
480// mention-stubs (measured 2026-07-24: bare "julia" ranked 25 onejav torrent stubs, /idols/julia/ absent from
481// page 0); the slug prior is the entity-understanding counterweight. Applied at most once per candidate, only
482// for terms the candidate MATCHED, never to a search/query-echo URL. Site/gate corpora: the authority block is
483// web-scope-only -> inert there.
484const DSS_SLUG_BOOST:  i64 = 6144   // Q10 6.0x: must clear the mention-wall (measured: ~140 short torrent stubs
                                  // at ~6.1M vs the profile page's ~1.5M BM25 base -- 2x left it below rank 60)
486const DSS_SLUG_MINLEN: i64 = 4      // ignore short segments/terms (/en/, /id/) -- too weak to mark an entity
487// RARITY GATE (measured 2026-07-24): the blanket slug prior REGRESSED the ruler 809->640 by 2x-boosting
488// generic nouns as segments ("survey" -> catalog.ihsn.org, "university" -> mlhmi.org, "pressure" -> nasa).
489// Entity NAMES are rare terms; generic nouns are common. Fire only when the term is in < corpus/RAREK docs
490// (0.2%): julia 0.08% fires; survey/university/pressure/household all miss.
491const DSS_SLUG_RAREK:  i64 = 512
492// R2d KEYWORD-STUFFING PENALTY (2026-08-04, debt 1785895889, MEASURED via nx_doc_lexstat over the
493// live SERP: spam blogspot maxdens=42, listicle 20, REAL content 2-11 incl. a hypertension paper at
494// 11 -- threshold 16 = 45% margin over the strongest legitimate doc). A COMMON term whose scan-tf
495// exceeds DSS_STUFF_DENS/1024 of the doc's tokens reads as stuffing -> ONE bounded 0.5x multiply.
496// RARE terms are EXEMPT (dcnt*DSS_SLUG_RAREK < bign = an entity name; profile/filmography pages
497// legitimately repeat the name -- the July julia work must not regress). Small corpora
498// (bign < DSS_STOP_MINCORPUS) and tiny docs are inert -> every gate fixture stays byte-identical.
499// (Declared BELOW DSS_SLUG_RAREK: module consts must precede their first reader.)
500const DSS_STUFF_DENS: i64 = 16
501const DSS_STUFF_PEN:  i64 = 512
502const DSS_STUFF_MINTOKS: i64 = 64
503func dss_stuff_factor(tf: i64, toks: i64, dcnt: i64, bign: i64) -> i64 {
  if bign < DSS_STOP_MINCORPUS { return DSS_MAGIC_1024 }
  if toks < DSS_STUFF_MINTOKS { return DSS_MAGIC_1024 }
  if dcnt * DSS_SLUG_RAREK < bign { return DSS_MAGIC_1024 }
  if tf * DSS_MAGIC_1024 > toks * DSS_STUFF_DENS { return DSS_STUFF_PEN }
  return DSS_MAGIC_1024
509}
510// R2d BRAND-LABEL EXEMPTION (the 368->297 navigational-regression fix, measured 2026-08-04):
511// a doc whose url HOST carries the query term as a whole '.'/'-'-bounded LABEL is the term's
512// OWNER, not a stuffer -- github.com is legitimately dense in "github", rust-lang.org in "rust".
513// "blood" inside highbloodpressure67.blogspot.com has NO boundary -> the spam stays penalized.
514// (Known honest limitation: a hyphenated keyword domain like blood-pressure-tips.example earns
515// the exemption -- '-' must be a boundary or rust-lang/git-scm class navigation breaks, and the
516// hyphen-domain class is better answered by authority than by this penalty.)
517func dss_host_label_match(u: *u8, ul: i64, term: *u8, scratch: *u8) -> i64 {
  let hl: i64 = dss_url_host(u, ul, scratch)
  if hl <= 0 { return 0 }
  var tl: i64 = 0
  while term[tl] != (0 as u8) { tl = tl + 1 }
  if tl == 0 { return 0 }
  var i: i64 = 0
  while i < hl {
      // label start = position 0 or preceded by '.'/'-'
      var isstart: i64 = 0
      if i == 0 { isstart = 1 } else {
          if scratch[i - 1] == (46 as u8) { isstart = 1 }
          if scratch[i - 1] == (45 as u8) { isstart = 1 }
      }
      if isstart == 1 { if i + tl <= hl {
          var m: i64 = 1
          var k: i64 = 0
          while k < tl { if scratch[i + k] != term[k] { m = 0; k = tl } else { k = k + 1 } }
          if m == 1 {
              // label end = host end or followed by '.'/'-'
              if i + tl == hl { return 1 }
              if scratch[i + tl] == (46 as u8) { return 1 }
              if scratch[i + tl] == (45 as u8) { return 1 }
          }
      } }
      i = i + 1
  }
  return 0
545}
546func dss_lower(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c }
547// does the null-terminated lowercase term equal the URL path segment starting at u[i] (whole segment, or a
548// prefix ending at / ? _ - . )? URL side is case-folded.
549func dss_slug_seg_eq(u: *u8, n: i64, i: i64, term: *u8) -> i64 {
  var j: i64 = 0
  while term[j] != (0 as u8) {
      if i + j >= n { return 0 }
      if dss_lower(u[i + j] as i64) != (term[j] as i64) { return 0 }
      j = j + 1
  }
  if j < DSS_SLUG_MINLEN { return 0 }
  if i + j >= n { return 1 }
  let c: i64 = u[i + j] as i64
  if c == 47 { return 1 }
  if c == 63 { return 1 }
  if c == 95 { return 1 }
  if c == 45 { return 1 }
  if c == 46 { return 1 }
  return 0
565}
566// scan the URL PATH (after the scheme+host) for any '/'-anchored segment equal to term; stop at the query string.
567func dss_slug_match_term(u: *u8, n: i64, term: *u8) -> i64 {
  var s: i64 = 0
  var k: i64 = 0
  while k + 2 < n { if u[k]==(58 as u8) { if u[k+1]==(47 as u8) { if u[k+2]==(47 as u8) { s = k + 3; k = n } } } k = k + 1 }
  var i: i64 = s
  while i < n {
      if u[i] == (63 as u8) { return 0 }
      if u[i] == (47 as u8) { if dss_slug_seg_eq(u, n, i + 1, term) == 1 { return 1 } }
      i = i + 1
  }
  return 0
578}
579func dss_prox_all(doc: *u8, dn: i64, termptrs: *i64, nterms: i64, tbl: *u8) -> i64 {
  if nterms < 1 { return 0 }
  let tbuf: *u8 = sys_mmap(DSC_TOKBUF)
  let pos: *i64 = sys_mmap(DSC_POSBUF) as *i64
  pos[0] = 0
  let fb: *u8 = sys_mmap(DSS_MAXTERMS + 8)
  let seen: *i64 = sys_mmap(DSS_MAXTERMS * 8) as *i64
  let tseen: *i64 = sys_mmap(DSS_MAXTERMS * 8) as *i64   // R1c: query term seen in the TITLE (lead tokens)
  var t0: i64 = 0
  while t0 < nterms {
      let tp0: *u8 = termptrs[t0] as *u8
      fb[t0] = tp0[0]
      seen[t0] = 0
      tseen[t0] = 0
      t0 = t0 + 1
  }
  let opos: *i64 = sys_mmap(DSS_PROX_OCAP * 8) as *i64
  let oidx: *i64 = sys_mmap(DSS_PROX_OCAP * 8) as *i64
  var nocc: i64 = 0
  var present: i64 = 0
  var tokidx: i64 = 0
  var go: i64 = 1
  while go == 1 {
      let l: i64 = ss_tok_next2(doc, dn, pos, tbuf, tbl)
      if l < 0 { go = 0 } else {
          let b0: u8 = tbuf[0]
          var t: i64 = 0
          while t < nterms {
              if fb[t] == b0 {
                  if dss_streq(tbuf, termptrs[t] as *u8) == 1 {
                      if nocc < DSS_PROX_OCAP { opos[nocc] = tokidx; oidx[nocc] = t; nocc = nocc + 1 }
                      if seen[t] == 0 { seen[t] = 1; present = present + 1 }
                      if tokidx < DSS_TITLE_TOKENS { tseen[t] = 1 }   // R1c: a hit in the title/lead field
                  }
              }
              t = t + 1
          }
          tokidx = tokidx + 1
          if nocc >= DSS_PROX_OCAP { go = 0 }
      }
  }
  // R1c TITLE-FIELD BONUS (BM25F-lite): distinct query terms in the leading field boost relevance.
  var thits: i64 = 0
  var tj: i64 = 0
  while tj < nterms { if tseen[tj] == 1 { thits = thits + 1 } tj = tj + 1 }
  var title_bonus: i64 = thits * DSS_TITLE_BONUS
  if title_bonus > DSS_PROX_SCALE { title_bonus = DSS_PROX_SCALE }
  // PROXIMITY (needs >= 2 distinct present terms; single-term docs get the title bonus only)
  var prox: i64 = 0
  if present >= 2 {
      let cnt: *i64 = sys_mmap(DSS_MAXTERMS * 8) as *i64
      var zz: i64 = 0
      while zz < nterms { cnt[zz] = 0; zz = zz + 1 }
      var distinct: i64 = 0
      var lft: i64 = 0
      var minspan: i64 = tokidx + 1
      var r: i64 = 0
      while r < nocc {
          let tr: i64 = oidx[r]
          if cnt[tr] == 0 { distinct = distinct + 1 }
          cnt[tr] = cnt[tr] + 1
          while distinct == present {
              let span: i64 = opos[r] - opos[lft]
              if span < minspan { minspan = span }
              let tl: i64 = oidx[lft]
              cnt[tl] = cnt[tl] - 1
              if cnt[tl] == 0 { distinct = distinct - 1 }
              lft = lft + 1
          }
          r = r + 1
      }
      let base: i64 = (present * DSS_PROX_SCALE) / nterms
      prox = (base * DSS_PROX_WREF) / (DSS_PROX_WREF + minspan)
      // freed at the END OF ITS OWN BLOCK -- cnt is block-scoped to `if present >= 2` and is not in
      // scope at the function tail, so this is the only correct place to release it.
      sys_munmap(cnt as *u8, DSS_MAXTERMS * 8)
  }
  var total: i64 = prox + title_bonus
  if total > DSS_PROX_SCALE { total = DSS_PROX_SCALE }
  // 7 function-level scratch buffers freed here; the 8th (cnt) is freed inside its own block above.
  sys_munmap(tbuf, DSC_TOKBUF)
  sys_munmap(pos as *u8, DSC_POSBUF)
  sys_munmap(fb, DSS_MAXTERMS + 8)
  sys_munmap(seen as *u8, DSS_MAXTERMS * 8)
  sys_munmap(tseen as *u8, DSS_MAXTERMS * 8)
  sys_munmap(opos as *u8, DSS_PROX_OCAP * 8)
  sys_munmap(oidx as *u8, DSS_PROX_OCAP * 8)
  return total
667}
668
669// DID-YOU-MEAN: for each query term UNKNOWN to the shard (dcount 0), scan the index's OWN sorted term
670// dictionary (ss_term_at -- the .terms blobs ARE the dictionary, no derived artifact) for the closest
671// term within edit distance 2, tiebroken by highest segment dcount. Writes the corrected query
672// (space-joined, known terms kept) into out; returns its length, or 0 when nothing needed/found.
673func dss_correct(domain: *u8, q: *u8, qn: i64, out: *u8, outcap: i64) -> i64 {
  let prefix: *u8 = sys_mmap(512)
  dss_prefix(domain, prefix)
  let h: *i64 = dss_open_maybe_cached(prefix)
  if (h as i64) == 0 { return 0 }
  let tbl: *u8 = sys_mmap(272)
  ss_tok_table(tbl)
  let termstore: *u8 = sys_mmap(DSS_MAXTERMS * 64)
  let termptrs: *i64 = sys_mmap(DSS_MAXTERMS * 8) as *i64
  var nterms: i64 = 0
  let qpos: *i64 = sys_mmap(16) as *i64
  qpos[0] = 0
  let tb: *u8 = sys_mmap(64)
  var qgo: i64 = 1
  while qgo == 1 {
      let l: i64 = ss_tok_next2(q, qn, qpos, tb, tbl)
      if l < 0 { qgo = 0 } else {
          if nterms < DSS_MAXTERMS {
              let dst: *u8 = (termstore as i64 + nterms * 64) as *u8
              var i: i64 = 0
              while tb[i] != (0 as u8) { dst[i] = tb[i]; i = i + 1 }
              dst[i] = 0 as u8
              termptrs[nterms] = dst as i64
              nterms = nterms + 1
          }
      }
  }
  if nterms == 0 { return 0 }
  let fixstore: *u8 = sys_mmap(DSS_MAXTERMS * 64)
  var corrected: i64 = 0
  let tpb: *i64 = sys_mmap(16) as *i64
  let tlb: *i64 = sys_mmap(16) as *i64
  let dcb: *i64 = sys_mmap(16) as *i64
  var t: i64 = 0
  while t < nterms {
      let term: *u8 = termptrs[t] as *u8
      let fix: *u8 = (fixstore as i64 + t * 64) as *u8
      var fi: i64 = 0
      while term[fi] != (0 as u8) { fix[fi] = term[fi]; fi = fi + 1 }
      fix[fi] = 0 as u8
      if ss_term_dcount(h, term) <= 0 {
          let tl0: i64 = fi
          var bestd: i64 = 3
          var bestdc: i64 = 0 - 1
          let ns: i64 = h[0]
          var s: i64 = 0
          while s < ns {
              let tc: i64 = ss_term_count(h, s)
              var e: i64 = 0
              while e < tc {
                  if ss_term_at(h, s, e, tpb, tlb, dcb) == 1 {
                      let d: i64 = ed_bounded(term, tl0, tpb[0] as *u8, tlb[0], 2)
                      var better: i64 = 0
                      if d < bestd { better = 1 }
                      if d == bestd { if dcb[0] > bestdc { better = 1 } }
                      if d <= 2 { if better == 1 {
                          bestd = d
                          bestdc = dcb[0]
                          let cp: *u8 = tpb[0] as *u8
                          var x: i64 = 0
                          while x < tlb[0] { fix[x] = cp[x]; x = x + 1 }
                          fix[tlb[0]] = 0 as u8
                      } }
                  }
                  e = e + 1
              }
              s = s + 1
          }
          if bestd <= 2 { corrected = corrected + 1 }
      }
      t = t + 1
  }
  if corrected == 0 { return 0 }
  // assemble the corrected query
  var o: i64 = 0
  t = 0
  while t < nterms {
      if t > 0 { if o < outcap { out[o] = 32 as u8; o = o + 1 } }
      let fx: *u8 = (fixstore as i64 + t * 64) as *u8
      var x2: i64 = 0
      while fx[x2] != (0 as u8) { if o < outcap { out[o] = fx[x2]; o = o + 1 } x2 = x2 + 1 }
      t = t + 1
  }
  if o < outcap { out[o] = 0 as u8 }
  sys_munmap(prefix, DSC_PREFIXBUF)
  sys_munmap(tbl, 272)
  sys_munmap(termstore, DSS_MAXTERMS * 64)
  sys_munmap(termptrs as *u8, DSS_MAXTERMS * 8)
  sys_munmap(qpos as *u8, DSC_POSBUF)
  sys_munmap(tb, DSC_TOKBUF)
  sys_munmap(fixstore, DSS_MAXTERMS * 64)
  sys_munmap(tpb as *u8, DSC_POSBUF)
  sys_munmap(tlb as *u8, DSC_POSBUF)
  sys_munmap(dcb as *u8, DSC_POSBUF)
  return o
768}
769// SUGGEST: rank the dictionary's completions of `pfx` by summed dcount (the store's own statistics; no
770// derived artifact). Fills up to `maxn` NUL-terminated terms packed into out (64 bytes apart); returns n.
771func dss_suggest(domain: *u8, pfx: *u8, pn: i64, out: *u8, maxn: i64) -> i64 {
  if pn < 1 { return 0 }
  let prefix: *u8 = sys_mmap(512)
  dss_prefix(domain, prefix)
  let h: *i64 = dss_open_maybe_cached(prefix)
  if (h as i64) == 0 { return 0 }
  let cand: *u8 = sys_mmap(64 * 40)
  let cdc: *i64 = sys_mmap(64 * 8) as *i64
  var nc: i64 = 0
  let tpb: *i64 = sys_mmap(16) as *i64
  let tlb: *i64 = sys_mmap(16) as *i64
  let dcb: *i64 = sys_mmap(16) as *i64
  let ns: i64 = h[0]
  var s: i64 = 0
  while s < ns {
      let tc: i64 = ss_term_count(h, s)
      var e: i64 = 0
      while e < tc {
          if ss_term_at(h, s, e, tpb, tlb, dcb) == 1 {
              if tlb[0] >= pn { if tlb[0] < 39 {
                  let tp: *u8 = tpb[0] as *u8
                  var m: i64 = 1
                  var x: i64 = 0
                  while x < pn { if tp[x] != pfx[x] { m = 0; x = pn } else { x = x + 1 } }
                  if m == 1 {
                      // merge into candidates (dedup across segments, dcounts summed)
                      var f: i64 = 0 - 1
                      var c: i64 = 0
                      while c < nc {
                          let cb: *u8 = (cand as i64 + c * 40) as *u8
                          var eq: i64 = 1
                          var y: i64 = 0
                          while y < tlb[0] { if cb[y] != tp[y] { eq = 0; y = tlb[0] } else { y = y + 1 } }
                          if eq == 1 { if cb[tlb[0]] == (0 as u8) { f = c } }
                          c = c + 1
                      }
                      if f >= 0 { cdc[f] = cdc[f] + dcb[0] }
                      else { if nc < 64 {
                          let nb: *u8 = (cand as i64 + nc * 40) as *u8
                          var y2: i64 = 0
                          while y2 < tlb[0] { nb[y2] = tp[y2]; y2 = y2 + 1 }
                          nb[tlb[0]] = 0 as u8
                          cdc[nc] = dcb[0]
                          nc = nc + 1
                      } }
                  }
              } }
          }
          e = e + 1
      }
      s = s + 1
  }
  // top maxn by dcount (selection)
  let used: *u8 = sys_mmap(64)
  var uz: i64 = 0
  while uz < nc { used[uz] = 0 as u8; uz = uz + 1 }
  var outn: i64 = 0
  while outn < maxn {
      var best: i64 = 0 - 1
      var bdc: i64 = 0 - 1
      var c2: i64 = 0
      while c2 < nc {
          if used[c2] == (0 as u8) { if cdc[c2] > bdc { bdc = cdc[c2]; best = c2 } }
          c2 = c2 + 1
      }
      if best < 0 { return outn }
      used[best] = 1 as u8
      let sb: *u8 = (cand as i64 + best * 40) as *u8
      let db: *u8 = (out as i64 + outn * 64) as *u8
      var y3: i64 = 0
      while sb[y3] != (0 as u8) { db[y3] = sb[y3]; y3 = y3 + 1 }
      db[y3] = 0 as u8
      outn = outn + 1
  }
  sys_munmap(prefix, DSC_PREFIXBUF)
  sys_munmap(cand, 64 * 40)
  sys_munmap(cdc as *u8, 64 * 8)
  sys_munmap(tpb as *u8, DSC_POSBUF)
  sys_munmap(tlb as *u8, DSC_POSBUF)
  sys_munmap(dcb as *u8, DSC_POSBUF)
  sys_munmap(used, DSC_TOKBUF)
  return outn
853}
854func dss_tlen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
855// STEM EXPANSION (recall; the sota_stemming rung, ZERO index change): for query term `term`, scan the live
856// term dictionary (ss_term_at over each segment's .terms) for terms whose Porter-lite stem == stem(term),
857// appending each NEW dictionary term (not already in the term list) to termstore/termptrs. Bounded by
858// DSS_MAXTERMS. Returns the (possibly grown) term count. `orig_nterms` = the pre-expansion count (only
859// those get stem-expanded, so we never expand an already-added variant).
860func dss_stem_expand(h: *i64, termstore: *u8, termptrs: *i64, nterms: i64, orig_nterms: i64) -> i64 {
  var nt: i64 = nterms
  let stemq: *u8 = sys_mmap(DSC_TOKBUF)
  let stemc: *u8 = sys_mmap(DSC_TOKBUF)
  let tpb: *i64 = sys_mmap(DSC_POSBUF) as *i64
  let tlb: *i64 = sys_mmap(DSC_POSBUF) as *i64
  let dcb: *i64 = sys_mmap(DSC_POSBUF) as *i64
  let cbuf: *u8 = sys_mmap(DSC_TOKBUF)
  var qt: i64 = 0
  while qt < orig_nterms {
      let term: *u8 = termptrs[qt] as *u8
      let qsn: i64 = nx_stem(term, dss_tlen(term), stemq)
      // prefix-prune width: Porter only strips/mutates SUFFIXES, so a dict term sharing this stem must
      // share the stem's leading bytes. Comparing the first min(qsn,3) bytes skips ~99% of the dictionary
      // BEFORE the expensive per-term stem -- the fix for the query-time dictionary-scan cost.
      var pw: i64 = qsn
      if pw > 3 { pw = 3 }
      let ns: i64 = h[0]
      var s: i64 = 0
      while s < ns {
          if nt >= DSS_MAXTERMS { s = ns } else {
              let tc: i64 = ss_term_count(h, s)
              var e: i64 = 0
              while e < tc {
                  if nt >= DSS_MAXTERMS { e = tc } else {
                      if ss_term_at(h, s, e, tpb, tlb, dcb) == 1 {
                          let dtp: *u8 = tpb[0] as *u8
                          // cheap prefix gate
                          var pfxok: i64 = 0
                          if tlb[0] >= pw {
                              pfxok = 1
                              var pk: i64 = 0
                              while pk < pw { if dtp[pk] != stemq[pk] { pfxok = 0; pk = pw } else { pk = pk + 1 } }
                          }
                          if pfxok == 1 { if tlb[0] < 40 {
                              // copy the dict term (NUL-terminate), stem it, compare stems
                              var ci: i64 = 0
                              while ci < tlb[0] { cbuf[ci] = dtp[ci]; ci = ci + 1 }
                              cbuf[tlb[0]] = 0 as u8
                              let csn: i64 = nx_stem(cbuf, tlb[0], stemc)
                              var samestem: i64 = 0
                              if csn == qsn {
                                  samestem = 1
                                  var x: i64 = 0
                                  while x < qsn { if stemc[x] != stemq[x] { samestem = 0; x = qsn } else { x = x + 1 } }
                              }
                              if samestem == 1 {
                                  // already in the term list?
                                  var dup: i64 = 0
                                  var y: i64 = 0
                                  while y < nt { if dss_streq(cbuf, termptrs[y] as *u8) == 1 { dup = 1; y = nt } else { y = y + 1 } }
                                  if dup == 0 {
                                      let dst: *u8 = (termstore as i64 + nt * 64) as *u8
                                      var z: i64 = 0
                                      while z < tlb[0] { dst[z] = cbuf[z]; z = z + 1 }
                                      dst[tlb[0]] = 0 as u8
                                      termptrs[nt] = dst as i64
                                      nt = nt + 1
                                  }
                              }
                          } }
                      }
                      e = e + 1
                  }
              }
              s = s + 1
          }
      }
      qt = qt + 1
  }
  sys_munmap(stemq, DSC_TOKBUF)
  sys_munmap(stemc, DSC_TOKBUF)
  sys_munmap(tpb as *u8, DSC_POSBUF)
  sys_munmap(tlb as *u8, DSC_POSBUF)
  sys_munmap(dcb as *u8, DSC_POSBUF)
  sys_munmap(cbuf, DSC_TOKBUF)
  return nt
937}
938// THE SOVEREIGN SEARCH (paged). (domain, query, offset) -> up to `max` ranked cids + scores STARTING at
939// rank `offset` (0 = the top); totalout[0] = total consent-passing candidates (the "about N results"
940// figure + the pager's has-more truth); totalout[1] = phrase exactness (2 = no phrase in the query,
941// 1 = "quoted phrase" enforced with positional adjacency everywhere, 0 = a pre-phrase segment degraded
942// to AND -- compaction upgrades it). Returns nresults emitted; -2 = a live segment lacks .terms.
943// QUERY SYNTAX: +term is REQUIRED (AND); -term is EXCLUDED (NOT); "quoted words" is a PHRASE (adjacent,
944// in order, via the NXQ1 positions sidecar; phrase terms also score normally); site:<host> scopes results
945// to pages whose url:<cid> host matches (dot-suffix: site:wikipedia.org covers en.wikipedia.org).
946func dss_search_off_div(domain: *u8, q: *u8, qn: i64, cids_out: *i64, scores_out: *i64, max: i64, offset: i64, totalout: *i64, webdiv: i64) -> i64 {
  totalout[0] = 0
  totalout[1] = 2
  let prefix: *u8 = sys_mmap(512)
  dss_prefix(domain, prefix)
  let h: *i64 = dss_open_maybe_cached(prefix)
  if (h as i64) == 0 { return 0 }   // empty/absent shard -> no results (also the cross-shard isolation guarantee)
  let tbl: *u8 = sys_mmap(272)
  ss_tok_table(tbl)
  // site:<host> FIELD FILTER (the faceted-search rung): parse + STRIP the clause up front so its tokens
  // ("site", the host words) never reach scoring; the host lands in sitehost (lowercased) and filters
  // candidates against their url:<cid> rows below. qq/qqn = the stripped query all downstream scans use.
  let sitehost: *u8 = sys_mmap(256)
  sitehost[0] = 0 as u8
  let qq: *u8 = sys_mmap(qn + 8)
  var qqn: i64 = 0
  var si0: i64 = 0
  while si0 < qn {
      var issite: i64 = 0
      if si0 + 5 <= qn {
          var bnd: i64 = 0
          if si0 == 0 { bnd = 1 } else { if tbl[q[si0 - 1]] == (0 as u8) { bnd = 1 } }
          if bnd == 1 {
              if q[si0] == (115 as u8) { if q[si0+1] == (105 as u8) { if q[si0+2] == (116 as u8) { if q[si0+3] == (101 as u8) { if q[si0+4] == (58 as u8) { issite = 1 } } } } }
          }
      }
      if issite == 1 {
          var hj: i64 = si0 + 5
          var hw: i64 = 0
          var hgo: i64 = 1
          while hgo == 1 {
              if hj >= qn { hgo = 0 } else {
                  var hc: i64 = q[hj] as i64
                  if hc == 32 { hgo = 0 } else {
                      if hc >= 65 { if hc <= 90 { hc = hc + 32 } }
                      if hw < 250 { sitehost[hw] = hc as u8; hw = hw + 1 }
                      hj = hj + 1
                  }
              }
          }
          sitehost[hw] = 0 as u8
          si0 = hj
      } else {
          qq[qqn] = q[si0]
          qqn = qqn + 1
          si0 = si0 + 1
      }
  }
  qq[qqn] = 0 as u8
  // R2b URL-SCOPE UNDERSTANDING (2026-08-04, operator: "search reddit and a specific subreddit brings
  // barely anything back"): two query idioms scope results by URL PATH, sharing one filter below.
  //   inurl:<frag> -- explicit operator, ANY site: keep candidates whose url contains <frag>.
  //   r/<name>     -- the universal subreddit idiom: implies site:reddit.com (only when no explicit
  //                   site: clause was given) + segment-anchored path /r/<name>; the bare name stays
  //                   in the query as a scoring term. Without this rewrite the tokenizer's 2-char
  //                   floor dropped the "r" and the whole scoping intent of "r/StableDiffusion"
  //                   silently died -- the query degraded to a bag of boilerplate-matchable words.
  // First clause of each kind wins; queries carrying neither are BYTE-IDENTICAL downstream (ruler-safe).
  let pathpat: *u8 = sys_mmap(256)
  pathpat[0] = 0 as u8
  var seganchor: i64 = 0
  let qr: *u8 = sys_mmap(qqn + 16)
  var qrn: i64 = 0
  var pi0: i64 = 0
  while pi0 < qqn {
      var bnd2: i64 = 0
      if pi0 == 0 { bnd2 = 1 } else { if tbl[qq[pi0 - 1]] == (0 as u8) { bnd2 = 1 } }
      var consumed: i64 = 0
      // inurl:<frag>
      if bnd2 == 1 { if pathpat[0] == (0 as u8) { if pi0 + 6 <= qqn {
          if qq[pi0] == (105 as u8) { if qq[pi0+1] == (110 as u8) { if qq[pi0+2] == (117 as u8) { if qq[pi0+3] == (114 as u8) { if qq[pi0+4] == (108 as u8) { if qq[pi0+5] == (58 as u8) {
              var fj: i64 = pi0 + 6
              var fw: i64 = 0
              var fgo: i64 = 1
              while fgo == 1 {
                  if fj >= qqn { fgo = 0 } else {
                      var fc: i64 = qq[fj] as i64
                      if fc == 32 { fgo = 0 } else {
                          if fc >= 65 { if fc <= 90 { fc = fc + 32 } }
                          if fw < 250 { pathpat[fw] = fc as u8; fw = fw + 1 }
                          fj = fj + 1
                      }
                  }
              }
              pathpat[fw] = 0 as u8
              seganchor = 0
              if fw > 0 { consumed = 1; pi0 = fj }
          } } } } } }
      } } }
      // r/<name> (subreddit idiom): name = [A-Za-z0-9_]{2,30}
      if consumed == 0 { if bnd2 == 1 { if pathpat[0] == (0 as u8) { if pi0 + 3 < qqn {
          var isr: i64 = 0
          if qq[pi0] == (114 as u8) { isr = 1 }
          if qq[pi0] == (82 as u8) { isr = 1 }
          if isr == 1 { if qq[pi0+1] == (47 as u8) {
              var nj: i64 = pi0 + 2
              var nlen: i64 = 0
              var ngo: i64 = 1
              while ngo == 1 {
                  if nj >= qqn { ngo = 0 } else {
                      let nc: i64 = qq[nj] as i64
                      var isw: i64 = 0
                      if nc >= 97 { if nc <= 122 { isw = 1 } }
                      if nc >= 65 { if nc <= 90 { isw = 1 } }
                      if nc >= 48 { if nc <= 57 { isw = 1 } }
                      if nc == 95 { isw = 1 }
                      if isw == 1 { nlen = nlen + 1; nj = nj + 1 } else { ngo = 0 }
                  }
              }
              if nlen >= 2 { if nlen <= 30 {
                  pathpat[0] = 47 as u8    // '/'
                  pathpat[1] = 114 as u8   // 'r'
                  pathpat[2] = 47 as u8    // '/'
                  var nz: i64 = 0
                  while nz < nlen {
                      var vc: i64 = qq[pi0 + 2 + nz] as i64
                      if vc >= 65 { if vc <= 90 { vc = vc + 32 } }
                      pathpat[3 + nz] = vc as u8
                      nz = nz + 1
                  }
                  pathpat[3 + nlen] = 0 as u8
                  seganchor = 1
                  if sitehost[0] == (0 as u8) {
                      let rh: *u8 = "reddit.com" as *u8
                      var rz: i64 = 0
                      while rh[rz] != (0 as u8) { sitehost[rz] = rh[rz]; rz = rz + 1 }
                      sitehost[rz] = 0 as u8
                  }
                  var cz: i64 = 0
                  while cz < nlen { qr[qrn] = qq[pi0 + 2 + cz]; qrn = qrn + 1; cz = cz + 1 }
                  consumed = 1
                  pi0 = nj
              } }
          } }
      } } } }
      if consumed == 0 {
          qr[qrn] = qq[pi0]
          qrn = qrn + 1
          pi0 = pi0 + 1
      }
  }
  var qc9: i64 = 0
  while qc9 < qrn { qq[qc9] = qr[qc9]; qc9 = qc9 + 1 }
  qqn = qrn
  qq[qqn] = 0 as u8
  // R6 BARE-site: FALSE-ZERO FIX (2026-08-05, debt 1785934453): a site:-only query stripped to
  // zero terms and hit the nterms==0 return -- an unconditional 0 indistinguishable from "host
  // absent", which poisoned every coverage census taken with bare site:. When the residual query
  // holds NO token characters and a site: clause was given, synthesize the host's own labels as
  // the query ("commoncrawl org" for site:commoncrawl.org): pages carry their host's brand
  // labels, so this returns an HONEST lower-bound listing through the NORMAL pipeline -- BM25,
  // authority, consent (pol:) checks all apply, purged docs stay purged. True url:-walk
  // enumeration remains the debt's next rung. Queries with any real term are byte-identical.
  if sitehost[0] != (0 as u8) {
      var r6only: i64 = 1
      var r6i: i64 = 0
      while r6i < qqn { if tbl[qq[r6i]] != (0 as u8) { r6only = 0; r6i = qqn } else { r6i = r6i + 1 } }
      if r6only == 1 {
          qqn = 0
          var r6j: i64 = 0
          while sitehost[r6j] != (0 as u8) {
              var r6c: i64 = sitehost[r6j] as i64
              if r6c == 46 { r6c = 32 }
              if r6c == 45 { r6c = 32 }
              qq[qqn] = r6c as u8
              qqn = qqn + 1
              r6j = r6j + 1
          }
          qq[qqn] = 0 as u8
      }
  }
  // tokenize the query with the index's OWN tokenizer so terms align exactly with the postings
  let termstore: *u8 = sys_mmap(DSS_MAXTERMS * 64)
  let termptrs: *i64 = sys_mmap(DSS_MAXTERMS * 8) as *i64
  var nterms: i64 = 0
  let qpos: *i64 = sys_mmap(16) as *i64
  qpos[0] = 0
  let tb: *u8 = sys_mmap(64)
  var qgo: i64 = 1
  while qgo == 1 {
      let l: i64 = ss_tok_next2(qq, qqn, qpos, tb, tbl)
      if l < 0 { qgo = 0 } else {
          if nterms < DSS_MAXTERMS {
              let dst: *u8 = (termstore as i64 + nterms * 64) as *u8
              var i: i64 = 0
              while tb[i] != (0 as u8) { dst[i] = tb[i]; i = i + 1 }
              dst[i] = 0 as u8
              termptrs[nterms] = dst as i64
              nterms = nterms + 1
          }
      }
  }
  if nterms == 0 { return 0 }
  // +term REQUIRED / -term EXCLUDED masks: '+' and '-' are separators to the tokenizer (never reach
  // the term list), so scan the RAW query -- at a token boundary they mark the NEXT token as required
  // (AND) or excluded (NOT). A '-' INSIDE a word (e-mail) is not a boundary and stays inert.
  let reqmask: *i64 = sys_mmap(DSS_MAXTERMS * 8) as *i64
  let exclmask: *i64 = sys_mmap(DSS_MAXTERMS * 8) as *i64
  var rz: i64 = 0
  while rz < nterms { reqmask[rz] = 0; exclmask[rz] = 0; rz = rz + 1 }
  let rtok: *u8 = sys_mmap(64)
  let rpos: *i64 = sys_mmap(16) as *i64
  var qi: i64 = 0
  while qi < qqn {
      var op: i64 = 0
      if qq[qi] == (43 as u8) { op = 1 }
      if qq[qi] == (45 as u8) { op = 2 }
      if op != 0 {
          var boundary: i64 = 0
          if qi == 0 { boundary = 1 } else { if tbl[qq[qi - 1]] == (0 as u8) { boundary = 1 } }
          if boundary == 1 { if qi + 1 < qqn { if tbl[qq[qi + 1]] != (0 as u8) {
              rpos[0] = qi + 1
              let rl: i64 = ss_tok_next2(qq, qqn, rpos, rtok, tbl)
              if rl >= 2 {
                  var tt2: i64 = 0
                  while tt2 < nterms {
                      if dss_streq(rtok, termptrs[tt2] as *u8) == 1 {
                          if op == 1 { reqmask[tt2] = 1 } else { exclmask[tt2] = 1 }
                      }
                      tt2 = tt2 + 1
                  }
              }
          } } }
      }
      qi = qi + 1
  }
  // "quoted phrase" parse over the RAW query (quotes are tokenizer separators, so the terms list is
  // unaffected -- the phrase only narrows CANDIDACY). First quoted pair honored; 2..8 terms.
  let phterms: *u8 = sys_mmap(8 * 64)
  let phptrs: *i64 = sys_mmap(8 * 8) as *i64
  var nph: i64 = 0
  var q1: i64 = 0 - 1
  var q2: i64 = 0 - 1
  var qi2: i64 = 0
  while qi2 < qqn {
      if qq[qi2] == (34 as u8) {
          if q1 < 0 { q1 = qi2 } else { if q2 < 0 { q2 = qi2 } }
      }
      qi2 = qi2 + 1
  }
  if q1 >= 0 { if q2 > q1 + 1 {
      let ppos: *i64 = sys_mmap(16) as *i64
      ppos[0] = q1 + 1
      let ptok: *u8 = sys_mmap(64)
      var pgo: i64 = 1
      while pgo == 1 {
          if ppos[0] >= q2 { pgo = 0 } else {
              let pl: i64 = ss_tok_next2(qq, q2, ppos, ptok, tbl)
              if pl < 0 { pgo = 0 } else {
                  if nph < 8 {
                      let pdst: *u8 = (phterms as i64 + nph * 64) as *u8
                      var px: i64 = 0
                      while ptok[px] != (0 as u8) { pdst[px] = ptok[px]; px = px + 1 }
                      pdst[px] = 0 as u8
                      phptrs[nph] = pdst as i64
                      nph = nph + 1
                  }
              }
          }
      }
  } }
  // STEM EXPANSION (recall; the sota_stemming rung) -- only for plain OR queries. A phrase needs exact
  // adjacency; a query carrying +required/-excluded operators has exact term semantics; expanding those
  // would blur the operator. Runs BEFORE idf so the added variants get real idf weights (a variant match
  // scores its own idf*tf, ranking below an exact match but above a non-match). reqmask/exclmask stay
  // sized to the ORIGINAL terms -- variants are never required/excluded (they are pure OR recall).
  var hasop0: i64 = 0
  var opz: i64 = 0
  while opz < nterms { if reqmask[opz] == 1 { hasop0 = 1 } if exclmask[opz] == 1 { hasop0 = 1 } opz = opz + 1 }
  let r1g_onterms: i64 = nterms   // R1g: ORIGINAL term count (pre-stem-expansion) -- the entity-shape gate
  if nph < 2 { if hasop0 == 0 {
      nterms = dss_stem_expand(h, termstore, termptrs, nterms, nterms)
  } }
  // idf per term (stage-1 needs it): the store's own persisted statistics -- covers expanded variants too.
  // dcnt[] kept alongside: the rare-term tf-floor (DSS_RARE_DCOUNT) needs the raw document frequency.
  let bign: i64 = ss_doc_count(h)
  let idf: *i64 = sys_mmap(DSS_MAXTERMS * 8) as *i64
  let dcnt: *i64 = sys_mmap(DSS_MAXTERMS * 8) as *i64
  // stopword threshold = DSS_STOP_DFPCT% of the corpus (0 disables when the corpus is too small)
  var stopdf: i64 = 0
  if bign >= DSS_STOP_MINCORPUS { stopdf = (bign * DSS_STOP_DFPCT) / 100 }
  var ti0: i64 = 0
  while ti0 < nterms {
      let nt0: i64 = ss_term_dcount(h, termptrs[ti0] as *u8)
      dcnt[ti0] = nt0
      var v0: i64 = idf_q10(bign, nt0)
      // corpus-derived stopword discount: a term in >DSS_STOP_DFPCT% of docs loses ranking weight so
      // content terms drive the result (fixes function-word domination; language-agnostic by construction)
      if stopdf > 0 { if nt0 > stopdf { v0 = v0 / DSS_STOP_DISCOUNT } }
      idf[ti0] = v0
      ti0 = ti0 + 1
  }
  let cand_cid: *i64 = sys_mmap(DSS_MAXCAND * 8) as *i64
  let cand_matched: *i64 = sys_mmap(DSS_MAXCAND * 8) as *i64
  let cand_tmask: *i64 = sys_mmap(DSS_MAXCAND * 8) as *i64   // bit t set = postings say term t is IN this doc (full-doc truth)
  var ncand: i64 = 0
  var candsat: i64 = 0   // 1 = candidacy truncated somewhere -> totals become df-derived estimates
  let satbox: *i64 = sys_mmap(16) as *i64
  let kp: *i64 = sys_mmap(DSS_MAXHITS * 8) as *i64
  let kl: *i64 = sys_mmap(DSS_MAXHITS * 8) as *i64
  let ktf: *i64 = sys_mmap(DSS_MAXHITS * 8) as *i64   // WAND rung: per-hit tf from the .imp sidecar
  // NOTE (2026-07-25, ruler-refuted): overlaying the sidecar's TRUE full-doc tf into stage-2 BM25
  // (max with the DSS_TFSCAN-capped scan) measured MRR@10 701 -> 566 and was REVERTED: full-doc tf
  // rewards long keyword-dense boilerplate (tf-spam), while the scan cap doubles as a head-of-doc
  // quality prior. Sidecar tf stays candidacy-only until learned-sparse impact weights replace raw tf.
  if nph >= 2 {
      // PHRASE candidacy: only docs where the quoted terms are adjacent in order (per-segment NXQ1
      // ladder; sidecar-less segments degrade to AND and clear the exact flag). Dedup cids: the
      // manifest's dup-line wart can walk a segment twice.
      let prefix0: *u8 = sys_mmap(512)
      dss_prefix(domain, prefix0)
      let ebox: *i64 = sys_mmap(16) as *i64
      let np2: i64 = ss_phrase(prefix0, h, phptrs, nph, kp, kl, DSS_PHRASEHITS, ebox)
      if np2 == (0 - 2) { return 0 - 2 }
      totalout[1] = ebox[0]
      var j2: i64 = 0
      while j2 < np2 {
          let cid2: i64 = dss_key_cid(kp[j2] as *u8, kl[j2])
          if cid2 >= 0 {
              var f2: i64 = 0 - 1
              var c9: i64 = 0
              while c9 < ncand { if cand_cid[c9] == cid2 { f2 = c9 } c9 = c9 + 1 }
              if f2 < 0 { if ncand < DSS_MAXCAND { cand_cid[ncand] = cid2; cand_matched[ncand] = 1; cand_tmask[ncand] = 0; ncand = ncand + 1 } }
          }
          j2 = j2 + 1
      }
  } else {
      // candidate accumulation via per-term postings (OR) -- EXCLUDED terms contribute no candidates.
      // cand_matched accumulates the STAGE-1 score: sum of matched-term idfs straight from the
      // postings (no doc reads) -- the cheap ranking that picks the stage-2 shortlist.
      // RAREST-TERM-FIRST (2026-07-24, the "toy-grade" candidacy fix): process terms by ASCENDING document
      // frequency. A common term ("julia", hundreds of postings) must NOT saturate DSS_MAXCAND before the
      // rare DISCRIMINATIVE terms ("measurements", "idols") claim their slots -- otherwise the ONE page that
      // best matches the rare terms (the entity's profile) never enters the candidate set and is unrankable
      // at any score. Ordering is candidacy-only: the OR score/tmask are order-independent (commutative sums),
      // so this is byte-identical on any query whose candidates all fit under the cap (e.g. the gate corpus).
      let torder: *i64 = sys_mmap(DSS_MAXTERMS * 8) as *i64
      var norder: i64 = 0
      var to0: i64 = 0
      while to0 < nterms { if exclmask[to0] == 0 { torder[norder] = to0; norder = norder + 1 } to0 = to0 + 1 }
      var oi: i64 = 1
      while oi < norder {
          let keyt: i64 = torder[oi]
          var oj: i64 = oi - 1
          var moving: i64 = 1
          while moving == 1 {
              if oj >= 0 { if dcnt[torder[oj]] > dcnt[keyt] { torder[oj + 1] = torder[oj]; oj = oj - 1 } else { moving = 0 } } else { moving = 0 }
          }
          torder[oj + 1] = keyt
          oi = oi + 1
      }
      var oidx: i64 = 0
      while oidx < norder {
          let t: i64 = torder[oidx]
          // IMPACT-ORDERED candidacy (2026-07-25, WAND rung, seq606): the per-term cap keeps the
          // HIGHEST-tf postings (an entity's profile page has the highest tf of its name), not the
          // first 512 in doc-id order. -3 = shard not yet .imp-upgraded -> exact legacy behavior.
          var usedimp: i64 = 1   // seq871: ktf[] is only meaningful on the impact path
          var nm: i64 = ss_term_top(prefix, h, termptrs[t] as *u8, kp, kl, ktf, DSS_MAXHITS, satbox)
          // candidacy SATURATION (2026-07-25 honest-total rung): trust the READER's own truncation
          // flag (build-capped list / collect / emit) -- NOT dcnt>nm, which false-positives on any
          // shard with shadowed re-committed docs (write-time dcount counts stale versions; the
          // dup-segment gate fixture proved it). A full candidate table saturates too.
          if satbox[0] == 1 { candsat = 1 }
          if ncand >= DSS_MAXCAND { candsat = 1 }
          if nm == (0 - 3) { usedimp = 0; nm = ss_term(h, termptrs[t] as *u8, kp, kl, DSS_MAXHITS) }
          if nm == (0 - 2) { return 0 - 2 }
          var j: i64 = 0
          while j < nm {
              let cid: i64 = dss_key_cid(kp[j] as *u8, kl[j])
              if cid >= 0 {
                  // IMPACT-AWARE STAGE-1 (seq871): the shortlist score weights each matched term's idf
                  // by a SATURATING tf factor carried in the impact list. The old sum-of-idf was
                  // tf-BLIND, so among candidates matching the SAME terms the shortlist kept whichever
                  // was walked first -- which is exactly why a 4x deeper candidate pool ranked WORSE
                  // (measured 705->534: nsf.gov fell out of the shortlist and became unrankable).
                  // Saturating, never linear: see DSS_S1_TFK. Fallback (no .imp) keeps the legacy sum.
                  var s1w: i64 = idf[t]
                  if usedimp == 1 { let tfv: i64 = ktf[j]; if tfv > 0 { s1w = (idf[t] * tfv) / (tfv + DSS_S1_TFK) } }
                  var f: i64 = 0 - 1
                  var c: i64 = 0
                  while c < ncand { if cand_cid[c] == cid { f = c } c = c + 1 }
                  if f < 0 {
                      if ncand < DSS_MAXCAND { cand_cid[ncand] = cid; cand_matched[ncand] = s1w; cand_tmask[ncand] = 1 << t; ncand = ncand + 1 }
                  } else {
                      cand_matched[f] = cand_matched[f] + s1w
                      cand_tmask[f] = cand_tmask[f] | (1 << t)
                  }
              }
              j = j + 1
          }
          oidx = oidx + 1
      }
  }
  if ncand == 0 { return 0 }
  // OWNER-CONSENT at query time. The retired tsv emitter enforced DP_USE_PUB_SEARCH at EMISSION (an
  // unflagged doc never entered <domain>_src.tsv); the store-native path enforces it LIVE per hit, so a
  // consent amendment takes effect at once (the docportal_lib doctrine). A candidate whose stored
  // pol:<cid> row LACKS the search bit is dropped; an ABSENT pol: row stays searchable -- that matches
  // dp_default_policy(PUBLIC) and covers migrated/legacy shards whose docs were consent-derived already.
  let polkey: *u8 = sys_mmap(64)
  let pp: *i64 = sys_mmap(16) as *i64
  let pl: *i64 = sys_mmap(16) as *i64
  var wkeep: i64 = 0
  var rcand: i64 = 0
  while rcand < ncand {
      dss_mkpolkey(cand_cid[rcand], polkey)
      var keep: i64 = 1
      if ss_hget(h, polkey, pp, pl) == 1 {
          var fv: i64 = 0
          let fp: *u8 = pp[0] as *u8
          var fi: i64 = 0
          while fi < pl[0] { let ch: i64 = fp[fi] as i64; if ch >= 48 { if ch <= 57 { fv = fv * 10 + (ch - 48) } } fi = fi + 1 }
          if (fv & DSS_POL_SEARCH) == 0 { keep = 0 }
      }
      if keep == 1 {
          cand_cid[wkeep] = cand_cid[rcand]
          cand_matched[wkeep] = cand_matched[rcand]
          cand_tmask[wkeep] = cand_tmask[rcand]
          wkeep = wkeep + 1
      }
      rcand = rcand + 1
  }
  ncand = wkeep
  if ncand == 0 { return 0 }
  // site:<host> filter (the faceted rung): keep only candidates whose url:<cid> row's host matches
  // (dot-suffix). Docs WITHOUT a url row (library texts) are not site-attributable -> dropped when a
  // site: clause is present. Runs BEFORE totals so "N matched" stays honest. Cheap: ss_hget per
  // candidate, no doc reads.
  if sitehost[0] != (0 as u8) {
      let sukey: *u8 = sys_mmap(64)
      let sup: *i64 = sys_mmap(16) as *i64
      let sul: *i64 = sys_mmap(16) as *i64
      var wsite: i64 = 0
      var rsite: i64 = 0
      while rsite < ncand {
          var skeep: i64 = 0
          dss_mkurlkey(cand_cid[rsite], sukey)
          if ss_hget(h, sukey, sup, sul) == 1 { if sul[0] > 0 {
              if dss_url_host_match(sup[0] as *u8, sul[0], sitehost) == 1 { skeep = 1 }
          } }
          if skeep == 1 {
              cand_cid[wsite] = cand_cid[rsite]
              cand_matched[wsite] = cand_matched[rsite]
              cand_tmask[wsite] = cand_tmask[rsite]
              wsite = wsite + 1
          }
          rsite = rsite + 1
      }
      ncand = wsite
      if ncand == 0 { return 0 }
  }
  // R2b URL-PATH filter (inurl: / r/<subreddit>): same contract as site: -- candidates whose url row
  // lacks the pattern (or that have no url row at all) drop BEFORE totals, so "N matched" stays honest.
  if pathpat[0] != (0 as u8) {
      let pukey: *u8 = sys_mmap(64)
      let pup: *i64 = sys_mmap(16) as *i64
      let pul: *i64 = sys_mmap(16) as *i64
      var wpath: i64 = 0
      var rpath: i64 = 0
      while rpath < ncand {
          var pkeep: i64 = 0
          dss_mkurlkey(cand_cid[rpath], pukey)
          if ss_hget(h, pukey, pup, pul) == 1 { if pul[0] > 0 {
              if dss_url_path_has(pup[0] as *u8, pul[0], pathpat, seganchor) == 1 { pkeep = 1 }
          } }
          if pkeep == 1 {
              cand_cid[wpath] = cand_cid[rpath]
              cand_matched[wpath] = cand_matched[rpath]
              cand_tmask[wpath] = cand_tmask[rpath]
              wpath = wpath + 1
          }
          rpath = rpath + 1
      }
      ncand = wpath
      if ncand == 0 { return 0 }
  }
  // the honest "N matched" (all consent-passing candidates) BEFORE the shortlist caps the set.
  // HONEST-TOTAL rung (2026-07-25, operator: "when do we get more than a few hundred results"): when
  // candidacy SATURATED, the walk count under-states reality -- the store's own per-term df (dcnt,
  // exact, corpus-wide) is a true lower bound on the OR-union match count, so report the largest
  // content-term df instead. Estimate applies ONLY to plain OR queries (no site:, no phrase, no
  // +/- operators -- those narrow the set below df). Unsaturated queries keep the exact count, so
  // small shards and every gate fixture are byte-identical. Downstream drops (score-0, url-dedup)
  // SUBTRACT from this total instead of resetting it to the shortlist remnant -- the old behavior
  // displayed ~128 "matched" on ANY corpus, which was the shortlist size, not the match count.
  var matched_total: i64 = ncand
  if candsat == 1 { if sitehost[0] == (0 as u8) { if pathpat[0] == (0 as u8) { if nph < 2 { if hasop0 == 0 {
      var dfm: i64 = 0
      var dft: i64 = 0
      while dft < nterms {
          if exclmask[dft] == 0 { if dcnt[dft] > dfm { dfm = dcnt[dft] } }
          dft = dft + 1
      }
      if dfm > matched_total { matched_total = dfm }
  } } } } }
  totalout[0] = matched_total
  // STAGE-1 SHORTLIST (WAND / BlockMax two-stage retrieval -- the sota_* BlockMax pattern): cand_matched
  // holds the CHEAP postings-only score (summed matched-term idf, ZERO doc reads). Full BM25 below reads
  // each survivor's TEXT (the p95 cost on many-candidate queries), so cap survivors to the top
  // (offset+max+margin) by stage-1 score. Deep pagination widens the shortlist so it stays correct;
  // the common first pages get 4x fewer doc walks. Phrase candidates carry a flat stage-1 score but are
  // already adjacency-narrowed (rarely > shortn), so this is a no-op for them.
  // MEASURED 2026-07-24: trimming this pool (54 vs 126) cut only ~17% latency but dropped MRR 834->809 -- the
  // p95 web cost is NOT the candidate count, it's page-faults on the cold 2.4GB mmap (stage-1 postings + serve
  // reads). Real speed = the precomputed-tf-in-postings + compact per-doc sidecar rung. Kept the WIDE, correct
  // shortlist (quality first): offset+max+margin, deep pages widen it.
  // SHORTLIST FLOOR (seq877, 2026-07-25): the round-4 three-point measurement isolated THIS as the
  // depth blocker -- the pool grew 4x while the sieve handed to full BM25 stayed 128, so more
  // candidates had to be discriminated by a same-size filter. Floor raised to DSS_SHORT_FLOOR: at
  // the shipping depth (DSS_MAXCAND=512) that means EVERY candidate gets full BM25 and the cheap
  // stage-1 stops being a lossy gate at all. Costs one doc read per extra candidate -- ruler AND
  // latency gated, because paying 4x the reads for a flat number is not an improvement.
  var shortn: i64 = offset + max + 96
  if shortn < DSS_SHORT_FLOOR { shortn = DSS_SHORT_FLOOR }
  if ncand > shortn {
      var ssel: i64 = 0
      while ssel < shortn {
          var best: i64 = ssel
          var sj: i64 = ssel + 1
          while sj < ncand {
              if cand_matched[sj] > cand_matched[best] { best = sj }
              sj = sj + 1
          }
          if best != ssel {
              let tc: i64 = cand_cid[ssel]; cand_cid[ssel] = cand_cid[best]; cand_cid[best] = tc
              let tm: i64 = cand_matched[ssel]; cand_matched[ssel] = cand_matched[best]; cand_matched[best] = tm
              let tk: i64 = cand_tmask[ssel]; cand_tmask[ssel] = cand_tmask[best]; cand_tmask[best] = tk
          }
          ssel = ssel + 1
      }
      ncand = shortn
  }
  // RANKING (the BM25/IDF rung, integer-only): stage-2 reranks the shortlist with FULL BM25 --
  // score(d) = SUM idf_q10(N,n_t) * tfnorm_q10(tf, |d|/avgdl). idf[] was computed pre-candidacy.
  let cand_score: *i64 = sys_mmap(DSS_MAXCAND * 8) as *i64
  let keybuf: *u8 = sys_mmap(64)
  let dptr: *i64 = sys_mmap(16) as *i64
  let dlen: *i64 = sys_mmap(16) as *i64
  // PASS 1 -- one walk per candidate: all term tfs + doc token length (the BM25 |d| statistic)
  let tfmat: *i64 = sys_mmap(DSS_MAXCAND * DSS_MAXTERMS * 8) as *i64
  let cand_dl: *i64 = sys_mmap(DSS_MAXCAND * 8) as *i64
  let cand_toks: *i64 = sys_mmap(DSS_MAXCAND * 8) as *i64   // R2d: pass-1 token count (was discarded)
  let cand_capped: *i64 = sys_mmap(DSS_MAXCAND * 8) as *i64
  let cand_prox: *i64 = sys_mmap(DSS_MAXCAND * 8) as *i64   // R1b: per-candidate term-proximity strength
  var dlsum: i64 = 0
  var c1: i64 = 0
  while c1 < ncand {
      dss_mkkey(cand_cid[c1], keybuf)
      var dl: i64 = 0
      cand_capped[c1] = 0
      cand_prox[c1] = 0
      if ss_hget(h, keybuf, dptr, dlen) == 1 {
          // scan-cap the tf walk: term frequencies saturate (k1=1.2) and the leading text carries the
          // relevant occurrences, so bounding the per-doc walk to DSS_TFSCAN bytes caps the query cost on
          // pathological docs (1MB wiki dumps) with negligible ranking impact. Precomputing tf INTO the
          // postings would remove the walk entirely = the perf store rung (conceded in the census).
          var dn: i64 = dlen[0]
          if dn > DSS_TFSCAN { dn = DSS_TFSCAN; cand_capped[c1] = 1 }
          cand_toks[c1] = dss_tf_all(dptr[0] as *u8, dn, termptrs, nterms, tbl, (tfmat as i64 + c1 * DSS_MAXTERMS * 8) as *i64)   // fills tf_out; R2d keeps the token-|d| return
          cand_prox[c1] = dss_prox_all(dptr[0] as *u8, dn, termptrs, nterms, tbl)   // R1b: token-aligned proximity, same pass semantics as tf
          // |d| = the doc's BYTE length (from ss_hget -- cap-INDEPENDENT, no extra scan): the shrunk tf-scan cap
          // no longer distorts length-norm, and |d|/avgdl is scale-invariant so BM25 b-norm is preserved (2026-07-24).
          dl = dlen[0]
      }
      cand_dl[c1] = dl
      dlsum = dlsum + dl
      c1 = c1 + 1
  }
  // avgdl over the CANDIDATE SET (self-consistent per query; corpus-persisted avgdl = the store-rung
  // refinement, conceded in the census). Guard: never 0.
  var avgdl: i64 = dlsum / ncand
  if avgdl < 1 { avgdl = 1 }
  // PASS 2 -- FULL BM25 (b=0.75): score = SUM idf * tfnorm(tf, 1024*|d|/avgdl); +term still disqualifies
  var c2: i64 = 0
  while c2 < ncand {
      let tfrow: *i64 = (tfmat as i64 + c2 * DSS_MAXTERMS * 8) as *i64
      let normq10: i64 = (cand_dl[c2] * DSS_MAGIC_1024) / avgdl
      var sc: i64 = 0
      var reqok: i64 = 1
      var tt: i64 = 0
      while tt < nterms {
          var tf: i64 = tfrow[tt]
          // RARE-TERM tf FLOOR: postings (full-doc truth) say the term IS here, the capped scan just
          // never reached it, and the term is near-unique corpus-wide -> floor tf to 1 so the doc stays
          // findable (ranked low: tail match of a rare term beats nothing, loses to head matches).
          if tf == 0 { if cand_capped[c2] == 1 { if (cand_tmask[c2] & (1 << tt)) != 0 { if dcnt[tt] >= 1 { if dcnt[tt] <= DSS_RARE_DCOUNT { tf = 1 } } } } }
          if reqmask[tt] == 1 { if tf == 0 { reqok = 0 } }
          if exclmask[tt] == 1 {
              if tf > 0 { reqok = 0 }   // -term PRESENT disqualifies (boolean NOT)
          } else {
              sc = sc + idf[tt] * tfnorm_q10(tf, normq10)
          }
          tt = tt + 1
      }
      if reqok == 0 { sc = 0 - 1 }   // a missing REQUIRED or present EXCLUDED term disqualifies outright
      // R1b PROXIMITY BOOST: bounded multiply on POSITIVE scores only (mirrors the PageRank authority
      // fusion below); clustered query terms rank higher. prox=0 (single-term / no-cluster) -> factor
      // 1024 -> NO CHANGE, so single-term queries + non-clustered docs are byte-identical to before.
      if sc > 0 { let pf: i64 = DSS_MAGIC_1024 + (cand_prox[c2] * DSS_PROX_BOOST) / DSS_PROX_SCALE; sc = (sc * pf) / DSS_MAGIC_1024 }
      cand_score[c2] = sc
      c2 = c2 + 1
  }
  // R1d COORDINATION -- IDF-WEIGHTED (2026-07-24 upgrade, ruler-driven): down-weight docs that cover less of
  // the query's IDF-MASS. Matching the rare/salient terms (e.g. "blood pressure") counts FAR more than matching
  // common ones ("how"/"to"). covfac = FLOOR + (1024-FLOOR)*matched_idf/total_idf; FULL idf coverage -> 1024
  // (byte-identical, so single-term queries + full matches -- incl. the #1 canonical entity -- are UNCHANGED).
  // The nx_web_relevance_bench ruler demanded this: every loss was an NL query where function-word/boilerplate
  // matches out-ranked salient-term content (blood-pressure -> nasa how-to; nsf -> spam). Multi-term NON-phrase.
  if nph < 2 { if nterms >= 2 {
      var totidf: i64 = 0; var ti: i64 = 0
      while ti < nterms { if exclmask[ti] == 0 { totidf = totidf + idf[ti] } ti = ti + 1 }
      if totidf > 0 {
          var cc: i64 = 0
          while cc < ncand {
              if cand_score[cc] > 0 { if cand_tmask[cc] != 0 {
                  var mi: i64 = 0; var mt: i64 = 0
                  while mt < nterms { if (cand_tmask[cc] & (1 << mt)) != 0 { if exclmask[mt] == 0 { mi = mi + idf[mt] } } mt = mt + 1 }
                  if mi < totidf {
                      let covfac: i64 = DSS_COV_FLOOR + ((DSS_MAGIC_1024 - DSS_COV_FLOOR) * mi) / totidf
                      cand_score[cc] = (cand_score[cc] * covfac) / DSS_MAGIC_1024
                  }
              } }
              cc = cc + 1
          }
      }
  } }
  // P1 AUTHORITY FUSION (web scope only): multiply each candidate's BM25 by a PageRank prior. Resolve the page
  // node: content_cid -> url:<cid> -> ci_hash(url) [== the crawler's out:/pr: node id] -> pr:<cid>. rank in
  // [0,DSS_PR_SCALE] ppb -> factor = 1024 + rank*BOOST/SCALE (capped DSS_PR_MAXFAC). No pr: row (or non-web) = 1x,
  // so this is ADDITIVE authority ON TOP of relevance -- and makes the host-diversity cap a FLOOR, not the signal.
  if webdiv == 1 {
      let aukey: *u8 = sys_mmap(64)
      let aup: *i64 = sys_mmap(16) as *i64
      let aul: *i64 = sys_mmap(16) as *i64
      let apkey: *u8 = sys_mmap(64)
      let app: *i64 = sys_mmap(16) as *i64
      let apl: *i64 = sys_mmap(16) as *i64
      // TOPICAL-RELEVANCE FLOOR (seq636 fix, 2026-07-23): authority may only boost docs that matched a
      // CONTENT (non-stopword) query term. Otherwise a high-authority domain (science.nasa.gov "How To
      // Guide", wikipedia, bbc) matching ONLY function words (how/to) rode a 4x boost above the actual
      // hypertension pages. contentmask = bits of non-excluded terms that are NOT corpus-derived stopwords
      // (dcnt <= stopdf). Authority now breaks TIES AMONG RELEVANT docs, never overrides relevance. In the
      // gate/onsite corpora stopword detection is off (stopdf=0) -> every term is content -> the floor is a
      // no-op = byte-identical there. Phrase candidates (tmask=0) keep authority (phrase already ensures
      // relevance).
      var contentmask: i64 = 0
      var totcidf: i64 = 0
      var tcm: i64 = 0
      while tcm < nterms {
          if exclmask[tcm] == 0 {
              var iscontent: i64 = 1
              if stopdf > 0 { if dcnt[tcm] > stopdf { iscontent = 0 } }
              // R1f DENOMINATOR FIX (2026-08-05): totcidf once summed EVERY term incl. the
              // stem-expanded variants, so on natural-lang queries no doc could reach
              // DSS_AUTH_IDF_MIN of the mass and the whole authority/trust branch was DEAD CODE
              // (proven by two byte-invisible deploys, 2026-08-04). The OBLIGATION (denominator)
              // now covers only the ORIGINAL terms; the CREDIT (contentmask -> mcidf) still spans
              // variants, so a doc can earn the floor by matching a variant. Pattern queries and
              // the gate corpora do not stem-expand -> byte-identical there.
              if iscontent == 1 { contentmask = contentmask | (1 << tcm); if tcm < r1g_onterms { totcidf = totcidf + idf[tcm] } }
          }
          tcm = tcm + 1
      }
      // R2c trust-prior host list: parse each seed line's host once per query (fork-per-request
      // children; ~2KB file; ss_loadfile absent/empty -> ntshost=0 -> the prior is inert).
      let tsz: *i64 = sys_mmap(16) as *i64
      tsz[0] = 0
      let tsbuf: *u8 = ss_loadfile("knowledge/status/canonical_seeds.txt" as *u8, tsz, 0)
      let tshosts: *u8 = sys_mmap(DSS_TRUST_MAXH * 64)
      var ntshost: i64 = 0
      if (tsbuf as i64) != 0 {
          var tp: i64 = 0
          while tp < tsz[0] {
              var te: i64 = tp
              var tgo: i64 = 1
              while tgo == 1 {
                  if te >= tsz[0] { tgo = 0 } else { if tsbuf[te] == (10 as u8) { tgo = 0 } else { te = te + 1 } }
              }
              if te > tp { if ntshost < DSS_TRUST_MAXH { if tsbuf[tp] != (35 as u8) {
                  let td: *u8 = (tshosts as i64 + ntshost * 64) as *u8
                  let thl: i64 = dss_url_host(((tsbuf as i64) + tp) as *u8, te - tp, td)
                  if thl > 0 { if thl < 60 { ntshost = ntshost + 1 } }
              } } }
              tp = te + 1
          }
      }
      let stscr: *u8 = sys_mmap(256)   // R2d: host-label scratch for the brand exemption
      var ca: i64 = 0
      while ca < ncand {
          // R2d STUFFING PENALTY -- RETIRED UNWIRED after a full two-round ruler campaign
          // (2026-08-04). Round 1 (no exemption): spam halved but MRR@10 368->297, navigational
          // brand pages are dense in their own name. Round 2 (brand-label exemption, T42): nav
          // healed only to 429/532 and the per-query ranks named the terminal class -- kernel.org
          // is legitimately dense in "linux" (rank 20!), londonmet.ac.uk in "london" (compound
          // label), a Lovelace biography in "lovelace". MEASURED CONCLUSION: query-term density
          // CANNOT separate stuffing from topical authority at any reachable threshold; the
          // legitimate hub/brand/biography class occupies the spam's density band. The anti-slop
          // judgment belongs at INGEST (content-quality axes: ad/affiliate density, template
          // mass) or to authority priors -- NOT at serve over query terms. dss_stuff_factor,
          // dss_host_label_match, T41/T42 and nx_doc_lexstat stay as the campaign's evidence.
          var authok: i64 = 1
          // R1f SALIENT-IDF authority floor: earn the PR boost only by matching >= DSS_AUTH_IDF_MIN of the
          // query's CONTENT idf-mass (a peripheral-term-only match on a high-authority domain forfeits it).
          if nph < 2 { if contentmask != 0 {
              if totcidf > 0 {
                  var mcidf: i64 = 0; var mc: i64 = 0
                  while mc < nterms { if (cand_tmask[ca] & contentmask & (1 << mc)) != 0 { mcidf = mcidf + idf[mc] } mc = mc + 1 }
                  if mcidf * DSS_MAGIC_1024 < totcidf * DSS_AUTH_IDF_MIN { authok = 0 }
              } else { if (cand_tmask[ca] & contentmask) == 0 { authok = 0 } }
          } }
          if cand_score[ca] > 0 { if authok == 1 {
              var rank: i64 = 0
              var searchpen: i64 = DSS_MAGIC_1024   // R1e: DSS_MAGIC_1024 = content; DSS_SEARCHPAGE_PEN if a search/query-echo URL
              var slugf: i64 = DSS_MAGIC_1024       // R1g: DSS_SLUG_BOOST if a matched content term is a URL path segment
              dss_mkurlkey(cand_cid[ca], aukey)
              if ss_hget(h, aukey, aup, aul) == 1 { if aul[0] > 0 {
                  if dss_is_search_url(aup[0] as *u8, aul[0]) == 1 { searchpen = DSS_SEARCHPAGE_PEN }
                  // ENTITY-SHAPE GATE (measured 2026-07-24): the prior is for NAME lookups (1-2 terms);
                  // informational 3+-term queries ("malawi household survey") regressed under it -- in this
                  // skewed 150k corpus even "survey"/"university" pass a df-rarity test, so query SHAPE is
                  // the reliable entity signal, rarity the secondary one.
                  if searchpen == DSS_MAGIC_1024 { if r1g_onterms <= 2 {
                      var st: i64 = 0
                      while st < nterms {
                          if slugf == DSS_MAGIC_1024 { if (cand_tmask[ca] & contentmask & (1 << st)) != 0 {
                              if dcnt[st] * DSS_SLUG_RAREK < bign {
                                  if dss_slug_match_term(aup[0] as *u8, aul[0], termptrs[st] as *u8) == 1 { slugf = DSS_SLUG_BOOST }
                              }
                          } }
                          st = st + 1
                      }
                  } }
                  let cidu: i64 = dss_urlcid(aup[0] as *u8, aul[0])
                  dss_prkey(cidu, apkey)
                  if ss_hget(h, apkey, app, apl) == 1 { if apl[0] == 8 {
                      let vp: *i64 = app[0] as *i64
                      rank = vp[0]
                  } }
              } }
              var authf: i64 = DSS_MAGIC_1024 + (rank * DSS_PR_BOOST) / DSS_PR_SCALE
              if authf > DSS_PR_MAXFAC { authf = DSS_PR_MAXFAC }
              cand_score[ca] = (cand_score[ca] * authf) / DSS_MAGIC_1024
              // R2c TRUST PRIOR: bounded multiply for a curated canonical host (same authok gate
              // as the PR boost; url row already in hand -- zero extra reads; dot-suffix match)
              if ntshost > 0 { if ss_hget(h, aukey, aup, aul) == 1 { if aul[0] > 0 {
                  var th9: i64 = 0
                  while th9 < ntshost {
                      if dss_url_host_match(aup[0] as *u8, aul[0], (tshosts as i64 + th9 * 64) as *u8) == 1 {
                          cand_score[ca] = (cand_score[ca] * DSS_TRUST_BOOST) / DSS_MAGIC_1024
                          th9 = ntshost
                      } else { th9 = th9 + 1 }
                  }
              } } }
              // R1e SEARCH-PAGE DE-RANK: half-weight a query-echo URL so real content outranks it when it exists.
              if searchpen != DSS_MAGIC_1024 { cand_score[ca] = (cand_score[ca] * searchpen) / DSS_MAGIC_1024 }
              // R1g URL-SLUG ENTITY PRIOR: pages ABOUT the entity (term = URL path segment) beat mere mentions.
              if slugf != DSS_MAGIC_1024 { cand_score[ca] = (cand_score[ca] * slugf) / DSS_MAGIC_1024 }
          } }
          ca = ca + 1
      }
  }
  // DROP candidates with NO POSITIVE relevance (score <= 0). A score of 0 means the term(s) the postings
  // said this doc matched were either past the tf-scan cap (a deep, weak match in a huge doc -- which BM25
  // length-norm penalizes anyway) OR only zero-idf terms (in every doc, no discriminative value). Either
  // way it is NOISE, not a result -- this was the "julia kyoka returns Dance/YouTube/Quaternion at score 0"
  // bug (2026-07-03, operator-reported). Required/excluded failures (sc=-1) drop here too. totalout[0] is
  // recomputed to the surviving POSITIVE count so "N matched" is honest (no phantom score-0 results).
  var wk2: i64 = 0
  var rc3: i64 = 0
  while rc3 < ncand {
      if cand_score[rc3] > 0 {
          cand_cid[wk2] = cand_cid[rc3]
          cand_score[wk2] = cand_score[rc3]
          wk2 = wk2 + 1
      }
      rc3 = rc3 + 1
  }
  // honest-total: subtract the dropped phantoms from the running matched estimate (never reset to the
  // shortlist remnant); floor at the surviving count so the page is never larger than its own total
  matched_total = matched_total - (ncand - wk2)
  ncand = wk2
  if ncand == 0 { return 0 }
  if matched_total < ncand { matched_total = ncand }
  totalout[0] = matched_total
  // HOST-DIVERSITY CAP (web scope only): precompute each surviving candidate's host fingerprint so one
  // crawl-heavy host cannot dominate a page (the "trust law -> 9 plato.stanford.edu" skew of a small,
  // seed-biased web index). 0 = onsite/trusted shard OR a doc with no url host -> never capped. Serve-time
  // rerank only: totals + scores are unchanged, and over-cap hits still appear (phase B) = down-rank not
  // delete (the neutrality charter). At real web scale (P1) authority + anti-slop-at-ingest supersede this.
  // webdiv is a CALLER-KNOWN flag (the serve layer knows the scope); dss_search_off auto-detects "web".
  let cand_hh: *i64 = sys_mmap(DSS_MAXCAND * 8) as *i64
  var hz: i64 = 0
  while hz < ncand { cand_hh[hz] = 0; hz = hz + 1 }
  let cand_uc: *i64 = sys_mmap(DSS_MAXCAND * 8) as *i64
  if webdiv == 1 {
      let hukey: *u8 = sys_mmap(64)
      let hup: *i64 = sys_mmap(16) as *i64
      let hul: *i64 = sys_mmap(16) as *i64
      let hscr: *u8 = sys_mmap(256)
      var hc0: i64 = 0
      while hc0 < ncand {
          cand_uc[hc0] = 0
          dss_mkurlkey(cand_cid[hc0], hukey)
          if ss_hget(h, hukey, hup, hul) == 1 { if hul[0] > 0 {
              cand_hh[hc0] = dss_hosthash(hup[0] as *u8, hul[0], hscr)
              cand_uc[hc0] = dss_urlcid(hup[0] as *u8, hul[0])
          } }
          hc0 = hc0 + 1
      }
      // URL DEDUP (web scope only): a re-crawled page leaves MULTIPLE content snapshots (distinct content
      // cids, SAME url row value). One page = one result: keep only the best-scored snapshot per full-url
      // cid (score tie -> first candidate), and recount totals to DISTINCT pages (the operator-visible
      // "Trust_law at #2 AND #3" dup, 2026-07-14). Flags are decided over the PRISTINE arrays first, THEN
      // one compaction pass -- deciding while compacting would compare against already-moved slots.
      let keepf: *u8 = sys_mmap(DSS_MAXCAND)
      var d0: i64 = 0
      while d0 < ncand {
          var keep: i64 = 1
          if cand_uc[d0] != 0 {
              var d1: i64 = 0
              while d1 < ncand {
                  if d1 != d0 { if cand_uc[d1] == cand_uc[d0] {
                      if cand_score[d1] > cand_score[d0] { keep = 0 }
                      if cand_score[d1] == cand_score[d0] { if d1 < d0 { keep = 0 } }
                  } }
                  d1 = d1 + 1
              }
          }
          if keep == 1 { keepf[d0] = 1 as u8 } else { keepf[d0] = 0 as u8 }
          d0 = d0 + 1
      }
      var wk4: i64 = 0
      var d2: i64 = 0
      while d2 < ncand {
          if keepf[d2] == (1 as u8) {
              cand_cid[wk4] = cand_cid[d2]
              cand_score[wk4] = cand_score[d2]
              cand_hh[wk4] = cand_hh[d2]
              wk4 = wk4 + 1
          }
          d2 = d2 + 1
      }
      // honest-total: dedup drops subtract from the running matched estimate (see the candidacy note)
      matched_total = matched_total - (ncand - wk4)
      ncand = wk4
      if matched_total < ncand { matched_total = ncand }
      totalout[0] = matched_total
  }
  // select ranks [offset, offset+max) by score desc, honoring the per-host cap (selection; sets are small)
  let selhh:  *i64 = sys_mmap(DSS_MAXCAND * 8) as *i64
  let selcnt: *i64 = sys_mmap(DSS_MAXCAND * 8) as *i64
  var nsel: i64 = 0
  let used: *u8 = sys_mmap(DSS_MAXCAND)
  var u: i64 = 0
  while u < ncand { used[u] = 0 as u8; u = u + 1 }
  var rank: i64 = 0
  var out_n: i64 = 0
  while rank < offset + max {
      // PHASE A: highest-scored unused candidate whose host is still under DSS_HOSTCAP on this page.
      var best: i64 = 0 - 1
      var bestsc: i64 = 0 - 1
      var c3: i64 = 0
      while c3 < ncand {
          if used[c3] == (0 as u8) { if cand_score[c3] > bestsc {
              var okcap: i64 = 1
              if cand_hh[c3] != 0 {
                  var sk: i64 = 0
                  while sk < nsel { if selhh[sk] == cand_hh[c3] { if selcnt[sk] >= DSS_HOSTCAP { okcap = 0 } } sk = sk + 1 }
              }
              if okcap == 1 { bestsc = cand_score[c3]; best = c3 }
          } }
          c3 = c3 + 1
      }
      // PHASE B (overflow fallback): the cap left nothing selectable -> take the best unused of ANY host,
      // so a dominant host still fills the page once diversity is exhausted (never returns fewer results).
      if best < 0 {
          var bs2: i64 = 0 - 1
          var c4: i64 = 0
          while c4 < ncand {
              if used[c4] == (0 as u8) { if cand_score[c4] > bs2 { bs2 = cand_score[c4]; best = c4 } }
              c4 = c4 + 1
          }
      }
      if best < 0 { return out_n }
      used[best] = 1 as u8
      if cand_hh[best] != 0 {
          var fk: i64 = 0 - 1
          var sk2: i64 = 0
          while sk2 < nsel { if selhh[sk2] == cand_hh[best] { fk = sk2 } sk2 = sk2 + 1 }
          if fk < 0 { selhh[nsel] = cand_hh[best]; selcnt[nsel] = 1; nsel = nsel + 1 }
          else { selcnt[fk] = selcnt[fk] + 1 }
      }
      if rank >= offset {
          cids_out[out_n] = cand_cid[best]
          scores_out[out_n] = cand_score[best]
          out_n = out_n + 1
      }
      rank = rank + 1
  }
  return out_n
1847}
1848// contract-stable 8-arg entry (rule 19): auto-detects web scope from the shard name so EVERY caller (serve,
1849// api, census) gets host-crowding diversity on the "web" shard with ZERO call-site changes; site/trusted
1850// pass webdiv=0 and stay byte-identical. A caller that already knows its scope can call ..._div directly.
1851func dss_search_off(domain: *u8, q: *u8, qn: i64, cids_out: *i64, scores_out: *i64, max: i64, offset: i64, totalout: *i64) -> i64 {
  return dss_search_off_div(domain, q, qn, cids_out, scores_out, max, offset, totalout, dss_is_web(domain))
1853}
1854// contract-stable wrapper: the original signature, rank 0, total discarded (rule 19: additive evolution)
1855func dss_search(domain: *u8, q: *u8, qn: i64, cids_out: *i64, scores_out: *i64, max: i64) -> i64 {
  // tb0 is a scratch OUT-param for the total; the callee writes it and this wrapper discards it, so
  // it is dead the moment dss_search_off returns. Free after the call, never before.
  let tb0: *i64 = sys_mmap(DSC_POSBUF) as *i64
  let r: i64 = dss_search_off(domain, q, qn, cids_out, scores_out, max, 0, tb0)
  sys_munmap(tb0 as *u8, DSC_POSBUF)
  return r
1862}
nx_docportal_search_seg.nx source

`nx_docportal_search_seg.nx` source
