code wiki / _hdl_build / nx_qrels_bench.nx
`nx_qrels_bench.nx` source

↩ module page · 773 lines · 45601 B
1// nx_qrels_bench.nx -- THE PER-DOCUMENT RELEVANCE RULER (Q7 runner, 2026-09-04).
2//
3// WHY IT EXISTS. The estate's incumbent web ruler judges by EXPECTED HOST, and that is not a detail --
4// it is why it could not see any defect found on 2026-09-04. "Icing (aeronautics)" at rank 1 for the
5// query "ice cream" sits on en.wikipedia.org and therefore scored a PERFECT hit@1, while the stored
6// document does not contain the word "cream" anywhere. A HOST IS NOT AN ANSWER. This runner scores
7// against knowledge/search_qrels.conf, whose judgements are per URL.
8//
9// COMPOSED, NOT REIMPLEMENTED, ON BOTH SIDES:
10//   retrieval -- dss_search over the SAME code path /search serves, so the number describes the live
11//                engine rather than a reimplementation of it. Deliberately NO HTTP: an organ that fetches
12//                its own daemon measures the transport too, and would queue behind the serve it is grading.
13//   metrics   -- nx_ir_eval (ie_ndcg_at_k, ie_mrr, ie_hits_at_k, ie_precision_at_k, ie_ap), the estate's
14//                Q16.16 integer-exact floor. A second metric implementation is the duplicate-ruler defect.
15//
16// SCOPE, INHERITED FROM THE QRELS FILE AND NOT WEAKENED HERE: that set is a BOOTSTRAP -- its candidates
17// were pooled from the engine's own required-term arm, so it measures whether known-good documents rank UP
18// and it CANNOT prove recall over documents that arm never retrieved. The recall figure is therefore recall
19// over the JUDGED POOL, and it is emitted as judged_recall_permil so nobody reads it as the other thing.
20//
21// GRADE 0 IS A JUDGEMENT, NOT A GAP. A result absent from the file scores 0 deliberately: the file records
22// every document examined per query, so "unlisted" means "examined and judged irrelevant" -- which is
23// exactly how a stem-variant-only match (icing for ice, warring for wars) is meant to score.
24import "nx_docportal_search_seg.nx"
25import "nx_ir_eval.nx"
26import "nx_gate_verdict.nx"
27
28const QB_PATH:    *u8 = "knowledge/search_qrels.conf"
29const QB_PATH_UP: *u8 = "../knowledge/search_qrels.conf"   // same file seen from a buildroot CWD
30const QB_DOMAIN:  *u8 = "web"
31const QB_MAXQ:     i64 = 64
32const QB_MAXREL:   i64 = 1024
33const QB_K:        i64 = 10       // DSS_MAXR is 30, so scoring @5 would discard two thirds of the page
34const QB_TOPN:     i64 = 30
35const QB_QBUF:     i64 = 128
36const QB_UBUF:     i64 = 512
37const QB_PATHCAP:  i64 = 1024
38const QB_BOX:      i64 = 16
39const QB_PIPE:     i64 = 124
40const QB_NL:       i64 = 10
41const QB_CR:       i64 = 13       // CRLF-generated data files leave this as the last byte of a field
42const QB_HASH:     i64 = 35
43const QB_D0:       i64 = 48
44const QB_D9:       i64 = 57
45const QB_B10:      i64 = 10
46const QB_PERMIL:   i64 = 1000
47const QB_Q16:      i64 = 65536    // nx_ir_eval returns Q16.16 fixed point, NOT permil (mrr=65536 == 1.0)
48const QB_FAMCAP:   i64 = 32       // a family is a COARSE signal class (lexical/human/neural), not a label
49const QB_LC_AUTO:  i64 = 0
50const QB_LC_HUMAN: i64 = 1
51const QB_SP:          i64 = 32
52const QB_CONTENT_MIN: i64 = 3           // S4: a CONTENT term. 1-2 char tokens in these queries are
                                      // function words, and counting them would let a rewrite score
                                      // alignment for preserving "of" while dropping "carburettor".
55const QB_CROSSOVER_TOKENS: i64 = 10000000   // S1: the published BM25 scale crossover, in corpus tokens
56const QB_EPOCH_FLOOR: i64 = 1750000000  // S2: below this is not a date, it is an UNSET CLOCK. A run
                                      // stamped 0 must fail benchmark identity rather than publish a
                                      // score that claims to have been taken at the epoch.
59
60func qb_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
61func qb_eqz(a: *u8, b: *u8) -> i64 {
  var i: i64 = 0
  var ok: i64 = 1
  var go: i64 = 1
  while go == 1 {
      if a[i] == (0 as u8) { go = 0 } else {
          if a[i] != b[i] { ok = 0; go = 0 } else { i = i + 1 }
      }
  }
  if ok == 1 { if b[i] != (0 as u8) { ok = 0 } }
  return ok
72}
73func qb_cpn(dst: *u8, cap: i64, src: *u8, n: i64) -> i64 {
  var k: i64 = n
  if k > cap - 1 { k = cap - 1 }
  var i: i64 = 0
  while i < k { dst[i] = src[i]; i = i + 1 }
  dst[k] = 0 as u8
  return k
80}
81// END OF LINE via an explicit flag, never by clobbering the cursor: a loop that exits by overwriting its
82// own index cannot also report where it stopped, and that idiom has cost this estate four defects.
83// END OF LINE, CR-TRIMMED. The CR is not a nicety: the judged set was generated on Windows, so every
84// record ends \r\n, and a scanner that stops only at \n leaves the CR as the LAST BYTE OF THE URL FIELD.
85// MEASURED, and it is the reason this ruler read a perfect zero on its first green run: retrieved and
86// judged urls printed CHARACTER-FOR-CHARACTER IDENTICAL in the diagnostic and compared UNEQUAL, because
87// gv_puts renders a trailing CR as a carriage return -- invisible in a log, decisive in a memcmp. The
88// index fields were all correct (rqi0=0 rgr0=2 qi=0 nr=174) and only streq=0 disagreed, which is what
89// localised it to the string and not to the bookkeeping.
90// ★★★★★★TWO STRINGS THAT PRINT IDENTICALLY CAN STILL DIFFER, AND A DIAGNOSTIC THAT ONLY PRINTS THEM WILL
91// AGREE WITH THE WRONG ANSWER -- PRINT THE LENGTH AND THE COMPARISON'S OWN VERDICT, NOT JUST THE VALUE.
92// Trimming here rather than at the generator is deliberate: a data file may arrive from any host, and a
93// parser that tolerates CRLF cannot be broken again by whoever next edits the set on a different OS.
94func qb_eol(buf: *u8, n: i64, i: i64) -> i64 {
  var e: i64 = i
  var go: i64 = 1
  while go == 1 {
      if e >= n { go = 0 } else { if buf[e] == (QB_NL as u8) { go = 0 } else { e = e + 1 } }
  }
  if e > i { if buf[e - 1] == (QB_CR as u8) { e = e - 1 } }
  return e
102}
103// offset of the next '|' at or after i, bounded by e
104func qb_pipe(buf: *u8, e: i64, i: i64) -> i64 {
  var p: i64 = i
  var go: i64 = 1
  while go == 1 {
      if p >= e { go = 0 } else { if buf[p] == (QB_PIPE as u8) { go = 0 } else { p = p + 1 } }
  }
  return p
111}
112func qb_atoi(buf: *u8, s: i64, e: i64) -> i64 {
  var v: i64 = 0
  var i: i64 = s
  while i < e {
      let c: i64 = buf[i] as i64
      if c >= QB_D0 { if c <= QB_D9 { v = v * QB_B10 + (c - QB_D0) } }
      i = i + 1
  }
  return v
121}
122// WHOLE-WORD CONTAINMENT. Needed by S4 because "dog" must not be satisfied by "dogs": the whole point of
123// a lexical alignment figure is that it counts the terms that genuinely survived, and a substring test
124// would score a rewrite for a word it did not keep. Boundaries are non-alphanumeric on both sides.
125func qb_isword(c: i64) -> i64 {
  if c >= 48 { if c <= 57 { return 1 } }
  if c >= 65 { if c <= 90 { return 1 } }
  if c >= 97 { if c <= 122 { return 1 } }
  return 0
130}
131func qb_hasword(hay: *u8, needle: *u8) -> i64 {
  let hn: i64 = qb_slen(hay)
  let nn: i64 = qb_slen(needle)
  if nn == 0 { return 0 }
  if nn > hn { return 0 }
  var i: i64 = 0
  while i <= hn - nn {
      var j: i64 = 0
      var same: i64 = 1
      while j < nn { if hay[i + j] != needle[j] { same = 0; j = nn } else { j = j + 1 } }
      if same == 1 {
          var lok: i64 = 1
          var rok: i64 = 1
          if i > 0 { if qb_isword(hay[i - 1] as i64) == 1 { lok = 0 } }
          if i + nn < hn { if qb_isword(hay[i + nn] as i64) == 1 { rok = 0 } }
          if lok == 1 { if rok == 1 { return 1 } }
      }
      i = i + 1
  }
  return 0
151}
152// S1 CONTRACT SYMBOL cs_crossover -- WHICH SIDE OF THE PUBLISHED BM25 SCALE CROSSOVER THIS CORPUS SITS ON.
153// The finding it encodes: BM25 overtakes agentic and dense retrieval at roughly 10M corpus tokens and
154// leads at every larger tier. That is load-bearing for this estate, because it says a lexical engine is
155// the RIGHT default at our scale rather than a placeholder for a model we have not built.
156// THREE STATES, NOT TWO. -1 is UNDECIDABLE and it is returned whenever either input is unmeasured, so an
157// unknown corpus size can never be silently reported as BELOW -- abstain, never acquit. The threshold is
158// a parameter rather than a constant precisely so the negative control can plant a corpus on either side.
159// ⚠SCOPE CAVEAT, CARRIED BESIDE THE NUMBER AS THE CONTRACT REQUIRES: the crossover is reported by its
160// authors over their own corpora and task mix; it is a published landmark, not a law of this index.
161// cs_crossover LIVES IN nx_intlog.nx since 2026-09-14 (the S1 contract symbol, reached through nx_docportal_search_seg's
162// import) with the same three states; a second copy here collided with it on rebuild -- ONE ruler.
163// S6 CONTRACT SYMBOL qb_tier -- THE LABEL TIER OF A JUDGED SET. 1 when the set carries at least one HUMAN
164// label, 0 when it is auto-only. It is deliberately SEPARATE from the publish decision, because the tier
165// is a property of the DATA while publishability also depends on WHICH RANKER this run measured; folding
166// them would make a property of the labels depend on the subject under test. It is on the critical path,
167// not a name written to satisfy a watch contract: the publish decision below consumes its return value.
168func qb_tier(rlc: *i64, nr: i64) -> i64 {
  var i: i64 = 0
  while i < nr {
      if rlc[i] == QB_LC_HUMAN { return 1 }
      i = i + 1
  }
  return 0
175}
176// S2 CONTRACT SYMBOL qb_benchver -- BENCHMARK IDENTITY. A retrieval score without the benchmark it was
177// taken on and the date it was taken is not comparable with anything, and it is the shape in which stale
178// numbers survive: "nDCG 581" outlives the index, the qrels revision and the scorer that produced it.
179// Returns 1 only when BOTH are present. The epoch floor is what makes this non-vacuous -- an unset clock
180// reads as 0 and would otherwise pass as "a date".
181func qb_benchver(bench: *u8, epoch: i64) -> i64 {
  if qb_slen(bench) == 0 { return 0 }
  if epoch < QB_EPOCH_FLOOR { return 0 }
  return 1
185}
186// S4 CONTRACT SYMBOL qb_lexalign -- LEXICAL ALIGNMENT of a rewrite, in permil, computed with NO MODEL.
187// Of the original query CONTENT terms, what fraction survived into the rewrite AND actually occurs in a
188// judged-relevant document? Both conjuncts are required on purpose: surviving into the rewrite alone
189// rewards a rewrite that keeps a term the corpus does not contain, and occurring in the document alone
190// rewards a rewrite that dropped it. Published evidence for why this is gated at all: LLM query rewriting
191// made the STRONGEST model worse on every metric with every technique, so a rewrite must EARN its place.
192// Returns 0 when there are no content terms -- a figure over an empty denominator is not a measurement.
193func qb_lexalign(orig: *u8, rewrite: *u8, doctext: *u8) -> i64 {
  let n: i64 = qb_slen(orig)
  var total: i64 = 0
  var kept:  i64 = 0
  var i: i64 = 0
  while i < n {
      // SEPARATOR SKIP AND TOKEN SCAN VIA AN EXPLICIT FLAG, NEVER A SENTINEL WRITTEN INTO THE CURSOR.
      // The first cut of this function exited both loops with i = n + n and then subtracted n back --
      // which is the idiom this estate has recorded FOUR separate defects against, because a loop that
      // exits by clobbering its own index cannot also report where it stopped. qb_eol above already
      // uses the flag form; matching it is the whole point of having one.
      var go: i64 = 1
      while go == 1 {
          if i >= n { go = 0 } else { if orig[i] == (QB_SP as u8) { i = i + 1 } else { go = 0 } }
      }
      var e: i64 = i
      go = 1
      while go == 1 {
          if e >= n { go = 0 } else { if orig[e] == (QB_SP as u8) { go = 0 } else { e = e + 1 } }
      }
      if e > i {
          if e - i >= QB_CONTENT_MIN {
              let tok: *u8 = sys_mmap(QB_QBUF)
              qb_cpn(tok, QB_QBUF, ((orig as i64) + i) as *u8, e - i)
              total = total + 1
              if qb_hasword(rewrite, tok) == 1 {
                  if qb_hasword(doctext, tok) == 1 { kept = kept + 1 }
              }
              sys_munmap(tok, QB_QBUF)
          }
      }
      i = e + 1
  }
  if total == 0 { return 0 }
  return (kept * QB_PERMIL) / total
228}
229// S6 (2026-09-04): THE PUBLISH DECISION, DELIBERATELY A PURE FUNCTION OF ALREADY-MEASURED VALUES so a
230// tooth can drive it with PLANTED inputs without touching the index, the conf or the network -- the
231// controls below therefore cannot pass while the live run takes some other path, because there is only
232// one of it. It answers ONE question: may this run publish an ABSOLUTE relevance metric? It says no in
233// two cases, and the caller NAMES which one fired.
234//   CIRCULAR   judge_family == ranker_family. A judged set graded by the same signal class the ranker
235//              ranks by measures TRANSCRIPTION, not relevance, and it flatters precisely the change that
236//              leans hardest on that signal. MEASURED ON THIS ESTATE 2026-09-04, which is why this exists:
237//              the set was graded "every query term present as a whole word" -- lexical -- against a
238//              lexical BM25 ranker. It graded a porn-spam page RELEVANT for the query "hot dog" and a
239//              Streamlit tutorial HIGHLY relevant, and a scorer change that raised its mean by 28 permil
240//              simultaneously made a user-visible result worse. The number rose while the engine got
241//              worse. That is what circularity buys, and no amount of extra queries fixes it.
242//   AUTO-ONLY  the set carries no human label at all. An automatic judgement is a research object, not a
243//              substitute for a human qrel, so an absolute figure drawn only from auto labels is withheld.
244// THE FAMILY IS COARSE ON PURPOSE. "lexical-termpresence" and "lexical-bm25" are different STRINGS and the
245// same FAMILY, so an exact match on the full label would have cleared the very case this was built to
246// catch. Circularity is a property of the signal class, never of the spelling.
247func qb_publish_ok(judgefam: *u8, rankerfam: *u8, tier: i64) -> i64 {
  if qb_slen(judgefam) == 0 { return 0 }
  if qb_slen(rankerfam) == 0 { return 0 }
  if qb_eqz(judgefam, rankerfam) == 1 { return 0 }
  if tier <= 0 { return 0 }
  return 1
253}
254// THE PARSER. Records: q|<query>|<class>|<pool>  and  rel|<query>|<grade>|<url>[|<label_class>]
255// plus the S6 provenance header rows judge|<family> and ranker|<family>.
256// Comments and blanks are skipped AND COUNTED -- a parser that silently drops rows turns a shrinking
257// judged set into a rising score, which is the flattering direction nobody audits.
258func qb_load(buf: *u8, n: i64, qtext: *u8, nq: *i64,
           rqi: *i64, rgr: *i64, rurl: *u8, rlc: *i64, nr: *i64, skipped: *i64,
           judgefam: *u8, rankerfam: *u8, benchstr: *u8) -> i64 {
  nq[0] = 0
  nr[0] = 0
  skipped[0] = 0
  judgefam[0] = 0 as u8
  rankerfam[0] = 0 as u8
  benchstr[0] = 0 as u8
  var i: i64 = 0
  while i < n {
      let e: i64 = qb_eol(buf, n, i)
      if e > i {
          if buf[i] != (QB_HASH as u8) {
              let p1: i64 = qb_pipe(buf, e, i)
              if p1 < e {
                  let kind: *u8 = sys_mmap(QB_BOX)
                  qb_cpn(kind, QB_BOX, ((buf as i64) + i) as *u8, p1 - i)
                  let p2: i64 = qb_pipe(buf, e, p1 + 1)
                  if qb_eqz(kind, "q") == 1 {
                      if nq[0] < QB_MAXQ {
                          let dst: *u8 = ((qtext as i64) + nq[0] * QB_QBUF) as *u8
                          qb_cpn(dst, QB_QBUF, ((buf as i64) + p1 + 1) as *u8, p2 - p1 - 1)
                          nq[0] = nq[0] + 1
                      }
                  } else { if qb_eqz(kind, "rel") == 1 {
                      let p3: i64 = qb_pipe(buf, e, p2 + 1)
                      let p4: i64 = qb_pipe(buf, e, p3 + 1)
                      if nr[0] < QB_MAXREL {
                          // resolve the query name to its index; a rel row for an unknown query is
                          // DROPPED AND COUNTED rather than attached to whatever was parsed last.
                          let qn: *u8 = sys_mmap(QB_QBUF)
                          qb_cpn(qn, QB_QBUF, ((buf as i64) + p1 + 1) as *u8, p2 - p1 - 1)
                          var qi: i64 = 0 - 1
                          var t: i64 = 0
                          while t < nq[0] {
                              if qb_eqz(qn, ((qtext as i64) + t * QB_QBUF) as *u8) == 1 { qi = t }
                              t = t + 1
                          }
                          sys_munmap(qn, QB_QBUF)
                          if qi >= 0 {
                              rqi[nr[0]] = qi
                              rgr[nr[0]] = qb_atoi(buf, p2 + 1, p3)
                              qb_cpn(((rurl as i64) + nr[0] * QB_UBUF) as *u8, QB_UBUF,
                                     ((buf as i64) + p3 + 1) as *u8, p4 - p3 - 1)
                              // LABEL CLASS is the optional 5th field and ABSENT MEANS auto. That
                              // default is a MEASUREMENT, not a convenience: every row in the shipped
                              // set was emitted by a lexical rule, so auto is what those rows ARE.
                              // p4 == e when the field is absent, so the url extent above is unchanged
                              // for every pre-existing row -- this is additive, per the contract rule.
                              var lc: i64 = QB_LC_AUTO
                              if p4 < e {
                                  let lcb: *u8 = sys_mmap(QB_BOX)
                                  qb_cpn(lcb, QB_BOX, ((buf as i64) + p4 + 1) as *u8, e - p4 - 1)
                                  if qb_eqz(lcb, "human") == 1 { lc = QB_LC_HUMAN }
                                  sys_munmap(lcb, QB_BOX)
                              }
                              rlc[nr[0]] = lc
                              nr[0] = nr[0] + 1
                          } else { skipped[0] = skipped[0] + 1 }
                      }
                  } else { if qb_eqz(kind, "judge") == 1 {
                      qb_cpn(judgefam, QB_FAMCAP, ((buf as i64) + p1 + 1) as *u8, p2 - p1 - 1)
                  } else { if qb_eqz(kind, "ranker") == 1 {
                      qb_cpn(rankerfam, QB_FAMCAP, ((buf as i64) + p1 + 1) as *u8, p2 - p1 - 1)
                  } else { if qb_eqz(kind, "bench") == 1 {
                      qb_cpn(benchstr, QB_QBUF, ((buf as i64) + p1 + 1) as *u8, p2 - p1 - 1)
                  } else { skipped[0] = skipped[0] + 1 } } } } }
                  sys_munmap(kind, QB_BOX)
              } else { skipped[0] = skipped[0] + 1 }
          }
      }
      i = e + 1
  }
  return 0
333}
334// grade for (query index, url), 0 when unlisted -- which the header explains is a JUDGEMENT
335func qb_grade(qi: i64, url: *u8, rqi: *i64, rgr: *i64, rurl: *u8, nr: i64) -> i64 {
  var i: i64 = 0
  var g: i64 = 0
  while i < nr {
      if rqi[i] == qi {
          if qb_eqz(url, ((rurl as i64) + i * QB_UBUF) as *u8) == 1 { g = rgr[i] }
      }
      i = i + 1
  }
  return g
345}
346static qb_mode_ablate: i64
347// THE SCORING ARM, EXTRACTED 2026-09-04 SO THERE IS EXACTLY ONE OF IT. S3 needs the judged set scored
348// TWICE -- once with stem expansion enabled and once without -- and the only honest way to compare two
349// arms is to run the SAME ruler over both. A second copy of this loop would be the duplicate-ruler defect
350// in its purest form: two scorers that agree today and drift apart on the first edit anybody forgets to
351// mirror. out[] carries the five aggregates the caller needs; the per-query lines are printed by design,
352// because the contract requires BOTH ARMS TO PRINT THEIR OWN NUMBERS rather than only a verdict.
353func qb_arm(h: *i64, qtext: *u8, nq: i64, rqi: *i64, rgr: *i64, rurl: *u8, nr: i64, out: *i64) -> i64 {
  let cids:   *i64 = sys_mmap(QB_TOPN * 8) as *i64
  let scores: *i64 = sys_mmap(QB_TOPN * 8) as *i64
  let gains:  *i64 = sys_mmap(QB_TOPN * 8) as *i64
  let rels:   *i64 = sys_mmap(QB_TOPN * 8) as *i64
  let ukey:   *u8 = sys_mmap(QB_PATHCAP)
  let up:    *i64 = sys_mmap(QB_BOX) as *i64
  let ul:    *i64 = sys_mmap(QB_BOX) as *i64
  var sum_ndcg: i64 = 0
  var sum_mrr:  i64 = 0
  var sum_p:    i64 = 0
  var scored:   i64 = 0
  var rank1:    i64 = 0
  var qi: i64 = 0
  while qi < nq {
      let q: *u8 = ((qtext as i64) + qi * QB_QBUF) as *u8
      let n: i64 = dss_search(QB_DOMAIN, q, qb_slen(q), cids, scores, QB_TOPN)
      var j: i64 = 0
      while j < QB_TOPN { gains[j] = 0; rels[j] = 0; j = j + 1 }
      var got: i64 = 0
      if n > 0 {
          var r: i64 = 0
          while r < n {
              dss_mkurlkey(cids[r], ukey)
              var g: i64 = 0
              if ss_hget(h, ukey, up, ul) == 1 { if ul[0] > 0 {
                  let ub: *u8 = sys_mmap(QB_UBUF)
                  qb_cpn(ub, QB_UBUF, up[0] as *u8, ul[0])
                  g = qb_grade(qi, ub, rqi, rgr, rurl, nr)
                  // DIAGNOSTIC, and it exists because every gain came back 0 on the first green run:
                  // 17 queries retrieved 30 results each and NOT ONE url matched a judged url, which is
                  // either a real miss or a string-shape mismatch, and a verdict cannot tell those apart.
                  // ★PRINT THE VALUES, NOT JUST PASS/FAIL -- a gate that reports a boolean cannot say why.
                  // Bounded to the first query's first two results so it cannot flood a beat log.
                  if qi == 0 { if r < 2 {
                      gv_puts("    DIAG retrieved[")
                      gv_num(r)
                      gv_puts("] len=")
                      gv_num(ul[0])
                      gv_puts(" url=")
                      gv_puts(ub)
                      gv_puts("\n    DIAG judged[0]  url=")
                      gv_puts(rurl)
                      // The two urls printed IDENTICAL and the grade was still 0, so the strings are not
                      // the defect and the index fields are. Print them: rqi[0] is the query this row was
                      // bound to at parse time, rgr[0] its grade, and streq the comparison's own answer.
                      gv_puts("\n    DIAG rqi0=")
                      gv_num(rqi[0])
                      gv_puts(" rgr0=")
                      gv_num(rgr[0])
                      gv_puts(" qi=")
                      gv_num(qi)
                      gv_puts(" nr=")
                      gv_num(nr)
                      gv_puts(" streq=")
                      gv_num(qb_eqz(ub, rurl))
                      gv_puts(" grade=")
                      gv_num(g)
                      gv_puts("\n")
                  } }
                  sys_munmap(ub, QB_UBUF)
              } }
              gains[r] = g
              if g > 0 { rels[r] = 1 }
              r = r + 1
          }
          got = n
      }
      if got > 0 {
          let nd: i64 = ie_ndcg_at_k(gains, got, QB_K)
          let mr: i64 = ie_mrr(rels, got)
          let pr: i64 = ie_precision_at_k(rels, got, QB_K)
          sum_ndcg = sum_ndcg + nd
          sum_mrr = sum_mrr + mr
          sum_p = sum_p + pr
          scored = scored + 1
          if gains[0] > 0 { rank1 = rank1 + 1 }
          gv_puts("  q=")
          gv_puts(q)
          gv_puts("  n=")
          gv_num(got)
          gv_puts(" ndcg=")
          gv_num(nd)
          gv_puts(" mrr=")
          gv_num(mr)
          gv_puts(" p@k=")
          gv_num(pr)
          gv_puts(" g1=")
          gv_num(gains[0])
          gv_puts("\n")
      } else {
          // A QUERY THAT RETRIEVED NOTHING IS NOT A ZERO, IT IS AN ABSTENTION: averaging it in would
          // let an index outage read as a relevance regression, which sends the next reader at the
          // wrong subject entirely.
          gv_puts("  q=")
          gv_puts(q)
          gv_puts("  NO-RESULTS -- excluded from the mean, not scored 0\n")
      }
      qi = qi + 1
  }
  out[0] = sum_ndcg
  out[1] = sum_mrr
  out[2] = sum_p
  out[3] = scored
  out[4] = rank1
  return scored
459}
460// S3 CONTRACT SYMBOL qb_expand_ab -- THE STEM-EXPANSION ABLATION. The estate spent a day repairing the
461// ACCOUNTING of query-time stem expansion without once asking whether expanding beats not expanding.
462// Published evidence says that question must be asked: query rewriting made the strongest model WORSE on
463// every metric with every technique, so a rewrite has to EARN its wiring.
464// RETURNS: 1 expansion wins, 0 expansion loses, 2 tie, -1 undecidable (an arm scored nothing).
465// ⚠THE ARMS SHARE ONE RULER AND ONE LABEL SET, so this comparison is sound even while the ABSOLUTE
466// figures from that set are unpublishable under S6 -- a difference over one biased ruler is still a
467// difference. The bias direction is stated in the caller: a lexical judge under-credits exactly the
468// non-whole-word variants expansion exists to match, so this test is HARSH on expansion by construction.
469func qb_expand_ab(h: *i64, qtext: *u8, nq: i64, rqi: *i64, rgr: *i64, rurl: *u8, nr: i64,
                outA: *i64, outB: *i64) -> i64 {
  dss_set_expand_off(0)
  qb_arm(h, qtext, nq, rqi, rgr, rurl, nr, outA)
  dss_set_expand_off(1)
  qb_arm(h, qtext, nq, rqi, rgr, rurl, nr, outB)
  dss_set_expand_off(0)
  if outA[3] <= 0 { return 0 - 1 }
  if outB[3] <= 0 { return 0 - 1 }
  let a: i64 = (outA[0] * QB_PERMIL) / (outA[3] * QB_Q16)
  let b: i64 = (outB[0] * QB_PERMIL) / (outB[3] * QB_Q16)
  if a > b { return 1 }
  if a < b { return 0 }
  return 2
483}
484func qb_main() -> i64 {
  gv_head("nx_qrels_bench -- per-DOCUMENT relevance over the live index, scored by the estate's own nx_ir_eval")
  let ctr: *i64 = gv_ctr()
  let szb: *i64 = sys_mmap(QB_BOX) as *i64
  // TWO-ROOT RESOLVE, AND IT ANNOUNCES WHICH ONE ANSWERED. A bare knowledge/ path is CWD-RELATIVE, and
  // this estate runs the same elf from two different working directories: the build/run lane chdirs into
  // buildroot/ while the daemons run from the nishihost root, so the IDENTICAL literal names two
  // different trees depending on who forked it. MEASURED ON THIS ORGAN'S FIRST RUN: it compiled, ran,
  // and returned passed 0/1 RED at this exact tooth because knowledge/search_qrels.conf resolved under
  // buildroot/ where the file does not exist. ★A RESOLVER THAT RETURNS BYTES WITHOUT SAYING WHERE THEY
  // CAME FROM REPRODUCES THE ORIGINAL DEFECT ONE LAYER UP, so qrels_root is emitted with every run and
  // a MISS is reported as a miss rather than as an empty judged set scoring a perfect zero.
  // ALIGN THE CWD ONCE, DO NOT RESOLVE EACH PATH SEPARATELY. The first cut read the qrels from a second
  // candidate root and left everything else alone -- and the very next tooth, web-shard-opened, FAILED
  // for the SAME reason, because dss_prefix builds the seg-store path relative to the CWD too. Patching
  // one literal would have fixed the symptom I could see and left the identical defect in every path
  // this organ touches. ★A PER-PATH WORKAROUND FOR A CWD PROBLEM IS THE DUPLICATE-RULER DEFECT WEARING A
  // FILENAME: the wrong thing is the working directory, so correct THAT and every path follows.
  // The probe is the qrels file itself because it is the one artifact this organ cannot run without.
  var buf: *u8 = sys_read_file(QB_PATH, szb)
  var whichroot: i64 = 1
  if (buf as i64) == 0 {
      // started under buildroot/ (the build-and-run lane chdirs there); the serving root is one up,
      // and it carries BOTH knowledge/search_qrels.conf AND knowledge/store/, so one chdir aligns them.
      sys_chdir("..")
      buf = sys_read_file(QB_PATH, szb)
      whichroot = 2
  }
  if (buf as i64) == 0 { whichroot = 0 }
  gv_kv("qrels_root", whichroot)
  gv_kv("qrels_bytes", szb[0])
  gv_check("qrels-file-readable", (buf as i64) != 0, ctr)
  if (buf as i64) == 0 { return gv_verdict("nx_qrels_bench", ctr, "the judged set is the subject; without it this organ has nothing to measure and says so rather than scoring an empty set") }
517
  let qtext: *u8 = sys_mmap(QB_MAXQ * QB_QBUF)
  let rqi:  *i64 = sys_mmap(QB_MAXREL * 8) as *i64
  let rgr:  *i64 = sys_mmap(QB_MAXREL * 8) as *i64
  let rurl: *u8 = sys_mmap(QB_MAXREL * QB_UBUF)
  let nqb: *i64 = sys_mmap(QB_BOX) as *i64
  let nrb: *i64 = sys_mmap(QB_BOX) as *i64
  let skb: *i64 = sys_mmap(QB_BOX) as *i64
  let rlc:  *i64 = sys_mmap(QB_MAXREL * 8) as *i64
  let judgefam:  *u8 = sys_mmap(QB_FAMCAP)
  let rankerfam: *u8 = sys_mmap(QB_FAMCAP)
  let benchstr:  *u8 = sys_mmap(QB_QBUF)
  qb_load(buf, szb[0], qtext, nqb, rqi, rgr, rurl, rlc, nrb, skb, judgefam, rankerfam, benchstr)
  let nq: i64 = nqb[0]
  let nr: i64 = nrb[0]
  gv_kv("queries_loaded", nq)
  gv_kv("judgements_loaded", nr)
  gv_kv("rows_skipped", skb[0])
  // BIND THE AGGREGATE TO ITS DENOMINATOR: a tooth that passes on the empty set is not a tooth.
  gv_check("qrels-carries-queries", nq > 0, ctr)
  gv_check("qrels-carries-judgements", nr > 0, ctr)
538
  // S6 LABEL PROVENANCE. Counted, printed, and PARTITIONED -- a class census whose parts do not sum to
  // the population is a leak, and here the leak would be a silently unclassified label deciding whether
  // an absolute figure may be published at all.
  var nhuman: i64 = 0
  var nauto:  i64 = 0
  var li: i64 = 0
  while li < nr {
      if rlc[li] == QB_LC_HUMAN { nhuman = nhuman + 1 } else { nauto = nauto + 1 }
      li = li + 1
  }
  gv_puts("  judge_family=")
  gv_puts(judgefam)
  gv_puts("  ranker_family=")
  gv_puts(rankerfam)
  gv_puts("
554")
  gv_kv("labels_human", nhuman)
  gv_kv("labels_auto", nauto)
  let tier: i64 = qb_tier(rlc, nr)
  gv_kv("label_tier", tier)
  // S2 BENCHMARK IDENTITY. The clock is read from the host and NOT trusted blindly: an unset clock
  // reads 0 and must fail identity rather than stamp a score with the epoch.
  let ts: *i64 = sys_mmap(QB_BOX) as *i64
  ts[0] = 0
  sys_clock_gettime_real(ts)
  let runepoch: i64 = ts[0]
  gv_puts("  benchmark=")
  gv_puts(benchstr)
  gv_puts("
568")
  gv_kv("run_epoch", runepoch)
  let bver: i64 = qb_benchver(benchstr, runepoch)
  gv_kv("benchmark_identity_ok", bver)
  var publish: i64 = 0
  if qb_publish_ok(judgefam, rankerfam, tier) == 1 { if bver == 1 { publish = 1 } }
  gv_kv("publish_absolute_metrics", publish)
  gv_check("judge-family-declared", qb_slen(judgefam) > 0, ctr)
  gv_check("ranker-family-declared", qb_slen(rankerfam) > 0, ctr)
  gv_check("label-class-partition-sums", nhuman + nauto == nr, ctr)
  // THE CONTROLS RUN THROUGH THE SAME PURE FUNCTION THE LIVE DECISION USES, with planted inputs.
  // The third one is a POSITIVE control and it is the load-bearing one: a guard that refuses
  // EVERYTHING passes every negative test, so without it these teeth would stay green on a decision
  // function that had been broken to always return 0.
  gv_check("refuses-when-judge-family-equals-ranker-family", qb_publish_ok("lexical", "lexical", 1) == 0, ctr)
  gv_check("refuses-absolute-metrics-from-auto-only-labels", qb_publish_ok("human", "lexical", 0) == 0, ctr)
  gv_check("neg-control-distinct-families-with-human-labels-publishes", qb_publish_ok("human", "lexical", 1) == 1, ctr)
  // S6 qb_tier is on the critical path above; this binds its return to the independently counted census.
  var tierexp: i64 = 0
  if nhuman > 0 { tierexp = 1 }
  gv_check("tier-agrees-with-the-independently-counted-human-labels", tier == tierexp, ctr)
  // S2 CONTROLS. The third is the positive one -- without it these two would stay green on a benchver
  // that had been broken to always return 0.
  gv_check("refuses-a-score-with-no-benchmark-string", qb_benchver("", runepoch) == 0, ctr)
  gv_check("refuses-a-score-stamped-by-an-unset-clock", qb_benchver("nishi-search-qrels", 0) == 0, ctr)
  gv_check("neg-control-complete-benchmark-identity-passes", qb_benchver("nishi-search-qrels", QB_EPOCH_FLOOR) == 1, ctr)
  // S4 CONTROLS. The declared bite is the first one. The rest exist because a single bite cannot tell a
  // real alignment measure from a function that returns 0, and because BOTH conjuncts must be shown to
  // be load-bearing -- a measure that only checks the rewrite would score the fourth case 1000.
  gv_check("lexalign-a-rewrite-that-drops-every-content-term-scores-zero",
           qb_lexalign("machine learning", "zzz qqq", "a machine learning document") == 0, ctr)
  gv_check("lexalign-full-survival-scores-1000",
           qb_lexalign("machine learning", "machine learning", "a machine learning document") == QB_PERMIL, ctr)
  gv_check("neg-control-lexalign-partial-is-strictly-between",
           qb_lexalign("machine learning", "machine", "a machine learning document") == 500, ctr)
  gv_check("lexalign-requires-the-term-in-the-document-not-only-in-the-rewrite",
           qb_lexalign("machine learning", "machine learning", "a machine document") == 500, ctr)
  // The hot dog case, as a unit test: a rewrite keeping "dog" earns nothing from a document that only
  // says "dogs". This is the whole reason alignment counts WHOLE WORDS.
  gv_check("lexalign-counts-whole-words-so-dogs-does-not-satisfy-dog",
           qb_lexalign("dog", "dog", "many dogs here") == 0, ctr)
  // S1 CONTROLS. The contract's declared negative control is that a PLANTED corpus on the other side
  // must print the other side, so both directions are planted here through the same pure function the
  // live report calls. The third asserts the abstention, which is the state the live shard is in.
  gv_check("crossover-a-corpus-above-the-threshold-reads-ABOVE",
           cs_crossover(50000000, 10000000) == 1, ctr)
  gv_check("neg-control-crossover-a-planted-corpus-below-reads-BELOW",
           cs_crossover(9999999, 10000000) == 0, ctr)
  gv_check("crossover-an-unmeasured-corpus-abstains-rather-than-reading-BELOW",
           cs_crossover(0, 10000000) == (0 - 1), ctr)
  // S5 CONTROLS. The anti-vacuity property is ARITHMETIC, not an assertion we remembered to write: at a
  // query-term frequency of 1 the saturation qtfsat_q10(1) is exactly 1024 (nx_intlog, the ONE ruler since S7 retired the
  // 3-argument twin this file used to call), so it is exactly the identity and every
  // query in this judged set scores identically under both scorers. The second tooth proves it is not
  // merely the identity everywhere, which is what would make the first vacuous.
  gv_check("bm25q-a-single-occurrence-query-term-is-scored-identically",
           idf_bm25q(1000, 1) == 1000, ctr)
  gv_check("bm25q-a-repeated-query-term-saturates-above-its-base-weight-but-below-linear",
           idf_bm25q(1000, 3) > 1000, ctr)
  gv_check("neg-control-bm25q-saturates-strictly-below-linear-growth",
           idf_bm25q(1000, 3) < 3000, ctr)
629
  let pfx:    *u8 = sys_mmap(QB_PATHCAP)
  dss_prefix(QB_DOMAIN, pfx)
  let h: *i64 = dss_open_maybe_cached(pfx)
  gv_check("web-shard-opened", (h as i64) != 0, ctr)
  let sums: *i64 = sys_mmap(QB_BOX * 8) as *i64
  if qb_mode_ablate != 0 {
      // S3 ABLATION. FOUR ARMS: the A/B pair, then a CONTROL pair run with expansion disabled on BOTH
      // sides. The control is the tooth that matters -- without it, an A/B that came back identical
      // would be indistinguishable from a toggle that never took effect, which is precisely how a
      // vacuous ablation reports "no difference" and gets believed.
      let outA: *i64 = sys_mmap(QB_BOX * 8) as *i64
      let outB: *i64 = sys_mmap(QB_BOX * 8) as *i64
      gv_puts("  S3 ABLATION -- arm A is expansion ENABLED, arm B is expansion DISABLED; both print per-query lines below
643")
      let win: i64 = qb_expand_ab(h, qtext, nq, rqi, rgr, rurl, nr, outA, outB)
      if outA[3] > 0 { gv_kv("armA_expansion_on_ndcg_permil", (outA[0] * QB_PERMIL) / (outA[3] * QB_Q16)) }
      if outA[3] > 0 { gv_kv("armA_expansion_on_rank1", outA[4]) }
      if outB[3] > 0 { gv_kv("armB_expansion_off_ndcg_permil", (outB[0] * QB_PERMIL) / (outB[3] * QB_Q16)) }
      if outB[3] > 0 { gv_kv("armB_expansion_off_rank1", outB[4]) }
      // THE ABLATION VERDICT IS GATED BY THE SAME RULE AS THE ABSOLUTE METRICS, AND IT HAS TO BE.
      // MEASURED 2026-09-04 on the first real run: arm B (expansion OFF) scored 912 permil ndcg and
      // 17 of 17 rank-1-relevant against arm A's 553 and 8. That reads like a 65% win and it is NOT
      // actionable, because the judged pool was drawn from this engine's own whole-word required-term
      // arm -- and disabling expansion makes the ranker behave like exactly that arm. The ruler
      // rewards precisely what the change does, which is circularity in its purest form and is the
      // very condition S6 refuses on. Publishing "expansion_wins" here unqualified would hand the next
      // reader a number engineered to be believed.
      // ★A DRAMATIC RESULT FROM A CIRCULAR RULER IS MORE DANGEROUS THAN A NULL ONE, BECAUSE ITS SIZE
      // IS MISTAKEN FOR ITS STRENGTH.
      if publish == 1 { gv_kv("expansion_wins", win) }
      else {
          gv_kv("expansion_wins_UNPUBLISHABLE", win)
          gv_puts("  ⚠S3 IS NOT SETTLED BY THIS RUN. The judged pool came from the engine's own whole-word required-term arm, so an expansion-off arm is scored by a ruler that shares its definition -- the arms differ, but the comparison is circular. What settles it is a human-labelled subset, which is exactly the label_class=human field S6 added; until one exists this verdict is evidence about the RULER, not about expansion.
663")
      }
      gv_puts("  S3 VERDICT -- ")
      if win == 1 { gv_puts("expansion EARNS its wiring on this judged set") }
      else { if win == 0 { gv_puts("expansion LOSES on this judged set and its done-rule says do not keep it wired") }
      else { if win == 2 { gv_puts("TIE -- expansion changes nothing measurable here, which is not a win and does not earn the cost") }
      else { gv_puts("UNDECIDABLE -- an arm scored no queries at all") } } }
      gv_puts("
671")
      let c1: *i64 = sys_mmap(QB_BOX * 8) as *i64
      let c2: *i64 = sys_mmap(QB_BOX * 8) as *i64
      dss_set_expand_off(1)
      qb_arm(h, qtext, nq, rqi, rgr, rurl, nr, c1)
      qb_arm(h, qtext, nq, rqi, rgr, rurl, nr, c2)
      dss_set_expand_off(0)
      gv_check("neg-control-two-arms-both-with-expansion-off-score-identically",
               c1[0] == c2[0], ctr)
      // Written as explicit branches rather than a bitwise OR of two comparisons: this dialect's | is
      // bitwise and the operand types of a comparison are not something to assume in a tooth whose whole
      // job is to catch a toggle that silently did nothing.
      var toggled: i64 = 0
      if outA[0] != outB[0] { toggled = 1 }
      if c1[0] == outB[0] { toggled = 1 }
      gv_check("neg-control-the-expansion-toggle-actually-took-effect-somewhere", toggled == 1, ctr)
      sums[0] = outA[0]
      sums[1] = outA[1]
      sums[2] = outA[2]
      sums[3] = outA[3]
      sums[4] = outA[4]
  } else {
      qb_arm(h, qtext, nq, rqi, rgr, rurl, nr, sums)
  }
  let sum_ndcg: i64 = sums[0]
  let sum_mrr:  i64 = sums[1]
  let sum_p:    i64 = sums[2]
  let scored:   i64 = sums[3]
  let rank1:    i64 = sums[4]
  gv_values_head()
  gv_kv("queries_scored", scored)
  gv_kv("queries_no_results", nq - scored)
  if scored > 0 {
      // UNIT CORRECTION, CAUGHT BY READING THE VALUE RATHER THAN THE LABEL. nx_ir_eval returns Q16.16
      // FIXED POINT, not permil -- the tell was mrr=65536 on a perfect query, which is exactly 2^16 and
      // cannot be a per-mille figure. The first cut emitted those raw under keys ending _permil, which
      // published 36264 where the true value is 553. ★★★★★A NUMBER UNDER THE WRONG UNIT IS WORSE THAN NO
      // NUMBER: it is precise, quotable, and wrong by a factor of 65.5, and nothing about its shape says
      // so. Converting here (not renaming the key) keeps these comparable with nx_recall_eval, which
      // already publishes permil over the same lib -- two rulers on one scale, never two scales.
      if publish == 1 {
          gv_kv("mean_ndcg_permil", (sum_ndcg * QB_PERMIL) / (scored * QB_Q16))
          gv_kv("mean_mrr_permil", (sum_mrr * QB_PERMIL) / (scored * QB_Q16))
          gv_kv("mean_precision_at_k_permil", (sum_p * QB_PERMIL) / (scored * QB_Q16))
          gv_kv("rank1_relevant", rank1)
          gv_kv("rank1_of", scored)
      } else {
          // REFUSED, AND THE RULE THAT FIRED IS NAMED. A refusal that will not say which conjunct
          // failed is a false-alarm generator, and the reader always guesses the most alarming one.
          gv_puts("  PUBLISH REFUSED -- ")
          if qb_slen(judgefam) == 0 {
              gv_puts("no judge_family declared, so the provenance of these labels is unknown and an absolute figure cannot be attributed")
          } else { if qb_slen(rankerfam) == 0 {
              gv_puts("no ranker_family declared, so circularity against the ranker cannot be ruled out")
          } else { if bver == 0 {
              gv_puts("benchmark identity incomplete -- a retrieval score with no benchmark string or no run date is not comparable with anything and is exactly how a stale number outlives the index that produced it")
          } else { if qb_eqz(judgefam, rankerfam) == 1 {
              gv_puts("judge_family equals ranker_family, so this set measures TRANSCRIPTION of the ranker own signal class rather than relevance")
          } else {
              gv_puts("no human label in the judged set, and an automatic judgement is not a substitute for a human qrel")
          } } } }
          gv_puts("
733")
          // THE DIFFERENCE BETWEEN TWO RUNS OVER ONE LABEL SET REMAINS SOUND EVEN WHEN THE LEVEL IS NOT,
          // so the figures are emitted rather than destroyed -- an ablation needs them and deleting them
          // would make every A/B on this estate unmeasurable. They are keyed UNPUBLISHABLE so no reader
          // can lift one into a headline without carrying the word that says it is not one. The BIAS HAS
          // A DIRECTION and it must be stated: a lexical judge under-credits exactly the mechanisms that
          // match non-whole-word variants, so a stem-expansion arm is penalised by construction here.
          gv_kv("diag_mean_ndcg_permil_UNPUBLISHABLE", (sum_ndcg * QB_PERMIL) / (scored * QB_Q16))
          gv_kv("diag_mean_mrr_permil_UNPUBLISHABLE", (sum_mrr * QB_PERMIL) / (scored * QB_Q16))
          gv_kv("diag_mean_precision_at_k_permil_UNPUBLISHABLE", (sum_p * QB_PERMIL) / (scored * QB_Q16))
          gv_kv("diag_rank1_relevant_UNPUBLISHABLE", rank1)
          gv_kv("rank1_of", scored)
      }
  }
  gv_kv("k", QB_K)
  // S1 LIVE REPORT. corpus_docs is EXACT from the store. corpus_tokens is NOT, and it is reported as
  // unmeasured rather than estimated: this scorer normalises with a precomputed per-doc normq10 and
  // computes avgdl over the CANDIDATE SET per query, so no corpus-wide token total is persisted --
  // the scorer's own comment already names that gap as "the store-rung". An estimate here would be a
  // constant wearing the shape of a measurement, and the whole purpose of this rung is to say which
  // side of a published threshold the corpus is on, which an estimate cannot honestly settle.
  let corpus_docs: i64 = ss_doc_count(h)
  gv_kv("corpus_docs", corpus_docs)
  gv_kv("corpus_tokens_measured", 0)
  let side: i64 = cs_crossover(0, QB_CROSSOVER_TOKENS)
  gv_kv("bm25_crossover_tokens", QB_CROSSOVER_TOKENS)
  gv_kv("bm25_crossover_side", side)
  gv_puts("  bm25_crossover: UNDECIDABLE -- corpus_tokens is not persisted by this index, so the side is abstained rather than guessed; the blocking rung is a corpus-wide token total in the store
761")
  // The scored-count tooth is what stops this reporting a clean mean over two queries.
  gv_check("scored-most-of-the-set", scored * 2 > nq, ctr)
  gv_check("some-query-has-a-relevant-rank1", rank1 > 0, ctr)
  return gv_verdict("nx_qrels_bench", ctr, "retrieval via dss_search on the live serve path and metrics via nx_ir_eval, so neither half is reimplemented here; recall over the JUDGED POOL only, never the corpus")
766}
767func main(argc: i64, argv: *i64) -> i64 {
  // The ablation is a VERB, not a per-beat tooth, and that is deliberate: it runs the judged set FOUR
  // times, so wiring it into the default path would make every beat four times slower to answer a
  // question that only changes when the scorer changes.
  if argc > 1 { if qb_eqz(argv[1] as *u8, "ablate") == 1 { qb_mode_ablate = 1 } }
  return qb_main()
773}
nx_qrels_bench.nx source

`nx_qrels_bench.nx` source
