code wiki / _hdl_build / nx_beir_eval.nx
`nx_beir_eval.nx` source

↩ module page · 505 lines · 22111 B
1// nx_beir_eval.nx -- SOVEREIGN BEIR harness: real external ground truth (BEIR/nfcorpus) scored by our own
2// integer BM25 -> mean nDCG@10 (the standard BEIR metric). The HONEST external row the maturity ladder is
3// gated on (F236): real 3633-doc corpus + 323 test queries + 12335 graded qrels, vs the published BEIR BM25
4// baseline (nfcorpus nDCG@10 ~0.32, Anserini). BM25 k1=0.9 b=0.4 (BEIR defaults). IDF=log2 (ranking-invariant
5// vs ln -> identical nDCG). Integer/fixed-point throughout (no float) = bit-reproducible. Forward index
6// (per-doc sorted term,tf). Tokenizer = lowercase [a-z0-9] runs, NO stem / NO stopwords (v1 -- the gap to the
7// stemmed 0.32 baseline IS the stemming rung, honestly, not a defect).
8// expect_exit: 0  license_tier: ORIGINAL
9import "nx_qabench_engine.nx"
10import "nx_ppmi_lib.nx"
11// seq1494 RERANK ARM: BM25 retrieves a shortlist, the PPMI late-interaction model re-orders it, and BOTH
12// nDCG@10 values are printed FROM THE SAME RUN so the delta cannot be confounded by corpus/qrel/tokeniser
13// drift. The BM25 arm is untouched: top[0..9] keeps its exact BM25 order, we merely widen the candidate
14// selection past 10 so the reranker has something to re-order.
15// derived: 50 candidates is the standard first-stage depth for late-interaction reranking -- deep enough
16// that recall@50 >> recall@10 (so the reranker CAN win) and shallow enough to stay O(50) maxsim per query.
17const BE_KCAND: i64 = 50
18// sized: query/doc PPMI id buffers; nfcorpus docs are abstracts, far under this
19const BE_IDCAP: i64 = 4096
20const K_MAGIC_1024: i64 = 1024
21const K_MAGIC_8192: i64 = 8192
22const K_MAGIC_4096: i64 = 4096
23const K_MAGIC_1900: i64 = 1900
24
25const MAXDOC: i64 = 4096
26const CAPFILE: i64 = 67108864
27const NTCAP: i64 = 2000000
28const DTCAP: i64 = 2000000
29const NJCAP: i64 = 20000
30
31func be_read(path: *u8, slot: *i64) -> i64 {
  let fd: i64 = sys_openat_rd(path)
  if fd < 0 { return 0 - 1 }
  let buf: *u8 = sys_mmap(CAPFILE)
  var total: i64 = 0
  var r: i64 = 1
  while r > 0 {
      if total >= CAPFILE { r = 0 } else {
          r = sys_read(fd, (buf as i64 + total) as *u8, CAPFILE - total)
          if r > 0 { total = total + r }
      }
  }
  sys_close(fd)
  slot[0] = buf as i64
  return total
46}
47
48// read pathA, else fall back to pathB (NAS stable path first, local /tmp second)
49func be_read_fb(pa: *u8, pb: *u8, slot: *i64) -> i64 {
  let n: i64 = be_read(pa, slot)
  if n > 0 { return n }
  return be_read(pb, slot)
53}
54
55// index of byte ch in buf[from,limit), else limit
56func be_find(buf: *u8, from: i64, limit: i64, ch: i64) -> i64 {
  var i: i64 = from
  var res: i64 = limit
  var go: i64 = 1
  while go == 1 {
      if i >= limit { go = 0 } else {
          if buf[i] == (ch as u8) { res = i; go = 0 } else { i = i + 1 }
      }
  }
  return res
66}
67
68// lowercase+hash [a-z0-9] tokens in buf[start,end); append hashes to out at noff[0]; return token count
69func be_tok(buf: *u8, start: i64, end: i64, out: *i64, noff: *i64, scr: *u8) -> i64 {
  var i: i64 = start
  var cnt: i64 = 0
  var sl: i64 = 0
  while i <= end {
      var c: i64 = 0
      if i < end { c = buf[i] as i64 }
      var isc: i64 = 0
      var lc: i64 = c
      if c >= 65 { if c <= 90 { lc = c + 32; isc = 1 } }
      if c >= 97 { if c <= 122 { isc = 1 } }
      if c >= 48 { if c <= 57 { isc = 1 } }
      if isc == 1 {
          if sl < 500 { scr[sl] = lc as u8; sl = sl + 1 }
      } else {
          if sl > 0 {
              out[noff[0]] = db_semhash(scr, 0, sl)
              noff[0] = noff[0] + 1
              cnt = cnt + 1
              sl = 0
          }
      }
      i = i + 1
  }
  return cnt
94}
95
96func be_qs1(a: *i64, lo: i64, hi: i64) -> i64 {
  if lo >= hi { return 0 }
  var i: i64 = lo
  var j: i64 = hi
  let p: i64 = a[(lo + hi) / 2]
  while i <= j {
      while a[i] < p { i = i + 1 }
      while a[j] > p { j = j - 1 }
      if i <= j {
          let t: i64 = a[i]; a[i] = a[j]; a[j] = t
          i = i + 1; j = j - 1
      }
  }
  be_qs1(a, lo, j)
  be_qs1(a, i, hi)
  return 0
112}
113
114func be_qs2(key: *i64, pay: *i64, lo: i64, hi: i64) -> i64 {
  if lo >= hi { return 0 }
  var i: i64 = lo
  var j: i64 = hi
  let p: i64 = key[(lo + hi) / 2]
  while i <= j {
      while key[i] < p { i = i + 1 }
      while key[j] > p { j = j - 1 }
      if i <= j {
          let t: i64 = key[i]; key[i] = key[j]; key[j] = t
          let u: i64 = pay[i]; pay[i] = pay[j]; pay[j] = u
          i = i + 1; j = j - 1
      }
  }
  be_qs2(key, pay, lo, j)
  be_qs2(key, pay, i, hi)
  return 0
131}
132
133// log2(q) in 1/1024ths for q>=1
134func be_log2_1024(q: i64) -> i64 {
  if q <= 1 { return 0 }
  var bl: i64 = 0
  var t: i64 = q
  while t > 1 { t = t / 2; bl = bl + 1 }
  var frac: i64 = 0
  if bl >= 6 { frac = (q / (1 << (bl - 6))) - 64 }
  if bl < 6 { frac = (q << (6 - bl)) - 64 }
  if frac < 0 { frac = 0 }
  return bl * K_MAGIC_1024 + frac * 16
144}
145
146func be_disc(r: i64) -> i64 {
  if r == 1 { return 1000 }
  if r == 2 { return 631 }
  if r == 3 { return 500 }
  if r == 4 { return 431 }
  if r == 5 { return 387 }
  if r == 6 { return 356 }
  if r == 7 { return 333 }
  if r == 8 { return 315 }
  if r == 9 { return 301 }
  if r == 10 { return 289 }
  return 0
158}
159
160func main() -> i64 {
  let G: *i64 = sys_mmap(64) as *i64
  let scr: *u8 = sys_mmap(K_MAGIC_1024)
163
  // ---------- load corpus ----------
  let cn: i64 = be_read_fb("knowledge/beir/nfcorpus/corpus.tsv" as *u8, "/tmp/beir/nfcorpus/corpus.tsv" as *u8, G)
  if cn <= 0 { db_w("{\"error\":\"corpus.tsv missing\"}\n" as *u8); return 1 }
  let cbuf: *u8 = G[0] as *u8
168
  let thash: *i64 = sys_mmap(NTCAP * 8) as *i64
  let noff: *i64 = sys_mmap(8) as *i64
  noff[0] = 0
  let doc_start: *i64 = sys_mmap((MAXDOC + 1) * 8) as *i64
  let doc_len: *i64 = sys_mmap(MAXDOC * 8) as *i64
  let doc_idh: *i64 = sys_mmap(MAXDOC * 8) as *i64
  // seq1494 RERANK ARM: keep each doc's RAW TEXT SPAN in cbuf. BM25 works on this harness's own hashed
  // vocab, but the PPMI late-interaction model has its OWN word ids, so the reranker needs the original
  // characters. Two i64 arrays is the whole cost of making rung 2 measurable instead of asserted.
  let doc_ts: *i64 = sys_mmap(MAXDOC * 8) as *i64
  let doc_te: *i64 = sys_mmap(MAXDOC * 8) as *i64
  var nd: i64 = 0
181
  var p: i64 = 0
  while p < cn {
      let eol: i64 = be_find(cbuf, p, cn, 10)
      let tab: i64 = be_find(cbuf, p, eol, 9)
      if tab < eol { if nd < MAXDOC {
          doc_idh[nd] = db_semhash(cbuf, p, tab - p)
          doc_start[nd] = noff[0]
          doc_ts[nd] = tab + 1
          doc_te[nd] = eol
          doc_len[nd] = be_tok(cbuf, tab + 1, eol, thash, noff, scr)
          nd = nd + 1
      } }
      p = eol + 1
  }
  doc_start[nd] = noff[0]
  let ntok: i64 = noff[0]
198
  // ---------- vocab = sort+unique ----------
  let vocab: *i64 = sys_mmap(ntok * 8) as *i64
  var vi: i64 = 0
  while vi < ntok { vocab[vi] = thash[vi]; vi = vi + 1 }
  be_qs1(vocab, 0, ntok - 1)
  var nv: i64 = 0
  var k: i64 = 0
  while k < ntok {
      if nv == 0 { vocab[nv] = vocab[k]; nv = nv + 1 }
      else { if vocab[k] != vocab[nv - 1] { vocab[nv] = vocab[k]; nv = nv + 1 } }
      k = k + 1
  }
211
  // ---------- forward index + df ----------
  let dterm: *i64 = sys_mmap(DTCAP * 8) as *i64
  let dtf: *i64 = sys_mmap(DTCAP * 8) as *i64
  let doc_off: *i64 = sys_mmap((MAXDOC + 1) * 8) as *i64
  let df: *i64 = sys_mmap(nv * 8) as *i64
  var z: i64 = 0
  while z < nv { df[z] = 0; z = z + 1 }
  let tmp: *i64 = sys_mmap(K_MAGIC_8192 * 8) as *i64
  var dpos: i64 = 0
  var d: i64 = 0
  while d < nd {
      let s0: i64 = doc_start[d]
      let s1: i64 = doc_start[d + 1]
      var m: i64 = 0
      var q: i64 = s0
      while q < s1 {
          if m < K_MAGIC_8192 { tmp[m] = db_bsearch_i64(vocab, nv, thash[q]); m = m + 1 }
          q = q + 1
      }
      be_qs1(tmp, 0, m - 1)
      doc_off[d] = dpos
      var a: i64 = 0
      while a < m {
          var b: i64 = a + 1
          var go3: i64 = 1
          while go3 == 1 { if b < m { if tmp[b] == tmp[a] { b = b + 1 } else { go3 = 0 } } else { go3 = 0 } }
          if dpos < DTCAP { dterm[dpos] = tmp[a]; dtf[dpos] = b - a; dpos = dpos + 1 }
          df[tmp[a]] = df[tmp[a]] + 1
          a = b
      }
      d = d + 1
  }
  doc_off[nd] = dpos
245
  var totlen: i64 = 0
  z = 0
  while z < nd { totlen = totlen + doc_len[z]; z = z + 1 }
  var avgdl: i64 = 1
  if nd > 0 { avgdl = totlen / nd }
  if avgdl < 1 { avgdl = 1 }
252
  // ---------- doc-id -> docidx (sorted idhash) ----------
  let sidh: *i64 = sys_mmap(nd * 8) as *i64
  let sidx: *i64 = sys_mmap(nd * 8) as *i64
  z = 0
  while z < nd { sidh[z] = doc_idh[z]; sidx[z] = z; z = z + 1 }
  be_qs2(sidh, sidx, 0, nd - 1)
259
  // ---------- qrels ----------
  let qn: i64 = be_read_fb("knowledge/beir/nfcorpus/qrels/test.tsv" as *u8, "/tmp/beir/nfcorpus/qrels/test.tsv" as *u8, (G as i64 + 8) as *i64)
  if qn <= 0 { db_w("{\"error\":\"qrels test.tsv missing\"}\n" as *u8); return 1 }
  let qbuf: *u8 = G[1] as *u8
  let j_qid: *i64 = sys_mmap(NJCAP * 8) as *i64
  let j_doc: *i64 = sys_mmap(NJCAP * 8) as *i64
  let j_scr: *i64 = sys_mmap(NJCAP * 8) as *i64
  var nj: i64 = 0
  var qp: i64 = 0
  var hdr: i64 = 1
  while qp < qn {
      let e2: i64 = be_find(qbuf, qp, qn, 10)
      if hdr == 1 { hdr = 0 } else {
          let t1: i64 = be_find(qbuf, qp, e2, 9)
          if t1 < e2 {
              let t2: i64 = be_find(qbuf, t1 + 1, e2, 9)
              if t2 < e2 {
                  var sc: i64 = 0
                  var sp: i64 = t2 + 1
                  while sp < e2 { let dch: i64 = qbuf[sp] as i64; if dch >= 48 { if dch <= 57 { sc = sc * 10 + (dch - 48) } } sp = sp + 1 }
                  let dh: i64 = db_semhash(qbuf, t1 + 1, t2 - t1 - 1)
                  let dpos2: i64 = db_bsearch_i64(sidh, nd, dh)
                  if dpos2 >= 0 { if nj < NJCAP {
                      j_qid[nj] = db_semhash(qbuf, qp, t1 - qp)
                      j_doc[nj] = sidx[dpos2]
                      j_scr[nj] = sc
                      nj = nj + 1
                  } }
              }
          }
      }
      qp = e2 + 1
  }
  // sort judgments by qid, carrying an index permutation (so j_doc/j_scr stay addressable via jidx)
  let jidx: *i64 = sys_mmap(nj * 8) as *i64
  z = 0
  while z < nj { jidx[z] = z; z = z + 1 }
  be_qs2(j_qid, jidx, 0, nj - 1)
298
  // ---------- per query ----------
  let dqn: i64 = be_read_fb("knowledge/beir/nfcorpus/queries.tsv" as *u8, "/tmp/beir/nfcorpus/queries.tsv" as *u8, (G as i64 + 16) as *i64)
  if dqn <= 0 { db_w("{\"error\":\"queries.tsv missing\"}\n" as *u8); return 1 }
  let dbuf: *u8 = G[2] as *u8
  let score: *i64 = sys_mmap(MAXDOC * 8) as *i64
  let qtid: *i64 = sys_mmap(K_MAGIC_1024 * 8) as *i64
  let top: *i64 = sys_mmap(16 * 8) as *i64
  let goldsc: *i64 = sys_mmap(K_MAGIC_4096 * 8) as *i64
  let used: *u8 = sys_mmap(MAXDOC)
  let qhbuf: *i64 = sys_mmap(K_MAGIC_1024 * 8) as *i64
  let qnoff: *i64 = sys_mmap(8) as *i64
310
  // seq1494: load the PPMI model ONCE. HONEST-ABSENT: if it is missing the rerank arm reports
  // UNAVAILABLE rather than silently scoring 0 -- an absent model must never read as "reranking is bad".
  let PG: *i64 = sys_mmap(64) as *i64
  let rr_ok: i64 = ppl_load(PG, "knowledge/index/semppmi_v1.bin" as *u8)
  let qids: *i64 = sys_mmap(BE_IDCAP * 8) as *i64
  let dids: *i64 = sys_mmap(BE_IDCAP * 8) as *i64
  let cscore: *i64 = sys_mmap(BE_KCAND * 8) as *i64
  let cused: *u8 = sys_mmap(BE_KCAND)
  var sum_ndcg_rr: i64 = 0
  var q_improved: i64 = 0
  var q_worsened: i64 = 0
  var q_same: i64 = 0
  var sum_ndcg: i64 = 0
  var nq: i64 = 0
  var n_noqrel: i64 = 0
  var n_noterm: i64 = 0
  var n_noidcg: i64 = 0
  var qpp: i64 = 0
  while qpp < dqn {
      let e3: i64 = be_find(dbuf, qpp, dqn, 10)
      let tb: i64 = be_find(dbuf, qpp, e3, 9)
      if tb < e3 {
          let qh2: i64 = db_semhash(dbuf, qpp, tb - qpp)
          qnoff[0] = 0
          be_tok(dbuf, tb + 1, e3, qhbuf, qnoff, scr)
          var nqt: i64 = 0
          var qi: i64 = 0
          while qi < qnoff[0] {
              let tid2: i64 = db_bsearch_i64(vocab, nv, qhbuf[qi])
              if tid2 >= 0 { if nqt < K_MAGIC_1024 { qtid[nqt] = tid2; nqt = nqt + 1 } }
              qi = qi + 1
          }
          let lo0: i64 = db_bsearch_i64(j_qid, nj, qh2)
          if lo0 < 0 { n_noqrel = n_noqrel + 1 } else {
              var jlo: i64 = lo0
              var g1: i64 = 1
              while g1 == 1 { if jlo > 0 { if j_qid[jlo - 1] == qh2 { jlo = jlo - 1 } else { g1 = 0 } } else { g1 = 0 } }
              var jhi: i64 = lo0
              var g2: i64 = 1
              while g2 == 1 { if jhi < nj - 1 { if j_qid[jhi + 1] == qh2 { jhi = jhi + 1 } else { g2 = 0 } } else { g2 = 0 } }
351
              // gold + IDCG@10 FIRST (methodology: every query with relevant judgments is counted)
              var ng: i64 = 0
              var jjg: i64 = jlo
              while jjg <= jhi { if ng < K_MAGIC_4096 { goldsc[ng] = j_scr[jidx[jjg]]; ng = ng + 1 } jjg = jjg + 1 }
              be_qs1(goldsc, 0, ng - 1)
              var idcg: i64 = 0
              var ri: i64 = 0
              while ri < 10 { if ri < ng { idcg = idcg + goldsc[ng - 1 - ri] * be_disc(ri + 1) } ri = ri + 1 }
360
              if idcg <= 0 { n_noidcg = n_noidcg + 1 } else {
                  var dcg: i64 = 0
                  if nqt <= 0 { n_noterm = n_noterm + 1 } else {
                      z = 0
                      while z < nd { score[z] = 0; z = z + 1 }
                      var qt: i64 = 0
                      while qt < nqt {
                          let tid3: i64 = qtid[qt]
                          let dfi: i64 = df[tid3]
                          if dfi > 0 {
                              let idf: i64 = be_log2_1024((2 * nd + 2) / (2 * dfi + 1))
                              var dd: i64 = 0
                              while dd < nd {
                                  let o0: i64 = doc_off[dd]
                                  let o1: i64 = doc_off[dd + 1]
                                  let pp: i64 = db_bsearch_i64((dterm as i64 + o0 * 8) as *i64, o1 - o0, tid3)
                                  if pp >= 0 {
                                      let tf: i64 = dtf[o0 + pp]
                                      let norm1000: i64 = 600 + (400 * doc_len[dd]) / avgdl
                                      let denom1000: i64 = tf * 1000 + (900 * norm1000) / 1000
                                      if denom1000 > 0 { score[dd] = score[dd] + (idf * tf * K_MAGIC_1900) / denom1000 }
                                  }
                                  dd = dd + 1
                              }
                          }
                          qt = qt + 1
                      }
                      z = 0
                      while z < nd { used[z] = 0 as u8; z = z + 1 }
                      var rr: i64 = 0
                      while rr < BE_KCAND {
                          var best: i64 = 0 - 1
                          var dd2: i64 = 0
                          while dd2 < nd {
                              if used[dd2] == (0 as u8) {
                                  if best < 0 { best = dd2 } else { if score[dd2] > score[best] { best = dd2 } }
                              }
                              dd2 = dd2 + 1
                          }
                          top[rr] = best
                          if best >= 0 { used[best] = 1 as u8 }
                          rr = rr + 1
                      }
                      rr = 0
                      while rr < 10 {
                          let dpk: i64 = top[rr]
                          var gv: i64 = 0
                          if dpk >= 0 {
                              var jj2: i64 = jlo
                              while jj2 <= jhi {
                                  if j_doc[jidx[jj2]] == dpk { gv = j_scr[jidx[jj2]]; jj2 = jhi + 1 } else { jj2 = jj2 + 1 }
                              }
                          }
                          dcg = dcg + gv * be_disc(rr + 1)
                          rr = rr + 1
                      }
                  }
                  // ---- seq1494 RERANK ARM: re-order the SAME BM25 shortlist by PPMI late-interaction ----
                  // Identical qrels, identical idcg, identical candidate pool -> the ONLY difference is
                  // the ordering function, which is what makes this a controlled comparison.
                  var dcg_rr: i64 = dcg
                  if rr_ok == 1 { if nqt > 0 {
                      // same (buf,len) fix on the query side: the query text is dbuf[tb+1 .. e3)
                      let nqi: i64 = ppl_tokenize_ids(PG, ((dbuf as i64) + tb + 1) as *u8, e3 - tb - 1, qids, BE_IDCAP)
                      var ci: i64 = 0
                      while ci < BE_KCAND {
                          cscore[ci] = 0 - 1
                          cused[ci] = 0 as u8
                          let dpc: i64 = top[ci]
                          if dpc >= 0 {
                              let dsx: i64 = doc_ts[dpc]
                              let dex: i64 = doc_te[dpc]
                              // BUGFIX (caught by an EMPTY result file, not by a passing test): this took
                              // (buf, len) but was handed (cbuf, dex) -- the corpus BASE and an ABSOLUTE
                              // END offset -- so every candidate re-tokenised the WHOLE corpus from byte 0.
                              // 323 queries x 50 candidates x 3633 docs never finished. Pass the doc's own
                              // span. ★a silent non-finish is a LOUDER bug than a wrong number.
                              let ndi: i64 = ppl_tokenize_ids(PG, ((cbuf as i64) + dsx) as *u8, dex - dsx, dids, BE_IDCAP)
                              cscore[ci] = ppl_maxsim_idf(PG, qids, nqi, dids, ndi)
                          }
                          ci = ci + 1
                      }
                      var dcg2: i64 = 0
                      var rk: i64 = 0
                      while rk < 10 {
                          var bi: i64 = 0 - 1
                          var cj: i64 = 0
                          while cj < BE_KCAND {
                              if cused[cj] == (0 as u8) { if top[cj] >= 0 {
                                  if bi < 0 { bi = cj } else { if cscore[cj] > cscore[bi] { bi = cj } }
                              } }
                              cj = cj + 1
                          }
                          if bi >= 0 {
                              cused[bi] = 1 as u8
                              let dpk2: i64 = top[bi]
                              var gv2: i64 = 0
                              var jj3: i64 = jlo
                              while jj3 <= jhi {
                                  if j_doc[jidx[jj3]] == dpk2 { gv2 = j_scr[jidx[jj3]]; jj3 = jhi + 1 } else { jj3 = jj3 + 1 }
                              }
                              dcg2 = dcg2 + gv2 * be_disc(rk + 1)
                          }
                          rk = rk + 1
                      }
                      dcg_rr = dcg2
                  } }
                  if dcg_rr > dcg { q_improved = q_improved + 1 } else { if dcg_rr < dcg { q_worsened = q_worsened + 1 } else { q_same = q_same + 1 } }
                  sum_ndcg_rr = sum_ndcg_rr + (dcg_rr * 1000) / idcg
                  sum_ndcg = sum_ndcg + (dcg * 1000) / idcg
                  nq = nq + 1
              }
          }
      }
      qpp = e3 + 1
  }
477
  var mean: i64 = 0
  if nq > 0 { mean = sum_ndcg / nq }
  db_w("{\"tool\":\"nx_beir_eval\",\"dataset\":\"BEIR/nfcorpus\",\"docs\":" as *u8); db_n(nd)
  db_w(",\"vocab\":" as *u8); db_n(nv)
  db_w(",\"test_queries_scored\":" as *u8); db_n(nq)
  db_w(",\"dropped_no_qrels\":" as *u8); db_n(n_noqrel)
  db_w(",\"dropped_no_relevant\":" as *u8); db_n(n_noidcg)
  db_w(",\"counted_zero_no_query_term_match\":" as *u8); db_n(n_noterm)
  db_w(",\"avgdl\":" as *u8); db_n(avgdl)
  db_w(",\"bm25\":{\"k1x1000\":900,\"bx1000\":400,\"idf\":\"log2\",\"stemming\":false,\"stopwords\":false}" as *u8)
  db_w(",\"ndcg_at_10_permil\":" as *u8); db_n(mean)
  db_w(",\"published_bm25_baseline_permil\":320" as *u8)
  // seq1494: the rerank arm, printed from the SAME run. UNAVAILABLE (not 0) when the model is absent.
  var mean_rr: i64 = 0
  if nq > 0 { mean_rr = sum_ndcg_rr / nq }
  db_w(",\"rerank\":{\"model\":" as *u8)
  if rr_ok == 1 { db_w("\"semppmi_v1 late-interaction maxsim(idf)\"" as *u8) } else { db_w("\"UNAVAILABLE -- semppmi_v1.bin did not load; rerank arm NOT measured (this is not a score of 0)\"" as *u8) }
  db_w(",\"candidates\":" as *u8); db_n(BE_KCAND)
  db_w(",\"ndcg_at_10_permil\":" as *u8); db_n(mean_rr)
  db_w(",\"delta_permil\":" as *u8); db_n(mean_rr - mean)
  db_w(",\"queries_improved\":" as *u8); db_n(q_improved)
  db_w(",\"queries_worsened\":" as *u8); db_n(q_worsened)
  db_w(",\"queries_unchanged\":" as *u8); db_n(q_same)
  db_w(",\"accept_rule\":\"declared BEFORE the run: rerank must EXCEED the BM25 arm on these same queries or it does NOT get wired into search (seq1494)\"}" as *u8)
  db_w(",\"note\":\"real external ground truth; integer/bit-reproducible; no-stem/no-stopword v1 (gap to the 0.32 stemmed baseline = the stemming rung)\"}" as *u8)
  db_w("\n" as *u8)
  return 0
505}
nx_beir_eval.nx source

`nx_beir_eval.nx` source
