`nx_qabench.nx` source

↩ module page · 2146 lines · 95267 B
1// nx_qabench.nx -- the QA-benchmark LADDER (operator 2026-07-07: "use others like beerqa, climb the ladder to
2// the top"). Runs the sovereign researcher's mechanical retrieve+extract+multi-hop pipeline (shared engine =
3// nx_qabench_engine, extracted from nx_drbench) across a DIFFICULTY GRADIENT and prints ONE scoreboard:
4//   RUNG 1-hop  = SQuAD v1.1 (single context paragraph -> isolates EXTRACTION from retrieval; should score HIGH)
5//   RUNG 2-hop  = HotpotQA distractor (retrieve 2 of 10 paras + bridge reasoning -> the reasoning wall)
6// Same F1/EM metric (nx_qa_score_lib, integer permille), same extractor -> the F1 DROP as hops rise is the
7// honest, MEASURED "where mechanical ends and reasoning/LLM begins" signal. Each benchmark: T-parse + T-negctl
8// (rotated-gold F1 ~0 = metric not rigged). Formats dispatched by `fmt` (0=hotpot distractor, 1=squad single).
9// expect_exit: 0  license_tier: ORIGINAL
10import "nx_qabench_engine.nx"
11import "nx_qwen_extract.nx"     // R5: opt-in pretrained-Qwen extraction (marker file knowledge/index/qwen_reader.on)
12const SEM_MAGIC_260000: i64 = 260000
13const SEM_MAGIC_32760: i64 = 32760
14const SEM_MAGIC_4096: i64 = 4096
15const SEM_MAGIC_536870912: i64 = 536870912
16const SEM_MAGIC_262144: i64 = 262144
17const SEM_MAGIC_4000: i64 = 4000
18const SEM_MAGIC_200000: i64 = 200000
19const SEM_MAGIC_2000: i64 = 2000
20const SEM_MAGIC_8000: i64 = 8000
21const SEM_MAGIC_1024: i64 = 1024
22const SEM_MAGIC_2000000000: i64 = 2000000000
23const SEM_MAGIC_16777216: i64 = 16777216
24const SEM_MAGIC_402653184: i64 = 402653184
25const SEM_MAGIC_10000: i64 = 10000
26const SEM_MAGIC_2026: i64 = 2026
27const SEM_MAGIC_2500: i64 = 2500
28const SEM_MAGIC_1000000: i64 = 1000000
29const SEM_MAGIC_2048: i64 = 2048
30const SEM_MAGIC_32768: i64 = 32768
31const SEM_MAGIC_1048576: i64 = 1048576
32const SEM_MAGIC_524288: i64 = 524288
33const SEM_MAGIC_8192: i64 = 8192
34const SEM_MAGIC_65536: i64 = 65536
35const SEM_MAGIC_8388608: i64 = 8388608
36const SEM_MAGIC_16384: i64 = 16384
37const SEM_MAGIC_3999: i64 = 3999
38
39// ---- SQuAD: split a single context string into sentences in sb/so/sl/sp (all para 0). returns nsent ----
40// is src[start..start+ln) prose (real answerable text) vs web CHROME/wikitext/JSON? Real web pages (esp.
41// Wikipedia) carry nav menus, language lists, and infobox markup ({{...}}, [[...]], {"wt":...}) that pollute
42// retrieval -- the measured binding constraint for live web research. Prose = letter-dense, sane length, few
43// markup markers. General (not site-specific): keeps the article body, drops the scaffolding.
44func qa_is_prose(src: *u8, start: i64, ln: i64) -> i64 {
  if ln < 25 { return 0 }
  if ln > 600 { return 0 }                            // giant runs w/o '. ' = nav blob / language list
  var letters: i64 = 0
  var spaces: i64 = 0
  var bad: i64 = 0
  var i: i64 = 0
  while i < ln {
      let c: i64 = src[start+i] as i64
      if c >= 65 { if c <= 90 { letters = letters + 1 } }
      if c >= 97 { if c <= 122 { letters = letters + 1 } }
      if c == 32 { spaces = spaces + 1 }
      if c == 123 { bad = bad + 1 }                   // {
      if c == 125 { bad = bad + 1 }                   // }
      if c == 124 { bad = bad + 1 }                   // |
      if c == 61 { bad = bad + 1 }                    // =
      if c == 91 { bad = bad + 1 }                    // [
      if c == 93 { bad = bad + 1 }                    // ]
      i = i + 1
  }
  if bad > 2 { return 0 }                             // wikitext/JSON markup
  if (letters + spaces) * 100 < ln * 78 { return 0 }  // < 78% letters+spaces = not prose
  var wc: i64 = spaces + 1
  if wc < 5 { return 0 }                              // need >= ~5 words to be a real sentence
  return 1
69}
70
71// copy [start,start+ln) into dst, DROPPING content inside ( ) and [ ] (pronunciation IPA / citation / date
72// parentheticals that pollute real web prose and get whole sentences dropped by the letter-ratio filter).
73// flat ifs (no else-chain: deep else{if} miscompiles per nx_cc). returns cleaned length.
74func qa_clean_sentence(src: *u8, start: i64, ln: i64, dst: *u8) -> i64 {
  var depth: i64 = 0
  var o: i64 = 0
  var k: i64 = 0
  while k < ln {
      let c: i64 = src[start+k] as i64
      var open: i64 = 0
      if c==40 { open=1 }                                 // (
      if c==91 { open=1 }                                 // [
      var close: i64 = 0
      if c==41 { close=1 }                                // )
      if c==93 { close=1 }                                // ]
      if open==1 { depth=depth+1 }
      if close==1 { if depth>0 { depth=depth-1 } }
      if open==0 { if close==0 { if depth==0 { if o<1000 { dst[o]=src[start+k]; o=o+1 } } } }
      k=k+1
  }
  dst[o]=(0 as u8)
  return o
93}
94
95// prose-filtered sentence splitter for the LIVE answer mode (higher cap; drops non-prose). Isolated from the
96// benchmark's qab_split_ctx (whose inputs are already clean JSON-extracted contexts).
97func qa_split_prose(g: *i64, src: *u8, slen: i64) -> i64 {
  let sb: *u8 = g[15] as *u8
  let so: *i64 = g[16] as *i64
  let sl: *i64 = g[17] as *i64
  let sp: *i64 = g[18] as *i64
  var ns: i64 = 0
  var bump: i64 = 0
  var start: i64 = 0
  var i: i64 = 0
  while i < slen {
      var boundary: i64 = 0
      let c: i64 = src[i] as i64
      if c==46 { boundary=1 }
      if c==63 { boundary=1 }
      if c==33 { boundary=1 }
      if boundary==1 {
          var nxsp: i64 = 1
          if i+1 < slen { let d: i64 = src[i+1] as i64; if d!=32 { if d!=10 { nxsp=0 } } }
          if nxsp==0 { boundary=0 }
      }
      if boundary==1 {
          let ln: i64 = (i+1) - start
          let cbuf: *u8 = g[130] as *u8                    // clean parenthetical/IPA/citation junk first
          let cln: i64 = qa_clean_sentence(src, start, ln, cbuf)
          if qa_is_prose(cbuf, 0, cln) == 1 { if ns < 400 { if bump+cln < SEM_MAGIC_260000 {
              var w: i64 = 0
              while w < cln { sb[bump+w] = cbuf[w]; w = w + 1 }
              sb[bump+cln] = (0 as u8)
              so[ns]=bump; sl[ns]=cln; sp[ns]=0; ns=ns+1; bump=bump+cln+1
          } } }
          var sk: i64 = i+1
          var go: i64 = 1
          while go == 1 { if sk < slen { if src[sk]==(32 as u8) { sk=sk+1 } else { go=0 } } else { go=0 } }
          start = sk
          i = sk
      } else { i = i + 1 }
  }
  return ns
135}
136
137func qab_split_ctx(g: *i64, src: *u8, slen: i64) -> i64 {
  let sb: *u8 = g[15] as *u8
  let so: *i64 = g[16] as *i64
  let sl: *i64 = g[17] as *i64
  let sp: *i64 = g[18] as *i64
  var ns: i64 = 0
  var bump: i64 = 0
  var start: i64 = 0
  var i: i64 = 0
  while i < slen {
      var boundary: i64 = 0
      let c: i64 = src[i] as i64
      if c==46 { boundary=1 }
      if c==63 { boundary=1 }
      if c==33 { boundary=1 }
      if boundary==1 {
          var nxsp: i64 = 1
          if i+1 < slen { let d: i64 = src[i+1] as i64; if d!=32 { if d!=10 { nxsp=0 } } }
          if nxsp==0 { boundary=0 }
      }
      if boundary==1 {
          let ln: i64 = (i+1) - start
          if ln > 2 { if ns < 80 { if bump+ln < SEM_MAGIC_260000 {
              var w: i64 = 0
              while w < ln { sb[bump+w] = src[start+w]; w = w + 1 }
              sb[bump+ln] = (0 as u8)                 // terminate: qs_norm reads to null; sb is reused across rows
              so[ns]=bump; sl[ns]=ln; sp[ns]=0; ns=ns+1; bump=bump+ln+1
          } } }
          var sk: i64 = i+1
          var go: i64 = 1
          while go == 1 { if sk < slen { if src[sk]==(32 as u8) { sk=sk+1 } else { go=0 } } else { go=0 } }
          start = sk
          i = sk
      } else { i = i + 1 }
  }
  if start < slen { let ln2: i64 = slen - start
      if ln2 > 2 { if ns < 80 { if bump+ln2 < SEM_MAGIC_260000 {
          var w2: i64 = 0
          while w2 < ln2 { sb[bump+w2] = src[start+w2]; w2 = w2 + 1 }
          sb[bump+ln2] = (0 as u8)
          so[ns]=bump; sl[ns]=ln2; sp[ns]=0; ns=ns+1
      } } }
  }
  return ns
181}
182
183// match a JSON bracket: pos at '[' or '{' -> index of the matching close (string-aware). -1 if unbalanced.
184func db_match_bracket(g: *i64, pos: i64) -> i64 {
  let open: i64 = db_b(g, pos)
  var close: i64 = 93
  if open == 123 { close = 125 }
  var depth: i64 = 0
  var i: i64 = pos
  var instr: i64 = 0
  while i < g[1] {
      let c: i64 = db_b(g, i)
      if instr == 1 {
          if c == 92 { i = i + 1 } else { if c == 34 { instr = 0 } }
      } else {
          if c == 34 { instr = 1 } else {
              if c == open { depth = depth + 1 }
              if c == close { depth = depth - 1; if depth == 0 { return i } }
          }
      }
      i = i + 1
  }
  return 0 - 1
204}
205
206// append sentences from src[0..slen) into sb/so/sl/sp with sp=para; st=[ns,bump] updated in place.
207func qab_split_append(g: *i64, src: *u8, slen: i64, para: i64, st: *i64) -> i64 {
  let sb: *u8 = g[15] as *u8
  let so: *i64 = g[16] as *i64
  let sl: *i64 = g[17] as *i64
  let sp: *i64 = g[18] as *i64
  var ns: i64 = st[0]
  var bump: i64 = st[1]
  var start: i64 = 0
  var i: i64 = 0
  while i < slen {
      var boundary: i64 = 0
      let c: i64 = src[i] as i64
      if c==46 { boundary=1 }
      if c==63 { boundary=1 }
      if c==33 { boundary=1 }
      if boundary==1 {
          var nxsp: i64 = 1
          if i+1 < slen { let d: i64 = src[i+1] as i64; if d!=32 { if d!=10 { nxsp=0 } } }
          if nxsp==0 { boundary=0 }
      }
      if boundary==1 {
          let ln: i64 = (i+1) - start
          if ln > 2 { if ns < 1000 { if bump+ln < SEM_MAGIC_260000 {
              var w: i64 = 0
              while w < ln { sb[bump+w] = src[start+w]; w = w + 1 }
              sb[bump+ln] = (0 as u8)                 // terminate: qs_norm reads to null; sb is reused across rows
              so[ns]=bump; sl[ns]=ln; sp[ns]=para; ns=ns+1; bump=bump+ln+1
          } } }
          var sk: i64 = i+1
          var go: i64 = 1
          while go == 1 { if sk < slen { if src[sk]==(32 as u8) { sk=sk+1 } else { go=0 } } else { go=0 } }
          start = sk
          i = sk
      } else { i = i + 1 }
  }
  if start < slen { let ln2: i64 = slen - start
      if ln2 > 2 { if ns < 1000 { if bump+ln2 < SEM_MAGIC_260000 {
          var w2: i64 = 0
          while w2 < ln2 { sb[bump+w2] = src[start+w2]; w2 = w2 + 1 }
          sb[bump+ln2] = (0 as u8)
          so[ns]=bump; sl[ns]=ln2; sp[ns]=para; ns=ns+1
      } } }
  }
  st[0]=ns; st[1]=bump
  return 0
252}
253
254// ==================== R3-LITE: SEMANTIC sentence selection (MODE2) ====================
255// Distributional co-occurrence embedding built from the question's OWN paragraphs (the nx_distrib_embed
256// method, per-row, zero training, sovereign): vocab of content words -> windowed co-occurrence matrix ->
257// sentence/question vectors = sums of member-word rows -> integer cosine. Extraction UNCHANGED -> isolates
258// the SELECTION variable.
259// ★MEASURED 2026-07-08 (kept as the R3 baseline column, NOT a win): F1_SEM vs F1_lex = SQuAD 149 vs 158,
260// HotpotQA 130 vs 146, MuSiQue 24 vs 45 -- per-question co-occurrence is WORSE everywhere and degrades MOST
261// where paragraphs are many (distractor mass poisons the stats; the question vector drifts to hub sentences).
262// 3rd principled climb, 3rd measured negative => the semantic wall needs CORPUS-LEVEL embeddings (R3b-scale
263// 519-doc thread) or a trained bi-encoder (R1-adjacent), NOT per-row stats. This column is the plug-in point:
264// swap sem_build_cooc's source for a persisted corpus matrix and the delta shows here.
265// ctx slots: 60 C(vc*vc i64) 61 qvec 62 svec 63 voff 64 vlen 65 vcount 66 vbuf 67 vbump
266//            68 pred2(256B) 69 vidx(per-sentence vocab ids)
267const SEM_VMAX: i64 = 1024
268const SEM_W: i64 = 6
269
270func sem_isqrt(v: i64) -> i64 {
  if v <= 0 { return 0 }
  var x: i64 = v
  var y: i64 = (x + 1) / 2
  while y < x { x = y; let q: i64 = v / x; y = (x + q) / 2 }
  return x
276}
277
278// find token (buf,off,len) in the vocab; -1 if absent
279func sem_vfind(g: *i64, buf: *u8, off: i64, len: i64) -> i64 {
  let vbuf: *u8 = g[66] as *u8
  let voff: *i64 = g[63] as *i64
  let vlen: *i64 = g[64] as *i64
  var i: i64 = 0
  while i < g[65] {
      if qs_tok_eq2(buf, off, len, vbuf, voff[i], vlen[i]) == 1 { return i }
      i = i + 1
  }
  return 0-1
289}
290
291// find-or-append; returns idx or -1 (caps hit)
292func sem_vadd(g: *i64, buf: *u8, off: i64, len: i64) -> i64 {
  let f: i64 = sem_vfind(g, buf, off, len)
  if f >= 0 { return f }
  if g[65] >= SEM_VMAX { return 0-1 }
  if g[67] + len + 1 >= SEM_MAGIC_32760 { return 0-1 }
  let vbuf: *u8 = g[66] as *u8
  let voff: *i64 = g[63] as *i64
  let vlen: *i64 = g[64] as *i64
  let idx: i64 = g[65]
  let bump: i64 = g[67]
  var w: i64 = 0
  while w < len { vbuf[bump+w] = buf[off+w]; w = w + 1 }
  voff[idx] = bump; vlen[idx] = len
  g[65] = idx + 1
  g[67] = bump + len + 1
  return idx
308}
309
310// vocab = question content words + all sentence content words (stop-filtered, len>=2)
311func sem_build_vocab(g: *i64) -> i64 {
  g[65] = 0
  g[67] = 0
  let qnb: *u8 = g[28] as *u8
  let qto: *i64 = g[29] as *i64
  let qtl: *i64 = g[30] as *i64
  let qk: *i64 = g[32] as *i64
  var k: i64 = 0
  while k < g[31] {
      if qk[k] == 1 { if qtl[k] >= 2 { sem_vadd(g, qnb, qto[k], qtl[k]) } }
      k = k + 1
  }
  let nb: *u8 = g[20] as *u8
  let tko: *i64 = g[23] as *i64
  let tkl: *i64 = g[24] as *i64
  let tks: *i64 = g[25] as *i64
  let tkc: *i64 = g[26] as *i64
  var s: i64 = 0
  while s < g[19] {
      var j: i64 = 0
      while j < tkc[s] {
          let ti: i64 = tks[s] + j
          if tkl[ti] >= 2 { if db_is_stop(nb, tko[ti], tkl[ti]) == 0 { sem_vadd(g, nb, tko[ti], tkl[ti]) } }
          j = j + 1
      }
      s = s + 1
  }
  return g[65]
339}
340
341// windowed co-occurrence over each sentence's vocab-member tokens
342func sem_build_cooc(g: *i64) -> i64 {
  let C: *i64 = g[60] as *i64
  let vc: i64 = g[65]
  var z: i64 = 0
  let zn: i64 = vc * vc
  while z < zn { C[z] = 0; z = z + 1 }
  let nb: *u8 = g[20] as *u8
  let tko: *i64 = g[23] as *i64
  let tkl: *i64 = g[24] as *i64
  let tks: *i64 = g[25] as *i64
  let tkc: *i64 = g[26] as *i64
  let vidx: *i64 = g[69] as *i64
  var s: i64 = 0
  while s < g[19] {
      var n: i64 = tkc[s]
      if n > 250 { n = 250 }
      var j: i64 = 0
      while j < n {
          let ti: i64 = tks[s] + j
          let v: i64 = sem_vfind(g, nb, tko[ti], tkl[ti])
          vidx[j] = v
          j = j + 1
      }
      var a: i64 = 0
      while a < n {
          if vidx[a] >= 0 {
              var b: i64 = a + 1
              var bend: i64 = a + SEM_W
              if bend > n-1 { bend = n-1 }
              while b <= bend {
                  if vidx[b] >= 0 {
                      let ia: i64 = vidx[a]*vc + vidx[b]
                      let ib: i64 = vidx[b]*vc + vidx[a]
                      C[ia] = C[ia] + 1
                      C[ib] = C[ib] + 1
                  }
                  b = b + 1
              }
          }
          a = a + 1
      }
      s = s + 1
  }
  return 0
386}
387
388// vec += C-row of vocab id t
389func sem_addrow(g: *i64, vec: *i64, t: i64) -> i64 {
  let C: *i64 = g[60] as *i64
  let vc: i64 = g[65]
  let base: i64 = t * vc
  var k: i64 = 0
  while k < vc { vec[k] = vec[k] + C[base + k]; k = k + 1 }
  return 0
396}
397
398// question vector = sum of rows of its content words present in vocab. returns #hits.
399func sem_qvec(g: *i64, vec: *i64) -> i64 {
  let vc: i64 = g[65]
  var k: i64 = 0
  while k < vc { vec[k] = 0; k = k + 1 }
  let qnb: *u8 = g[28] as *u8
  let qto: *i64 = g[29] as *i64
  let qtl: *i64 = g[30] as *i64
  let qk: *i64 = g[32] as *i64
  var hits: i64 = 0
  var i: i64 = 0
  while i < g[31] {
      if qk[i] == 1 {
          let v: i64 = sem_vfind(g, qnb, qto[i], qtl[i])
          if v >= 0 { sem_addrow(g, vec, v); hits = hits + 1 }
      }
      i = i + 1
  }
  return hits
417}
418
419func sem_svec(g: *i64, s: i64, vec: *i64) -> i64 {
  let vc: i64 = g[65]
  var k: i64 = 0
  while k < vc { vec[k] = 0; k = k + 1 }
  let nb: *u8 = g[20] as *u8
  let tko: *i64 = g[23] as *i64
  let tkl: *i64 = g[24] as *i64
  let tks: *i64 = g[25] as *i64
  let tkc: *i64 = g[26] as *i64
  var hits: i64 = 0
  var j: i64 = 0
  while j < tkc[s] {
      let ti: i64 = tks[s] + j
      let v: i64 = sem_vfind(g, nb, tko[ti], tkl[ti])
      if v >= 0 { sem_addrow(g, vec, v); hits = hits + 1 }
      j = j + 1
  }
  return hits
437}
438
439// integer cosine in permille
440func sem_cos(a: *i64, b: *i64, vc: i64) -> i64 {
  var dot: i64 = 0
  var na: i64 = 0
  var nb2: i64 = 0
  var k: i64 = 0
  while k < vc {
      dot = dot + a[k]*b[k]
      na = na + a[k]*a[k]
      nb2 = nb2 + b[k]*b[k]
      k = k + 1
  }
  if dot <= 0 { return 0 }
  let d1: i64 = sem_isqrt(na)
  let d2: i64 = sem_isqrt(nb2)
  return (dot/(d1+1))*1000/(d2+1)
455}
456
457// ---------- PPMI-soft path (loads knowledge/index/semppmi_v1.bin when present; else per-row fallback) ----------
458// slots: 70 blob base, 71 nv, 72 nt, 73 vh, 74 ridx, 75 nrm2, 76 tctx, 77 tval, 78 loaded-flag
459func ppmi_load(g: *i64) -> i64 {
  g[78] = 0
  let fd: i64 = sys_openat_rd("knowledge/index/semppmi_v1.bin" as *u8)
  if fd < 0 { return 0 }
  // TWO-PHASE read (no fixed whole-file cap -- the exact "grew past a fixed buffer" debt class): read the
  // 32B header first, size the blob from nv/nt, then read the rest. Sanity-capped at 512MB.
  let hdrb: *u8 = sys_mmap(SEM_MAGIC_4096)
  var hgot: i64 = 0
  var hr: i64 = 1
  while hr > 0 { if hgot >= 32 { hr = 0 } else { hr = sys_read(fd, (hdrb as i64 + hgot) as *u8, 32 - hgot); if hr > 0 { hgot = hgot + hr } } }
  if hgot < 32 { sys_close(fd); return 0 }
  let hh: *i64 = (hdrb as i64 + 8) as *i64
  let hnv: i64 = hh[0]
  let hnt: i64 = hh[1]
  let need0: i64 = 32 + (hnv*8) + ((hnv+1)*8) + (hnv*8) + (hnt*8) + (hnt*8)
  if need0 <= 32 { sys_close(fd); return 0 }
  if need0 > SEM_MAGIC_536870912 { sys_close(fd); return 0 }
  let cap: i64 = need0 + SEM_MAGIC_4096
  let blob: *u8 = sys_mmap(cap)
  var i0: i64 = 0
  while i0 < 32 { blob[i0] = hdrb[i0]; i0 = i0 + 1 }
  var total: i64 = 32
  var r: i64 = 1
  while r > 0 {
      let left: i64 = need0 - total
      if left <= 0 { r = 0 } else {
          var want: i64 = SEM_MAGIC_262144
          if want > left { want = left }
          r = sys_read(fd, (blob as i64 + total) as *u8, want)
          if r > 0 { total = total + r }
      }
  }
  sys_close(fd)
  if total < 64 { return 0 }
  if blob[0] != (78 as u8) { return 0 }
  if blob[6] != (49 as u8) { return 0 }
  let hi: *i64 = (blob as i64 + 8) as *i64
  let nv: i64 = hi[0]
  let nt: i64 = hi[1]
  let need: i64 = 32 + (nv*8) + ((nv+1)*8) + (nv*8) + (nt*8) + (nt*8)
  if total < need { return 0 }
  g[70] = blob as i64
  g[71] = nv
  g[72] = nt
  var off: i64 = 32
  g[73] = (blob as i64) + off; off = off + nv*8
  g[74] = (blob as i64) + off; off = off + (nv+1)*8
  g[75] = (blob as i64) + off; off = off + nv*8
  g[76] = (blob as i64) + off; off = off + nt*8
  g[77] = (blob as i64) + off
  g[78] = 1
  return 1
511}
512
513// cosine between two PPMI rows (sorted sparse merge), permille
514func ppmi_cos(g: *i64, a: i64, b: i64) -> i64 {
  let ridx: *i64 = g[74] as *i64
  let nrm2: *i64 = g[75] as *i64
  let tctx: *i64 = g[76] as *i64
  let tval: *i64 = g[77] as *i64
  var ia: i64 = ridx[a]
  var ib: i64 = ridx[b]
  let ea: i64 = ridx[a+1]
  let eb: i64 = ridx[b+1]
  var dot: i64 = 0
  while ia < ea {
      if ib >= eb { ia = ea } else {
          if tctx[ia] == tctx[ib] { dot = dot + tval[ia]*tval[ib]; ia = ia + 1; ib = ib + 1 }
          else { if tctx[ia] < tctx[ib] { ia = ia + 1 } else { ib = ib + 1 } }
      }
  }
  if dot <= 0 { return 0 }
  let d1: i64 = sem_isqrt(nrm2[a])
  let d2: i64 = sem_isqrt(nrm2[b])
  if d1 == 0 { return 0 }
  if d2 == 0 { return 0 }
  var cv: i64 = (dot*1000)/(d1*d2)
  if cv > 1000 { cv = 1000 }
  return cv
538}
539
540// resolve token (buf,off,len) -> PPMI vocab id or -1
541func ppmi_wid(g: *i64, buf: *u8, off: i64, len: i64) -> i64 {
  let hv: i64 = db_semhash(buf, off, len)
  return db_bsearch_i64(g[73] as *i64, g[71], hv)
544}
545
546// PPMI-SOFT sentence selection: score(s) = SUM over question content words q of MAX over sentence words w of
547// cos_ppmi(q,w) -- soft lexical alignment ("won" aligns to "defeated"). Identity pairs count 1000 (exact match
548// keeps its full weight). RETRIEVE-THEN-READ scope: when pa>=0, only sentences in paragraphs {pa,pb} are
549// scored (paragraph retrieval stays IDF-lexical -- semantics only refines WITHIN the retrieved scope; global
550// soft-matching inflated distractor hits on 20-para MuSiQue). Returns best sentence or -1.
551func ppmi_best_sentence(g: *i64, pa: i64, pb: i64) -> i64 {
  let qnb: *u8 = g[28] as *u8
  let qto: *i64 = g[29] as *i64
  let qtl: *i64 = g[30] as *i64
  let qk: *i64 = g[32] as *i64
  let nb: *u8 = g[20] as *u8
  let tko: *i64 = g[23] as *i64
  let tkl: *i64 = g[24] as *i64
  let tks: *i64 = g[25] as *i64
  let tkc: *i64 = g[26] as *i64
  // resolve question ids once
  let qid: *i64 = g[61] as *i64                    // reuse qvec buffer as id scratch
  var nq: i64 = 0
  var k: i64 = 0
  while k < g[31] {
      if qk[k] == 1 { if nq < 64 {
          let wid: i64 = ppmi_wid(g, qnb, qto[k], qtl[k])
          if wid >= 0 { qid[nq] = wid; nq = nq + 1 }
      } }
      k = k + 1
  }
  if nq < 1 { return 0-1 }
  let sid: *i64 = g[62] as *i64                    // per-sentence word ids scratch
  let spp: *i64 = g[18] as *i64                    // sentence -> paragraph
  var best: i64 = 0-1
  var bs: i64 = 0
  var s: i64 = 0
  while s < g[19] {
      var inscope: i64 = 1
      if pa >= 0 { inscope = 0; if spp[s] == pa { inscope = 1 } if pb >= 0 { if spp[s] == pb { inscope = 1 } } }
      if inscope == 1 {
      var nsids: i64 = 0
      var j: i64 = 0
      while j < tkc[s] {
          if nsids < 250 {
              let ti: i64 = tks[s] + j
              let wj: i64 = ppmi_wid(g, nb, tko[ti], tkl[ti])
              sid[nsids] = wj
              nsids = nsids + 1
          }
          j = j + 1
      }
      var score: i64 = 0
      var qi: i64 = 0
      while qi < nq {
          var mx: i64 = 0
          var si: i64 = 0
          while si < nsids {
              if sid[si] >= 0 {
                  var cv: i64 = 0
                  if sid[si] == qid[qi] { cv = 1000 } else { cv = ppmi_cos(g, qid[qi], sid[si]) }
                  if cv > mx { mx = cv }
              }
              si = si + 1
          }
          score = score + mx
          qi = qi + 1
      }
      if score > bs { bs = score; best = s }
      }
      s = s + 1
  }
  return best
614}
615
616// MODE2 core: best sentence by semantic similarity to the question. PPMI-soft when the corpus file is loaded
617// (g[78]==1), SCOPED to the top-2 retrieved paragraphs (retrieve-then-read); else the per-row co-occurrence
618// fallback (the recorded measured-negative baseline). -1 if degenerate.
619func sem_best_sentence(g: *i64, pa: i64, pb: i64) -> i64 {
  if g[78] == 1 { return ppmi_best_sentence(g, pa, pb) }
  return sem_best_sentence_perrow(g)
622}
623
624func sem_best_sentence_perrow(g: *i64) -> i64 {
  let vcnt: i64 = sem_build_vocab(g)
  if vcnt < 8 { return 0-1 }
  sem_build_cooc(g)
  let qvec: *i64 = g[61] as *i64
  let svec: *i64 = g[62] as *i64
  let qh: i64 = sem_qvec(g, qvec)
  if qh < 1 { return 0-1 }
  var best: i64 = 0-1
  var bs: i64 = 0
  var s: i64 = 0
  while s < g[19] {
      let sh: i64 = sem_svec(g, s, svec)
      if sh >= 1 {
          let c: i64 = sem_cos(qvec, svec, g[65])
          if c > bs { bs = c; best = s }
      }
      s = s + 1
  }
  return best
644}
645// ==================== end R3-LITE ====================
646
647// ---- MuSiQue (2-4 hop) row loader. -1 EOF, 0 bad, 1 ok. JSON key order: paragraphs, question,
648//      question_decomposition (nested q/a -> bracket-skip), answer, answer_aliases, answerable.
649//      gt* slots = answer_aliases (fmt=3 max-F1). Skips unanswerable rows. ----
650func qab_load_musique(g: *i64) -> i64 {
  if db_find_key(g, "paragraphs" as *u8) < 0 { return 0-1 }
  db_skip_ws(g)
  if db_b(g, g[3]) != 91 { return 0 }
  let arr_open: i64 = g[3]
  let arr_end: i64 = db_match_bracket(g, arr_open)
  if arr_end < 0 { return 0 }
  let ptb: *u8 = g[11] as *u8
  let pto: *i64 = g[12] as *i64
  let ptl: *i64 = g[13] as *i64
  let ctxb: *u8 = g[59] as *u8
  let stbox: *i64 = g[100] as *i64                 // preallocated (was sys_mmap per row -> leak over training)
  stbox[0]=0; stbox[1]=0                            // ns, sb bump
  var np: i64 = 0
  var tbump: i64 = 0
  g[3] = arr_open + 1
  var live: i64 = 1
  while live == 1 {
      if np >= 24 { live = 0 } else {
      if db_find_key(g, "title" as *u8) < 0 { live = 0 } else {
      if g[3] >= arr_end { live = 0 } else {
          let tl: i64 = db_dec_str(g, (ptb as i64 + tbump) as *u8, SEM_MAGIC_4000 - tbump)
          pto[np] = tbump; ptl[np] = tl; tbump = tbump + tl + 1
          if db_find_key(g, "paragraph_text" as *u8) < 0 { live = 0 } else {
              let cl: i64 = db_dec_str(g, ctxb, SEM_MAGIC_200000)
              qab_split_append(g, ctxb, cl, np, stbox)
              np = np + 1
          }
      } } }
  }
  g[14] = np
  g[19] = stbox[0]
  g[3] = arr_end + 1
  // question
  if db_find_key(g, "question" as *u8) < 0 { return 0 }
  let ql: i64 = db_dec_str(g, g[4] as *u8, SEM_MAGIC_4000)
  // skip question_decomposition (nested question/answer)
  if db_find_key(g, "question_decomposition" as *u8) >= 0 {
      db_skip_ws(g)
      if db_b(g, g[3]) == 91 { let qd_end: i64 = db_match_bracket(g, g[3]); if qd_end >= 0 { g[3] = qd_end + 1 } }
  }
  // main answer
  if db_find_key(g, "answer" as *u8) < 0 { return 0 }
  let al: i64 = db_dec_str(g, g[5] as *u8, SEM_MAGIC_2000)
  // answer_aliases -> gt* (refs)
  var nref: i64 = 0
  if db_find_key(g, "answer_aliases" as *u8) >= 0 { nref = db_parse_str_arr(g, g[7] as *u8, SEM_MAGIC_4000, g[8] as *i64, g[9] as *i64, 20) }
  if nref < 0 { nref = 0 }
  g[10] = nref
  // answerable -> skip unanswerable
  if db_find_key(g, "answerable" as *u8) >= 0 { db_skip_ws(g); if db_b(g, g[3]) == 102 { return 0 } }   // 'f' = false
  if ql <= 0 { return 0 }
  if al <= 0 { return 0 }
  if np < 2 { return 0 }
  if g[19] <= 0 { return 0 }
  return 1
706}
707
708// ---- SQuAD row loader. -1 EOF, 0 bad, 1 ok. gt* slots repurposed as ANSWER REFS (fmt=1). ----
709func qab_load_squad(g: *i64) -> i64 {
  if db_find_key(g, "context" as *u8) < 0 { return 0-1 }
  let ctxb: *u8 = g[59] as *u8
  let cl: i64 = db_dec_str(g, ctxb, SEM_MAGIC_200000)
  db_find_key(g, "question" as *u8)
  let ql: i64 = db_dec_str(g, g[4] as *u8, SEM_MAGIC_4000)
  db_find_key(g, "answers" as *u8)
  db_find_key(g, "text" as *u8)
  let nref: i64 = db_parse_str_arr(g, g[7] as *u8, SEM_MAGIC_4000, g[8] as *i64, g[9] as *i64, 20)
  g[10] = nref
  // primary gold answer -> a (g[5])
  if nref > 0 {
      let gtb: *u8 = g[7] as *u8
      let gto: *i64 = g[8] as *i64
      let gtl: *i64 = g[9] as *i64
      let a: *u8 = g[5] as *u8
      var w: i64 = 0
      while w < gtl[0] { if w < 250 { a[w] = gtb[gto[0]+w] } w = w + 1 }
      var e: i64 = gtl[0]; if e > 250 { e = 250 }
      a[e] = (0 as u8)
  }
  // one paragraph, empty title, sentences = split(context)
  g[14] = 1
  let pto: *i64 = g[12] as *i64
  let ptl: *i64 = g[13] as *i64
  let ptb: *u8 = g[11] as *u8
  pto[0]=0; ptl[0]=0; ptb[0]=(0 as u8)
  let ns: i64 = qab_split_ctx(g, ctxb, cl)
  g[19] = ns
  if ql <= 0 { return 0 }
  if nref < 1 { return 0 }
  if ns <= 0 { return 0 }
  return 1
742}
743
744// ---- HotpotQA distractor row loader (mirrors nx_drbench). -1 EOF, 0 bad, 1 ok. gt*=gold titles. ----
745func qab_load_hotpot(g: *i64) -> i64 {
  if db_find_key(g, "question" as *u8) < 0 { return 0-1 }
  let ql: i64 = db_dec_str(g, g[4] as *u8, SEM_MAGIC_4000)
  db_find_key(g, "answer" as *u8)
  let al: i64 = db_dec_str(g, g[5] as *u8, SEM_MAGIC_2000)
  db_find_key(g, "type" as *u8)
  db_dec_str(g, g[6] as *u8, 60)
  db_find_key(g, "supporting_facts" as *u8)
  db_find_key(g, "title" as *u8)
  let gtb: *u8 = g[7] as *u8
  let gto: *i64 = g[8] as *i64
  let gtl: *i64 = g[9] as *i64
  let rawgt: i64 = db_parse_str_arr(g, gtb, SEM_MAGIC_4000, gto, gtl, 30)
  g[10] = 0
  var gi: i64 = 0
  while gi < rawgt {
      var dup: i64 = 0
      var gj: i64 = 0
      while gj < g[10] {
          if gtl[gi] == gtl[gj] {
              var eq: i64 = 1
              var w: i64 = 0
              while w < gtl[gi] { if gtb[gto[gi]+w] != gtb[gto[gj]+w] { eq = 0 } w = w + 1 }
              if eq == 1 { dup = 1 }
          }
          gj = gj + 1
      }
      if dup == 0 { let nd: i64 = g[10]; gto[nd] = gto[gi]; gtl[nd] = gtl[gi]; g[10] = nd + 1 }
      gi = gi + 1
  }
  db_find_key(g, "context" as *u8)
  db_find_key(g, "title" as *u8)
  let np0: i64 = db_parse_str_arr(g, g[11] as *u8, SEM_MAGIC_8000, g[12] as *i64, g[13] as *i64, 16)
  g[14] = np0
  db_find_key(g, "sentences" as *u8)
  let npara2: i64 = db_parse_sentences(g)
  if ql <= 0 { return 0 }
  if al <= 0 { return 0 }
  if g[14] < 2 { return 0 }
  if npara2 != g[14] { return 0 }
  if g[19] <= 0 { return 0 }
  if g[10] < 1 { return 0 }
  return 1
788}
789
790func qab_load_row(g: *i64, fmt: i64) -> i64 {
  if fmt == 1 { return qab_load_squad(g) }
  if fmt == 3 { return qab_load_musique(g) }
  return qab_load_hotpot(g)
794}
795
796// F1 (permille): for SQuAD (fmt=1) take MAX over answer refs (SQuAD protocol); else single gold answer.
797func qab_f1(g: *i64, fmt: i64, pred: *u8) -> i64 {
  var best: i64 = qs_f1(pred, g[5] as *u8)
  if fmt >= 1 {
      let gtb: *u8 = g[7] as *u8
      let gto: *i64 = g[8] as *i64
      let gtl: *i64 = g[9] as *i64
      var r: i64 = 0
      while r < g[10] {
          let f: i64 = qs_f1(pred, (gtb as i64 + gto[r]) as *u8)
          if f > best { best = f }
          r = r + 1
      }
  }
  return best
811}
812func qab_em(g: *i64, fmt: i64, pred: *u8) -> i64 {
  var best: i64 = qs_em(pred, g[5] as *u8)
  if fmt >= 1 {
      let gtb: *u8 = g[7] as *u8
      let gto: *i64 = g[8] as *i64
      let gtl: *i64 = g[9] as *i64
      var r: i64 = 0
      while r < g[10] {
          let e: i64 = qs_em(pred, (gtb as i64 + gto[r]) as *u8)
          if e > best { best = e }
          r = r + 1
      }
  }
  return best
826}
827
828// ==================== MODE-R: SUPERVISED SPAN READER (SQuAD-baseline lineage) ====================
829// The 6x-corpus datapoint proved SELECTION is no longer the binding stage -- EXTRACTION (which candidate span
830// to emit) is. This is the trained reader for that stage: a structured (averaged) perceptron over candidate
831// answer spans, exactly the Rajpurkar-2016 SQuAD-baseline recipe (candidate spans + lexical/type features +
832// a learned linear scorer, which got 51 F1 where sliding-window heuristics got 20). Sovereign/integer/no-float.
833// TRAIN on the qab_*_c* corpus rows (DISJOINT from the eval subsets) -> persist reader_v1.bin -> eval MODE-R.
834// Replaces db_extract's fixed pass-order (multiword>noncommon>any) with a learned ranking; the extractor's
835// candidate GENERATION is unchanged, so this isolates the scoring variable. Reports candidate-recall ceiling.
836// ctx: 79 loaded 80 w[NF] 81 featmat[300*NF] 82 meta[300] 84 predR 85 wsum[NF] 86 candtmp 87 avg-count
837// NF=14, featmat stride = NF*8 = 112. TWO principled scoring features were tried against the measured binding
838// stage (oracle 727 vs reader 251) and BOTH failed the linear perceptron: (a) positional proximity-to-anchor
839// (NF16) = NET-NEGATIVE (SQuAD 251->214); (b) corpus-PPMI semantic relatedness cand<->question (NF15) = INERT
840// (perceptron learned weight 0, F1 250~=251). => the gold-vs-distractor signal is NOT a linear function of
841// hand-features; the scoring gap needs LEARNED REPRESENTATIONS = the neural R1 no-float reader into this socket.
842const RD_NF: i64 = 14
843
844func rd_wants_entity(g: *i64) -> i64 {
  let qnb: *u8 = g[28] as *u8
  let qto: *i64 = g[29] as *i64
  let qtl: *i64 = g[30] as *i64
  var i: i64 = 0
  while i < g[31] {
      let o: i64 = qto[i]; let l: i64 = qtl[i]
      if db_tok_is(qnb, o, l, "who" as *u8) == 1 { return 1 }
      if db_tok_is(qnb, o, l, "whom" as *u8) == 1 { return 1 }
      if db_tok_is(qnb, o, l, "whose" as *u8) == 1 { return 1 }
      if db_tok_is(qnb, o, l, "which" as *u8) == 1 { return 1 }
      i = i + 1
  }
  return 0
858}
859
860func rd_wants_place(g: *i64) -> i64 {
  let qnb: *u8 = g[28] as *u8
  let qto: *i64 = g[29] as *i64
  let qtl: *i64 = g[30] as *i64
  var i: i64 = 0
  while i < g[31] {
      let o: i64 = qto[i]; let l: i64 = qtl[i]
      if db_tok_is(qnb, o, l, "where" as *u8) == 1 { return 1 }
      if db_tok_is(qnb, o, l, "city" as *u8) == 1 { return 1 }
      if db_tok_is(qnb, o, l, "country" as *u8) == 1 { return 1 }
      if db_tok_is(qnb, o, l, "town" as *u8) == 1 { return 1 }
      if db_tok_is(qnb, o, l, "state" as *u8) == 1 { return 1 }
      if db_tok_is(qnb, o, l, "capital" as *u8) == 1 { return 1 }
      if db_tok_is(qnb, o, l, "located" as *u8) == 1 { return 1 }
      if db_tok_is(qnb, o, l, "venue" as *u8) == 1 { return 1 }
      i = i + 1
  }
  return 0
878}
879
880// word count of candidate ci
881func rd_candwc(g: *i64, ci: i64) -> i64 {
  let cnd: *u8 = g[48] as *u8
  let cno: *i64 = g[49] as *i64
  let cnl: *i64 = g[50] as *i64
  let o: i64 = cno[ci]; let l: i64 = cnl[ci]
  var wc: i64 = 1
  var i: i64 = 0
  while i < l { if cnd[o+i] == (32 as u8) { wc = wc + 1 } i = i + 1 }
  return wc
890}
891
892// featurize candidate ci (db_candidates for its sentence must be current). best_s = the top sentence.
893func rd_feat(g: *i64, ci: i64, s: i64, best_s: i64, feat: *i64) -> i64 {
  let cnt: *i64 = g[51] as *i64
  let shit: *i64 = g[39] as *i64
  let mw: i64 = db_cand_multiword(g, ci)
  var isnum: i64 = 0
  if cnt[ci] == 2 { isnum = 1 }
  if cnt[ci] == 3 { isnum = 1 }
  var isyr: i64 = 0
  if cnt[ci] == 3 { isyr = 1 }
  let ech: i64 = db_cand_is_echo(g, ci)
  let com: i64 = db_cand_is_common(g, ci)
  let wn: i64 = db_prefer_numeric(g)
  let wy: i64 = db_wants_year(g)
  let we: i64 = rd_wants_entity(g)
  let wp: i64 = rd_wants_place(g)
  let wc: i64 = rd_candwc(g, ci)
  feat[0] = 10
  var f1v: i64 = shit[s]
  if f1v > 10 { f1v = 10 }
  feat[1] = f1v
  feat[2] = mw * 10
  feat[3] = isnum * 10
  feat[4] = isyr * 10
  feat[5] = ech * 10
  feat[6] = com * 10
  var f7v: i64 = wc
  if f7v > 6 { f7v = 6 }
  feat[7] = f7v
  var f8v: i64 = 0
  if wn == 1 { if isnum == 1 { f8v = 10 } }
  feat[8] = f8v
  var f9v: i64 = 0
  if we == 1 { if mw == 1 { f9v = 10 } }
  feat[9] = f9v
  var f10v: i64 = 0
  if wp == 1 { if mw == 1 { f10v = 10 } }
  feat[10] = f10v
  var f11v: i64 = 0
  if wy == 1 { if isyr == 1 { f11v = 10 } }
  feat[11] = f11v
  var f12v: i64 = 0
  if s == best_s { f12v = 10 }
  feat[12] = f12v
  var f13v: i64 = 0
  if ci == 0 { f13v = 5 }
  feat[13] = f13v
  return 0
940}
941
942func rd_score(g: *i64, feat: *i64) -> i64 {
  let w: *i64 = g[80] as *i64
  var acc: i64 = 0
  var k: i64 = 0
  while k < RD_NF { acc = acc + w[k]*feat[k]; k = k + 1 }
  return acc
948}
949
950// reconstruct candidate ci string of sentence s into out (re-runs db_candidates for s). returns length.
951// out buffers are 2048B; a candidate can be up to ~2000B (db_candidates bump cap) so BOUND the copy at 250
952// (a real answer span is short; a longer candidate is never the gold and only needs a stable non-overflowing key).
953func rd_candstr(g: *i64, s: i64, ci: i64, out: *u8) -> i64 {
  let so: *i64 = g[16] as *i64
  let sl: *i64 = g[17] as *i64
  db_candidates(g, so[s], sl[s])
  if ci >= g[52] { out[0] = (0 as u8); return 0 }
  let cnd: *u8 = g[48] as *u8
  let cno: *i64 = g[49] as *i64
  let cnl: *i64 = g[50] as *i64
  let o: i64 = cno[ci]
  var l: i64 = cnl[ci]
  if l > 250 { l = 250 }
  var i: i64 = 0
  while i < l { out[i] = cnd[o+i]; i = i + 1 }
  out[l] = (0 as u8)
  return l
968}
969
970// training-side F1 (word-F1 permille, same formula as qs_f1) using PREALLOCATED ctx buffers -- qs_f1 mmaps 7
971// buffers per call, which is fine for eval's few-thousand calls but OOMs at training's millions. Tokenize gold
972// ONCE per row (rd_setgold), then score each candidate with rd_f1 -- zero per-call allocation.
973// slots: 92 gold-norm 93 gold-toff 94 gold-tlen 95 cand-norm 96 cand-toff 97 cand-tlen 98 used 99 gold-ntok
974func rd_setgold(g: *i64, gold: *u8) -> i64 {
  let gnb: *u8 = g[92] as *u8
  let gto: *i64 = g[93] as *i64
  let gtl: *i64 = g[94] as *i64
  let gl: i64 = qs_norm(gold, gnb)
  let ng: i64 = qs_tok(gnb, gl, gto, gtl, 256)
  g[99] = ng
  return ng
982}
983func rd_f1(g: *i64, cand: *u8) -> i64 {
  let cnb: *u8 = g[95] as *u8
  let cto: *i64 = g[96] as *i64
  let ctl: *i64 = g[97] as *i64
  let cl: i64 = qs_norm(cand, cnb)
  let np: i64 = qs_tok(cnb, cl, cto, ctl, 256)
  let ng: i64 = g[99]
  if np == 0 { if ng == 0 { return 1000 } return 0 }
  if ng == 0 { return 0 }
  let used: *i64 = g[98] as *i64
  let gnb: *u8 = g[92] as *u8
  let gto: *i64 = g[93] as *i64
  let gtl: *i64 = g[94] as *i64
  let common: i64 = qs_common(cnb, cto, ctl, np, gnb, gto, gtl, ng, used)
  if common == 0 { return 0 }
  let prec: i64 = (common*1000)/np
  let rec: i64 = (common*1000)/ng
  return (2*prec*rec)/(prec+rec)
1001}
1002
1003// enumerate all candidates across the top-2 retrieved paragraphs; fill featmat + meta. returns count.
1004func rd_enumerate(g: *i64, top1: i64, top2: i64, best_s: i64) -> i64 {
  let so: *i64 = g[16] as *i64
  let sl: *i64 = g[17] as *i64
  let sp: *i64 = g[18] as *i64
  let meta: *i64 = g[82] as *i64
  var nc: i64 = 0
  var s: i64 = 0
  while s < g[19] {
      var inscope: i64 = 1
      if top1 >= 0 { inscope = 0; if sp[s] == top1 { inscope = 1 } if top2 >= 0 { if sp[s] == top2 { inscope = 1 } } }
      if inscope == 1 {
          db_candidates(g, so[s], sl[s])
          let ncand: i64 = g[52]
          var ci: i64 = 0
          while ci < ncand {
              if nc < 300 {
                  let fp: *i64 = ((g[81] as i64) + nc*112) as *i64
                  rd_feat(g, ci, s, best_s, fp)
                  meta[nc] = s*SEM_MAGIC_1024 + ci
                  nc = nc + 1
              }
              ci = ci + 1
          }
      }
      s = s + 1
  }
  return nc
1031}
1032
1033// ORACLE ceiling: max F1 over the enumerated candidate set vs gold. If this >> the reader's F1, SCORING has
1034// headroom; if ~= the reader's F1, candidate GENERATION (recall) is the wall. The measure-first probe that
1035// decides which stage to attack next (per the "measure the real path before optimizing" lesson).
1036func rd_oracle(g: *i64, top1: i64, top2: i64, best_s: i64, gold: *u8) -> i64 {
  let nc: i64 = rd_enumerate(g, top1, top2, best_s)
  if nc == 0 { return 0 }
  rd_setgold(g, gold)
  let meta: *i64 = g[82] as *i64
  let tmp: *u8 = g[86] as *u8
  var best: i64 = 0
  var i: i64 = 0
  while i < nc {
      let mv: i64 = meta[i]
      let s: i64 = mv / SEM_MAGIC_1024
      let ci: i64 = mv % SEM_MAGIC_1024
      rd_candstr(g, s, ci, tmp)
      let f: i64 = rd_f1(g, tmp)
      if f > best { best = f }
      i = i + 1
  }
  return best
1054}
1055
1056// inference: pick best-scored candidate into pred. 1 ok, 0 = no candidates (caller falls back).
1057func rd_pick(g: *i64, top1: i64, top2: i64, best_s: i64, pred: *u8) -> i64 {
  let nc: i64 = rd_enumerate(g, top1, top2, best_s)
  if nc == 0 { return 0 }
  var best: i64 = 0-1
  var bs: i64 = 0-SEM_MAGIC_2000000000
  var i: i64 = 0
  while i < nc {
      let sc: i64 = rd_score(g, ((g[81] as i64) + i*112) as *i64)
      if sc > bs { bs = sc; best = i }
      i = i + 1
  }
  if best < 0 { return 0 }
  let meta: *i64 = g[82] as *i64
  let mv: i64 = meta[best]
  let s: i64 = mv / SEM_MAGIC_1024
  let ci: i64 = mv % SEM_MAGIC_1024
  let n: i64 = rd_candstr(g, s, ci, pred)
  if n <= 0 { return 0 }
  return 1
1076}
1077
1078// one training row: label candidates by F1 vs gold; structured-perceptron update. returns 1 if gold recoverable.
1079func rd_train_row(g: *i64, gold: *u8, top1: i64, top2: i64, best_s: i64) -> i64 {
  let nc: i64 = rd_enumerate(g, top1, top2, best_s)
  if nc == 0 { return 0 }
  let meta: *i64 = g[82] as *i64
  let tmp: *u8 = g[86] as *u8
  rd_setgold(g, gold)                                 // tokenize gold ONCE (rd_f1 reuses ctx buffers, no leak)
  var goldi: i64 = 0-1
  var goldf: i64 = 0
  var i: i64 = 0
  while i < nc {
      let mv: i64 = meta[i]
      let s: i64 = mv / SEM_MAGIC_1024
      let ci: i64 = mv % SEM_MAGIC_1024
      rd_candstr(g, s, ci, tmp)
      let f: i64 = rd_f1(g, tmp)
      if f > goldf { goldf = f; goldi = i }
      i = i + 1
  }
  if dbg == 1 { db_w("      [rr] labeled goldi="); db_n(goldi); db_w(" goldf="); db_n(goldf); db_w("\n" as *u8) }
  if goldf < 500 { return 0 }                        // gold not in candidate set -> unlearnable (recall miss)
  var predi: i64 = 0-1
  var bs: i64 = 0-SEM_MAGIC_2000000000
  i = 0
  while i < nc {
      let sc: i64 = rd_score(g, ((g[81] as i64) + i*112) as *i64)
      if sc > bs { bs = sc; predi = i }
      i = i + 1
  }
  if predi != goldi {
      let w: *i64 = g[80] as *i64
      let fg: *i64 = ((g[81] as i64) + goldi*112) as *i64
      let fp: *i64 = ((g[81] as i64) + predi*112) as *i64
      var k: i64 = 0
      while k < RD_NF { w[k] = w[k] + fg[k] - fp[k]; k = k + 1 }
  }
  return 1
1115}
1116
1117// walk one corpus file, train per row (averaged into wsum). returns [trained, recoverable] via g[88]/g[89] add.
1118func rd_train_file(g: *i64, path: *u8, fmt: i64) -> i64 {
  let total: i64 = db_read_raw(g, path, SEM_MAGIC_16777216)
  if total <= 0 { return 0 }
  g[2] = db_body_start(g, total)
  g[1] = total - g[2]
  g[3] = 0
  var trained: i64 = 0
  var live: i64 = 1
  while live == 1 {
      let rc: i64 = qab_load_row(g, fmt)
      if rc < 0 { live = 0 } else {
          if rc == 1 {
              db_norm_row(g)
              db_score_row(g)
              let top1: i64 = db_top_para(g, 0-1)
              let top2: i64 = db_top_para(g, top1)
              let s0: i64 = db_best_sentence(g)
              let yn: i64 = db_is_yesno(g)
              g[89] = g[89] + 1                        // rows seen (non-degenerate)
              if yn == 0 {
                  let did: i64 = rd_train_row(g, g[5] as *u8, top1, top2, s0)
                  if did == 1 {
                      trained = trained + 1
                      g[88] = g[88] + 1                // recoverable-gold rows
                      let w: *i64 = g[80] as *i64
                      let ws: *i64 = g[85] as *i64
                      var k: i64 = 0
                      while k < RD_NF { ws[k] = ws[k] + w[k]; k = k + 1 }
                      g[87] = g[87] + 1
                  }
              }
          }
      }
  }
  return trained
1153}
1154
1155func rd_persist(g: *i64) -> i64 {
  let hdr: *u8 = sys_mmap(256)
  hdr[0]=78 as u8; hdr[1]=88 as u8; hdr[2]=82 as u8; hdr[3]=68 as u8; hdr[4]=82 as u8; hdr[5]=49 as u8; hdr[6]=0 as u8; hdr[7]=0 as u8
  let hi: *i64 = (hdr as i64 + 8) as *i64
  hi[0] = RD_NF
  let wb: *i64 = (hdr as i64 + 16) as *i64
  let w: *i64 = g[80] as *i64
  var k: i64 = 0
  while k < RD_NF { wb[k] = w[k]; k = k + 1 }
  let fd: i64 = sys_openat_wr("knowledge/index/reader_v1.bin" as *u8, 0x1a4)
  if fd < 0 { return 1 }
  var off: i64 = 0
  let tot: i64 = 16 + RD_NF*8
  while off < tot { let ww: i64 = sys_write(fd, (hdr as i64 + off) as *u8, tot - off); if ww <= 0 { off = tot } else { off = off + ww } }
  sys_close(fd)
  return 0
1171}
1172
1173// train the reader on the disjoint corpus chunks (E averaged-perceptron epochs), persist, load into g[80].
1174func rd_train(g: *i64) -> i64 {
  let w: *i64 = g[80] as *i64
  let ws: *i64 = g[85] as *i64
  var k: i64 = 0
  while k < RD_NF { w[k] = 0; ws[k] = 0; k = k + 1 }
  g[87] = 0; g[88] = 0; g[89] = 0
  let E: i64 = 6
  var ep: i64 = 0
  while ep < E {
      rd_train_file(g, "knowledge/fetched/qab_squad_c0.raw" as *u8, 1)
      rd_train_file(g, "knowledge/fetched/qab_squad_c1.raw" as *u8, 1)
      rd_train_file(g, "knowledge/fetched/qab_squad_c2.raw" as *u8, 1)
      rd_train_file(g, "knowledge/fetched/qab_hotpot_c0.raw" as *u8, 0)
      rd_train_file(g, "knowledge/fetched/qab_hotpot_c1.raw" as *u8, 0)
      rd_train_file(g, "knowledge/fetched/qab_hotpot_c2.raw" as *u8, 0)
      rd_train_file(g, "knowledge/fetched/qab_musique_c0.raw" as *u8, 3)
      rd_train_file(g, "knowledge/fetched/qab_musique_c1.raw" as *u8, 3)
      ep = ep + 1
  }
  if g[87] > 0 { k = 0; while k < RD_NF { w[k] = ws[k] / g[87]; k = k + 1 } }
  var recall: i64 = 0
  if g[89] > 0 { recall = (g[88] * 1000) / g[89] }
  db_w("  reader trained: "); db_n(g[88]); db_w(" recoverable-gold rows / "); db_n(g[89]); db_w(" seen (candidate-recall="); db_n(recall); db_w("permille, over E="); db_n(E); db_w(" epochs)\n" as *u8)
  rd_persist(g)
  g[79] = 1
  return 0
1200}
1201
1202func rd_load(g: *i64) -> i64 {
  g[79] = 0
  let fd: i64 = sys_openat_rd("knowledge/index/reader_v1.bin" as *u8)
  if fd < 0 { return 0 }
  let buf: *u8 = sys_mmap(256)
  var got: i64 = 0
  var r: i64 = 1
  while r > 0 { if got >= 128 { r = 0 } else { r = sys_read(fd, (buf as i64 + got) as *u8, 128 - got); if r > 0 { got = got + r } } }
  sys_close(fd)
  if got < 16 { return 0 }
  if buf[0] != (78 as u8) { return 0 }
  if buf[5] != (49 as u8) { return 0 }
  let hi: *i64 = (buf as i64 + 8) as *i64
  if hi[0] != RD_NF { return 0 }
  let wb: *i64 = (buf as i64 + 16) as *i64
  let w: *i64 = g[80] as *i64
  var k: i64 = 0
  while k < RD_NF { w[k] = wb[k]; k = k + 1 }
  g[79] = 1
  return 1
1222}
1223
1224// DUMP_NF = 14 hand-features + 3 corpus-PPMI EMBEDDING features (semantic relatedness the hand-features lack).
1225// The rung-2 finding was that hand-features are the ceiling -> test whether embedding features break it. Kept
1226// in the DUMP only (not rd_feat), so the main perceptron reader stays at RD_NF=14/251 unchanged.
1227const DUMP_NF: i64 = 17
1228
1229// ---- TRAINED dense embeddings (embed_v1.bin from nx_embed_train): Q10 integer E[nv*DIM]; integer cosine =
1230//      no-float inference. Loaded alongside semppmi (same vocab ids from ppmi vh). slots: 115 E 116 DIM 117 flag
1231func embed_load(g: *i64) -> i64 {
  g[117] = 0
  let fd: i64 = sys_openat_rd("knowledge/index/embed_v1.bin" as *u8)
  if fd < 0 { return 0 }
  let hb: *u8 = sys_mmap(64)
  var hg: i64 = 0
  var hr: i64 = 1
  while hr > 0 { if hg >= 24 { hr = 0 } else { hr = sys_read(fd, (hb as i64 + hg) as *u8, 24 - hg); if hr > 0 { hg = hg + hr } } }
  if hg < 24 { sys_close(fd); return 0 }
  if hb[0] != (78 as u8) { sys_close(fd); return 0 }        // 'N' (NXEMB1)
  let hi: *i64 = (hb as i64 + 8) as *i64
  let nv: i64 = hi[0]
  let dim: i64 = hi[1]
  let need: i64 = nv * dim * 8
  if need <= 0 { sys_close(fd); return 0 }
  if need > SEM_MAGIC_402653184 { sys_close(fd); return 0 }           // 384MB sanity
  let buf: *u8 = sys_mmap(need + SEM_MAGIC_4096)
  var got: i64 = 0
  var r: i64 = 1
  while r > 0 { let left: i64 = need - got; if left <= 0 { r = 0 } else { r = sys_read(fd, (buf as i64 + got) as *u8, left); if r > 0 { got = got + r } } }
  sys_close(fd)
  if got < need { return 0 }
  g[115] = buf as i64
  g[116] = dim
  g[117] = 1
  return 1
1257}
1258
1259// integer cosine (permille) of trained dense embedding rows a,b (Q10 ints; scale cancels in cosine).
1260func dense_cos_int(g: *i64, a: i64, b: i64) -> i64 {
  let E: *i64 = g[115] as *i64
  let dim: i64 = g[116]
  var dot: i64 = 0
  var na: i64 = 0
  var nb: i64 = 0
  var k: i64 = 0
  let ba: i64 = a * dim
  let bb: i64 = b * dim
  while k < dim {
      let ea: i64 = E[ba+k]
      let eb: i64 = E[bb+k]
      dot = dot + ea*eb
      na = na + ea*ea
      nb = nb + eb*eb
      k = k + 1
  }
  if dot <= 0 { return 0 }
  let d1: i64 = sem_isqrt(na)
  let d2: i64 = sem_isqrt(nb)
  if d1 == 0 { return 0 }
  if d2 == 0 { return 0 }
  var cv: i64 = (dot*1000)/(d1*d2)
  if cv > 1000 { cv = 1000 }
  return cv
1285}
1286
1287// resolve the question's content-word PPMI ids into g[108] (count g[109]) once per row.
1288func rd_setqppmi(g: *i64) -> i64 {
  g[109] = 0
  if g[78] != 1 { return 0 }
  let qnb: *u8 = g[28] as *u8
  let qto: *i64 = g[29] as *i64
  let qtl: *i64 = g[30] as *i64
  let qk: *i64 = g[32] as *i64
  let qid: *i64 = g[108] as *i64
  var n: i64 = 0
  var k: i64 = 0
  while k < g[31] {
      if qk[k] == 1 { if n < 64 { let wid: i64 = ppmi_wid(g, qnb, qto[k], qtl[k]); if wid >= 0 { qid[n] = wid; n = n + 1 } } }
      k = k + 1
  }
  g[109] = n
  return n
1304}
1305
1306// 3 corpus-PPMI embedding features for a candidate STRING vs the question: out[0]=max token-alignment,
1307// out[1]=mean token-alignment, out[2]=#candidate tokens strongly related to a q-word. All ~0..10.
1308func rd_emb_feats(g: *i64, candstr: *u8, out: *i64) -> i64 {
  out[0] = 0; out[1] = 0; out[2] = 0
  if g[78] != 1 { return 0 }
  if g[109] == 0 { return 0 }
  let nrm: *u8 = g[111] as *u8
  let cto: *i64 = g[112] as *i64
  let ctl: *i64 = g[113] as *i64
  let nl: i64 = qs_norm(candstr, nrm)
  let nt: i64 = qs_tok(nrm, nl, cto, ctl, 16)
  let qid: *i64 = g[108] as *i64
  var best: i64 = 0
  var sum: i64 = 0
  var ntok: i64 = 0
  var cnt: i64 = 0
  var t: i64 = 0
  while t < nt {
      let cid: i64 = ppmi_wid(g, nrm, cto[t], ctl[t])
      if cid >= 0 {
          var tb: i64 = 0
          var q: i64 = 0
          while q < g[109] {
              var c: i64 = 0
              if g[117] == 1 { c = dense_cos_int(g, cid, qid[q]) } else { c = ppmi_cos(g, cid, qid[q]) }
              if c > tb { tb = c }
              q = q + 1
          }
          if tb > best { best = tb }
          sum = sum + tb; ntok = ntok + 1
          if tb > 300 { cnt = cnt + 1 }
      }
      t = t + 1
  }
  out[0] = best / 100
  if ntok > 0 { out[1] = (sum / ntok) / 100 }
  out[2] = cnt
  if out[2] > 10 { out[2] = 10 }
  return 0
1345}
1346
1347// ---- FULL-VECTOR embedding features (the R1 crux): instead of collapsing to a scalar cosine, feed the MLP
1348//      the element-wise product of the L2-normalized question and candidate embedding vectors (DIM values) so
1349//      it learns a per-dimension weighted metric. slots: 118 q_emb_unit  119 cand_unit scratch ----
1350func rd_setqemb(g: *i64) -> i64 {
  let dim: i64 = g[116]
  let qu: *i64 = g[118] as *i64
  var d: i64 = 0
  while d < dim { qu[d] = 0; d = d + 1 }
  if g[117] != 1 { return 0 }
  if g[109] == 0 { return 0 }
  let qid: *i64 = g[108] as *i64
  let E: *i64 = g[115] as *i64
  var n: i64 = 0
  while n < g[109] {
      let id: i64 = qid[n]
      d = 0
      while d < dim { qu[d] = qu[d] + E[id*dim+d]; d = d + 1 }
      n = n + 1
  }
  var nrm: i64 = 0
  d = 0
  while d < dim { nrm = nrm + qu[d]*qu[d]; d = d + 1 }
  let rt: i64 = sem_isqrt(nrm)
  if rt == 0 { return 0 }
  d = 0
  while d < dim { qu[d] = qu[d]*SEM_MAGIC_1024/rt; d = d + 1 }    // Q10 unit vector
  return 1
1374}
1375
1376// element-wise product of q_emb_unit and cand_emb_unit, each component scaled to ~0..10. out must hold DIM.
1377func rd_emb_vec(g: *i64, candstr: *u8, out: *i64) -> i64 {
  let dim: i64 = g[116]
  var d: i64 = 0
  while d < dim { out[d] = 5; d = d + 1 }               // neutral default
  if g[117] != 1 { return 0 }
  if g[109] == 0 { return 0 }
  let nrm2: *u8 = g[111] as *u8
  let cto: *i64 = g[112] as *i64
  let ctl: *i64 = g[113] as *i64
  let cu: *i64 = g[119] as *i64
  d = 0
  while d < dim { cu[d] = 0; d = d + 1 }
  let nl: i64 = qs_norm(candstr, nrm2)
  let nt: i64 = qs_tok(nrm2, nl, cto, ctl, 16)
  let E: *i64 = g[115] as *i64
  var nseen: i64 = 0
  var t: i64 = 0
  while t < nt {
      let cid: i64 = ppmi_wid(g, nrm2, cto[t], ctl[t])
      if cid >= 0 { d = 0; while d < dim { cu[d] = cu[d] + E[cid*dim+d]; d = d + 1 } nseen = nseen + 1 }
      t = t + 1
  }
  if nseen == 0 { return 0 }
  var cn: i64 = 0
  d = 0
  while d < dim { cn = cn + cu[d]*cu[d]; d = d + 1 }
  let crt: i64 = sem_isqrt(cn)
  if crt == 0 { return 0 }
  d = 0
  while d < dim { cu[d] = cu[d]*SEM_MAGIC_1024/crt; d = d + 1 }
  let qu: *i64 = g[118] as *i64
  d = 0
  while d < dim {
      let p: i64 = qu[d]*cu[d]/SEM_MAGIC_1024                     // Q10 product, ~+-SEM_MAGIC_1024
      var f: i64 = p/20 + 5                             // scale to ~0..10 (matches hand-feature range)
      if f < 0 { f = 0 }
      if f > 10 { f = 10 }
      out[d] = f
      d = d + 1
  }
  return 1
1418}
1419
1420// ---- TEXT DUMP (R3 neural reader): export per-row RAW (question, context, gold-answer) strings so the
1421//      offline neural trainer (nx_reader_squad_gate on nx_nofloat_autograd) does its own tokenization + gold
1422//      span matching. File: knowledge/index/reader_rows.bin = [magic "NXRR1"] then per row:
1423//      [qlen i64][q bytes][clen i64][ctx bytes = prose sentences space-joined][alen i64][gold bytes].
1424//      Selected by g[131]=1 (dumptext mode 'dt'); the feats path is untouched when the flag is 0. ----
1425func rd_dump_text_row(g: *i64, gold: *u8, fd: i64) -> i64 {
  let q: *u8 = g[4] as *u8
  let qlen: i64 = qa_slen(q)
  let alen: i64 = qa_slen(gold)
  if qlen < 4 { return 0 }
  if alen < 1 { return 0 }
  if g[19] <= 0 { return 0 }
  let sb: *u8 = g[15] as *u8
  let so: *i64 = g[16] as *i64
  var clen: i64 = 0
  var s: i64 = 0
  while s < g[19] { let sl: i64 = qa_slen((sb as i64 + so[s]) as *u8); clen = clen + sl + 1; s = s + 1 }
  if clen < 8 { return 0 }
  let hb: *i64 = g[86] as *i64
  hb[0] = qlen
  sys_write(fd, g[86] as *u8, 8)
  sys_write(fd, q, qlen)
  hb[0] = clen
  sys_write(fd, g[86] as *u8, 8)
  s = 0
  while s < g[19] {
      let sp0: *u8 = (sb as i64 + so[s]) as *u8
      let sl: i64 = qa_slen(sp0)
      sys_write(fd, sp0, sl)
      let sc: *u8 = g[86] as *u8
      sc[0] = 32 as u8
      sys_write(fd, sc, 1)
      s = s + 1
  }
  hb[0] = alen
  sys_write(fd, g[86] as *u8, 8)
  sys_write(fd, gold, alen)
  return 1
1458}
1459
1460func rd_dump_texts(g: *i64) -> i64 {
  let fd: i64 = sys_openat_wr("knowledge/index/reader_rows.bin" as *u8, 0x1a4)
  if fd < 0 { db_w("dumptext: cannot open reader_rows.bin\n" as *u8); return 1 }
  let mgb: *u8 = g[86] as *u8
  mgb[0]=78 as u8; mgb[1]=88 as u8; mgb[2]=82 as u8; mgb[3]=82 as u8; mgb[4]=49 as u8; mgb[5]=0 as u8; mgb[6]=0 as u8; mgb[7]=0 as u8
  sys_write(fd, g[86] as *u8, 8)
  g[88] = 0
  g[131] = 1
  rd_dump_file(g, "knowledge/fetched/qab_squad_c0.raw" as *u8, 1, fd, SEM_MAGIC_10000)
  rd_dump_file(g, "knowledge/fetched/qab_squad_c1.raw" as *u8, 1, fd, SEM_MAGIC_10000)
  g[131] = 0
  sys_close(fd)
  db_w("dumptext: wrote "); db_n(g[88]); db_w(" SQuAD rows (raw q/ctx/gold text) -> knowledge/index/reader_rows.bin\n" as *u8)
  return 0
1474}
1475
1476// XL text dump ('dx'): ALL banked SQuAD chunks -> reader_rows_xl.bin. SEPARATE FILE by design: the in-flight
1477// f32 run's checkpoint is keyed to reader_rows.bin's vocab (nw) -- overwriting it mid-arc would invalidate the
1478// checkpoint on the next watchdog relaunch. The XL set is the R3f data-scale lever, staged in advance.
1479func rd_dump_texts_xl(g: *i64) -> i64 {
  let fd: i64 = sys_openat_wr("knowledge/index/reader_rows_xl.bin" as *u8, 0x1a4)
  if fd < 0 { db_w("dumptext-xl: cannot open reader_rows_xl.bin\n" as *u8); return 1 }
  let mgb: *u8 = g[86] as *u8
  mgb[0]=78 as u8; mgb[1]=88 as u8; mgb[2]=82 as u8; mgb[3]=82 as u8; mgb[4]=49 as u8; mgb[5]=0 as u8; mgb[6]=0 as u8; mgb[7]=0 as u8
  sys_write(fd, g[86] as *u8, 8)
  g[88] = 0
  g[131] = 1
  rd_dump_file(g, "knowledge/fetched/qab_squad_c0.raw" as *u8, 1, fd, SEM_MAGIC_10000)
  rd_dump_file(g, "knowledge/fetched/qab_squad_c1.raw" as *u8, 1, fd, SEM_MAGIC_10000)
  rd_dump_file(g, "knowledge/fetched/qab_squad_c2.raw" as *u8, 1, fd, SEM_MAGIC_10000)
  rd_dump_file(g, "knowledge/fetched/qab_squad_c3.raw" as *u8, 1, fd, SEM_MAGIC_10000)
  rd_dump_file(g, "knowledge/fetched/qab_squad_c4.raw" as *u8, 1, fd, SEM_MAGIC_10000)
  rd_dump_file(g, "knowledge/fetched/qab_squad_c5.raw" as *u8, 1, fd, SEM_MAGIC_10000)
  rd_dump_file(g, "knowledge/fetched/qab_squad_c6.raw" as *u8, 1, fd, SEM_MAGIC_10000)
  rd_dump_file(g, "knowledge/fetched/qab_squad_c7.raw" as *u8, 1, fd, SEM_MAGIC_10000)
  rd_dump_file(g, "knowledge/fetched/qab_squad_c8.raw" as *u8, 1, fd, SEM_MAGIC_10000)
  rd_dump_file(g, "knowledge/fetched/qab_squad_big.raw" as *u8, 1, fd, SEM_MAGIC_10000)
  g[131] = 0
  sys_close(fd)
  db_w("dumptext-xl: wrote "); db_n(g[88]); db_w(" SQuAD rows -> knowledge/index/reader_rows_xl.bin\n" as *u8)
  return 0
1501}
1502
1503// ---- feature DUMP: export per-row candidate features + gold index for OFFLINE MLP training (nx_reader_mlp_train
1504//      trains the neural reader on nx_autograd; nx_qabench stays integer/no-float). File: knowledge/index/
1505//      reader_feats.bin = [magic "NXRF1", DUMP_NF] then per row: [nc, goldi, feat[nc*DUMP_NF]] (14 hand + 3 emb). ----
1506func rd_dump_row(g: *i64, gold: *u8, top1: i64, top2: i64, best_s: i64, fd: i64) -> i64 {
  if g[131] == 1 { return rd_dump_text_row(g, gold, fd) }
  let nc: i64 = rd_enumerate(g, top1, top2, best_s)
  if nc < 2 { return 0 }
  if nc > 64 { return 0 }
  rd_setgold(g, gold)
  let meta: *i64 = g[82] as *i64
  let tmp: *u8 = g[86] as *u8
  var goldi: i64 = 0-1
  var goldf: i64 = 0
  var i: i64 = 0
  while i < nc {
      let mv: i64 = meta[i]
      let s: i64 = mv / SEM_MAGIC_1024
      let ci: i64 = mv % SEM_MAGIC_1024
      rd_candstr(g, s, ci, tmp)
      let f: i64 = rd_f1(g, tmp)
      if f > goldf { goldf = f; goldi = i }
      i = i + 1
  }
  if goldf < 500 { return 0 }                          // gold not recoverable -> skip
  rd_setqppmi(g)                                        // question PPMI ids
  let vecmode: i64 = g[117]                             // 1 = dense embeddings loaded -> FULL-VECTOR features
  if vecmode == 1 { rd_setqemb(g) }                    // q_emb_unit once per row
  let hb: *i64 = g[86] as *i64
  hb[0] = nc; hb[1] = goldi
  sys_write(fd, g[86] as *u8, 16)
  let emb: *i64 = g[114] as *i64                        // 3-scalar buffer (sparse mode)
  let vbuf: *i64 = g[121] as *i64                       // DIM-vector buffer (vector mode)
  var wc: i64 = 0
  while wc < nc {
      sys_write(fd, (g[81] as i64 + wc*RD_NF*8) as *u8, RD_NF * 8)   // 14 hand-features for candidate wc
      let mv: i64 = meta[wc]
      let s: i64 = mv / SEM_MAGIC_1024
      let ci: i64 = mv % SEM_MAGIC_1024
      rd_candstr(g, s, ci, g[110] as *u8)
      if vecmode == 1 { rd_emb_vec(g, g[110] as *u8, vbuf); sys_write(fd, g[121] as *u8, g[116] * 8) }   // DIM vector feats
      else { rd_emb_feats(g, g[110] as *u8, emb); sys_write(fd, g[114] as *u8, 24) }                     // 3 scalar feats
      wc = wc + 1
  }
  return 1
1547}
1548
1549func rd_dump_file(g: *i64, path: *u8, fmt: i64, fd: i64, cap: i64) -> i64 {
  let total: i64 = db_read_raw(g, path, SEM_MAGIC_16777216)
  if total <= 0 { return 0 }
  g[2] = db_body_start(g, total)
  g[1] = total - g[2]
  g[3] = 0
  let start88: i64 = g[88]                     // debt eaten SEM_MAGIC_2026-07-10: cap is now PER-FILE. It was compared
  var live: i64 = 1                            // against the GLOBAL row counter, so a multi-file dump silently
  while live == 1 {                            // truncated at `cap` TOTAL rows (the XL dump lost half its data).
      if g[88] - start88 >= cap { live = 0 } else {
          let rc: i64 = qab_load_row(g, fmt)
          if rc < 0 { live = 0 } else {
              if rc == 1 {
                  db_norm_row(g)
                  db_score_row(g)
                  let top1: i64 = db_top_para(g, 0-1)
                  let top2: i64 = db_top_para(g, top1)
                  let s0: i64 = db_best_sentence(g)
                  let yn: i64 = db_is_yesno(g)
                  if yn == 0 { if rd_dump_row(g, g[5] as *u8, top1, top2, s0, fd) == 1 { g[88] = g[88] + 1 } }
              }
          }
      }
  }
  return g[88]
1574}
1575
1576func rd_dump_feats(g: *i64) -> i64 {
  let fd: i64 = sys_openat_wr("knowledge/index/reader_feats.bin" as *u8, 0x1a4)
  if fd < 0 { db_w("dumpfeats: cannot open reader_feats.bin\n" as *u8); return 1 }
  let mgb: *u8 = g[86] as *u8
  mgb[0]=78 as u8; mgb[1]=88 as u8; mgb[2]=82 as u8; mgb[3]=70 as u8; mgb[4]=49 as u8; mgb[5]=0 as u8; mgb[6]=0 as u8; mgb[7]=0 as u8
  let mg: *i64 = g[86] as *i64
  var nf: i64 = DUMP_NF                                 // sparse: 14 hand + 3 scalar embed
  if g[117] == 1 { nf = RD_NF + g[116] }               // vector mode: 14 hand + DIM full-vector product
  mg[1] = nf
  db_w("dumpfeats: feature count nf="); db_n(nf); db_w("\n" as *u8)
  sys_write(fd, g[86] as *u8, 16)
  g[88] = 0
  let CAP: i64 = SEM_MAGIC_2500
  rd_dump_file(g, "knowledge/fetched/qab_squad_c0.raw" as *u8, 1, fd, CAP)
  rd_dump_file(g, "knowledge/fetched/qab_squad_c1.raw" as *u8, 1, fd, CAP)
  rd_dump_file(g, "knowledge/fetched/qab_hotpot_c0.raw" as *u8, 0, fd, CAP)
  rd_dump_file(g, "knowledge/fetched/qab_hotpot_c1.raw" as *u8, 0, fd, CAP)
  sys_close(fd)
  db_w("dumpfeats: wrote "); db_n(g[88]); db_w(" rows (DUMP_NF="); db_n(DUMP_NF); db_w(", incl 3 PPMI-embed) -> knowledge/index/reader_feats.bin\n" as *u8)
  return 0
1596}
1597// ==================== end MODE-R ====================
1598
1599// run one benchmark end-to-end; write [rows, f1_single, f1_multi, em] into res[base..base+3]. returns rows.
1600func qab_run(g: *i64, path: *u8, fmt: i64, label: *u8, res: *i64, base: i64) -> i64 {
  let total: i64 = db_read_raw(g, path, SEM_MAGIC_16777216)
  if total == 0-2 { db_w("  "); db_w(label); db_w(": [file EXCEEDS 16MB raw buffer -- raise m0, skipped]\n" as *u8); res[base]=0; return 0 }
  if total <= 0 { db_w("  "); db_w(label); db_w(": [absent -- fetch first, skipped]\n" as *u8); res[base]=0; return 0 }
  g[2] = db_body_start(g, total)
  g[1] = total - g[2]
  let p0a: *u8 = g[54] as *u8
  let gaa: *u8 = g[56] as *u8
1608
  var rows: i64 = 0
  var sum_f0: i64 = 0; var sum_f1: i64 = 0; var sum_f2: i64 = 0; var sum_fR: i64 = 0; var sum_orc: i64 = 0; var em0: i64 = 0
  g[3] = 0
  var live: i64 = 1
  let DBGL: i64 = 0
  var QLIMIT: i64 = 0                       // optional row cap (knowledge/index/qab_limit.txt) -> bounded on-box runs
  let lfd: i64 = sys_openat_rd("knowledge/index/qab_limit.txt" as *u8)
  if lfd >= 0 {
      let lb: *u8 = sys_mmap(32)
      let lrn: i64 = sys_read(lfd, lb, 31)
      sys_close(lfd)
      var lci: i64 = 0
      while lci < lrn { if lb[lci]>=(48 as u8) { if lb[lci]<=(57 as u8) { QLIMIT = QLIMIT*10 + (lb[lci] as i64 - 48) } } lci=lci+1 }
  }
  while live == 1 {
      let rc: i64 = qab_load_row(g, fmt)
      if DBGL == 1 { if fmt == 3 { db_w("MQ load rows="); db_n(rows); db_w(" rc="); db_n(rc); db_w(" np="); db_n(g[14]); db_w(" ns="); db_n(g[19]); db_w(" nqt="); db_n(g[31]); db_w("\n" as *u8) } }
      if rc < 0 { live = 0 } else {
      if rc == 1 {
          db_norm_row(g)
          if DBGL == 1 { if fmt == 3 { db_w("  MQ normed\n" as *u8) } }
          db_score_row(g)
          if DBGL == 1 { if fmt == 3 { db_w("  MQ scored\n" as *u8) } }
          let top1: i64 = db_top_para(g, 0-1)
          let top2: i64 = db_top_para(g, top1)
          let s0: i64 = db_best_sentence(g)
          let yn: i64 = db_is_yesno(g)
          let wantnum0: i64 = db_prefer_numeric(g)
          let pd0: *u8 = (p0a as i64 + rows*256) as *u8
          var qw0: i64 = 0
          if g[132] == 1 { if qwx_ready() == 1 { if s0 >= 0 { qw0 = qwx_extract(g, s0, pd0) } } }   // R5: Qwen reads context centered on best sentence s0 (mode0/SQuAD = 251 rung)
          if qw0 == 0 {
              if yn == 1 { db_setpred(pd0, "yes" as *u8, 3) } else {
                  var got0: i64 = 0
                  if wantnum0 == 1 { got0 = db_sweep_numeric(g, top1, top2, pd0) }
                  if got0 == 0 { if s0 >= 0 { db_extract(g, s0, pd0) } else { db_setpred(pd0, "none" as *u8, 4) } }
              }
          }
          // multi-hop (mode1)
          var x2: i64 = 0-1
          if s0 >= 0 { if top1 >= 0 {
              db_bridge_from(g, s0, top1)
              let psc: *i64 = g[42] as *i64
              let xsc: *i64 = g[43] as *i64
              var pp: i64 = 0
              while pp < g[14] {
                  if pp == top1 { xsc[pp] = 0-1000 } else {
                      let bt: i64 = db_bridge_title_hits(g, pp)
                      let bb: i64 = db_bridge_body_hits(g, pp)
                      xsc[pp] = psc[pp] + 3*bt + 2*bb
                  }
                  pp = pp + 1
              }
              var bx2: i64 = 0-1
              var bxs: i64 = 0-SEM_MAGIC_1000000
              pp = 0
              while pp < g[14] { if xsc[pp] > bxs { bxs = xsc[pp]; bx2 = pp } pp = pp + 1 }
              x2 = bx2
          } }
          let pd1: *u8 = (g[55] as i64 + rows*256) as *u8
          if yn == 1 { db_setpred(pd1, "yes" as *u8, 3) } else {
              var got1: i64 = 0
              if wantnum0 == 1 { got1 = db_sweep_numeric(g, top1, x2, pd1) }
              if got1 == 0 {
                  var s2: i64 = 0-1
                  if x2 >= 0 { s2 = db_best_sentence_in(g, x2) }
                  var sfin: i64 = s2
                  if sfin < 0 { sfin = s0 }
                  let shit: *i64 = g[39] as *i64
                  if s0 >= 0 { if s2 >= 0 { if shit[s0] > shit[s2] + 1 { sfin = s0 } } }
                  if sfin >= 0 { db_extract(g, sfin, pd1) } else { db_setpred(pd1, "none" as *u8, 4) }
              }
          }
          // ---- mode 2: SEMANTIC sentence selection (R3-lite; same extraction, different selection) ----
          let pd2: *u8 = g[68] as *u8
          if yn == 1 { db_setpred(pd2, "yes" as *u8, 3) } else {
              var got2: i64 = 0
              if wantnum0 == 1 { got2 = db_sweep_numeric(g, top1, top2, pd2) }
              if got2 == 0 {
                  let ss: i64 = sem_best_sentence(g, top1, top2)
                  var sf2: i64 = ss
                  if sf2 < 0 { sf2 = s0 }
                  if sf2 >= 0 { db_extract(g, sf2, pd2) } else { db_setpred(pd2, "none" as *u8, 4) }
              }
          }
          // ---- mode R: TRAINED span reader (learned candidate ranking; same candidate GENERATION) ----
          let pdR: *u8 = g[84] as *u8
          if g[79] == 1 {
              if yn == 1 { db_setpred(pdR, "yes" as *u8, 3) } else {
                  var gotR: i64 = 0
                  if wantnum0 == 1 { gotR = db_sweep_numeric(g, top1, top2, pdR) }
                  if gotR == 0 {
                      let okR: i64 = rd_pick(g, top1, top2, s0, pdR)
                      if okR == 0 { if s0 >= 0 { db_extract(g, s0, pdR) } else { db_setpred(pdR, "none" as *u8, 4) } }
                  }
              }
              if yn == 0 { sum_orc = sum_orc + rd_oracle(g, top1, top2, s0, g[5] as *u8) }
          }
1707
          // gold copy (for neg-control)
          let gd: *u8 = (gaa as i64 + rows*256) as *u8
          db_setpred(gd, g[5] as *u8, 250)
          sum_f0 = sum_f0 + qab_f1(g, fmt, pd0)
          sum_f1 = sum_f1 + qab_f1(g, fmt, pd1)
          sum_f2 = sum_f2 + qab_f1(g, fmt, pd2)
          if g[79] == 1 { sum_fR = sum_fR + qab_f1(g, fmt, pdR) }
          em0 = em0 + qab_em(g, fmt, pd0)
          let DBG: i64 = 0
          if DBG == 1 { if fmt == 1 { if rows < 14 {
              db_w("    [SQ] f0="); db_n(qab_f1(g,fmt,pd0)); db_w(" | Q="); db_w(g[4] as *u8); db_w(" | P="); db_w(pd0); db_w(" | G="); db_w(g[5] as *u8); db_w("\n" as *u8)
          } } }
          rows = rows + 1
          if QLIMIT > 0 { if rows >= QLIMIT { live = 0 } }
      }
      }
  }
1725
  // neg-control: mode0 preds vs ROTATED golds -> must be ~0
  var neg: i64 = 0
  var r: i64 = 0
  while r < rows {
      let pd: *u8 = (p0a as i64 + r*256) as *u8
      var r2: i64 = r + 1
      if r2 >= rows { r2 = 0 }
      neg = neg + qs_f1(pd, (gaa as i64 + r2*256) as *u8)
      r = r + 1
  }
  var af0: i64 = 0; var af1: i64 = 0; var af2: i64 = 0; var afR: i64 = 0; var afO: i64 = 0; var naneg: i64 = 0
  if rows > 0 { af0 = sum_f0/rows; af1 = sum_f1/rows; af2 = sum_f2/rows; afR = sum_fR/rows; afO = sum_orc/rows; naneg = neg/rows }
  db_w("  "); db_w(label)
  db_w("  rows="); db_n(rows)
  db_w("  F1_lex="); db_n(af0)
  db_w("  F1_hop="); db_n(af1)
  db_w("  F1_SEM="); db_n(af2)
  db_w("  F1_rdr="); db_n(afR)
  db_w("  ORACLE="); db_n(afO)
  db_w("  EM="); db_n(em0); db_w("/"); db_n(rows)
  db_w("  negctl="); db_n(naneg); db_w("\n" as *u8)
  res[base]=rows; res[base+1]=af0; res[base+2]=af1; res[base+3]=af2; res[base+4]=em0; res[base+5]=afR; res[base+6]=afO
  return rows
1749}
1750
1751func main(argc: i64, argv: *i64) -> i64 {
  if argc < 4 { db_w("=== nx_qabench -- QA benchmark LADDER: SQuAD 1-hop (extraction floor) vs HotpotQA 2-hop (reasoning wall) ===\n" as *u8) }
  if argc >= 4 { db_w("=== nx_qabench ANSWER (retrieve-then-read; trained span reader) ===\n" as *u8) }
  let g: *i64 = sys_mmap(SEM_MAGIC_2048) as *i64                        // 256 slots (was 128; multi-source research uses 125-127)
  let m0: *u8 = sys_mmap(SEM_MAGIC_16777216); g[0] = m0 as i64      // 16MB raw-file buffer: paginated sets grow past the old 1.5MB (HotpotQA-300 = 1.9MB)
  let m4: *u8 = sys_mmap(SEM_MAGIC_4096); g[4] = m4 as i64
  let m5: *u8 = sys_mmap(SEM_MAGIC_2048); g[5] = m5 as i64
  let m6: *u8 = sys_mmap(64); g[6] = m6 as i64
  let m7: *u8 = sys_mmap(SEM_MAGIC_4096); g[7] = m7 as i64
  let m8: *u8 = sys_mmap(256); g[8] = m8 as i64
  let m9: *u8 = sys_mmap(256); g[9] = m9 as i64
  let m11: *u8 = sys_mmap(SEM_MAGIC_32768); g[11] = m11 as i64          // title arena (MuSiQue: 20 paras vs HotpotQA 10)
  let m12: *u8 = sys_mmap(SEM_MAGIC_4096); g[12] = m12 as i64           // pto: 512 paragraph slots
  let m13: *u8 = sys_mmap(SEM_MAGIC_4096); g[13] = m13 as i64           // ptl
  let m15: *u8 = sys_mmap(SEM_MAGIC_1048576); g[15] = m15 as i64        // sentence arena (20 long paras)
  let m16: *u8 = sys_mmap(SEM_MAGIC_32768); g[16] = m16 as i64          // so: SEM_MAGIC_4096 sentence slots
  let m17: *u8 = sys_mmap(SEM_MAGIC_32768); g[17] = m17 as i64          // sl
  let m18: *u8 = sys_mmap(SEM_MAGIC_32768); g[18] = m18 as i64          // sp
  let m20: *u8 = sys_mmap(SEM_MAGIC_1048576); g[20] = m20 as i64        // normalized sentence arena
  let m21: *u8 = sys_mmap(SEM_MAGIC_32768); g[21] = m21 as i64          // no
  let m22: *u8 = sys_mmap(SEM_MAGIC_32768); g[22] = m22 as i64          // nl
  let m23: *u8 = sys_mmap(SEM_MAGIC_524288); g[23] = m23 as i64         // tko: 64k token slots
  let m24: *u8 = sys_mmap(SEM_MAGIC_524288); g[24] = m24 as i64         // tkl
  let m25: *u8 = sys_mmap(SEM_MAGIC_32768); g[25] = m25 as i64          // tks: per-sentence token start
  let m26: *u8 = sys_mmap(SEM_MAGIC_32768); g[26] = m26 as i64          // tkc
  let m28: *u8 = sys_mmap(SEM_MAGIC_4096); g[28] = m28 as i64
  let m29: *u8 = sys_mmap(SEM_MAGIC_2048); g[29] = m29 as i64
  let m30: *u8 = sys_mmap(SEM_MAGIC_2048); g[30] = m30 as i64
  let m32: *u8 = sys_mmap(SEM_MAGIC_2048); g[32] = m32 as i64
  let m33: *u8 = sys_mmap(SEM_MAGIC_32768); g[33] = m33 as i64          // title normalized arena
  let m34: *u8 = sys_mmap(SEM_MAGIC_32768); g[34] = m34 as i64          // tto
  let m35: *u8 = sys_mmap(SEM_MAGIC_32768); g[35] = m35 as i64          // ttl
  let m36: *u8 = sys_mmap(SEM_MAGIC_4096); g[36] = m36 as i64           // tts: 512 paragraph slots
  let m37: *u8 = sys_mmap(SEM_MAGIC_4096); g[37] = m37 as i64           // ttc
  let m39: *u8 = sys_mmap(SEM_MAGIC_32768); g[39] = m39 as i64          // shit: per-sentence
  let m40: *u8 = sys_mmap(SEM_MAGIC_4096); g[40] = m40 as i64           // thit: per-paragraph
  let m41: *u8 = sys_mmap(SEM_MAGIC_4096); g[41] = m41 as i64           // bhit
  let m42: *u8 = sys_mmap(SEM_MAGIC_4096); g[42] = m42 as i64           // psc
  let m43: *u8 = sys_mmap(SEM_MAGIC_4096); g[43] = m43 as i64           // xsc
  let m44: *u8 = sys_mmap(SEM_MAGIC_1024); g[44] = m44 as i64
  let m45: *u8 = sys_mmap(128); g[45] = m45 as i64
  let m46: *u8 = sys_mmap(128); g[46] = m46 as i64
  let m48: *u8 = sys_mmap(SEM_MAGIC_2048); g[48] = m48 as i64
  let m49: *u8 = sys_mmap(256); g[49] = m49 as i64
  let m50: *u8 = sys_mmap(256); g[50] = m50 as i64
  let m51: *u8 = sys_mmap(256); g[51] = m51 as i64
  let m53: *u8 = sys_mmap(SEM_MAGIC_8192); g[53] = m53 as i64
  let m54: *u8 = sys_mmap(SEM_MAGIC_262144); g[54] = m54 as i64      // pred/gold arenas: 256B/row -> SEM_MAGIC_1024-row capacity (grown for paginated eval sets)
  let m55: *u8 = sys_mmap(SEM_MAGIC_262144); g[55] = m55 as i64
  let m56: *u8 = sys_mmap(SEM_MAGIC_262144); g[56] = m56 as i64
  let m57: *u8 = sys_mmap(SEM_MAGIC_65536); g[57] = m57 as i64          // H presence matrix p*128+k (20 paras)
  let m58: *u8 = sys_mmap(SEM_MAGIC_8192); g[58] = m58 as i64           // df per content qterm
  let m59: *u8 = sys_mmap(SEM_MAGIC_262144); g[59] = m59 as i64      // SQuAD single-context temp
  let m60: *u8 = sys_mmap(SEM_MAGIC_8388608); g[60] = m60 as i64        // sem C matrix vc*vc i64 (vc<=SEM_MAGIC_1024)
  let m61: *u8 = sys_mmap(SEM_MAGIC_8192); g[61] = m61 as i64           // sem qvec
  let m62: *u8 = sys_mmap(SEM_MAGIC_8192); g[62] = m62 as i64           // sem svec
  let m63: *u8 = sys_mmap(SEM_MAGIC_8192); g[63] = m63 as i64           // sem voff
  let m64: *u8 = sys_mmap(SEM_MAGIC_8192); g[64] = m64 as i64           // sem vlen
  g[65] = 0                                                    // sem vcount
  let m66: *u8 = sys_mmap(SEM_MAGIC_32768); g[66] = m66 as i64          // sem vbuf
  g[67] = 0                                                    // sem vbump
  let m68: *u8 = sys_mmap(256); g[68] = m68 as i64            // pred2 (MODE2, reused per row)
  let m69: *u8 = sys_mmap(SEM_MAGIC_2048); g[69] = m69 as i64           // sem per-sentence vidx
  // reader (MODE-R) slots
  let m80: *u8 = sys_mmap(256); g[80] = m80 as i64            // reader weights w[14]
  let m81: *u8 = sys_mmap(SEM_MAGIC_65536); g[81] = m81 as i64          // cand feature matrix 300*14 i64
  let m82: *u8 = sys_mmap(SEM_MAGIC_4096); g[82] = m82 as i64           // cand meta 300
  let m84: *u8 = sys_mmap(SEM_MAGIC_2048); g[84] = m84 as i64           // reader prediction (candidates can be long)
  let m85: *u8 = sys_mmap(256); g[85] = m85 as i64            // averaged-perceptron weight sum
  let m86: *u8 = sys_mmap(SEM_MAGIC_2048); g[86] = m86 as i64           // candidate-string tmp (bounded copy 250)
  // reader training-F1 scratch (avoids qs_f1's 7-mmap-per-call OOM over millions of training scores)
  let m92: *u8 = sys_mmap(SEM_MAGIC_2048); g[92] = m92 as i64           // gold normalized
  let m93: *u8 = sys_mmap(SEM_MAGIC_2048); g[93] = m93 as i64           // gold token offsets
  let m94: *u8 = sys_mmap(SEM_MAGIC_2048); g[94] = m94 as i64           // gold token lengths
  let m95: *u8 = sys_mmap(SEM_MAGIC_2048); g[95] = m95 as i64           // cand normalized
  let m96: *u8 = sys_mmap(SEM_MAGIC_2048); g[96] = m96 as i64           // cand token offsets
  let m97: *u8 = sys_mmap(SEM_MAGIC_2048); g[97] = m97 as i64           // cand token lengths
  let m98: *u8 = sys_mmap(SEM_MAGIC_2048); g[98] = m98 as i64           // used array (qs_common)
  let m100: *u8 = sys_mmap(64); g[100] = m100 as i64          // MuSiQue loader stbox (hoisted from per-row mmap)
  let m108: *u8 = sys_mmap(SEM_MAGIC_2048); g[108] = m108 as i64        // dump: question PPMI ids (embed feats)
  let m110: *u8 = sys_mmap(SEM_MAGIC_2048); g[110] = m110 as i64        // dump: candidate string
  let m111: *u8 = sys_mmap(SEM_MAGIC_2048); g[111] = m111 as i64        // dump: candidate normalized
  let m112: *u8 = sys_mmap(SEM_MAGIC_2048); g[112] = m112 as i64        // dump: candidate token offsets
  let m113: *u8 = sys_mmap(SEM_MAGIC_2048); g[113] = m113 as i64        // dump: candidate token lengths
  let m114: *u8 = sys_mmap(64); g[114] = m114 as i64          // dump: 3-i64 embedding feature buffer
  let m118: *u8 = sys_mmap(SEM_MAGIC_1024); g[118] = m118 as i64        // full-vec: q_emb_unit (DIM i64)
  let m119: *u8 = sys_mmap(SEM_MAGIC_1024); g[119] = m119 as i64        // full-vec: cand_unit scratch
  let m121: *u8 = sys_mmap(SEM_MAGIC_1024); g[121] = m121 as i64        // full-vec: DIM-vector feature buffer
  let m126: *u8 = sys_mmap(256); g[126] = m126 as i64         // research: best answer buffer
  let m127: *u8 = sys_mmap(SEM_MAGIC_16384); g[127] = m127 as i64       // research: per-source answers (64 x 256 fixed-width)
  let m128: *u8 = sys_mmap(512); g[128] = m128 as i64         // research: per-source confidence (i64)
  let m129: *u8 = sys_mmap(512); g[129] = m129 as i64         // research: per-source argv index (i64)
  let m130: *u8 = sys_mmap(SEM_MAGIC_4096); g[130] = m130 as i64        // live prose: sentence-clean scratch
  g[79] = 0
1845
  // ---- DUMPFEATS MODE: nx_qabench dumpfeats -> export candidate features for offline neural-reader training ----
  // dumpfeats: sparse-PPMI embed features are the current BEST (measured: sparse 279 > dense@24 262 -- the
  //   24-dim factorization is a lossy compression, weaker than the full sparse signal AS A SCALAR feature).
  //   embed_load()/dense_cos_int() are committed infra for the next rung (bigger embeddings / full-vector
  //   features); 'de' selects dense to re-measure. Default 'd' = sparse.
  if argc >= 2 { let a1: *u8 = argv[1] as *u8; if a1[0] == (100 as u8) {
      if a1[1] == (116 as u8) { return rd_dump_texts(g) }   // 'dt' = raw text rows for the neural reader (no ppmi needed)
      if a1[1] == (120 as u8) { return rd_dump_texts_xl(g) }   // 'dx' = XL rows (all chunks) -> reader_rows_xl.bin
      ppmi_load(g)
      if a1[1] == (101 as u8) { let el: i64 = embed_load(g); if el == 1 { db_w("dumpfeats: TRAINED dense embeddings (embed_v1.bin)\n" as *u8) } }   // 'de' = dense
      return rd_dump_feats(g)
  } }
1858
  // R5 opt-in: if the marker file exists, route the researcher's EXTRACTION through the pretrained sovereign
  // Qwen (qwx_extract) instead of the mechanical db_extract. Boots the model once (fail-fast, before serving).
  g[132] = 0
  let qw_on: i64 = sys_openat_rd("knowledge/index/qwen_reader.on" as *u8)
  if qw_on >= 0 {
      sys_close(qw_on)
      db_w("QWEN-READER = marker present -> booting pretrained Qwen for extraction (nx_qwen_extract)...\n" as *u8)
      let qrc: i64 = qwx_load()
      if qrc == 0 { g[132] = 1; db_w("QWEN-READER = LIVE (qwx_extract routes qa_read_source extraction)\n" as *u8) }
      else { db_w("QWEN-READER = load FAILED -> mechanical db_extract fallback\n" as *u8) }
  }
1870
  // ---- LIVE ANSWER MODE: nx_qabench answer "<question>" <context-file> ----
  // Turns the benchmark pipeline into a callable reading-comprehension capability on ARBITRARY input
  // (retrieve-then-read with the trained span reader). Same organ, no benchmark files needed.
  if argc >= 4 {
      ppmi_load(g)                                            // silent (loads semppmi_v1.bin if present)
      rd_load(g)                                              // silent (loads reader_v1.bin if present; else db_extract fallback)
      let a1: *u8 = argv[1] as *u8
      if a1[0] == (114 as u8) { return qa_research(g, argv[2] as *u8, argv, 3, argc) }   // 'r' = research "<q>" ctx1 ctx2...
      return qa_answer(g, argv[2] as *u8, argv[3] as *u8)                                // else answer "<q>" ctx
  }
1881
  let res: *i64 = sys_mmap(256) as *i64
  let pl: i64 = ppmi_load(g)
  if pl == 1 { db_w("\nSEM path = CORPUS sparse-PPMI soft-alignment (semppmi_v1.bin loaded: vocab="); db_n(g[71]); db_w(" triples="); db_n(g[72]); db_w(")\n" as *u8) }
  else { db_w("\nSEM path = per-row co-occurrence FALLBACK (semppmi_v1.bin absent; run nx_semppmi_build)\n" as *u8) }
  // reader: load persisted weights, else train on disjoint corpus chunks
  let rl: i64 = rd_load(g)
  if rl == 0 { db_w("READER = no reader_v1.bin -> training span-scorer on DISJOINT corpus rows (SQuAD-baseline recipe)...\n" as *u8); rd_train(g) }
  if g[79] == 1 {
      let w: *i64 = g[80] as *i64
      db_w("READER = trained span-scorer LIVE. w=[")
      var wi: i64 = 0
      while wi < RD_NF { db_n(w[wi]); if wi < RD_NF-1 { db_w(" " as *u8) } wi = wi + 1 }
      db_w("]\n" as *u8)
  } else { db_w("READER = unavailable (corpus absent); MODE-R column will read 0\n" as *u8) }
  db_w("\n-- per-benchmark (same extractor+metric; F1 permille; lex=IDF-lexical hop=bridge SEM=distributional MODE2) --\n" as *u8)
  qab_run(g, "knowledge/fetched/qab_squad_big.raw" as *u8, 1, "SQuAD-v1.1  [1-hop, single-ctx]" as *u8, res, 0)
  qab_run(g, "knowledge/fetched/qab_hotpot_big.raw" as *u8, 0, "HotpotQA    [2-hop, distractor]" as *u8, res, 8)
  qab_run(g, "knowledge/fetched/qab_musique_big.raw" as *u8, 3, "MuSiQue     [2-4-hop, 20 paras]" as *u8, res, 16)
1900
  db_w("\n=== LADDER (mechanical researcher vs published SOTA; harder = more hops/distractors) ===\n" as *u8)
  db_w("  RUNG           F1_lex  F1_hop  F1_SEM  F1_rdr   published-SOTA F1   gap(best)\n" as *u8)
  db_w("  SQuAD 1-hop      "); db_n(res[1]); db_w("     "); db_n(res[2]); db_w("     "); db_n(res[3]); db_w("     "); db_n(res[5]); db_w("     ~900 (BiDAF+ 2016..)   "); db_n(900 - qab_max4(res[1],res[2],res[3],res[5])); db_w("\n" as *u8)
  db_w("  HotpotQA 2-hop   "); db_n(res[9]); db_w("     "); db_n(res[10]); db_w("     "); db_n(res[11]); db_w("     "); db_n(res[13]); db_w("     ~700-820 (strong)      "); db_n(760 - qab_max4(res[9],res[10],res[11],res[13])); db_w("\n" as *u8)
  db_w("  MuSiQue 2-4-hop  "); db_n(res[17]); db_w("     "); db_n(res[18]); db_w("     "); db_n(res[19]); db_w("     "); db_n(res[21]); db_w("     ~490-690 (SOTA/RoHT)   "); db_n(590 - qab_max4(res[17],res[18],res[19],res[21])); db_w("\n" as *u8)
1906
  // best-mode per rung + which mode won (honest: name the winner, never hide a losing column)
  let bS: i64 = qab_max4(res[1],res[2],res[3],res[5])
  let bH: i64 = qab_max4(res[9],res[10],res[11],res[13])
  let bM: i64 = qab_max4(res[17],res[18],res[19],res[21])
  db_w("  BEST-MODE ladder: SQuAD "); db_n(bS); db_w(" / HotpotQA "); db_n(bH); db_w(" / MuSiQue "); db_n(bM); db_w("  (rdr=trained span reader)\n" as *u8)
  // ORACLE ceiling (best candidate in the set) vs reader -> names the binding stage: gap(oracle-rdr) big = SCORING
  // headroom; oracle ~= reader = candidate GENERATION is the wall.
  db_w("  ORACLE ceiling:   SQuAD "); db_n(res[6]); db_w(" / HotpotQA "); db_n(res[14]); db_w(" / MuSiQue "); db_n(res[22]); db_w("  (best candidate available; rdr-to-oracle gap = scoring headroom)\n" as *u8)
  db_w("  -> binding stage: SQuAD "); qab_bind(res[5], res[6]); db_w(" / HotpotQA "); qab_bind(res[13], res[14]); db_w(" / MuSiQue "); qab_bind(res[21], res[22]); db_w("\n" as *u8)
1916
  var pass: i64 = 0
  if res[0] > 0 { pass = pass + 1 }
  if res[8] > 0 { pass = pass + 1 }
  if res[16] > 0 { pass = pass + 1 }
  // difficulty gradient must hold on the BEST mode: SQuAD >= HotpotQA >= MuSiQue -> the hops ARE the gap
  if bS >= bH { if bH >= bM { pass = pass + 1 } }
  db_w("\nTEETH (squad+hotpot+musique ran + best-mode difficulty-gradient holds) = "); db_n(pass); db_w("/4\n" as *u8)
  if pass == 4 {
      db_w("GREEN -- 4-mode ladder measured (lex/hop/SEM/trained-reader). Best-mode F1 falls monotonically as\n" as *u8)
      db_w("HOPS + DISTRACTORS rise. The trained span reader (SQuAD-baseline recipe) attacks the EXTRACTION stage\n" as *u8)
      db_w("the 6x-corpus datapoint proved is now binding. Honest: no rung at SOTA; next lever = R1 no-float reader.\n" as *u8)
      return 0
  }
  db_w("RED -- a rung did not run (fetch its subset) or the best-mode difficulty gradient did not hold\n" as *u8)
  return 1
1932}
1933
1934func qab_max4(a: i64, b: i64, c: i64, d: i64) -> i64 {
  var m: i64 = a
  if b > m { m = b }
  if c > m { m = c }
  if d > m { m = d }
  return m
1940}
1941
1942// name the binding stage from (reader F1, oracle F1): a wide gap = the right candidate exists but the reader
1943// mis-scores it (SCORING); a narrow gap = the reader is near-optimal and the answer isn't a candidate (GENERATION).
1944func qab_bind(rdr: i64, orc: i64) -> i64 {
  let gap: i64 = orc - rdr
  if gap > 150 { db_w("SCORING(gap "); db_n(gap); db_w(")" as *u8) } else { db_w("GENERATION(gap "); db_n(gap); db_w(")" as *u8) }
  return 0
1948}
1949
1950// read a whole file into buf (null-terminated); returns byte length, -1 on open failure.
1951func qa_read_file(path: *u8, buf: *u8, cap: i64) -> i64 {
  let fd: i64 = sys_openat_rd(path)
  if fd < 0 { return 0-1 }
  var total: i64 = 0
  var r: i64 = 1
  while r > 0 {
      let left: i64 = cap - 1 - total
      if left <= 0 { r = 0 } else {
          r = sys_read(fd, (buf as i64 + total) as *u8, left)
          if r > 0 { total = total + r }
      }
  }
  sys_close(fd)
  buf[total] = (0 as u8)
  return total
1966}
1967
1968// LIVE answer: run the retrieve-then-read pipeline on an arbitrary (question, context) and print the answer +
1969// its source sentence (citation). Reuses the exact benchmark machinery; reader if loaded, else db_extract.
1970// core: read ONE source vs the question already in g[4]; extract the answer into pred; stash the best-sentence
1971// index in g[125]; return CONFIDENCE = best-sentence question-hit score (2*sentence-hits + title-hits), which is
1972// comparable across sources. Returns -1 on read/parse failure. Shared by single-source answer + multi-source.
1973func qa_read_source(g: *i64, ctxpath: *u8, pred: *u8) -> i64 {
  let ctxb: *u8 = g[59] as *u8
  let cl: i64 = qa_read_file(ctxpath, ctxb, SEM_MAGIC_200000)
  if cl <= 0 { return 0-1 }
  g[14] = 1
  let pto: *i64 = g[12] as *i64
  let ptl: *i64 = g[13] as *i64
  let ptb: *u8 = g[11] as *u8
  pto[0]=0; ptl[0]=0; ptb[0]=(0 as u8)
  let ns: i64 = qa_split_prose(g, ctxb, cl)            // prose-filtered: drops web chrome/wikitext/JSON
  g[19] = ns
  if ns <= 0 { return 0-1 }
  db_norm_row(g)
  db_score_row(g)
  let top1: i64 = db_top_para(g, 0-1)
  let top2: i64 = db_top_para(g, top1)
  let s0: i64 = db_best_sentence(g)
  g[125] = s0
  var conf: i64 = 0
  if s0 >= 0 {
      let shit: *i64 = g[39] as *i64
      let thit: *i64 = g[40] as *i64
      let sp: *i64 = g[18] as *i64
      conf = 2*shit[s0] + thit[sp[s0]]
  }
  let yn: i64 = db_is_yesno(g)
  let wantnum: i64 = db_prefer_numeric(g)
  var qwdone: i64 = 0
  if g[132] == 1 { if qwx_ready() == 1 { if s0 >= 0 { qwdone = qwx_extract(g, s0, pred) } } }   // R5: Qwen reads context centered on best sentence s0
  if qwdone == 0 {
      if yn == 1 { db_setpred(pred, "yes" as *u8, 3) } else {
          var got: i64 = 0
          if wantnum == 1 { got = db_sweep_numeric(g, top1, top2, pred) }
          if got == 0 { if db_wants_location(g) == 1 { got = db_sweep_location(g, top1, top2, pred) } }   // where -> place answer
          if got == 0 { if s0 >= 0 { db_extract(g, s0, pred) } else { db_setpred(pred, "(no answer found)" as *u8, 17) } }
      }
  }
  // ACCURATE CITATION: a sweep may extract from a different sentence than the top-match s0, so cite the
  // sentence that actually CONTAINS the answer (fall back to s0 if not found).
  if yn == 0 {
      let so2: *i64 = g[16] as *i64
      let sb2: *u8 = g[15] as *u8
      var cs: i64 = 0
      var found: i64 = 0
      while cs < g[19] {
          if found == 0 {
              if qa_contains_ci((sb2 as i64 + so2[cs]) as *u8, pred) == 1 { g[125] = cs; found = 1 }
          }
          cs = cs + 1
      }
  }
  return conf
2025}
2026
2027func qa_streq(a: *u8, b: *u8) -> i64 {
  var i: i64 = 0
  while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 }
  if b[i] != (0 as u8) { return 0 }
  return 1
2032}
2033
2034func qa_lc(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c }
2035func qa_slen(s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { i = i + 1 } return i }
2036
2037// case-insensitive: is `needle` a substring of `hay`?
2038func qa_contains_ci(hay: *u8, needle: *u8) -> i64 {
  let hl: i64 = qa_slen(hay)
  let nl: i64 = qa_slen(needle)
  if nl == 0 { return 0 }
  if nl > hl { return 0 }
  var i: i64 = 0
  while i + nl <= hl {
      var j: i64 = 0
      var m: i64 = 1
      while j < nl { if m == 1 { if qa_lc(hay[i+j] as i64) != qa_lc(needle[j] as i64) { m = 0 } } j = j + 1 }
      if m == 1 { return 1 }
      i = i + 1
  }
  return 0
2052}
2053
2054// two answers CORROBORATE if one is contained in the other (case-insensitive): "Einstein" ~ "Albert Einstein".
2055func qa_answer_agree(a: *u8, b: *u8) -> i64 {
  if qa_contains_ci(a, b) == 1 { return 1 }
  if qa_contains_ci(b, a) == 1 { return 1 }
  return 0
2059}
2060// (a definitional hit-density sentence selector was tried to fix the Eiffel "New York Times" miss -- it was
2061//  INERT (the concise lead still did not win; deeper debug = single-example whack-a-mole) so it was removed.
2062//  The Eiffel-class selection miss needs a trained neural reader, not a mechanical density heuristic.)
2063
2064func qa_answer(g: *i64, question: *u8, ctxpath: *u8) -> i64 {
  let q: *u8 = g[4] as *u8
  var i: i64 = 0
  while question[i] != (0 as u8) { if i < SEM_MAGIC_3999 { q[i] = question[i] } i = i + 1 }
  var qn: i64 = i
  if qn > SEM_MAGIC_3999 { qn = SEM_MAGIC_3999 }
  q[qn] = (0 as u8)
  let pred: *u8 = g[84] as *u8
  let conf: i64 = qa_read_source(g, ctxpath, pred)
  if conf < 0 { db_w("ERROR: cannot read/parse context file: "); db_w(ctxpath); db_w("\n" as *u8); return 1 }
  db_w("Q: "); db_w(q); db_w("\n" as *u8)
  db_w("A: "); db_w(pred); db_w("\n" as *u8)
  let s0: i64 = g[125]
  if s0 >= 0 {
      let sb: *u8 = g[15] as *u8
      let so: *i64 = g[16] as *i64
      db_w("   source: "); db_w((sb as i64 + so[s0]) as *u8); db_w("\n" as *u8)
  }
  return 0
2083}
2084
2085// DEEP RESEARCH: read the question across MULTIPLE fetched sources, pick the highest-confidence answer, and
2086// report the cross-source picture + AGREEMENT (how many sources produced the same answer = corroboration). This
2087// is the researcher's synthesis stage -- multi-source, the actual "deep research" loop.
2088func qa_research(g: *i64, question: *u8, argv: *i64, cstart: i64, argc: i64) -> i64 {
  let q: *u8 = g[4] as *u8
  var i: i64 = 0
  while question[i] != (0 as u8) { if i < SEM_MAGIC_3999 { q[i] = question[i] } i = i + 1 }
  var qn: i64 = i
  if qn > SEM_MAGIC_3999 { qn = SEM_MAGIC_3999 }
  q[qn] = (0 as u8)
  db_w("=== DEEP RESEARCH (multi-source) ===\nQ: "); db_w(q); db_w("\n" as *u8)
  let pred: *u8 = g[84] as *u8
  let bestpred: *u8 = g[126] as *u8
  let ansb: *u8 = g[127] as *u8                        // per-source answers, 256-byte fixed-width records
  let confs: *i64 = g[128] as *i64                     // per-source confidence
  let sidx: *i64 = g[129] as *i64                      // per-source argv index (for citation)
  var nsrc: i64 = 0
  var si: i64 = cstart
  while si < argc {
      if nsrc < 64 {
          let ctxpath: *u8 = argv[si] as *u8
          let conf: i64 = qa_read_source(g, ctxpath, pred)
          if conf >= 0 {
              db_w("  [src "); db_n(si-cstart); db_w("] conf="); db_n(conf); db_w("  A: "); db_w(pred); db_w("  <- "); db_w(ctxpath); db_w("\n" as *u8)
              qa_strcpy((ansb as i64 + nsrc*256) as *u8, pred)
              confs[nsrc] = conf
              sidx[nsrc] = si
              nsrc = nsrc + 1
          } else {
              db_w("  [src "); db_n(si-cstart); db_w("] (unreadable / no prose) <- "); db_w(ctxpath); db_w("\n" as *u8)
          }
      }
      si = si + 1
  }
  if nsrc == 0 { db_w("NO ANSWER: no readable source\n" as *u8); return 1 }
  // CORROBORATION-FIRST aggregation: the answer the MOST sources support wins (tie-break by confidence).
  // This is the deep-research principle -- agreement across independent sources beats a lone high score.
  var best: i64 = 0-1
  var bestagree: i64 = 0-1
  var bestconf: i64 = 0-1
  var a: i64 = 0
  while a < nsrc {
      let ca: *u8 = (ansb as i64 + a*256) as *u8
      var agree: i64 = 0
      var b: i64 = 0
      while b < nsrc { if qa_answer_agree(ca, (ansb as i64 + b*256) as *u8) == 1 { agree = agree + 1 } b = b + 1 }
      var take: i64 = 0
      if agree > bestagree { take = 1 } else { if agree == bestagree { if confs[a] > bestconf { take = 1 } } }
      if take == 1 { bestagree = agree; bestconf = confs[a]; best = a }
      a = a + 1
  }
  qa_strcpy(bestpred, (ansb as i64 + best*256) as *u8)
  db_w("\n=== ANSWER: "); db_w(bestpred); db_w("\n    "); db_n(bestagree); db_w("/"); db_n(nsrc); db_w(" sources corroborate (confidence "); db_n(bestconf); db_w(")\n    source: "); db_w(argv[sidx[best]] as *u8); db_w("\n" as *u8)
  return 0
2139}
2140
2141func qa_strcpy(dst: *u8, src: *u8) -> i64 {
  var i: i64 = 0
  while src[i] != (0 as u8) { if i < 250 { dst[i] = src[i] } i = i + 1 }
  var n: i64 = i; if n > 250 { n = 250 } dst[n] = (0 as u8)
  return 0
2146}
nx_qabench.nx source

`nx_qabench.nx` source
