code wiki / (root) / nx_qwen_reader_bench_gate.nx
`nx_qwen_reader_bench_gate.nx` source

↩ module page · 419 lines · 18508 B
1// nx_qwen_reader_bench_gate.nx -- IN-PROCESS sovereign benchmark of the pretrained no-float Qwen2.5-0.5B as an
2// extractive SQuAD reader. Loads the model ONCE (nsv_init -> dequant-once caches), loops the real
3// (question, context, gold) rows from knowledge/index/reader_rows.bin, greedy-generates the answer IN-PROCESS
4// (NO HTTP, NO daemon, NO socket), scores integer token-F1 vs gold, prints per-row detail + the mean permille.
5//
6// WHY THIS EXISTS: the prior PowerShell->HTTP->:8032 harness was fragile (the daemon's request path could not
7// take the long SQuAD contexts -> connection dropped mid-batch) AND it violated doctrine ("sovereign gates,
8// not curl harnesses"). Rule#3: stop patching the HTTP path -- rewrite the measurement as an in-process gate.
9//
10// Baselines for context (printed with the result): from-scratch neural reader F1~81 ; mechanical reader F1=251.
11// usage: nx_qwen_reader_bench_gate [N]   (default N=30 ; i32 lossless ; greedy)   Sovereign: nx_nofloat_serve_core.
12import "nx_syscalls.nx"
13import "nx_nofloat_serve_core.nx"
14import "nx_qa_score_lib.nx"     // qs_f1 = the CANONICAL SQuAD scorer behind the 251/148/101 baseline (strips articles)
15
16const QRB_MODEL: *u8 = "/home/elderwesto/nx_stage/nx_real_model.gguf"
17const QRB_CTXCAP: i64 = 440     // context truncation (chars); 1-shot @ 440 (440 ctx + ~370 prefix/q <= 900 guard). ⚠2-shot@380 REVERTED — the standalone +104 (722 vs 618) was a SINGLE-GOLD artifact; on-protocol multi-alias it did NOT hold (SQuAD 427 < 510). See arc memory.
18const QRB_MAXPROMPT: i64 = 900  // hard char guard: sized for the FIXED tokenizer + NSV_MAXT=384 (900/2.5-worst-bpe = 360 tok + MAXNEW 16 = 376 < 384). Was 600 = a stale relic of the old 192-tok window that spuriously SKIPPED rows 8/9 (po=607) to 0.
19const QRB_MAXNEW: i64 = 16
20const QRB_MODE: i64 = 0         // 0 = i32 lossless (faithful), not i8 fast -- measure THE model, no approximation
21const QRB_RESULTS: *u8 = "knowledge/index/qwen_reader_results.tsv"   // resume-accumulate ledger: "idx<TAB>f1\n"
22
23func qrb_pn(v: i64) -> i64 { let b: *u8=sys_mmap(32); let e: i64=nsv_catn(b,0,v); sys_write(1,b,e); return 0 }
24func qrb_ps(s: *u8) -> i64 { nsv_log(s); return 0 }
25
26// SQuAD token normalization: lowercase A-Z, keep a-z/0-9, everything else -> space(32).
27func qrb_lc(c: i64) -> i64 {
  if c >= 65 { if c <= 90 { return c + 32 } }
  if c >= 97 { if c <= 122 { return c } }
  if c >= 48 { if c <= 57 { return c } }
  return 32
32}
33func qrb_norm(src: *u8, n: i64, dst: *u8) -> i64 { var i: i64=0; while i<n { dst[i]=qrb_lc(src[i] as i64) as u8; i=i+1 } return n }
34
35// extract word boundaries (maximal non-space runs) from a normalized buffer into starts[]/lens[]; cap 256.
36func qrb_words(buf: *u8, n: i64, starts: *i64, lens: *i64) -> i64 {
  var w: i64=0; var i: i64=0; var inw: i64=0; var st: i64=0
  while i < n {
      if (buf[i] as i64) == 32 {
          if inw == 1 { if w < 256 { starts[w]=st; lens[w]=i-st; w=w+1 } inw=0 }
      } else {
          if inw == 0 { st=i; inw=1 }
      }
      i = i + 1
  }
  if inw == 1 { if w < 256 { starts[w]=st; lens[w]=i-st; w=w+1 } }
  return w
48}
49func qrb_weq(a: *u8, ao: i64, al: i64, b: *u8, bo: i64, bl: i64) -> i64 {
  if al != bl { return 0 }
  var i: i64=0
  while i < al { if a[ao+i] != b[bo+i] { return 0 } i=i+1 }
  return 1
54}
55
56// question-aware context selection (retrieve-then-read): split ctx into sentences, score each by question-term
57// hits (normalized exact word match, len>=3 -- keeps AFC/NFC/50-class discriminators), emit best sentence +/- 1
58// neighbor shrunk-to-cap around the best. WHY: blind truncation is answer-POSITION-dependent -- MEASURED
59// ctx=260 hid deep answers (rows 4/5/10 = 0) while ctx=440 pulled in parallel-fact distractor sentences
60// (rows 7/8/9 answered the WRONG GAME). Selection wins both sides. No term hits -> prefix-truncate fallback.
61// sel packs preallocated scratch (>6-arg clobber gotcha): sel[0]=ctx-norm(4096) sel[1]=q-norm(1024)
62// sel[2..5]=ctx/q word starts+lens (256*8 each) sel[6..8]=sent off/len/score (64*8) sel[9]=dst(512)
63// sel[10]=word->sentence map (256*8).
64func qrb_sel_ctx(cp: *u8, cn: i64, qp: *u8, qn: i64, sel: *i64) -> i64 {
  let nb: *u8 = sel[0] as *u8
  let qb: *u8 = sel[1] as *u8
  let cs: *i64 = sel[2] as *i64
  let cl: *i64 = sel[3] as *i64
  let qs: *i64 = sel[4] as *i64
  let ql: *i64 = sel[5] as *i64
  let soff: *i64 = sel[6] as *i64
  let slen: *i64 = sel[7] as *i64
  let ssc: *i64 = sel[8] as *i64
  let dst: *u8 = sel[9] as *u8
  let wsent: *i64 = sel[10] as *i64
  var cn2: i64 = cn
  if cn2 > 4000 { cn2 = 4000 }              // norm-buffer bound (sel[0]=4096)
  var qn2: i64 = qn
  if qn2 > 1000 { qn2 = 1000 }
  // 1) sentence split on ./!/? followed by space (raw bytes; abbrev over-splits are noise the +/-1 window absorbs)
  var ns: i64 = 0
  var st: i64 = 0
  var i: i64 = 0
  while i < cn2 {
      let c: i64 = cp[i] as i64
      var isend: i64 = 0
      if c == 46 { isend = 1 }
      if c == 33 { isend = 1 }
      if c == 63 { isend = 1 }
      if isend == 1 {
          var nxt: i64 = 32
          if i + 1 < cn2 { nxt = cp[i+1] as i64 }
          if nxt == 32 {
              if ns < 64 { soff[ns]=st; slen[ns]=i+1-st; ssc[ns]=0; ns=ns+1 }
              st = i + 2
          }
      }
      i = i + 1
  }
  if st < cn2 { if ns < 64 { soff[ns]=st; slen[ns]=cn2-st; ssc[ns]=0; ns=ns+1 } }
  if ns == 0 {
      var w0: i64 = cn2
      if w0 > QRB_CTXCAP { w0 = QRB_CTXCAP }
      i=0
      while i < w0 { dst[i]=cp[i]; i=i+1 }
      return w0
  }
  // 2) normalized word tables (qrb_norm is position-aligned 1:1 with the raw bytes)
  qrb_norm(cp, cn2, nb)
  qrb_norm(qp, qn2, qb)
  let ncw: i64 = qrb_words(nb, cn2, cs, cl)
  let nqw: i64 = qrb_words(qb, qn2, qs, ql)
  // 3a) map each ctx word to its sentence
  var scur: i64 = 0
  var w: i64 = 0
  while w < ncw {
      var adv: i64 = 1
      while adv == 1 {
          adv = 0
          if scur + 1 < ns { if cs[w] >= soff[scur] + slen[scur] { scur = scur + 1; adv = 1 } }
      }
      wsent[w] = scur
      w = w + 1
  }
  // 3b) per-sentence score = number of DISTINCT question terms (len>=3) PRESENT in the sentence.
  // Term-presence (not word-frequency) scoring: MEASURED failure row 0 -- a long sentence full of
  // the/was outscored the true MVP sentence under per-word crediting; dedup + presence kills the
  // stopword-frequency inflation while rare terms (named/valuable/player, AFC/NFC/50) decide.
  var t: i64 = 0
  while t < nqw {
      if ql[t] >= 3 {
          var dup: i64 = 0
          var j2: i64 = 0
          while j2 < t {
              if dup == 0 { if qrb_weq(qb, qs[j2], ql[j2], qb, qs[t], ql[t]) == 1 { dup = 1 } }
              j2 = j2 + 1
          }
          if dup == 0 {
              var s2: i64 = 0
              while s2 < ns {
                  var hit: i64 = 0
                  var w2: i64 = 0
                  while w2 < ncw {
                      if hit == 0 { if wsent[w2] == s2 { if qrb_weq(nb, cs[w2], cl[w2], qb, qs[t], ql[t]) == 1 { hit = 1 } } }
                      w2 = w2 + 1
                  }
                  ssc[s2] = ssc[s2] + hit
                  s2 = s2 + 1
              }
          }
      }
      t = t + 1
  }
  // 4) best sentence = first max; zero hits anywhere -> prefix-truncate fallback (old behavior)
  var best: i64 = 0
  var bsc: i64 = 0-1
  i = 0
  while i < ns { if ssc[i] > bsc { bsc = ssc[i]; best = i } i = i + 1 }
  if bsc <= 0 {
      var w1: i64 = cn2
      if w1 > QRB_CTXCAP { w1 = QRB_CTXCAP }
      i=0
      while i < w1 { dst[i]=cp[i]; i=i+1 }
      return w1
  }
  // 5) window best +/- 1 neighbor, shrunk to the cap dropping neighbors before the best sentence itself
  var lo: i64 = best - 1
  if lo < 0 { lo = 0 }
  var hi: i64 = best + 1
  if hi >= ns { hi = ns - 1 }
  var tot: i64 = 0
  i = lo
  while i <= hi { tot = tot + slen[i] + 1; i = i + 1 }
  var shrink: i64 = 1
  while shrink == 1 {
      shrink = 0
      if tot > QRB_CTXCAP {
          if lo < best { tot = tot - slen[lo] - 1; lo = lo + 1; shrink = 1 } else {
              if hi > best { tot = tot - slen[hi] - 1; hi = hi - 1; shrink = 1 }
          }
      }
  }
  var o: i64 = 0
  i = lo
  while i <= hi {
      var k: i64 = 0
      while k < slen[i] { if o < QRB_CTXCAP { dst[o] = cp[soff[i]+k]; o = o + 1 } k = k + 1 }
      if o < QRB_CTXCAP { dst[o] = 32 as u8; o = o + 1 }
      i = i + 1
  }
  return o
192}
193
194// token-F1 permille between pred[0,pn) and gold[0,gn). sc packs 7 preallocated scratch ptrs (bundled to stay
195// under the >6-arg clobber gotcha): sc[0]=pb sc[1]=gb (norm bufs) sc[2..5]=ps/pl/gs/gl (word tables) sc[6]=gu.
196func qrb_f1(pred: *u8, pn: i64, gold: *u8, gn: i64, sc: *i64) -> i64 {
  let pb: *u8 = sc[0] as *u8
  let gb: *u8 = sc[1] as *u8
  let ps: *i64 = sc[2] as *i64
  let pl: *i64 = sc[3] as *i64
  let gs: *i64 = sc[4] as *i64
  let gl: *i64 = sc[5] as *i64
  let gu: *i64 = sc[6] as *i64
  qrb_norm(pred, pn, pb)
  qrb_norm(gold, gn, gb)
  let np: i64 = qrb_words(pb, pn, ps, pl)
  let ng: i64 = qrb_words(gb, gn, gs, gl)
  if np == 0 { return 0 }
  if ng == 0 { return 0 }
  var k: i64=0
  while k < ng { gu[k]=0; k=k+1 }
  var common: i64=0
  var i: i64=0
  while i < np {
      var j: i64=0
      var hit: i64=0
      while j < ng {
          if hit == 0 { if gu[j]==0 { if qrb_weq(pb,ps[i],pl[i],gb,gs[j],gl[j])==1 { gu[j]=1; common=common+1; hit=1 } } }
          j = j + 1
      }
      i = i + 1
  }
  if common == 0 { return 0 }
  return (2000*common) / (np + ng)
225}
226
227// resume ledger: read QRB_RESULTS, count completed rows (one per '\n') and sum their f1 (digits after the tab).
228// returns done-count; writes the running sum to sumptr[0]. Absent file -> done=0, sum=0.
229func qrb_read_done(sumptr: *i64) -> i64 {
  let fd: i64 = sys_openat_rd(QRB_RESULTS)
  if fd < 0 { sumptr[0]=0; return 0 }
  let cap: i64 = 1048576
  let buf: *u8 = sys_mmap(cap)
  var got: i64=0
  var r: i64=1
  while r > 0 { r=sys_read(fd,(buf as i64+got) as *u8, cap-got); if r>0 { got=got+r } }
  sys_close(fd)
  var done: i64=0
  var sum: i64=0
  var f1: i64=0
  var seentab: i64=0
  var i: i64=0
  while i < got {
      let c: i64 = buf[i] as i64
      if c == 9 { seentab=1 } else {
          if c == 10 { done=done+1; sum=sum+f1; f1=0; seentab=0 } else {
              if seentab == 1 { if c>=48 { if c<=57 { f1=f1*10+(c-48) } } }
          }
      }
      i = i + 1
  }
  sumptr[0]=sum
  return done
254}
255// append one completed row to the ledger (open-append + write + close = durable before any later crash).
256func qrb_append_result(idx: i64, f1: i64) -> i64 {
  let fd: i64 = sys_openat_append(QRB_RESULTS, 420)   // 0644
  if fd < 0 { return 0-1 }
  let b: *u8 = sys_mmap(64)
  var o: i64 = nsv_catn(b, 0, idx)
  b[o]=9 as u8
  o=o+1
  o = nsv_catn(b, o, f1)
  b[o]=10 as u8
  o=o+1
  sys_write(fd, b, o)
  sys_close(fd)
  return 0
269}
270
271func main(argc: i64, argv: *i64) -> i64 {
  var N: i64 = 30
  var MODE: i64 = QRB_MODE                 // arg2: "1" -> i8 fast, else i32 lossless (default)
  if argc >= 2 {
      let a: *u8 = argv[1] as *u8
      var v: i64=0
      var j: i64=0
      while a[j] != (0 as u8) { if a[j]>=(48 as u8){ if a[j]<=(57 as u8){ v=v*10+(a[j] as i64 - 48) } } j=j+1 }
      if v > 0 { N = v }
  }
  if argc >= 3 { let b: *u8 = argv[2] as *u8; if b[0]==(49 as u8) { MODE = 1 } }
  var mpath: *u8 = QRB_MODEL                 // arg3: model path override (NAS-exec: Synology home != /home/elderwesto)
  if argc >= 4 { mpath = argv[3] as *u8 }
  qrb_ps("=== QWEN-AS-READER in-process benchmark (greedy, ctxcap=400) mode=" as *u8); qrb_pn(MODE); qrb_ps(" (0=i32-lossless 1=i8-fast) ===\n" as *u8)
  qrb_ps("loading model + dequant-once caches...\n" as *u8)
  let rc: i64 = nsv_init(mpath)
  if rc != 0 { qrb_ps("MODEL INIT FAILED rc=" as *u8); qrb_pn(rc); qrb_ps("\n" as *u8); return 1 }
  qrb_ps("model ready; reading rows...\n" as *u8)
289
  let fd: i64 = sys_openat_rd("knowledge/index/reader_rows.bin" as *u8)
  if fd < 0 { qrb_ps("reader_rows.bin open FAILED\n" as *u8); return 1 }
  let cap: i64 = 16777216
  let raw: *u8 = sys_mmap(cap)
  var got: i64=0
  var r: i64=1
  while r > 0 { r=sys_read(fd,(raw as i64+got) as *u8, cap-got); if r>0 { got=got+r } }
  sys_close(fd)
  if got < 16 { qrb_ps("reader_rows.bin too small\n" as *u8); return 1 }
299
  // preallocate ALL per-row scratch ONCE (no per-row/per-token mmap -> no leak in the loop)
  let pr: *u8 = sys_mmap(4096)
  let out: *u8 = sys_mmap(4096)
  let meta: *i64 = sys_mmap(8*8) as *i64
  let gp: *i64 = sys_mmap(12*8) as *i64
  let sc: *i64 = sys_mmap(7*8) as *i64
  sc[0]=sys_mmap(1024) as i64
  sc[1]=sys_mmap(1024) as i64
  sc[2]=sys_mmap(256*8) as i64
  sc[3]=sys_mmap(256*8) as i64
  sc[4]=sys_mmap(256*8) as i64
  sc[5]=sys_mmap(256*8) as i64
  sc[6]=sys_mmap(256*8) as i64
  let goldbuf: *u8 = sys_mmap(4096)          // NUL-terminated copy of gold for qs_f1 (gold in the mmap is not NUL-term)
  let sel: *i64 = sys_mmap(12*8) as *i64     // qrb_sel_ctx scratch (see its header comment)
  sel[0]=sys_mmap(4096) as i64
  sel[1]=sys_mmap(1024) as i64
  sel[2]=sys_mmap(256*8) as i64
  sel[3]=sys_mmap(256*8) as i64
  sel[4]=sys_mmap(256*8) as i64
  sel[5]=sys_mmap(256*8) as i64
  sel[6]=sys_mmap(64*8) as i64
  sel[7]=sys_mmap(64*8) as i64
  sel[8]=sys_mmap(64*8) as i64
  sel[9]=sys_mmap(512) as i64
  sel[10]=sys_mmap(256*8) as i64
  let selbuf: *u8 = sel[9] as *u8            // bind-before-index (inline-cast-index gotcha)
327
  var sumbox: *i64 = sys_mmap(8) as *i64
  let done0: i64 = qrb_read_done(sumbox)
  var sum: i64 = sumbox[0]
  qrb_ps("[resume] prior rows done=" as *u8); qrb_pn(done0)
  qrb_ps(" (sum=" as *u8); qrb_pn(sum); qrb_ps(") ; target N=" as *u8); qrb_pn(N); qrb_ps("\n" as *u8)
333
  var off: i64 = 8
  var idx: i64 = 0
  var newdone: i64 = 0
  while off + 24 < got {
      if idx >= N { off = got } else {
          let hp: *i64 = (raw as i64 + off) as *i64
          let qlen: i64 = hp[0]
          if qlen < 0 { off = got } else { if qlen > 100000 { off = got } else {
              let qp: *u8 = (raw as i64 + off + 8) as *u8
              let hp2: *i64 = (raw as i64 + off + 8 + qlen) as *i64
              let clen: i64 = hp2[0]
              let cp: *u8 = (raw as i64 + off + 16 + qlen) as *u8
              let hp3: *i64 = (raw as i64 + off + 16 + qlen + clen) as *i64
              let alen: i64 = hp3[0]
              let ap: *u8 = (raw as i64 + off + 24 + qlen + clen) as *u8
              off = off + 24 + qlen + clen + alen
350
              if idx >= done0 {
                  let cc: i64 = qrb_sel_ctx(cp, clen, qp, qlen, sel)   // question-aware window (retrieve-then-read)
                  var po: i64 = 0
                  po = nsv_cat(pr, po, "Extract the answer as the shortest exact phrase from the context, not a sentence.\n\nContext: The Eiffel Tower was completed in 1889 and stands in the city of Paris.\nQuestion: In what city is the Eiffel Tower?\nAnswer: Paris\n\nContext: " as *u8)
                  po = nsv_catb(pr, po, selbuf, cc)
                  po = nsv_cat(pr, po, "\n\nQuestion: " as *u8)
                  var qc: i64 = qlen
                  if qc > 120 { qc = 120 }
                  po = nsv_catb(pr, po, qp, qc)
                  po = nsv_cat(pr, po, "\nAnswer:" as *u8)
361
                  gp[0]=pr as i64
                  gp[1]=po
                  gp[2]=QRB_MAXNEW
                  gp[3]=MODE
                  gp[4]=out as i64
                  gp[5]=4096
                  gp[6]=meta as i64
                  gp[7]=0-1
                  gp[8]=0
                  gp[9]=0
                  gp[10]=0
                  gp[11]=0
                  var tlen: i64 = 0-9                       // sentinel: -9 = skipped (prompt over the token-window guard)
                  if po <= QRB_MAXPROMPT { tlen = nsv_generate(gp) }
376
                  var f1: i64 = 0
                  var el: i64 = 0
                  if tlen > 0 {
                      el = tlen
                      var i2: i64=0
                      var nlf: i64=0
                      while i2 < tlen { if nlf==0 { if (out[i2] as i64)==10 { el=i2; nlf=1 } } i2=i2+1 }
                      out[el] = 0 as u8                          // NUL-terminate the answer's first line for qs_f1
                      var gi: i64 = 0
                      while gi < alen { goldbuf[gi] = ap[gi]; gi=gi+1 }
                      goldbuf[alen] = 0 as u8
                      f1 = qs_f1(out, goldbuf)                   // CANONICAL SQuAD F1 (article-stripping) = exact 251-baseline metric
                  }
390
                  qrb_append_result(idx, f1)
                  sum = sum + f1
                  newdone = newdone + 1
394
                  qrb_ps("[" as *u8); qrb_pn(idx); qrb_ps("] F1=" as *u8); qrb_pn(f1)
                  if tlen == 0-9 { qrb_ps(" (skipped po=" as *u8); qrb_pn(po); qrb_ps(">guard)" as *u8) } else { if tlen <= 0 { qrb_ps(" (gen-err=" as *u8); qrb_pn(meta[5]); qrb_ps(")" as *u8) } }
                  qrb_ps("  gold=[" as *u8); sys_write(1, ap, alen); qrb_ps("]  qwen=[" as *u8)
                  if el > 0 { sys_write(1, out, el) }
                  qrb_ps("]\n" as *u8)
              }
              idx = idx + 1
          } }
      }
  }
405
  let total: i64 = done0 + newdone
  if total >= N {
      var mean: i64 = 0
      if N > 0 { mean = sum / N }
      qrb_ps("\n=== QWEN-AS-READER: mean token-F1 = " as *u8); qrb_pn(mean)
      qrb_ps(" permille over " as *u8); qrb_pn(N); qrb_ps(" real SQuAD rows ===\n" as *u8)
      qrb_ps("=== baselines: from-scratch neural reader F1~81 ; mechanical reader F1=251 ===\n" as *u8)
      qrb_ps("QRB-COMPLETE\n" as *u8)
  } else {
      qrb_ps("\n[progress] " as *u8); qrb_pn(total); qrb_ps("/" as *u8); qrb_pn(N)
      qrb_ps(" rows (this session +" as *u8); qrb_pn(newdone); qrb_ps(") ; relaunch to continue\n" as *u8)
  }
  return 0
419}
nx_qwen_reader_bench_gate.nx source

`nx_qwen_reader_bench_gate.nx` source
