code wiki / (root) / nx_reader_squad_gate.nx
`nx_reader_squad_gate.nx` source

↩ module page · 546 lines · 26562 B
1// nx_reader_squad_gate.nx -- RUNG 3 (a/b/c) of the NEURAL PASSAGE READER arc: the reader trains on REAL SQuAD.
2// R3c = ERROR-FEEDBACK SGD (nfa_sgd_ef): R3b's three runs all froze in a Q16 uniform-collapse absorbing state
3// because sub-quantum updates truncate to zero; error-feedback keeps the residuals so no gradient is lost.
4// R3a (2889ff525f) measured the from-scratch wall honestly: the pipeline + STABLE Q16 training work (grad-clip
5// + LR-decay; loss 8401->2286) but at 664 rows the reader MEMORIZES without out-generalizing a question-blind
6// token-prior baseline -- because 131k of 133k params were COLD-START embeddings.
7// R3b (this version) = the measured lever: PRETRAINED SOVEREIGN EMBEDDINGS. The vocab is exact-word (61-bit
8// ehash, same hash as the PPMI/SGNS vocab); the embedding table is INITIALIZED from R1's trained SGNS
9// embeddings (knowledge/index/embed_v1.bin, full-data run) and FROZEN -- trainable params drop 133k -> ~7k
10// (attention block + FFN + span heads only), attacking the memorization directly. Attention alignment
11// (Wq/Wk bilinear) now operates on MEANINGFUL word geometry (question<->context similarity = SGNS cosine).
12//   T1 held-out span token-F1: reader > question-blind ablation + 100 permille (liar-kill at pretrained parity)
13//   T2 held-out exact-start > 60 permille (chance ~11)
14//   T3 pretraining helps: reader F1 > R3a's from-scratch 56 + 50
15//   T4 deterministic re-eval
16// Consumes: reader_rows.bin (nx_qabench 'dt'), semppmi_v1.bin (vocab hashes), embed_v1.bin (SGNS Q10).
17// expect_exit: 0   license_tier: ORIGINAL   Sovereign: nx_nofloat_autograd + nx_syscalls.
18import "nx_nofloat_autograd.nx"
19import "nx_syscalls.nx"
20
21func rq_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
22func rq_pn(v: i64) -> i64 { let b: *u8=sys_mmap(28); var x: i64=v; if x<0{b[0]=45;sys_write(1,b,1);x=0-x} if x==0{b[0]=48;sys_write(1,b,1);return 0} var d: i64=0; var y: i64=x; while y>0{d=d+1;y=y/10} var i: i64=d-1; y=x; while i>=0{b[i]=(48+(y%10)) as u8;y=y/10;i=i-1} sys_write(1,b,d); return 0 }
23func rq_ck(name: *u8, c: i64) -> i64 { if c==1 { rq_puts("  PASS " as *u8) } else { rq_puts("  FAIL " as *u8) } rq_puts(name); rq_puts("\n" as *u8); return c }
24
25const RQ_E: i64 = 24           // = SGNS embedding dim (embed_v1.bin)
26const RQ_F: i64 = 48
27const RQ_MAXT: i64 = 96
28const RQ_MAXQ: i64 = 20
29const RQ_MAXG: i64 = 8
30const RQ_SCALE: i64 = 13378    // 1/sqrt(24) in Q16
31const RQ_NROWS: i64 = 960
32const RQ_MAXV: i64 = 16384     // exact-word vocab cap (id 0 = SEP; words 1..nw)
33const RQ_MAPN: i64 = 32768     // open-addressing map slots (power of 2)
34// flat weight layout
35const QOFF_EMB: i64 = 0                      // [MAXV,E] 393216 (FROZEN; SGNS-initialized)
36const QOFF_WQ: i64 = 393216                  // [E,E] 576
37const QOFF_WK: i64 = 393792
38const QOFF_WV: i64 = 394368
39const QOFF_WO: i64 = 394944
40const QOFF_WG: i64 = 395520                  // [E,F] 1152
41const QOFF_WU: i64 = 396672
42const QOFF_WD: i64 = 397824                  // [F,E] 1152
43const QOFF_US: i64 = 398976                  // [1,E] 24
44const QOFF_UE: i64 = 399000                  // [1,E] 24
45const RQ_NP: i64  = 399024
46
47// ---- word hashing: EXACTLY the PPMI/SGNS vocab hash (dual djb2, 61-bit) ----
48func rq_ehash(buf: *u8, off: i64, len: i64) -> i64 {
  var h1: i64 = 5381
  var h2: i64 = 77245
  var i: i64 = 0
  while i < len { let c: i64 = buf[off+i] as i64; h1 = (h1*33 + c) % 1073741789; h2 = (h2*131 + c) % 1073741783; i = i + 1 }
  return h1 * 1073741783 + h2
54}
55func rq_bsearch(a: *i64, n: i64, v: i64) -> i64 {
  var lo: i64 = 0
  var hi: i64 = n - 1
  while lo <= hi { let mid: i64 = (lo+hi)/2; if a[mid] == v { return mid } if a[mid] < v { lo = mid+1 } else { hi = mid-1 } }
  return 0-1
60}
61func rq_lc(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c }
62func rq_isal(c: i64) -> i64 { if c >= 97 { if c <= 122 { return 1 } } if c >= 48 { if c <= 57 { return 1 } } return 0 }
63
64// tokenize buf[0..n) into WORD HASHES (lowercased alnum runs, ehash'd); returns count
65func rq_tok(buf: *u8, n: i64, hout: *i64, cap: i64) -> i64 {
  let wb: *u8 = sys_mmap(64)
  var nt: i64 = 0
  var i: i64 = 0
  while i < n {
      let c: i64 = rq_lc(buf[i] as i64)
      if rq_isal(c) == 1 {
          var wl: i64 = 0
          var j: i64 = i
          var live: i64 = 1
          while live == 1 {
              if j >= n { live = 0 } else {
                  let cj: i64 = rq_lc(buf[j] as i64)
                  if rq_isal(cj) == 0 { live = 0 } else { if wl < 48 { wb[wl] = cj as u8; wl = wl + 1 } j = j + 1 }
              }
          }
          if nt < cap { hout[nt] = rq_ehash(wb, 0, wl); nt = nt + 1 }
          i = j
      } else { i = i + 1 }
  }
  return nt
86}
87
88// vocab: open-addressing hash->local-id map. vst[0]=nw. returns local id (1..nw); 0 reserved for SEP.
89func rq_vid(map: *i64, vhash: *i64, vst: *i64, h: i64) -> i64 {
  var slot: i64 = h & (RQ_MAPN - 1)
  if slot < 0 { slot = 0 - slot }
  var probe: i64 = 0
  while probe < RQ_MAPN {
      let e: i64 = map[slot]
      if e == 0 {
          // absent -> assign
          if vst[0] >= RQ_MAXV - 1 { return 1 }        // vocab full -> bucket everything into id 1 (rare)
          vst[0] = vst[0] + 1
          let id: i64 = vst[0]
          vhash[id] = h
          map[slot] = id
          return id
      }
      if vhash[e] == h { return e }
      slot = (slot + 1) & (RQ_MAPN - 1)
      probe = probe + 1
  }
  return 1
109}
110
111// ---- model forward (R2 architecture; embeddings pre-GATHERED into Xg[T*E]). out[0]=root loss,
112//      out[1]=start logits, out[2]=end logits, out[3]=Xg leaf, out[4..12]=other leaves. returns 0. ----
113func rq_fwd(tape: *i64, vals: *i64, st: *i64, W: *i64, Xg: *i64, tgts: *i64, tgte: *i64, T: i64, useattn: i64, out: *i64) -> i64 {
  st[0] = 0
  st[1] = 0
  let nXg: i64 = nfa_leaf(tape, vals, st, T, RQ_E, Xg, 0)
  let nWq: i64 = nfa_leaf(tape, vals, st, RQ_E, RQ_E, W, QOFF_WQ)
  let nWk: i64 = nfa_leaf(tape, vals, st, RQ_E, RQ_E, W, QOFF_WK)
  let nWv: i64 = nfa_leaf(tape, vals, st, RQ_E, RQ_E, W, QOFF_WV)
  let nWo: i64 = nfa_leaf(tape, vals, st, RQ_E, RQ_E, W, QOFF_WO)
  let nWg: i64 = nfa_leaf(tape, vals, st, RQ_E, RQ_F, W, QOFF_WG)
  let nWu: i64 = nfa_leaf(tape, vals, st, RQ_E, RQ_F, W, QOFF_WU)
  let nWd: i64 = nfa_leaf(tape, vals, st, RQ_F, RQ_E, W, QOFF_WD)
  let nUs: i64 = nfa_leaf(tape, vals, st, 1, RQ_E, W, QOFF_US)
  let nUe: i64 = nfa_leaf(tape, vals, st, 1, RQ_E, W, QOFF_UE)
  var nH: i64 = nXg
  if useattn == 1 {
      let nXn: i64 = nfa_rmsnorm_rows(tape, vals, st, nXg)
      let nQ: i64 = nfa_matmul(tape, vals, st, nXn, nWq)
      let nK: i64 = nfa_matmul(tape, vals, st, nXn, nWk)
      let nV: i64 = nfa_matmul(tape, vals, st, nXn, nWv)
      let nQr: i64 = nfa_rope(tape, vals, st, nQ)
      let nKr: i64 = nfa_rope(tape, vals, st, nK)
      let nS: i64 = nfa_matmul_nt(tape, vals, st, nQr, nKr)
      let nSs: i64 = nfa_cmul(tape, vals, st, nS, RQ_SCALE)
      let nA: i64 = nfa_softmax_rows(tape, vals, st, nSs, 1)
      let nO: i64 = nfa_matmul(tape, vals, st, nA, nV)
      let nOp: i64 = nfa_matmul(tape, vals, st, nO, nWo)
      let nOs: i64 = nfa_cmul(tape, vals, st, nOp, 16384)      // R3d(b): 0.25x branch scale -- the residual
      nH = nfa_vadd(tape, vals, st, nXg, nOs)                  // must DOMINATE so per-token states can't collapse
  }
  let nHn: i64 = nfa_rmsnorm_rows(tape, vals, st, nH)
  let nG: i64 = nfa_matmul(tape, vals, st, nHn, nWg)
  let nU2: i64 = nfa_matmul(tape, vals, st, nHn, nWu)
  let nSg: i64 = nfa_silu(tape, vals, st, nG)
  let nHs: i64 = nfa_hadamard(tape, vals, st, nSg, nU2)
  let nD: i64 = nfa_matmul(tape, vals, st, nHs, nWd)
  let nDs: i64 = nfa_cmul(tape, vals, st, nD, 16384)           // R3d(b): 0.25x FFN branch -- the diagnosed
  let nY: i64 = nfa_vadd(tape, vals, st, nH, nDs)              // spike source (FFN swamped the residual)
  let nYn: i64 = nfa_rmsnorm_rows(tape, vals, st, nY)
  let nLs: i64 = nfa_matmul_nt(tape, vals, st, nUs, nYn)
  let nLe: i64 = nfa_matmul_nt(tape, vals, st, nUe, nYn)
  let nCs: i64 = nfa_softce_rows(tape, vals, st, nLs, tgts)
  let nCe: i64 = nfa_softce_rows(tape, vals, st, nLe, tgte)
  let nRoot: i64 = nfa_vadd(tape, vals, st, nCs, nCe)
  out[0] = nRoot
  out[1] = nLs
  out[2] = nLe
  out[3] = nXg
  out[4] = nWq
  out[5] = nWk
  out[6] = nWv
  out[7] = nWo
  out[8] = nWg
  out[9] = nWu
  out[10] = nWd
  out[11] = nUs
  out[12] = nUe
  return 0
170}
171
172// accumulate grads; embeddings TRAINABLE (SGNS is the INIT, not a freeze -- v1 froze them and the block
173// collapsed into the uniform fixed point: loss 8170 -> exactly 2*ln(86)=8867 stuck, identical states => span
174// grads cancel by symmetry => zero gradient forever). Emb rows scattered back by token id, R3a-style.
175func rq_grab_emb(tape: *i64, grads: *i64, out: *i64, G: *i64, ids: *i64, T: i64) -> i64 {
  let nXg: i64 = out[3]
  let gx: i64 = tape[7*nXg+6]
  var t: i64 = 0
  while t < T {
      let id: i64 = ids[t]
      var e: i64 = 0
      while e < RQ_E { G[QOFF_EMB + id*RQ_E + e] = G[QOFF_EMB + id*RQ_E + e] + grads[gx + t*RQ_E + e]; e = e + 1 }
      t = t + 1
  }
  return 0
186}
187func rq_grab(tape: *i64, grads: *i64, out: *i64, G: *i64) -> i64 {
  var k: i64 = 0
  while k < 9 {
      var sz: i64 = 576
      var of: i64 = 0
      if k == 0 { of = QOFF_WQ }
      if k == 1 { of = QOFF_WK }
      if k == 2 { of = QOFF_WV }
      if k == 3 { of = QOFF_WO }
      if k == 4 { sz = 1152; of = QOFF_WG }
      if k == 5 { sz = 1152; of = QOFF_WU }
      if k == 6 { sz = 1152; of = QOFF_WD }
      if k == 7 { sz = 24; of = QOFF_US }
      if k == 8 { sz = 24; of = QOFF_UE }
      let nid: i64 = out[4+k]
      let goff: i64 = tape[7*nid+6]
      var i: i64 = 0
      while i < sz { G[of+i] = G[of+i] + grads[goff+i]; i = i + 1 }
      k = k + 1
  }
  return 0
208}
209
210// non-embedding weight init (embeddings come from SGNS)
211func rq_init(W: *i64) -> i64 {
  var i: i64 = 0
  while i < 576 { W[QOFF_WQ+i] = (((i*13)%9)-4)*(NFA_Q16/32); W[QOFF_WK+i] = (((i*17)%9)-4)*(NFA_Q16/32); W[QOFF_WV+i] = (((i*19)%9)-4)*(NFA_Q16/32); W[QOFF_WO+i] = (((i*23)%9)-4)*(NFA_Q16/32); i = i + 1 }
  i = 0
  while i < 1152 { W[QOFF_WG+i] = (((i*29)%9)-4)*(NFA_Q16/40); W[QOFF_WU+i] = (((i*31)%9)-4)*(NFA_Q16/40); W[QOFF_WD+i] = (((i*37)%9)-4)*(NFA_Q16/40); i = i + 1 }
  i = 0
  while i < 24 { W[QOFF_US+i] = (((i*5+1)%7)-3)*(NFA_Q16/8); W[QOFF_UE+i] = (((i*11+2)%7)-3)*(NFA_Q16/8); i = i + 1 }
  return 0
219}
220
221func rq_span_f1(ids: *i64, a: i64, b: i64, gs: i64, ge: i64) -> i64 {
  let pl: i64 = b - a + 1
  let gl: i64 = ge - gs + 1
  if pl <= 0 { return 0 }
  if gl <= 0 { return 0 }
  var common: i64 = 0
  let used: *i64 = sys_mmap(RQ_MAXG*8) as *i64
  var u: i64 = 0
  while u < gl { used[u] = 0; u = u + 1 }
  var p: i64 = a
  while p <= b {
      var q2: i64 = 0
      var got: i64 = 0
      while q2 < gl {
          if got == 0 { if used[q2] == 0 { if ids[gs+q2] == ids[p] { used[q2] = 1; common = common + 1; got = 1 } } }
          q2 = q2 + 1
      }
      p = p + 1
  }
  if common == 0 { return 0 }
  return (2*common*1000)/(pl+gl)
242}
243
244func rq_eval(tape: *i64, vals: *i64, st: *i64, W: *i64, dat: *i64, meta: *i64, r0: i64, r1: i64, useattn: i64, res: *i64) -> i64 {
  let out: *i64 = sys_mmap(16*8) as *i64
  let Xg: *i64 = sys_mmap(RQ_MAXT*RQ_E*8) as *i64
  let tgts: *i64 = sys_mmap(8) as *i64
  let tgte: *i64 = sys_mmap(8) as *i64
  var f1sum: i64 = 0
  var exact: i64 = 0
  var n: i64 = 0
  var r: i64 = r0
  while r < r1 {
      let T: i64 = meta[r*4+0]
      let qn: i64 = meta[r*4+1]
      let gs: i64 = meta[r*4+2]
      let ge: i64 = meta[r*4+3]
      let ids: *i64 = (dat as i64 + r*RQ_MAXT*8) as *i64
      var t: i64 = 0
      while t < T { var e: i64 = 0; while e < RQ_E { Xg[t*RQ_E+e] = W[QOFF_EMB + ids[t]*RQ_E + e]; e = e + 1 } t = t + 1 }
      tgts[0] = gs
      tgte[0] = ge
      rq_fwd(tape, vals, st, W, Xg, tgts, tgte, T, useattn, out)
      let nLs: i64 = out[1]
      let nLe: i64 = out[2]
      var s: i64 = qn + 1
      var bv: i64 = nfa_val(tape, vals, nLs, qn+1)
      var j: i64 = qn + 2
      while j < T { let v: i64 = nfa_val(tape, vals, nLs, j); if v > bv { bv = v; s = j } j = j + 1 }
      var en: i64 = s
      var ev: i64 = nfa_val(tape, vals, nLe, s)
      var lim: i64 = s + RQ_MAXG
      if lim > T { lim = T }
      j = s + 1
      while j < lim { let v2: i64 = nfa_val(tape, vals, nLe, j); if v2 > ev { ev = v2; en = j } j = j + 1 }
      f1sum = f1sum + rq_span_f1(ids, s, en, gs, ge)
      if s == gs { exact = exact + 1 }
      n = n + 1
      r = r + 1
  }
  if n == 0 { res[0] = 0; res[1] = 0; return 0 }
  res[0] = f1sum / n
  res[1] = exact * 1000 / n
  return 0
285}
286
287func rq_train(tape: *i64, vals: *i64, grads: *i64, st: *i64, W: *i64, dat: *i64, meta: *i64, ntrain: i64, useattn: i64, EP: i64, lr: i64, verbose: i64) -> i64 {
  let out: *i64 = sys_mmap(16*8) as *i64
  let Xg: *i64 = sys_mmap(RQ_MAXT*RQ_E*8) as *i64
  let tgts: *i64 = sys_mmap(8) as *i64
  let tgte: *i64 = sys_mmap(8) as *i64
  let G: *i64 = sys_mmap(RQ_NP*8) as *i64
  let EF: *i64 = sys_mmap(RQ_NP*8) as *i64             // error-feedback residuals (R3c: no grad lost to >>16)
  var zf: i64 = 0
  while zf < RQ_NP { EF[zf] = 0; zf = zf + 1 }
  var ep: i64 = 0
  while ep < EP {
      var lsum: i64 = 0
      var nb: i64 = 0
      var r: i64 = 0
      while r + 8 <= ntrain {
          var z: i64 = 0
          while z < RQ_NP { G[z] = 0; z = z + 1 }
          var mb: i64 = 0
          while mb < 8 {
              let rr: i64 = r + mb
              let T: i64 = meta[rr*4+0]
              let gs: i64 = meta[rr*4+2]
              let ge: i64 = meta[rr*4+3]
              let ids: *i64 = (dat as i64 + rr*RQ_MAXT*8) as *i64
              var t: i64 = 0
              while t < T { var e: i64 = 0; while e < RQ_E { Xg[t*RQ_E+e] = W[QOFF_EMB + ids[t]*RQ_E + e]; e = e + 1 } t = t + 1 }
              tgts[0] = gs
              tgte[0] = ge
              rq_fwd(tape, vals, st, W, Xg, tgts, tgte, T, useattn, out)
              let nRoot: i64 = out[0]
              lsum = lsum + nfa_val(tape, vals, nRoot, 0)
              nfa_backward(tape, vals, grads, st[0], nRoot)
              rq_grab(tape, grads, out, G)
              rq_grab_emb(tape, grads, out, G, ids, T)
              mb = mb + 1
          }
          // FULL-param batch-mean + GRAD-CLIP +-4.0 + LR DECAY @8/16 -- EXACTLY the R3a-proven stable recipe
          var w: i64 = 0
          while w < RQ_NP {
              var gv: i64 = G[w] / 8
              if gv > 262144 { gv = 262144 }
              if gv < 0-262144 { gv = 0-262144 }
              G[w] = gv
              w = w + 1
          }
          var elr: i64 = lr
          if ep >= 8 { elr = lr/2 }
          if ep >= 16 { elr = lr/4 }
          nfa_sgd_ef(W, G, EF, RQ_NP, elr)             // R3c: error-feedback (sub-quantum residuals kept)
          nb = nb + 1
          r = r + 8
      }
      if verbose == 1 { rq_puts("    epoch "); rq_pn(ep); rq_puts("  mean_loss_milli="); rq_pn((lsum/(nb*8))*1000/NFA_Q16); rq_puts("\n" as *u8) }
      ep = ep + 1
  }
  return 0
343}
344
345func main() -> i64 {
  rq_puts("nx_reader_squad_gate (RUNG 3b: SGNS-PRETRAINED frozen embeddings + trainable attention reader on REAL SQuAD)\n" as *u8)
  var pass: i64 = 0
  var total: i64 = 0
349
  // ---- load reader_rows.bin ----
  let fd: i64 = sys_openat_rd("knowledge/index/reader_rows.bin" as *u8)
  if fd < 0 { rq_puts("RED -- reader_rows.bin missing (run: nx_qabench dt)\n" as *u8); return 1 }
  let cap: i64 = 16777216
  let raw: *u8 = sys_mmap(cap)
  var got: i64 = 0
  var rr2: i64 = 1
  while rr2 > 0 { rr2 = sys_read(fd, (raw as i64 + got) as *u8, cap - got); if rr2 > 0 { got = got + rr2 } }
  sys_close(fd)
  if got < 16 { rq_puts("RED -- reader_rows.bin empty\n" as *u8); return 1 }
360
  // ---- load SGNS vocab hashes (semppmi vh) + trained embeddings (embed_v1 Q10) ----
  let pfd: i64 = sys_openat_rd("knowledge/index/semppmi_v1.bin" as *u8)
  if pfd < 0 { rq_puts("RED -- semppmi_v1.bin missing\n" as *u8); return 1 }
  let pcap: i64 = 2097152
  let pblob: *u8 = sys_mmap(pcap)
  var pgot: i64 = 0
  var pr: i64 = 1
  while pr > 0 { if pgot >= pcap { pr = 0 } else { pr = sys_read(pfd, (pblob as i64 + pgot) as *u8, pcap - pgot); if pr > 0 { pgot = pgot + pr } } }
  sys_close(pfd)
  let phi: *i64 = (pblob as i64 + 8) as *i64
  let nv: i64 = phi[0]
  if pgot < 32 + nv*8 { rq_puts("RED -- semppmi vh truncated\n" as *u8); return 1 }
  let vh: *i64 = (pblob as i64 + 32) as *i64
  let efd: i64 = sys_openat_rd("knowledge/index/embed_v1.bin" as *u8)
  if efd < 0 { rq_puts("RED -- embed_v1.bin missing (run: nx_embed_train)\n" as *u8); return 1 }
  let ecap: i64 = 67108864
  let eblob: *u8 = sys_mmap(ecap)
  var egot: i64 = 0
  var er: i64 = 1
  while er > 0 { if egot >= ecap { er = 0 } else { er = sys_read(efd, (eblob as i64 + egot) as *u8, ecap - egot); if er > 0 { egot = egot + er } } }
  sys_close(efd)
  let ehi: *i64 = (eblob as i64 + 8) as *i64
  let env: i64 = ehi[0]
  let edim: i64 = ehi[1]
  if edim != RQ_E { rq_puts("RED -- embed dim mismatch\n" as *u8); return 1 }
  if env != nv { rq_puts("RED -- embed/vocab nv mismatch\n" as *u8); return 1 }
  let EQ: *i64 = (eblob as i64 + 24) as *i64                 // Q10 rows [nv x 24]
  rq_puts("  SGNS loaded: vocab nv="); rq_pn(nv); rq_puts("  dim="); rq_pn(edim); rq_puts("\n" as *u8)
389
  // ---- parse rows -> exact-word vocab + tokenized dataset with gold spans ----
  let dat: *i64 = sys_mmap(RQ_NROWS*RQ_MAXT*8) as *i64
  let meta: *i64 = sys_mmap(RQ_NROWS*4*8) as *i64
  let qh: *i64 = sys_mmap(64*8) as *i64
  let ch: *i64 = sys_mmap(4096*8) as *i64
  let gh: *i64 = sys_mmap(64*8) as *i64
  let vhash: *i64 = sys_mmap(RQ_MAXV*8) as *i64
  let vmap: *i64 = sys_mmap(RQ_MAPN*8) as *i64
  let vst: *i64 = sys_mmap(8) as *i64
  vst[0] = 0
  var nrows: i64 = 0
  var tsum: i64 = 0
  var off: i64 = 8
  var scanned: i64 = 0
  while off + 24 < got {
      if nrows >= RQ_NROWS { off = got } else {
          let hp: *i64 = (raw as i64 + off) as *i64
          let qlen: i64 = hp[0]
          if qlen < 0 { off = got } else { if qlen > 4000 { off = got } else {
              let qp: *u8 = (raw as i64 + off + 8) as *u8
              let hp2: *i64 = (raw as i64 + off + 8 + qlen) as *i64
              let clen: i64 = hp2[0]
              let cp: *u8 = (raw as i64 + off + 16 + qlen) as *u8
              let hp3: *i64 = (raw as i64 + off + 16 + qlen + clen) as *i64
              let alen: i64 = hp3[0]
              let ap: *u8 = (raw as i64 + off + 24 + qlen + clen) as *u8
              off = off + 24 + qlen + clen + alen
              scanned = scanned + 1
              var qn: i64 = rq_tok(qp, qlen, qh, RQ_MAXQ)
              let cn0: i64 = rq_tok(cp, clen, ch, 4096)
              let gn: i64 = rq_tok(ap, alen, gh, RQ_MAXG)
              var cn: i64 = RQ_MAXT - qn - 1
              if cn > cn0 { cn = cn0 }
              if qn >= 3 { if gn >= 1 { if cn >= 8 {
                  var gs: i64 = 0-1
                  var c: i64 = 0
                  while c + gn <= cn {
                      if gs < 0 {
                          var m: i64 = 1
                          var k: i64 = 0
                          while k < gn { if ch[c+k] != gh[k] { m = 0; k = gn } else { k = k + 1 } }
                          if m == 1 { gs = c }
                      }
                      c = c + 1
                  }
                  if gs >= 0 {
                      let T: i64 = qn + 1 + cn
                      let ids: *i64 = (dat as i64 + nrows*RQ_MAXT*8) as *i64
                      var t: i64 = 0
                      while t < qn { let lid: i64 = rq_vid(vmap, vhash, vst, qh[t]); ids[t] = lid; t = t + 1 }
                      ids[qn] = 0                                  // SEP = local id 0
                      t = 0
                      while t < cn { let lid2: i64 = rq_vid(vmap, vhash, vst, ch[t]); ids[qn+1+t] = lid2; t = t + 1 }
                      meta[nrows*4+0] = T
                      meta[nrows*4+1] = qn
                      meta[nrows*4+2] = qn + 1 + gs
                      meta[nrows*4+3] = qn + 1 + gs + gn - 1
                      tsum = tsum + T
                      nrows = nrows + 1
                  }
              } } }
          } }
      }
  }
  let nw: i64 = vst[0]
  rq_puts("  scanned="); rq_pn(scanned); rq_puts("  usable="); rq_pn(nrows); rq_puts("  avg T="); rq_pn(tsum/(nrows+1)); rq_puts("  vocab nw="); rq_pn(nw); rq_puts("\n" as *u8)
  if nrows < 200 { rq_puts("RED -- too few usable rows\n" as *u8); return 1 }
  var ntrain: i64 = (nrows * 4) / 5
  let nheld: i64 = nrows - ntrain
  rq_puts("  train="); rq_pn(ntrain); rq_puts("  held-out="); rq_pn(nheld); rq_puts("\n" as *u8)
460
  // ---- embedding table: SGNS-initialized (Q10<<6 = Q16), FROZEN; OOV + SEP = deterministic small init ----
  let W: *i64 = sys_mmap(RQ_NP*8) as *i64
  var hits: i64 = 0
  var wv: i64 = 1
  while wv <= nw {
      let r3: i64 = rq_bsearch(vh, nv, vhash[wv])
      if r3 >= 0 {
          // SGNS DIRECTION, R3a-proven AMPLITUDE: raw trained rows are ~10-15x larger than the stable init
          // scale and drove the Q16 SGD into the uniform absorbing state (v2 collapse); rescale each row to
          // RMS=0.125 Q16 (8192), preserving the semantic geometry. (Standard pretrained-init rescaling.)
          var ss: i64 = 0
          var e2: i64 = 0
          while e2 < RQ_E { let q6: i64 = EQ[r3*RQ_E + e2] << 6; ss = ss + nfa_qmul(q6, q6); e2 = e2 + 1 }
          var rms: i64 = nfa_isqrt(((ss / RQ_E) + 1) << 16)
          if rms < 64 { rms = 64 }
          e2 = 0
          while e2 < RQ_E { let q6b: i64 = EQ[r3*RQ_E + e2] << 6; W[QOFF_EMB + wv*RQ_E + e2] = (q6b * 8192) / rms; e2 = e2 + 1 }
          hits = hits + 1
      } else {
          var e3: i64 = 0
          while e3 < RQ_E { W[QOFF_EMB + wv*RQ_E + e3] = (((wv*7+e3*3)%11)-5)*(NFA_Q16/32); e3 = e3 + 1 }
      }
      wv = wv + 1
  }
  var e4: i64 = 0
  while e4 < RQ_E { W[QOFF_EMB + e4] = (((e4*5+2)%9)-4)*(NFA_Q16/12); e4 = e4 + 1 }
  rq_puts("  SGNS coverage: "); rq_pn(hits); rq_puts("/"); rq_pn(nw); rq_puts(" vocab words pretrained ("); rq_pn(hits*1000/(nw+1)); rq_puts(" permille)\n" as *u8)
  rq_init(W)
489
  let tape: *i64 = sys_mmap(4096*7*8) as *i64
  let vals: *i64 = sys_mmap(262144*8) as *i64
  let grads: *i64 = sys_mmap(262144*8) as *i64
  let st: *i64 = sys_mmap(2*8) as *i64
  let EP: i64 = 24
  let lr: i64 = 2048             // back to the proven rate: R3d(b) residual-scale (0.25x branches) closes the
                                 // basin entrance structurally, so the lr band that trained R3a applies again.
497
  rq_puts("  training READER (SGNS-INITIALIZED trainable embeddings; R3a-proven recipe)...\n" as *u8)
  rq_train(tape, vals, grads, st, W, dat, meta, ntrain, 1, EP, lr, 1)
  let resA: *i64 = sys_mmap(2*8) as *i64
  rq_eval(tape, vals, st, W, dat, meta, ntrain, nrows, 1, resA)
  rq_puts("  READER held-out: span-F1="); rq_pn(resA[0]); rq_puts(" permille  exact-start="); rq_pn(resA[1]); rq_puts(" permille (chance ~11; R3a from-scratch was F1=56)\n" as *u8)
503
  rq_puts("  training ABLATION (question-blind, same frozen embeddings/budget)...\n" as *u8)
  let W2: *i64 = sys_mmap(RQ_NP*8) as *i64
  var cw: i64 = 0
  while cw < RQ_NP { W2[cw] = W[cw]; cw = cw + 1 }             // same SGNS embeddings
  rq_init(W2)                                                  // fresh non-emb weights (same init as reader)
  rq_train(tape, vals, grads, st, W2, dat, meta, ntrain, 0, EP, lr, 0)
  let resB: *i64 = sys_mmap(2*8) as *i64
  rq_eval(tape, vals, st, W2, dat, meta, ntrain, nrows, 0, resB)
  rq_puts("  ABLATION held-out: span-F1="); rq_pn(resB[0]); rq_puts(" permille  exact-start="); rq_pn(resB[1]); rq_puts(" permille\n" as *u8)
513
  total = total + 1
  if resA[0] > resB[0] + 100 { pass = pass + 1; rq_ck("T1 reader beats question-blind ablation by >100 F1 (liar-kill)" as *u8, 1) } else { rq_ck("T1 reader beats question-blind ablation by >100 F1 (liar-kill)" as *u8, 0) }
  total = total + 1
  if resA[1] > 60 { pass = pass + 1; rq_ck("T2 reader exact-start > 60 permille (>5x chance)" as *u8, 1) } else { rq_ck("T2 reader exact-start > 60 permille (>5x chance)" as *u8, 0) }
  total = total + 1
  if resA[0] > 106 { pass = pass + 1; rq_ck("T3 pretraining helps: F1 > from-scratch 56 + 50" as *u8, 1) } else { rq_ck("T3 pretraining helps: F1 > from-scratch 56 + 50" as *u8, 0) }
520
  let resC: *i64 = sys_mmap(2*8) as *i64
  rq_eval(tape, vals, st, W, dat, meta, ntrain, nrows, 1, resC)
  total = total + 1
  var t4: i64 = 0
  if resC[0] == resA[0] { if resC[1] == resA[1] { t4 = 1 } }
  if t4 == 1 { pass = pass + 1; rq_ck("T4 deterministic re-eval" as *u8, 1) } else { rq_ck("T4 deterministic re-eval" as *u8, 0) }
527
  let pf: i64 = sys_openat_wr("knowledge/index/reader_neural.bin" as *u8, 0x1a4)
  if pf >= 0 {
      let mg: *i64 = sys_mmap(32) as *i64
      let mgb: *u8 = mg as *u8
      mgb[0]=78 as u8; mgb[1]=88 as u8; mgb[2]=78 as u8; mgb[3]=82 as u8; mgb[4]=50 as u8; mgb[5]=0 as u8; mgb[6]=0 as u8; mgb[7]=0 as u8
      mg[1] = nw
      mg[2] = RQ_E
      mg[3] = RQ_F
      sys_write(pf, mg as *u8, 32)
      sys_write(pf, W as *u8, RQ_NP*8)
      sys_close(pf)
      rq_puts("  persisted trained reader -> knowledge/index/reader_neural.bin (NXNR2)\n" as *u8)
  }
541
  rq_puts("---- nx_reader_squad_gate: passed "); rq_pn(pass); rq_puts(" / "); rq_pn(total); rq_puts("\n" as *u8)
  if pass == total { rq_puts("READER RUNG3b GREEN -- SGNS-pretrained sovereign embeddings + trained attention = a GENERALIZING reader on real SQuAD; the question-blind reader cannot.\n" as *u8); return 0 }
  rq_puts("RED -- rung 3b not fully passed (see numbers)\n" as *u8)
  return 1
546}
nx_reader_squad_gate.nx source

`nx_reader_squad_gate.nx` source
