code wiki / _hdl_build / nx_nofloat_prose_gate.nx
`nx_nofloat_prose_gate.nx` source

↩ module page · 146 lines · 8482 B
1// nx_nofloat_prose_gate.nx -- CAP-NF-PROSE: the no-float char-LM trains on REAL ENGLISH PROSE (words + spaces,
2// not the alphabet) at a LARGER scale (dm=24/ffn=48). Real prose is genuinely harder than a periodic sequence:
3// the space char precedes many different words, so the model MUST use causal-attention history to disambiguate.
4// Trains the full pre-norm transformer block (all weights) with clipped SGD. Honest: an INCREMENTAL step into
5// CAP-AI-FRONTIER (real corpus + scale), not a claim to finish it.
6//   T1 started untrained (CE > 0) ; T2 CE drops substantially (>= 60%) ; T3 next-char accuracy >> chance (1/V)
7//   + prints the model's predicted text vs target (visible evidence it learned the prose).
8// Lean: 2 clm_fwd call sites (train-loop + accuracy/print). Sovereign: nx_nofloat_autograd + nx_syscalls. expect_exit: 0
9import "nx_nofloat_autograd.nx"
10import "nx_syscalls.nx"
11import "nx_gate_emit_lib.nx"
12const Q16: i64 = 65536
13
14
15func slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
16func dini(a: *i64, n: i64, sd: i64) -> i64 { var i: i64=0; while i<n { a[i]=(((i*7+sd*13+1)%11)-5)*13107; i=i+1 } return 0 }
17
18func clm_fwd(tape: *i64, vals: *i64, st: *i64, W: *i64, ids: *i64, tgt: *i64, T: i64, dm: i64, ffn: i64, V: i64, scale: i64, leaves: *i64) -> i64 {
  let E: *i64=W[0] as *i64; let Wq: *i64=W[1] as *i64; let Wk: *i64=W[2] as *i64; let Wv: *i64=W[3] as *i64; let Wo: *i64=W[4] as *i64
  let Wg: *i64=W[5] as *i64; let Wu: *i64=W[6] as *i64; let Wd: *i64=W[7] as *i64; let Wlm: *i64=W[8] as *i64
  st[0]=0; st[1]=0
  let nE: i64=nfa_leaf(tape,vals,st,V,dm,E,0)
  let nWq: i64=nfa_leaf(tape,vals,st,dm,dm,Wq,0)
  let nWk: i64=nfa_leaf(tape,vals,st,dm,dm,Wk,0)
  let nWv: i64=nfa_leaf(tape,vals,st,dm,dm,Wv,0)
  let nWo: i64=nfa_leaf(tape,vals,st,dm,dm,Wo,0)
  let nWg: i64=nfa_leaf(tape,vals,st,dm,ffn,Wg,0)
  let nWu: i64=nfa_leaf(tape,vals,st,dm,ffn,Wu,0)
  let nWd: i64=nfa_leaf(tape,vals,st,ffn,dm,Wd,0)
  let nWlm: i64=nfa_leaf(tape,vals,st,dm,V,Wlm,0)
  let nX: i64=nfa_embed(tape,vals,st,nE,ids,T)
  let nXn: i64=nfa_rmsnorm_rows(tape,vals,st,nX)
  let nQ: i64=nfa_matmul(tape,vals,st,nXn,nWq)
  let nK: i64=nfa_matmul(tape,vals,st,nXn,nWk)
  let nV: i64=nfa_matmul(tape,vals,st,nXn,nWv)
  let nQr: i64=nfa_rope(tape,vals,st,nQ)
  let nKr: i64=nfa_rope(tape,vals,st,nK)
  let nS: i64=nfa_matmul_nt(tape,vals,st,nQr,nKr)
  let nSs: i64=nfa_cmul(tape,vals,st,nS,scale)
  let nA: i64=nfa_softmax_rows(tape,vals,st,nSs,1)
  let nO: i64=nfa_matmul(tape,vals,st,nA,nV)
  let nOp: i64=nfa_matmul(tape,vals,st,nO,nWo)
  let nH: i64=nfa_vadd(tape,vals,st,nX,nOp)
  let nHn: i64=nfa_rmsnorm_rows(tape,vals,st,nH)
  let nG: i64=nfa_matmul(tape,vals,st,nHn,nWg)
  let nU: i64=nfa_matmul(tape,vals,st,nHn,nWu)
  let nSg: i64=nfa_silu(tape,vals,st,nG)
  let nHs: i64=nfa_hadamard(tape,vals,st,nSg,nU)
  let nDp: i64=nfa_matmul(tape,vals,st,nHs,nWd)
  let nY: i64=nfa_vadd(tape,vals,st,nH,nDp)
  let nYn: i64=nfa_rmsnorm_rows(tape,vals,st,nY)
  let nLg: i64=nfa_matmul(tape,vals,st,nYn,nWlm)
  let nLoss: i64=nfa_softce_rows(tape,vals,st,nLg,tgt)
  leaves[0]=nE; leaves[1]=nWq; leaves[2]=nWk; leaves[3]=nWv; leaves[4]=nWo
  leaves[5]=nWg; leaves[6]=nWu; leaves[7]=nWd; leaves[8]=nWlm; leaves[9]=nLg
  return nLoss
57}
58func step_all(tape: *i64, grads: *i64, W: *i64, WN: *i64, leaves: *i64, nW: i64, lr: i64, clip: i64, gb: *i64) -> i64 {
  var i: i64=0
  while i<nW { let ar: *i64=W[i] as *i64; let cn: i64=WN[i]; let nd: i64=leaves[i]; var c: i64=0; while c<cn { var g: i64=nfa_grad(tape,grads,nd,c); if g>clip{g=clip} if g<0-clip{g=0-clip} gb[c]=g; c=c+1 } nfa_sgd(ar,gb,cn,lr); i=i+1 }
  return 0
62}
63// call site 1: train all weights
64func do_train(tape: *i64, vals: *i64, grads: *i64, st: *i64, W: *i64, WN: *i64, ids: *i64, tgt: *i64, T: i64, dm: i64, ffn: i64, V: i64, scale: i64, leaves: *i64, gb: *i64, steps: i64, cf: *i64, cl: *i64) -> i64 {
  var ep: i64=0
  while ep < steps {
      let nl: i64=clm_fwd(tape,vals,st,W,ids,tgt,T,dm,ffn,V,scale,leaves)
      nfa_backward(tape,vals,grads,st[0],nl)
      if ep==0 { cf[0]=nfa_val(tape,vals,nl,0) }
      cl[0]=nfa_val(tape,vals,nl,0)
      step_all(tape,grads,W,WN,leaves,9,6554,262144,gb)
      ep=ep+1
  }
  return 0
75}
76
77func main() -> i64 {
  g_puts("nx_nofloat_prose gate (char-LM trains on REAL ENGLISH PROSE at scale dm=24, pure integer Q16)\n" as *u8)
  var pass: i64=0; var total: i64=0
  let corpus: *u8 = "the cat sat on the mat" as *u8
  let L: i64 = slen(corpus)
  let c2i: *i64 = sys_mmap(256*8) as *i64
  var iz: i64=0; while iz<256 { c2i[iz]=0-1; iz=iz+1 }
  let i2c: *i64 = sys_mmap(256*8) as *i64
  var V: i64=0; var pz: i64=0
  while pz<L { let ch: i64=corpus[pz] as i64; if c2i[ch]<0 { c2i[ch]=V; i2c[V]=ch; V=V+1 } pz=pz+1 }
  let seq: *i64 = sys_mmap(L*8) as *i64
  pz=0; while pz<L { seq[pz]=c2i[corpus[pz] as i64]; pz=pz+1 }
  let T: i64 = L-1
  let ids: *i64 = sys_mmap(T*8) as *i64; let tgt: *i64 = sys_mmap(T*8) as *i64
  pz=0; while pz<T { ids[pz]=seq[pz]; tgt[pz]=seq[pz+1]; pz=pz+1 }
  g_puts("  corpus=\"" as *u8); g_puts(corpus); g_puts("\" len=" as *u8); g_pn(L); g_puts(" vocab=" as *u8); g_pn(V); g_puts(" (real words+spaces; space is ambiguous -> needs history)\n" as *u8)
93
  let dm: i64=24; let ffn: i64=48; let scale: i64=13377   // 1/sqrt(24)
  let tape:  *i64 = sys_mmap(512*7*8) as *i64
  let vals:  *i64 = sys_mmap(262144*8) as *i64
  let grads: *i64 = sys_mmap(262144*8) as *i64
  let st:    *i64 = sys_mmap(2*8) as *i64
  let nW: i64=9
  let W: *i64 = sys_mmap(nW*8) as *i64
  let WN: *i64 = sys_mmap(nW*8) as *i64
  WN[0]=V*dm; WN[1]=dm*dm; WN[2]=dm*dm; WN[3]=dm*dm; WN[4]=dm*dm; WN[5]=dm*ffn; WN[6]=dm*ffn; WN[7]=ffn*dm; WN[8]=dm*V
  var wi: i64=0
  while wi<nW { let a: *i64=sys_mmap(WN[wi]*8) as *i64; dini(a,WN[wi],wi+1); W[wi]=a as i64; wi=wi+1 }
  let leaves: *i64 = sys_mmap(10*8) as *i64
  let gbuf: *i64 = sys_mmap(8192*8) as *i64
  let cf: *i64 = sys_mmap(8) as *i64; let cl: *i64 = sys_mmap(8) as *i64
108
  do_train(tape,vals,grads,st,W,WN,ids,tgt,T,dm,ffn,V,scale,leaves,gbuf,16000,cf,cl)
110
  // accuracy + predicted text (call site 2)
  let nLf: i64 = clm_fwd(tape,vals,st,W,ids,tgt,T,dm,ffn,V,scale,leaves)
  let off: i64 = tape[7*leaves[9]+5]
  var correct: i64=0; var tt: i64=0
  while tt<T {
      var best: i64=0; var bv: i64=vals[off+tt*V]; var j: i64=1
      while j<V { if vals[off+tt*V+j]>bv { bv=vals[off+tt*V+j]; best=j } j=j+1 }
      if best==tgt[tt] { correct=correct+1 }
      tt=tt+1
  }
  g_puts("  [measure] CE start=" as *u8); g_pn(cf[0]); g_puts(" end=" as *u8); g_pn(cl[0]); g_puts("   next-char acc=" as *u8); g_pn(correct); g_puts("/" as *u8); g_pn(T); g_puts(" (chance~" as *u8); g_pn(T/V); g_puts(")\n" as *u8)
  g_puts("  target   : " as *u8); g_puts(corpus); g_puts("\n  predicted: " as *u8)
  let ch0: *u8 = sys_mmap(2); ch0[0]=corpus[0]; sys_write(1,ch0,1)
  tt=0
  while tt<T {
      var best: i64=0; var bv: i64=vals[off+tt*V]; var j: i64=1
      while j<V { if vals[off+tt*V+j]>bv { bv=vals[off+tt*V+j]; best=j } j=j+1 }
      let cb: *u8 = sys_mmap(2); cb[0]=i2c[best] as u8; sys_write(1,cb,1)
      tt=tt+1
  }
  g_puts("\n" as *u8)
132
  var t1: i64=0; if cf[0] > 0 { t1=1 }
  pass=pass+g_check("T1: started genuinely untrained (CE > 0)" as *u8, t1); total=total+1
  var t2: i64=0; if cl[0]*10 <= cf[0]*4 { t2=1 }      // CE dropped >= 60%
  pass=pass+g_check("T2: CE drops >= 60% on real prose (the LM learned the text)" as *u8, t2); total=total+1
  var t3: i64=0; if correct*V >= T*3 { t3=1 }          // accuracy >= 3x chance (3/V); chance=1/V
  pass=pass+g_check("T3: next-char accuracy >= 3x chance on real prose (learned real-text structure)" as *u8, t3); total=total+1
139
  var okall: i64=0; if pass==total { okall=1 }
  let logf: i64 = sys_openat_append("knowledge/status/nofloat_prose.log" as *u8, 420)
  if logf >= 0 { let w0: i64=sys_write(logf,"NOFLOATPROSE trained on real english prose\n" as *u8,42); sys_close(logf) }
  g_puts("---- prose gate: passed " as *u8); g_pn(pass); g_puts(" / " as *u8); g_pn(total); g_puts(" ----\n" as *u8)
  if okall==1 { g_puts("verdict=GREEN (no-float char-LM trains on real English prose at scale; predicts next char >> chance)\n" as *u8); sys_exit(0); return 0 }
  g_puts("verdict=RED\n" as *u8); sys_exit(1); return 1
146}
nx_nofloat_prose_gate.nx source

`nx_nofloat_prose_gate.nx` source
