code wiki / _hdl_build / nx_nofloat_deepstack_gate.nx
`nx_nofloat_deepstack_gate.nx` source

↩ module page · 215 lines · 13668 B
1// nx_nofloat_deepstack_gate.nx -- HARD-EVIDENCE gate that the no-float transformer scales in DEPTH: a
2// 2-BLOCK (depth-2) transformer LM trains end-to-end on next-token prediction, pure integer Q16. Each block is
3// the verified pre-norm block (attn+RoPE+causal-softmax+SwiGLU+residuals); stacking = composing blk forward N
4// times (NO new ops -- depth is pure composition). Trained with the Q16 AdamW optimizer.
5//
6//   A1 depth-2 gradcheck wrt Wlm : the LM-head gradient == finite differences.
7//   A2 depth-2 gradcheck wrt E   : E's gradient flows back through BOTH blocks (the deep-backprop proof) == FD.
8//   D  neg-control teeth (E, largest component) ; C bit-exact.
9//   B  the depth-2 LM TRAINS     : next-token CE drops substantially (AdamW on E+Wlm through the 2-block stack).
10//   (informational) depth-1 vs depth-2 final CE printed for transparency.
11//
12// Evidence -> knowledge/status/nofloat_deepstack.log. Sovereign: nx_nofloat_autograd + nx_syscalls. expect_exit: 0
13import "nx_nofloat_autograd.nx"
14import "nx_syscalls.nx"
15import "nx_gate_emit_lib.nx"
16
17const DLOG: *u8 = "knowledge/status/nofloat_deepstack.log"
18const Q16:  i64 = 65536
19
20
21func g_abs(v: i64) -> i64 { if v < 0 { return 0 - v } return v }
22func d_ws(fd: i64, s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(fd,s,n); return 0 }
23func d_wn(fd: i64, v: i64) -> i64 { let b: *u8=sys_mmap(28); var m: i64=v; if m<0{sys_write(fd,"-" as *u8,1);m=0-m} let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i: i64=0; while i<k{b[i]=t[k-1-i];i=i+1} sys_write(fd,b,k); return 0 }
24func dini(arr: *i64, n: i64, seed: i64) -> i64 { var i: i64=0; while i<n { arr[i] = (((i*7 + seed*13 + 1) % 11) - 5) * 13107; i=i+1 } return 0 }
25
26// ---- depth-D stack LM forward; W = ptr-array [E, Wlm, then per block b: Wq,Wk,Wv,Wo,Wg,Wu,Wd]; leaves[0]=nE,[1]=nWlm ----
27func stack_fwd(tape: *i64, vals: *i64, st: *i64, W: *i64, D: i64, ids: *i64, tgt: *i64, T: i64, dm: i64, ffn: i64, V: i64, scale: i64, leaves: *i64) -> i64 {
  let E: *i64 = W[0] as *i64; let Wlm: *i64 = W[1] as *i64
  st[0]=0; st[1]=0
  let nE: i64 = nfa_leaf(tape,vals,st,V,dm,E,0)
  let nWlm: i64 = nfa_leaf(tape,vals,st,dm,V,Wlm,0)
  var cur: i64 = nfa_embed(tape,vals,st,nE,ids,T)
  var b: i64 = 0
  while b < D {
      let base: i64 = 2 + b*7
      let Wq: *i64 = W[base+0] as *i64; let Wk: *i64 = W[base+1] as *i64; let Wv: *i64 = W[base+2] as *i64; let Wo: *i64 = W[base+3] as *i64
      let Wg: *i64 = W[base+4] as *i64; let Wu: *i64 = W[base+5] as *i64; let Wd: *i64 = W[base+6] as *i64
      let nWq: i64 = nfa_leaf(tape,vals,st,dm,dm,Wq,0)
      let nWk: i64 = nfa_leaf(tape,vals,st,dm,dm,Wk,0)
      let nWv: i64 = nfa_leaf(tape,vals,st,dm,dm,Wv,0)
      let nWo: i64 = nfa_leaf(tape,vals,st,dm,dm,Wo,0)
      let nWg: i64 = nfa_leaf(tape,vals,st,dm,ffn,Wg,0)
      let nWu: i64 = nfa_leaf(tape,vals,st,dm,ffn,Wu,0)
      let nWd: i64 = nfa_leaf(tape,vals,st,ffn,dm,Wd,0)
      let nXn: i64 = nfa_rmsnorm_rows(tape,vals,st,cur)
      let nQ: i64 = nfa_matmul(tape,vals,st,nXn,nWq)
      let nK: i64 = nfa_matmul(tape,vals,st,nXn,nWk)
      let nV: i64 = nfa_matmul(tape,vals,st,nXn,nWv)
      let nQr: i64 = nfa_rope(tape,vals,st,nQ)
      let nKr: i64 = nfa_rope(tape,vals,st,nK)
      let nS: i64 = nfa_matmul_nt(tape,vals,st,nQr,nKr)
      let nSs: i64 = nfa_cmul(tape,vals,st,nS,scale)
      let nA: i64 = nfa_softmax_rows(tape,vals,st,nSs,1)
      let nO: i64 = nfa_matmul(tape,vals,st,nA,nV)
      let nOp: i64 = nfa_matmul(tape,vals,st,nO,nWo)
      let nH: i64 = nfa_vadd(tape,vals,st,cur,nOp)
      let nHn: i64 = nfa_rmsnorm_rows(tape,vals,st,nH)
      let nG: i64 = nfa_matmul(tape,vals,st,nHn,nWg)
      let nU: i64 = nfa_matmul(tape,vals,st,nHn,nWu)
      let nSg: i64 = nfa_silu(tape,vals,st,nG)
      let nHs: i64 = nfa_hadamard(tape,vals,st,nSg,nU)
      let nDp: i64 = nfa_matmul(tape,vals,st,nHs,nWd)
      cur = nfa_vadd(tape,vals,st,nH,nDp)
      b = b + 1
  }
  let nYn: i64 = nfa_rmsnorm_rows(tape,vals,st,cur)
  let nLg: i64 = nfa_matmul(tape,vals,st,nYn,nWlm)
  let nLoss: i64 = nfa_softce_rows(tape,vals,st,nLg,tgt)
  leaves[0]=nE; leaves[1]=nWlm
  return nLoss
71}
72func stack_lossval(tape: *i64, vals: *i64, st: *i64, W: *i64, D: i64, ids: *i64, tgt: *i64, T: i64, dm: i64, ffn: i64, V: i64, scale: i64) -> i64 {
  let lv: *i64 = sys_mmap(2*8) as *i64
  let nLoss: i64 = stack_fwd(tape,vals,st,W,D,ids,tgt,T,dm,ffn,V,scale,lv)
  return nfa_val(tape,vals,nLoss,0)
76}
77// gradcheck a leaf weight array (widx: 0=E via leaves[0], 1=Wlm via leaves[1]); extract analytic grads first.
78func stack_gc(tape: *i64, vals: *i64, grads: *i64, st: *i64, W: *i64, D: i64, ids: *i64, tgt: *i64, T: i64, dm: i64, ffn: i64, V: i64, scale: i64, leaves: *i64, lidx: i64, arr: *i64, cnt: i64, h: i64, tol_q: i64, floor_q: i64, worst: *i64, anabuf: *i64) -> i64 {
  let nLoss: i64 = stack_fwd(tape,vals,st,W,D,ids,tgt,T,dm,ffn,V,scale,leaves)
  nfa_backward(tape,vals,grads,st[0],nLoss)
  let nnode: i64 = leaves[lidx]
  var c: i64=0
  while c<cnt { anabuf[c]=nfa_grad(tape,grads,nnode,c); c=c+1 }
  var ok: i64=1; worst[0]=0
  var i: i64=0
  while i<cnt {
      let old: i64=arr[i]
      arr[i]=old+h; let lp: i64=stack_lossval(tape,vals,st,W,D,ids,tgt,T,dm,ffn,V,scale)
      arr[i]=old-h; let lm2: i64=stack_lossval(tape,vals,st,W,D,ids,tgt,T,dm,ffn,V,scale)
      arr[i]=old
      let fd: i64=((lp-lm2)*Q16)/(2*h); let num: i64=g_abs(fd-anabuf[i]); var den: i64=g_abs(anabuf[i]); if den<floor_q{den=floor_q}
      if num >= ((tol_q*den)>>16) { ok=0 }
      let rel: i64=(num*1000)/den; if rel>worst[0]{worst[0]=rel}
      i=i+1
  }
  return ok
97}
98// train E + Wlm with AdamW for `steps`; returns final CE; *lf0 = first CE.
99func stack_train(tape: *i64, vals: *i64, grads: *i64, st: *i64, W: *i64, D: i64, ids: *i64, tgt: *i64, T: i64, dm: i64, ffn: i64, V: i64, scale: i64, steps: i64, lf0: *i64) -> i64 {
  let E: *i64 = W[0] as *i64; let Wlm: *i64 = W[1] as *i64
  let gE: *i64 = sys_mmap(64*8) as *i64; let mE: *i64 = sys_mmap(64*8) as *i64; let vE: *i64 = sys_mmap(64*8) as *i64
  let gL: *i64 = sys_mmap(64*8) as *i64; let mL: *i64 = sys_mmap(64*8) as *i64; let vL: *i64 = sys_mmap(64*8) as *i64
  var z: i64=0; while z<V*dm { mE[z]=0; vE[z]=0; z=z+1 } z=0; while z<dm*V { mL[z]=0; vL[z]=0; z=z+1 }
  let leaves: *i64 = sys_mmap(2*8) as *i64
  var ll: i64=0
  var ep: i64=0
  while ep < steps {
      let nLoss: i64 = stack_fwd(tape,vals,st,W,D,ids,tgt,T,dm,ffn,V,scale,leaves)
      nfa_backward(tape,vals,grads,st[0],nLoss)
      if ep==0 { *lf0 = nfa_val(tape,vals,nLoss,0) }
      ll = nfa_val(tape,vals,nLoss,0)
      let nE: i64=leaves[0]; z=0; while z<V*dm { gE[z]=nfa_grad(tape,grads,nE,z); z=z+1 }
      nfa_adamw(E, gE, mE, vE, V*dm, 3277, 58982, 65470, 66, 0, ep+1)
      let nWl: i64=leaves[1]; z=0; while z<dm*V { gL[z]=nfa_grad(tape,grads,nWl,z); z=z+1 }
      nfa_adamw(Wlm, gL, mL, vL, dm*V, 3277, 58982, 65470, 66, 0, ep+1)
      ep=ep+1
  }
  return ll
119}
120// build a fresh weight pointer-array of D blocks, all arrays freshly mmap'd + init'd (deterministic).
121func build_W(D: i64, dm: i64, ffn: i64, V: i64) -> *i64 {
  let W: *i64 = sys_mmap((2 + D*7)*8) as *i64
  let E: *i64 = sys_mmap(V*dm*8) as *i64; dini(E,V*dm,1); W[0]=E as i64
  let Wlm: *i64 = sys_mmap(dm*V*8) as *i64; dini(Wlm,dm*V,2); W[1]=Wlm as i64
  var b: i64=0
  while b<D {
      let base: i64 = 2 + b*7
      let Wq: *i64=sys_mmap(dm*dm*8) as *i64; dini(Wq,dm*dm,10+b*7+0); W[base+0]=Wq as i64
      let Wk: *i64=sys_mmap(dm*dm*8) as *i64; dini(Wk,dm*dm,10+b*7+1); W[base+1]=Wk as i64
      let Wv: *i64=sys_mmap(dm*dm*8) as *i64; dini(Wv,dm*dm,10+b*7+2); W[base+2]=Wv as i64
      let Wo: *i64=sys_mmap(dm*dm*8) as *i64; dini(Wo,dm*dm,10+b*7+3); W[base+3]=Wo as i64
      let Wg: *i64=sys_mmap(dm*ffn*8) as *i64; dini(Wg,dm*ffn,10+b*7+4); W[base+4]=Wg as i64
      let Wu: *i64=sys_mmap(dm*ffn*8) as *i64; dini(Wu,dm*ffn,10+b*7+5); W[base+5]=Wu as i64
      let Wd: *i64=sys_mmap(ffn*dm*8) as *i64; dini(Wd,ffn*dm,10+b*7+6); W[base+6]=Wd as i64
      b=b+1
  }
  return W
138}
139
140func main() -> i64 {
  g_puts("nx_nofloat_deepstack gate (a DEPTH-2 transformer LM trains end-to-end, PURE INTEGER Q16)\n" as *u8)
  var pass: i64=0; var total: i64=0
  let tape:  *i64 = sys_mmap(1024*7*8) as *i64
  let vals:  *i64 = sys_mmap(32768*8) as *i64
  let grads: *i64 = sys_mmap(32768*8) as *i64
  let st:    *i64 = sys_mmap(2*8) as *i64
  let h: i64=512; let floor_q: i64=4096
  let worst: *i64 = sys_mmap(8) as *i64
  let anabuf: *i64 = sys_mmap(64*8) as *i64
  let T: i64=4; let dm: i64=4; let ffn: i64=8; let V: i64=4; let scale: i64=32768; let D: i64=2
  let ids: *i64 = sys_mmap(T*8) as *i64; ids[0]=0; ids[1]=1; ids[2]=2; ids[3]=3
  let tgt: *i64 = sys_mmap(T*8) as *i64; tgt[0]=1; tgt[1]=2; tgt[2]=3; tgt[3]=0
  let leaves: *i64 = sys_mmap(2*8) as *i64
154
  // ---- A1: depth-2 gradcheck wrt Wlm ----
  let Wa: *i64 = build_W(D,dm,ffn,V)
  let a1_ok: i64 = stack_gc(tape,vals,grads,st,Wa,D,ids,tgt,T,dm,ffn,V,scale,leaves,1,Wa[1] as *i64,dm*V,h,8192,floor_q,worst,anabuf)
  g_puts("  [measure] depth-2 dL/dWlm worst rel grad err = " as *u8); g_pn(worst[0]); g_puts(" /1000 (tol=125)\n" as *u8)
  pass=pass+g_check("A1: depth-2 gradcheck wrt Wlm (LM head through CE) == finite differences" as *u8, a1_ok); total=total+1
160
  // ---- A2: depth-2 gradcheck wrt E (through BOTH blocks) ----
  let a2_ok: i64 = stack_gc(tape,vals,grads,st,Wa,D,ids,tgt,T,dm,ffn,V,scale,leaves,0,Wa[0] as *i64,V*dm,h,16384,floor_q,worst,anabuf)
  g_puts("  [measure] depth-2 dL/dE (through BOTH blocks) worst rel grad err = " as *u8); g_pn(worst[0]); g_puts(" /1000 (tol=250)\n" as *u8)
  pass=pass+g_check("A2: depth-2 gradcheck wrt E through BOTH blocks == finite differences (deep backprop)" as *u8, a2_ok); total=total+1
165
  // ---- D: neg-control teeth (E, largest grad component, scale-free) ----
  let nLd: i64 = stack_fwd(tape,vals,st,Wa,D,ids,tgt,T,dm,ffn,V,scale,leaves)
  nfa_backward(tape,vals,grads,st[0],nLd)
  let Ea: *i64 = Wa[0] as *i64
  var imax: i64=0; var vmax: i64=0; var ii: i64=0
  while ii<V*dm { let gg: i64=g_abs(nfa_grad(tape,grads,leaves[0],ii)); if gg>vmax { vmax=gg; imax=ii } ii=ii+1 }
  let dana: i64 = nfa_grad(tape,grads,leaves[0],imax)
  let o0: i64=Ea[imax]; Ea[imax]=o0+h; let lpd: i64=stack_lossval(tape,vals,st,Wa,D,ids,tgt,T,dm,ffn,V,scale); Ea[imax]=o0-h; let lmd: i64=stack_lossval(tape,vals,st,Wa,D,ids,tgt,T,dm,ffn,V,scale); Ea[imax]=o0
  let dfd: i64=((lpd-lmd)*Q16)/(2*h); let dbad: i64=0-dana
  let dgood: i64=g_abs(dfd-dana); let dneg: i64=g_abs(dfd-dbad)
  var caught: i64=0; if vmax>64 { if dneg > dgood*4 { caught=1 } }
  pass=pass+g_check("D: neg-control -- FD is >4x closer to the true E-grad than to the negated one (teeth)" as *u8, caught); total=total+1
178
  // ---- B: depth-2 LM trains (CE drops) with AdamW ----
  let Wb: *i64 = build_W(D,dm,ffn,V)
  let lf2: *i64 = sys_mmap(8) as *i64
  let ce2: i64 = stack_train(tape,vals,grads,st,Wb,D,ids,tgt,T,dm,ffn,V,scale,3000,lf2)
  // informational: depth-1 final CE
  let W1: *i64 = build_W(1,dm,ffn,V)
  let lf1: *i64 = sys_mmap(8) as *i64
  let ce1: i64 = stack_train(tape,vals,grads,st,W1,1,ids,tgt,T,dm,ffn,V,scale,3000,lf1)
  g_puts("  [measure] depth-2 CE: start=" as *u8); g_pn(*lf2); g_puts(" end=" as *u8); g_pn(ce2); g_puts("   (info: depth-1 CE start=" as *u8); g_pn(*lf1); g_puts(" end=" as *u8); g_pn(ce1); g_puts(")\n" as *u8)
  var fits: i64=1
  if ce2*2 > (*lf2) { fits=0 }       // depth-2 CE drops >= 50%
  if (*lf2) <= 0 { fits=0 }
  pass=pass+g_check("B: the DEPTH-2 LM TRAINS end-to-end -- CE drops >=50% (AdamW thru 2 blocks; FITS/memorizes a FIXED sequence, not generalization)" as *u8, fits); total=total+1
192
  // ---- C: bit-exact (depth-2 trained twice) ----
  let Wc: *i64 = build_W(D,dm,ffn,V)
  let lfc: *i64 = sys_mmap(8) as *i64
  let cec: i64 = stack_train(tape,vals,grads,st,Wc,D,ids,tgt,T,dm,ffn,V,scale,3000,lfc)
  var bitexact: i64=1
  let Eb: *i64 = Wb[0] as *i64; let Ec: *i64 = Wc[0] as *i64
  var bz: i64=0; while bz<V*dm { if Eb[bz]!=Ec[bz] { bitexact=0 } bz=bz+1 }
  let Wlb: *i64 = Wb[1] as *i64; let Wlc: *i64 = Wc[1] as *i64
  bz=0; while bz<dm*V { if Wlb[bz]!=Wlc[bz] { bitexact=0 } bz=bz+1 }
  pass=pass+g_check("C: bit-exact -- training the depth-2 LM twice gives IDENTICAL integer weights (determinism)" as *u8, bitexact); total=total+1
203
  var okall: i64=0; if pass==total { okall=1 }
  let logf: i64 = sys_openat_append(DLOG, 420)
  if logf >= 0 {
      d_ws(logf,"NOFLOATDEEPSTACK D=2 A1_Wlm=" as *u8); d_wn(logf,a1_ok); d_ws(logf," A2_E_bothblocks=" as *u8); d_wn(logf,a2_ok)
      d_ws(logf," D=" as *u8); d_wn(logf,caught); d_ws(logf," B_fits=" as *u8); d_wn(logf,fits); d_ws(logf," CE2_start=" as *u8); d_wn(logf,*lf2); d_ws(logf," CE2_end=" as *u8); d_wn(logf,ce2); d_ws(logf," C_bitexact=" as *u8); d_wn(logf,bitexact)
      if okall==1 { d_ws(logf," verdict=GREEN\n" as *u8) } else { d_ws(logf," verdict=RED\n" as *u8) }
      sys_close(logf)
  }
  g_puts("---- nofloat_deepstack gate: passed " as *u8); g_pn(pass); g_puts(" / " as *u8); g_pn(total); g_puts(" ----\n" as *u8)
  if okall==1 { g_puts("verdict=GREEN (a depth-2 transformer LM trains end-to-end in pure integer Q16 -- depth scales by composition)\n" as *u8); sys_exit(0); return 0 }
  g_puts("verdict=RED\n" as *u8); sys_exit(1); return 1
215}
nx_nofloat_deepstack_gate.nx source

`nx_nofloat_deepstack_gate.nx` source
