nx_nofloat_scale_ffn_gate.nx source

1// nx_nofloat_scale_ffn_gate.nx -- R4 FINAL test: the COMPLETE transformer block (attention + FFN) on the richer 2// grammar. All prior scale attempts were ATTENTION-ONLY (no FFN) -- but a real transformer block has an FFN, the 3// per-token nonlinear map that can turn a token's category into a SHARP output distribution. Honesty requires 4// testing the complete architecture before concluding the floor is unreachable. Strongest single shot = FFN + 5// mini-batch averaging (lowest-noise training). Richer 4-cat grammar (vocab 16), dm=32, ffn=64. 6// floor = avg(ln2,ln4,ln5,ln5)=1324 milli-nats (ppl 3.76); uniform = ln(16)=2773; attention-only plateaued ~2135. 7// T1 held-out CE << uniform. T2 held-out CE ~= floor (near-OPTIMAL -> the FFN was the missing piece -> R4 lands). 8// If T2 fails too, the tractable levers (optimizer x6 + complete architecture) are EXHAUSTED -> R4-FULL genuinely 9// needs compute-scale (operator-gated), the honest end of the road. expect_exit: 0 Sovereign: nofloat_autograd. 10import "nx_nofloat_autograd.nx" 11import "nx_syscalls.nx" 12import "nx_gate_emit_lib.nx" 13const Q16: i64 = 65536 14const UNIFORM_MNAT: i64 = 2773 15const FLOOR_MNAT: i64 = 1324 16 17 18func dini(a: *i64, n: i64, sd: i64) -> i64 { var i: i64=0; while i<n { a[i]=(((i*7+sd*13+1)%11)-5)*13107; i=i+1 } return 0 } 19func lcg(st: *i64) -> i64 { st[0]=(st[0]*1103515245 + 12345) & 2147483647; return (st[0] >> 15) } 20func make_stream4(S: *i64, tgt: *i64, P: i64, st: *i64) -> i64 { 21 var i: i64=0 22 while i<P { let c: i64=i%4; if c==0 { S[i]=lcg(st)%2 } if c==1 { S[i]=2+lcg(st)%4 } if c==2 { S[i]=6+lcg(st)%5 } if c==3 { S[i]=11+lcg(st)%5 } i=i+1 } 23 i=0; while i<P-1 { tgt[i]=S[i+1]; i=i+1 } tgt[P-1]=S[0] 24 return 0 25} 26// COMPLETE block: attention residual, THEN FFN residual (W2 . silu(W1 . norm(h))), then head. weights: 27// E(0) Wq(1) Wk(2) Wv(3) Wo(4) W1(5)[dm,ffn] W2(6)[ffn,dm] Wlm(7)[dm,V]. leaves[0..7]=weights, leaves[8]=logits. 28func clm_ffn(tape: *i64, vals: *i64, st: *i64, W: *i64, ids: *i64, tgt: *i64, T: i64, dm: i64, ffn: i64, V: i64, scale: i64, lv: *i64) -> i64 { 29 st[0]=0; st[1]=0 30 let nE: i64=nfa_leaf(tape,vals,st,V,dm,W[0] as *i64,0) 31 let nWq: i64=nfa_leaf(tape,vals,st,dm,dm,W[1] as *i64,0) 32 let nWk: i64=nfa_leaf(tape,vals,st,dm,dm,W[2] as *i64,0) 33 let nWv: i64=nfa_leaf(tape,vals,st,dm,dm,W[3] as *i64,0) 34 let nWo: i64=nfa_leaf(tape,vals,st,dm,dm,W[4] as *i64,0) 35 let nW1: i64=nfa_leaf(tape,vals,st,dm,ffn,W[5] as *i64,0) 36 let nW2: i64=nfa_leaf(tape,vals,st,ffn,dm,W[6] as *i64,0) 37 let nWlm: i64=nfa_leaf(tape,vals,st,dm,V,W[7] as *i64,0) 38 let nX: i64=nfa_embed(tape,vals,st,nE,ids,T) 39 let nXn: i64=nfa_rmsnorm_rows(tape,vals,st,nX) 40 let nQ: i64=nfa_matmul(tape,vals,st,nXn,nWq) 41 let nK: i64=nfa_matmul(tape,vals,st,nXn,nWk) 42 let nV: i64=nfa_matmul(tape,vals,st,nXn,nWv) 43 let nQr: i64=nfa_rope(tape,vals,st,nQ) 44 let nKr: i64=nfa_rope(tape,vals,st,nK) 45 let nS: i64=nfa_matmul_nt(tape,vals,st,nQr,nKr) 46 let nSs: i64=nfa_cmul(tape,vals,st,nS,scale) 47 let nA: i64=nfa_softmax_rows(tape,vals,st,nSs,1) 48 let nO: i64=nfa_matmul(tape,vals,st,nA,nV) 49 let nOp: i64=nfa_matmul(tape,vals,st,nO,nWo) 50 let nH: i64=nfa_vadd(tape,vals,st,nX,nOp) 51 let nHn: i64=nfa_rmsnorm_rows(tape,vals,st,nH) 52 let nF1: i64=nfa_matmul(tape,vals,st,nHn,nW1) 53 let nFs: i64=nfa_silu(tape,vals,st,nF1) 54 let nF2: i64=nfa_matmul(tape,vals,st,nFs,nW2) 55 let nH2: i64=nfa_vadd(tape,vals,st,nH,nF2) 56 let nH2n: i64=nfa_rmsnorm_rows(tape,vals,st,nH2) 57 let nLg: i64=nfa_matmul(tape,vals,st,nH2n,nWlm) 58 let nLoss: i64=nfa_softce_rows(tape,vals,st,nLg,tgt) 59 lv[0]=nE; lv[1]=nWq; lv[2]=nWk; lv[3]=nWv; lv[4]=nWo; lv[5]=nW1; lv[6]=nW2; lv[7]=nWlm; lv[8]=nLg 60 return nLoss 61} 62func do_train_batch(tape: *i64, vals: *i64, grads: *i64, st: *i64, W: *i64, WN: *i64, gacc: *i64, S: *i64, tgt: *i64, P: i64, dm: i64, ffn: i64, V: i64, scale: i64, lv: *i64, gb: *i64, outer: i64, B: i64, lr: i64, sdat: *i64) -> i64 { 63 var s: i64=0 64 while s<outer { 65 var i: i64=0 66 while i<8 { let ga: *i64=gacc[i] as *i64; let cn: i64=WN[i]; var c: i64=0; while c<cn { ga[c]=0; c=c+1 } i=i+1 } 67 var b: i64=0 68 while b<B { 69 make_stream4(S,tgt,P,sdat) 70 let nl: i64=clm_ffn(tape,vals,st,W,S,tgt,P-1,dm,ffn,V,scale,lv) 71 nfa_backward(tape,vals,grads,st[0],nl) 72 i=0 73 while i<8 { let ga: *i64=gacc[i] as *i64; let cn: i64=WN[i]; let nd: i64=lv[i]; var c: i64=0; while c<cn { var g: i64=nfa_grad(tape,grads,nd,c); if g>262144{g=262144} if g<0-262144{g=0-262144} ga[c]=ga[c]+g; c=c+1 } i=i+1 } 74 b=b+1 75 } 76 i=0 77 while i<8 { let ar: *i64=W[i] as *i64; let ga: *i64=gacc[i] as *i64; let cn: i64=WN[i]; var c: i64=0; while c<cn { gb[c]=ga[c]/B; c=c+1 } nfa_sgd(ar,gb,cn,lr); i=i+1 } 78 s=s+1 79 } 80 return 0 81} 82func eval_ce(tape: *i64, vals: *i64, st: *i64, W: *i64, S: *i64, tgt: *i64, P: i64, dm: i64, ffn: i64, V: i64, scale: i64, lv: *i64, N: i64, sdat: *i64) -> i64 { 83 var acc: i64=0; var e: i64=0 84 while e<N { make_stream4(S,tgt,P,sdat); let nl: i64=clm_ffn(tape,vals,st,W,S,tgt,P-1,dm,ffn,V,scale,lv); acc=acc+nfa_val(tape,vals,nl,0); e=e+1 } 85 let mq: i64=acc/N 86 return (mq*1000)/Q16 87} 88 89func main() -> i64 { 90 g_puts("nx_nofloat_scale_ffn gate (R4 FINAL: COMPLETE block attention+FFN on the richer 16-word grammar)\n" as *u8) 91 let V: i64=16; let P: i64=16; let dm: i64=32; let ffn: i64=64; let scale: i64=11585 92 let tape: *i64=sys_mmap(2048*7*8) as *i64 93 let vals: *i64=sys_mmap(262144*8) as *i64 94 let grads: *i64=sys_mmap(262144*8) as *i64 95 let st: *i64=sys_mmap(2*8) as *i64 96 let nW: i64=8 97 let W: *i64=sys_mmap(nW*8) as *i64; let WN: *i64=sys_mmap(nW*8) as *i64 98 WN[0]=V*dm; WN[1]=dm*dm; WN[2]=dm*dm; WN[3]=dm*dm; WN[4]=dm*dm; WN[5]=dm*ffn; WN[6]=ffn*dm; WN[7]=dm*V 99 var wi: i64=0; while wi<nW { let a: *i64=sys_mmap(WN[wi]*8) as *i64; dini(a,WN[wi],wi+1); W[wi]=a as i64; wi=wi+1 } 100 let gacc: *i64=sys_mmap(nW*8) as *i64; wi=0; while wi<nW { gacc[wi]=(sys_mmap(WN[wi]*8) as *i64) as i64; wi=wi+1 } 101 let lv: *i64=sys_mmap(10*8) as *i64; let gbuf: *i64=sys_mmap(4096*8) as *i64 102 let S: *i64=sys_mmap(P*8) as *i64; let tgt: *i64=sys_mmap(P*8) as *i64; let sdat: *i64=sys_mmap(8) as *i64 103 104 g_puts(" complete block (attn+FFN, ffn=64), batch B=16, dm=32, SGD lr=0.2, 1500 outer steps\n" as *u8) 105 sdat[0]=12345 106 do_train_batch(tape,vals,grads,st,W,WN,gacc,S,tgt,P,dm,ffn,V,scale,lv,gbuf,1500,16,13107,sdat) 107 sdat[0]=24682468 108 let ce: i64=eval_ce(tape,vals,st,W,S,tgt,P,dm,ffn,V,scale,lv,150,sdat) 109 110 g_puts(" [measure] held-out CE="); g_pn(ce); g_puts(" milli-nats uniform=2773 floor=1324 (attention-only plateaued ~2135)\n") 111 112 var pass: i64=0; var total: i64=0 113 var t1: i64=0; if ce*10 <= UNIFORM_MNAT*7 { t1=1 } 114 pass=pass+g_check("T1: held-out CE << uniform (the complete block learned the richer language)" as *u8, t1); total=total+1 115 var t2: i64=0; if ce <= FLOOR_MNAT+200 { t2=1 } 116 pass=pass+g_check("T2: held-out CE ~= floor (near-OPTIMAL -> the FFN was the missing piece -> R4 lands)" as *u8, t2); total=total+1 117 118 var okall: i64=0; if pass==total { okall=1 } 119 g_puts("---- scale_ffn gate: passed "); g_pn(pass); g_puts(" / "); g_pn(total); g_puts(" ----\n") 120 if okall==1 { g_puts("verdict=GREEN (the complete attn+FFN block reached the richer floor -- R4 lands at scale, pure no-float)\n" as *u8); sys_exit(0); return 0 } 121 g_puts("verdict=RED\n" as *u8); sys_exit(1); return 1 122}