nx_nofloat_scale_ffn_gate.nx source

1// nx_nofloat_scale_ffn_gate.nx -- R4 FINAL test: the COMPLETE transformer block (attention + FFN) on the richer 2// grammar. All prior scale attempts were ATTENTION-ONLY (no FFN) -- but a real transformer block has an FFN, the 3// per-token nonlinear map that can turn a token's category into a SHARP output distribution. Honesty requires 4// testing the complete architecture before concluding the floor is unreachable. Strongest single shot = FFN + 5// mini-batch averaging (lowest-noise training). Richer 4-cat grammar (vocab 16), dm=32, ffn=64. 6// floor = avg(ln2,ln4,ln5,ln5)=1324 milli-nats (ppl 3.76); uniform = ln(16)=2773; attention-only plateaued ~2135. 7// T1 held-out CE << uniform. T2 held-out CE ~= floor (near-OPTIMAL -> the FFN was the missing piece -> R4 lands). 8// If T2 fails too, the tractable levers (optimizer x6 + complete architecture) are EXHAUSTED -> R4-FULL genuinely 9// needs compute-scale (operator-gated), the honest end of the road. expect_exit: 0 Sovereign: nofloat_autograd. 10// 11// D001 MIGRATION 2026-08-25: this gate hand-rolled g_check plus a pass/total counter pair and printed its verdict 12// as prose, so its EXIT CODE did not carry its verdict. It now inherits nx_gate_verdict: gv_ctr makes declared and 13// executed tooth counts ONE number, and gv_verdict's return value IS the exit code. BOTH original assertions are 14// preserved verbatim; the only other change is binding their bare comparison literals to named bounds, which 15// leaves the arithmetic byte-for-byte identical. 16import "nx_nofloat_autograd.nx" 17import "nx_syscalls.nx" 18import "nx_gate_emit_lib.nx" 19import "nx_gate_verdict.nx" 20const Q16: i64 = 65536 21const UNIFORM_MNAT: i64 = 2773 22const FLOOR_MNAT: i64 = 1324 23// T1's bound: to count as "learned the language" the held-out CE must sit at or below this fraction of the 24// uniform-distribution cross-entropy. Held as an exact rational so the comparison stays pure-integer. 25const LEARNED_FRAC_NUM: i64 = 7 26const LEARNED_FRAC_DEN: i64 = 10 27// T2's bound: how far above the grammar's own entropy floor still counts as near-OPTIMAL, in milli-nats. 28const FLOOR_TOL_MNAT: i64 = 200 29 30 31func dini(a: *i64, n: i64, sd: i64) -> i64 { var i: i64=0; while i<n { a[i]=(((i*7+sd*13+1)%11)-5)*13107; i=i+1 } return 0 } 32func lcg(st: *i64) -> i64 { st[0]=(st[0]*1103515245 + 12345) & 2147483647; return (st[0] >> 15) } 33func make_stream4(S: *i64, tgt: *i64, P: i64, st: *i64) -> i64 { 34 var i: i64=0 35 while i<P { let c: i64=i%4; if c==0 { S[i]=lcg(st)%2 } if c==1 { S[i]=2+lcg(st)%4 } if c==2 { S[i]=6+lcg(st)%5 } if c==3 { S[i]=11+lcg(st)%5 } i=i+1 } 36 i=0; while i<P-1 { tgt[i]=S[i+1]; i=i+1 } tgt[P-1]=S[0] 37 return 0 38} 39// COMPLETE block: attention residual, THEN FFN residual (W2 . silu(W1 . norm(h))), then head. weights: 40// E(0) Wq(1) Wk(2) Wv(3) Wo(4) W1(5)[dm,ffn] W2(6)[ffn,dm] Wlm(7)[dm,V]. leaves[0..7]=weights, leaves[8]=logits. 41func clm_ffn(tape: *i64, vals: *i64, st: *i64, W: *i64, ids: *i64, tgt: *i64, T: i64, dm: i64, ffn: i64, V: i64, scale: i64, lv: *i64) -> i64 { 42 st[0]=0; st[1]=0 43 let nE: i64=nfa_leaf(tape,vals,st,V,dm,W[0] as *i64,0) 44 let nWq: i64=nfa_leaf(tape,vals,st,dm,dm,W[1] as *i64,0) 45 let nWk: i64=nfa_leaf(tape,vals,st,dm,dm,W[2] as *i64,0) 46 let nWv: i64=nfa_leaf(tape,vals,st,dm,dm,W[3] as *i64,0) 47 let nWo: i64=nfa_leaf(tape,vals,st,dm,dm,W[4] as *i64,0) 48 let nW1: i64=nfa_leaf(tape,vals,st,dm,ffn,W[5] as *i64,0) 49 let nW2: i64=nfa_leaf(tape,vals,st,ffn,dm,W[6] as *i64,0) 50 let nWlm: i64=nfa_leaf(tape,vals,st,dm,V,W[7] as *i64,0) 51 let nX: i64=nfa_embed(tape,vals,st,nE,ids,T) 52 let nXn: i64=nfa_rmsnorm_rows(tape,vals,st,nX) 53 let nQ: i64=nfa_matmul(tape,vals,st,nXn,nWq) 54 let nK: i64=nfa_matmul(tape,vals,st,nXn,nWk) 55 let nV: i64=nfa_matmul(tape,vals,st,nXn,nWv) 56 let nQr: i64=nfa_rope(tape,vals,st,nQ) 57 let nKr: i64=nfa_rope(tape,vals,st,nK) 58 let nS: i64=nfa_matmul_nt(tape,vals,st,nQr,nKr) 59 let nSs: i64=nfa_cmul(tape,vals,st,nS,scale) 60 let nA: i64=nfa_softmax_rows(tape,vals,st,nSs,1) 61 let nO: i64=nfa_matmul(tape,vals,st,nA,nV) 62 let nOp: i64=nfa_matmul(tape,vals,st,nO,nWo) 63 let nH: i64=nfa_vadd(tape,vals,st,nX,nOp) 64 let nHn: i64=nfa_rmsnorm_rows(tape,vals,st,nH) 65 let nF1: i64=nfa_matmul(tape,vals,st,nHn,nW1) 66 let nFs: i64=nfa_silu(tape,vals,st,nF1) 67 let nF2: i64=nfa_matmul(tape,vals,st,nFs,nW2) 68 let nH2: i64=nfa_vadd(tape,vals,st,nH,nF2) 69 let nH2n: i64=nfa_rmsnorm_rows(tape,vals,st,nH2) 70 let nLg: i64=nfa_matmul(tape,vals,st,nH2n,nWlm) 71 let nLoss: i64=nfa_softce_rows(tape,vals,st,nLg,tgt) 72 lv[0]=nE; lv[1]=nWq; lv[2]=nWk; lv[3]=nWv; lv[4]=nWo; lv[5]=nW1; lv[6]=nW2; lv[7]=nWlm; lv[8]=nLg 73 return nLoss 74} 75func do_train_batch(tape: *i64, vals: *i64, grads: *i64, st: *i64, W: *i64, WN: *i64, gacc: *i64, S: *i64, tgt: *i64, P: i64, dm: i64, ffn: i64, V: i64, scale: i64, lv: *i64, gb: *i64, outer: i64, B: i64, lr: i64, sdat: *i64) -> i64 { 76 var s: i64=0 77 while s<outer { 78 var i: i64=0 79 while i<8 { let ga: *i64=gacc[i] as *i64; let cn: i64=WN[i]; var c: i64=0; while c<cn { ga[c]=0; c=c+1 } i=i+1 } 80 var b: i64=0 81 while b<B { 82 make_stream4(S,tgt,P,sdat) 83 let nl: i64=clm_ffn(tape,vals,st,W,S,tgt,P-1,dm,ffn,V,scale,lv) 84 nfa_backward(tape,vals,grads,st[0],nl) 85 i=0 86 while i<8 { let ga: *i64=gacc[i] as *i64; let cn: i64=WN[i]; let nd: i64=lv[i]; var c: i64=0; while c<cn { var g: i64=nfa_grad(tape,grads,nd,c); if g>262144{g=262144} if g<0-262144{g=0-262144} ga[c]=ga[c]+g; c=c+1 } i=i+1 } 87 b=b+1 88 } 89 i=0 90 while i<8 { let ar: *i64=W[i] as *i64; let ga: *i64=gacc[i] as *i64; let cn: i64=WN[i]; var c: i64=0; while c<cn { gb[c]=ga[c]/B; c=c+1 } nfa_sgd(ar,gb,cn,lr); i=i+1 } 91 s=s+1 92 } 93 return 0 94} 95func eval_ce(tape: *i64, vals: *i64, st: *i64, W: *i64, S: *i64, tgt: *i64, P: i64, dm: i64, ffn: i64, V: i64, scale: i64, lv: *i64, N: i64, sdat: *i64) -> i64 { 96 var acc: i64=0; var e: i64=0 97 while e<N { make_stream4(S,tgt,P,sdat); let nl: i64=clm_ffn(tape,vals,st,W,S,tgt,P-1,dm,ffn,V,scale,lv); acc=acc+nfa_val(tape,vals,nl,0); e=e+1 } 98 let mq: i64=acc/N 99 return (mq*1000)/Q16 100} 101 102func main() -> i64 { 103 let ctr: *i64 = gv_ctr() 104 gv_head("nx_nofloat_scale_ffn gate (R4 FINAL: COMPLETE block attention+FFN on the richer 16-word grammar)" as *u8) 105 let V: i64=16; let P: i64=16; let dm: i64=32; let ffn: i64=64; let scale: i64=11585 106 let tape: *i64=sys_mmap(2048*7*8) as *i64 107 let vals: *i64=sys_mmap(262144*8) as *i64 108 let grads: *i64=sys_mmap(262144*8) as *i64 109 let st: *i64=sys_mmap(2*8) as *i64 110 let nW: i64=8 111 let W: *i64=sys_mmap(nW*8) as *i64; let WN: *i64=sys_mmap(nW*8) as *i64 112 WN[0]=V*dm; WN[1]=dm*dm; WN[2]=dm*dm; WN[3]=dm*dm; WN[4]=dm*dm; WN[5]=dm*ffn; WN[6]=ffn*dm; WN[7]=dm*V 113 var wi: i64=0; while wi<nW { let a: *i64=sys_mmap(WN[wi]*8) as *i64; dini(a,WN[wi],wi+1); W[wi]=a as i64; wi=wi+1 } 114 let gacc: *i64=sys_mmap(nW*8) as *i64; wi=0; while wi<nW { gacc[wi]=(sys_mmap(WN[wi]*8) as *i64) as i64; wi=wi+1 } 115 let lv: *i64=sys_mmap(10*8) as *i64; let gbuf: *i64=sys_mmap(4096*8) as *i64 116 let S: *i64=sys_mmap(P*8) as *i64; let tgt: *i64=sys_mmap(P*8) as *i64; let sdat: *i64=sys_mmap(8) as *i64 117 118 g_puts(" complete block (attn+FFN, ffn=64), batch B=16, dm=32, SGD lr=0.2, 1500 outer steps\n" as *u8) 119 sdat[0]=12345 120 do_train_batch(tape,vals,grads,st,W,WN,gacc,S,tgt,P,dm,ffn,V,scale,lv,gbuf,1500,16,13107,sdat) 121 sdat[0]=24682468 122 let ce: i64=eval_ce(tape,vals,st,W,S,tgt,P,dm,ffn,V,scale,lv,150,sdat) 123 124 g_puts(" [measure] held-out CE="); g_pn(ce); g_puts(" milli-nats uniform=2773 floor=1324 (attention-only plateaued ~2135)\n") 125 126 var t1: i64=0; if ce*LEARNED_FRAC_DEN <= UNIFORM_MNAT*LEARNED_FRAC_NUM { t1=1 } 127 gv_check("T1: held-out CE << uniform (the complete block learned the richer language)" as *u8, t1, ctr) 128 var t2: i64=0; if ce <= FLOOR_MNAT+FLOOR_TOL_MNAT { t2=1 } 129 gv_check("T2: held-out CE ~= floor (near-OPTIMAL -> the FFN was the missing piece -> R4 lands)" as *u8, t2, ctr) 130 131 let rc: i64 = gv_verdict("NOFLOAT-SCALE-FFN-GATE" as *u8, ctr, "a complete attention+FFN transformer block, trained and evaluated in pure integer arithmetic with no float anywhere, reached the 4-category grammar's own entropy floor on held-out text" as *u8) 132 sys_exit(rc) 133 return rc 134}