nx_nofloat_scale_batch_gate.nx source

1// nx_nofloat_scale_batch_gate.nx -- R4 retry via the ONE untested fundamental lever: MINI-BATCH GRADIENT 2// AVERAGING. All prior richer-grammar training was BATCH-1 (one fresh-random stream/step = very noisy gradients 3// -> SGD oscillates, can't sharpen to the floor). Standard fix: average gradients over a mini-batch before 4// stepping. Same richer 4-cat grammar (DET2->ADJ4->NOUN5->VERB5, vocab 16) + dm=32 that plateaued at CE ~2135 5// (ppl 8.5); here we accumulate grads over B=16 fresh streams per step (low-noise), then ONE SGD step. 6// floor = avg(ln2,ln4,ln5,ln5)=1324 milli-nats (ppl 3.76); uniform = ln(16)=2773. 7// T1 held-out CE << uniform (learned). T2 held-out CE ~= floor (near-OPTIMAL -> batch-averaging cracked it). 8// If T2 fails too, batch size is NOT the lever and the tractable training levers are genuinely exhausted (honest). 9// expect_exit: 0 Sovereign: nx_nofloat_autograd + nx_syscalls. 10import "nx_nofloat_autograd.nx" 11import "nx_syscalls.nx" 12import "nx_gate_emit_lib.nx" 13const Q16: i64 = 65536 14const UNIFORM_MNAT: i64 = 2773 15const FLOOR_MNAT: i64 = 1324 16 17 18func dini(a: *i64, n: i64, sd: i64) -> i64 { var i: i64=0; while i<n { a[i]=(((i*7+sd*13+1)%11)-5)*13107; i=i+1 } return 0 } 19func lcg(st: *i64) -> i64 { st[0]=(st[0]*1103515245 + 12345) & 2147483647; return (st[0] >> 15) } 20func make_stream4(S: *i64, tgt: *i64, P: i64, st: *i64) -> i64 { 21 var i: i64=0 22 while i<P { let c: i64=i%4; if c==0 { S[i]=lcg(st)%2 } if c==1 { S[i]=2+lcg(st)%4 } if c==2 { S[i]=6+lcg(st)%5 } if c==3 { S[i]=11+lcg(st)%5 } i=i+1 } 23 i=0; while i<P-1 { tgt[i]=S[i+1]; i=i+1 } tgt[P-1]=S[0] 24 return 0 25} 26func clm_fwd(tape: *i64, vals: *i64, st: *i64, W: *i64, ids: *i64, tgt: *i64, T: i64, dm: i64, V: i64, scale: i64, leaves: *i64) -> i64 { 27 let E: *i64=W[0] as *i64; let Wq: *i64=W[1] as *i64; let Wk: *i64=W[2] as *i64; let Wv: *i64=W[3] as *i64; let Wo: *i64=W[4] as *i64; let Wlm: *i64=W[5] as *i64 28 st[0]=0; st[1]=0 29 let nE: i64=nfa_leaf(tape,vals,st,V,dm,E,0) 30 let nWq: i64=nfa_leaf(tape,vals,st,dm,dm,Wq,0) 31 let nWk: i64=nfa_leaf(tape,vals,st,dm,dm,Wk,0) 32 let nWv: i64=nfa_leaf(tape,vals,st,dm,dm,Wv,0) 33 let nWo: i64=nfa_leaf(tape,vals,st,dm,dm,Wo,0) 34 let nWlm: i64=nfa_leaf(tape,vals,st,dm,V,Wlm,0) 35 let nX: i64=nfa_embed(tape,vals,st,nE,ids,T) 36 let nXn: i64=nfa_rmsnorm_rows(tape,vals,st,nX) 37 let nQ: i64=nfa_matmul(tape,vals,st,nXn,nWq) 38 let nK: i64=nfa_matmul(tape,vals,st,nXn,nWk) 39 let nV: i64=nfa_matmul(tape,vals,st,nXn,nWv) 40 let nQr: i64=nfa_rope(tape,vals,st,nQ) 41 let nKr: i64=nfa_rope(tape,vals,st,nK) 42 let nS: i64=nfa_matmul_nt(tape,vals,st,nQr,nKr) 43 let nSs: i64=nfa_cmul(tape,vals,st,nS,scale) 44 let nA: i64=nfa_softmax_rows(tape,vals,st,nSs,1) 45 let nO: i64=nfa_matmul(tape,vals,st,nA,nV) 46 let nOp: i64=nfa_matmul(tape,vals,st,nO,nWo) 47 let nH: i64=nfa_vadd(tape,vals,st,nX,nOp) 48 let nHn: i64=nfa_rmsnorm_rows(tape,vals,st,nH) 49 let nLg: i64=nfa_matmul(tape,vals,st,nHn,nWlm) 50 let nLoss: i64=nfa_softce_rows(tape,vals,st,nLg,tgt) 51 leaves[0]=nE; leaves[1]=nWq; leaves[2]=nWk; leaves[3]=nWv; leaves[4]=nWo; leaves[5]=nWlm; leaves[6]=nLg 52 return nLoss 53} 54// BATCH-AVERAGED training: per outer step, sum clipped grads over B fresh streams into gacc, then one SGD step 55// with the AVERAGED gradient (gacc/B). Lower-noise gradient -> can sharpen where batch-1 oscillated. 56func do_train_batch(tape: *i64, vals: *i64, grads: *i64, st: *i64, W: *i64, WN: *i64, gacc: *i64, S: *i64, tgt: *i64, P: i64, dm: i64, V: i64, scale: i64, leaves: *i64, gb: *i64, outer: i64, B: i64, lr: i64, sdat: *i64) -> i64 { 57 var s: i64=0 58 while s<outer { 59 var i: i64=0 60 while i<6 { let ga: *i64=gacc[i] as *i64; let cn: i64=WN[i]; var c: i64=0; while c<cn { ga[c]=0; c=c+1 } i=i+1 } // zero accumulators 61 var b: i64=0 62 while b<B { 63 make_stream4(S,tgt,P,sdat) 64 let nl: i64=clm_fwd(tape,vals,st,W,S,tgt,P-1,dm,V,scale,leaves) 65 nfa_backward(tape,vals,grads,st[0],nl) 66 i=0 67 while i<6 { let ga: *i64=gacc[i] as *i64; let cn: i64=WN[i]; let nd: i64=leaves[i]; var c: i64=0; while c<cn { var g: i64=nfa_grad(tape,grads,nd,c); if g>262144{g=262144} if g<0-262144{g=0-262144} ga[c]=ga[c]+g; c=c+1 } i=i+1 } 68 b=b+1 69 } 70 i=0 71 while i<6 { let ar: *i64=W[i] as *i64; let ga: *i64=gacc[i] as *i64; let cn: i64=WN[i]; var c: i64=0; while c<cn { gb[c]=ga[c]/B; c=c+1 } nfa_sgd(ar,gb,cn,lr); i=i+1 } 72 s=s+1 73 } 74 return 0 75} 76func eval_ce(tape: *i64, vals: *i64, st: *i64, W: *i64, S: *i64, tgt: *i64, P: i64, dm: i64, V: i64, scale: i64, leaves: *i64, N: i64, sdat: *i64) -> i64 { 77 var acc: i64=0; var e: i64=0 78 while e<N { make_stream4(S,tgt,P,sdat); let nl: i64=clm_fwd(tape,vals,st,W,S,tgt,P-1,dm,V,scale,leaves); acc=acc+nfa_val(tape,vals,nl,0); e=e+1 } 79 let mq: i64=acc/N 80 return (mq*1000)/Q16 81} 82 83func main() -> i64 { 84 g_puts("nx_nofloat_scale_batch gate (R4 retry: MINI-BATCH gradient averaging on the richer 16-word grammar)\n" as *u8) 85 let V: i64=16; let P: i64=16; let dm: i64=32; let scale: i64=11585 86 let tape: *i64=sys_mmap(1024*7*8) as *i64 87 let vals: *i64=sys_mmap(131072*8) as *i64 88 let grads: *i64=sys_mmap(131072*8) as *i64 89 let st: *i64=sys_mmap(2*8) as *i64 90 let nW: i64=6 91 let W: *i64=sys_mmap(nW*8) as *i64; let WN: *i64=sys_mmap(nW*8) as *i64 92 WN[0]=V*dm; WN[1]=dm*dm; WN[2]=dm*dm; WN[3]=dm*dm; WN[4]=dm*dm; WN[5]=dm*V 93 var wi: i64=0; while wi<nW { let a: *i64=sys_mmap(WN[wi]*8) as *i64; dini(a,WN[wi],wi+1); W[wi]=a as i64; wi=wi+1 } 94 let gacc: *i64=sys_mmap(nW*8) as *i64; wi=0; while wi<nW { gacc[wi]=(sys_mmap(WN[wi]*8) as *i64) as i64; wi=wi+1 } 95 let leaves: *i64=sys_mmap(8*8) as *i64; let gbuf: *i64=sys_mmap(4096*8) as *i64 96 let S: *i64=sys_mmap(P*8) as *i64; let tgt: *i64=sys_mmap(P*8) as *i64; let sdat: *i64=sys_mmap(8) as *i64 97 98 g_puts(" batch B=16 averaged grads, dm=32, SGD lr=0.2, 2200 outer steps (~35k forwards)\n" as *u8) 99 sdat[0]=12345 100 do_train_batch(tape,vals,grads,st,W,WN,gacc,S,tgt,P,dm,V,scale,leaves,gbuf,2200,16,13107,sdat) 101 sdat[0]=24682468 102 let ce: i64=eval_ce(tape,vals,st,W,S,tgt,P,dm,V,scale,leaves,150,sdat) 103 104 g_puts(" [measure] held-out CE="); g_pn(ce); g_puts(" milli-nats uniform=2773 floor=1324 (batch-1 plateaued ~2135)\n") 105 106 var pass: i64=0; var total: i64=0 107 var t1: i64=0; if ce*10 <= UNIFORM_MNAT*7 { t1=1 } 108 pass=pass+g_check("T1: held-out CE << uniform (learned the richer language)" as *u8, t1); total=total+1 109 var t2: i64=0; if ce <= FLOOR_MNAT+200 { t2=1 } 110 pass=pass+g_check("T2: held-out CE ~= floor (near-OPTIMAL -> mini-batch averaging cracked the plateau)" as *u8, t2); total=total+1 111 112 var okall: i64=0; if pass==total { okall=1 } 113 g_puts("---- scale_batch gate: passed "); g_pn(pass); g_puts(" / "); g_pn(total); g_puts(" ----\n") 114 if okall==1 { g_puts("verdict=GREEN (mini-batch averaging reached the richer floor -- R4 lands at scale, pure no-float)\n" as *u8); sys_exit(0); return 0 } 115 g_puts("verdict=RED\n" as *u8); sys_exit(1); return 1 116}