nx_nofloat_scale_batch_gate.nx source

1// nx_nofloat_scale_batch_gate.nx -- R4 retry via the ONE untested fundamental lever: MINI-BATCH GRADIENT 2// AVERAGING. All prior richer-grammar training was BATCH-1 (one fresh-random stream/step = very noisy gradients 3// -> SGD oscillates, can't sharpen to the floor). Standard fix: average gradients over a mini-batch before 4// stepping. Same richer 4-cat grammar (DET2->ADJ4->NOUN5->VERB5, vocab 16) + dm=32 that plateaued at CE ~2135 5// (ppl 8.5); here we accumulate grads over B=16 fresh streams per step (low-noise), then ONE SGD step. 6// floor = avg(ln2,ln4,ln5,ln5)=1324 milli-nats (ppl 3.76); uniform = ln(16)=2773. 7// T1 held-out CE << uniform (learned). T2 held-out CE ~= floor (near-OPTIMAL -> batch-averaging cracked it). 8// If T2 fails too, batch size is NOT the lever and the tractable training levers are genuinely exhausted (honest). 9// expect_exit: 0 Sovereign: nx_nofloat_autograd + nx_syscalls. 10import "nx_nofloat_autograd.nx" 11import "nx_syscalls.nx" 12import "nx_gate_emit_lib.nx" 13import "nx_gate_verdict.nx" 14const Q16: i64 = 65536 15const UNIFORM_MNAT: i64 = 2773 16const FLOOR_MNAT: i64 = 1324 17 18 19func dini(a: *i64, n: i64, sd: i64) -> i64 { var i: i64=0; while i<n { a[i]=(((i*7+sd*13+1)%11)-5)*13107; i=i+1 } return 0 } 20func lcg(st: *i64) -> i64 { st[0]=(st[0]*1103515245 + 12345) & 2147483647; return (st[0] >> 15) } 21func make_stream4(S: *i64, tgt: *i64, P: i64, st: *i64) -> i64 { 22 var i: i64=0 23 while i<P { let c: i64=i%4; if c==0 { S[i]=lcg(st)%2 } if c==1 { S[i]=2+lcg(st)%4 } if c==2 { S[i]=6+lcg(st)%5 } if c==3 { S[i]=11+lcg(st)%5 } i=i+1 } 24 i=0; while i<P-1 { tgt[i]=S[i+1]; i=i+1 } tgt[P-1]=S[0] 25 return 0 26} 27func clm_fwd(tape: *i64, vals: *i64, st: *i64, W: *i64, ids: *i64, tgt: *i64, T: i64, dm: i64, V: i64, scale: i64, leaves: *i64) -> i64 { 28 let E: *i64=W[0] as *i64; let Wq: *i64=W[1] as *i64; let Wk: *i64=W[2] as *i64; let Wv: *i64=W[3] as *i64; let Wo: *i64=W[4] as *i64; let Wlm: *i64=W[5] as *i64 29 st[0]=0; st[1]=0 30 let nE: i64=nfa_leaf(tape,vals,st,V,dm,E,0) 31 let nWq: i64=nfa_leaf(tape,vals,st,dm,dm,Wq,0) 32 let nWk: i64=nfa_leaf(tape,vals,st,dm,dm,Wk,0) 33 let nWv: i64=nfa_leaf(tape,vals,st,dm,dm,Wv,0) 34 let nWo: i64=nfa_leaf(tape,vals,st,dm,dm,Wo,0) 35 let nWlm: i64=nfa_leaf(tape,vals,st,dm,V,Wlm,0) 36 let nX: i64=nfa_embed(tape,vals,st,nE,ids,T) 37 let nXn: i64=nfa_rmsnorm_rows(tape,vals,st,nX) 38 let nQ: i64=nfa_matmul(tape,vals,st,nXn,nWq) 39 let nK: i64=nfa_matmul(tape,vals,st,nXn,nWk) 40 let nV: i64=nfa_matmul(tape,vals,st,nXn,nWv) 41 let nQr: i64=nfa_rope(tape,vals,st,nQ) 42 let nKr: i64=nfa_rope(tape,vals,st,nK) 43 let nS: i64=nfa_matmul_nt(tape,vals,st,nQr,nKr) 44 let nSs: i64=nfa_cmul(tape,vals,st,nS,scale) 45 let nA: i64=nfa_softmax_rows(tape,vals,st,nSs,1) 46 let nO: i64=nfa_matmul(tape,vals,st,nA,nV) 47 let nOp: i64=nfa_matmul(tape,vals,st,nO,nWo) 48 let nH: i64=nfa_vadd(tape,vals,st,nX,nOp) 49 let nHn: i64=nfa_rmsnorm_rows(tape,vals,st,nH) 50 let nLg: i64=nfa_matmul(tape,vals,st,nHn,nWlm) 51 let nLoss: i64=nfa_softce_rows(tape,vals,st,nLg,tgt) 52 leaves[0]=nE; leaves[1]=nWq; leaves[2]=nWk; leaves[3]=nWv; leaves[4]=nWo; leaves[5]=nWlm; leaves[6]=nLg 53 return nLoss 54} 55// BATCH-AVERAGED training: per outer step, sum clipped grads over B fresh streams into gacc, then one SGD step 56// with the AVERAGED gradient (gacc/B). Lower-noise gradient -> can sharpen where batch-1 oscillated. 57func do_train_batch(tape: *i64, vals: *i64, grads: *i64, st: *i64, W: *i64, WN: *i64, gacc: *i64, S: *i64, tgt: *i64, P: i64, dm: i64, V: i64, scale: i64, leaves: *i64, gb: *i64, outer: i64, B: i64, lr: i64, sdat: *i64) -> i64 { 58 var s: i64=0 59 while s<outer { 60 var i: i64=0 61 while i<6 { let ga: *i64=gacc[i] as *i64; let cn: i64=WN[i]; var c: i64=0; while c<cn { ga[c]=0; c=c+1 } i=i+1 } // zero accumulators 62 var b: i64=0 63 while b<B { 64 make_stream4(S,tgt,P,sdat) 65 let nl: i64=clm_fwd(tape,vals,st,W,S,tgt,P-1,dm,V,scale,leaves) 66 nfa_backward(tape,vals,grads,st[0],nl) 67 i=0 68 while i<6 { let ga: *i64=gacc[i] as *i64; let cn: i64=WN[i]; let nd: i64=leaves[i]; var c: i64=0; while c<cn { var g: i64=nfa_grad(tape,grads,nd,c); if g>262144{g=262144} if g<0-262144{g=0-262144} ga[c]=ga[c]+g; c=c+1 } i=i+1 } 69 b=b+1 70 } 71 i=0 72 while i<6 { let ar: *i64=W[i] as *i64; let ga: *i64=gacc[i] as *i64; let cn: i64=WN[i]; var c: i64=0; while c<cn { gb[c]=ga[c]/B; c=c+1 } nfa_sgd(ar,gb,cn,lr); i=i+1 } 73 s=s+1 74 } 75 return 0 76} 77func eval_ce(tape: *i64, vals: *i64, st: *i64, W: *i64, S: *i64, tgt: *i64, P: i64, dm: i64, V: i64, scale: i64, leaves: *i64, N: i64, sdat: *i64) -> i64 { 78 var acc: i64=0; var e: i64=0 79 while e<N { make_stream4(S,tgt,P,sdat); let nl: i64=clm_fwd(tape,vals,st,W,S,tgt,P-1,dm,V,scale,leaves); acc=acc+nfa_val(tape,vals,nl,0); e=e+1 } 80 let mq: i64=acc/N 81 return (mq*1000)/Q16 82} 83 84func main() -> i64 { 85 g_puts("nx_nofloat_scale_batch gate (R4 retry: MINI-BATCH gradient averaging on the richer 16-word grammar)\n" as *u8) 86 let V: i64=16; let P: i64=16; let dm: i64=32; let scale: i64=11585 87 let tape: *i64=sys_mmap(1024*7*8) as *i64 88 let vals: *i64=sys_mmap(131072*8) as *i64 89 let grads: *i64=sys_mmap(131072*8) as *i64 90 let st: *i64=sys_mmap(2*8) as *i64 91 let nW: i64=6 92 let W: *i64=sys_mmap(nW*8) as *i64; let WN: *i64=sys_mmap(nW*8) as *i64 93 WN[0]=V*dm; WN[1]=dm*dm; WN[2]=dm*dm; WN[3]=dm*dm; WN[4]=dm*dm; WN[5]=dm*V 94 var wi: i64=0; while wi<nW { let a: *i64=sys_mmap(WN[wi]*8) as *i64; dini(a,WN[wi],wi+1); W[wi]=a as i64; wi=wi+1 } 95 let gacc: *i64=sys_mmap(nW*8) as *i64; wi=0; while wi<nW { gacc[wi]=(sys_mmap(WN[wi]*8) as *i64) as i64; wi=wi+1 } 96 let leaves: *i64=sys_mmap(8*8) as *i64; let gbuf: *i64=sys_mmap(4096*8) as *i64 97 let S: *i64=sys_mmap(P*8) as *i64; let tgt: *i64=sys_mmap(P*8) as *i64; let sdat: *i64=sys_mmap(8) as *i64 98 99 g_puts(" batch B=16 averaged grads, dm=32, SGD lr=0.2, 2200 outer steps (~35k forwards)\n" as *u8) 100 sdat[0]=12345 101 do_train_batch(tape,vals,grads,st,W,WN,gacc,S,tgt,P,dm,V,scale,leaves,gbuf,2200,16,13107,sdat) 102 sdat[0]=24682468 103 let ce: i64=eval_ce(tape,vals,st,W,S,tgt,P,dm,V,scale,leaves,150,sdat) 104 105 g_puts(" [measure] held-out CE="); g_pn(ce); g_puts(" milli-nats uniform=2773 floor=1324 (batch-1 plateaued ~2135)\n") 106 107 var pass: i64=0; var total: i64=0 108 var t1: i64=0; if ce*10 <= UNIFORM_MNAT*7 { t1=1 } 109 pass=pass+g_check("T1: held-out CE << uniform (learned the richer language)" as *u8, t1); total=total+1 110 var t2: i64=0; if ce <= FLOOR_MNAT+200 { t2=1 } 111 pass=pass+g_check("T2: held-out CE ~= floor (near-OPTIMAL -> mini-batch averaging cracked the plateau)" as *u8, t2); total=total+1 112 113 var okall: i64=0; if pass==total { okall=1 } 114 g_puts("---- scale_batch gate: passed "); g_pn(pass); g_puts(" / "); g_pn(total); g_puts(" ----\n") 115 // MIGRATED onto nx_gate_verdict by nx_gate_dry_apply (D001, minimal form): every check 116 // row above is untouched, so the PASS/FAIL vector cannot change; only the hand-rolled 117 // verdict emission is replaced by the ONE shared base class. Proven by nx_gate_migrate verify. 118 let ctr__dry: *i64 = gv_ctr() 119 ctr__dry[0] = pass 120 ctr__dry[1] = total 121 let rc__dry: i64 = gv_verdict("NOFLOAT-SCALE-BATCH-GATE" as *u8, ctr__dry, "mini-batch averaging reached the richer floor -- R4 lands at scale, pure no-float)" as *u8) 122 sys_exit(rc__dry) 123 return rc__dry 124}