nx_nofloat_scale_wide_gate.nx source

1// nx_nofloat_scale_wide_gate.nx -- R4 the genuine SCALE test: real WIDTH scale-up. Prior R4 tests stayed at 2// dm=24/32 and ruled out optimizer/depth/batch/FFN -- but WIDTH (features per token) is distinct from depth and 3// is exactly what producing SHARP 4-way-conditional distributions needs. Maximal tractable model: dm=64, 4// ffn=128, the COMPLETE block (attn+FFN), mini-batch averaged grads (lowest noise). Richer 4-cat grammar. 5// floor = avg(ln2,ln4,ln5,ln5)=1324 milli-nats (ppl 3.76); uniform=2773; dm=32 plateaued ~2135 (ppl 8.5). 6// T1 held-out CE << uniform. T2 held-out CE ~= floor (near-OPTIMAL -> WIDTH-scale closes it -> R4 progresses). 7// If T2 fails too, even 2x width is insufficient at tractable compute -> R4-FULL needs massive (GPU-scale) compute. 8// expect_exit: 0 Sovereign: nx_nofloat_autograd + nx_syscalls. (run in background -- longer compute) 9import "nx_nofloat_autograd.nx" 10import "nx_syscalls.nx" 11import "nx_gate_emit_lib.nx" 12const Q16: i64 = 65536 13const UNIFORM_MNAT: i64 = 2773 14const FLOOR_MNAT: i64 = 1324 15 16 17func dini(a: *i64, n: i64, sd: i64) -> i64 { var i: i64=0; while i<n { a[i]=(((i*7+sd*13+1)%11)-5)*9362; i=i+1 } return 0 } 18func lcg(st: *i64) -> i64 { st[0]=(st[0]*1103515245 + 12345) & 2147483647; return (st[0] >> 15) } 19func make_stream4(S: *i64, tgt: *i64, P: i64, st: *i64) -> i64 { 20 var i: i64=0 21 while i<P { let c: i64=i%4; if c==0 { S[i]=lcg(st)%2 } if c==1 { S[i]=2+lcg(st)%4 } if c==2 { S[i]=6+lcg(st)%5 } if c==3 { S[i]=11+lcg(st)%5 } i=i+1 } 22 i=0; while i<P-1 { tgt[i]=S[i+1]; i=i+1 } tgt[P-1]=S[0] 23 return 0 24} 25func clm_ffn(tape: *i64, vals: *i64, st: *i64, W: *i64, ids: *i64, tgt: *i64, T: i64, dm: i64, ffn: i64, V: i64, scale: i64, lv: *i64) -> i64 { 26 st[0]=0; st[1]=0 27 let nE: i64=nfa_leaf(tape,vals,st,V,dm,W[0] as *i64,0) 28 let nWq: i64=nfa_leaf(tape,vals,st,dm,dm,W[1] as *i64,0) 29 let nWk: i64=nfa_leaf(tape,vals,st,dm,dm,W[2] as *i64,0) 30 let nWv: i64=nfa_leaf(tape,vals,st,dm,dm,W[3] as *i64,0) 31 let nWo: i64=nfa_leaf(tape,vals,st,dm,dm,W[4] as *i64,0) 32 let nW1: i64=nfa_leaf(tape,vals,st,dm,ffn,W[5] as *i64,0) 33 let nW2: i64=nfa_leaf(tape,vals,st,ffn,dm,W[6] as *i64,0) 34 let nWlm: i64=nfa_leaf(tape,vals,st,dm,V,W[7] as *i64,0) 35 let nX: i64=nfa_embed(tape,vals,st,nE,ids,T) 36 let nXn: i64=nfa_rmsnorm_rows(tape,vals,st,nX) 37 let nQ: i64=nfa_matmul(tape,vals,st,nXn,nWq) 38 let nK: i64=nfa_matmul(tape,vals,st,nXn,nWk) 39 let nV: i64=nfa_matmul(tape,vals,st,nXn,nWv) 40 let nQr: i64=nfa_rope(tape,vals,st,nQ) 41 let nKr: i64=nfa_rope(tape,vals,st,nK) 42 let nS: i64=nfa_matmul_nt(tape,vals,st,nQr,nKr) 43 let nSs: i64=nfa_cmul(tape,vals,st,nS,scale) 44 let nA: i64=nfa_softmax_rows(tape,vals,st,nSs,1) 45 let nO: i64=nfa_matmul(tape,vals,st,nA,nV) 46 let nOp: i64=nfa_matmul(tape,vals,st,nO,nWo) 47 let nH: i64=nfa_vadd(tape,vals,st,nX,nOp) 48 let nHn: i64=nfa_rmsnorm_rows(tape,vals,st,nH) 49 let nF1: i64=nfa_matmul(tape,vals,st,nHn,nW1) 50 let nFs: i64=nfa_silu(tape,vals,st,nF1) 51 let nF2: i64=nfa_matmul(tape,vals,st,nFs,nW2) 52 let nH2: i64=nfa_vadd(tape,vals,st,nH,nF2) 53 let nH2n: i64=nfa_rmsnorm_rows(tape,vals,st,nH2) 54 let nLg: i64=nfa_matmul(tape,vals,st,nH2n,nWlm) 55 let nLoss: i64=nfa_softce_rows(tape,vals,st,nLg,tgt) 56 lv[0]=nE; lv[1]=nWq; lv[2]=nWk; lv[3]=nWv; lv[4]=nWo; lv[5]=nW1; lv[6]=nW2; lv[7]=nWlm; lv[8]=nLg 57 return nLoss 58} 59func do_train_batch(tape: *i64, vals: *i64, grads: *i64, st: *i64, W: *i64, WN: *i64, gacc: *i64, S: *i64, tgt: *i64, P: i64, dm: i64, ffn: i64, V: i64, scale: i64, lv: *i64, gb: *i64, outer: i64, B: i64, lr: i64, sdat: *i64) -> i64 { 60 var s: i64=0 61 while s<outer { 62 var i: i64=0 63 while i<8 { let ga: *i64=gacc[i] as *i64; let cn: i64=WN[i]; var c: i64=0; while c<cn { ga[c]=0; c=c+1 } i=i+1 } 64 var b: i64=0 65 while b<B { 66 make_stream4(S,tgt,P,sdat) 67 let nl: i64=clm_ffn(tape,vals,st,W,S,tgt,P-1,dm,ffn,V,scale,lv) 68 nfa_backward(tape,vals,grads,st[0],nl) 69 i=0 70 while i<8 { let ga: *i64=gacc[i] as *i64; let cn: i64=WN[i]; let nd: i64=lv[i]; var c: i64=0; while c<cn { var g: i64=nfa_grad(tape,grads,nd,c); if g>262144{g=262144} if g<0-262144{g=0-262144} ga[c]=ga[c]+g; c=c+1 } i=i+1 } 71 b=b+1 72 } 73 i=0 74 while i<8 { let ar: *i64=W[i] as *i64; let ga: *i64=gacc[i] as *i64; let cn: i64=WN[i]; var c: i64=0; while c<cn { gb[c]=ga[c]/B; c=c+1 } nfa_sgd(ar,gb,cn,lr); i=i+1 } 75 s=s+1 76 } 77 return 0 78} 79func eval_ce(tape: *i64, vals: *i64, st: *i64, W: *i64, S: *i64, tgt: *i64, P: i64, dm: i64, ffn: i64, V: i64, scale: i64, lv: *i64, N: i64, sdat: *i64) -> i64 { 80 var acc: i64=0; var e: i64=0 81 while e<N { make_stream4(S,tgt,P,sdat); let nl: i64=clm_ffn(tape,vals,st,W,S,tgt,P-1,dm,ffn,V,scale,lv); acc=acc+nfa_val(tape,vals,nl,0); e=e+1 } 82 let mq: i64=acc/N 83 return (mq*1000)/Q16 84} 85 86func main() -> i64 { 87 g_puts("nx_nofloat_scale_wide gate (R4 SCALE: WIDTH dm=64 + FFN-complete + batch on the richer grammar)\n" as *u8) 88 let V: i64=16; let P: i64=16; let dm: i64=64; let ffn: i64=128; let scale: i64=8192 89 let tape: *i64=sys_mmap(2048*7*8) as *i64 90 let vals: *i64=sys_mmap(524288*8) as *i64 91 let grads: *i64=sys_mmap(524288*8) as *i64 92 let st: *i64=sys_mmap(2*8) as *i64 93 let nW: i64=8 94 let W: *i64=sys_mmap(nW*8) as *i64; let WN: *i64=sys_mmap(nW*8) as *i64 95 WN[0]=V*dm; WN[1]=dm*dm; WN[2]=dm*dm; WN[3]=dm*dm; WN[4]=dm*dm; WN[5]=dm*ffn; WN[6]=ffn*dm; WN[7]=dm*V 96 var wi: i64=0; while wi<nW { let a: *i64=sys_mmap(WN[wi]*8) as *i64; dini(a,WN[wi],wi+1); W[wi]=a as i64; wi=wi+1 } 97 let gacc: *i64=sys_mmap(nW*8) as *i64; wi=0; while wi<nW { gacc[wi]=(sys_mmap(WN[wi]*8) as *i64) as i64; wi=wi+1 } 98 let lv: *i64=sys_mmap(10*8) as *i64; let gbuf: *i64=sys_mmap(16384*8) as *i64 99 let S: *i64=sys_mmap(P*8) as *i64; let tgt: *i64=sys_mmap(P*8) as *i64; let sdat: *i64=sys_mmap(8) as *i64 100 101 g_puts(" WIDTH scale: dm=64, ffn=128, complete block, batch B=16, SGD lr=0.2, 2500 outer steps (~40k forwards)\n" as *u8) 102 sdat[0]=12345 103 do_train_batch(tape,vals,grads,st,W,WN,gacc,S,tgt,P,dm,ffn,V,scale,lv,gbuf,2500,16,13107,sdat) 104 sdat[0]=24682468 105 let ce: i64=eval_ce(tape,vals,st,W,S,tgt,P,dm,ffn,V,scale,lv,150,sdat) 106 107 g_puts(" [measure] held-out CE="); g_pn(ce); g_puts(" milli-nats uniform=2773 floor=1324 (dm=32 plateaued ~2135)\n") 108 109 var pass: i64=0; var total: i64=0 110 var t1: i64=0; if ce*10 <= UNIFORM_MNAT*7 { t1=1 } 111 pass=pass+g_check("T1: held-out CE << uniform (the wide model learned the richer language)" as *u8, t1); total=total+1 112 var t2: i64=0; if ce <= FLOOR_MNAT+200 { t2=1 } 113 pass=pass+g_check("T2: held-out CE ~= floor (near-OPTIMAL -> WIDTH-scale closes the richer floor -> R4 progresses)" as *u8, t2); total=total+1 114 115 var okall: i64=0; if pass==total { okall=1 } 116 let logf: i64=sys_openat_append("knowledge/status/nofloat_scale_wide.log" as *u8, 420) 117 if logf>=0 { let x0: i64=sys_write(logf,"NOFLOATSCALEWIDE dm64 width scale-up measured\n" as *u8,46); sys_close(logf) } 118 g_puts("---- scale_wide gate: passed "); g_pn(pass); g_puts(" / "); g_pn(total); g_puts(" ----\n") 119 if okall==1 { g_puts("verdict=GREEN (WIDTH scale-up reached the richer floor -- scale IS the lever, R4 progresses, pure no-float)\n" as *u8); sys_exit(0); return 0 } 120 g_puts("verdict=RED\n" as *u8); sys_exit(1); return 1 121}