nx_nofloat_scale_gate.nx source

1// STATUS 2026-06-23: PARTIAL / NOT REGISTERED -- HONEST documented attempt mapping the frontier. On the richer 2// 4-category grammar (vocab 16) at dm=32, a SINGLE-BLOCK no-float LM SCALES and LEARNS STRUCTURE but does NOT 3// reach optimality at tractable compute: 4// - low-lr clipped SGD (0.1): plateaus at uniform (CE ~2738 ~= ln16 2773) -- too-small steps. 5// - higher-lr clipped SGD (0.2): ESCAPES the plateau, CE ~2135 (perplexity ~8.5) = clearly BELOW uniform 6// (2773 / ppl 16) so it learned the category structure, but still well ABOVE the floor (1324 / ppl 3.76) 7// = NOT near-optimal. lr 0.3 / more steps did not improve (plateaus ~2135-2295). THIS IS THE BEST RESULT. 8// - lr DECAY (0.2->0.02): WORSE (CE 2510) -- the low late-lr can't progress; not the fix. 9// - AdamW: diverges at lr>=0.01 / stalls at uniform at lr<=0.003 on this model (a narrow unstable band). 10// NOTE: I hypothesized this was a Q16 g^2-UNDERFLOW and built a Q32-moment Adam to fix it -- but a CONTROLLED 11// small-gradient fit CONVERGES under BOTH Q16 and Q32 moments (loss 112 vs 85), so that hypothesis is NOT 12// confirmed; the transformer-specific divergence cause is NOT isolated. (over-claim retracted; gate deleted.) 13// - DEPTH-2 (two stacked blocks, SGD 0.2): CE 2236 ~= single-block 2135 -- DEPTH DOES NOT HELP (tested, then 14// deleted). So capacity/depth is NOT the bottleneck. 15// So R4-FULL = PARTIAL, and the boundary is now precisely located: NOT a missing mechanism, NOT capacity/depth 16// (depth tested), but fixed-point OPTIMIZATION reaching sharp output distributions + compute -- a novel-optimizer 17// / much-more-compute investment (operator-gated). The 3-category LANDING (nx_nofloat_landing_gate) IS 18// near-optimal and stands. (don't-lie: not registered, NOT claimed GREEN -- near-optimal genuinely fails here.) 19// 20// nx_nofloat_scale_gate.nx -- R4 SCALE-UP (the frontier step): grow BOTH the language and the model, and show 21// the no-float LM stays NEAR-OPTIMAL. Richer grammar (4 categories, vocab 16): DET(2) -> ADJ(4) -> NOUN(5) -> 22// VERB(5) -> DET ... (e.g. "the big cat ran a red dog sat"). Bigger model dm=32. Fresh-random members each 23// stream -> the model learns the GRAMMAR, measured by HELD-OUT perplexity vs the richer information floor. 24// Floor = avg(ln2,ln4,ln5,ln5) = 1.324 nats (perplexity 3.76; members are random within category so unbeatable); 25// uniform baseline = ln(16) = 2.773 nats (perplexity 16). 26// T1 held-out CE << uniform (learned the richer language). 27// T2 held-out CE ~= the richer floor (near-OPTIMAL at scale = the model scaled up cleanly). 28// Pure integer Q16. HONEST: richer but still a CONSTRAINED domain (open multi-topic prose is the remaining 29// compute frontier). Sovereign: nx_nofloat_autograd + nx_syscalls. expect_exit: 0 30import "nx_nofloat_autograd.nx" 31import "nx_syscalls.nx" 32import "nx_gate_emit_lib.nx" 33const Q16: i64 = 65536 34const UNIFORM_MNAT: i64 = 2773 // ln(16) 35const FLOOR_MNAT: i64 = 1324 // avg(ln2,ln4,ln5,ln5) 36 37 38func dini(a: *i64, n: i64, sd: i64) -> i64 { var i: i64=0; while i<n { a[i]=(((i*7+sd*13+1)%11)-5)*13107; i=i+1 } return 0 } 39func lcg(st: *i64) -> i64 { st[0]=(st[0]*1103515245 + 12345) & 2147483647; return (st[0] >> 15) } 40// richer grammar stream: cat=i%4; DET 0..1, ADJ 2..5, NOUN 6..10, VERB 11..15. 41func make_stream4(S: *i64, tgt: *i64, P: i64, st: *i64) -> i64 { 42 var i: i64=0 43 while i<P { 44 let c: i64=i%4 45 if c==0 { S[i]=lcg(st)%2 } 46 if c==1 { S[i]=2+lcg(st)%4 } 47 if c==2 { S[i]=6+lcg(st)%5 } 48 if c==3 { S[i]=11+lcg(st)%5 } 49 i=i+1 50 } 51 i=0; while i<P-1 { tgt[i]=S[i+1]; i=i+1 } tgt[P-1]=S[0] 52 return 0 53} 54func clm_fwd(tape: *i64, vals: *i64, st: *i64, W: *i64, ids: *i64, tgt: *i64, T: i64, dm: i64, V: i64, scale: i64, leaves: *i64) -> i64 { 55 let E: *i64=W[0] as *i64; let Wq: *i64=W[1] as *i64; let Wk: *i64=W[2] as *i64; let Wv: *i64=W[3] as *i64; let Wo: *i64=W[4] as *i64; let Wlm: *i64=W[5] as *i64 56 st[0]=0; st[1]=0 57 let nE: i64=nfa_leaf(tape,vals,st,V,dm,E,0) 58 let nWq: i64=nfa_leaf(tape,vals,st,dm,dm,Wq,0) 59 let nWk: i64=nfa_leaf(tape,vals,st,dm,dm,Wk,0) 60 let nWv: i64=nfa_leaf(tape,vals,st,dm,dm,Wv,0) 61 let nWo: i64=nfa_leaf(tape,vals,st,dm,dm,Wo,0) 62 let nWlm: i64=nfa_leaf(tape,vals,st,dm,V,Wlm,0) 63 let nX: i64=nfa_embed(tape,vals,st,nE,ids,T) 64 let nXn: i64=nfa_rmsnorm_rows(tape,vals,st,nX) 65 let nQ: i64=nfa_matmul(tape,vals,st,nXn,nWq) 66 let nK: i64=nfa_matmul(tape,vals,st,nXn,nWk) 67 let nV: i64=nfa_matmul(tape,vals,st,nXn,nWv) 68 let nQr: i64=nfa_rope(tape,vals,st,nQ) 69 let nKr: i64=nfa_rope(tape,vals,st,nK) 70 let nS: i64=nfa_matmul_nt(tape,vals,st,nQr,nKr) 71 let nSs: i64=nfa_cmul(tape,vals,st,nS,scale) 72 let nA: i64=nfa_softmax_rows(tape,vals,st,nSs,1) 73 let nO: i64=nfa_matmul(tape,vals,st,nA,nV) 74 let nOp: i64=nfa_matmul(tape,vals,st,nO,nWo) 75 let nH: i64=nfa_vadd(tape,vals,st,nX,nOp) 76 let nHn: i64=nfa_rmsnorm_rows(tape,vals,st,nH) 77 let nLg: i64=nfa_matmul(tape,vals,st,nHn,nWlm) 78 let nLoss: i64=nfa_softce_rows(tape,vals,st,nLg,tgt) 79 leaves[0]=nE; leaves[1]=nWq; leaves[2]=nWk; leaves[3]=nWv; leaves[4]=nWo; leaves[5]=nWlm; leaves[6]=nLg 80 return nLoss 81} 82// Q32-MOMENT AdamW per param array -- THE FIX for the Q16 AdamW divergence: store v (2nd moment) in Q32 and 83// compute g^2 at FULL precision (g*g, no >>16) so small gradients don't underflow to 0. In Q16, g=0.01 gives 84// g^2 that rounds to ~0, v collapses, denom -> eps, update explodes. Q32 v keeps g^2 (=g*g) exact. 85// m: Q16 (g-scale, fine). v: Q32 (= g^2 scale). bias-correction power loop early-stops when c2 underflows. 86// sqrt(vhat_Q32) via isqrt gives the Q16 sqrt directly (isqrt(V*2^32) = sqrt(V)*2^16). 87func adamw32_one(w: *i64, g: *i64, m: *i64, v: *i64, n: i64, lr: i64, b1: i64, b2: i64, eps: i64, t: i64) -> i64 { 88 var c1: i64=65536; var c2: i64=65536; var k: i64=0 89 while k<t { if c2==0 { k=t } else { c1=(c1*b1)>>16; c2=(c2*b2)>>16; k=k+1 } } // b1^t,b2^t (early-stop when c2->0) 90 let bc1: i64=65536-c1; let bc2: i64=65536-c2 91 var i: i64=0 92 while i<n { 93 m[i] = ((b1*m[i])>>16) + (((65536-b1)*g[i])>>16) // m Q16 94 let g2: i64 = g[i]*g[i] // Q32 -- full precision, no underflow 95 v[i] = ((b2*v[i])>>16) + (((65536-b2)*g2)>>16) // v Q32 96 var mh: i64=m[i]; if bc1>0 { mh=(m[i]<<16)/bc1 } // mhat Q16 97 var vh: i64=v[i]; if bc2>0 { vh=(v[i]<<16)/bc2 } // vhat Q32 98 if vh<0 { vh=0 } 99 let sq: i64 = nfa_isqrt(vh) // sqrt of Q32 = Q16 sqrt directly 100 let denom: i64 = sq + eps 101 var upd: i64=0; if denom>0 { upd=(mh<<16)/denom } // mhat/(sqrt(vhat)+eps), Q16 102 w[i] = w[i] - ((lr*upd)>>16) 103 i=i+1 104 } 105 return 0 106} 107// clipped Q32-AdamW over all weights 108func adamw_all(tape: *i64, grads: *i64, W: *i64, WN: *i64, M: *i64, Vv: *i64, leaves: *i64, nW: i64, t: i64, lr: i64, clip: i64, gb: *i64) -> i64 { 109 var i: i64=0 110 while i<nW { 111 let ar: *i64=W[i] as *i64; let m: *i64=M[i] as *i64; let v: *i64=Vv[i] as *i64; let cn: i64=WN[i]; let nd: i64=leaves[i] 112 var c: i64=0; while c<cn { var g: i64=nfa_grad(tape,grads,nd,c); if g>clip{g=clip} if g<0-clip{g=0-clip} gb[c]=g; c=c+1 } 113 adamw32_one(ar, gb, m, v, cn, lr, 58982, 65470, 66, t) 114 i=i+1 115 } 116 return 0 117} 118func do_train(tape: *i64, vals: *i64, grads: *i64, st: *i64, W: *i64, WN: *i64, M: *i64, Vv: *i64, S: *i64, tgt: *i64, P: i64, dm: i64, V: i64, scale: i64, leaves: *i64, gb: *i64, steps: i64, sdat: *i64) -> i64 { 119 var ep: i64=0 120 while ep < steps { 121 make_stream4(S,tgt,P,sdat) 122 let nl: i64=clm_fwd(tape,vals,st,W,S,tgt,P-1,dm,V,scale,leaves) 123 nfa_backward(tape,vals,grads,st[0],nl) 124 // clipped SGD with lr DECAY 0.2->0.02 (escape the uniform plateau early, fine-converge toward the floor late) 125 let lr: i64 = 13107 - (11796*ep)/steps 126 var i2: i64=0 127 while i2<6 { 128 let arr: *i64=W[i2] as *i64; let cn: i64=WN[i2]; let nd: i64=leaves[i2] 129 var c: i64=0; while c<cn { var g: i64=nfa_grad(tape,grads,nd,c); if g>262144{g=262144} if g<0-262144{g=0-262144} gb[c]=g; c=c+1 } 130 nfa_sgd(arr, gb, cn, lr) 131 i2=i2+1 132 } 133 ep=ep+1 134 } 135 return 0 136} 137func eval_ce(tape: *i64, vals: *i64, st: *i64, W: *i64, S: *i64, tgt: *i64, P: i64, dm: i64, V: i64, scale: i64, leaves: *i64, N: i64, sdat: *i64) -> i64 { 138 var acc: i64=0; var e: i64=0 139 while e<N { make_stream4(S,tgt,P,sdat); let nl: i64=clm_fwd(tape,vals,st,W,S,tgt,P-1,dm,V,scale,leaves); acc=acc+nfa_val(tape,vals,nl,0); e=e+1 } 140 let mean_q16: i64 = acc / N 141 return (mean_q16 * 1000) / Q16 142} 143 144func main() -> i64 { 145 g_puts("nx_nofloat_scale gate (R4 SCALE-UP: richer 16-word grammar + bigger model dm=32, near-optimal)\n" as *u8) 146 let V: i64=16; let P: i64=16; let dm: i64=32; let scale: i64=11585 // 1/sqrt(32) ~ 0.1768 147 let tape: *i64 = sys_mmap(1024*7*8) as *i64 148 let vals: *i64 = sys_mmap(131072*8) as *i64 149 let grads: *i64 = sys_mmap(131072*8) as *i64 150 let st: *i64 = sys_mmap(2*8) as *i64 151 let nW: i64=6 152 let W: *i64 = sys_mmap(nW*8) as *i64 153 let WN: *i64 = sys_mmap(nW*8) as *i64 154 WN[0]=V*dm; WN[1]=dm*dm; WN[2]=dm*dm; WN[3]=dm*dm; WN[4]=dm*dm; WN[5]=dm*V 155 let M: *i64 = sys_mmap(nW*8) as *i64; let Vv: *i64 = sys_mmap(nW*8) as *i64 156 var wi: i64=0 157 while wi<nW { let a: *i64=sys_mmap(WN[wi]*8) as *i64; dini(a,WN[wi],wi+1); W[wi]=a as i64; let m: *i64=sys_mmap(WN[wi]*8) as *i64; let v: *i64=sys_mmap(WN[wi]*8) as *i64; var z: i64=0; while z<WN[wi] { m[z]=0; v[z]=0; z=z+1 } M[wi]=m as i64; Vv[wi]=v as i64; wi=wi+1 } 158 let leaves: *i64 = sys_mmap(8*8) as *i64 159 let gbuf: *i64 = sys_mmap(4096*8) as *i64 160 let S: *i64 = sys_mmap(P*8) as *i64; let tgt: *i64 = sys_mmap(P*8) as *i64 161 let sdat: *i64 = sys_mmap(8) as *i64 162 163 g_puts(" language: DET(2) ADJ(4) NOUN(5) VERB(5), vocab=16, 4-slot sentences; model dm=32\n" as *u8) 164 sdat[0]=12345 165 do_train(tape,vals,grads,st,W,WN,M,Vv,S,tgt,P,dm,V,scale,leaves,gbuf,30000,sdat) 166 sdat[0]=24682468 167 let ce: i64 = eval_ce(tape,vals,st,W,S,tgt,P,dm,V,scale,leaves,150,sdat) 168 169 g_puts(" [measure] held-out CE=" as *u8); g_pn(ce); g_puts(" milli-nats uniform ln(16)=" as *u8); g_pn(UNIFORM_MNAT); g_puts(" richer floor avg(ln2,ln4,ln5,ln5)=" as *u8); g_pn(FLOOR_MNAT); g_puts(" (perplexity=exp(CE))\n" as *u8) 170 171 var pass: i64=0; var total: i64=0 172 var t1: i64=0; if ce*10 <= UNIFORM_MNAT*7 { t1=1 } // CE <= 0.7x uniform = learned the richer language 173 pass=pass+g_check("T1: held-out CE << uniform ln(16) (the no-float LM SCALED to the richer language)" as *u8, t1); total=total+1 174 var t2: i64=0; if ce <= FLOOR_MNAT+200 { t2=1 } // within 0.2 nats of the richer floor = near-optimal 175 pass=pass+g_check("T2: held-out CE ~= the richer information FLOOR (near-OPTIMAL at scale)" as *u8, t2); total=total+1 176 177 var okall: i64=0; if pass==total { okall=1 } 178 // (no status-log: this gate is the documented RED R4 attempt -- writing a "measured" log would be false run-evidence, WAVE-A) 179 g_puts("---- scale gate: passed " as *u8); g_pn(pass); g_puts(" / " as *u8); g_pn(total); g_puts(" ----\n" as *u8) 180 if okall==1 { g_puts("verdict=GREEN (R4 scale-up: a bigger no-float LM stays near-optimal on a richer language)\n" as *u8); sys_exit(0); return 0 } 181 g_puts("verdict=RED\n" as *u8); sys_exit(1); return 1 182}