nx_nofloat_scale_gate.nx source

1// STATUS 2026-06-23: PARTIAL / NOT REGISTERED -- HONEST documented attempt mapping the frontier. On the richer 2// 4-category grammar (vocab 16) at dm=32, a SINGLE-BLOCK no-float LM SCALES and LEARNS STRUCTURE but does NOT 3// reach optimality at tractable compute: 4// - low-lr clipped SGD (0.1): plateaus at uniform (CE ~2738 ~= ln16 2773) -- too-small steps. 5// - higher-lr clipped SGD (0.2): ESCAPES the plateau, CE ~2135 (perplexity ~8.5) = clearly BELOW uniform 6// (2773 / ppl 16) so it learned the category structure, but still well ABOVE the floor (1324 / ppl 3.76) 7// = NOT near-optimal. lr 0.3 / more steps did not improve (plateaus ~2135-2295). THIS IS THE BEST RESULT. 8// - lr DECAY (0.2->0.02): WORSE (CE 2510) -- the low late-lr can't progress; not the fix. 9// - AdamW: diverges at lr>=0.01 / stalls at uniform at lr<=0.003 on this model (a narrow unstable band). 10// NOTE: I hypothesized this was a Q16 g^2-UNDERFLOW and built a Q32-moment Adam to fix it -- but a CONTROLLED 11// small-gradient fit CONVERGES under BOTH Q16 and Q32 moments (loss 112 vs 85), so that hypothesis is NOT 12// confirmed; the transformer-specific divergence cause is NOT isolated. (over-claim retracted; gate deleted.) 13// - DEPTH-2 (two stacked blocks, SGD 0.2): CE 2236 ~= single-block 2135 -- DEPTH DOES NOT HELP (tested, then 14// deleted). So capacity/depth is NOT the bottleneck. 15// So R4-FULL = PARTIAL, and the boundary is now precisely located: NOT a missing mechanism, NOT capacity/depth 16// (depth tested), but fixed-point OPTIMIZATION reaching sharp output distributions + compute -- a novel-optimizer 17// / much-more-compute investment (operator-gated). The 3-category LANDING (nx_nofloat_landing_gate) IS 18// near-optimal and stands. (don't-lie: not registered, NOT claimed GREEN -- near-optimal genuinely fails here.) 19// 20// nx_nofloat_scale_gate.nx -- R4 SCALE-UP (the frontier step): grow BOTH the language and the model, and show 21// the no-float LM stays NEAR-OPTIMAL. Richer grammar (4 categories, vocab 16): DET(2) -> ADJ(4) -> NOUN(5) -> 22// VERB(5) -> DET ... (e.g. "the big cat ran a red dog sat"). Bigger model dm=32. Fresh-random members each 23// stream -> the model learns the GRAMMAR, measured by HELD-OUT perplexity vs the richer information floor. 24// Floor = avg(ln2,ln4,ln5,ln5) = 1.324 nats (perplexity 3.76; members are random within category so unbeatable); 25// uniform baseline = ln(16) = 2.773 nats (perplexity 16). 26// T1 held-out CE << uniform (learned the richer language). 27// T2 held-out CE ~= the richer floor (near-OPTIMAL at scale = the model scaled up cleanly). 28// Pure integer Q16. HONEST: richer but still a CONSTRAINED domain (open multi-topic prose is the remaining 29// compute frontier). Sovereign: nx_nofloat_autograd + nx_syscalls. expect_exit: 0 30import "nx_nofloat_autograd.nx" 31import "nx_syscalls.nx" 32import "nx_gate_emit_lib.nx" 33import "nx_gate_verdict.nx" 34const Q16: i64 = 65536 35const UNIFORM_MNAT: i64 = 2773 // ln(16) 36const FLOOR_MNAT: i64 = 1324 // avg(ln2,ln4,ln5,ln5) 37 38 39func dini(a: *i64, n: i64, sd: i64) -> i64 { var i: i64=0; while i<n { a[i]=(((i*7+sd*13+1)%11)-5)*13107; i=i+1 } return 0 } 40func lcg(st: *i64) -> i64 { st[0]=(st[0]*1103515245 + 12345) & 2147483647; return (st[0] >> 15) } 41// richer grammar stream: cat=i%4; DET 0..1, ADJ 2..5, NOUN 6..10, VERB 11..15. 42func make_stream4(S: *i64, tgt: *i64, P: i64, st: *i64) -> i64 { 43 var i: i64=0 44 while i<P { 45 let c: i64=i%4 46 if c==0 { S[i]=lcg(st)%2 } 47 if c==1 { S[i]=2+lcg(st)%4 } 48 if c==2 { S[i]=6+lcg(st)%5 } 49 if c==3 { S[i]=11+lcg(st)%5 } 50 i=i+1 51 } 52 i=0; while i<P-1 { tgt[i]=S[i+1]; i=i+1 } tgt[P-1]=S[0] 53 return 0 54} 55func clm_fwd(tape: *i64, vals: *i64, st: *i64, W: *i64, ids: *i64, tgt: *i64, T: i64, dm: i64, V: i64, scale: i64, leaves: *i64) -> i64 { 56 let E: *i64=W[0] as *i64; let Wq: *i64=W[1] as *i64; let Wk: *i64=W[2] as *i64; let Wv: *i64=W[3] as *i64; let Wo: *i64=W[4] as *i64; let Wlm: *i64=W[5] as *i64 57 st[0]=0; st[1]=0 58 let nE: i64=nfa_leaf(tape,vals,st,V,dm,E,0) 59 let nWq: i64=nfa_leaf(tape,vals,st,dm,dm,Wq,0) 60 let nWk: i64=nfa_leaf(tape,vals,st,dm,dm,Wk,0) 61 let nWv: i64=nfa_leaf(tape,vals,st,dm,dm,Wv,0) 62 let nWo: i64=nfa_leaf(tape,vals,st,dm,dm,Wo,0) 63 let nWlm: i64=nfa_leaf(tape,vals,st,dm,V,Wlm,0) 64 let nX: i64=nfa_embed(tape,vals,st,nE,ids,T) 65 let nXn: i64=nfa_rmsnorm_rows(tape,vals,st,nX) 66 let nQ: i64=nfa_matmul(tape,vals,st,nXn,nWq) 67 let nK: i64=nfa_matmul(tape,vals,st,nXn,nWk) 68 let nV: i64=nfa_matmul(tape,vals,st,nXn,nWv) 69 let nQr: i64=nfa_rope(tape,vals,st,nQ) 70 let nKr: i64=nfa_rope(tape,vals,st,nK) 71 let nS: i64=nfa_matmul_nt(tape,vals,st,nQr,nKr) 72 let nSs: i64=nfa_cmul(tape,vals,st,nS,scale) 73 let nA: i64=nfa_softmax_rows(tape,vals,st,nSs,1) 74 let nO: i64=nfa_matmul(tape,vals,st,nA,nV) 75 let nOp: i64=nfa_matmul(tape,vals,st,nO,nWo) 76 let nH: i64=nfa_vadd(tape,vals,st,nX,nOp) 77 let nHn: i64=nfa_rmsnorm_rows(tape,vals,st,nH) 78 let nLg: i64=nfa_matmul(tape,vals,st,nHn,nWlm) 79 let nLoss: i64=nfa_softce_rows(tape,vals,st,nLg,tgt) 80 leaves[0]=nE; leaves[1]=nWq; leaves[2]=nWk; leaves[3]=nWv; leaves[4]=nWo; leaves[5]=nWlm; leaves[6]=nLg 81 return nLoss 82} 83// Q32-MOMENT AdamW per param array -- THE FIX for the Q16 AdamW divergence: store v (2nd moment) in Q32 and 84// compute g^2 at FULL precision (g*g, no >>16) so small gradients don't underflow to 0. In Q16, g=0.01 gives 85// g^2 that rounds to ~0, v collapses, denom -> eps, update explodes. Q32 v keeps g^2 (=g*g) exact. 86// m: Q16 (g-scale, fine). v: Q32 (= g^2 scale). bias-correction power loop early-stops when c2 underflows. 87// sqrt(vhat_Q32) via isqrt gives the Q16 sqrt directly (isqrt(V*2^32) = sqrt(V)*2^16). 88func adamw32_one(w: *i64, g: *i64, m: *i64, v: *i64, n: i64, lr: i64, b1: i64, b2: i64, eps: i64, t: i64) -> i64 { 89 var c1: i64=65536; var c2: i64=65536; var k: i64=0 90 while k<t { if c2==0 { k=t } else { c1=(c1*b1)>>16; c2=(c2*b2)>>16; k=k+1 } } // b1^t,b2^t (early-stop when c2->0) 91 let bc1: i64=65536-c1; let bc2: i64=65536-c2 92 var i: i64=0 93 while i<n { 94 m[i] = ((b1*m[i])>>16) + (((65536-b1)*g[i])>>16) // m Q16 95 let g2: i64 = g[i]*g[i] // Q32 -- full precision, no underflow 96 v[i] = ((b2*v[i])>>16) + (((65536-b2)*g2)>>16) // v Q32 97 var mh: i64=m[i]; if bc1>0 { mh=(m[i]<<16)/bc1 } // mhat Q16 98 var vh: i64=v[i]; if bc2>0 { vh=(v[i]<<16)/bc2 } // vhat Q32 99 if vh<0 { vh=0 } 100 let sq: i64 = nfa_isqrt(vh) // sqrt of Q32 = Q16 sqrt directly 101 let denom: i64 = sq + eps 102 var upd: i64=0; if denom>0 { upd=(mh<<16)/denom } // mhat/(sqrt(vhat)+eps), Q16 103 w[i] = w[i] - ((lr*upd)>>16) 104 i=i+1 105 } 106 return 0 107} 108// clipped Q32-AdamW over all weights 109func adamw_all(tape: *i64, grads: *i64, W: *i64, WN: *i64, M: *i64, Vv: *i64, leaves: *i64, nW: i64, t: i64, lr: i64, clip: i64, gb: *i64) -> i64 { 110 var i: i64=0 111 while i<nW { 112 let ar: *i64=W[i] as *i64; let m: *i64=M[i] as *i64; let v: *i64=Vv[i] as *i64; let cn: i64=WN[i]; let nd: i64=leaves[i] 113 var c: i64=0; while c<cn { var g: i64=nfa_grad(tape,grads,nd,c); if g>clip{g=clip} if g<0-clip{g=0-clip} gb[c]=g; c=c+1 } 114 adamw32_one(ar, gb, m, v, cn, lr, 58982, 65470, 66, t) 115 i=i+1 116 } 117 return 0 118} 119func do_train(tape: *i64, vals: *i64, grads: *i64, st: *i64, W: *i64, WN: *i64, M: *i64, Vv: *i64, S: *i64, tgt: *i64, P: i64, dm: i64, V: i64, scale: i64, leaves: *i64, gb: *i64, steps: i64, sdat: *i64) -> i64 { 120 var ep: i64=0 121 while ep < steps { 122 make_stream4(S,tgt,P,sdat) 123 let nl: i64=clm_fwd(tape,vals,st,W,S,tgt,P-1,dm,V,scale,leaves) 124 nfa_backward(tape,vals,grads,st[0],nl) 125 // clipped SGD with lr DECAY 0.2->0.02 (escape the uniform plateau early, fine-converge toward the floor late) 126 let lr: i64 = 13107 - (11796*ep)/steps 127 var i2: i64=0 128 while i2<6 { 129 let arr: *i64=W[i2] as *i64; let cn: i64=WN[i2]; let nd: i64=leaves[i2] 130 var c: i64=0; while c<cn { var g: i64=nfa_grad(tape,grads,nd,c); if g>262144{g=262144} if g<0-262144{g=0-262144} gb[c]=g; c=c+1 } 131 nfa_sgd(arr, gb, cn, lr) 132 i2=i2+1 133 } 134 ep=ep+1 135 } 136 return 0 137} 138func eval_ce(tape: *i64, vals: *i64, st: *i64, W: *i64, S: *i64, tgt: *i64, P: i64, dm: i64, V: i64, scale: i64, leaves: *i64, N: i64, sdat: *i64) -> i64 { 139 var acc: i64=0; var e: i64=0 140 while e<N { make_stream4(S,tgt,P,sdat); let nl: i64=clm_fwd(tape,vals,st,W,S,tgt,P-1,dm,V,scale,leaves); acc=acc+nfa_val(tape,vals,nl,0); e=e+1 } 141 let mean_q16: i64 = acc / N 142 return (mean_q16 * 1000) / Q16 143} 144 145func main() -> i64 { 146 g_puts("nx_nofloat_scale gate (R4 SCALE-UP: richer 16-word grammar + bigger model dm=32, near-optimal)\n" as *u8) 147 let V: i64=16; let P: i64=16; let dm: i64=32; let scale: i64=11585 // 1/sqrt(32) ~ 0.1768 148 let tape: *i64 = sys_mmap(1024*7*8) as *i64 149 let vals: *i64 = sys_mmap(131072*8) as *i64 150 let grads: *i64 = sys_mmap(131072*8) as *i64 151 let st: *i64 = sys_mmap(2*8) as *i64 152 let nW: i64=6 153 let W: *i64 = sys_mmap(nW*8) as *i64 154 let WN: *i64 = sys_mmap(nW*8) as *i64 155 WN[0]=V*dm; WN[1]=dm*dm; WN[2]=dm*dm; WN[3]=dm*dm; WN[4]=dm*dm; WN[5]=dm*V 156 let M: *i64 = sys_mmap(nW*8) as *i64; let Vv: *i64 = sys_mmap(nW*8) as *i64 157 var wi: i64=0 158 while wi<nW { let a: *i64=sys_mmap(WN[wi]*8) as *i64; dini(a,WN[wi],wi+1); W[wi]=a as i64; let m: *i64=sys_mmap(WN[wi]*8) as *i64; let v: *i64=sys_mmap(WN[wi]*8) as *i64; var z: i64=0; while z<WN[wi] { m[z]=0; v[z]=0; z=z+1 } M[wi]=m as i64; Vv[wi]=v as i64; wi=wi+1 } 159 let leaves: *i64 = sys_mmap(8*8) as *i64 160 let gbuf: *i64 = sys_mmap(4096*8) as *i64 161 let S: *i64 = sys_mmap(P*8) as *i64; let tgt: *i64 = sys_mmap(P*8) as *i64 162 let sdat: *i64 = sys_mmap(8) as *i64 163 164 g_puts(" language: DET(2) ADJ(4) NOUN(5) VERB(5), vocab=16, 4-slot sentences; model dm=32\n" as *u8) 165 sdat[0]=12345 166 do_train(tape,vals,grads,st,W,WN,M,Vv,S,tgt,P,dm,V,scale,leaves,gbuf,30000,sdat) 167 sdat[0]=24682468 168 let ce: i64 = eval_ce(tape,vals,st,W,S,tgt,P,dm,V,scale,leaves,150,sdat) 169 170 g_puts(" [measure] held-out CE=" as *u8); g_pn(ce); g_puts(" milli-nats uniform ln(16)=" as *u8); g_pn(UNIFORM_MNAT); g_puts(" richer floor avg(ln2,ln4,ln5,ln5)=" as *u8); g_pn(FLOOR_MNAT); g_puts(" (perplexity=exp(CE))\n" as *u8) 171 172 var pass: i64=0; var total: i64=0 173 var t1: i64=0; if ce*10 <= UNIFORM_MNAT*7 { t1=1 } // CE <= 0.7x uniform = learned the richer language 174 pass=pass+g_check("T1: held-out CE << uniform ln(16) (the no-float LM SCALED to the richer language)" as *u8, t1); total=total+1 175 var t2: i64=0; if ce <= FLOOR_MNAT+200 { t2=1 } // within 0.2 nats of the richer floor = near-optimal 176 pass=pass+g_check("T2: held-out CE ~= the richer information FLOOR (near-OPTIMAL at scale)" as *u8, t2); total=total+1 177 178 var okall: i64=0; if pass==total { okall=1 } 179 // (no status-log: this gate is the documented RED R4 attempt -- writing a "measured" log would be false run-evidence, WAVE-A) 180 g_puts("---- scale gate: passed " as *u8); g_pn(pass); g_puts(" / " as *u8); g_pn(total); g_puts(" ----\n" as *u8) 181 // MIGRATED onto nx_gate_verdict by nx_gate_dry_apply (D001, minimal form): every check 182 // row above is untouched, so the PASS/FAIL vector cannot change; only the hand-rolled 183 // verdict emission is replaced by the ONE shared base class. Proven by nx_gate_migrate verify. 184 let ctr__dry: *i64 = gv_ctr() 185 ctr__dry[0] = pass 186 ctr__dry[1] = total 187 let rc__dry: i64 = gv_verdict("NOFLOAT-SCALE-GATE" as *u8, ctr__dry, "R4 scale-up: a bigger no-float LM stays near-optimal on a richer language)" as *u8) 188 sys_exit(rc__dry) 189 return rc__dry 190}