nx_nofloat_adamw_gate.nx source

1// nx_nofloat_adamw_gate.nx -- HARD-EVIDENCE gate for the Q16 AdamW optimizer (nfa_adamw), with a MEASURED 2// exceed over plain SGD. On an ILL-CONDITIONED linear regression (two features of very different scales: 3// col0 ~ O(1), col1 ~ O(1/40)), one global SGD learning-rate either crawls on the small-curvature direction 4// or diverges on the large one. AdamW normalizes each parameter's step by its own running gradient scale, so 5// it converges FAR faster. Pure integer Q16 -> bit-exact/deterministic. 6// 7// T1 AdamW converges : from zero, AdamW drives the SSE loss to ~0 (it works). 8// T2 MEASURED exceed : at the SAME step budget, AdamW's final loss is MUCH lower than tuned SGD's 9// (and AdamW reaches the loss threshold in FAR fewer steps). Numbers printed. 10// T3 SGD really struggles: SGD's final loss on this conditioning stays high (the problem is genuinely hard 11// for one global rate) -- so T2 is a real win, not a strawman. 12// T4 bit-exact : two AdamW runs give IDENTICAL integer weights (determinism). 13// 14// Evidence -> knowledge/status/nofloat_adamw.log. Sovereign: nx_nofloat_autograd + nx_syscalls. expect_exit: 0 15import "nx_nofloat_autograd.nx" 16import "nx_syscalls.nx" 17import "nx_gate_emit_lib.nx" 18 19const ALOG: *u8 = "knowledge/status/nofloat_adamw.log" 20const Q16: i64 = 65536 21 22 23func a_ws(fd: i64, s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(fd,s,n); return 0 } 24func a_wn(fd: i64, v: i64) -> i64 { let b: *u8=sys_mmap(28); var m: i64=v; if m<0{sys_write(fd,"-" as *u8,1);m=0-m} let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i: i64=0; while i<k{b[i]=t[k-1-i];i=i+1} sys_write(fd,b,k); return 0 } 25 26// one forward+backward of the 1x2 linear regression (SSE over T samples); fills g[0..1] = dW, returns loss. 27func reg_grad(tape: *i64, vals: *i64, grads: *i64, st: *i64, Wp: *i64, X: *i64, Y: *i64, T: i64, g: *i64) -> i64 { 28 st[0]=0; st[1]=0 29 let nW: i64 = nfa_leaf(tape,vals,st,1,2,Wp,0) 30 var root: i64 = 0 - 1 31 var t: i64 = 0 32 while t < T { 33 let nx: i64 = nfa_leaf(tape,vals,st,2,1,X,t*2) 34 let np: i64 = nfa_matmul(tape,vals,st,nW,nx) 35 let ny: i64 = nfa_leaf(tape,vals,st,1,1,Y,t) 36 let nm: i64 = nfa_mse(tape,vals,st,np,ny) 37 if root < 0 { root = nm } else { root = nfa_vadd(tape,vals,st,root,nm) } 38 t = t + 1 39 } 40 nfa_backward(tape,vals,grads,st[0],root) 41 g[0] = nfa_grad(tape,grads,nW,0); g[1] = nfa_grad(tape,grads,nW,1) 42 return nfa_val(tape,vals,root,0) 43} 44 45func main() -> i64 { 46 g_puts("nx_nofloat_adamw gate (Q16 AdamW vs SGD on an ILL-CONDITIONED regression -- MEASURED exceed)\n" as *u8) 47 var pass: i64=0; var total: i64=0 48 let tape: *i64 = sys_mmap(256*7*8) as *i64 49 let vals: *i64 = sys_mmap(2048*8) as *i64 50 let grads: *i64 = sys_mmap(2048*8) as *i64 51 let st: *i64 = sys_mmap(2*8) as *i64 52 let T: i64 = 4 53 54 // ill-conditioned: col0 ~ O(1), col1 ~ O(1/40) BUT its target weight is LARGE (16.0) so it genuinely 55 // matters -> direction w1 has tiny curvature (sum x1^2) yet must travel far. One global SGD rate crawls 56 // on w1 (a ~1000x condition number); AdamW's per-param normalization fits both. Y = (W*.X)>>16. 57 let Wt: *i64 = sys_mmap(2*8) as *i64; Wt[0]=65536; Wt[1]=1048576 // w0=1.0, w1=16.0 58 let X: *i64 = sys_mmap(T*2*8) as *i64 59 X[0]=65536; X[1]=1638; X[2]=131072; X[3]=1311; X[4]=98304; X[5]=1966; X[6]=49152; X[7]=819 // col0~1, col1~0.02 60 let Y: *i64 = sys_mmap(T*8) as *i64 61 var t: i64=0 62 while t<T { Y[t]=(Wt[0]*X[t*2]+Wt[1]*X[t*2+1])>>16; t=t+1 } 63 let STEPS: i64 = 4000 64 let thresh: i64 = 2000 65 66 // ---- SGD (tuned lr; bigger diverges on col0) ---- 67 let Ws: *i64 = sys_mmap(2*8) as *i64; Ws[0]=0; Ws[1]=0 68 let gs: *i64 = sys_mmap(2*8) as *i64 69 var sgd_first: i64 = 0; var sgd_last: i64 = 0; var sgd_steps: i64 = STEPS 70 var sgd_reached: i64 = 0 71 var i: i64 = 0 72 while i < STEPS { 73 let L: i64 = reg_grad(tape,vals,grads,st,Ws,X,Y,T,gs) 74 if i==0 { sgd_first=L } 75 sgd_last=L 76 if sgd_reached==0 { if L < thresh { sgd_reached=1; sgd_steps=i } } 77 nfa_sgd(Ws, gs, 2, 655) // lr ~0.01 (larger overshoots col0; this is stable but crawls on w1) 78 i=i+1 79 } 80 81 // ---- AdamW (same budget) ---- 82 let Wa: *i64 = sys_mmap(2*8) as *i64; Wa[0]=0; Wa[1]=0 83 let ga: *i64 = sys_mmap(2*8) as *i64 84 let m: *i64 = sys_mmap(2*8) as *i64; m[0]=0; m[1]=0 85 let v: *i64 = sys_mmap(2*8) as *i64; v[0]=0; v[1]=0 86 var adam_first: i64=0; var adam_last: i64=0; var adam_steps: i64=STEPS; var adam_reached: i64=0 87 i = 0 88 while i < STEPS { 89 let L: i64 = reg_grad(tape,vals,grads,st,Wa,X,Y,T,ga) 90 if i==0 { adam_first=L } 91 adam_last=L 92 if adam_reached==0 { if L < thresh { adam_reached=1; adam_steps=i } } 93 nfa_adamw(Wa, ga, m, v, 2, 3277, 58982, 65470, 66, 0, i+1) // lr~0.05, b1=0.9, b2=0.999, eps~0.001, wd=0 94 i=i+1 95 } 96 97 g_puts(" [measure] SGD : first_loss=" as *u8); g_pn(sgd_first); g_puts(" final_loss=" as *u8); g_pn(sgd_last); g_puts(" steps_to<" as *u8); g_pn(thresh); g_puts("=" as *u8); if sgd_reached==1 { g_pn(sgd_steps) } else { g_puts(">" as *u8); g_pn(STEPS) } g_puts("\n" as *u8) 98 g_puts(" [measure] Adam: first_loss=" as *u8); g_pn(adam_first); g_puts(" final_loss=" as *u8); g_pn(adam_last); g_puts(" steps_to<" as *u8); g_pn(thresh); g_puts("=" as *u8); if adam_reached==1 { g_pn(adam_steps) } else { g_puts(">" as *u8); g_pn(STEPS) } g_puts("\n" as *u8) 99 g_puts(" learned W: SGD=[" as *u8); g_pn(Ws[0]); g_puts("," as *u8); g_pn(Ws[1]); g_puts("] Adam=[" as *u8); g_pn(Wa[0]); g_puts("," as *u8); g_pn(Wa[1]); g_puts("] target=[65536,1048576]\n" as *u8) 100 101 var t1: i64=0; if adam_last < thresh { t1=1 } 102 pass=pass+g_check("T1: AdamW converges -- drives the SSE loss below threshold from zero" as *u8, t1); total=total+1 103 var t2: i64=0; if adam_last*4 < sgd_last { t2=1 } 104 pass=pass+g_check("T2: MEASURED exceed -- AdamW's final loss is >=4x lower than SGD's at the same step budget" as *u8, t2); total=total+1 105 var t3: i64=0; if sgd_last > thresh { t3=1 } 106 pass=pass+g_check("T3: the problem is genuinely ill-conditioned -- SGD stays stuck above threshold (real win)" as *u8, t3); total=total+1 107 108 // ---- T4: bit-exact AdamW ---- 109 let Wa2: *i64 = sys_mmap(2*8) as *i64; Wa2[0]=0; Wa2[1]=0 110 let ga2: *i64 = sys_mmap(2*8) as *i64 111 let m2: *i64 = sys_mmap(2*8) as *i64; m2[0]=0; m2[1]=0 112 let v2: *i64 = sys_mmap(2*8) as *i64; v2[0]=0; v2[1]=0 113 i=0 114 while i < STEPS { reg_grad(tape,vals,grads,st,Wa2,X,Y,T,ga2); nfa_adamw(Wa2,ga2,m2,v2,2,3277,58982,65470,66,0,i+1); i=i+1 } 115 var t4: i64=1; if Wa2[0]!=Wa[0] { t4=0 } if Wa2[1]!=Wa[1] { t4=0 } 116 pass=pass+g_check("T4: bit-exact -- two AdamW runs give IDENTICAL integer weights (determinism)" as *u8, t4); total=total+1 117 118 var okall: i64=0; if pass==total { okall=1 } 119 let logf: i64 = sys_openat_append(ALOG, 420) 120 if logf >= 0 { 121 a_ws(logf,"NOFLOATADAMW illcond-regression sgd_final=" as *u8); a_wn(logf,sgd_last); a_ws(logf," adam_final=" as *u8); a_wn(logf,adam_last) 122 a_ws(logf," sgd_steps=" as *u8); a_wn(logf,sgd_steps); a_ws(logf," adam_steps=" as *u8); a_wn(logf,adam_steps); a_ws(logf," sgd_reached=" as *u8); a_wn(logf,sgd_reached) 123 a_ws(logf," T1=" as *u8); a_wn(logf,t1); a_ws(logf," T2=" as *u8); a_wn(logf,t2); a_ws(logf," T3=" as *u8); a_wn(logf,t3); a_ws(logf," T4=" as *u8); a_wn(logf,t4) 124 if okall==1 { a_ws(logf," verdict=GREEN\n" as *u8) } else { a_ws(logf," verdict=RED\n" as *u8) } 125 sys_close(logf) 126 } 127 g_puts("---- nofloat_adamw gate: passed " as *u8); g_pn(pass); g_puts(" / " as *u8); g_pn(total); g_puts(" ----\n" as *u8) 128 if okall==1 { g_puts("verdict=GREEN (Q16 AdamW converges + beats SGD on ill-conditioned training, bit-exact)\n" as *u8); sys_exit(0); return 0 } 129 g_puts("verdict=RED\n" as *u8); sys_exit(1); return 1 130}