nx_nofloat_adamw_gate.nx source

1// nx_nofloat_adamw_gate.nx -- HARD-EVIDENCE gate for the Q16 AdamW optimizer (nfa_adamw), with a MEASURED 2// exceed over plain SGD. On an ILL-CONDITIONED linear regression (two features of very different scales: 3// col0 ~ O(1), col1 ~ O(1/40)), one global SGD learning-rate either crawls on the small-curvature direction 4// or diverges on the large one. AdamW normalizes each parameter's step by its own running gradient scale, so 5// it converges FAR faster. Pure integer Q16 -> bit-exact/deterministic. 6// 7// T1 AdamW converges : from zero, AdamW drives the SSE loss to ~0 (it works). 8// T2 MEASURED exceed : at the SAME step budget, AdamW's final loss is MUCH lower than tuned SGD's 9// (and AdamW reaches the loss threshold in FAR fewer steps). Numbers printed. 10// T3 SGD really struggles: SGD's final loss on this conditioning stays high (the problem is genuinely hard 11// for one global rate) -- so T2 is a real win, not a strawman. 12// T4 bit-exact : two AdamW runs give IDENTICAL integer weights (determinism). 13// 14// Evidence -> knowledge/status/nofloat_adamw.log. Sovereign: nx_nofloat_autograd + nx_syscalls. expect_exit: 0 15import "nx_nofloat_autograd.nx" 16import "nx_syscalls.nx" 17import "nx_gate_emit_lib.nx" 18import "nx_gate_verdict.nx" 19 20const ALOG: *u8 = "knowledge/status/nofloat_adamw.log" 21const Q16: i64 = 65536 22 23 24func a_ws(fd: i64, s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(fd,s,n); return 0 } 25func a_wn(fd: i64, v: i64) -> i64 { let b: *u8=sys_mmap(28); var m: i64=v; if m<0{sys_write(fd,"-" as *u8,1);m=0-m} let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i: i64=0; while i<k{b[i]=t[k-1-i];i=i+1} sys_write(fd,b,k); return 0 } 26 27// one forward+backward of the 1x2 linear regression (SSE over T samples); fills g[0..1] = dW, returns loss. 28func reg_grad(tape: *i64, vals: *i64, grads: *i64, st: *i64, Wp: *i64, X: *i64, Y: *i64, T: i64, g: *i64) -> i64 { 29 st[0]=0; st[1]=0 30 let nW: i64 = nfa_leaf(tape,vals,st,1,2,Wp,0) 31 var root: i64 = 0 - 1 32 var t: i64 = 0 33 while t < T { 34 let nx: i64 = nfa_leaf(tape,vals,st,2,1,X,t*2) 35 let np: i64 = nfa_matmul(tape,vals,st,nW,nx) 36 let ny: i64 = nfa_leaf(tape,vals,st,1,1,Y,t) 37 let nm: i64 = nfa_mse(tape,vals,st,np,ny) 38 if root < 0 { root = nm } else { root = nfa_vadd(tape,vals,st,root,nm) } 39 t = t + 1 40 } 41 nfa_backward(tape,vals,grads,st[0],root) 42 g[0] = nfa_grad(tape,grads,nW,0); g[1] = nfa_grad(tape,grads,nW,1) 43 return nfa_val(tape,vals,root,0) 44} 45 46func main() -> i64 { 47 g_puts("nx_nofloat_adamw gate (Q16 AdamW vs SGD on an ILL-CONDITIONED regression -- MEASURED exceed)\n" as *u8) 48 var pass: i64=0; var total: i64=0 49 let tape: *i64 = sys_mmap(256*7*8) as *i64 50 let vals: *i64 = sys_mmap(2048*8) as *i64 51 let grads: *i64 = sys_mmap(2048*8) as *i64 52 let st: *i64 = sys_mmap(2*8) as *i64 53 let T: i64 = 4 54 55 // ill-conditioned: col0 ~ O(1), col1 ~ O(1/40) BUT its target weight is LARGE (16.0) so it genuinely 56 // matters -> direction w1 has tiny curvature (sum x1^2) yet must travel far. One global SGD rate crawls 57 // on w1 (a ~1000x condition number); AdamW's per-param normalization fits both. Y = (W*.X)>>16. 58 let Wt: *i64 = sys_mmap(2*8) as *i64; Wt[0]=65536; Wt[1]=1048576 // w0=1.0, w1=16.0 59 let X: *i64 = sys_mmap(T*2*8) as *i64 60 X[0]=65536; X[1]=1638; X[2]=131072; X[3]=1311; X[4]=98304; X[5]=1966; X[6]=49152; X[7]=819 // col0~1, col1~0.02 61 let Y: *i64 = sys_mmap(T*8) as *i64 62 var t: i64=0 63 while t<T { Y[t]=(Wt[0]*X[t*2]+Wt[1]*X[t*2+1])>>16; t=t+1 } 64 let STEPS: i64 = 4000 65 let thresh: i64 = 2000 66 67 // ---- SGD (tuned lr; bigger diverges on col0) ---- 68 let Ws: *i64 = sys_mmap(2*8) as *i64; Ws[0]=0; Ws[1]=0 69 let gs: *i64 = sys_mmap(2*8) as *i64 70 var sgd_first: i64 = 0; var sgd_last: i64 = 0; var sgd_steps: i64 = STEPS 71 var sgd_reached: i64 = 0 72 var i: i64 = 0 73 while i < STEPS { 74 let L: i64 = reg_grad(tape,vals,grads,st,Ws,X,Y,T,gs) 75 if i==0 { sgd_first=L } 76 sgd_last=L 77 if sgd_reached==0 { if L < thresh { sgd_reached=1; sgd_steps=i } } 78 nfa_sgd(Ws, gs, 2, 655) // lr ~0.01 (larger overshoots col0; this is stable but crawls on w1) 79 i=i+1 80 } 81 82 // ---- AdamW (same budget) ---- 83 let Wa: *i64 = sys_mmap(2*8) as *i64; Wa[0]=0; Wa[1]=0 84 let ga: *i64 = sys_mmap(2*8) as *i64 85 let m: *i64 = sys_mmap(2*8) as *i64; m[0]=0; m[1]=0 86 let v: *i64 = sys_mmap(2*8) as *i64; v[0]=0; v[1]=0 87 var adam_first: i64=0; var adam_last: i64=0; var adam_steps: i64=STEPS; var adam_reached: i64=0 88 i = 0 89 while i < STEPS { 90 let L: i64 = reg_grad(tape,vals,grads,st,Wa,X,Y,T,ga) 91 if i==0 { adam_first=L } 92 adam_last=L 93 if adam_reached==0 { if L < thresh { adam_reached=1; adam_steps=i } } 94 nfa_adamw(Wa, ga, m, v, 2, 3277, 58982, 65470, 66, 0, i+1) // lr~0.05, b1=0.9, b2=0.999, eps~0.001, wd=0 95 i=i+1 96 } 97 98 g_puts(" [measure] SGD : first_loss=" as *u8); g_pn(sgd_first); g_puts(" final_loss=" as *u8); g_pn(sgd_last); g_puts(" steps_to<" as *u8); g_pn(thresh); g_puts("=" as *u8); if sgd_reached==1 { g_pn(sgd_steps) } else { g_puts(">" as *u8); g_pn(STEPS) } g_puts("\n" as *u8) 99 g_puts(" [measure] Adam: first_loss=" as *u8); g_pn(adam_first); g_puts(" final_loss=" as *u8); g_pn(adam_last); g_puts(" steps_to<" as *u8); g_pn(thresh); g_puts("=" as *u8); if adam_reached==1 { g_pn(adam_steps) } else { g_puts(">" as *u8); g_pn(STEPS) } g_puts("\n" as *u8) 100 g_puts(" learned W: SGD=[" as *u8); g_pn(Ws[0]); g_puts("," as *u8); g_pn(Ws[1]); g_puts("] Adam=[" as *u8); g_pn(Wa[0]); g_puts("," as *u8); g_pn(Wa[1]); g_puts("] target=[65536,1048576]\n" as *u8) 101 102 var t1: i64=0; if adam_last < thresh { t1=1 } 103 pass=pass+g_check("T1: AdamW converges -- drives the SSE loss below threshold from zero" as *u8, t1); total=total+1 104 var t2: i64=0; if adam_last*4 < sgd_last { t2=1 } 105 pass=pass+g_check("T2: MEASURED exceed -- AdamW's final loss is >=4x lower than SGD's at the same step budget" as *u8, t2); total=total+1 106 var t3: i64=0; if sgd_last > thresh { t3=1 } 107 pass=pass+g_check("T3: the problem is genuinely ill-conditioned -- SGD stays stuck above threshold (real win)" as *u8, t3); total=total+1 108 109 // ---- T4: bit-exact AdamW ---- 110 let Wa2: *i64 = sys_mmap(2*8) as *i64; Wa2[0]=0; Wa2[1]=0 111 let ga2: *i64 = sys_mmap(2*8) as *i64 112 let m2: *i64 = sys_mmap(2*8) as *i64; m2[0]=0; m2[1]=0 113 let v2: *i64 = sys_mmap(2*8) as *i64; v2[0]=0; v2[1]=0 114 i=0 115 while i < STEPS { reg_grad(tape,vals,grads,st,Wa2,X,Y,T,ga2); nfa_adamw(Wa2,ga2,m2,v2,2,3277,58982,65470,66,0,i+1); i=i+1 } 116 var t4: i64=1; if Wa2[0]!=Wa[0] { t4=0 } if Wa2[1]!=Wa[1] { t4=0 } 117 pass=pass+g_check("T4: bit-exact -- two AdamW runs give IDENTICAL integer weights (determinism)" as *u8, t4); total=total+1 118 119 var okall: i64=0; if pass==total { okall=1 } 120 let logf: i64 = sys_openat_append(ALOG, 420) 121 if logf >= 0 { 122 a_ws(logf,"NOFLOATADAMW illcond-regression sgd_final=" as *u8); a_wn(logf,sgd_last); a_ws(logf," adam_final=" as *u8); a_wn(logf,adam_last) 123 a_ws(logf," sgd_steps=" as *u8); a_wn(logf,sgd_steps); a_ws(logf," adam_steps=" as *u8); a_wn(logf,adam_steps); a_ws(logf," sgd_reached=" as *u8); a_wn(logf,sgd_reached) 124 a_ws(logf," T1=" as *u8); a_wn(logf,t1); a_ws(logf," T2=" as *u8); a_wn(logf,t2); a_ws(logf," T3=" as *u8); a_wn(logf,t3); a_ws(logf," T4=" as *u8); a_wn(logf,t4) 125 if okall==1 { a_ws(logf," verdict=GREEN\n" as *u8) } else { a_ws(logf," verdict=RED\n" as *u8) } 126 sys_close(logf) 127 } 128 g_puts("---- nofloat_adamw gate: passed " as *u8); g_pn(pass); g_puts(" / " as *u8); g_pn(total); g_puts(" ----\n" as *u8) 129 // MIGRATED onto nx_gate_verdict by nx_gate_dry_apply (D001, minimal form): every check 130 // row above is untouched, so the PASS/FAIL vector cannot change; only the hand-rolled 131 // verdict emission is replaced by the ONE shared base class. Proven by nx_gate_migrate verify. 132 let ctr__dry: *i64 = gv_ctr() 133 ctr__dry[0] = pass 134 ctr__dry[1] = total 135 let rc__dry: i64 = gv_verdict("NOFLOAT-ADAMW-GATE" as *u8, ctr__dry, "Q16 AdamW converges + beats SGD on ill-conditioned training, bit-exact)" as *u8) 136 sys_exit(rc__dry) 137 return rc__dry 138}