code wiki / _hdl_build / nx_autograd_mlp_gate.nx
nx_autograd_mlp_gate.nx source
↩ module page · 133 lines · 9094 B
1// nx_autograd_mlp_gate.nx -- R3.5->R4 bridge: prove the sovereign autograd trains a MULTI-LAYER MLP (the
2// COOL-CHIC decoder is a tiny MLP, so this is the prerequisite for the lead neural-codec avenue). Builds a
3// 1->4->1 ReLU net + MSE loss entirely from the autograd primitives (define-by-run; rebuilt each step), then:
4// (a) GRADIENT-CHECKS weights across both layers vs central finite differences (proves backprop through the
5// full MLP graph -- the engine), and (b) runs SGD and shows the loss TRAINS DOWN overfitting |x| (a
6// nonlinear target a linear model cannot fit). Fixed-point Q16, no FPU/GPU/3rd-party. license_tier: ORIGINAL
7import "nx_syscalls.nx"
8import "nx_gate_emit_lib.nx"
9
10const Q: i64 = 65536
11// node: [0]=val [1]=grad [2]=op [3]=inA [4]=inB ; op 0=LEAF 1=ADD 2=MUL 3=RELU 4=SUB
12func g_num(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=(48 as u8);k=1}; while m>0{t[k]=((48+(m%10)) as u8);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(1,bb,k); return 0 }
13func g_w(fd: i64, s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(fd,s,n); return 0 }
14func g_wn(fd: i64, v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=(48 as u8);k=1}; while m>0{t[k]=((48+(m%10)) as u8);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(fd,bb,k); return 0 }
15func iabs(v: i64) -> i64 { if v<0 { return 0-v } return v }
16
17func ag_leaf(tp: *i64, np: *i64, v: i64) -> i64 { let k: i64=np[0]; tp[k*5]=v; tp[k*5+2]=0; tp[k*5+3]=0-1; tp[k*5+4]=0-1; np[0]=k+1; return k }
18func ag_add(tp: *i64, np: *i64, i: i64, j: i64) -> i64 { let k: i64=np[0]; tp[k*5]=tp[i*5]+tp[j*5]; tp[k*5+2]=1; tp[k*5+3]=i; tp[k*5+4]=j; np[0]=k+1; return k }
19func ag_sub(tp: *i64, np: *i64, i: i64, j: i64) -> i64 { let k: i64=np[0]; tp[k*5]=tp[i*5]-tp[j*5]; tp[k*5+2]=4; tp[k*5+3]=i; tp[k*5+4]=j; np[0]=k+1; return k }
20func ag_mul(tp: *i64, np: *i64, i: i64, j: i64) -> i64 { let k: i64=np[0]; tp[k*5]=(tp[i*5]*tp[j*5])>>16; tp[k*5+2]=2; tp[k*5+3]=i; tp[k*5+4]=j; np[0]=k+1; return k }
21func ag_relu(tp: *i64, np: *i64, i: i64) -> i64 { let k: i64=np[0]; var v: i64=tp[i*5]; if v<0 { v=0 } tp[k*5]=v; tp[k*5+2]=3; tp[k*5+3]=i; tp[k*5+4]=0-1; np[0]=k+1; return k }
22func ag_backward(tp: *i64, np: *i64, out: i64) -> i64 {
23 var i: i64=0; while i<np[0] { tp[i*5+1]=0; i=i+1 }
24 tp[out*5+1]=Q
25 i=np[0]-1
26 while i>=0 {
27 let g: i64=tp[i*5+1]; let o: i64=tp[i*5+2]; let ia: i64=tp[i*5+3]; let ib: i64=tp[i*5+4]
28 if o==1 { tp[ia*5+1]=tp[ia*5+1]+g; tp[ib*5+1]=tp[ib*5+1]+g }
29 if o==4 { tp[ia*5+1]=tp[ia*5+1]+g; tp[ib*5+1]=tp[ib*5+1]-g }
30 if o==2 { tp[ia*5+1]=tp[ia*5+1]+((g*tp[ib*5])>>16); tp[ib*5+1]=tp[ib*5+1]+((g*tp[ia*5])>>16) }
31 if o==3 { if tp[ia*5]>0 { tp[ia*5+1]=tp[ia*5+1]+g } }
32 i=i-1
33 }
34 return 0
35}
36// 1->H->1 ReLU MLP + squared error, built define-by-run. Weight leaves at fixed indices: w1[j]=j, b1[j]=H+j,
37// w2[j]=2H+j, b2=3H. Returns the sq-loss node index. (val of sq = tp[ret*5].)
38func build_mlp(tp: *i64, np: *i64, w1: *i64, b1: *i64, w2: *i64, b2v: i64, H: i64, xv: i64, tgtv: i64) -> i64 {
39 np[0]=0
40 var j: i64=0; while j<H { ag_leaf(tp,np,w1[j]); j=j+1 }
41 j=0; while j<H { ag_leaf(tp,np,b1[j]); j=j+1 }
42 j=0; while j<H { ag_leaf(tp,np,w2[j]); j=j+1 }
43 ag_leaf(tp,np,b2v)
44 let xi: i64=ag_leaf(tp,np,xv)
45 let ti: i64=ag_leaf(tp,np,tgtv)
46 var y: i64=3*H
47 j=0
48 while j<H {
49 let t1: i64=ag_mul(tp,np, j, xi)
50 let t2: i64=ag_add(tp,np, t1, H+j)
51 let h: i64=ag_relu(tp,np, t2)
52 let p: i64=ag_mul(tp,np, 2*H+j, h)
53 y=ag_add(tp,np, y, p)
54 j=j+1
55 }
56 let d: i64=ag_sub(tp,np, y, ti)
57 return ag_mul(tp,np, d, d)
58}
59func close(auto: i64, num: i64) -> i64 { let dd: i64=iabs(auto-num); let tol: i64=(iabs(auto)*8)/100 + 96; if dd<=tol { return 1 } return 0 }
60
61func main() -> i64 {
62 g_puts("=== AUTOGRAD-MLP GATE: sovereign autograd trains a 1->4->1 ReLU MLP (COOL-CHIC decoder prerequisite) ===\n" as *u8)
63 let H: i64=4
64 let tp: *i64=sys_mmap(256*5*8) as *i64
65 let np: *i64=sys_mmap(8) as *i64
66 let w1: *i64=sys_mmap(H*8) as *i64; let b1: *i64=sys_mmap(H*8) as *i64; let w2: *i64=sys_mmap(H*8) as *i64
67 // init weights (small, varied, nonzero -> break symmetry; relu units active at x=1 for a clean grad-check)
68 w1[0]=Q/4; w1[1]=Q/2; w1[2]=(3*Q)/4; w1[3]=Q
69 b1[0]=0; b1[1]=0; b1[2]=0; b1[3]=0
70 w2[0]=Q/4; w2[1]=Q/4; w2[2]=Q/4; w2[3]=Q/4
71 var b2: i64=0
72 let eps: i64=655
73 var pass: i64=0; let rows: i64=6
74
75 // GRADIENT-CHECK at (x=1.0, tgt=1.0): all relu active -> smooth -> finite-diff must match autograd
76 let sq: i64=build_mlp(tp, np, w1, b1, w2, b2, H, Q, Q)
77 ag_backward(tp, np, sq)
78 let gw1: i64=tp[0*5+1]; let gw2: i64=tp[(2*H)*5+1]; let gb1: i64=tp[H*5+1]
79 // numerical d/dw1[0]
80 let sv: i64=w1[0]; w1[0]=sv+eps; let sp1: i64=tp[build_mlp(tp,np,w1,b1,w2,b2,H,Q,Q)*5]; w1[0]=sv-eps; let sm1: i64=tp[build_mlp(tp,np,w1,b1,w2,b2,H,Q,Q)*5]; w1[0]=sv
81 let nw1: i64=((sp1-sm1)*Q)/(2*eps)
82 let sv2: i64=w2[0]; w2[0]=sv2+eps; let sp2: i64=tp[build_mlp(tp,np,w1,b1,w2,b2,H,Q,Q)*5]; w2[0]=sv2-eps; let sm2: i64=tp[build_mlp(tp,np,w1,b1,w2,b2,H,Q,Q)*5]; w2[0]=sv2
83 let nw2: i64=((sp2-sm2)*Q)/(2*eps)
84 let sv3: i64=b1[0]; b1[0]=sv3+eps; let sp3: i64=tp[build_mlp(tp,np,w1,b1,w2,b2,H,Q,Q)*5]; b1[0]=sv3-eps; let sm3: i64=tp[build_mlp(tp,np,w1,b1,w2,b2,H,Q,Q)*5]; b1[0]=sv3
85 let nb1: i64=((sp3-sm3)*Q)/(2*eps)
86 g_puts("-- grad-check: d/dw1[0] auto=" as *u8); g_num(gw1); g_puts(" num=" as *u8); g_num(nw1)
87 g_puts(" d/dw2[0] auto=" as *u8); g_num(gw2); g_puts(" num=" as *u8); g_num(nw2)
88 g_puts(" d/db1[0] auto=" as *u8); g_num(gb1); g_puts(" num=" as *u8); g_num(nb1); g_puts("\n" as *u8)
89 pass=pass+g_check(" T1 d/dw1[0] (layer-1 weight) autograd == finite-diff" as *u8, close(gw1, nw1))
90 pass=pass+g_check(" T2 d/dw2[0] (layer-2 weight) autograd == finite-diff" as *u8, close(gw2, nw2))
91 pass=pass+g_check(" T3 d/db1[0] (hidden bias) autograd == finite-diff" as *u8, close(gb1, nb1))
92 pass=pass+g_check(" T4 NEG: a wrong gradient is REJECTED (liar-kill armed)" as *u8, 1-close(gw1, nw1+40000))
93
94 // TRAIN: overfit |x| over 5 samples via SGD on autograd grads (nonlinear target a linear model can't fit)
95 let xs: *i64=sys_mmap(8*8) as *i64; let ys: *i64=sys_mmap(8*8) as *i64
96 xs[0]=0-(2*Q); xs[1]=0-Q; xs[2]=0; xs[3]=Q; xs[4]=2*Q
97 var s: i64=0; while s<5 { ys[s]=iabs(xs[s]); s=s+1 }
98 // reset to a fresh small init for training
99 w1[0]=Q/3; w1[1]=0-(Q/3); w1[2]=Q/2; w1[3]=0-(Q/2)
100 b1[0]=0; b1[1]=0; b1[2]=0; b1[3]=0
101 w2[0]=Q/4; w2[1]=Q/4; w2[2]=Q/4; w2[3]=Q/4; b2=0
102 var loss0: i64=0; s=0; while s<5 { loss0=loss0+tp[build_mlp(tp,np,w1,b1,w2,b2,H,xs[s],ys[s])*5]; s=s+1 }
103 let g1: *i64=sys_mmap(H*8) as *i64; let gb1a: *i64=sys_mmap(H*8) as *i64; let g2: *i64=sys_mmap(H*8) as *i64
104 let lr: i64=16
105 var step: i64=0
106 while step<40000 {
107 var j: i64=0; while j<H { g1[j]=0; gb1a[j]=0; g2[j]=0; j=j+1 }
108 var gb2: i64=0
109 s=0
110 while s<5 {
111 let sqi: i64=build_mlp(tp,np,w1,b1,w2,b2,H,xs[s],ys[s])
112 ag_backward(tp,np,sqi)
113 j=0; while j<H { g1[j]=g1[j]+tp[j*5+1]; gb1a[j]=gb1a[j]+tp[(H+j)*5+1]; g2[j]=g2[j]+tp[(2*H+j)*5+1]; j=j+1 }
114 gb2=gb2+tp[(3*H)*5+1]
115 s=s+1
116 }
117 j=0; while j<H { w1[j]=w1[j]-(g1[j]>>lr); b1[j]=b1[j]-(gb1a[j]>>lr); w2[j]=w2[j]-(g2[j]>>lr); j=j+1 }
118 b2=b2-(gb2>>lr)
119 step=step+1
120 }
121 var loss1: i64=0; s=0; while s<5 { loss1=loss1+tp[build_mlp(tp,np,w1,b1,w2,b2,H,xs[s],ys[s])*5]; s=s+1 }
122 g_puts("-- TRAIN |x|: loss start=" as *u8); g_num(loss0); g_puts(" after 40000 steps=" as *u8); g_num(loss1); g_puts(" (Q16 SSE over 5 samples)\n" as *u8)
123 var t5: i64=0; if loss1*2 < loss0 { t5=1 } // at least halved = it is genuinely training the MLP
124 pass=pass+g_check(" T5 SGD on autograd grads TRAINS the MLP down (loss >=50% reduced)" as *u8, t5)
125 var t6: i64=0; if loss1 < 20000 { t6=1 } // final SSE small -> the MLP actually FITS the nonlinear |x|
126 pass=pass+g_check(" T6 trained MLP genuinely FITS |x| (final SSE<20000 -- a linear model cannot)" as *u8, t6)
127
128 g_puts("----\nMLP rows=" as *u8); g_num(rows); g_puts(" pass=" as *u8); g_num(pass); g_puts("\n" as *u8)
129 let lg: i64=sys_openat_append("knowledge/status/autograd_mlp_gate.log" as *u8, 0x1a4)
130 if lg>=0 { g_w(lg,"MLP rows=" as *u8); g_wn(lg,rows); g_w(lg," pass=" as *u8); g_wn(lg,pass); g_w(lg," loss0=" as *u8); g_wn(lg,loss0); g_w(lg," loss1=" as *u8); g_wn(lg,loss1); if pass==rows { g_w(lg," verdict=GREEN\n" as *u8) } else { g_w(lg," verdict=RED\n" as *u8) } sys_close(lg) }
131 if pass==rows { g_puts("MLP GREEN (sovereign autograd trains a multi-layer ReLU MLP -- COOL-CHIC decoder is now buildable)\n" as *u8); sys_exit(0); return 0 }
132 g_puts("MLP RED\n" as *u8); sys_exit(1); return 1
133}