code wiki / _hdl_build / nx_nofloat_deepstack_gate.nx
nx_nofloat_deepstack_gate.nx source
↩ module page · 223 lines · 14053 B
1// nx_nofloat_deepstack_gate.nx -- HARD-EVIDENCE gate that the no-float transformer scales in DEPTH: a
2// 2-BLOCK (depth-2) transformer LM trains end-to-end on next-token prediction, pure integer Q16. Each block is
3// the verified pre-norm block (attn+RoPE+causal-softmax+SwiGLU+residuals); stacking = composing blk forward N
4// times (NO new ops -- depth is pure composition). Trained with the Q16 AdamW optimizer.
5//
6// A1 depth-2 gradcheck wrt Wlm : the LM-head gradient == finite differences.
7// A2 depth-2 gradcheck wrt E : E's gradient flows back through BOTH blocks (the deep-backprop proof) == FD.
8// D neg-control teeth (E, largest component) ; C bit-exact.
9// B the depth-2 LM TRAINS : next-token CE drops substantially (AdamW on E+Wlm through the 2-block stack).
10// (informational) depth-1 vs depth-2 final CE printed for transparency.
11//
12// Evidence -> knowledge/status/nofloat_deepstack.log. Sovereign: nx_nofloat_autograd + nx_syscalls. expect_exit: 0
13import "nx_nofloat_autograd.nx"
14import "nx_syscalls.nx"
15import "nx_gate_emit_lib.nx"
16import "nx_gate_verdict.nx"
17
18const DLOG: *u8 = "knowledge/status/nofloat_deepstack.log"
19const Q16: i64 = 65536
20
21
22func g_abs(v: i64) -> i64 { if v < 0 { return 0 - v } return v }
23func d_ws(fd: i64, s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(fd,s,n); return 0 }
24func d_wn(fd: i64, v: i64) -> i64 { let b: *u8=sys_mmap(28); var m: i64=v; if m<0{sys_write(fd,"-" as *u8,1);m=0-m} let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i: i64=0; while i<k{b[i]=t[k-1-i];i=i+1} sys_write(fd,b,k); return 0 }
25func dini(arr: *i64, n: i64, seed: i64) -> i64 { var i: i64=0; while i<n { arr[i] = (((i*7 + seed*13 + 1) % 11) - 5) * 13107; i=i+1 } return 0 }
26
27// ---- depth-D stack LM forward; W = ptr-array [E, Wlm, then per block b: Wq,Wk,Wv,Wo,Wg,Wu,Wd]; leaves[0]=nE,[1]=nWlm ----
28func stack_fwd(tape: *i64, vals: *i64, st: *i64, W: *i64, D: i64, ids: *i64, tgt: *i64, T: i64, dm: i64, ffn: i64, V: i64, scale: i64, leaves: *i64) -> i64 {
29 let E: *i64 = W[0] as *i64; let Wlm: *i64 = W[1] as *i64
30 st[0]=0; st[1]=0
31 let nE: i64 = nfa_leaf(tape,vals,st,V,dm,E,0)
32 let nWlm: i64 = nfa_leaf(tape,vals,st,dm,V,Wlm,0)
33 var cur: i64 = nfa_embed(tape,vals,st,nE,ids,T)
34 var b: i64 = 0
35 while b < D {
36 let base: i64 = 2 + b*7
37 let Wq: *i64 = W[base+0] as *i64; let Wk: *i64 = W[base+1] as *i64; let Wv: *i64 = W[base+2] as *i64; let Wo: *i64 = W[base+3] as *i64
38 let Wg: *i64 = W[base+4] as *i64; let Wu: *i64 = W[base+5] as *i64; let Wd: *i64 = W[base+6] as *i64
39 let nWq: i64 = nfa_leaf(tape,vals,st,dm,dm,Wq,0)
40 let nWk: i64 = nfa_leaf(tape,vals,st,dm,dm,Wk,0)
41 let nWv: i64 = nfa_leaf(tape,vals,st,dm,dm,Wv,0)
42 let nWo: i64 = nfa_leaf(tape,vals,st,dm,dm,Wo,0)
43 let nWg: i64 = nfa_leaf(tape,vals,st,dm,ffn,Wg,0)
44 let nWu: i64 = nfa_leaf(tape,vals,st,dm,ffn,Wu,0)
45 let nWd: i64 = nfa_leaf(tape,vals,st,ffn,dm,Wd,0)
46 let nXn: i64 = nfa_rmsnorm_rows(tape,vals,st,cur)
47 let nQ: i64 = nfa_matmul(tape,vals,st,nXn,nWq)
48 let nK: i64 = nfa_matmul(tape,vals,st,nXn,nWk)
49 let nV: i64 = nfa_matmul(tape,vals,st,nXn,nWv)
50 let nQr: i64 = nfa_rope(tape,vals,st,nQ)
51 let nKr: i64 = nfa_rope(tape,vals,st,nK)
52 let nS: i64 = nfa_matmul_nt(tape,vals,st,nQr,nKr)
53 let nSs: i64 = nfa_cmul(tape,vals,st,nS,scale)
54 let nA: i64 = nfa_softmax_rows(tape,vals,st,nSs,1)
55 let nO: i64 = nfa_matmul(tape,vals,st,nA,nV)
56 let nOp: i64 = nfa_matmul(tape,vals,st,nO,nWo)
57 let nH: i64 = nfa_vadd(tape,vals,st,cur,nOp)
58 let nHn: i64 = nfa_rmsnorm_rows(tape,vals,st,nH)
59 let nG: i64 = nfa_matmul(tape,vals,st,nHn,nWg)
60 let nU: i64 = nfa_matmul(tape,vals,st,nHn,nWu)
61 let nSg: i64 = nfa_silu(tape,vals,st,nG)
62 let nHs: i64 = nfa_hadamard(tape,vals,st,nSg,nU)
63 let nDp: i64 = nfa_matmul(tape,vals,st,nHs,nWd)
64 cur = nfa_vadd(tape,vals,st,nH,nDp)
65 b = b + 1
66 }
67 let nYn: i64 = nfa_rmsnorm_rows(tape,vals,st,cur)
68 let nLg: i64 = nfa_matmul(tape,vals,st,nYn,nWlm)
69 let nLoss: i64 = nfa_softce_rows(tape,vals,st,nLg,tgt)
70 leaves[0]=nE; leaves[1]=nWlm
71 return nLoss
72}
73func stack_lossval(tape: *i64, vals: *i64, st: *i64, W: *i64, D: i64, ids: *i64, tgt: *i64, T: i64, dm: i64, ffn: i64, V: i64, scale: i64) -> i64 {
74 let lv: *i64 = sys_mmap(2*8) as *i64
75 let nLoss: i64 = stack_fwd(tape,vals,st,W,D,ids,tgt,T,dm,ffn,V,scale,lv)
76 return nfa_val(tape,vals,nLoss,0)
77}
78// gradcheck a leaf weight array (widx: 0=E via leaves[0], 1=Wlm via leaves[1]); extract analytic grads first.
79func stack_gc(tape: *i64, vals: *i64, grads: *i64, st: *i64, W: *i64, D: i64, ids: *i64, tgt: *i64, T: i64, dm: i64, ffn: i64, V: i64, scale: i64, leaves: *i64, lidx: i64, arr: *i64, cnt: i64, h: i64, tol_q: i64, floor_q: i64, worst: *i64, anabuf: *i64) -> i64 {
80 let nLoss: i64 = stack_fwd(tape,vals,st,W,D,ids,tgt,T,dm,ffn,V,scale,leaves)
81 nfa_backward(tape,vals,grads,st[0],nLoss)
82 let nnode: i64 = leaves[lidx]
83 var c: i64=0
84 while c<cnt { anabuf[c]=nfa_grad(tape,grads,nnode,c); c=c+1 }
85 var ok: i64=1; worst[0]=0
86 var i: i64=0
87 while i<cnt {
88 let old: i64=arr[i]
89 arr[i]=old+h; let lp: i64=stack_lossval(tape,vals,st,W,D,ids,tgt,T,dm,ffn,V,scale)
90 arr[i]=old-h; let lm2: i64=stack_lossval(tape,vals,st,W,D,ids,tgt,T,dm,ffn,V,scale)
91 arr[i]=old
92 let fd: i64=((lp-lm2)*Q16)/(2*h); let num: i64=g_abs(fd-anabuf[i]); var den: i64=g_abs(anabuf[i]); if den<floor_q{den=floor_q}
93 if num >= ((tol_q*den)>>16) { ok=0 }
94 let rel: i64=(num*1000)/den; if rel>worst[0]{worst[0]=rel}
95 i=i+1
96 }
97 return ok
98}
99// train E + Wlm with AdamW for `steps`; returns final CE; *lf0 = first CE.
100func stack_train(tape: *i64, vals: *i64, grads: *i64, st: *i64, W: *i64, D: i64, ids: *i64, tgt: *i64, T: i64, dm: i64, ffn: i64, V: i64, scale: i64, steps: i64, lf0: *i64) -> i64 {
101 let E: *i64 = W[0] as *i64; let Wlm: *i64 = W[1] as *i64
102 let gE: *i64 = sys_mmap(64*8) as *i64; let mE: *i64 = sys_mmap(64*8) as *i64; let vE: *i64 = sys_mmap(64*8) as *i64
103 let gL: *i64 = sys_mmap(64*8) as *i64; let mL: *i64 = sys_mmap(64*8) as *i64; let vL: *i64 = sys_mmap(64*8) as *i64
104 var z: i64=0; while z<V*dm { mE[z]=0; vE[z]=0; z=z+1 } z=0; while z<dm*V { mL[z]=0; vL[z]=0; z=z+1 }
105 let leaves: *i64 = sys_mmap(2*8) as *i64
106 var ll: i64=0
107 var ep: i64=0
108 while ep < steps {
109 let nLoss: i64 = stack_fwd(tape,vals,st,W,D,ids,tgt,T,dm,ffn,V,scale,leaves)
110 nfa_backward(tape,vals,grads,st[0],nLoss)
111 if ep==0 { *lf0 = nfa_val(tape,vals,nLoss,0) }
112 ll = nfa_val(tape,vals,nLoss,0)
113 let nE: i64=leaves[0]; z=0; while z<V*dm { gE[z]=nfa_grad(tape,grads,nE,z); z=z+1 }
114 nfa_adamw(E, gE, mE, vE, V*dm, 3277, 58982, 65470, 66, 0, ep+1)
115 let nWl: i64=leaves[1]; z=0; while z<dm*V { gL[z]=nfa_grad(tape,grads,nWl,z); z=z+1 }
116 nfa_adamw(Wlm, gL, mL, vL, dm*V, 3277, 58982, 65470, 66, 0, ep+1)
117 ep=ep+1
118 }
119 return ll
120}
121// build a fresh weight pointer-array of D blocks, all arrays freshly mmap'd + init'd (deterministic).
122func build_W(D: i64, dm: i64, ffn: i64, V: i64) -> *i64 {
123 let W: *i64 = sys_mmap((2 + D*7)*8) as *i64
124 let E: *i64 = sys_mmap(V*dm*8) as *i64; dini(E,V*dm,1); W[0]=E as i64
125 let Wlm: *i64 = sys_mmap(dm*V*8) as *i64; dini(Wlm,dm*V,2); W[1]=Wlm as i64
126 var b: i64=0
127 while b<D {
128 let base: i64 = 2 + b*7
129 let Wq: *i64=sys_mmap(dm*dm*8) as *i64; dini(Wq,dm*dm,10+b*7+0); W[base+0]=Wq as i64
130 let Wk: *i64=sys_mmap(dm*dm*8) as *i64; dini(Wk,dm*dm,10+b*7+1); W[base+1]=Wk as i64
131 let Wv: *i64=sys_mmap(dm*dm*8) as *i64; dini(Wv,dm*dm,10+b*7+2); W[base+2]=Wv as i64
132 let Wo: *i64=sys_mmap(dm*dm*8) as *i64; dini(Wo,dm*dm,10+b*7+3); W[base+3]=Wo as i64
133 let Wg: *i64=sys_mmap(dm*ffn*8) as *i64; dini(Wg,dm*ffn,10+b*7+4); W[base+4]=Wg as i64
134 let Wu: *i64=sys_mmap(dm*ffn*8) as *i64; dini(Wu,dm*ffn,10+b*7+5); W[base+5]=Wu as i64
135 let Wd: *i64=sys_mmap(ffn*dm*8) as *i64; dini(Wd,ffn*dm,10+b*7+6); W[base+6]=Wd as i64
136 b=b+1
137 }
138 return W
139}
140
141func main() -> i64 {
142 g_puts("nx_nofloat_deepstack gate (a DEPTH-2 transformer LM trains end-to-end, PURE INTEGER Q16)\n" as *u8)
143 var pass: i64=0; var total: i64=0
144 let tape: *i64 = sys_mmap(1024*7*8) as *i64
145 let vals: *i64 = sys_mmap(32768*8) as *i64
146 let grads: *i64 = sys_mmap(32768*8) as *i64
147 let st: *i64 = sys_mmap(2*8) as *i64
148 let h: i64=512; let floor_q: i64=4096
149 let worst: *i64 = sys_mmap(8) as *i64
150 let anabuf: *i64 = sys_mmap(64*8) as *i64
151 let T: i64=4; let dm: i64=4; let ffn: i64=8; let V: i64=4; let scale: i64=32768; let D: i64=2
152 let ids: *i64 = sys_mmap(T*8) as *i64; ids[0]=0; ids[1]=1; ids[2]=2; ids[3]=3
153 let tgt: *i64 = sys_mmap(T*8) as *i64; tgt[0]=1; tgt[1]=2; tgt[2]=3; tgt[3]=0
154 let leaves: *i64 = sys_mmap(2*8) as *i64
155
156 // ---- A1: depth-2 gradcheck wrt Wlm ----
157 let Wa: *i64 = build_W(D,dm,ffn,V)
158 let a1_ok: i64 = stack_gc(tape,vals,grads,st,Wa,D,ids,tgt,T,dm,ffn,V,scale,leaves,1,Wa[1] as *i64,dm*V,h,8192,floor_q,worst,anabuf)
159 g_puts(" [measure] depth-2 dL/dWlm worst rel grad err = " as *u8); g_pn(worst[0]); g_puts(" /1000 (tol=125)\n" as *u8)
160 pass=pass+g_check("A1: depth-2 gradcheck wrt Wlm (LM head through CE) == finite differences" as *u8, a1_ok); total=total+1
161
162 // ---- A2: depth-2 gradcheck wrt E (through BOTH blocks) ----
163 let a2_ok: i64 = stack_gc(tape,vals,grads,st,Wa,D,ids,tgt,T,dm,ffn,V,scale,leaves,0,Wa[0] as *i64,V*dm,h,16384,floor_q,worst,anabuf)
164 g_puts(" [measure] depth-2 dL/dE (through BOTH blocks) worst rel grad err = " as *u8); g_pn(worst[0]); g_puts(" /1000 (tol=250)\n" as *u8)
165 pass=pass+g_check("A2: depth-2 gradcheck wrt E through BOTH blocks == finite differences (deep backprop)" as *u8, a2_ok); total=total+1
166
167 // ---- D: neg-control teeth (E, largest grad component, scale-free) ----
168 let nLd: i64 = stack_fwd(tape,vals,st,Wa,D,ids,tgt,T,dm,ffn,V,scale,leaves)
169 nfa_backward(tape,vals,grads,st[0],nLd)
170 let Ea: *i64 = Wa[0] as *i64
171 var imax: i64=0; var vmax: i64=0; var ii: i64=0
172 while ii<V*dm { let gg: i64=g_abs(nfa_grad(tape,grads,leaves[0],ii)); if gg>vmax { vmax=gg; imax=ii } ii=ii+1 }
173 let dana: i64 = nfa_grad(tape,grads,leaves[0],imax)
174 let o0: i64=Ea[imax]; Ea[imax]=o0+h; let lpd: i64=stack_lossval(tape,vals,st,Wa,D,ids,tgt,T,dm,ffn,V,scale); Ea[imax]=o0-h; let lmd: i64=stack_lossval(tape,vals,st,Wa,D,ids,tgt,T,dm,ffn,V,scale); Ea[imax]=o0
175 let dfd: i64=((lpd-lmd)*Q16)/(2*h); let dbad: i64=0-dana
176 let dgood: i64=g_abs(dfd-dana); let dneg: i64=g_abs(dfd-dbad)
177 var caught: i64=0; if vmax>64 { if dneg > dgood*4 { caught=1 } }
178 pass=pass+g_check("D: neg-control -- FD is >4x closer to the true E-grad than to the negated one (teeth)" as *u8, caught); total=total+1
179
180 // ---- B: depth-2 LM trains (CE drops) with AdamW ----
181 let Wb: *i64 = build_W(D,dm,ffn,V)
182 let lf2: *i64 = sys_mmap(8) as *i64
183 let ce2: i64 = stack_train(tape,vals,grads,st,Wb,D,ids,tgt,T,dm,ffn,V,scale,3000,lf2)
184 // informational: depth-1 final CE
185 let W1: *i64 = build_W(1,dm,ffn,V)
186 let lf1: *i64 = sys_mmap(8) as *i64
187 let ce1: i64 = stack_train(tape,vals,grads,st,W1,1,ids,tgt,T,dm,ffn,V,scale,3000,lf1)
188 g_puts(" [measure] depth-2 CE: start=" as *u8); g_pn(*lf2); g_puts(" end=" as *u8); g_pn(ce2); g_puts(" (info: depth-1 CE start=" as *u8); g_pn(*lf1); g_puts(" end=" as *u8); g_pn(ce1); g_puts(")\n" as *u8)
189 var fits: i64=1
190 if ce2*2 > (*lf2) { fits=0 } // depth-2 CE drops >= 50%
191 if (*lf2) <= 0 { fits=0 }
192 pass=pass+g_check("B: the DEPTH-2 LM TRAINS end-to-end -- CE drops >=50% (AdamW thru 2 blocks; FITS/memorizes a FIXED sequence, not generalization)" as *u8, fits); total=total+1
193
194 // ---- C: bit-exact (depth-2 trained twice) ----
195 let Wc: *i64 = build_W(D,dm,ffn,V)
196 let lfc: *i64 = sys_mmap(8) as *i64
197 let cec: i64 = stack_train(tape,vals,grads,st,Wc,D,ids,tgt,T,dm,ffn,V,scale,3000,lfc)
198 var bitexact: i64=1
199 let Eb: *i64 = Wb[0] as *i64; let Ec: *i64 = Wc[0] as *i64
200 var bz: i64=0; while bz<V*dm { if Eb[bz]!=Ec[bz] { bitexact=0 } bz=bz+1 }
201 let Wlb: *i64 = Wb[1] as *i64; let Wlc: *i64 = Wc[1] as *i64
202 bz=0; while bz<dm*V { if Wlb[bz]!=Wlc[bz] { bitexact=0 } bz=bz+1 }
203 pass=pass+g_check("C: bit-exact -- training the depth-2 LM twice gives IDENTICAL integer weights (determinism)" as *u8, bitexact); total=total+1
204
205 var okall: i64=0; if pass==total { okall=1 }
206 let logf: i64 = sys_openat_append(DLOG, 420)
207 if logf >= 0 {
208 d_ws(logf,"NOFLOATDEEPSTACK D=2 A1_Wlm=" as *u8); d_wn(logf,a1_ok); d_ws(logf," A2_E_bothblocks=" as *u8); d_wn(logf,a2_ok)
209 d_ws(logf," D=" as *u8); d_wn(logf,caught); d_ws(logf," B_fits=" as *u8); d_wn(logf,fits); d_ws(logf," CE2_start=" as *u8); d_wn(logf,*lf2); d_ws(logf," CE2_end=" as *u8); d_wn(logf,ce2); d_ws(logf," C_bitexact=" as *u8); d_wn(logf,bitexact)
210 if okall==1 { d_ws(logf," verdict=GREEN\n" as *u8) } else { d_ws(logf," verdict=RED\n" as *u8) }
211 sys_close(logf)
212 }
213 g_puts("---- nofloat_deepstack gate: passed " as *u8); g_pn(pass); g_puts(" / " as *u8); g_pn(total); g_puts(" ----\n" as *u8)
214 // MIGRATED onto nx_gate_verdict by nx_gate_dry_apply (D001, minimal form): every check
215 // row above is untouched, so the PASS/FAIL vector cannot change; only the hand-rolled
216 // verdict emission is replaced by the ONE shared base class. Proven by nx_gate_migrate verify.
217 let ctr__dry: *i64 = gv_ctr()
218 ctr__dry[0] = pass
219 ctr__dry[1] = total
220 let rc__dry: i64 = gv_verdict("NOFLOAT-DEEPSTACK-GATE" as *u8, ctr__dry, "a depth-2 transformer LM trains end-to-end in pure integer Q16 -- depth scales by composition)" as *u8)
221 sys_exit(rc__dry)
222 return rc__dry
223}