code wiki / _hdl_build / nx_intfp_lm_simd_gate.nx

nx_intfp_lm_simd_gate.nx

buildroot/runtime/_hdl_build/nx_intfp_lm_simd_gate.nx

22279 B182 linesdepth 7pulls 9 transitivereach 0 importersview sourcekind gate/prooftopic intfp
docsdependenciesstructsconstsfunctions

about

nx_intfp_lm_real_gate.nx -- REAL-SCALE sovereign integer transformer LM. d=64, ffn=256 (4x), token+POSITION embeddings, full transformer layer (attn+SwiGLU+2 RMSNorm+2 residual) + head + cross-entropy, integer SGD + error-feedback. ALL Q20 integer, NO float. Task REQUIRES positions: token 0 sits at every even position and is followed by a DIFFERENT token each time -> only positional info disambiguates. If teacher-forced accuracy -> T/T, a real-scale integer transformer with working positional encoding has LEARNED a position-dependent sequence. This is the sovereign trainer at real width. license_tier: ORIGINAL

dependencies 2 imports · 0 importers

nx_syscalls.nx nx_thread_pool.nx nx_intfp_lm_simd_gate.nx

imports: nx_syscalls.nxnx_thread_pool.nx

imported by: nobody (leaf or entry point)

call flow from main pre-order; caps 40 nodes / depth 6 declared; ↻ = already shown

main w sys_write sys_mmap embed_fwd layer_core_fwd rmsnorm_fwd isqrt attn_fwd smm hsum8 fp_exp swiglu_fwd smm ↻ sigmoid fp_exp ↻ head_fwd smm ↻ fp_exp ↻ wn sys_write ↻ sys_mmap ↻ sys_now_us sys_mmap ↻ sys_clock_gettime_mono head_bwd sys_mmap ↻ layer_core_bwd sys_mmap ↻ swiglu_bwd sys_mmap ↻ smm_tt hsum8 ↻ smm_wt hsum8 ↻ rmsnorm_bwd attn_bwd sys_mmap ↻ smm_tt ↻ smm_wt ↻

structs

none

consts

13const S: i64 = 1048576
14const T: i64 = 24
15const DM: i64 = 64
16const HF: i64 = 256
17const VOC: i64 = 12
18const SCALE: i64 = 131072 // 1/sqrt(64) = 0.125
19const EPS: i64 = 1048576

functions

10func w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
called by 1: main calls 1: sys_write
11func wn(v: i64) -> i64 { if v==0 { sys_write(1,"0" as *u8,1); return 0 } var m: i64=v; if m<0{sys_write(1,"-" as *u8,1);m=0-m} let t: *u8=sys_mmap(24); var k: i64=0; while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} let o: *u8=sys_mmap(24); var q: i64=k-1; var i: i64=0; while q>=0{o[i]=t[q];i=i+1;q=q-1} sys_write(1,o,i); return 0 }
called by 1: main calls 2: sys_writesys_mmap
21func isqrt(n: i64) -> i64 { if n<=0 { return 0 } var bit: i64=1; while bit*4<=n { bit=bit*4 } var res: i64=0; var num: i64=n; while bit!=0 { if num>=res+bit { num=num-(res+bit); res=(res/2)+bit } else { res=res/2 } bit=bit/4 } return res }
called by 1: rmsnorm_fwd
22func fp_exp(xq: i64) -> i64 { let y: i64=(xq*1512776)/S; var yi: i64=0; if y>=0 { yi=y/S } else { yi=0-(((0-y)+S-1)/S) } let yf: i64=y-yi*S; var p: i64=10085; p=58197+(p*yf)/S; p=251882+(p*yf)/S; p=726817+(p*yf)/S; p=S+(p*yf)/S; if yi>=0 { if yi>=31 { return 2000000000 } return p*(1<<yi) } let k: i64=0-yi; if k>=31 { return 0 } return p/(1<<k) }
called by 3: sigmoidattn_fwdhead_fwd
23func sigmoid(z: i64) -> i64 { let e: i64=fp_exp(0-z); return (S*S)/(S+e) }
called by 1: swiglu_fwd calls 1: fp_exp
27func hsum8(acc: *u8) -> i64 { let pp: *i32=acc as *i32; var s: i64=0; var i: i64=0; while i<8 { s=s+(pp[i] as i64); i=i+1 } return s }
called by 4: smm_workersmmsmm_wtsmm_tt
29func smm_worker(ctx_i: i64) -> i64
calls 1: hsum8
37func smm(X: *i64, W: *i64, dst: *i64, TT: i64, IN: i64, OUT: i64) -> i64
called by 3: attn_fwdswiglu_fwdhead_fwd calls 1: hsum8
50func smm_wt(X: *i64, W: *i64, dst: *i64, TT: i64, IN: i64, OUT: i64) -> i64
called by 2: attn_bwdswiglu_bwd calls 1: hsum8
62func smm_tt(A: *i64, B: *i64, dst: *i64, TT: i64, DA: i64, DB: i64) -> i64
called by 2: attn_bwdswiglu_bwd calls 1: hsum8
74func rmsnorm_fwd(x: *i64, gm: *i64, out: *i64, rms: *i64) -> i64 { var t: i64=0; while t<T { var ms: i64=0; var i: i64=0; while i<DM { ms=ms+x[t*DM+i]*x[t*DM+i]; i=i+1 } ms=ms/DM+EPS; var r: i64=isqrt(ms); if r<1 { r=1 } rms[t]=r; let inv: i64=(S*S)/r; i=0; while i<DM { let nrm: i64=(x[t*DM+i]*inv)/S; out[t*DM+i]=(nrm*gm[i])/S; i=i+1 } t=t+1 } return 0 }
called by 1: layer_core_fwd calls 1: isqrt
75func rmsnorm_bwd(x: *i64, gm: *i64, dh: *i64, rms: *i64, dxacc: *i64, dgm: *i64) -> i64 { var t: i64=0; while t<T { let r: i64=rms[t]; let inv: i64=(S*S)/r; let invr3: i64=(((inv*inv)/S)*inv)/S; var c: i64=0; var i: i64=0; while i<DM { let nrm: i64=(x[t*DM+i]*inv)/S; dgm[i]=dgm[i]+(dh[t*DM+i]*nrm)/S; let dn: i64=(dh[t*DM+i]*gm[i])/S; c=c+(dn*x[t*DM+i])/S; i=i+1 } i=0; while i<DM { let dn: i64=(dh[t*DM+i]*gm[i])/S; let t1: i64=(dn*inv)/S; let tt: i64=(x[t*DM+i]*c)/S; let t2: i64=(((tt*invr3)/S))/DM; dxacc[t*DM+i]=dxacc[t*DM+i]+(t1-t2); i=i+1 } t=t+1 } return 0 }
called by 1: layer_core_bwd
76func attn_fwd(h: *i64, Wq: *i64, Wk: *i64, Wv: *i64, Q: *i64, K: *i64, V: *i64, A: *i64, O: *i64) -> i64 { smm(h, Wq, Q, T, DM, DM); smm(h, Wk, K, T, DM, DM); smm(h, Wv, V, T, DM, DM); var t: i64=0; while t<T { var mx: i64=0-2000000000; var s: i64=0; while s<=t { var dot: i64=0; var i: i64=0; while i<DM { dot=dot+Q[t*DM+i]*K[s*DM+i]; i=i+1 } let sc: i64=((dot/S)*SCALE)/S; A[t*T+s]=sc; if sc>mx { mx=sc } s=s+1 } var sum: i64=0; s=0; while s<=t { let e: i64=fp_exp(A[t*T+s]-mx); A[t*T+s]=e; sum=sum+e; s=s+1 } s=0; while s<=t { A[t*T+s]=(A[t*T+s]*S+sum/2)/sum; s=s+1 } t=t+1 } t=0; while t<T { var i: i64=0; while i<DM { var acc: i64=0; var s: i64=0; while s<=t { acc=acc+A[t*T+s]*V[s*DM+i]; s=s+1 } O[t*DM+i]=acc/S; i=i+1 } t=t+1 } return 0 }
called by 1: layer_core_fwd calls 2: smmfp_exp
77func attn_bwd(h: *i64, Wq: *i64, Wk: *i64, Wv: *i64, Q: *i64, K: *i64, V: *i64, A: *i64, dO: *i64, dWq: *i64, dWk: *i64, dWv: *i64, dh: *i64) -> i64 { let dV: *i64=sys_mmap(T*DM*8) as *i64; let dA: *i64=sys_mmap(T*T*8) as *i64; let dsc: *i64=sys_mmap(T*T*8) as *i64; let dQ: *i64=sys_mmap(T*DM*8) as *i64; let dK: *i64=sys_mmap(T*DM*8) as *i64; var s: i64=0; while s<DM*T { dV[s]=0; s=s+1 } s=0; while s<T { var i: i64=0; while i<DM { var acc: i64=0; var t: i64=s; while t<T { acc=acc+(A[t*T+s]*dO[t*DM+i])/S; t=t+1 } dV[s*DM+i]=acc; i=i+1 } s=s+1 } var t: i64=0; while t<T { s=0; while s<=t { var acc: i64=0; var i: i64=0; while i<DM { acc=acc+(dO[t*DM+i]*V[s*DM+i])/S; i=i+1 } dA[t*T+s]=acc; s=s+1 } t=t+1 } t=0; while t<T { var dot: i64=0; s=0; while s<=t { dot=dot+(A[t*T+s]*dA[t*T+s])/S; s=s+1 } s=0; while s<=t { dsc[t*T+s]=(A[t*T+s]*(dA[t*T+s]-dot))/S; s=s+1 } t=t+1 } t=0; while t<T*DM { dQ[t]=0; dK[t]=0; t=t+1 } t=0; while t<T { var i: i64=0; while i<DM { var acc: i64=0; s=0; while s<=t { let dqk: i64=(SCALE*dsc[t*T+s])/S; acc=acc+(dqk*K[s*DM+i])/S; s=s+1 } dQ[t*DM+i]=acc; i=i+1 } t=t+1 } s=0; while s<T { var i: i64=0; while i<DM { var acc: i64=0; t=s; while t<T { let dqk: i64=(SCALE*dsc[t*T+s])/S; acc=acc+(dqk*Q[t*DM+i])/S; t=t+1 } dK[s*DM+i]=acc; i=i+1 } s=s+1 } smm_tt(h, dQ, dWq, T, DM, DM); smm_tt(h, dK, dWk, T, DM, DM); smm_tt(h, dV, dWv, T, DM, DM) // dWq/dWk/dWv (reduce T)
called by 1: layer_core_bwd calls 3: sys_mmapsmm_ttsmm_wt
82func swiglu_fwd(h: *i64, Wg: *i64, Wu: *i64, Wd: *i64, g: *i64, a: *i64, u: *i64, m: *i64, ffn: *i64, sigc: *i64) -> i64 { smm(h, Wg, g, T, DM, HF); smm(h, Wu, u, T, DM, HF); var i: i64=0; while i<T*HF { let gv: i64=g[i]; let sig: i64=sigmoid(gv); sigc[i]=sig; let av: i64=(gv*sig)/S; a[i]=av; m[i]=(av*u[i])/S; i=i+1 } smm(m, Wd, ffn, T, HF, DM); return 0 }
called by 1: layer_core_fwd calls 2: smmsigmoid
83func swiglu_bwd(h: *i64, Wg: *i64, Wu: *i64, Wd: *i64, g: *i64, a: *i64, u: *i64, m: *i64, dOut: *i64, dWg: *i64, dWu: *i64, dWd: *i64, dh: *i64, sigc: *i64) -> i64
called by 1: layer_core_bwd calls 3: sys_mmapsmm_ttsmm_wt
95func layer_core_fwd(P: *i64) -> i64
called by 1: main calls 3: rmsnorm_fwdattn_fwdswiglu_fwd
105func layer_core_bwd(P: *i64) -> i64
called by 1: main calls 4: sys_mmapswiglu_bwdrmsnorm_bwdattn_bwd
117func embed_fwd(P: *i64) -> i64 { let X: *i64=P[0] as *i64; let Em: *i64=P[9] as *i64; let Pe: *i64=P[43] as *i64; let tok: *i64=P[40] as *i64; var t: i64=0; while t<T { let id: i64=tok[t]; var i: i64=0; while i<DM { X[t*DM+i]=Em[id*DM+i]+Pe[t*DM+i]; i=i+1 } t=t+1 } return 0 }
called by 1: main
118func head_fwd(P: *i64) -> i64 { let Wout: *i64=P[35] as *i64; let lg: *i64=P[36] as *i64; let p: *i64=P[37] as *i64; smm(P[25] as *i64, Wout, lg, T, DM, VOC); var t: i64=0; while t<T { var mx: i64=0-2000000000; var c: i64=0; while c<VOC { if lg[t*VOC+c]>mx { mx=lg[t*VOC+c] } c=c+1 } var sum: i64=0; c=0; while c<VOC { let e: i64=fp_exp(lg[t*VOC+c]-mx); p[t*VOC+c]=e; sum=sum+e; c=c+1 } c=0; while c<VOC { p[t*VOC+c]=(p[t*VOC+c]*S+sum/2)/sum; c=c+1 } t=t+1 } return 0 }
called by 1: main calls 2: smmfp_exp
119func head_bwd(P: *i64) -> i64 { let y: *i64=P[25] as *i64; let Wout: *i64=P[35] as *i64; let p: *i64=P[37] as *i64; let tgt: *i64=P[41] as *i64; let dWout: *i64=P[39] as *i64; let dy: *i64=P[42] as *i64; let dlg: *i64=sys_mmap(T*VOC*8) as *i64; var t: i64=0; while t<T { var c: i64=0; while c<VOC { var d: i64=p[t*VOC+c]; if c==tgt[t] { d=d-S } dlg[t*VOC+c]=d; c=c+1 } t=t+1 } var i: i64=0; while i<DM { var c: i64=0; while c<VOC { var acc: i64=0; t=0; while t<T { acc=acc+(y[t*DM+i]*dlg[t*VOC+c])/S; t=t+1 } dWout[i*VOC+c]=acc; c=c+1 } i=i+1 } t=0; while t<T { i=0; while i<DM { var acc: i64=0; var c: i64=0; while c<VOC { acc=acc+(dlg[t*VOC+c]*Wout[i*VOC+c])/S; c=c+1 } dy[t*DM+i]=acc; i=i+1 } t=t+1 } return 0 }
called by 1: main calls 1: sys_mmap
120func embed_bwd(P: *i64) -> i64 { let dEm: *i64=P[38] as *i64; let dPe: *i64=P[44] as *i64; let dX: *i64=P[34] as *i64; let tok: *i64=P[40] as *i64; var z: i64=0; while z<VOC*DM { dEm[z]=0; z=z+1 } var t: i64=0; while t<T { let id: i64=tok[t]; var i: i64=0; while i<DM { dEm[id*DM+i]=dEm[id*DM+i]+dX[t*DM+i]; dPe[t*DM+i]=dX[t*DM+i]; i=i+1 } t=t+1 } return 0 }
called by 1: main
122func main() -> i64
calls 10: wsys_mmapembed_fwdlayer_core_fwdhead_fwdwn+4