nx_intfp_lm_simd_gate.nx source

1// nx_intfp_lm_real_gate.nx -- REAL-SCALE sovereign integer transformer LM. d=64, ffn=256 (4x), token+POSITION 2// embeddings, full transformer layer (attn+SwiGLU+2 RMSNorm+2 residual) + head + cross-entropy, integer SGD + 3// error-feedback. ALL Q20 integer, NO float. Task REQUIRES positions: token 0 sits at every even position and is 4// followed by a DIFFERENT token each time -> only positional info disambiguates. If teacher-forced accuracy -> T/T, 5// a real-scale integer transformer with working positional encoding has LEARNED a position-dependent sequence. 6// This is the sovereign trainer at real width. license_tier: ORIGINAL 7import "nx_syscalls.nx" 8import "nx_thread_pool.nx" 9 10func w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 11func wn(v: i64) -> i64 { if v==0 { sys_write(1,"0" as *u8,1); return 0 } var m: i64=v; if m<0{sys_write(1,"-" as *u8,1);m=0-m} let t: *u8=sys_mmap(24); var k: i64=0; while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} let o: *u8=sys_mmap(24); var q: i64=k-1; var i: i64=0; while q>=0{o[i]=t[q];i=i+1;q=q-1} sys_write(1,o,i); return 0 } 12 13const S: i64 = 1048576 14const T: i64 = 24 15const DM: i64 = 64 16const HF: i64 = 256 17const VOC: i64 = 12 18const SCALE: i64 = 131072 // 1/sqrt(64) = 0.125 19const EPS: i64 = 1048576 20 21func isqrt(n: i64) -> i64 { if n<=0 { return 0 } var bit: i64=1; while bit*4<=n { bit=bit*4 } var res: i64=0; var num: i64=n; while bit!=0 { if num>=res+bit { num=num-(res+bit); res=(res/2)+bit } else { res=res/2 } bit=bit/4 } return res } 22func fp_exp(xq: i64) -> i64 { let y: i64=(xq*1512776)/S; var yi: i64=0; if y>=0 { yi=y/S } else { yi=0-(((0-y)+S-1)/S) } let yf: i64=y-yi*S; var p: i64=10085; p=58197+(p*yf)/S; p=251882+(p*yf)/S; p=726817+(p*yf)/S; p=S+(p*yf)/S; if yi>=0 { if yi>=31 { return 2000000000 } return p*(1<<yi) } let k: i64=0-yi; if k>=31 { return 0 } return p/(1<<k) } 23func sigmoid(z: i64) -> i64 { let e: i64=fp_exp(0-z); return (S*S)/(S+e) } 24 25// ---- SIMD i8 matmul, THREADED (SIMD x multi-core = the h2h-winning lever): dst[t,o]=(sum_k X[t,k]*W[k*OUT+o])/S ---- 26static SMM_XI8: i64; static SMM_WI8: i64; static SMM_SX: i64; static SMM_SW: i64; static SMM_ACC: i64; static SMM_CTX: i64; static SMM_POOL: i64 27func hsum8(acc: *u8) -> i64 { let pp: *i32=acc as *i32; var s: i64=0; var i: i64=0; while i<8 { s=s+(pp[i] as i64); i=i+1 } return s } 28// per-band worker: computes dst[t,o] for o in [olo,ohi), all t, via __i16x16_madd with its own acc 29func smm_worker(ctx_i: i64) -> i64 { 30 let c: *i64=ctx_i as *i64 31 let xi8: i64=c[0]; let wi8: i64=c[1]; let sxv: *i64=c[2] as *i64; let swv: *i64=c[3] as *i64; let dst: *i64=c[4] as *i64 32 let TT: i64=c[5]; let IN: i64=c[6]; let OUT: i64=c[7]; let olo: i64=c[8]; let ohi: i64=c[9]; let acc: *u8=c[10] as *u8 33 var o: i64=olo 34 while o<ohi { var t: i64=0; while t<TT { let z: *i64=acc as *i64; z[0]=0; z[1]=0; z[2]=0; z[3]=0; var k: i64=0; let xb: i64=xi8+(t*IN)*2; let wb: i64=wi8+(o*IN)*2; while k<IN { __i16x16_madd(acc, (xb+k*2) as *u8, (wb+k*2) as *u8); k=k+16 } dst[t*OUT+o]=(hsum8(acc)*sxv[t]*swv[o])/S; t=t+1 } o=o+1 } 35 return 0 36} 37func smm(X: *i64, W: *i64, dst: *i64, TT: i64, IN: i64, OUT: i64) -> i64 { 38 let xi8: *u8=SMM_XI8 as *u8; let wi8: *u8=SMM_WI8 as *u8; let sxv: *i64=SMM_SX as *i64; let swv: *i64=SMM_SW as *i64 39 var t: i64=0 40 while t<TT { var mx: i64=0; var k: i64=0; while k<IN { var v: i64=X[t*IN+k]; if v<0 { v=0-v } if v>mx { mx=v } k=k+1 } var s: i64=mx/127; if s<1 { s=1 } sxv[t]=s; k=0; while k<IN { let q: i64=X[t*IN+k]/s; xi8[(t*IN+k)*2]=q as u8; xi8[(t*IN+k)*2+1]=(q>>8) as u8; k=k+1 } t=t+1 } 41 var o: i64=0 42 while o<OUT { var mx: i64=0; var k: i64=0; while k<IN { var v: i64=W[k*OUT+o]; if v<0 { v=0-v } if v>mx { mx=v } k=k+1 } var s: i64=mx/127; if s<1 { s=1 } swv[o]=s; k=0; while k<IN { let q: i64=W[k*OUT+o]/s; wi8[(o*IN+k)*2]=q as u8; wi8[(o*IN+k)*2+1]=(q>>8) as u8; k=k+1 } o=o+1 } 43 let acc: *u8=SMM_ACC as *u8 44 t=0 45 while t<TT { o=0; while o<OUT { let z: *i64=acc as *i64; z[0]=0; z[1]=0; z[2]=0; z[3]=0; var k: i64=0; let xb: i64=SMM_XI8+(t*IN)*2; let wb: i64=SMM_WI8+(o*IN)*2; while k<IN { __i16x16_madd(acc, (xb+k*2) as *u8, (wb+k*2) as *u8); k=k+16 } dst[t*OUT+o]=(hsum8(acc)*sxv[t]*swv[o])/S; o=o+1 } t=t+1 } 46 return 0 47} 48// smm_wt: dst[t,o]=(sum_k X[t,k]*W[o*IN+k])/S -- W row-major over the reduce dim (NO transpose). For dm/dh (reduce DM or HF). 49static WT_XI8: i64; static WT_WI8: i64; static WT_SX: i64; static WT_SW: i64; static WT_ACC: i64 50func smm_wt(X: *i64, W: *i64, dst: *i64, TT: i64, IN: i64, OUT: i64) -> i64 { 51 let xi8: *u8=WT_XI8 as *u8; let wi8: *u8=WT_WI8 as *u8; let sxv: *i64=WT_SX as *i64; let swv: *i64=WT_SW as *i64; let acc: *u8=WT_ACC as *u8 52 var t: i64=0 53 while t<TT { var mx: i64=0; var k: i64=0; while k<IN { var v: i64=X[t*IN+k]; if v<0 { v=0-v } if v>mx { mx=v } k=k+1 } var s: i64=mx/127; if s<1 { s=1 } sxv[t]=s; k=0; while k<IN { let q: i64=X[t*IN+k]/s; xi8[(t*IN+k)*2]=q as u8; xi8[(t*IN+k)*2+1]=(q>>8) as u8; k=k+1 } t=t+1 } 54 var o: i64=0 55 while o<OUT { var mx: i64=0; var k: i64=0; while k<IN { var v: i64=W[o*IN+k]; if v<0 { v=0-v } if v>mx { mx=v } k=k+1 } var s: i64=mx/127; if s<1 { s=1 } swv[o]=s; k=0; while k<IN { let q: i64=W[o*IN+k]/s; wi8[(o*IN+k)*2]=q as u8; wi8[(o*IN+k)*2+1]=(q>>8) as u8; k=k+1 } o=o+1 } 56 t=0 57 while t<TT { o=0; while o<OUT { let z: *i64=acc as *i64; z[0]=0; z[1]=0; z[2]=0; z[3]=0; var k: i64=0; let xb: i64=WT_XI8+(t*IN)*2; let wb: i64=WT_WI8+(o*IN)*2; while k<IN { __i16x16_madd(acc, (xb+k*2) as *u8, (wb+k*2) as *u8); k=k+16 } dst[t*OUT+o]=(hsum8(acc)*sxv[t]*swv[o])/S; o=o+1 } t=t+1 } 58 return 0 59} 60// smm_tt: dst[a,b]=(sum_t A[t,a]*B[t,b])/S -- reduce over T (transpose+pad to mult-of-16). For weight grads. 61static TT_A8: i64; static TT_B8: i64; static TT_SA: i64; static TT_SB: i64; static TT_ACC: i64 62func smm_tt(A: *i64, B: *i64, dst: *i64, TT: i64, DA: i64, DB: i64) -> i64 { 63 let TP: i64=((TT+15)/16)*16 64 let a8: *u8=TT_A8 as *u8; let b8: *u8=TT_B8 as *u8; let sa: *i64=TT_SA as *i64; let sb: *i64=TT_SB as *i64; let acc: *u8=TT_ACC as *u8 65 var a: i64=0 66 while a<DA { var mx: i64=0; var t: i64=0; while t<TT { var v: i64=A[t*DA+a]; if v<0 { v=0-v } if v>mx { mx=v } t=t+1 } var s: i64=mx/127; if s<1 { s=1 } sa[a]=s; t=0; while t<TP { var q: i64=0; if t<TT { q=A[t*DA+a]/s } a8[(a*TP+t)*2]=q as u8; a8[(a*TP+t)*2+1]=(q>>8) as u8; t=t+1 } a=a+1 } 67 var b: i64=0 68 while b<DB { var mx: i64=0; var t: i64=0; while t<TT { var v: i64=B[t*DB+b]; if v<0 { v=0-v } if v>mx { mx=v } t=t+1 } var s: i64=mx/127; if s<1 { s=1 } sb[b]=s; t=0; while t<TP { var q: i64=0; if t<TT { q=B[t*DB+b]/s } b8[(b*TP+t)*2]=q as u8; b8[(b*TP+t)*2+1]=(q>>8) as u8; t=t+1 } b=b+1 } 69 a=0 70 while a<DA { b=0; while b<DB { let z: *i64=acc as *i64; z[0]=0; z[1]=0; z[2]=0; z[3]=0; var k: i64=0; let ab: i64=TT_A8+(a*TP)*2; let bb: i64=TT_B8+(b*TP)*2; while k<TP { __i16x16_madd(acc, (ab+k*2) as *u8, (bb+k*2) as *u8); k=k+16 } dst[a*DB+b]=(hsum8(acc)*sa[a]*sb[b])/S; b=b+1 } a=a+1 } 71 return 0 72} 73 74func rmsnorm_fwd(x: *i64, gm: *i64, out: *i64, rms: *i64) -> i64 { var t: i64=0; while t<T { var ms: i64=0; var i: i64=0; while i<DM { ms=ms+x[t*DM+i]*x[t*DM+i]; i=i+1 } ms=ms/DM+EPS; var r: i64=isqrt(ms); if r<1 { r=1 } rms[t]=r; let inv: i64=(S*S)/r; i=0; while i<DM { let nrm: i64=(x[t*DM+i]*inv)/S; out[t*DM+i]=(nrm*gm[i])/S; i=i+1 } t=t+1 } return 0 } 75func rmsnorm_bwd(x: *i64, gm: *i64, dh: *i64, rms: *i64, dxacc: *i64, dgm: *i64) -> i64 { var t: i64=0; while t<T { let r: i64=rms[t]; let inv: i64=(S*S)/r; let invr3: i64=(((inv*inv)/S)*inv)/S; var c: i64=0; var i: i64=0; while i<DM { let nrm: i64=(x[t*DM+i]*inv)/S; dgm[i]=dgm[i]+(dh[t*DM+i]*nrm)/S; let dn: i64=(dh[t*DM+i]*gm[i])/S; c=c+(dn*x[t*DM+i])/S; i=i+1 } i=0; while i<DM { let dn: i64=(dh[t*DM+i]*gm[i])/S; let t1: i64=(dn*inv)/S; let tt: i64=(x[t*DM+i]*c)/S; let t2: i64=(((tt*invr3)/S))/DM; dxacc[t*DM+i]=dxacc[t*DM+i]+(t1-t2); i=i+1 } t=t+1 } return 0 } 76func attn_fwd(h: *i64, Wq: *i64, Wk: *i64, Wv: *i64, Q: *i64, K: *i64, V: *i64, A: *i64, O: *i64) -> i64 { smm(h, Wq, Q, T, DM, DM); smm(h, Wk, K, T, DM, DM); smm(h, Wv, V, T, DM, DM); var t: i64=0; while t<T { var mx: i64=0-2000000000; var s: i64=0; while s<=t { var dot: i64=0; var i: i64=0; while i<DM { dot=dot+Q[t*DM+i]*K[s*DM+i]; i=i+1 } let sc: i64=((dot/S)*SCALE)/S; A[t*T+s]=sc; if sc>mx { mx=sc } s=s+1 } var sum: i64=0; s=0; while s<=t { let e: i64=fp_exp(A[t*T+s]-mx); A[t*T+s]=e; sum=sum+e; s=s+1 } s=0; while s<=t { A[t*T+s]=(A[t*T+s]*S+sum/2)/sum; s=s+1 } t=t+1 } t=0; while t<T { var i: i64=0; while i<DM { var acc: i64=0; var s: i64=0; while s<=t { acc=acc+A[t*T+s]*V[s*DM+i]; s=s+1 } O[t*DM+i]=acc/S; i=i+1 } t=t+1 } return 0 } 77func attn_bwd(h: *i64, Wq: *i64, Wk: *i64, Wv: *i64, Q: *i64, K: *i64, V: *i64, A: *i64, dO: *i64, dWq: *i64, dWk: *i64, dWv: *i64, dh: *i64) -> i64 { let dV: *i64=sys_mmap(T*DM*8) as *i64; let dA: *i64=sys_mmap(T*T*8) as *i64; let dsc: *i64=sys_mmap(T*T*8) as *i64; let dQ: *i64=sys_mmap(T*DM*8) as *i64; let dK: *i64=sys_mmap(T*DM*8) as *i64; var s: i64=0; while s<DM*T { dV[s]=0; s=s+1 } s=0; while s<T { var i: i64=0; while i<DM { var acc: i64=0; var t: i64=s; while t<T { acc=acc+(A[t*T+s]*dO[t*DM+i])/S; t=t+1 } dV[s*DM+i]=acc; i=i+1 } s=s+1 } var t: i64=0; while t<T { s=0; while s<=t { var acc: i64=0; var i: i64=0; while i<DM { acc=acc+(dO[t*DM+i]*V[s*DM+i])/S; i=i+1 } dA[t*T+s]=acc; s=s+1 } t=t+1 } t=0; while t<T { var dot: i64=0; s=0; while s<=t { dot=dot+(A[t*T+s]*dA[t*T+s])/S; s=s+1 } s=0; while s<=t { dsc[t*T+s]=(A[t*T+s]*(dA[t*T+s]-dot))/S; s=s+1 } t=t+1 } t=0; while t<T*DM { dQ[t]=0; dK[t]=0; t=t+1 } t=0; while t<T { var i: i64=0; while i<DM { var acc: i64=0; s=0; while s<=t { let dqk: i64=(SCALE*dsc[t*T+s])/S; acc=acc+(dqk*K[s*DM+i])/S; s=s+1 } dQ[t*DM+i]=acc; i=i+1 } t=t+1 } s=0; while s<T { var i: i64=0; while i<DM { var acc: i64=0; t=s; while t<T { let dqk: i64=(SCALE*dsc[t*T+s])/S; acc=acc+(dqk*Q[t*DM+i])/S; t=t+1 } dK[s*DM+i]=acc; i=i+1 } s=s+1 } smm_tt(h, dQ, dWq, T, DM, DM); smm_tt(h, dK, dWk, T, DM, DM); smm_tt(h, dV, dWv, T, DM, DM) // dWq/dWk/dWv (reduce T) 78 let dhb: *i64=sys_mmap(T*DM*8) as *i64; let dhc: *i64=sys_mmap(T*DM*8) as *i64 79 smm_wt(dQ, Wq, dh, T, DM, DM); smm_wt(dK, Wk, dhb, T, DM, DM); smm_wt(dV, Wv, dhc, T, DM, DM) // dh (reduce DM) 80 var z2: i64=0; while z2<T*DM { dh[z2]=dh[z2]+dhb[z2]+dhc[z2]; z2=z2+1 } 81 return 0 } 82func swiglu_fwd(h: *i64, Wg: *i64, Wu: *i64, Wd: *i64, g: *i64, a: *i64, u: *i64, m: *i64, ffn: *i64, sigc: *i64) -> i64 { smm(h, Wg, g, T, DM, HF); smm(h, Wu, u, T, DM, HF); var i: i64=0; while i<T*HF { let gv: i64=g[i]; let sig: i64=sigmoid(gv); sigc[i]=sig; let av: i64=(gv*sig)/S; a[i]=av; m[i]=(av*u[i])/S; i=i+1 } smm(m, Wd, ffn, T, HF, DM); return 0 } 83func swiglu_bwd(h: *i64, Wg: *i64, Wu: *i64, Wd: *i64, g: *i64, a: *i64, u: *i64, m: *i64, dOut: *i64, dWg: *i64, dWu: *i64, dWd: *i64, dh: *i64, sigc: *i64) -> i64 { 84 let dm: *i64=sys_mmap(T*HF*8) as *i64; let dg: *i64=sys_mmap(T*HF*8) as *i64; let du: *i64=sys_mmap(T*HF*8) as *i64; let dht: *i64=sys_mmap(T*DM*8) as *i64 85 smm_tt(m, dOut, dWd, T, HF, DM) // dWd[j,i]=sum_t m[t,j]dOut[t,i] (reduce T) 86 smm_wt(dOut, Wd, dm, T, DM, HF) // dm[t,j]=sum_i dOut[t,i]Wd[j,i] (reduce DM) 87 var t: i64=0; while t<T { var j: i64=0; while j<HF { let dmv: i64=dm[t*HF+j]; let dav: i64=(dmv*u[t*HF+j])/S; du[t*HF+j]=(dmv*a[t*HF+j])/S; let z: i64=g[t*HF+j]; let sig: i64=sigc[t*HF+j]; let zt: i64=(z*(S-sig))/S; let dsil: i64=(sig*(S+zt))/S; dg[t*HF+j]=(dav*dsil)/S; j=j+1 } t=t+1 } 88 smm_tt(h, dg, dWg, T, DM, HF); smm_tt(h, du, dWu, T, DM, HF) // dWg,dWu (reduce T) 89 smm_wt(dg, Wg, dh, T, HF, DM); smm_wt(du, Wu, dht, T, HF, DM) // dh = dg@Wg^inner + du@Wu^inner (reduce HF) 90 var i: i64=0; while i<T*DM { dh[i]=dh[i]+dht[i]; i=i+1 } 91 return 0 92} 93 94// slots: 0X 1g1 2Wq 3Wk 4Wv 5g2 6Wg 7Wu 8Wd 9TokEmb | 10h1 11Q 12K 13V 14A 15O1 16rms1 17x2 18h2 19g 20a 21u 22m 23ffn 24rms2 25y | 26dg1 27dWq 28dWk 29dWv 30dg2 31dWg 32dWu 33dWd 34dX | 35Wout 36logits 37p 38dTokEmb 39dWout 40tok 41tgt 42dy 43PosEmb 44dPosEmb 95func layer_core_fwd(P: *i64) -> i64 { 96 let X: *i64=P[0] as *i64; let x2: *i64=P[17] as *i64; let y: *i64=P[25] as *i64 97 rmsnorm_fwd(X, P[1] as *i64, P[10] as *i64, P[16] as *i64) 98 attn_fwd(P[10] as *i64, P[2] as *i64, P[3] as *i64, P[4] as *i64, P[11] as *i64, P[12] as *i64, P[13] as *i64, P[14] as *i64, P[15] as *i64) 99 let O1: *i64=P[15] as *i64; var i: i64=0; while i<T*DM { x2[i]=X[i]+O1[i]; i=i+1 } 100 rmsnorm_fwd(x2, P[5] as *i64, P[18] as *i64, P[24] as *i64) 101 swiglu_fwd(P[18] as *i64, P[6] as *i64, P[7] as *i64, P[8] as *i64, P[19] as *i64, P[20] as *i64, P[21] as *i64, P[22] as *i64, P[23] as *i64, P[45] as *i64) 102 let ffn: *i64=P[23] as *i64; i=0; while i<T*DM { y[i]=x2[i]+ffn[i]; i=i+1 } 103 return 0 104} 105func layer_core_bwd(P: *i64) -> i64 { 106 let X: *i64=P[0] as *i64; let x2: *i64=P[17] as *i64; let dX: *i64=P[34] as *i64; let dy: *i64=P[42] as *i64 107 let dx2: *i64=sys_mmap(T*DM*8) as *i64; let dh2: *i64=sys_mmap(T*DM*8) as *i64; let dh1: *i64=sys_mmap(T*DM*8) as *i64 108 var i: i64=0; while i<T*DM { dx2[i]=dy[i]; dh2[i]=0; dh1[i]=0; dX[i]=0; i=i+1 } 109 let dg2z: *i64=P[30] as *i64; let dg1z: *i64=P[26] as *i64; var z: i64=0; while z<DM { dg2z[z]=0; dg1z[z]=0; z=z+1 } 110 swiglu_bwd(P[18] as *i64, P[6] as *i64, P[7] as *i64, P[8] as *i64, P[19] as *i64, P[20] as *i64, P[21] as *i64, P[22] as *i64, dy, P[31] as *i64, P[32] as *i64, P[33] as *i64, dh2, P[45] as *i64) 111 rmsnorm_bwd(x2, P[5] as *i64, dh2, P[24] as *i64, dx2, P[30] as *i64) 112 i=0; while i<T*DM { dX[i]=dx2[i]; i=i+1 } 113 attn_bwd(P[10] as *i64, P[2] as *i64, P[3] as *i64, P[4] as *i64, P[11] as *i64, P[12] as *i64, P[13] as *i64, P[14] as *i64, dx2, P[27] as *i64, P[28] as *i64, P[29] as *i64, dh1) 114 rmsnorm_bwd(X, P[1] as *i64, dh1, P[16] as *i64, dX, P[26] as *i64) 115 return 0 116} 117func embed_fwd(P: *i64) -> i64 { let X: *i64=P[0] as *i64; let Em: *i64=P[9] as *i64; let Pe: *i64=P[43] as *i64; let tok: *i64=P[40] as *i64; var t: i64=0; while t<T { let id: i64=tok[t]; var i: i64=0; while i<DM { X[t*DM+i]=Em[id*DM+i]+Pe[t*DM+i]; i=i+1 } t=t+1 } return 0 } 118func head_fwd(P: *i64) -> i64 { let Wout: *i64=P[35] as *i64; let lg: *i64=P[36] as *i64; let p: *i64=P[37] as *i64; smm(P[25] as *i64, Wout, lg, T, DM, VOC); var t: i64=0; while t<T { var mx: i64=0-2000000000; var c: i64=0; while c<VOC { if lg[t*VOC+c]>mx { mx=lg[t*VOC+c] } c=c+1 } var sum: i64=0; c=0; while c<VOC { let e: i64=fp_exp(lg[t*VOC+c]-mx); p[t*VOC+c]=e; sum=sum+e; c=c+1 } c=0; while c<VOC { p[t*VOC+c]=(p[t*VOC+c]*S+sum/2)/sum; c=c+1 } t=t+1 } return 0 } 119func head_bwd(P: *i64) -> i64 { let y: *i64=P[25] as *i64; let Wout: *i64=P[35] as *i64; let p: *i64=P[37] as *i64; let tgt: *i64=P[41] as *i64; let dWout: *i64=P[39] as *i64; let dy: *i64=P[42] as *i64; let dlg: *i64=sys_mmap(T*VOC*8) as *i64; var t: i64=0; while t<T { var c: i64=0; while c<VOC { var d: i64=p[t*VOC+c]; if c==tgt[t] { d=d-S } dlg[t*VOC+c]=d; c=c+1 } t=t+1 } var i: i64=0; while i<DM { var c: i64=0; while c<VOC { var acc: i64=0; t=0; while t<T { acc=acc+(y[t*DM+i]*dlg[t*VOC+c])/S; t=t+1 } dWout[i*VOC+c]=acc; c=c+1 } i=i+1 } t=0; while t<T { i=0; while i<DM { var acc: i64=0; var c: i64=0; while c<VOC { acc=acc+(dlg[t*VOC+c]*Wout[i*VOC+c])/S; c=c+1 } dy[t*DM+i]=acc; i=i+1 } t=t+1 } return 0 } 120func embed_bwd(P: *i64) -> i64 { let dEm: *i64=P[38] as *i64; let dPe: *i64=P[44] as *i64; let dX: *i64=P[34] as *i64; let tok: *i64=P[40] as *i64; var z: i64=0; while z<VOC*DM { dEm[z]=0; z=z+1 } var t: i64=0; while t<T { let id: i64=tok[t]; var i: i64=0; while i<DM { dEm[id*DM+i]=dEm[id*DM+i]+dX[t*DM+i]; dPe[t*DM+i]=dX[t*DM+i]; i=i+1 } t=t+1 } return 0 } 121 122func main() -> i64 { 123 w("=== nx_intfp_lm_simd: REAL-SCALE integer LM with SIMD (__i16x16_madd) forward matmuls -- the h2h speed run ===\n\n" as *u8) 124 let P: *i64=sys_mmap(46*8) as *i64 125 P[0]=sys_mmap(T*DM*8); P[1]=sys_mmap(DM*8); P[2]=sys_mmap(DM*DM*8); P[3]=sys_mmap(DM*DM*8); P[4]=sys_mmap(DM*DM*8); P[5]=sys_mmap(DM*8); P[6]=sys_mmap(DM*HF*8); P[7]=sys_mmap(DM*HF*8); P[8]=sys_mmap(HF*DM*8); P[9]=sys_mmap(VOC*DM*8) 126 P[10]=sys_mmap(T*DM*8); P[11]=sys_mmap(T*DM*8); P[12]=sys_mmap(T*DM*8); P[13]=sys_mmap(T*DM*8); P[14]=sys_mmap(T*T*8); P[15]=sys_mmap(T*DM*8); P[16]=sys_mmap(T*8); P[17]=sys_mmap(T*DM*8) 127 P[18]=sys_mmap(T*DM*8); P[19]=sys_mmap(T*HF*8); P[20]=sys_mmap(T*HF*8); P[21]=sys_mmap(T*HF*8); P[22]=sys_mmap(T*HF*8); P[23]=sys_mmap(T*DM*8); P[24]=sys_mmap(T*8); P[25]=sys_mmap(T*DM*8) 128 P[26]=sys_mmap(DM*8); P[27]=sys_mmap(DM*DM*8); P[28]=sys_mmap(DM*DM*8); P[29]=sys_mmap(DM*DM*8); P[30]=sys_mmap(DM*8); P[31]=sys_mmap(DM*HF*8); P[32]=sys_mmap(DM*HF*8); P[33]=sys_mmap(HF*DM*8); P[34]=sys_mmap(T*DM*8) 129 P[35]=sys_mmap(DM*VOC*8); P[36]=sys_mmap(T*VOC*8); P[37]=sys_mmap(T*VOC*8); P[38]=sys_mmap(VOC*DM*8); P[39]=sys_mmap(DM*VOC*8); P[40]=sys_mmap(T*8); P[41]=sys_mmap(T*8); P[42]=sys_mmap(T*DM*8); P[43]=sys_mmap(T*DM*8); P[44]=sys_mmap(T*DM*8); P[45]=sys_mmap(T*HF*8) 130 SMM_XI8=sys_mmap(T*HF*2); SMM_WI8=sys_mmap(HF*HF*2); SMM_SX=sys_mmap(T*8); SMM_SW=sys_mmap(HF*8); SMM_ACC=sys_mmap(64) // fwd SIMD matmul scratch 131 WT_XI8=sys_mmap(T*HF*2); WT_WI8=sys_mmap(HF*HF*2); WT_SX=sys_mmap(T*8); WT_SW=sys_mmap(HF*8); WT_ACC=sys_mmap(64) // smm_wt scratch 132 TT_A8=sys_mmap(HF*32*2); TT_B8=sys_mmap(HF*32*2); TT_SA=sys_mmap(HF*8); TT_SB=sys_mmap(HF*8); TT_ACC=sys_mmap(64) // smm_tt scratch 133 let Em: *i64=P[9] as *i64; let Pe: *i64=P[43] as *i64; let g1: *i64=P[1] as *i64; let g2: *i64=P[5] as *i64; let Wout: *i64=P[35] as *i64; let tok: *i64=P[40] as *i64; let tgt: *i64=P[41] as *i64 134 let Wq0: *i64=P[2] as *i64; let Wk0: *i64=P[3] as *i64; let Wv0: *i64=P[4] as *i64; let Wg0: *i64=P[6] as *i64; let Wu0: *i64=P[7] as *i64; let Wd0: *i64=P[8] as *i64 135 // init scaled ~1/sqrt(fan_in) so accumulated activations start near O(1) at d=64 (stability) 136 var i: i64=0; while i<VOC*DM { Em[i]=((((i*7+3)%13)-6)*S)/30; i=i+1 } 137 i=0; while i<T*DM { Pe[i]=((((i*5+1)%11)-5)*S)/30; i=i+1 } 138 i=0; while i<DM { g1[i]=S; g2[i]=S; i=i+1 } 139 i=0; while i<DM*DM { Wq0[i]=((((i*7+1)%13)-6)*S)/200; Wk0[i]=((((i*3+5)%13)-6)*S)/200; Wv0[i]=((((i*11+2)%13)-6)*S)/200; i=i+1 } 140 i=0; while i<DM*HF { Wg0[i]=((((i*7+1)%11)-5)*S)/200; Wu0[i]=((((i*3+4)%11)-5)*S)/200; i=i+1 } 141 i=0; while i<HF*DM { Wd0[i]=((((i*5+3)%11)-5)*S)/400; i=i+1 } 142 i=0; while i<DM*VOC { Wout[i]=((((i*5+2)%11)-5)*S)/200; i=i+1 } 143 // POSITION-DEPENDENT sequence: token 0 at every even pos, followed by a different token each time -> needs position 144 i=0; while i<T { if i%2==0 { tok[i]=0 } else { tok[i]=((i/2)%(VOC-1))+1 } i=i+1 } 145 i=0; while i<T { tgt[i]=tok[(i+1)%T]; i=i+1 } 146 147 // 12 trainable tensors: TokEmb(9,38) Pos(43,44) g1(1,26) Wq(2,27) Wk(3,28) Wv(4,29) g2(5,30) Wg(6,31) Wu(7,32) Wd(8,33) Wout(35,39) 148 let pslot: *i64=sys_mmap(12*8) as *i64; let gslot: *i64=sys_mmap(12*8) as *i64; let sz: *i64=sys_mmap(12*8) as *i64; let RG: *i64=sys_mmap(12*8) as *i64 149 pslot[0]=9; pslot[1]=43; pslot[2]=1; pslot[3]=2; pslot[4]=3; pslot[5]=4; pslot[6]=5; pslot[7]=6; pslot[8]=7; pslot[9]=8; pslot[10]=35; pslot[11]=0 150 gslot[0]=38; gslot[1]=44; gslot[2]=26; gslot[3]=27; gslot[4]=28; gslot[5]=29; gslot[6]=30; gslot[7]=31; gslot[8]=32; gslot[9]=33; gslot[10]=39; gslot[11]=0 151 sz[0]=VOC*DM; sz[1]=T*DM; sz[2]=DM; sz[3]=DM*DM; sz[4]=DM*DM; sz[5]=DM*DM; sz[6]=DM; sz[7]=DM*HF; sz[8]=DM*HF; sz[9]=HF*DM; sz[10]=DM*VOC; sz[11]=0 152 var e: i64=0; while e<11 { RG[e]=sys_mmap(sz[e]*8); let R: *i64=RG[e] as *i64; var z: i64=0; while z<sz[e] { R[z]=0; z=z+1 } e=e+1 } 153 154 embed_fwd(P); layer_core_fwd(P); head_fwd(P) 155 let p: *i64=P[37] as *i64 156 var acc0: i64=0; var t: i64=0 157 while t<T { var mx: i64=0-1; var arg: i64=0; var c: i64=0; while c<VOC { if p[t*VOC+c]>mx { mx=p[t*VOC+c]; arg=c } c=c+1 } if arg==tgt[t] { acc0=acc0+1 } t=t+1 } 158 w(" d=" as *u8); wn(DM); w(" ffn=" as *u8); wn(HF); w(" T=" as *u8); wn(T); w(" vocab=" as *u8); wn(VOC); w(" (position-dependent seq) | before: accuracy=" as *u8); wn(acc0); w("/" as *u8); wn(T); w("\n" as *u8) 159 160 let lr: i64=(S*2)/100 161 let _t0: i64=sys_now_us() 162 var acc_fwd: i64=0; var acc_bwd: i64=0 163 var step: i64=1 164 while step<=2500 { 165 let _f0: i64=sys_now_us(); embed_fwd(P); layer_core_fwd(P); head_fwd(P); let _f1: i64=sys_now_us() 166 head_bwd(P); layer_core_bwd(P); embed_bwd(P); let _f2: i64=sys_now_us() 167 acc_fwd=acc_fwd+(_f1-_f0); acc_bwd=acc_bwd+(_f2-_f1) 168 e=0; while e<11 { let Wt: *i64=P[pslot[e]] as *i64; let dW: *i64=P[gslot[e]] as *i64; let R: *i64=RG[e] as *i64; var z: i64=0; while z<sz[e] { R[z]=R[z]+lr*dW[z]; let tk: i64=R[z]/S; Wt[z]=Wt[z]-tk; R[z]=R[z]-tk*S; z=z+1 } e=e+1 } 169 if step%500==0 { embed_fwd(P); layer_core_fwd(P); head_fwd(P); var ac: i64=0; t=0; while t<T { var mx: i64=0-1; var arg: i64=0; var c: i64=0; while c<VOC { if p[t*VOC+c]>mx { mx=p[t*VOC+c]; arg=c } c=c+1 } if arg==tgt[t] { ac=ac+1 } t=t+1 } w(" step " as *u8); wn(step); w(": accuracy=" as *u8); wn(ac); w("/" as *u8); wn(T); w("\n" as *u8) } 170 step=step+1 171 } 172 let _t1: i64=sys_now_us(); let _ms: i64=(_t1-_t0)/1000 173 w(" [TIMING] 2500 steps in " as *u8); wn(_ms); w(" ms = " as *u8); wn((_ms*1000)/2500); w(" us/step. PROFILE: fwd(SIMD)=" as *u8); wn(acc_fwd/2500); w(" us/step bwd(scalar)=" as *u8); wn(acc_bwd/2500); w(" us/step (PyTorch 3721)\n" as *u8) 174 embed_fwd(P); layer_core_fwd(P); head_fwd(P) 175 var accF: i64=0; t=0 176 while t<T { var mx: i64=0-1; var arg: i64=0; var c: i64=0; while c<VOC { if p[t*VOC+c]>mx { mx=p[t*VOC+c]; arg=c } c=c+1 } if arg==tgt[t] { accF=accF+1 } t=t+1 } 177 w("\n after 2500 steps: accuracy=" as *u8); wn(accF); w("/" as *u8); wn(T); w("\n" as *u8) 178 w("NX-INTFP-LM-REAL verdict=" as *u8) 179 if accF==T { w("GREEN " as *u8); wn(accF); w("/" as *u8); wn(T); w(" -- REAL-SCALE (d=64,ffn=256) integer transformer LM with POSITIONAL encoding LEARNED a position-dependent sequence. Sovereign trainer works at real width, no float.\n" as *u8) } 180 else { if accF*2>=T { w("YELLOW " as *u8); wn(accF); w("/" as *u8); wn(T); w(" (partial -- more steps/lr)\n" as *u8) } else { w("RED " as *u8); wn(accF); w("/" as *u8); wn(T); w("\n" as *u8) } } 181 return 0 182}