nx_intfp_lm_real_gate.nx source

1// nx_intfp_lm_real_gate.nx -- REAL-SCALE sovereign integer transformer LM. d=64, ffn=256 (4x), token+POSITION 2// embeddings, full transformer layer (attn+SwiGLU+2 RMSNorm+2 residual) + head + cross-entropy, integer SGD + 3// error-feedback. ALL Q20 integer, NO float. Task REQUIRES positions: token 0 sits at every even position and is 4// followed by a DIFFERENT token each time -> only positional info disambiguates. If teacher-forced accuracy -> T/T, 5// a real-scale integer transformer with working positional encoding has LEARNED a position-dependent sequence. 6// This is the sovereign trainer at real width. license_tier: ORIGINAL 7import "nx_syscalls.nx" 8 9func w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 10func wn(v: i64) -> i64 { if v==0 { sys_write(1,"0" as *u8,1); return 0 } var m: i64=v; if m<0{sys_write(1,"-" as *u8,1);m=0-m} let t: *u8=sys_mmap(24); var k: i64=0; while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} let o: *u8=sys_mmap(24); var q: i64=k-1; var i: i64=0; while q>=0{o[i]=t[q];i=i+1;q=q-1} sys_write(1,o,i); return 0 } 11 12const S: i64 = 1048576 13const T: i64 = 24 14const DM: i64 = 64 15const HF: i64 = 256 16const VOC: i64 = 12 17const SCALE: i64 = 131072 // 1/sqrt(64) = 0.125 18const EPS: i64 = 1048576 19 20func isqrt(n: i64) -> i64 { if n<=0 { return 0 } var bit: i64=1; while bit*4<=n { bit=bit*4 } var res: i64=0; var num: i64=n; while bit!=0 { if num>=res+bit { num=num-(res+bit); res=(res/2)+bit } else { res=res/2 } bit=bit/4 } return res } 21func fp_exp(xq: i64) -> i64 { let y: i64=(xq*1512776)/S; var yi: i64=0; if y>=0 { yi=y/S } else { yi=0-(((0-y)+S-1)/S) } let yf: i64=y-yi*S; var p: i64=10085; p=58197+(p*yf)/S; p=251882+(p*yf)/S; p=726817+(p*yf)/S; p=S+(p*yf)/S; if yi>=0 { if yi>=31 { return 2000000000 } return p*(1<<yi) } let k: i64=0-yi; if k>=31 { return 0 } return p/(1<<k) } 22func sigmoid(z: i64) -> i64 { let e: i64=fp_exp(0-z); return (S*S)/(S+e) } 23 24func rmsnorm_fwd(x: *i64, gm: *i64, out: *i64, rms: *i64) -> i64 { var t: i64=0; while t<T { var ms: i64=0; var i: i64=0; while i<DM { ms=ms+x[t*DM+i]*x[t*DM+i]; i=i+1 } ms=ms/DM+EPS; var r: i64=isqrt(ms); if r<1 { r=1 } rms[t]=r; let inv: i64=(S*S)/r; i=0; while i<DM { let nrm: i64=(x[t*DM+i]*inv)/S; out[t*DM+i]=(nrm*gm[i])/S; i=i+1 } t=t+1 } return 0 } 25func rmsnorm_bwd(x: *i64, gm: *i64, dh: *i64, rms: *i64, dxacc: *i64, dgm: *i64) -> i64 { var t: i64=0; while t<T { let r: i64=rms[t]; let inv: i64=(S*S)/r; let invr3: i64=(((inv*inv)/S)*inv)/S; var c: i64=0; var i: i64=0; while i<DM { let nrm: i64=(x[t*DM+i]*inv)/S; dgm[i]=dgm[i]+(dh[t*DM+i]*nrm)/S; let dn: i64=(dh[t*DM+i]*gm[i])/S; c=c+(dn*x[t*DM+i])/S; i=i+1 } i=0; while i<DM { let dn: i64=(dh[t*DM+i]*gm[i])/S; let t1: i64=(dn*inv)/S; let tt: i64=(x[t*DM+i]*c)/S; let t2: i64=(((tt*invr3)/S))/DM; dxacc[t*DM+i]=dxacc[t*DM+i]+(t1-t2); i=i+1 } t=t+1 } return 0 } 26func attn_fwd(h: *i64, Wq: *i64, Wk: *i64, Wv: *i64, Q: *i64, K: *i64, V: *i64, A: *i64, O: *i64) -> i64 { var t: i64=0; while t<T { var i: i64=0; while i<DM { var aq: i64=0; var ak: i64=0; var av: i64=0; var k: i64=0; while k<DM { aq=aq+h[t*DM+k]*Wq[k*DM+i]; ak=ak+h[t*DM+k]*Wk[k*DM+i]; av=av+h[t*DM+k]*Wv[k*DM+i]; k=k+1 } Q[t*DM+i]=aq/S; K[t*DM+i]=ak/S; V[t*DM+i]=av/S; i=i+1 } t=t+1 } t=0; while t<T { var mx: i64=0-2000000000; var s: i64=0; while s<=t { var dot: i64=0; var i: i64=0; while i<DM { dot=dot+Q[t*DM+i]*K[s*DM+i]; i=i+1 } let sc: i64=((dot/S)*SCALE)/S; A[t*T+s]=sc; if sc>mx { mx=sc } s=s+1 } var sum: i64=0; s=0; while s<=t { let e: i64=fp_exp(A[t*T+s]-mx); A[t*T+s]=e; sum=sum+e; s=s+1 } s=0; while s<=t { A[t*T+s]=(A[t*T+s]*S+sum/2)/sum; s=s+1 } t=t+1 } t=0; while t<T { var i: i64=0; while i<DM { var acc: i64=0; var s: i64=0; while s<=t { acc=acc+A[t*T+s]*V[s*DM+i]; s=s+1 } O[t*DM+i]=acc/S; i=i+1 } t=t+1 } return 0 } 27func attn_bwd(h: *i64, Wq: *i64, Wk: *i64, Wv: *i64, Q: *i64, K: *i64, V: *i64, A: *i64, dO: *i64, dWq: *i64, dWk: *i64, dWv: *i64, dh: *i64) -> i64 { let dV: *i64=sys_mmap(T*DM*8) as *i64; let dA: *i64=sys_mmap(T*T*8) as *i64; let dsc: *i64=sys_mmap(T*T*8) as *i64; let dQ: *i64=sys_mmap(T*DM*8) as *i64; let dK: *i64=sys_mmap(T*DM*8) as *i64; var s: i64=0; while s<DM*T { dV[s]=0; s=s+1 } s=0; while s<T { var i: i64=0; while i<DM { var acc: i64=0; var t: i64=s; while t<T { acc=acc+(A[t*T+s]*dO[t*DM+i])/S; t=t+1 } dV[s*DM+i]=acc; i=i+1 } s=s+1 } var t: i64=0; while t<T { s=0; while s<=t { var acc: i64=0; var i: i64=0; while i<DM { acc=acc+(dO[t*DM+i]*V[s*DM+i])/S; i=i+1 } dA[t*T+s]=acc; s=s+1 } t=t+1 } t=0; while t<T { var dot: i64=0; s=0; while s<=t { dot=dot+(A[t*T+s]*dA[t*T+s])/S; s=s+1 } s=0; while s<=t { dsc[t*T+s]=(A[t*T+s]*(dA[t*T+s]-dot))/S; s=s+1 } t=t+1 } t=0; while t<T*DM { dQ[t]=0; dK[t]=0; t=t+1 } t=0; while t<T { var i: i64=0; while i<DM { var acc: i64=0; s=0; while s<=t { let dqk: i64=(SCALE*dsc[t*T+s])/S; acc=acc+(dqk*K[s*DM+i])/S; s=s+1 } dQ[t*DM+i]=acc; i=i+1 } t=t+1 } s=0; while s<T { var i: i64=0; while i<DM { var acc: i64=0; t=s; while t<T { let dqk: i64=(SCALE*dsc[t*T+s])/S; acc=acc+(dqk*Q[t*DM+i])/S; t=t+1 } dK[s*DM+i]=acc; i=i+1 } s=s+1 } var kk: i64=0; while kk<DM { var i: i64=0; while i<DM { var aq: i64=0; var ak: i64=0; var av: i64=0; t=0; while t<T { aq=aq+(h[t*DM+kk]*dQ[t*DM+i])/S; ak=ak+(h[t*DM+kk]*dK[t*DM+i])/S; av=av+(h[t*DM+kk]*dV[t*DM+i])/S; t=t+1 } dWq[kk*DM+i]=aq; dWk[kk*DM+i]=ak; dWv[kk*DM+i]=av; i=i+1 } kk=kk+1 } t=0; while t<T { var k2: i64=0; while k2<DM { var acc: i64=0; var i: i64=0; while i<DM { acc=acc+(dQ[t*DM+i]*Wq[k2*DM+i])/S+(dK[t*DM+i]*Wk[k2*DM+i])/S+(dV[t*DM+i]*Wv[k2*DM+i])/S; i=i+1 } dh[t*DM+k2]=acc; k2=k2+1 } t=t+1 } return 0 } 28func swiglu_fwd(h: *i64, Wg: *i64, Wu: *i64, Wd: *i64, g: *i64, a: *i64, u: *i64, m: *i64, ffn: *i64) -> i64 { var t: i64=0; while t<T { var j: i64=0; while j<HF { var gg: i64=0; var uu: i64=0; var k: i64=0; while k<DM { gg=gg+h[t*DM+k]*Wg[k*HF+j]; uu=uu+h[t*DM+k]*Wu[k*HF+j]; k=k+1 } let gv: i64=gg/S; g[t*HF+j]=gv; u[t*HF+j]=uu/S; let sig: i64=sigmoid(gv); let av: i64=(gv*sig)/S; a[t*HF+j]=av; m[t*HF+j]=(av*u[t*HF+j])/S; j=j+1 } t=t+1 } t=0; while t<T { var i: i64=0; while i<DM { var acc: i64=0; var j: i64=0; while j<HF { acc=acc+m[t*HF+j]*Wd[j*DM+i]; j=j+1 } ffn[t*DM+i]=acc/S; i=i+1 } t=t+1 } return 0 } 29func swiglu_bwd(h: *i64, Wg: *i64, Wu: *i64, Wd: *i64, g: *i64, a: *i64, u: *i64, m: *i64, dOut: *i64, dWg: *i64, dWu: *i64, dWd: *i64, dh: *i64) -> i64 { let dm: *i64=sys_mmap(T*HF*8) as *i64; let dg: *i64=sys_mmap(T*HF*8) as *i64; let du: *i64=sys_mmap(T*HF*8) as *i64; var j: i64=0; while j<HF { var i: i64=0; while i<DM { var acc: i64=0; var t: i64=0; while t<T { acc=acc+(m[t*HF+j]*dOut[t*DM+i])/S; t=t+1 } dWd[j*DM+i]=acc; i=i+1 } j=j+1 } var t: i64=0; while t<T { j=0; while j<HF { var acc: i64=0; var i: i64=0; while i<DM { acc=acc+(dOut[t*DM+i]*Wd[j*DM+i])/S; i=i+1 } dm[t*HF+j]=acc; j=j+1 } t=t+1 } t=0; while t<T { j=0; while j<HF { let dmv: i64=dm[t*HF+j]; let dav: i64=(dmv*u[t*HF+j])/S; du[t*HF+j]=(dmv*a[t*HF+j])/S; let z: i64=g[t*HF+j]; let sig: i64=sigmoid(z); let zt: i64=(z*(S-sig))/S; let dsil: i64=(sig*(S+zt))/S; dg[t*HF+j]=(dav*dsil)/S; j=j+1 } t=t+1 } var k: i64=0; while k<DM { j=0; while j<HF { var ag: i64=0; var au: i64=0; t=0; while t<T { ag=ag+(h[t*DM+k]*dg[t*HF+j])/S; au=au+(h[t*DM+k]*du[t*HF+j])/S; t=t+1 } dWg[k*HF+j]=ag; dWu[k*HF+j]=au; j=j+1 } k=k+1 } t=0; while t<T { k=0; while k<DM { var acc: i64=0; j=0; while j<HF { acc=acc+(dg[t*HF+j]*Wg[k*HF+j])/S+(du[t*HF+j]*Wu[k*HF+j])/S; j=j+1 } dh[t*DM+k]=acc; k=k+1 } t=t+1 } return 0 } 30 31// slots: 0X 1g1 2Wq 3Wk 4Wv 5g2 6Wg 7Wu 8Wd 9TokEmb | 10h1 11Q 12K 13V 14A 15O1 16rms1 17x2 18h2 19g 20a 21u 22m 23ffn 24rms2 25y | 26dg1 27dWq 28dWk 29dWv 30dg2 31dWg 32dWu 33dWd 34dX | 35Wout 36logits 37p 38dTokEmb 39dWout 40tok 41tgt 42dy 43PosEmb 44dPosEmb 32func layer_core_fwd(P: *i64) -> i64 { 33 let X: *i64=P[0] as *i64; let x2: *i64=P[17] as *i64; let y: *i64=P[25] as *i64 34 rmsnorm_fwd(X, P[1] as *i64, P[10] as *i64, P[16] as *i64) 35 attn_fwd(P[10] as *i64, P[2] as *i64, P[3] as *i64, P[4] as *i64, P[11] as *i64, P[12] as *i64, P[13] as *i64, P[14] as *i64, P[15] as *i64) 36 let O1: *i64=P[15] as *i64; var i: i64=0; while i<T*DM { x2[i]=X[i]+O1[i]; i=i+1 } 37 rmsnorm_fwd(x2, P[5] as *i64, P[18] as *i64, P[24] as *i64) 38 swiglu_fwd(P[18] as *i64, P[6] as *i64, P[7] as *i64, P[8] as *i64, P[19] as *i64, P[20] as *i64, P[21] as *i64, P[22] as *i64, P[23] as *i64) 39 let ffn: *i64=P[23] as *i64; i=0; while i<T*DM { y[i]=x2[i]+ffn[i]; i=i+1 } 40 return 0 41} 42func layer_core_bwd(P: *i64) -> i64 { 43 let X: *i64=P[0] as *i64; let x2: *i64=P[17] as *i64; let dX: *i64=P[34] as *i64; let dy: *i64=P[42] as *i64 44 let dx2: *i64=sys_mmap(T*DM*8) as *i64; let dh2: *i64=sys_mmap(T*DM*8) as *i64; let dh1: *i64=sys_mmap(T*DM*8) as *i64 45 var i: i64=0; while i<T*DM { dx2[i]=dy[i]; dh2[i]=0; dh1[i]=0; dX[i]=0; i=i+1 } 46 let dg2z: *i64=P[30] as *i64; let dg1z: *i64=P[26] as *i64; var z: i64=0; while z<DM { dg2z[z]=0; dg1z[z]=0; z=z+1 } 47 swiglu_bwd(P[18] as *i64, P[6] as *i64, P[7] as *i64, P[8] as *i64, P[19] as *i64, P[20] as *i64, P[21] as *i64, P[22] as *i64, dy, P[31] as *i64, P[32] as *i64, P[33] as *i64, dh2) 48 rmsnorm_bwd(x2, P[5] as *i64, dh2, P[24] as *i64, dx2, P[30] as *i64) 49 i=0; while i<T*DM { dX[i]=dx2[i]; i=i+1 } 50 attn_bwd(P[10] as *i64, P[2] as *i64, P[3] as *i64, P[4] as *i64, P[11] as *i64, P[12] as *i64, P[13] as *i64, P[14] as *i64, dx2, P[27] as *i64, P[28] as *i64, P[29] as *i64, dh1) 51 rmsnorm_bwd(X, P[1] as *i64, dh1, P[16] as *i64, dX, P[26] as *i64) 52 return 0 53} 54func embed_fwd(P: *i64) -> i64 { let X: *i64=P[0] as *i64; let Em: *i64=P[9] as *i64; let Pe: *i64=P[43] as *i64; let tok: *i64=P[40] as *i64; var t: i64=0; while t<T { let id: i64=tok[t]; var i: i64=0; while i<DM { X[t*DM+i]=Em[id*DM+i]+Pe[t*DM+i]; i=i+1 } t=t+1 } return 0 } 55func head_fwd(P: *i64) -> i64 { let y: *i64=P[25] as *i64; let Wout: *i64=P[35] as *i64; let lg: *i64=P[36] as *i64; let p: *i64=P[37] as *i64; var t: i64=0; while t<T { var mx: i64=0-2000000000; var c: i64=0; while c<VOC { var acc: i64=0; var i: i64=0; while i<DM { acc=acc+y[t*DM+i]*Wout[i*VOC+c]; i=i+1 } let lv: i64=acc/S; lg[t*VOC+c]=lv; if lv>mx { mx=lv } c=c+1 } var sum: i64=0; c=0; while c<VOC { let e: i64=fp_exp(lg[t*VOC+c]-mx); p[t*VOC+c]=e; sum=sum+e; c=c+1 } c=0; while c<VOC { p[t*VOC+c]=(p[t*VOC+c]*S+sum/2)/sum; c=c+1 } t=t+1 } return 0 } 56func head_bwd(P: *i64) -> i64 { let y: *i64=P[25] as *i64; let Wout: *i64=P[35] as *i64; let p: *i64=P[37] as *i64; let tgt: *i64=P[41] as *i64; let dWout: *i64=P[39] as *i64; let dy: *i64=P[42] as *i64; let dlg: *i64=sys_mmap(T*VOC*8) as *i64; var t: i64=0; while t<T { var c: i64=0; while c<VOC { var d: i64=p[t*VOC+c]; if c==tgt[t] { d=d-S } dlg[t*VOC+c]=d; c=c+1 } t=t+1 } var i: i64=0; while i<DM { var c: i64=0; while c<VOC { var acc: i64=0; t=0; while t<T { acc=acc+(y[t*DM+i]*dlg[t*VOC+c])/S; t=t+1 } dWout[i*VOC+c]=acc; c=c+1 } i=i+1 } t=0; while t<T { i=0; while i<DM { var acc: i64=0; var c: i64=0; while c<VOC { acc=acc+(dlg[t*VOC+c]*Wout[i*VOC+c])/S; c=c+1 } dy[t*DM+i]=acc; i=i+1 } t=t+1 } return 0 } 57func embed_bwd(P: *i64) -> i64 { let dEm: *i64=P[38] as *i64; let dPe: *i64=P[44] as *i64; let dX: *i64=P[34] as *i64; let tok: *i64=P[40] as *i64; var z: i64=0; while z<VOC*DM { dEm[z]=0; z=z+1 } var t: i64=0; while t<T { let id: i64=tok[t]; var i: i64=0; while i<DM { dEm[id*DM+i]=dEm[id*DM+i]+dX[t*DM+i]; dPe[t*DM+i]=dX[t*DM+i]; i=i+1 } t=t+1 } return 0 } 58 59func main() -> i64 { 60 w("=== nx_intfp_lm_real: REAL-SCALE integer transformer LM d=64 ffn=256, token+POSITION embeddings, Q20 no-float ===\n\n" as *u8) 61 let P: *i64=sys_mmap(46*8) as *i64 62 P[0]=sys_mmap(T*DM*8); P[1]=sys_mmap(DM*8); P[2]=sys_mmap(DM*DM*8); P[3]=sys_mmap(DM*DM*8); P[4]=sys_mmap(DM*DM*8); P[5]=sys_mmap(DM*8); P[6]=sys_mmap(DM*HF*8); P[7]=sys_mmap(DM*HF*8); P[8]=sys_mmap(HF*DM*8); P[9]=sys_mmap(VOC*DM*8) 63 P[10]=sys_mmap(T*DM*8); P[11]=sys_mmap(T*DM*8); P[12]=sys_mmap(T*DM*8); P[13]=sys_mmap(T*DM*8); P[14]=sys_mmap(T*T*8); P[15]=sys_mmap(T*DM*8); P[16]=sys_mmap(T*8); P[17]=sys_mmap(T*DM*8) 64 P[18]=sys_mmap(T*DM*8); P[19]=sys_mmap(T*HF*8); P[20]=sys_mmap(T*HF*8); P[21]=sys_mmap(T*HF*8); P[22]=sys_mmap(T*HF*8); P[23]=sys_mmap(T*DM*8); P[24]=sys_mmap(T*8); P[25]=sys_mmap(T*DM*8) 65 P[26]=sys_mmap(DM*8); P[27]=sys_mmap(DM*DM*8); P[28]=sys_mmap(DM*DM*8); P[29]=sys_mmap(DM*DM*8); P[30]=sys_mmap(DM*8); P[31]=sys_mmap(DM*HF*8); P[32]=sys_mmap(DM*HF*8); P[33]=sys_mmap(HF*DM*8); P[34]=sys_mmap(T*DM*8) 66 P[35]=sys_mmap(DM*VOC*8); P[36]=sys_mmap(T*VOC*8); P[37]=sys_mmap(T*VOC*8); P[38]=sys_mmap(VOC*DM*8); P[39]=sys_mmap(DM*VOC*8); P[40]=sys_mmap(T*8); P[41]=sys_mmap(T*8); P[42]=sys_mmap(T*DM*8); P[43]=sys_mmap(T*DM*8); P[44]=sys_mmap(T*DM*8) 67 let Em: *i64=P[9] as *i64; let Pe: *i64=P[43] as *i64; let g1: *i64=P[1] as *i64; let g2: *i64=P[5] as *i64; let Wout: *i64=P[35] as *i64; let tok: *i64=P[40] as *i64; let tgt: *i64=P[41] as *i64 68 let Wq0: *i64=P[2] as *i64; let Wk0: *i64=P[3] as *i64; let Wv0: *i64=P[4] as *i64; let Wg0: *i64=P[6] as *i64; let Wu0: *i64=P[7] as *i64; let Wd0: *i64=P[8] as *i64 69 // init scaled ~1/sqrt(fan_in) so accumulated activations start near O(1) at d=64 (stability) 70 var i: i64=0; while i<VOC*DM { Em[i]=((((i*7+3)%13)-6)*S)/30; i=i+1 } 71 i=0; while i<T*DM { Pe[i]=((((i*5+1)%11)-5)*S)/30; i=i+1 } 72 i=0; while i<DM { g1[i]=S; g2[i]=S; i=i+1 } 73 i=0; while i<DM*DM { Wq0[i]=((((i*7+1)%13)-6)*S)/200; Wk0[i]=((((i*3+5)%13)-6)*S)/200; Wv0[i]=((((i*11+2)%13)-6)*S)/200; i=i+1 } 74 i=0; while i<DM*HF { Wg0[i]=((((i*7+1)%11)-5)*S)/200; Wu0[i]=((((i*3+4)%11)-5)*S)/200; i=i+1 } 75 i=0; while i<HF*DM { Wd0[i]=((((i*5+3)%11)-5)*S)/400; i=i+1 } 76 i=0; while i<DM*VOC { Wout[i]=((((i*5+2)%11)-5)*S)/200; i=i+1 } 77 // POSITION-DEPENDENT sequence: token 0 at every even pos, followed by a different token each time -> needs position 78 i=0; while i<T { if i%2==0 { tok[i]=0 } else { tok[i]=((i/2)%(VOC-1))+1 } i=i+1 } 79 i=0; while i<T { tgt[i]=tok[(i+1)%T]; i=i+1 } 80 81 // 12 trainable tensors: TokEmb(9,38) Pos(43,44) g1(1,26) Wq(2,27) Wk(3,28) Wv(4,29) g2(5,30) Wg(6,31) Wu(7,32) Wd(8,33) Wout(35,39) 82 let pslot: *i64=sys_mmap(12*8) as *i64; let gslot: *i64=sys_mmap(12*8) as *i64; let sz: *i64=sys_mmap(12*8) as *i64; let RG: *i64=sys_mmap(12*8) as *i64 83 pslot[0]=9; pslot[1]=43; pslot[2]=1; pslot[3]=2; pslot[4]=3; pslot[5]=4; pslot[6]=5; pslot[7]=6; pslot[8]=7; pslot[9]=8; pslot[10]=35; pslot[11]=0 84 gslot[0]=38; gslot[1]=44; gslot[2]=26; gslot[3]=27; gslot[4]=28; gslot[5]=29; gslot[6]=30; gslot[7]=31; gslot[8]=32; gslot[9]=33; gslot[10]=39; gslot[11]=0 85 sz[0]=VOC*DM; sz[1]=T*DM; sz[2]=DM; sz[3]=DM*DM; sz[4]=DM*DM; sz[5]=DM*DM; sz[6]=DM; sz[7]=DM*HF; sz[8]=DM*HF; sz[9]=HF*DM; sz[10]=DM*VOC; sz[11]=0 86 var e: i64=0; while e<11 { RG[e]=sys_mmap(sz[e]*8); let R: *i64=RG[e] as *i64; var z: i64=0; while z<sz[e] { R[z]=0; z=z+1 } e=e+1 } 87 88 embed_fwd(P); layer_core_fwd(P); head_fwd(P) 89 let p: *i64=P[37] as *i64 90 var acc0: i64=0; var t: i64=0 91 while t<T { var mx: i64=0-1; var arg: i64=0; var c: i64=0; while c<VOC { if p[t*VOC+c]>mx { mx=p[t*VOC+c]; arg=c } c=c+1 } if arg==tgt[t] { acc0=acc0+1 } t=t+1 } 92 w(" d=" as *u8); wn(DM); w(" ffn=" as *u8); wn(HF); w(" T=" as *u8); wn(T); w(" vocab=" as *u8); wn(VOC); w(" (position-dependent seq) | before: accuracy=" as *u8); wn(acc0); w("/" as *u8); wn(T); w("\n" as *u8) 93 94 let lr: i64=(S*2)/100 95 let _t0: i64=sys_now_us() 96 var step: i64=1 97 while step<=2500 { 98 embed_fwd(P); layer_core_fwd(P); head_fwd(P) 99 head_bwd(P); layer_core_bwd(P); embed_bwd(P) 100 e=0; while e<11 { let Wt: *i64=P[pslot[e]] as *i64; let dW: *i64=P[gslot[e]] as *i64; let R: *i64=RG[e] as *i64; var z: i64=0; while z<sz[e] { R[z]=R[z]+lr*dW[z]; let tk: i64=R[z]/S; Wt[z]=Wt[z]-tk; R[z]=R[z]-tk*S; z=z+1 } e=e+1 } 101 if step%500==0 { embed_fwd(P); layer_core_fwd(P); head_fwd(P); var ac: i64=0; t=0; while t<T { var mx: i64=0-1; var arg: i64=0; var c: i64=0; while c<VOC { if p[t*VOC+c]>mx { mx=p[t*VOC+c]; arg=c } c=c+1 } if arg==tgt[t] { ac=ac+1 } t=t+1 } w(" step " as *u8); wn(step); w(": accuracy=" as *u8); wn(ac); w("/" as *u8); wn(T); w("\n" as *u8) } 102 step=step+1 103 } 104 let _t1: i64=sys_now_us(); let _ms: i64=(_t1-_t0)/1000 105 w(" [TIMING] 2500 steps in " as *u8); wn(_ms); w(" ms = " as *u8); wn((_ms*1000)/2500); w(" us/step (d=64 ffn=256 1-layer, scalar integer) <- h2h vs PyTorch\n" as *u8) 106 embed_fwd(P); layer_core_fwd(P); head_fwd(P) 107 var accF: i64=0; t=0 108 while t<T { var mx: i64=0-1; var arg: i64=0; var c: i64=0; while c<VOC { if p[t*VOC+c]>mx { mx=p[t*VOC+c]; arg=c } c=c+1 } if arg==tgt[t] { accF=accF+1 } t=t+1 } 109 w("\n after 2500 steps: accuracy=" as *u8); wn(accF); w("/" as *u8); wn(T); w("\n" as *u8) 110 w("NX-INTFP-LM-REAL verdict=" as *u8) 111 if accF==T { w("GREEN " as *u8); wn(accF); w("/" as *u8); wn(T); w(" -- REAL-SCALE (d=64,ffn=256) integer transformer LM with POSITIONAL encoding LEARNED a position-dependent sequence. Sovereign trainer works at real width, no float.\n" as *u8) } 112 else { if accF*2>=T { w("YELLOW " as *u8); wn(accF); w("/" as *u8); wn(T); w(" (partial -- more steps/lr)\n" as *u8) } else { w("RED " as *u8); wn(accF); w("/" as *u8); wn(T); w("\n" as *u8) } } 113 return 0 114}