nx_intfp_layer_gradcheck_gate.nx source

1// nx_intfp_layer_gradcheck_gate.nx -- FULL integer transformer LAYER: two pre-norm residual blocks stacked 2// x2 = x + Attn(RMSNorm1(x)) [block 1] 3// y = x2 + SwiGLU(RMSNorm2(x2)) [block 2] 4// with the COMPLETE end-to-end integer backward (dX chains block2 -> block1 through the residual stream), Q20, no 5// float. Gradchecked (integer finite-diff, max-|grad| metric) across BOTH blocks (g1,Wq,g2,Wg,Wd) and the input X, 6// then TRAINED (loss drops). This is the whole layer the real model stacks. Modular fns to fit the 16-arg budget. 7// license_tier: ORIGINAL 8import "nx_syscalls.nx" 9 10func w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 11func wn(v: i64) -> i64 { if v==0 { sys_write(1,"0" as *u8,1); return 0 } var m: i64=v; if m<0{sys_write(1,"-" as *u8,1);m=0-m} let t: *u8=sys_mmap(24); var k: i64=0; while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} let o: *u8=sys_mmap(24); var q: i64=k-1; var i: i64=0; while q>=0{o[i]=t[q];i=i+1;q=q-1} sys_write(1,o,i); return 0 } 12func iabs(v: i64) -> i64 { if v<0 { return 0-v } return v } 13 14const S: i64 = 1048576 15const T: i64 = 3 16const DM: i64 = 4 17const HF: i64 = 6 18const SCALE: i64 = 524288 19const EPS: i64 = 1048576 20 21func isqrt(n: i64) -> i64 { if n<=0 { return 0 } var bit: i64=1; while bit*4<=n { bit=bit*4 } var res: i64=0; var num: i64=n; while bit!=0 { if num>=res+bit { num=num-(res+bit); res=(res/2)+bit } else { res=res/2 } bit=bit/4 } return res } 22func fp_exp(xq: i64) -> i64 { let y: i64=(xq*1512776)/S; var yi: i64=0; if y>=0 { yi=y/S } else { yi=0-(((0-y)+S-1)/S) } let yf: i64=y-yi*S; var p: i64=10085; p=58197+(p*yf)/S; p=251882+(p*yf)/S; p=726817+(p*yf)/S; p=S+(p*yf)/S; if yi>=0 { if yi>=31 { return 2000000000 } return p*(1<<yi) } let k: i64=0-yi; if k>=31 { return 0 } return p/(1<<k) } 23func sigmoid(z: i64) -> i64 { let e: i64=fp_exp(0-z); return (S*S)/(S+e) } 24 25func rmsnorm_fwd(x: *i64, gm: *i64, out: *i64, rms: *i64) -> i64 { 26 var t: i64=0 27 while t<T { var ms: i64=0; var i: i64=0; while i<DM { ms=ms+x[t*DM+i]*x[t*DM+i]; i=i+1 } ms=ms/DM+EPS; let r: i64=isqrt(ms); rms[t]=r; let inv: i64=(S*S)/r 28 i=0; while i<DM { let nrm: i64=(x[t*DM+i]*inv)/S; out[t*DM+i]=(nrm*gm[i])/S; i=i+1 } t=t+1 } 29 return 0 30} 31// rmsnorm backward: dh -> ADD norm-path into dxacc, and dgamma (accumulate) 32func rmsnorm_bwd(x: *i64, gm: *i64, dh: *i64, rms: *i64, dxacc: *i64, dgm: *i64) -> i64 { 33 var t: i64=0 34 while t<T { 35 let r: i64=rms[t]; let inv: i64=(S*S)/r; let invr3: i64=(((inv*inv)/S)*inv)/S 36 var c: i64=0; var i: i64=0 37 while i<DM { let nrm: i64=(x[t*DM+i]*inv)/S; dgm[i]=dgm[i]+(dh[t*DM+i]*nrm)/S; let dn: i64=(dh[t*DM+i]*gm[i])/S; c=c+(dn*x[t*DM+i])/S; i=i+1 } 38 i=0; while i<DM { let dn: i64=(dh[t*DM+i]*gm[i])/S; let t1: i64=(dn*inv)/S; let tt: i64=(x[t*DM+i]*c)/S; let t2: i64=(((tt*invr3)/S))/DM; dxacc[t*DM+i]=dxacc[t*DM+i]+(t1-t2); i=i+1 } 39 t=t+1 40 } 41 return 0 42} 43func attn_fwd(h: *i64, Wq: *i64, Wk: *i64, Wv: *i64, Q: *i64, K: *i64, V: *i64, A: *i64, O: *i64) -> i64 { 44 var t: i64=0 45 while t<T { var i: i64=0; while i<DM { var aq: i64=0; var ak: i64=0; var av: i64=0; var k: i64=0; while k<DM { aq=aq+h[t*DM+k]*Wq[k*DM+i]; ak=ak+h[t*DM+k]*Wk[k*DM+i]; av=av+h[t*DM+k]*Wv[k*DM+i]; k=k+1 } Q[t*DM+i]=aq/S; K[t*DM+i]=ak/S; V[t*DM+i]=av/S; i=i+1 } t=t+1 } 46 t=0 47 while t<T { var mx: i64=0-2000000000; var s: i64=0; while s<=t { var dot: i64=0; var i: i64=0; while i<DM { dot=dot+Q[t*DM+i]*K[s*DM+i]; i=i+1 } let sc: i64=((dot/S)*SCALE)/S; A[t*T+s]=sc; if sc>mx { mx=sc } s=s+1 } var sum: i64=0; s=0; while s<=t { let e: i64=fp_exp(A[t*T+s]-mx); A[t*T+s]=e; sum=sum+e; s=s+1 } s=0; while s<=t { A[t*T+s]=(A[t*T+s]*S+sum/2)/sum; s=s+1 } t=t+1 } 48 t=0; while t<T { var i: i64=0; while i<DM { var acc: i64=0; var s: i64=0; while s<=t { acc=acc+A[t*T+s]*V[s*DM+i]; s=s+1 } O[t*DM+i]=acc/S; i=i+1 } t=t+1 } 49 return 0 50} 51// attention backward: dO -> dWq,dWk,dWv and dh (input grad) 52func attn_bwd(h: *i64, Wq: *i64, Wk: *i64, Wv: *i64, Q: *i64, K: *i64, V: *i64, A: *i64, dO: *i64, dWq: *i64, dWk: *i64, dWv: *i64, dh: *i64) -> i64 { 53 let dV: *i64=sys_mmap(T*DM*8) as *i64; let dA: *i64=sys_mmap(T*T*8) as *i64; let dsc: *i64=sys_mmap(T*T*8) as *i64; let dQ: *i64=sys_mmap(T*DM*8) as *i64; let dK: *i64=sys_mmap(T*DM*8) as *i64 54 var s: i64=0; while s<DM*T { dV[s]=0; s=s+1 } 55 s=0; while s<T { var i: i64=0; while i<DM { var acc: i64=0; var t: i64=s; while t<T { acc=acc+(A[t*T+s]*dO[t*DM+i])/S; t=t+1 } dV[s*DM+i]=acc; i=i+1 } s=s+1 } 56 var t: i64=0; while t<T { s=0; while s<=t { var acc: i64=0; var i: i64=0; while i<DM { acc=acc+(dO[t*DM+i]*V[s*DM+i])/S; i=i+1 } dA[t*T+s]=acc; s=s+1 } t=t+1 } 57 t=0; while t<T { var dot: i64=0; s=0; while s<=t { dot=dot+(A[t*T+s]*dA[t*T+s])/S; s=s+1 } s=0; while s<=t { dsc[t*T+s]=(A[t*T+s]*(dA[t*T+s]-dot))/S; s=s+1 } t=t+1 } 58 t=0; while t<T*DM { dQ[t]=0; dK[t]=0; t=t+1 } 59 t=0; while t<T { var i: i64=0; while i<DM { var acc: i64=0; s=0; while s<=t { let dqk: i64=(SCALE*dsc[t*T+s])/S; acc=acc+(dqk*K[s*DM+i])/S; s=s+1 } dQ[t*DM+i]=acc; i=i+1 } t=t+1 } 60 s=0; while s<T { var i: i64=0; while i<DM { var acc: i64=0; t=s; while t<T { let dqk: i64=(SCALE*dsc[t*T+s])/S; acc=acc+(dqk*Q[t*DM+i])/S; t=t+1 } dK[s*DM+i]=acc; i=i+1 } s=s+1 } 61 var kk: i64=0; while kk<DM { var i: i64=0; while i<DM { var aq: i64=0; var ak: i64=0; var av: i64=0; t=0; while t<T { aq=aq+(h[t*DM+kk]*dQ[t*DM+i])/S; ak=ak+(h[t*DM+kk]*dK[t*DM+i])/S; av=av+(h[t*DM+kk]*dV[t*DM+i])/S; t=t+1 } dWq[kk*DM+i]=aq; dWk[kk*DM+i]=ak; dWv[kk*DM+i]=av; i=i+1 } kk=kk+1 } 62 t=0; while t<T { var k2: i64=0; while k2<DM { var acc: i64=0; var i: i64=0; while i<DM { acc=acc+(dQ[t*DM+i]*Wq[k2*DM+i])/S+(dK[t*DM+i]*Wk[k2*DM+i])/S+(dV[t*DM+i]*Wv[k2*DM+i])/S; i=i+1 } dh[t*DM+k2]=acc; k2=k2+1 } t=t+1 } 63 return 0 64} 65func swiglu_fwd(h: *i64, Wg: *i64, Wu: *i64, Wd: *i64, g: *i64, a: *i64, u: *i64, m: *i64, ffn: *i64) -> i64 { 66 var t: i64=0 67 while t<T { var j: i64=0; while j<HF { var gg: i64=0; var uu: i64=0; var k: i64=0; while k<DM { gg=gg+h[t*DM+k]*Wg[k*HF+j]; uu=uu+h[t*DM+k]*Wu[k*HF+j]; k=k+1 } let gv: i64=gg/S; g[t*HF+j]=gv; u[t*HF+j]=uu/S; let sig: i64=sigmoid(gv); let av: i64=(gv*sig)/S; a[t*HF+j]=av; m[t*HF+j]=(av*u[t*HF+j])/S; j=j+1 } t=t+1 } 68 t=0; while t<T { var i: i64=0; while i<DM { var acc: i64=0; var j: i64=0; while j<HF { acc=acc+m[t*HF+j]*Wd[j*DM+i]; j=j+1 } ffn[t*DM+i]=acc/S; i=i+1 } t=t+1 } 69 return 0 70} 71// swiglu backward: dOut -> dWg,dWu,dWd and dh (input grad) 72func swiglu_bwd(h: *i64, Wg: *i64, Wu: *i64, Wd: *i64, g: *i64, a: *i64, u: *i64, m: *i64, dOut: *i64, dWg: *i64, dWu: *i64, dWd: *i64, dh: *i64) -> i64 { 73 let dm: *i64=sys_mmap(T*HF*8) as *i64; let dg: *i64=sys_mmap(T*HF*8) as *i64; let du: *i64=sys_mmap(T*HF*8) as *i64 74 var j: i64=0; while j<HF { var i: i64=0; while i<DM { var acc: i64=0; var t: i64=0; while t<T { acc=acc+(m[t*HF+j]*dOut[t*DM+i])/S; t=t+1 } dWd[j*DM+i]=acc; i=i+1 } j=j+1 } 75 var t: i64=0; while t<T { j=0; while j<HF { var acc: i64=0; var i: i64=0; while i<DM { acc=acc+(dOut[t*DM+i]*Wd[j*DM+i])/S; i=i+1 } dm[t*HF+j]=acc; j=j+1 } t=t+1 } 76 t=0; while t<T { j=0; while j<HF { let dmv: i64=dm[t*HF+j]; let dav: i64=(dmv*u[t*HF+j])/S; du[t*HF+j]=(dmv*a[t*HF+j])/S; let z: i64=g[t*HF+j]; let sig: i64=sigmoid(z); let zt: i64=(z*(S-sig))/S; let dsil: i64=(sig*(S+zt))/S; dg[t*HF+j]=(dav*dsil)/S; j=j+1 } t=t+1 } 77 var k: i64=0; while k<DM { j=0; while j<HF { var ag: i64=0; var au: i64=0; t=0; while t<T { ag=ag+(h[t*DM+k]*dg[t*HF+j])/S; au=au+(h[t*DM+k]*du[t*HF+j])/S; t=t+1 } dWg[k*HF+j]=ag; dWu[k*HF+j]=au; j=j+1 } k=k+1 } 78 t=0; while t<T { k=0; while k<DM { var acc: i64=0; j=0; while j<HF { acc=acc+(dg[t*HF+j]*Wg[k*HF+j])/S+(du[t*HF+j]*Wu[k*HF+j])/S; j=j+1 } dh[t*DM+k]=acc; k=k+1 } t=t+1 } 79 return 0 80} 81 82// slots: 0X 1g1 2Wq 3Wk 4Wv 5g2 6Wg 7Wu 8Wd 9Y | 10h1 11Q 12K 13V 14A 15O1 16rms1 17x2 | 18h2 19g 20a 21u 22m 23ffn 24rms2 25y | 26dg1 27dWq 28dWk 29dWv 30dg2 31dWg 32dWu 33dWd 34dX 83func layer_fwd(P: *i64) -> i64 { 84 let X: *i64=P[0] as *i64; let Y: *i64=P[9] as *i64; let x2: *i64=P[17] as *i64; let y: *i64=P[25] as *i64 85 rmsnorm_fwd(X, P[1] as *i64, P[10] as *i64, P[16] as *i64) 86 attn_fwd(P[10] as *i64, P[2] as *i64, P[3] as *i64, P[4] as *i64, P[11] as *i64, P[12] as *i64, P[13] as *i64, P[14] as *i64, P[15] as *i64) 87 let O1: *i64=P[15] as *i64; var i: i64=0; while i<T*DM { x2[i]=X[i]+O1[i]; i=i+1 } 88 rmsnorm_fwd(x2, P[5] as *i64, P[18] as *i64, P[24] as *i64) 89 swiglu_fwd(P[18] as *i64, P[6] as *i64, P[7] as *i64, P[8] as *i64, P[19] as *i64, P[20] as *i64, P[21] as *i64, P[22] as *i64, P[23] as *i64) 90 let ffn: *i64=P[23] as *i64; var Lq: i64=0; i=0; while i<T*DM { let yv: i64=x2[i]+ffn[i]; y[i]=yv; let e: i64=yv-Y[i]; Lq=Lq+e*e; i=i+1 } 91 return Lq 92} 93func layer_bwd(P: *i64) -> i64 { 94 let X: *i64=P[0] as *i64; let Y: *i64=P[9] as *i64; let x2: *i64=P[17] as *i64; let y: *i64=P[25] as *i64; let dX: *i64=P[34] as *i64 95 let dy: *i64=sys_mmap(T*DM*8) as *i64; let dx2: *i64=sys_mmap(T*DM*8) as *i64; let dh2: *i64=sys_mmap(T*DM*8) as *i64; let dh1: *i64=sys_mmap(T*DM*8) as *i64 96 var i: i64=0; while i<T*DM { dy[i]=2*(y[i]-Y[i]); dx2[i]=dy[i]; dh2[i]=0; dh1[i]=0; dX[i]=0; i=i+1 } // residual2: dx2 starts = dy 97 let dg2z: *i64=P[30] as *i64; let dg1z: *i64=P[26] as *i64 98 var z: i64=0; while z<DM { dg2z[z]=0; dg1z[z]=0; z=z+1 } // zero dg2,dg1 99 // block2: SwiGLU (input h2=P18), dOut=dy -> dWg,dWu,dWd, dh2 100 swiglu_bwd(P[18] as *i64, P[6] as *i64, P[7] as *i64, P[8] as *i64, P[19] as *i64, P[20] as *i64, P[21] as *i64, P[22] as *i64, dy, P[31] as *i64, P[32] as *i64, P[33] as *i64, dh2) 101 rmsnorm_bwd(x2, P[5] as *i64, dh2, P[24] as *i64, dx2, P[30] as *i64) // dh2 -> dx2 += norm2, dg2 102 // block1: residual1 dO1 = dx2 ; dX starts = dx2 103 i=0; while i<T*DM { dX[i]=dx2[i]; i=i+1 } 104 attn_bwd(P[10] as *i64, P[2] as *i64, P[3] as *i64, P[4] as *i64, P[11] as *i64, P[12] as *i64, P[13] as *i64, P[14] as *i64, dx2, P[27] as *i64, P[28] as *i64, P[29] as *i64, dh1) 105 rmsnorm_bwd(X, P[1] as *i64, dh1, P[16] as *i64, dX, P[26] as *i64) // dh1 -> dX += norm1, dg1 106 return 0 107} 108 109func gcheck(name: *u8, P: *i64, tgtslot: i64, gslot: i64, ncell: i64) -> i64 { 110 let Wtgt: *i64=P[tgtslot] as *i64; let dW: *i64=P[gslot] as *i64 111 let DELTA: i64=10486; let TOLP: i64=70 112 var maxabs: i64=1; var q: i64=0; while q<ncell { if iabs(dW[q])>maxabs { maxabs=iabs(dW[q]) } q=q+1 } 113 var npass: i64=0; var worst: i64=0; var i: i64=0 114 while i<ncell { 115 let save: i64=Wtgt[i] 116 Wtgt[i]=save+DELTA; let Lp: i64=layer_fwd(P) 117 Wtgt[i]=save-DELTA; let Lm: i64=layer_fwd(P) 118 Wtgt[i]=save; let dd: i64=layer_fwd(P) 119 let num: i64=(Lp-Lm)/(2*DELTA); let rel: i64=(iabs(num-dW[i])*1000)/maxabs 120 if rel<=TOLP { npass=npass+1 } else { w(" " as *u8); w(name); w(" cell " as *u8); wn(i); w(" ana=" as *u8); wn(dW[i]); w(" num=" as *u8); wn(num); w(" rel=" as *u8); wn(rel); w("\n" as *u8) } 121 if rel>worst { worst=rel } 122 i=i+1 123 } 124 w(" " as *u8); w(name); w(": " as *u8); wn(npass); w("/" as *u8); wn(ncell); w(" (worst=" as *u8); wn(worst); w("permil)\n" as *u8) 125 return npass 126} 127 128func main() -> i64 { 129 w("=== nx_intfp_layer_gradcheck: FULL transformer layer [RMSNorm->Attn->res][RMSNorm->SwiGLU->res], Q20 integer ===\n\n" as *u8) 130 let P: *i64=sys_mmap(36*8) as *i64 131 P[0]=sys_mmap(T*DM*8); P[1]=sys_mmap(DM*8); P[2]=sys_mmap(DM*DM*8); P[3]=sys_mmap(DM*DM*8); P[4]=sys_mmap(DM*DM*8); P[5]=sys_mmap(DM*8); P[6]=sys_mmap(DM*HF*8); P[7]=sys_mmap(DM*HF*8); P[8]=sys_mmap(HF*DM*8); P[9]=sys_mmap(T*DM*8) 132 P[10]=sys_mmap(T*DM*8); P[11]=sys_mmap(T*DM*8); P[12]=sys_mmap(T*DM*8); P[13]=sys_mmap(T*DM*8); P[14]=sys_mmap(T*T*8); P[15]=sys_mmap(T*DM*8); P[16]=sys_mmap(T*8); P[17]=sys_mmap(T*DM*8) 133 P[18]=sys_mmap(T*DM*8); P[19]=sys_mmap(T*HF*8); P[20]=sys_mmap(T*HF*8); P[21]=sys_mmap(T*HF*8); P[22]=sys_mmap(T*HF*8); P[23]=sys_mmap(T*DM*8); P[24]=sys_mmap(T*8); P[25]=sys_mmap(T*DM*8) 134 P[26]=sys_mmap(DM*8); P[27]=sys_mmap(DM*DM*8); P[28]=sys_mmap(DM*DM*8); P[29]=sys_mmap(DM*DM*8); P[30]=sys_mmap(DM*8); P[31]=sys_mmap(DM*HF*8); P[32]=sys_mmap(DM*HF*8); P[33]=sys_mmap(HF*DM*8); P[34]=sys_mmap(T*DM*8) 135 let X: *i64=P[0] as *i64; let g1: *i64=P[1] as *i64; let Wq: *i64=P[2] as *i64; let Wk: *i64=P[3] as *i64; let Wv: *i64=P[4] as *i64; let g2: *i64=P[5] as *i64; let Wg: *i64=P[6] as *i64; let Wu: *i64=P[7] as *i64; let Wd: *i64=P[8] as *i64; let Y: *i64=P[9] as *i64 136 var i: i64=0; while i<T*DM { X[i]=((((i*5+2)%11)-5)*S)/10; i=i+1 } 137 i=0; while i<DM { g1[i]=S; g2[i]=S; i=i+1 } 138 i=0; while i<DM*DM { Wq[i]=((((i*7+1)%13)-6)*S)/16; Wk[i]=((((i*3+5)%13)-6)*S)/16; Wv[i]=((((i*11+2)%13)-6)*S)/16; i=i+1 } 139 i=0; while i<DM*HF { Wg[i]=((((i*7+1)%11)-5)*S)/16; Wu[i]=((((i*3+4)%11)-5)*S)/16; i=i+1 } 140 i=0; while i<HF*DM { Wd[i]=((((i*5+3)%11)-5)*S)/16; i=i+1 } 141 i=0; while i<T*DM { Y[i]=((((i*3+1)%7)-3)*S)/8; i=i+1 } 142 143 let L0: i64=layer_fwd(P); layer_bwd(P) 144 w(" forward L_q32=" as *u8); wn(L0); w("\n gradcheck (assembled 2-block backward; only failing cells printed):\n" as *u8) 145 let p1: i64=gcheck("g1 " as *u8, P, 1, 26, DM) 146 let p2: i64=gcheck("Wq " as *u8, P, 2, 27, DM*DM) 147 let p3: i64=gcheck("g2 " as *u8, P, 5, 30, DM) 148 let p4: i64=gcheck("Wg " as *u8, P, 6, 31, DM*HF) 149 let p5: i64=gcheck("Wd " as *u8, P, 8, 33, HF*DM) 150 let p6: i64=gcheck("X " as *u8, P, 0, 34, T*DM) 151 let tot: i64=p1+p2+p3+p4+p5+p6; let want: i64=DM+DM*DM+DM+DM*HF+HF*DM+T*DM 152 w("\n FULL-LAYER gradcheck: " as *u8); wn(tot); w("/" as *u8); wn(want); w(" cells correct\n" as *u8) 153 154 // TRAIN the full layer (integer SGD + error-feedback on all 8 weight tensors) 155 let lr: i64=(S*8)/100 156 let RG: *i64=sys_mmap(9*8) as *i64 // residual-accum pointers per tensor (indices align to slot list) 157 RG[0]=sys_mmap(DM*8); RG[1]=sys_mmap(DM*DM*8); RG[2]=sys_mmap(DM*DM*8); RG[3]=sys_mmap(DM*DM*8); RG[4]=sys_mmap(DM*8); RG[5]=sys_mmap(DM*HF*8); RG[6]=sys_mmap(DM*HF*8); RG[7]=sys_mmap(HF*DM*8) 158 // (tensor, param-slot, grad-slot, size): g1(1,26),Wq(2,27),Wk(3,28),Wv(4,29),g2(5,30),Wg(6,31),Wu(7,32),Wd(8,33) 159 let pslot: *i64=sys_mmap(8*8) as *i64; let gslot: *i64=sys_mmap(8*8) as *i64; let sz: *i64=sys_mmap(8*8) as *i64 160 pslot[0]=1; pslot[1]=2; pslot[2]=3; pslot[3]=4; pslot[4]=5; pslot[5]=6; pslot[6]=7; pslot[7]=8 161 gslot[0]=26; gslot[1]=27; gslot[2]=28; gslot[3]=29; gslot[4]=30; gslot[5]=31; gslot[6]=32; gslot[7]=33 162 sz[0]=DM; sz[1]=DM*DM; sz[2]=DM*DM; sz[3]=DM*DM; sz[4]=DM; sz[5]=DM*HF; sz[6]=DM*HF; sz[7]=HF*DM 163 var e: i64=0; while e<8 { let R: *i64=RG[e] as *i64; var z: i64=0; while z<sz[e] { R[z]=0; z=z+1 } e=e+1 } 164 var step: i64=1 165 while step<=2000 { 166 let L: i64=layer_fwd(P); layer_bwd(P) 167 e=0; while e<8 { let Wt: *i64=P[pslot[e]] as *i64; let dW: *i64=P[gslot[e]] as *i64; let R: *i64=RG[e] as *i64; var z: i64=0; while z<sz[e] { R[z]=R[z]+lr*dW[z]; let tk: i64=R[z]/S; Wt[z]=Wt[z]-tk; R[z]=R[z]-tk*S; z=z+1 } e=e+1 } 168 step=step+1 169 } 170 let Lf: i64=layer_fwd(P) 171 w(" TRAIN: loss_q32 " as *u8); wn(L0); w(" -> " as *u8); wn(Lf); w(" over 2000 steps\n" as *u8) 172 w("NX-INTFP-LAYER verdict=" as *u8) 173 if tot==want { if Lf*4<L0 { w("GREEN gradcheck " as *u8); wn(tot); w("/" as *u8); wn(want); w(" + TRAINS (loss>4x drop) -- FULL integer transformer LAYER proven end-to-end (both blocks, residual stream, all 8 tensors)\n" as *u8) } else { w("YELLOW gradcheck ok, loss didn't fall >4x\n" as *u8) } } 174 else { w("RED gradcheck " as *u8); wn(tot); w("/" as *u8); wn(want); w(" -- 2-block assembly bug\n" as *u8) } 175 return 0 176}