code wiki / _hdl_build / nx_intfp_lm_train_gate.nx
nx_intfp_lm_train_gate.nx source
↩ module page · 130 lines · 16485 B
1// nx_intfp_lm_train_gate.nx -- COMPLETE integer transformer LM, trained on next-token prediction. Composes the
2// proven pieces: token EMBEDDING (lookup + scatter grad) -> full transformer LAYER (attn+FFN+2 norms+2 residuals,
3// gradchecked 84/84 elsewhere) -> output HEAD (y@Wout) -> CROSS-ENTROPY (softmax + the exact p-onehot gradient).
4// Everything Q20 integer, no float. Trains tokens [1,2,3,4,5,6] -> targets [2,3,4,5,6,7]; reports the (1-p_correct)
5// proxy loss + argmax ACCURACY. If accuracy -> T/T, a sovereign integer transformer LM has LEARNED. This is the
6// sovereign half of the PyTorch training h2h; only the run itself justifies a speed claim. license_tier: ORIGINAL
7import "nx_syscalls.nx"
8
9func w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
10func wn(v: i64) -> i64 { if v==0 { sys_write(1,"0" as *u8,1); return 0 } var m: i64=v; if m<0{sys_write(1,"-" as *u8,1);m=0-m} let t: *u8=sys_mmap(24); var k: i64=0; while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} let o: *u8=sys_mmap(24); var q: i64=k-1; var i: i64=0; while q>=0{o[i]=t[q];i=i+1;q=q-1} sys_write(1,o,i); return 0 }
11
12const S: i64 = 1048576
13const T: i64 = 6
14const DM: i64 = 4
15const HF: i64 = 8
16const VOC: i64 = 8
17const SCALE: i64 = 524288
18const EPS: i64 = 1048576
19
20func isqrt(n: i64) -> i64 { if n<=0 { return 0 } var bit: i64=1; while bit*4<=n { bit=bit*4 } var res: i64=0; var num: i64=n; while bit!=0 { if num>=res+bit { num=num-(res+bit); res=(res/2)+bit } else { res=res/2 } bit=bit/4 } return res }
21func fp_exp(xq: i64) -> i64 { let y: i64=(xq*1512776)/S; var yi: i64=0; if y>=0 { yi=y/S } else { yi=0-(((0-y)+S-1)/S) } let yf: i64=y-yi*S; var p: i64=10085; p=58197+(p*yf)/S; p=251882+(p*yf)/S; p=726817+(p*yf)/S; p=S+(p*yf)/S; if yi>=0 { if yi>=31 { return 2000000000 } return p*(1<<yi) } let k: i64=0-yi; if k>=31 { return 0 } return p/(1<<k) }
22func sigmoid(z: i64) -> i64 { let e: i64=fp_exp(0-z); return (S*S)/(S+e) }
23
24func rmsnorm_fwd(x: *i64, gm: *i64, out: *i64, rms: *i64) -> i64 { var t: i64=0; while t<T { var ms: i64=0; var i: i64=0; while i<DM { ms=ms+x[t*DM+i]*x[t*DM+i]; i=i+1 } ms=ms/DM+EPS; let r: i64=isqrt(ms); rms[t]=r; let inv: i64=(S*S)/r; i=0; while i<DM { let nrm: i64=(x[t*DM+i]*inv)/S; out[t*DM+i]=(nrm*gm[i])/S; i=i+1 } t=t+1 } return 0 }
25func rmsnorm_bwd(x: *i64, gm: *i64, dh: *i64, rms: *i64, dxacc: *i64, dgm: *i64) -> i64 { var t: i64=0; while t<T { let r: i64=rms[t]; let inv: i64=(S*S)/r; let invr3: i64=(((inv*inv)/S)*inv)/S; var c: i64=0; var i: i64=0; while i<DM { let nrm: i64=(x[t*DM+i]*inv)/S; dgm[i]=dgm[i]+(dh[t*DM+i]*nrm)/S; let dn: i64=(dh[t*DM+i]*gm[i])/S; c=c+(dn*x[t*DM+i])/S; i=i+1 } i=0; while i<DM { let dn: i64=(dh[t*DM+i]*gm[i])/S; let t1: i64=(dn*inv)/S; let tt: i64=(x[t*DM+i]*c)/S; let t2: i64=(((tt*invr3)/S))/DM; dxacc[t*DM+i]=dxacc[t*DM+i]+(t1-t2); i=i+1 } t=t+1 } return 0 }
26func attn_fwd(h: *i64, Wq: *i64, Wk: *i64, Wv: *i64, Q: *i64, K: *i64, V: *i64, A: *i64, O: *i64) -> i64 { var t: i64=0; while t<T { var i: i64=0; while i<DM { var aq: i64=0; var ak: i64=0; var av: i64=0; var k: i64=0; while k<DM { aq=aq+h[t*DM+k]*Wq[k*DM+i]; ak=ak+h[t*DM+k]*Wk[k*DM+i]; av=av+h[t*DM+k]*Wv[k*DM+i]; k=k+1 } Q[t*DM+i]=aq/S; K[t*DM+i]=ak/S; V[t*DM+i]=av/S; i=i+1 } t=t+1 } t=0; while t<T { var mx: i64=0-2000000000; var s: i64=0; while s<=t { var dot: i64=0; var i: i64=0; while i<DM { dot=dot+Q[t*DM+i]*K[s*DM+i]; i=i+1 } let sc: i64=((dot/S)*SCALE)/S; A[t*T+s]=sc; if sc>mx { mx=sc } s=s+1 } var sum: i64=0; s=0; while s<=t { let e: i64=fp_exp(A[t*T+s]-mx); A[t*T+s]=e; sum=sum+e; s=s+1 } s=0; while s<=t { A[t*T+s]=(A[t*T+s]*S+sum/2)/sum; s=s+1 } t=t+1 } t=0; while t<T { var i: i64=0; while i<DM { var acc: i64=0; var s: i64=0; while s<=t { acc=acc+A[t*T+s]*V[s*DM+i]; s=s+1 } O[t*DM+i]=acc/S; i=i+1 } t=t+1 } return 0 }
27func attn_bwd(h: *i64, Wq: *i64, Wk: *i64, Wv: *i64, Q: *i64, K: *i64, V: *i64, A: *i64, dO: *i64, dWq: *i64, dWk: *i64, dWv: *i64, dh: *i64) -> i64 { let dV: *i64=sys_mmap(T*DM*8) as *i64; let dA: *i64=sys_mmap(T*T*8) as *i64; let dsc: *i64=sys_mmap(T*T*8) as *i64; let dQ: *i64=sys_mmap(T*DM*8) as *i64; let dK: *i64=sys_mmap(T*DM*8) as *i64; var s: i64=0; while s<DM*T { dV[s]=0; s=s+1 } s=0; while s<T { var i: i64=0; while i<DM { var acc: i64=0; var t: i64=s; while t<T { acc=acc+(A[t*T+s]*dO[t*DM+i])/S; t=t+1 } dV[s*DM+i]=acc; i=i+1 } s=s+1 } var t: i64=0; while t<T { s=0; while s<=t { var acc: i64=0; var i: i64=0; while i<DM { acc=acc+(dO[t*DM+i]*V[s*DM+i])/S; i=i+1 } dA[t*T+s]=acc; s=s+1 } t=t+1 } t=0; while t<T { var dot: i64=0; s=0; while s<=t { dot=dot+(A[t*T+s]*dA[t*T+s])/S; s=s+1 } s=0; while s<=t { dsc[t*T+s]=(A[t*T+s]*(dA[t*T+s]-dot))/S; s=s+1 } t=t+1 } t=0; while t<T*DM { dQ[t]=0; dK[t]=0; t=t+1 } t=0; while t<T { var i: i64=0; while i<DM { var acc: i64=0; s=0; while s<=t { let dqk: i64=(SCALE*dsc[t*T+s])/S; acc=acc+(dqk*K[s*DM+i])/S; s=s+1 } dQ[t*DM+i]=acc; i=i+1 } t=t+1 } s=0; while s<T { var i: i64=0; while i<DM { var acc: i64=0; t=s; while t<T { let dqk: i64=(SCALE*dsc[t*T+s])/S; acc=acc+(dqk*Q[t*DM+i])/S; t=t+1 } dK[s*DM+i]=acc; i=i+1 } s=s+1 } var kk: i64=0; while kk<DM { var i: i64=0; while i<DM { var aq: i64=0; var ak: i64=0; var av: i64=0; t=0; while t<T { aq=aq+(h[t*DM+kk]*dQ[t*DM+i])/S; ak=ak+(h[t*DM+kk]*dK[t*DM+i])/S; av=av+(h[t*DM+kk]*dV[t*DM+i])/S; t=t+1 } dWq[kk*DM+i]=aq; dWk[kk*DM+i]=ak; dWv[kk*DM+i]=av; i=i+1 } kk=kk+1 } t=0; while t<T { var k2: i64=0; while k2<DM { var acc: i64=0; var i: i64=0; while i<DM { acc=acc+(dQ[t*DM+i]*Wq[k2*DM+i])/S+(dK[t*DM+i]*Wk[k2*DM+i])/S+(dV[t*DM+i]*Wv[k2*DM+i])/S; i=i+1 } dh[t*DM+k2]=acc; k2=k2+1 } t=t+1 } return 0 }
28func swiglu_fwd(h: *i64, Wg: *i64, Wu: *i64, Wd: *i64, g: *i64, a: *i64, u: *i64, m: *i64, ffn: *i64) -> i64 { var t: i64=0; while t<T { var j: i64=0; while j<HF { var gg: i64=0; var uu: i64=0; var k: i64=0; while k<DM { gg=gg+h[t*DM+k]*Wg[k*HF+j]; uu=uu+h[t*DM+k]*Wu[k*HF+j]; k=k+1 } let gv: i64=gg/S; g[t*HF+j]=gv; u[t*HF+j]=uu/S; let sig: i64=sigmoid(gv); let av: i64=(gv*sig)/S; a[t*HF+j]=av; m[t*HF+j]=(av*u[t*HF+j])/S; j=j+1 } t=t+1 } t=0; while t<T { var i: i64=0; while i<DM { var acc: i64=0; var j: i64=0; while j<HF { acc=acc+m[t*HF+j]*Wd[j*DM+i]; j=j+1 } ffn[t*DM+i]=acc/S; i=i+1 } t=t+1 } return 0 }
29func swiglu_bwd(h: *i64, Wg: *i64, Wu: *i64, Wd: *i64, g: *i64, a: *i64, u: *i64, m: *i64, dOut: *i64, dWg: *i64, dWu: *i64, dWd: *i64, dh: *i64) -> i64 { let dm: *i64=sys_mmap(T*HF*8) as *i64; let dg: *i64=sys_mmap(T*HF*8) as *i64; let du: *i64=sys_mmap(T*HF*8) as *i64; var j: i64=0; while j<HF { var i: i64=0; while i<DM { var acc: i64=0; var t: i64=0; while t<T { acc=acc+(m[t*HF+j]*dOut[t*DM+i])/S; t=t+1 } dWd[j*DM+i]=acc; i=i+1 } j=j+1 } var t: i64=0; while t<T { j=0; while j<HF { var acc: i64=0; var i: i64=0; while i<DM { acc=acc+(dOut[t*DM+i]*Wd[j*DM+i])/S; i=i+1 } dm[t*HF+j]=acc; j=j+1 } t=t+1 } t=0; while t<T { j=0; while j<HF { let dmv: i64=dm[t*HF+j]; let dav: i64=(dmv*u[t*HF+j])/S; du[t*HF+j]=(dmv*a[t*HF+j])/S; let z: i64=g[t*HF+j]; let sig: i64=sigmoid(z); let zt: i64=(z*(S-sig))/S; let dsil: i64=(sig*(S+zt))/S; dg[t*HF+j]=(dav*dsil)/S; j=j+1 } t=t+1 } var k: i64=0; while k<DM { j=0; while j<HF { var ag: i64=0; var au: i64=0; t=0; while t<T { ag=ag+(h[t*DM+k]*dg[t*HF+j])/S; au=au+(h[t*DM+k]*du[t*HF+j])/S; t=t+1 } dWg[k*HF+j]=ag; dWu[k*HF+j]=au; j=j+1 } k=k+1 } t=0; while t<T { k=0; while k<DM { var acc: i64=0; j=0; while j<HF { acc=acc+(dg[t*HF+j]*Wg[k*HF+j])/S+(du[t*HF+j]*Wu[k*HF+j])/S; j=j+1 } dh[t*DM+k]=acc; k=k+1 } t=t+1 } return 0 }
30
31// slots: 0X 1g1 2Wq 3Wk 4Wv 5g2 6Wg 7Wu 8Wd 9Embed | 10h1 11Q 12K 13V 14A 15O1 16rms1 17x2 18h2 19g 20a 21u 22m 23ffn 24rms2 25y | 26dg1 27dWq 28dWk 29dWv 30dg2 31dWg 32dWu 33dWd 34dX | 35Wout 36logits 37p 38dEmbed 39dWout 40tokens 41targets 42dy
32func layer_core_fwd(P: *i64) -> i64 {
33 let X: *i64=P[0] as *i64; let x2: *i64=P[17] as *i64; let y: *i64=P[25] as *i64
34 rmsnorm_fwd(X, P[1] as *i64, P[10] as *i64, P[16] as *i64)
35 attn_fwd(P[10] as *i64, P[2] as *i64, P[3] as *i64, P[4] as *i64, P[11] as *i64, P[12] as *i64, P[13] as *i64, P[14] as *i64, P[15] as *i64)
36 let O1: *i64=P[15] as *i64; var i: i64=0; while i<T*DM { x2[i]=X[i]+O1[i]; i=i+1 }
37 rmsnorm_fwd(x2, P[5] as *i64, P[18] as *i64, P[24] as *i64)
38 swiglu_fwd(P[18] as *i64, P[6] as *i64, P[7] as *i64, P[8] as *i64, P[19] as *i64, P[20] as *i64, P[21] as *i64, P[22] as *i64, P[23] as *i64)
39 let ffn: *i64=P[23] as *i64; i=0; while i<T*DM { y[i]=x2[i]+ffn[i]; i=i+1 }
40 return 0
41}
42func layer_core_bwd(P: *i64) -> i64 {
43 let X: *i64=P[0] as *i64; let x2: *i64=P[17] as *i64; let dX: *i64=P[34] as *i64; let dy: *i64=P[42] as *i64
44 let dx2: *i64=sys_mmap(T*DM*8) as *i64; let dh2: *i64=sys_mmap(T*DM*8) as *i64; let dh1: *i64=sys_mmap(T*DM*8) as *i64
45 var i: i64=0; while i<T*DM { dx2[i]=dy[i]; dh2[i]=0; dh1[i]=0; dX[i]=0; i=i+1 }
46 let dg2z: *i64=P[30] as *i64; let dg1z: *i64=P[26] as *i64; var z: i64=0; while z<DM { dg2z[z]=0; dg1z[z]=0; z=z+1 }
47 swiglu_bwd(P[18] as *i64, P[6] as *i64, P[7] as *i64, P[8] as *i64, P[19] as *i64, P[20] as *i64, P[21] as *i64, P[22] as *i64, dy, P[31] as *i64, P[32] as *i64, P[33] as *i64, dh2)
48 rmsnorm_bwd(x2, P[5] as *i64, dh2, P[24] as *i64, dx2, P[30] as *i64)
49 i=0; while i<T*DM { dX[i]=dx2[i]; i=i+1 }
50 attn_bwd(P[10] as *i64, P[2] as *i64, P[3] as *i64, P[4] as *i64, P[11] as *i64, P[12] as *i64, P[13] as *i64, P[14] as *i64, dx2, P[27] as *i64, P[28] as *i64, P[29] as *i64, dh1)
51 rmsnorm_bwd(X, P[1] as *i64, dh1, P[16] as *i64, dX, P[26] as *i64)
52 return 0
53}
54// embedding lookup: X[t] = Embed[tokens[t]]
55func embed_fwd(P: *i64) -> i64 { let X: *i64=P[0] as *i64; let Em: *i64=P[9] as *i64; let tok: *i64=P[40] as *i64; var t: i64=0; while t<T { let id: i64=tok[t]; var i: i64=0; while i<DM { X[t*DM+i]=Em[id*DM+i]; i=i+1 } t=t+1 } return 0 }
56// head + softmax(logits): logits=y@Wout, p=softmax(logits) over VOC
57func head_fwd(P: *i64) -> i64 {
58 let y: *i64=P[25] as *i64; let Wout: *i64=P[35] as *i64; let lg: *i64=P[36] as *i64; let p: *i64=P[37] as *i64
59 var t: i64=0
60 while t<T {
61 var mx: i64=0-2000000000; var c: i64=0
62 while c<VOC { var acc: i64=0; var i: i64=0; while i<DM { acc=acc+y[t*DM+i]*Wout[i*VOC+c]; i=i+1 } let lv: i64=acc/S; lg[t*VOC+c]=lv; if lv>mx { mx=lv } c=c+1 }
63 var sum: i64=0; c=0; while c<VOC { let e: i64=fp_exp(lg[t*VOC+c]-mx); p[t*VOC+c]=e; sum=sum+e; c=c+1 }
64 c=0; while c<VOC { p[t*VOC+c]=(p[t*VOC+c]*S+sum/2)/sum; c=c+1 }
65 t=t+1
66 }
67 return 0
68}
69// head backward: dlogits = p - onehot(target); dWout; dy (into slot 42)
70func head_bwd(P: *i64) -> i64 {
71 let y: *i64=P[25] as *i64; let Wout: *i64=P[35] as *i64; let p: *i64=P[37] as *i64; let tgt: *i64=P[41] as *i64
72 let dWout: *i64=P[39] as *i64; let dy: *i64=P[42] as *i64
73 let dlg: *i64=sys_mmap(T*VOC*8) as *i64
74 var t: i64=0; while t<T { var c: i64=0; while c<VOC { var d: i64=p[t*VOC+c]; if c==tgt[t] { d=d-S } dlg[t*VOC+c]=d; c=c+1 } t=t+1 }
75 var i: i64=0; while i<DM { var c: i64=0; while c<VOC { var acc: i64=0; t=0; while t<T { acc=acc+(y[t*DM+i]*dlg[t*VOC+c])/S; t=t+1 } dWout[i*VOC+c]=acc; c=c+1 } i=i+1 }
76 t=0; while t<T { i=0; while i<DM { var acc: i64=0; var c: i64=0; while c<VOC { acc=acc+(dlg[t*VOC+c]*Wout[i*VOC+c])/S; c=c+1 } dy[t*DM+i]=acc; i=i+1 } t=t+1 }
77 return 0
78}
79// embed backward: scatter dX into dEmbed
80func embed_bwd(P: *i64) -> i64 { let dEm: *i64=P[38] as *i64; let dX: *i64=P[34] as *i64; let tok: *i64=P[40] as *i64; var z: i64=0; while z<VOC*DM { dEm[z]=0; z=z+1 } var t: i64=0; while t<T { let id: i64=tok[t]; var i: i64=0; while i<DM { dEm[id*DM+i]=dEm[id*DM+i]+dX[t*DM+i]; i=i+1 } t=t+1 } return 0 }
81
82func main() -> i64 {
83 w("=== nx_intfp_lm_train: COMPLETE integer transformer LM (embed->layer->head->cross-entropy), next-token, Q20 ===\n\n" as *u8)
84 let P: *i64=sys_mmap(44*8) as *i64
85 P[0]=sys_mmap(T*DM*8); P[1]=sys_mmap(DM*8); P[2]=sys_mmap(DM*DM*8); P[3]=sys_mmap(DM*DM*8); P[4]=sys_mmap(DM*DM*8); P[5]=sys_mmap(DM*8); P[6]=sys_mmap(DM*HF*8); P[7]=sys_mmap(DM*HF*8); P[8]=sys_mmap(HF*DM*8); P[9]=sys_mmap(VOC*DM*8)
86 P[10]=sys_mmap(T*DM*8); P[11]=sys_mmap(T*DM*8); P[12]=sys_mmap(T*DM*8); P[13]=sys_mmap(T*DM*8); P[14]=sys_mmap(T*T*8); P[15]=sys_mmap(T*DM*8); P[16]=sys_mmap(T*8); P[17]=sys_mmap(T*DM*8)
87 P[18]=sys_mmap(T*DM*8); P[19]=sys_mmap(T*HF*8); P[20]=sys_mmap(T*HF*8); P[21]=sys_mmap(T*HF*8); P[22]=sys_mmap(T*HF*8); P[23]=sys_mmap(T*DM*8); P[24]=sys_mmap(T*8); P[25]=sys_mmap(T*DM*8)
88 P[26]=sys_mmap(DM*8); P[27]=sys_mmap(DM*DM*8); P[28]=sys_mmap(DM*DM*8); P[29]=sys_mmap(DM*DM*8); P[30]=sys_mmap(DM*8); P[31]=sys_mmap(DM*HF*8); P[32]=sys_mmap(DM*HF*8); P[33]=sys_mmap(HF*DM*8); P[34]=sys_mmap(T*DM*8)
89 P[35]=sys_mmap(DM*VOC*8); P[36]=sys_mmap(T*VOC*8); P[37]=sys_mmap(T*VOC*8); P[38]=sys_mmap(VOC*DM*8); P[39]=sys_mmap(DM*VOC*8); P[40]=sys_mmap(T*8); P[41]=sys_mmap(T*8); P[42]=sys_mmap(T*DM*8)
90 let Em: *i64=P[9] as *i64; let g1: *i64=P[1] as *i64; let g2: *i64=P[5] as *i64; let Wout: *i64=P[35] as *i64; let tok: *i64=P[40] as *i64; let tgt: *i64=P[41] as *i64
91 var i: i64=0; while i<VOC*DM { Em[i]=((((i*7+3)%13)-6)*S)/10; i=i+1 }
92 i=0; while i<DM { g1[i]=S; g2[i]=S; i=i+1 }
93 let Wq0: *i64=P[2] as *i64; let Wk0: *i64=P[3] as *i64; let Wv0: *i64=P[4] as *i64; let Wg0: *i64=P[6] as *i64; let Wu0: *i64=P[7] as *i64; let Wd0: *i64=P[8] as *i64
94 i=0; while i<DM*DM { Wq0[i]=((((i*7+1)%13)-6)*S)/16; Wk0[i]=((((i*3+5)%13)-6)*S)/16; Wv0[i]=((((i*11+2)%13)-6)*S)/16; i=i+1 }
95 i=0; while i<DM*HF { Wg0[i]=((((i*7+1)%11)-5)*S)/16; Wu0[i]=((((i*3+4)%11)-5)*S)/16; i=i+1 }
96 i=0; while i<HF*DM { Wd0[i]=((((i*5+3)%11)-5)*S)/16; i=i+1 }
97 i=0; while i<DM*VOC { Wout[i]=((((i*5+2)%11)-5)*S)/16; i=i+1 }
98 i=0; while i<T { tok[i]=i+1; tgt[i]=i+2; i=i+1 } // predict token+1
99
100 // 9 trainable tensors: Embed(9), g1(1),Wq(2),Wk(3),Wv(4),g2(5),Wg(6),Wu(7),Wd(8),Wout(35) -> use slot table
101 let pslot: *i64=sys_mmap(10*8) as *i64; let gslot: *i64=sys_mmap(10*8) as *i64; let sz: *i64=sys_mmap(10*8) as *i64; let RG: *i64=sys_mmap(10*8) as *i64
102 pslot[0]=9; pslot[1]=1; pslot[2]=2; pslot[3]=3; pslot[4]=4; pslot[5]=5; pslot[6]=6; pslot[7]=7; pslot[8]=8; pslot[9]=35
103 gslot[0]=38; gslot[1]=26; gslot[2]=27; gslot[3]=28; gslot[4]=29; gslot[5]=30; gslot[6]=31; gslot[7]=32; gslot[8]=33; gslot[9]=39
104 sz[0]=VOC*DM; sz[1]=DM; sz[2]=DM*DM; sz[3]=DM*DM; sz[4]=DM*DM; sz[5]=DM; sz[6]=DM*HF; sz[7]=DM*HF; sz[8]=HF*DM; sz[9]=DM*VOC
105 var e: i64=0; while e<10 { RG[e]=sys_mmap(sz[e]*8); let R: *i64=RG[e] as *i64; var z: i64=0; while z<sz[e] { R[z]=0; z=z+1 } e=e+1 }
106
107 // initial proxy loss + accuracy
108 embed_fwd(P); layer_core_fwd(P); head_fwd(P)
109 let p: *i64=P[37] as *i64
110 var loss0: i64=0; var acc0: i64=0; var t: i64=0
111 while t<T { loss0=loss0+(S-p[t*VOC+tgt[t]]); var mx: i64=0-1; var arg: i64=0; var c: i64=0; while c<VOC { if p[t*VOC+c]>mx { mx=p[t*VOC+c]; arg=c } c=c+1 } if arg==tgt[t] { acc0=acc0+1 } t=t+1 }
112 w(" before: proxy_loss(sum 1-p_correct, /1000)=" as *u8); wn(loss0/1000); w(" accuracy=" as *u8); wn(acc0); w("/" as *u8); wn(T); w("\n" as *u8)
113
114 let lr: i64=(S*10)/100
115 var step: i64=1
116 while step<=3000 {
117 embed_fwd(P); layer_core_fwd(P); head_fwd(P)
118 head_bwd(P); layer_core_bwd(P); embed_bwd(P)
119 e=0; while e<10 { let Wt: *i64=P[pslot[e]] as *i64; let dW: *i64=P[gslot[e]] as *i64; let R: *i64=RG[e] as *i64; var z: i64=0; while z<sz[e] { R[z]=R[z]+lr*dW[z]; let tk: i64=R[z]/S; Wt[z]=Wt[z]-tk; R[z]=R[z]-tk*S; z=z+1 } e=e+1 }
120 step=step+1
121 }
122 embed_fwd(P); layer_core_fwd(P); head_fwd(P)
123 var lossF: i64=0; var accF: i64=0; t=0
124 while t<T { lossF=lossF+(S-p[t*VOC+tgt[t]]); var mx: i64=0-1; var arg: i64=0; var c: i64=0; while c<VOC { if p[t*VOC+c]>mx { mx=p[t*VOC+c]; arg=c } c=c+1 } if arg==tgt[t] { accF=accF+1 } t=t+1 }
125 w(" after: proxy_loss=" as *u8); wn(lossF/1000); w(" accuracy=" as *u8); wn(accF); w("/" as *u8); wn(T); w(" (predicted next-token sequence)\n" as *u8)
126 w("NX-INTFP-LM verdict=" as *u8)
127 if accF==T { w("GREEN accuracy " as *u8); wn(accF); w("/" as *u8); wn(T); w(" -- COMPLETE integer transformer LM LEARNED next-token prediction end-to-end (embed+layer+head+cross-entropy, all no-float). Sovereign half of the PyTorch h2h READY.\n" as *u8) }
128 else { if lossF*2<loss0 { w("YELLOW loss fell but accuracy " as *u8); wn(accF); w("/" as *u8); wn(T); w(" (needs more steps/capacity)\n" as *u8) } else { w("RED did not learn\n" as *u8) } }
129 return 0
130}