nx_intfp_moe_gradcheck_gate.nx source

1// nx_intfp_moe_gradcheck_gate.nx -- 2026 FRONTIER capability in the sovereign integer stack: MIXTURE-OF-EXPERTS 2// routing, gradchecked, NO float. MoE is UNIVERSAL in 2026 (DeepSeek/Qwen/Llama/GLM/Kimi all MoE). The novel part 3// vs a dense FFN is ROUTING: router logits -> top-K experts -> softmax gate over the selected -> weighted expert 4// sum, with the gradient flowing back THROUGH the gate softmax INTO the router. Experts here are linear (the 5// SwiGLU expert is already proven) so the gate focuses on the routing mechanism. Q20 integer, integer finite-diff 6// gradcheck (max-|grad| metric) on the ROUTER Wr (the frontier novelty) AND an expert We. license_tier: ORIGINAL 7import "nx_syscalls.nx" 8 9func w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 10func wn(v: i64) -> i64 { if v==0 { sys_write(1,"0" as *u8,1); return 0 } var m: i64=v; if m<0{sys_write(1,"-" as *u8,1);m=0-m} let t: *u8=sys_mmap(24); var k: i64=0; while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} let o: *u8=sys_mmap(24); var q: i64=k-1; var i: i64=0; while q>=0{o[i]=t[q];i=i+1;q=q-1} sys_write(1,o,i); return 0 } 11func iabs(v: i64) -> i64 { if v<0 { return 0-v } return v } 12 13const S: i64 = 1048576 // Q20 14const T: i64 = 3 15const DM: i64 = 6 16const NE: i64 = 4 // experts 17// K=2 (top-2 routing, like DeepSeek/Mixtral) 18 19func fp_exp(xq: i64) -> i64 { let y: i64=(xq*1512776)/S; var yi: i64=0; if y>=0 { yi=y/S } else { yi=0-(((0-y)+S-1)/S) } let yf: i64=y-yi*S; var p: i64=10085; p=58197+(p*yf)/S; p=251882+(p*yf)/S; p=726817+(p*yf)/S; p=S+(p*yf)/S; if yi>=0 { if yi>=31 { return 2000000000 } return p*(1<<yi) } let k: i64=0-yi; if k>=31 { return 0 } return p/(1<<k) } 20 21// slots: 0x 1Wr 2We | 3r 4e1 5e2 6w1 7w2 8oe1 9oe2 10y | 11dWr 12dWe 22// forward: router -> top-2 -> gate softmax -> weighted expert-linear sum; returns L_q32 23func moe_fwd(P: *i64) -> i64 { 24 let x: *i64=P[0] as *i64; let Wr: *i64=P[1] as *i64; let We: *i64=P[2] as *i64 25 let r: *i64=P[3] as *i64; let e1: *i64=P[4] as *i64; let e2: *i64=P[5] as *i64; let w1: *i64=P[6] as *i64; let w2: *i64=P[7] as *i64 26 let oe1: *i64=P[8] as *i64; let oe2: *i64=P[9] as *i64; let y: *i64=P[10] as *i64 27 // router logits r[t,e] 28 var t: i64=0 29 while t<T { var e: i64=0; while e<NE { var acc: i64=0; var k: i64=0; while k<DM { acc=acc+x[t*DM+k]*Wr[k*NE+e]; k=k+1 } r[t*NE+e]=acc/S; e=e+1 } t=t+1 } 30 var Lq: i64=0; t=0 31 while t<T { 32 // top-2 experts by logit 33 var be1: i64=0; var br1: i64=r[t*NE+0]; var e: i64=1; while e<NE { if r[t*NE+e]>br1 { br1=r[t*NE+e]; be1=e } e=e+1 } 34 var be2: i64=0-1; var br2: i64=0-2000000000; e=0; while e<NE { if e!=be1 { if r[t*NE+e]>br2 { br2=r[t*NE+e]; be2=e } } e=e+1 } 35 e1[t]=be1; e2[t]=be2 36 // 2-way softmax gate over {br1, br2}; br1 is the max 37 let ex1: i64=fp_exp(br1-br1); let ex2: i64=fp_exp(br2-br1); let sum: i64=ex1+ex2 38 w1[t]=(ex1*S+sum/2)/sum; w2[t]=(ex2*S+sum/2)/sum 39 // expert-linear outputs oe = x @ We[expert] 40 var i: i64=0 41 while i<DM { var a1: i64=0; var a2: i64=0; var k: i64=0; while k<DM { a1=a1+x[t*DM+k]*We[be1*DM*DM+k*DM+i]; a2=a2+x[t*DM+k]*We[be2*DM*DM+k*DM+i]; k=k+1 } oe1[t*DM+i]=a1/S; oe2[t*DM+i]=a2/S; i=i+1 } 42 // weighted sum + loss 43 i=0; while i<DM { let yv: i64=(w1[t]*oe1[t*DM+i])/S + (w2[t]*oe2[t*DM+i])/S; y[t*DM+i]=yv; Lq=Lq+yv*yv; i=i+1 } 44 t=t+1 45 } 46 return Lq 47} 48// backward: fills dWr, dWe (gradient flows through gate softmax into the router) 49func moe_bwd(P: *i64) -> i64 { 50 let x: *i64=P[0] as *i64; let We: *i64=P[2] as *i64 51 let e1: *i64=P[4] as *i64; let e2: *i64=P[5] as *i64; let w1: *i64=P[6] as *i64; let w2: *i64=P[7] as *i64 52 let oe1: *i64=P[8] as *i64; let oe2: *i64=P[9] as *i64; let y: *i64=P[10] as *i64 53 let dWr: *i64=P[11] as *i64; let dWe: *i64=P[12] as *i64 54 var z: i64=0; while z<DM*NE { dWr[z]=0; z=z+1 } z=0; while z<NE*DM*DM { dWe[z]=0; z=z+1 } 55 var t: i64=0 56 while t<T { 57 let ee1: i64=e1[t]; let ee2: i64=e2[t]; let ww1: i64=w1[t]; let ww2: i64=w2[t] 58 // dy, d(expert outputs)=w*dy, gate-weight grads dw=sum_i dy*oe 59 var dw1: i64=0; var dw2: i64=0; var i: i64=0 60 while i<DM { let dy: i64=2*y[t*DM+i]; let do1: i64=(ww1*dy)/S; let do2: i64=(ww2*dy)/S 61 // dWe[expert][k,i] += x[t,k]*d(expert_out) 62 var k: i64=0; while k<DM { dWe[ee1*DM*DM+k*DM+i]=dWe[ee1*DM*DM+k*DM+i]+(x[t*DM+k]*do1)/S; dWe[ee2*DM*DM+k*DM+i]=dWe[ee2*DM*DM+k*DM+i]+(x[t*DM+k]*do2)/S; k=k+1 } 63 dw1=dw1+(dy*oe1[t*DM+i])/S; dw2=dw2+(dy*oe2[t*DM+i])/S; i=i+1 } 64 // 2-way softmax backward: gate=softmax(r_e1,r_e2); dr_e = w_e*(dw_e - dot), dot=sum w*dw 65 let dot: i64=(ww1*dw1)/S+(ww2*dw2)/S 66 let dre1: i64=(ww1*(dw1-dot))/S; let dre2: i64=(ww2*(dw2-dot))/S 67 // dWr[k,e] += x[t,k]*dr_e (selected experts only; unselected dr=0) 68 var k: i64=0; while k<DM { dWr[k*NE+ee1]=dWr[k*NE+ee1]+(x[t*DM+k]*dre1)/S; dWr[k*NE+ee2]=dWr[k*NE+ee2]+(x[t*DM+k]*dre2)/S; k=k+1 } 69 t=t+1 70 } 71 return 0 72} 73 74func gcheck(name: *u8, P: *i64, Wt: *i64, dW: *i64, ncell: i64) -> i64 { 75 let DELTA: i64=2097; let TOLP: i64=70 // delta~0.002 at Q20; err vs max|grad| 76 var maxabs: i64=1; var q: i64=0; while q<ncell { if iabs(dW[q])>maxabs { maxabs=iabs(dW[q]) } q=q+1 } 77 var npass: i64=0; var worst: i64=0; var i: i64=0 78 while i<ncell { 79 let save: i64=Wt[i] 80 Wt[i]=save+DELTA; let Lp: i64=moe_fwd(P) 81 Wt[i]=save-DELTA; let Lm: i64=moe_fwd(P) 82 Wt[i]=save; let dd: i64=moe_fwd(P) 83 let num: i64=(Lp-Lm)/(2*DELTA); let rel: i64=(iabs(num-dW[i])*1000)/maxabs 84 if rel<=TOLP { npass=npass+1 } else { w(" " as *u8); w(name); w(" cell " as *u8); wn(i); w(" ana=" as *u8); wn(dW[i]); w(" num=" as *u8); wn(num); w(" rel=" as *u8); wn(rel); w("\n" as *u8) } 85 if rel>worst { worst=rel } 86 i=i+1 87 } 88 w(" " as *u8); w(name); w(": " as *u8); wn(npass); w("/" as *u8); wn(ncell); w(" (worst=" as *u8); wn(worst); w("permil of max|grad|=" as *u8); wn(maxabs); w(")\n" as *u8) 89 return npass 90} 91 92func main() -> i64 { 93 w("=== nx_intfp_moe_gradcheck: 2026-FRONTIER Mixture-of-Experts routing (top-2) in Q20 INTEGER, no float ===\n\n" as *u8) 94 let P: *i64=sys_mmap(13*8) as *i64 95 P[0]=sys_mmap(T*DM*8); P[1]=sys_mmap(DM*NE*8); P[2]=sys_mmap(NE*DM*DM*8) 96 P[3]=sys_mmap(T*NE*8); P[4]=sys_mmap(T*8); P[5]=sys_mmap(T*8); P[6]=sys_mmap(T*8); P[7]=sys_mmap(T*8); P[8]=sys_mmap(T*DM*8); P[9]=sys_mmap(T*DM*8); P[10]=sys_mmap(T*DM*8) 97 P[11]=sys_mmap(DM*NE*8); P[12]=sys_mmap(NE*DM*DM*8) 98 let x: *i64=P[0] as *i64; let Wr: *i64=P[1] as *i64; let We: *i64=P[2] as *i64 99 var i: i64=0; while i<T*DM { x[i]=((((i*5+2)%11)-5)*S)/10; i=i+1 } 100 i=0; while i<DM*NE { Wr[i]=((((i*7+3)%13)-6)*S)/12; i=i+1 } // distinct-ish logits so top-2 is stable 101 i=0; while i<NE*DM*DM { We[i]=((((i*3+1)%11)-5)*S)/16; i=i+1 } 102 103 let L0: i64=moe_fwd(P); moe_bwd(P) 104 // report routing 105 let e1: *i64=P[4] as *i64; let e2: *i64=P[5] as *i64; let w1: *i64=P[6] as *i64 106 w(" routing: t0 -> experts [" as *u8); wn(e1[0]); w("," as *u8); wn(e2[0]); w("] gate w1=" as *u8); wn((w1[0]*1000)/S); w("permil ; t1->[" as *u8); wn(e1[1]); w("," as *u8); wn(e2[1]); w("] ; t2->[" as *u8); wn(e1[2]); w("," as *u8); wn(e2[2]); w("]\n" as *u8) 107 w(" forward L_q32=" as *u8); wn(L0); w("\n gradcheck (integer finite-diff):\n" as *u8) 108 let pr: i64=gcheck("Wr(router)" as *u8, P, Wr, P[11] as *i64, DM*NE) // THE frontier novelty: gradient into the router thru the gate 109 let pe: i64=gcheck("We(expert)" as *u8, P, We, P[12] as *i64, NE*DM*DM) // expert weights (per-token routed) 110 let tot: i64=pr+pe; let want: i64=DM*NE+NE*DM*DM 111 w("\n MoE gradcheck: " as *u8); wn(tot); w("/" as *u8); wn(want); w(" cells correct\n" as *u8) 112 w("NX-INTFP-MOE verdict=" as *u8) 113 if tot==want { w("GREEN " as *u8); wn(tot); w("/" as *u8); wn(want); w(" -- 2026-FRONTIER integer MoE routing PROVEN (top-2 + gate softmax + gradient into router), no float. Sovereign stack at the MoE frontier.\n" as *u8) } 114 else { w("RED " as *u8); wn(tot); w("/" as *u8); wn(want); w(" -- routing/gate Q-scaling bug\n" as *u8) } 115 return 0 116}