nx_nano_mlp_lm.nx source

1// nx_nano_mlp_lm.nx -- climb our sovereign nanoGPT from bigram -> CONTEXT MLP LANGUAGE MODEL (the makemore rung). 2// Learnable embedding table + K=2 context window -> concat embeddings -> MLP(W1->ReLU->W2) -> softmax over vocab; 3// FULL integer backprop incl the gradient scattered back to the embedding rows. Trained on a Fibonacci-mod-3 4// sequence where the next char depends on the LAST TWO chars: a bigram (last char only) provably CANNOT learn it; 5// an MLP with 2-char context CAN. All integer Q16 => deterministic, $0 on this laptop. 6// T1 MLP-LM P(correct) rises >0.8 (learns the context rule). T2 a bigram baseline stays ~chance (<0.6) = context wins. 7// T3 generation continues the sequence. T4 (EXCEED) deterministic bit-identical. 8// expect_exit: 0 Sovereign: nx_syscalls. 9import "nx_syscalls.nx" 10import "nx_itoa_lib.nx" // shared MSB-first emitter (zero-alloc) 11const K_MAGIC_3000: i64 = 3000 12 13func g_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 14// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer 15// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the 16// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls). 17// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign. 18func g_pn(v: i64) -> i64 { nxi_out(v); return 0 } 19func ck(name: *u8, c: i64) -> i64 { if c==1 { g_puts(" PASS " as *u8) } else { g_puts(" FAIL " as *u8) } g_puts(name); g_puts("\n" as *u8); return c } 20 21const Q: i64 = 16 22const ONE: i64 = 65536 23const V: i64 = 3 24const E: i64 = 2 // embedding dim 25const K: i64 = 2 // context length 26const CC: i64 = 4 // K*E 27const H: i64 = 6 // hidden 28const LOG2E: i64 = 94548 29const C1: i64 = 45426 30const C2: i64 = 15743 31func fxmul(a: i64, b: i64) -> i64 { return (a*b)>>Q } 32func relu(x: i64) -> i64 { if x>0 { return x } return 0 } 33func exp_fx(x: i64) -> i64 { var xx: i64=x; if xx>0 { xx=0 } let yabs: i64=fxmul(0-xx,LOG2E); let nabs: i64=yabs>>Q; let fabs: i64=yabs-(nabs<<Q); let f2: i64=fxmul(fabs,fabs); let p: i64=ONE+fxmul(fabs,C1)+fxmul(f2,C2); let inv: i64=(ONE*ONE)/p; if nabs>=31 { return 0 } return inv>>nabs } 34func softmax(logits: *i64, probs: *i64, n: i64) -> i64 { var mx: i64=logits[0]; var i: i64=1; while i<n { if logits[i]>mx { mx=logits[i] } i=i+1 } let e: *i64=sys_mmap(n*8) as *i64; var sum: i64=0; i=0; while i<n { e[i]=exp_fx(logits[i]-mx); sum=sum+e[i]; i=i+1 } i=0; while i<n { probs[i]=(e[i]*ONE)/sum; i=i+1 } return 0 } 35func argmaxn(a: *i64, n: i64) -> i64 { var bi: i64=0; var bv: i64=a[0]; var i: i64=1; while i<n { if a[i]>bv { bv=a[i]; bi=i } i=i+1 } return bi } 36 37// forward an MLP-LM step: ctx (prev2,prev1) -> logits[V]; fills concat[CC],z1[H],a1[H] for backprop. 38func fwd(Emb: *i64, W1: *i64, b1: *i64, W2: *i64, b2: *i64, p2: i64, p1: i64, concat: *i64, z1: *i64, a1: *i64, logits: *i64) -> i64 { 39 concat[0]=Emb[p2*E+0]; concat[1]=Emb[p2*E+1]; concat[2]=Emb[p1*E+0]; concat[3]=Emb[p1*E+1] 40 var h: i64=0; while h<H { var s: i64=b1[h]; var i: i64=0; while i<CC { s=s+fxmul(W1[h*CC+i], concat[i]); i=i+1 } z1[h]=s; a1[h]=relu(s); h=h+1 } 41 var v: i64=0; while v<V { var s: i64=b2[v]; h=0; while h<H { s=s+fxmul(W2[v*H+h], a1[h]); h=h+1 } logits[v]=s; v=v+1 } 42 return 0 43} 44 45func main() -> i64 { 46 g_puts("nx_nano_mlp_lm (CONTEXT MLP language model -- embeddings + 2-char context; beats bigram on a context task)\n" as *u8) 47 var pass: i64=0; var total: i64=0 48 // Fibonacci-mod-3 corpus: next depends on the last TWO chars 49 let LEN: i64=32; let cor: *i64=sys_mmap(LEN*8) as *i64; cor[0]=1; cor[1]=1; var i: i64=2; while i<LEN { cor[i]=(cor[i-1]+cor[i-2])%V; i=i+1 } 50 g_puts(" corpus = Fibonacci-mod-3 ("); g_pn(LEN); g_puts(" chars); next char depends on the last TWO chars\n" as *u8) 51 52 // ---- bigram baseline (last char only) ---- 53 let BW: *i64=sys_mmap(V*V*8) as *i64; i=0; while i<V*V { BW[i]=0; i=i+1 } 54 let bp: *i64=sys_mmap(V*8) as *i64; var ep: i64=0 55 while ep<600 { var k: i64=1; while k<LEN { let prev: i64=cor[k-1]; let nx: i64=cor[k]; let row: *i64=sys_mmap(V*8) as *i64; var m: i64=0; while m<V { row[m]=BW[prev*V+m]; m=m+1 } softmax(row,bp,V); var j: i64=0; while j<V { var g: i64=bp[j]; if j==nx { g=g-ONE } BW[prev*V+j]=BW[prev*V+j]-fxmul(ONE/4,g); j=j+1 } k=k+1 } ep=ep+1 } 56 var bpc: i64=0; var k: i64=1; while k<LEN { let prev: i64=cor[k-1]; let row: *i64=sys_mmap(V*8) as *i64; var m: i64=0; while m<V { row[m]=BW[prev*V+m]; m=m+1 } softmax(row,bp,V); bpc=bpc+bp[cor[k]]; k=k+1 } bpc=bpc/(LEN-1) 57 g_puts(" BIGRAM baseline P(correct)="); g_pn(bpc); g_puts(" (ONE=65536; context-blind -> stuck below ~0.6)\n" as *u8) 58 59 // ---- MLP-LM (2-char context) ---- 60 let Emb: *i64=sys_mmap(V*E*8) as *i64; let W1: *i64=sys_mmap(H*CC*8) as *i64; let b1: *i64=sys_mmap(H*8) as *i64; let W2: *i64=sys_mmap(V*H*8) as *i64; let b2: *i64=sys_mmap(V*8) as *i64 61 Emb[0]=ONE/4; Emb[1]=0-ONE/4; Emb[2]=ONE/3; Emb[3]=ONE/5; Emb[4]=0-ONE/3; Emb[5]=ONE/6 62 i=0; while i<H*CC { W1[i]=(((i%5)-2)*ONE)/8; i=i+1 } i=0; while i<H { b1[i]=0; i=i+1 } 63 i=0; while i<V*H { W2[i]=(((i%5)-2)*ONE)/10; i=i+1 } i=0; while i<V { b2[i]=0; i=i+1 } 64 let concat: *i64=sys_mmap(CC*8) as *i64; let z1: *i64=sys_mmap(H*8) as *i64; let a1: *i64=sys_mmap(H*8) as *i64; let logits: *i64=sys_mmap(V*8) as *i64; let probs: *i64=sys_mmap(V*8) as *i64 65 let lr: i64=ONE/8; let EP: i64=K_MAGIC_3000 66 ep=0 67 while ep<EP { 68 let gE: *i64=sys_mmap(V*E*8) as *i64; let gW1: *i64=sys_mmap(H*CC*8) as *i64; let gb1: *i64=sys_mmap(H*8) as *i64; let gW2: *i64=sys_mmap(V*H*8) as *i64; let gb2: *i64=sys_mmap(V*8) as *i64 69 var z: i64=0; while z<V*E { gE[z]=0; z=z+1 } z=0; while z<H*CC { gW1[z]=0; z=z+1 } z=0; while z<H { gb1[z]=0; z=z+1 } z=0; while z<V*H { gW2[z]=0; z=z+1 } z=0; while z<V { gb2[z]=0; z=z+1 } 70 var t: i64=2 71 while t<LEN { 72 let p2: i64=cor[t-2]; let p1: i64=cor[t-1]; let tgt: i64=cor[t] 73 fwd(Emb,W1,b1,W2,b2,p2,p1,concat,z1,a1,logits); softmax(logits,probs,V) 74 let dl: *i64=sys_mmap(V*8) as *i64; var v: i64=0; while v<V { dl[v]=probs[v]; if v==tgt { dl[v]=dl[v]-ONE } v=v+1 } 75 // dW2,db2 + da1 76 let da1: *i64=sys_mmap(H*8) as *i64; var h: i64=0; while h<H { da1[h]=0; h=h+1 } 77 v=0; while v<V { gb2[v]=gb2[v]+dl[v]; h=0; while h<H { gW2[v*H+h]=gW2[v*H+h]+fxmul(dl[v],a1[h]); da1[h]=da1[h]+fxmul(dl[v],W2[v*H+h]); h=h+1 } v=v+1 } 78 // dz1 (relu) -> dW1,db1 + dconcat 79 let dcc: *i64=sys_mmap(CC*8) as *i64; var c: i64=0; while c<CC { dcc[c]=0; c=c+1 } 80 h=0; while h<H { var dz: i64=0; if z1[h]>0 { dz=da1[h] } gb1[h]=gb1[h]+dz; var ii: i64=0; while ii<CC { gW1[h*CC+ii]=gW1[h*CC+ii]+fxmul(dz,concat[ii]); dcc[ii]=dcc[ii]+fxmul(dz,W1[h*CC+ii]); ii=ii+1 } h=h+1 } 81 // scatter dconcat -> embedding grads 82 gE[p2*E+0]=gE[p2*E+0]+dcc[0]; gE[p2*E+1]=gE[p2*E+1]+dcc[1]; gE[p1*E+0]=gE[p1*E+0]+dcc[2]; gE[p1*E+1]=gE[p1*E+1]+dcc[3] 83 t=t+1 84 } 85 let ns: i64=LEN-2 86 var u: i64=0; while u<V*E { Emb[u]=Emb[u]-fxmul(lr, gE[u]/ns); u=u+1 } u=0; while u<H*CC { W1[u]=W1[u]-fxmul(lr, gW1[u]/ns); u=u+1 } u=0; while u<H { b1[u]=b1[u]-fxmul(lr, gb1[u]/ns); u=u+1 } u=0; while u<V*H { W2[u]=W2[u]-fxmul(lr, gW2[u]/ns); u=u+1 } u=0; while u<V { b2[u]=b2[u]-fxmul(lr, gb2[u]/ns); u=u+1 } 87 ep=ep+1 88 } 89 var mpc: i64=0; var t: i64=2; while t<LEN { fwd(Emb,W1,b1,W2,b2,cor[t-2],cor[t-1],concat,z1,a1,logits); softmax(logits,probs,V); mpc=mpc+probs[cor[t]]; t=t+1 } mpc=mpc/(LEN-2) 90 g_puts(" MLP-LM P(correct)="); g_pn(mpc); g_puts(" (context-aware -> should rise high)\n" as *u8) 91 var t1: i64=0; if mpc>(ONE*8/10) { t1=1 } 92 pass=pass+ck("T1: MLP-LM P(correct) > 0.8 -- it LEARNED the 2-char-context rule" as *u8, t1); total=total+1 93 var t2: i64=0; if bpc<(ONE*6/10) { if mpc>bpc { t2=1 } } 94 pass=pass+ck("T2 (measured): bigram baseline stuck < 0.6 while MLP-LM beats it -- CONTEXT MODELING wins" as *u8, t2); total=total+1 95 96 // generation: from the first two chars, predict forward; compare to the true sequence 97 let gen: *i64=sys_mmap(8*8) as *i64; gen[0]=cor[0]; gen[1]=cor[1]; var gi: i64=2; while gi<8 { fwd(Emb,W1,b1,W2,b2,gen[gi-2],gen[gi-1],concat,z1,a1,logits); gen[gi]=argmaxn(logits,V); gi=gi+1 } 98 var genok: i64=1; gi=2; while gi<8 { if gen[gi]!=cor[gi] { genok=0 } gi=gi+1 } 99 g_puts(" generated[2..7]="); gi=2; while gi<8 { g_pn(gen[gi]); g_puts(" "); gi=gi+1 } g_puts(" vs true="); gi=2; while gi<8 { g_pn(cor[gi]); g_puts(" "); gi=gi+1 } g_puts("\n" as *u8) 100 var t3: i64=0; if genok==1 { t3=1 } 101 pass=pass+ck("T3: GENERATION continues the sequence correctly (the model predicts the next chars from context)" as *u8, t3); total=total+1 102 103 // determinism: P(correct) is a pure function of the integer training -> re-deriving argmax gen is identical (already deterministic by construction); assert gen stable 104 let gen2: *i64=sys_mmap(8*8) as *i64; gen2[0]=cor[0]; gen2[1]=cor[1]; gi=2; while gi<8 { fwd(Emb,W1,b1,W2,b2,gen2[gi-2],gen2[gi-1],concat,z1,a1,logits); gen2[gi]=argmaxn(logits,V); gi=gi+1 } 105 var same: i64=1; gi=2; while gi<8 { if gen[gi]!=gen2[gi] { same=0 } gi=gi+1 } 106 var t4: i64=0; if same==1 { t4=1 } 107 pass=pass+ck("T4 (EXCEED): DETERMINISTIC -- integer model gives identical predictions every run" as *u8, t4); total=total+1 108 109 g_puts(" >> nano_lm graduated bigram -> CONTEXT MLP-LM (learnable embeddings + context + backprop-to-embeddings).\n" as *u8) 110 g_puts(" CLIMB: longer context + more layers + full transformer backprop (attention/LayerNorm) = a real sovereign transformer-LM.\n" as *u8) 111 112 var okall: i64=0; if pass==total { okall=1 } 113 g_puts("---- nx_nano_mlp_lm: passed "); g_pn(pass); g_puts(" / "); g_pn(total); g_puts(" ----\n" as *u8) 114 if okall==1 { 115 let logf: i64=sys_openat_append("knowledge/status/nano_mlp_lm.log" as *u8, 420) 116 if logf>=0 { let z: i64=sys_write(logf,"NXNANOMLPLM GREEN: context MLP-LM (embeddings+2-char context+backprop-to-embeddings) learns Fib-mod-3, beats bigram, generates, deterministic\n" as *u8,137); sys_close(logf) } 117 g_puts("verdict=GREEN (context MLP language model: learnable embeddings + context window + backprop-to-embeddings; beats the bigram, learns + generates a context-dependent sequence, deterministic)\n" as *u8); sys_exit(0); return 0 118 } 119 g_puts("verdict=RED\n" as *u8); sys_exit(1); return 1 120}