code wiki / _hdl_build / nx_nano_mlp_lm.nx
nx_nano_mlp_lm.nx
buildroot/runtime/_hdl_build/nx_nano_mlp_lm.nx
about
nx_nano_mlp_lm.nx -- climb our sovereign nanoGPT from bigram -> CONTEXT MLP LANGUAGE MODEL (the makemore rung).
Learnable embedding table + K=2 context window -> concat embeddings -> MLP(W1->ReLU->W2) -> softmax over vocab;
FULL integer backprop incl the gradient scattered back to the embedding rows. Trained on a Fibonacci-mod-3
sequence where the next char depends on the LAST TWO chars: a bigram (last char only) provably CANNOT learn it;
an MLP with 2-char context CAN. All integer Q16 => deterministic, $0 on this laptop.
T1 MLP-LM P(correct) rises >0.8 (learns the context rule). T2 a bigram baseline stays ~chance (<0.6) = context wins.
T3 generation continues the sequence. T4 (EXCEED) deterministic bit-identical.
expect_exit: 0 Sovereign: nx_syscalls.
dependencies 2 imports · 0 importers
imports: nx_syscalls.nxnx_itoa_lib.nx
imported by: nobody (leaf or entry point)
call flow from main pre-order; caps 40 nodes / depth 6 declared; ↻ = already shown
structs
| none |
consts
| 11 | const K_MAGIC_3000: i64 = 3000 |
| 21 | const Q: i64 = 16 |
| 22 | const ONE: i64 = 65536 |
| 23 | const V: i64 = 3 |
| 24 | const E: i64 = 2 // embedding dim |
| 25 | const K: i64 = 2 // context length |
| 26 | const CC: i64 = 4 // K*E |
| 27 | const H: i64 = 6 // hidden |
| 28 | const LOG2E: i64 = 94548 |
| 29 | const C1: i64 = 45426 |
| 30 | const C2: i64 = 15743 |
functions
| 13 | func g_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } |
| 18 | func g_pn(v: i64) -> i64 { nxi_out(v); return 0 } |
| 19 | func ck(name: *u8, c: i64) -> i64 { if c==1 { g_puts(" PASS " as *u8) } else { g_puts(" FAIL " as *u8) } g_puts(name); g_puts("\n" as *u8); return c } |
| 31 | func fxmul(a: i64, b: i64) -> i64 { return (a*b)>>Q } |
| 32 | func relu(x: i64) -> i64 { if x>0 { return x } return 0 } called by 1: fwd |
| 33 | func exp_fx(x: i64) -> i64 { var xx: i64=x; if xx>0 { xx=0 } let yabs: i64=fxmul(0-xx,LOG2E); let nabs: i64=yabs>>Q; let fabs: i64=yabs-(nabs<<Q); let f2: i64=fxmul(fabs,fabs); let p: i64=ONE+fxmul(fabs,C1)+fxmul(f2,C2); let inv: i64=(ONE*ONE)/p; if nabs>=31 { return 0 } return inv>>nabs } |
| 34 | func softmax(logits: *i64, probs: *i64, n: i64) -> i64 { var mx: i64=logits[0]; var i: i64=1; while i<n { if logits[i]>mx { mx=logits[i] } i=i+1 } let e: *i64=sys_mmap(n*8) as *i64; var sum: i64=0; i=0; while i<n { e[i]=exp_fx(logits[i]-mx); sum=sum+e[i]; i=i+1 } i=0; while i<n { probs[i]=(e[i]*ONE)/sum; i=i+1 } return 0 } |
| 35 | func argmaxn(a: *i64, n: i64) -> i64 { var bi: i64=0; var bv: i64=a[0]; var i: i64=1; while i<n { if a[i]>bv { bv=a[i]; bi=i } i=i+1 } return bi } called by 1: main |
| 38 | func fwd(Emb: *i64, W1: *i64, b1: *i64, W2: *i64, b2: *i64, p2: i64, p1: i64, concat: *i64, z1: *i64, a1: *i64, logits: *i64) -> i64 |
| 45 | func main() -> i64 |