`nx_nano_lm.nx` source

↩ module page · 112 lines · 7582 B
1// nx_nano_lm.nx -- OUR OWN SOVEREIGN nanoGPT (the bigram/char rung where nanoGPT+makemore start), built from the
2// Nishi rung up. A char-level language model that ACTUALLY TRAINS (integer gradient descent) and GENERATES text back.
3// Model: a V*V logit table W (W[prev][next]); forward = softmax(W[prev]); cross-entropy gradient = probs - onehot
4// (NO log needed -- the softmax+CE gradient is just probs minus the target one-hot). 100% integer Q16 => bit-exact
5// deterministic, $0 on this laptop CPU (no GPU/CUDA/PyTorch). Trains on a tiny cyclic corpus, learns the transitions,
6// and generates the pattern back.
7//   T1 P(correct next char) INCREASES across training (it learns the statistics).
8//   T2 GENERATION reproduces the learned pattern (from 'a' -> a,b,c,a,b,c).
9//   T3 (EXCEED) deterministic bit-identical.   T4 argmax(W[prev]) == the correct next char for every char.
10// expect_exit: 0   Sovereign: nx_syscalls.
11import "nx_syscalls.nx"
12import "nx_itoa_lib.nx"   // shared MSB-first emitter (zero-alloc)
13import "nx_g_puts_lib.nx"
14
15// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
16// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
17// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
18// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
19func g_pn(v: i64) -> i64 { nxi_out(v); return 0 }
20func g_ch(id: i64) -> i64 { let b: *u8=sys_mmap(1); b[0]=(97+id) as u8; sys_write(1,b,1); return 0 }   // 0->a,1->b,2->c
21func ck(name: *u8, c: i64) -> i64 { if c==1 { g_puts("  PASS " as *u8) } else { g_puts("  FAIL " as *u8) } g_puts(name); g_puts("\n" as *u8); return c }
22
23const Q: i64 = 16
24const ONE: i64 = 65536
25const V: i64 = 3              // vocab: a,b,c
26const LOG2E: i64 = 94548
27const C1: i64 = 45426
28const C2: i64 = 15743
29func fxmul(a: i64, b: i64) -> i64 { return (a*b)>>Q }
30func exp_fx(x: i64) -> i64 {
31    var xx: i64=x; if xx>0 { xx=0 }
32    let yabs: i64=fxmul(0-xx, LOG2E); let nabs: i64=yabs>>Q; let fabs: i64=yabs-(nabs<<Q)
33    let f2: i64=fxmul(fabs,fabs); let p: i64=ONE+fxmul(fabs,C1)+fxmul(f2,C2); let inv: i64=(ONE*ONE)/p
34    if nabs>=31 { return 0 } return inv>>nabs
35}
36func softmax(logits: *i64, probs: *i64) -> i64 {
37    var mx: i64=logits[0]; var i: i64=1; while i<V { if logits[i]>mx { mx=logits[i] } i=i+1 }
38    let e: *i64=sys_mmap(V*8) as *i64; var sum: i64=0; i=0; while i<V { e[i]=exp_fx(logits[i]-mx); sum=sum+e[i]; i=i+1 }
39    i=0; while i<V { probs[i]=(e[i]*ONE)/sum; i=i+1 } return 0
40}
41func argmax(a: *i64) -> i64 { var bi: i64=0; var bv: i64=a[0]; var i: i64=1; while i<V { if a[i]>bv { bv=a[i]; bi=i } i=i+1 } return bi }
42
43// train the bigram LM on corpus[0..n); pcorr[0]=avg P(correct) epoch0, pcorr[1]=final. W is V*V, zero-init.
44func train(W: *i64, corpus: *i64, n: i64, epochs: i64, lr: i64, pcorr: *i64) -> i64 {
45    var i: i64=0; while i<V*V { W[i]=0; i=i+1 }
46    let probs: *i64=sys_mmap(V*8) as *i64
47    var ep: i64=0
48    while ep<epochs {
49        var pc: i64=0
50        var k: i64=0
51        while k<(n-1) {
52            let prev: i64=corpus[k]; let next: i64=corpus[k+1]
53            let row: *i64=sys_mmap(V*8) as *i64; var m: i64=0; while m<V { row[m]=W[prev*V+m]; m=m+1 } softmax(row, probs)
54            pc = pc + probs[next]
55            var j: i64=0; while j<V { var g: i64=probs[j]; if j==next { g=g-ONE } W[prev*V+j]=W[prev*V+j]-fxmul(lr,g); j=j+1 }
56            k=k+1
57        }
58        if ep==0 { pcorr[0]=pc/(n-1) }
59        if ep==(epochs-1) { pcorr[1]=pc/(n-1) }
60        ep=ep+1
61    }
62    return 0
63}
64func gen(W: *i64, start: i64, outids: *i64, steps: i64) -> i64 {
65    var cur: i64=start; var k: i64=0
66    while k<steps { outids[k]=cur; let row: *i64=sys_mmap(V*8) as *i64; var m: i64=0; while m<V { row[m]=W[cur*V+m]; m=m+1 } cur=argmax(row); k=k+1 }
67    return 0
68}
69
70func main() -> i64 {
71    g_puts("nx_nano_lm (OUR OWN SOVEREIGN nanoGPT -- char LM that TRAINS + GENERATES, integer/deterministic, $0)\n" as *u8)
72    var pass: i64=0; var total: i64=0
73    let N: i64=24
74    let corpus: *i64=sys_mmap(N*8) as *i64; var i: i64=0; while i<N { corpus[i]=i%V; i=i+1 }   // a,b,c,a,b,c,...
75    g_puts("  corpus = abcabc... ("); g_pn(N); g_puts(" chars, vocab a/b/c); model = bigram logit table, integer SGD\n" as *u8)
76    let W: *i64=sys_mmap(V*V*8) as *i64; let pc: *i64=sys_mmap(2*8) as *i64
77    train(W, corpus, N, 400, ONE/4, pc)
78    g_puts("  P(correct next): epoch0="); g_pn(pc[0]); g_puts(" final="); g_pn(pc[1]); g_puts(" (ONE=65536; ~22000=1/3 chance -> ~ONE=learned)\n" as *u8)
79    var t1: i64=0; if pc[1]>pc[0] { if pc[1]>(ONE*8/10) { t1=1 } }
80    pass=pass+ck("T1: P(correct next char) INCREASED to >0.8 -- the model LEARNED the character statistics" as *u8, t1); total=total+1
81
82    let gids: *i64=sys_mmap(6*8) as *i64; gen(W, 0, gids, 6)
83    g_puts("  GENERATED from 'a': "); var gi: i64=0; while gi<6 { g_ch(gids[gi]); gi=gi+1 } g_puts("  (want abcabc)\n" as *u8)
84    var t2: i64=0; if gids[0]==0 { if gids[1]==1 { if gids[2]==2 { if gids[3]==0 { if gids[4]==1 { if gids[5]==2 { t2=1 } } } } } }
85    pass=pass+ck("T2: GENERATION reproduces the learned pattern (a->b->c->a->b->c) -- it generates text it learned" as *u8, t2); total=total+1
86
87    let W2: *i64=sys_mmap(V*V*8) as *i64; let pc2: *i64=sys_mmap(2*8) as *i64; train(W2, corpus, N, 400, ONE/4, pc2)
88    var same: i64=1; i=0; while i<V*V { if W[i]!=W2[i] { same=0 } i=i+1 }
89    var t3: i64=0; if same==1 { if pc[1]==pc2[1] { t3=1 } }
90    pass=pass+ck("T3 (EXCEED): DETERMINISTIC -- two trainings give a BIT-IDENTICAL model (float/CUDA LM training cannot)" as *u8, t3); total=total+1
91
92    let r0: *i64=sys_mmap(V*8) as *i64; let r1: *i64=sys_mmap(V*8) as *i64; let r2: *i64=sys_mmap(V*8) as *i64
93    var m: i64=0; while m<V { r0[m]=W[0*V+m]; r1[m]=W[1*V+m]; r2[m]=W[2*V+m]; m=m+1 }
94    var t4: i64=0; if argmax(r0)==1 { if argmax(r1)==2 { if argmax(r2)==0 { t4=1 } } }
95    pass=pass+ck("T4: learned transitions -- argmax(W[a])=b, argmax(W[b])=c, argmax(W[c])=a (the bigram statistics)" as *u8, t4); total=total+1
96
97    g_puts("  -- OUR OPTIONS to train without thousands of $ (grounded ct_*.raw) --\n" as *u8)
98    g_puts("   1. DON'T train from scratch: transfer-learning + FINE-TUNE a free pretrained model (Hugging Face) -- minutes/$0-cheap.\n" as *u8)
99    g_puts("   2. LoRA / parameter-efficient fine-tune: train ~1% of weights -> single consumer GPU.\n" as *u8)
100    g_puts("   3. KNOWLEDGE DISTILLATION: train a small student from a big teacher's outputs (cheap, small final model).\n" as *u8)
101    g_puts("   4. FREE compute: Kaggle / Colab free GPU/TPU hours; or rent a SPOT instance (vast.ai/RunPod) by the hour.\n" as *u8)
102    g_puts("   5. SOVEREIGN (this organ): a tiny deterministic model trains on THIS laptop's CPU for $0 -- no GPU, no cloud, reproducible.\n" as *u8)
103
104    var okall: i64=0; if pass==total { okall=1 }
105    g_puts("---- nx_nano_lm: passed "); g_pn(pass); g_puts(" / "); g_pn(total); g_puts(" ----\n" as *u8)
106    if okall==1 {
107        let logf: i64=sys_openat_append("knowledge/status/nano_lm.log" as *u8, 420)
108        if logf>=0 { let z: i64=sys_write(logf,"NXNANOLM GREEN: sovereign char bigram LM -- trains by integer SGD (P-correct 1/3->~1), generates abcabc, deterministic, $0\n" as *u8,118); sys_close(logf) }
109        g_puts("verdict=GREEN (our own sovereign nanoGPT, bigram rung: trains by integer gradient descent + generates the learned pattern, deterministic, $0 on this laptop)\n" as *u8); sys_exit(0); return 0
110    }
111    g_puts("verdict=RED\n" as *u8); sys_exit(1); return 1
112}
nx_nano_lm.nx source

`nx_nano_lm.nx` source
