nx_nofloat_generalize_gate.nx source

1// nx_nofloat_generalize_gate.nx -- HELD-OUT RULE GENERALIZATION in pure integer Q16 (CAP-NF-GENERALIZE). The 2// real test of learning vs memorization: train the no-float char-LM on a periodic sequence "abcabc..." (it must 3// learn the first-order RULE a->b, b->c, c->a), then evaluate on data it NEVER trained on: 4// T1 a HELD-OUT string "cabcab..." (SAME rule, DIFFERENT string) -> high next-char accuracy (>> chance 1/3). 5// Succeeding on a SHIFTED string proves the model learned the CHAR-rule, not position-by-position memorization. 6// T2 (teeth) a CONTROL "acbacb..." (DIFFERENT rule a->c,c->b,b->a) -> ~chance accuracy. Proves the model applies 7// the SPECIFIC learned rule (it is not outputting constants or trivially-correct). 8// Lean (nx_cc .s ceiling): exactly 2 clm_fwd call sites -- do_train (helper) + eval_acc (helper, called twice). 9// HONEST: first-order periodic rule on a tiny task = a real but modest generalization (not deep in-context learning). 10// Sovereign: nx_nofloat_autograd + nx_syscalls. expect_exit: 0 11import "nx_nofloat_autograd.nx" 12import "nx_g_puts_lib.nx" 13import "nx_syscalls.nx" 14const Q16: i64 = 65536 15 16func g_pn(v: i64) -> i64 { let b: *u8=sys_mmap(28); var x: i64=v; if x<0{b[0]=45;sys_write(1,b,1);x=0-x} if x==0{b[0]=48;sys_write(1,b,1);return 0} var d: i64=0; var y: i64=x; while y>0{d=d+1;y=y/10} var i: i64=d-1; y=x; while i>=0{b[i]=(48+(y%10)) as u8;y=y/10;i=i-1} sys_write(1,b,d); return 0 } 17func slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n } 18func dini(a: *i64, n: i64, sd: i64) -> i64 { var i: i64=0; while i<n { a[i]=(((i*7+sd*13+1)%11)-5)*13107; i=i+1 } return 0 } 19func amx(tape: *i64, vals: *i64, logn: i64, r: i64, V: i64) -> i64 { let o: i64=tape[7*logn+5]; var b: i64=0; var bv: i64=vals[o+r*V]; var j: i64=1; while j<V { if vals[o+r*V+j]>bv { bv=vals[o+r*V+j]; b=j } j=j+1 } return b } 20 21func clm_fwd(tape: *i64, vals: *i64, st: *i64, W: *i64, ids: *i64, tgt: *i64, T: i64, dm: i64, V: i64, scale: i64, leaves: *i64) -> i64 { 22 let E: *i64=W[0] as *i64; let Wq: *i64=W[1] as *i64; let Wk: *i64=W[2] as *i64; let Wv: *i64=W[3] as *i64; let Wo: *i64=W[4] as *i64; let Wlm: *i64=W[5] as *i64 23 st[0]=0; st[1]=0 24 let nE: i64=nfa_leaf(tape,vals,st,V,dm,E,0) 25 let nWq: i64=nfa_leaf(tape,vals,st,dm,dm,Wq,0) 26 let nWk: i64=nfa_leaf(tape,vals,st,dm,dm,Wk,0) 27 let nWv: i64=nfa_leaf(tape,vals,st,dm,dm,Wv,0) 28 let nWo: i64=nfa_leaf(tape,vals,st,dm,dm,Wo,0) 29 let nWlm: i64=nfa_leaf(tape,vals,st,dm,V,Wlm,0) 30 let nX: i64=nfa_embed(tape,vals,st,nE,ids,T) 31 let nXn: i64=nfa_rmsnorm_rows(tape,vals,st,nX) 32 let nQ: i64=nfa_matmul(tape,vals,st,nXn,nWq) 33 let nK: i64=nfa_matmul(tape,vals,st,nXn,nWk) 34 let nV: i64=nfa_matmul(tape,vals,st,nXn,nWv) 35 let nQr: i64=nfa_rope(tape,vals,st,nQ) 36 let nKr: i64=nfa_rope(tape,vals,st,nK) 37 let nS: i64=nfa_matmul_nt(tape,vals,st,nQr,nKr) 38 let nSs: i64=nfa_cmul(tape,vals,st,nS,scale) 39 let nA: i64=nfa_softmax_rows(tape,vals,st,nSs,1) 40 let nO: i64=nfa_matmul(tape,vals,st,nA,nV) 41 let nOp: i64=nfa_matmul(tape,vals,st,nO,nWo) 42 let nH: i64=nfa_vadd(tape,vals,st,nX,nOp) 43 let nHn: i64=nfa_rmsnorm_rows(tape,vals,st,nH) 44 let nLg: i64=nfa_matmul(tape,vals,st,nHn,nWlm) 45 let nLoss: i64=nfa_softce_rows(tape,vals,st,nLg,tgt) 46 leaves[0]=nE; leaves[1]=nWq; leaves[2]=nWk; leaves[3]=nWv; leaves[4]=nWo; leaves[5]=nWlm; leaves[6]=nLg 47 return nLoss 48} 49func step_all(tape: *i64, grads: *i64, W: *i64, WN: *i64, leaves: *i64, nW: i64, lr: i64, clip: i64, gb: *i64) -> i64 { 50 var i: i64=0 51 while i<nW { let ar: *i64=W[i] as *i64; let cn: i64=WN[i]; let nd: i64=leaves[i]; var c: i64=0; while c<cn { var g: i64=nfa_grad(tape,grads,nd,c); if g>clip{g=clip} if g<0-clip{g=0-clip} gb[c]=g; c=c+1 } nfa_sgd(ar,gb,cn,lr); i=i+1 } 52 return 0 53} 54// call site 1: train all weights on (ids,tgt) for `steps` 55func do_train(tape: *i64, vals: *i64, grads: *i64, st: *i64, W: *i64, WN: *i64, ids: *i64, tgt: *i64, T: i64, dm: i64, V: i64, scale: i64, leaves: *i64, gb: *i64, steps: i64) -> i64 { 56 var ep: i64=0 57 while ep < steps { let nl: i64=clm_fwd(tape,vals,st,W,ids,tgt,T,dm,V,scale,leaves); nfa_backward(tape,vals,grads,st[0],nl); step_all(tape,grads,W,WN,leaves,6,6554,262144,gb); ep=ep+1 } 58 return 0 59} 60// call site 2: next-char accuracy of the trained model on a sequence (predict pos 0..T-1 -> compare to ids2[pos+1]) 61func eval_acc(tape: *i64, vals: *i64, st: *i64, W: *i64, ids2: *i64, T: i64, dm: i64, V: i64, scale: i64, leaves: *i64) -> i64 { 62 let nl: i64=clm_fwd(tape,vals,st,W,ids2,ids2,T,dm,V,scale,leaves) 63 var ok: i64=0; var p: i64=0 64 while p<T { if amx(tape,vals,leaves[6],p,V)==ids2[p+1] { ok=ok+1 } p=p+1 } 65 return ok 66} 67 68func main() -> i64 { 69 g_puts("nx_nofloat_generalize gate (HELD-OUT rule generalization, pure integer Q16)\n" as *u8) 70 let train: *u8 = "abcabcabcabcabcabc" as *u8 // learn rule a->b, b->c, c->a 71 let heldout:*u8 = "cabcabcabcabcabcab" as *u8 // SAME rule, DIFFERENT string (never trained) 72 let control:*u8 = "acbacbacbacbacbacb" as *u8 // DIFFERENT rule a->c, c->b, b->a (teeth) 73 let L: i64 = slen(train) 74 // shared char->id map built from train (a=0,b=1,c=2) 75 let c2i: *i64 = sys_mmap(256*8) as *i64 76 var iz: i64=0; while iz<256 { c2i[iz]=0-1; iz=iz+1 } 77 var V: i64=0; var pz: i64=0 78 while pz<L { let ch: i64=train[pz] as i64; if c2i[ch]<0 { c2i[ch]=V; V=V+1 } pz=pz+1 } 79 let s_tr: *i64 = sys_mmap(L*8) as *i64; let s_ho: *i64 = sys_mmap(L*8) as *i64; let s_ct: *i64 = sys_mmap(L*8) as *i64 80 pz=0; while pz<L { s_tr[pz]=c2i[train[pz] as i64]; s_ho[pz]=c2i[heldout[pz] as i64]; s_ct[pz]=c2i[control[pz] as i64]; pz=pz+1 } 81 let T: i64 = L-1 82 g_puts(" train=\"" as *u8); g_puts(train); g_puts("\" heldout=\"" as *u8); g_puts(heldout); g_puts("\" control=\"" as *u8); g_puts(control); g_puts("\" V=" as *u8); g_pn(V); g_puts("\n" as *u8) 83 84 let dm: i64=16; let scale: i64=16384 85 let tape: *i64 = sys_mmap(512*7*8) as *i64 86 let vals: *i64 = sys_mmap(65536*8) as *i64 87 let grads: *i64 = sys_mmap(65536*8) as *i64 88 let st: *i64 = sys_mmap(2*8) as *i64 89 let nW: i64=6 90 let W: *i64 = sys_mmap(nW*8) as *i64 91 let WN: *i64 = sys_mmap(nW*8) as *i64 92 WN[0]=V*dm; WN[1]=dm*dm; WN[2]=dm*dm; WN[3]=dm*dm; WN[4]=dm*dm; WN[5]=dm*V 93 var wi: i64=0 94 while wi<nW { let a: *i64=sys_mmap(WN[wi]*8) as *i64; dini(a,WN[wi],wi+1); W[wi]=a as i64; wi=wi+1 } 95 let leaves: *i64 = sys_mmap(8*8) as *i64 96 let gbuf: *i64 = sys_mmap(4096*8) as *i64 97 98 do_train(tape,vals,grads,st,W,WN,s_tr,((s_tr as i64)+8) as *i64,T,dm,V,scale,leaves,gbuf,25000) 99 let ho: i64 = eval_acc(tape,vals,st,W,s_ho,T,dm,V,scale,leaves) 100 let ct: i64 = eval_acc(tape,vals,st,W,s_ct,T,dm,V,scale,leaves) 101 g_puts(" [measure] held-out(same rule) acc=" as *u8); g_pn(ho); g_puts("/" as *u8); g_pn(T); g_puts(" control(diff rule) acc=" as *u8); g_pn(ct); g_puts("/" as *u8); g_pn(T); g_puts(" (chance~" as *u8); g_pn(T/V); g_puts(")\n" as *u8) 102 103 // chance = T/V (~29% for V=3). Deterministic training -> ho/ct are stable run-to-run. 104 var pass: i64=0; var total: i64=2 105 var t1: i64=0; if ho*2 >= T { t1=1 } // held-out >= 50% (>> ~29% chance) = generalizes above chance 106 if t1==1 { g_puts(" PASS T1 held-out generalization: same-rule string predicted well above chance (learned the rule, not positions)\n" as *u8); pass=pass+1 } else { g_puts(" FAIL T1\n" as *u8) } 107 var t2: i64=0; if ho >= ct+3 { t2=1 } // held-out clearly beats wrong-rule control = rule-SPECIFIC transfer (teeth) 108 if t2==1 { g_puts(" PASS T2 rule-specific: held-out (same rule) clearly beats the wrong-rule control (it transfers THE learned rule)\n" as *u8); pass=pass+1 } else { g_puts(" FAIL T2\n" as *u8) } 109 110 let logf: i64 = sys_openat_append("knowledge/status/nofloat_generalize.log" as *u8, 420) 111 if logf >= 0 { let w0: i64=sys_write(logf,"NOFLOATGENERALIZE heldout/control measured\n" as *u8,42); sys_close(logf) } 112 g_puts("---- generalize gate: passed " as *u8); g_pn(pass); g_puts(" / " as *u8); g_pn(total); g_puts(" ----\n" as *u8) 113 if pass==total { g_puts("verdict=GREEN (held-out rule generalization: learns the rule, predicts unseen same-rule data, fails wrong-rule control)\n" as *u8); sys_exit(0); return 0 } 114 g_puts("verdict=RED\n" as *u8); sys_exit(1); return 1 115}