nx_nofloat_weighttie_gate.nx source

1// nx_nofloat_weighttie_gate.nx -- CAP-NF-WEIGHTTIE: tied input-embedding / output-projection (GPT-style 2// parameter efficiency), pure no-float Q16. The SAME embedding tensor E is used both to embed tokens AND as the 3// output projection (logits = hn . E^T via matmul_nt). One leaf node nE is referenced by two ops, so the 4// autograd tape ACCUMULATES gradients from the embed path + the output path into E -- elegant and exact. 5// Removes the separate Wlm (saves dm*V params). Trained on the 3-category grammar. 6// T1 the tied LM LEARNS (held-out CE << uniform ln(8)=2079). 7// T2 the tied LM is NEAR-OPTIMAL (CE ~= floor 964) -- tying does NOT hurt quality, at fewer params. 8// T3 TEETH: an UNTRAINED tied model has ~uniform CE -> the low CE is from training, not the architecture. 9// Sovereign: nx_nofloat_autograd + nx_syscalls. expect_exit: 0 10import "nx_nofloat_autograd.nx" 11import "nx_syscalls.nx" 12import "nx_gate_emit_lib.nx" 13const Q16: i64 = 65536 14const UNIFORM_MNAT: i64 = 2079 // ln(8) 15const FLOOR_MNAT: i64 = 964 16 17 18func dini(a: *i64, n: i64, sd: i64) -> i64 { var i: i64=0; while i<n { a[i]=(((i*7+sd*13+1)%11)-5)*13107; i=i+1 } return 0 } 19func lcg(st: *i64) -> i64 { st[0]=(st[0]*1103515245 + 12345) & 2147483647; return (st[0] >> 15) } 20func make_stream(S: *i64, tgt: *i64, P: i64, st: *i64) -> i64 { 21 var i: i64=0 22 while i<P { let c: i64=i%3; if c==0 { S[i]=lcg(st)%2 } if c==1 { S[i]=2+lcg(st)%3 } if c==2 { S[i]=5+lcg(st)%3 } i=i+1 } 23 i=0; while i<P-1 { tgt[i]=S[i+1]; i=i+1 } tgt[P-1]=S[0] 24 return 0 25} 26// TIED LM: nE used for both embed and output (matmul_nt(hn, E) = hn . E^T = logits). weights: E,Wq,Wk,Wv,Wo (5). 27func clm_tied(tape: *i64, vals: *i64, st: *i64, W: *i64, ids: *i64, tgt: *i64, T: i64, dm: i64, V: i64, scale: i64, lv: *i64) -> i64 { 28 st[0]=0; st[1]=0 29 let nE: i64=nfa_leaf(tape,vals,st,V,dm,W[0] as *i64,0) 30 let nWq: i64=nfa_leaf(tape,vals,st,dm,dm,W[1] as *i64,0) 31 let nWk: i64=nfa_leaf(tape,vals,st,dm,dm,W[2] as *i64,0) 32 let nWv: i64=nfa_leaf(tape,vals,st,dm,dm,W[3] as *i64,0) 33 let nWo: i64=nfa_leaf(tape,vals,st,dm,dm,W[4] as *i64,0) 34 let nX: i64=nfa_embed(tape,vals,st,nE,ids,T) 35 let nXn: i64=nfa_rmsnorm_rows(tape,vals,st,nX) 36 let nQ: i64=nfa_matmul(tape,vals,st,nXn,nWq) 37 let nK: i64=nfa_matmul(tape,vals,st,nXn,nWk) 38 let nV: i64=nfa_matmul(tape,vals,st,nXn,nWv) 39 let nQr: i64=nfa_rope(tape,vals,st,nQ) 40 let nKr: i64=nfa_rope(tape,vals,st,nK) 41 let nS: i64=nfa_matmul_nt(tape,vals,st,nQr,nKr) 42 let nSs: i64=nfa_cmul(tape,vals,st,nS,scale) 43 let nA: i64=nfa_softmax_rows(tape,vals,st,nSs,1) 44 let nO: i64=nfa_matmul(tape,vals,st,nA,nV) 45 let nOp: i64=nfa_matmul(tape,vals,st,nO,nWo) 46 let nH: i64=nfa_vadd(tape,vals,st,nX,nOp) 47 let nHn: i64=nfa_rmsnorm_rows(tape,vals,st,nH) 48 let nLg: i64=nfa_matmul_nt(tape,vals,st,nHn,nE) // <-- TIED: output projection reuses nE 49 let nLoss: i64=nfa_softce_rows(tape,vals,st,nLg,tgt) 50 lv[0]=nE; lv[1]=nWq; lv[2]=nWk; lv[3]=nWv; lv[4]=nWo; lv[5]=nLg 51 return nLoss 52} 53func step_all(tape: *i64, grads: *i64, W: *i64, WN: *i64, lv: *i64, lr: i64, clip: i64, gb: *i64) -> i64 { 54 var i: i64=0 55 while i<5 { let ar: *i64=W[i] as *i64; let cn: i64=WN[i]; let nd: i64=lv[i]; var c: i64=0; while c<cn { var g: i64=nfa_grad(tape,grads,nd,c); if g>clip{g=clip} if g<0-clip{g=0-clip} gb[c]=g; c=c+1 } nfa_sgd(ar,gb,cn,lr); i=i+1 } 56 return 0 57} 58func do_train(tape: *i64, vals: *i64, grads: *i64, st: *i64, W: *i64, WN: *i64, S: *i64, tgt: *i64, P: i64, dm: i64, V: i64, scale: i64, lv: *i64, gb: *i64, steps: i64, sdat: *i64) -> i64 { 59 var ep: i64=0 60 while ep<steps { make_stream(S,tgt,P,sdat); let nl: i64=clm_tied(tape,vals,st,W,S,tgt,P-1,dm,V,scale,lv); nfa_backward(tape,vals,grads,st[0],nl); step_all(tape,grads,W,WN,lv,6554,262144,gb); ep=ep+1 } 61 return 0 62} 63func eval_ce(tape: *i64, vals: *i64, st: *i64, W: *i64, S: *i64, tgt: *i64, P: i64, dm: i64, V: i64, scale: i64, lv: *i64, N: i64, sdat: *i64) -> i64 { 64 var acc: i64=0; var e: i64=0 65 while e<N { make_stream(S,tgt,P,sdat); let nl: i64=clm_tied(tape,vals,st,W,S,tgt,P-1,dm,V,scale,lv); acc=acc+nfa_val(tape,vals,nl,0); e=e+1 } 66 let mq: i64=acc/N 67 return (mq*1000)/Q16 68} 69 70func main() -> i64 { 71 g_puts("nx_nofloat_weighttie gate (tied embedding/output projection -- GPT-style parameter efficiency)\n" as *u8) 72 let V: i64=8; let P: i64=12; let dm: i64=24; let scale: i64=13377 73 let tape: *i64=sys_mmap(512*7*8) as *i64 74 let vals: *i64=sys_mmap(65536*8) as *i64 75 let grads: *i64=sys_mmap(65536*8) as *i64 76 let st: *i64=sys_mmap(2*8) as *i64 77 let nW: i64=5 78 let W: *i64=sys_mmap(nW*8) as *i64; let WN: *i64=sys_mmap(nW*8) as *i64 79 WN[0]=V*dm; WN[1]=dm*dm; WN[2]=dm*dm; WN[3]=dm*dm; WN[4]=dm*dm 80 var wi: i64=0; while wi<nW { let a: *i64=sys_mmap(WN[wi]*8) as *i64; dini(a,WN[wi],wi+1); W[wi]=a as i64; wi=wi+1 } 81 let lv: *i64=sys_mmap(8*8) as *i64; let gbuf: *i64=sys_mmap(4096*8) as *i64 82 let S: *i64=sys_mmap(P*8) as *i64; let tgt: *i64=sys_mmap(P*8) as *i64; let sdat: *i64=sys_mmap(8) as *i64 83 84 // T3 neg-control: untrained tied model CE (fresh weights) 85 sdat[0]=70707070 86 let untrained_ce: i64=eval_ce(tape,vals,st,W,S,tgt,P,dm,V,scale,lv,150,sdat) 87 sdat[0]=12345 88 do_train(tape,vals,grads,st,W,WN,S,tgt,P,dm,V,scale,lv,gbuf,55000,sdat) 89 sdat[0]=70707070 90 let ce: i64=eval_ce(tape,vals,st,W,S,tgt,P,dm,V,scale,lv,150,sdat) 91 92 var tied_params: i64=V*dm + 4*dm*dm 93 var untied_params: i64=tied_params + dm*V // an untied model would also have a separate Wlm[dm,V] 94 g_puts(" [measure] tied LM held-out CE="); g_pn(ce); g_puts(" milli-nats (untrained="); g_pn(untrained_ce); g_puts(", uniform=2079, floor=964) params: tied="); g_pn(tied_params); g_puts(" vs untied="); g_pn(untied_params); g_puts(" (saves "); g_pn(dm*V); g_puts(")\n") 95 96 var pass: i64=0; var total: i64=0 97 var t1: i64=0; if ce*10 <= UNIFORM_MNAT*7 { t1=1 } 98 pass=pass+g_check("T1: tied LM LEARNED (held-out CE << uniform)" as *u8, t1); total=total+1 99 var t2: i64=0; if ce <= FLOOR_MNAT+250 { t2=1 } 100 pass=pass+g_check("T2: tied LM NEAR-OPTIMAL (CE ~= floor) -- tying preserves quality at fewer params" as *u8, t2); total=total+1 101 var t3: i64=0; if untrained_ce > 1800 { if ce*2 < untrained_ce { t3=1 } } 102 pass=pass+g_check("T3: TEETH -- untrained tied model ~uniform; low CE is from TRAINING (E really learns both roles)" as *u8, t3); total=total+1 103 104 var okall: i64=0; if pass==total { okall=1 } 105 let logf: i64=sys_openat_append("knowledge/status/nofloat_weighttie.log" as *u8, 420) 106 if logf>=0 { let x0: i64=sys_write(logf,"NOFLOATWEIGHTTIE tied embedding/output near-optimal measured\n" as *u8,60); sys_close(logf) } 107 g_puts("---- weighttie gate: passed "); g_pn(pass); g_puts(" / "); g_pn(total); g_puts(" ----\n") 108 if okall==1 { g_puts("verdict=GREEN (tied embedding/output: near-optimal at fewer params; autograd accumulates both roles into E)\n" as *u8); sys_exit(0); return 0 } 109 g_puts("verdict=RED\n" as *u8); sys_exit(1); return 1 110}