nx_nofloat_scale2_gate.nx source

1// nx_nofloat_scale2_gate.nx -- R4 DEPTH test: does a 2-BLOCK no-float LM reach the richer-grammar floor where 2// the single block plateaued (ppl ~8.5)? Empirical (the lesson: TEST, don't assert). Same richer grammar 3// (DET2->ADJ4->NOUN5->VERB5, vocab 16) + dm=32, but TWO stacked pre-norm attention blocks. SGD lr=0.2 (the 4// best single-block optimizer config). Floor = avg(ln2,ln4,ln5,ln5)=1324 milli-nats (ppl 3.76); uniform=2773. 5// T1 held-out CE << uniform (learned). T2 held-out CE ~= floor (near-OPTIMAL -> R4 lands). 6// If T2 fails too, depth is NOT the lever and the boundary is genuinely optimization/compute (honest stop). 7// expect_exit: 0 Sovereign: nx_nofloat_autograd + nx_syscalls. 8import "nx_nofloat_autograd.nx" 9import "nx_syscalls.nx" 10import "nx_gate_emit_lib.nx" 11const Q16: i64 = 65536 12const UNIFORM_MNAT: i64 = 2773 13const FLOOR_MNAT: i64 = 1324 14 15 16func dini(a: *i64, n: i64, sd: i64) -> i64 { var i: i64=0; while i<n { a[i]=(((i*7+sd*13+1)%11)-5)*13107; i=i+1 } return 0 } 17func lcg(st: *i64) -> i64 { st[0]=(st[0]*1103515245 + 12345) & 2147483647; return (st[0] >> 15) } 18func make_stream4(S: *i64, tgt: *i64, P: i64, st: *i64) -> i64 { 19 var i: i64=0 20 while i<P { let c: i64=i%4; if c==0 { S[i]=lcg(st)%2 } if c==1 { S[i]=2+lcg(st)%4 } if c==2 { S[i]=6+lcg(st)%5 } if c==3 { S[i]=11+lcg(st)%5 } i=i+1 } 21 i=0; while i<P-1 { tgt[i]=S[i+1]; i=i+1 } tgt[P-1]=S[0] 22 return 0 23} 24// one pre-norm attention block: returns the new residual-stream node id (H = Hin + Wo(attn(rope(QKV(norm(Hin)))))) 25func block(tape: *i64, vals: *i64, st: *i64, nHin: i64, Wq: *i64, Wk: *i64, Wv: *i64, Wo: *i64, dm: i64, scale: i64, lv: *i64, base: i64) -> i64 { 26 let nWq: i64=nfa_leaf(tape,vals,st,dm,dm,Wq,0) 27 let nWk: i64=nfa_leaf(tape,vals,st,dm,dm,Wk,0) 28 let nWv: i64=nfa_leaf(tape,vals,st,dm,dm,Wv,0) 29 let nWo: i64=nfa_leaf(tape,vals,st,dm,dm,Wo,0) 30 let nXn: i64=nfa_rmsnorm_rows(tape,vals,st,nHin) 31 let nQ: i64=nfa_matmul(tape,vals,st,nXn,nWq) 32 let nK: i64=nfa_matmul(tape,vals,st,nXn,nWk) 33 let nV: i64=nfa_matmul(tape,vals,st,nXn,nWv) 34 let nQr: i64=nfa_rope(tape,vals,st,nQ) 35 let nKr: i64=nfa_rope(tape,vals,st,nK) 36 let nS: i64=nfa_matmul_nt(tape,vals,st,nQr,nKr) 37 let nSs: i64=nfa_cmul(tape,vals,st,nS,scale) 38 let nA: i64=nfa_softmax_rows(tape,vals,st,nSs,1) 39 let nO: i64=nfa_matmul(tape,vals,st,nA,nV) 40 let nOp: i64=nfa_matmul(tape,vals,st,nO,nWo) 41 let nH: i64=nfa_vadd(tape,vals,st,nHin,nOp) 42 lv[base]=nWq; lv[base+1]=nWk; lv[base+2]=nWv; lv[base+3]=nWo 43 return nH 44} 45// 2-block causal LM. weights: E, (Wq,Wk,Wv,Wo)x2, Wlm (10 arrays). leaves[0..9]=weights, leaves[10]=logits. 46func clm2(tape: *i64, vals: *i64, st: *i64, W: *i64, ids: *i64, tgt: *i64, T: i64, dm: i64, V: i64, scale: i64, lv: *i64) -> i64 { 47 st[0]=0; st[1]=0 48 let nE: i64=nfa_leaf(tape,vals,st,V,dm,W[0] as *i64,0) 49 let nX: i64=nfa_embed(tape,vals,st,nE,ids,T) 50 let nH1: i64=block(tape,vals,st,nX, W[1] as *i64,W[2] as *i64,W[3] as *i64,W[4] as *i64, dm,scale,lv,1) 51 let nH2: i64=block(tape,vals,st,nH1, W[5] as *i64,W[6] as *i64,W[7] as *i64,W[8] as *i64, dm,scale,lv,5) 52 let nWlm: i64=nfa_leaf(tape,vals,st,dm,V,W[9] as *i64,0) 53 let nHn: i64=nfa_rmsnorm_rows(tape,vals,st,nH2) 54 let nLg: i64=nfa_matmul(tape,vals,st,nHn,nWlm) 55 let nLoss: i64=nfa_softce_rows(tape,vals,st,nLg,tgt) 56 lv[0]=nE; lv[9]=nWlm; lv[10]=nLg 57 return nLoss 58} 59func step_all(tape: *i64, grads: *i64, W: *i64, WN: *i64, lv: *i64, lr: i64, clip: i64, gb: *i64) -> i64 { 60 var i: i64=0 61 while i<10 { let ar: *i64=W[i] as *i64; let cn: i64=WN[i]; let nd: i64=lv[i]; var c: i64=0; while c<cn { var g: i64=nfa_grad(tape,grads,nd,c); if g>clip{g=clip} if g<0-clip{g=0-clip} gb[c]=g; c=c+1 } nfa_sgd(ar,gb,cn,lr); i=i+1 } 62 return 0 63} 64func do_train(tape: *i64, vals: *i64, grads: *i64, st: *i64, W: *i64, WN: *i64, S: *i64, tgt: *i64, P: i64, dm: i64, V: i64, scale: i64, lv: *i64, gb: *i64, steps: i64, sdat: *i64) -> i64 { 65 var ep: i64=0 66 while ep<steps { make_stream4(S,tgt,P,sdat); let nl: i64=clm2(tape,vals,st,W,S,tgt,P-1,dm,V,scale,lv); nfa_backward(tape,vals,grads,st[0],nl); step_all(tape,grads,W,WN,lv,13107,262144,gb); ep=ep+1 } 67 return 0 68} 69func eval_ce(tape: *i64, vals: *i64, st: *i64, W: *i64, S: *i64, tgt: *i64, P: i64, dm: i64, V: i64, scale: i64, lv: *i64, N: i64, sdat: *i64) -> i64 { 70 var acc: i64=0; var e: i64=0 71 while e<N { make_stream4(S,tgt,P,sdat); let nl: i64=clm2(tape,vals,st,W,S,tgt,P-1,dm,V,scale,lv); acc=acc+nfa_val(tape,vals,nl,0); e=e+1 } 72 let mq: i64=acc/N 73 return (mq*1000)/Q16 74} 75 76func main() -> i64 { 77 g_puts("nx_nofloat_scale2 gate (R4 DEPTH test: 2-block no-float LM on the richer 16-word grammar)\n" as *u8) 78 let V: i64=16; let P: i64=16; let dm: i64=32; let scale: i64=11585 79 let tape: *i64=sys_mmap(2048*7*8) as *i64 80 let vals: *i64=sys_mmap(262144*8) as *i64 81 let grads: *i64=sys_mmap(262144*8) as *i64 82 let st: *i64=sys_mmap(2*8) as *i64 83 let nW: i64=10 84 let W: *i64=sys_mmap(nW*8) as *i64; let WN: *i64=sys_mmap(nW*8) as *i64 85 WN[0]=V*dm; var bi: i64=1; while bi<9 { WN[bi]=dm*dm; bi=bi+1 } WN[9]=dm*V 86 var wi: i64=0; while wi<nW { let a: *i64=sys_mmap(WN[wi]*8) as *i64; dini(a,WN[wi],wi+1); W[wi]=a as i64; wi=wi+1 } 87 let lv: *i64=sys_mmap(12*8) as *i64; let gbuf: *i64=sys_mmap(4096*8) as *i64 88 let S: *i64=sys_mmap(P*8) as *i64; let tgt: *i64=sys_mmap(P*8) as *i64; let sdat: *i64=sys_mmap(8) as *i64 89 90 g_puts(" 2 stacked pre-norm attention blocks; vocab=16, dm=32, SGD lr=0.2\n" as *u8) 91 sdat[0]=12345 92 do_train(tape,vals,grads,st,W,WN,S,tgt,P,dm,V,scale,lv,gbuf,18000,sdat) 93 sdat[0]=24682468 94 let ce: i64=eval_ce(tape,vals,st,W,S,tgt,P,dm,V,scale,lv,150,sdat) 95 96 g_puts(" [measure] held-out CE="); g_pn(ce); g_puts(" milli-nats uniform=2773 floor=1324 (single-block plateaued ~2135)\n") 97 98 var pass: i64=0; var total: i64=0 99 var t1: i64=0; if ce*10 <= UNIFORM_MNAT*7 { t1=1 } 100 pass=pass+g_check("T1: held-out CE << uniform (the 2-block LM learned the richer language)" as *u8, t1); total=total+1 101 var t2: i64=0; if ce <= FLOOR_MNAT+200 { t2=1 } 102 pass=pass+g_check("T2: held-out CE ~= floor (near-OPTIMAL -> depth reached it -> R4 lands)" as *u8, t2); total=total+1 103 104 var okall: i64=0; if pass==total { okall=1 } 105 g_puts("---- scale2 gate: passed "); g_pn(pass); g_puts(" / "); g_pn(total); g_puts(" ----\n") 106 if okall==1 { g_puts("verdict=GREEN (depth reached the richer floor -- R4 lands at scale, pure no-float)\n" as *u8); sys_exit(0); return 0 } 107 g_puts("verdict=RED\n" as *u8); sys_exit(1); return 1 108}