nx_nofloat_scale2_gate.nx source

1// nx_nofloat_scale2_gate.nx -- R4 DEPTH test: does a 2-BLOCK no-float LM reach the richer-grammar floor where 2// the single block plateaued (ppl ~8.5)? Empirical (the lesson: TEST, don't assert). Same richer grammar 3// (DET2->ADJ4->NOUN5->VERB5, vocab 16) + dm=32, but TWO stacked pre-norm attention blocks. SGD lr=0.2 (the 4// best single-block optimizer config). Floor = avg(ln2,ln4,ln5,ln5)=1324 milli-nats (ppl 3.76); uniform=2773. 5// T1 held-out CE << uniform (learned). T2 held-out CE ~= floor (near-OPTIMAL -> R4 lands). 6// If T2 fails too, depth is NOT the lever and the boundary is genuinely optimization/compute (honest stop). 7// expect_exit: 0 Sovereign: nx_nofloat_autograd + nx_syscalls. 8import "nx_nofloat_autograd.nx" 9import "nx_syscalls.nx" 10import "nx_gate_emit_lib.nx" 11import "nx_gate_verdict.nx" 12const Q16: i64 = 65536 13const UNIFORM_MNAT: i64 = 2773 14const FLOOR_MNAT: i64 = 1324 15 16 17func dini(a: *i64, n: i64, sd: i64) -> i64 { var i: i64=0; while i<n { a[i]=(((i*7+sd*13+1)%11)-5)*13107; i=i+1 } return 0 } 18func lcg(st: *i64) -> i64 { st[0]=(st[0]*1103515245 + 12345) & 2147483647; return (st[0] >> 15) } 19func make_stream4(S: *i64, tgt: *i64, P: i64, st: *i64) -> i64 { 20 var i: i64=0 21 while i<P { let c: i64=i%4; if c==0 { S[i]=lcg(st)%2 } if c==1 { S[i]=2+lcg(st)%4 } if c==2 { S[i]=6+lcg(st)%5 } if c==3 { S[i]=11+lcg(st)%5 } i=i+1 } 22 i=0; while i<P-1 { tgt[i]=S[i+1]; i=i+1 } tgt[P-1]=S[0] 23 return 0 24} 25// one pre-norm attention block: returns the new residual-stream node id (H = Hin + Wo(attn(rope(QKV(norm(Hin)))))) 26func block(tape: *i64, vals: *i64, st: *i64, nHin: i64, Wq: *i64, Wk: *i64, Wv: *i64, Wo: *i64, dm: i64, scale: i64, lv: *i64, base: i64) -> i64 { 27 let nWq: i64=nfa_leaf(tape,vals,st,dm,dm,Wq,0) 28 let nWk: i64=nfa_leaf(tape,vals,st,dm,dm,Wk,0) 29 let nWv: i64=nfa_leaf(tape,vals,st,dm,dm,Wv,0) 30 let nWo: i64=nfa_leaf(tape,vals,st,dm,dm,Wo,0) 31 let nXn: i64=nfa_rmsnorm_rows(tape,vals,st,nHin) 32 let nQ: i64=nfa_matmul(tape,vals,st,nXn,nWq) 33 let nK: i64=nfa_matmul(tape,vals,st,nXn,nWk) 34 let nV: i64=nfa_matmul(tape,vals,st,nXn,nWv) 35 let nQr: i64=nfa_rope(tape,vals,st,nQ) 36 let nKr: i64=nfa_rope(tape,vals,st,nK) 37 let nS: i64=nfa_matmul_nt(tape,vals,st,nQr,nKr) 38 let nSs: i64=nfa_cmul(tape,vals,st,nS,scale) 39 let nA: i64=nfa_softmax_rows(tape,vals,st,nSs,1) 40 let nO: i64=nfa_matmul(tape,vals,st,nA,nV) 41 let nOp: i64=nfa_matmul(tape,vals,st,nO,nWo) 42 let nH: i64=nfa_vadd(tape,vals,st,nHin,nOp) 43 lv[base]=nWq; lv[base+1]=nWk; lv[base+2]=nWv; lv[base+3]=nWo 44 return nH 45} 46// 2-block causal LM. weights: E, (Wq,Wk,Wv,Wo)x2, Wlm (10 arrays). leaves[0..9]=weights, leaves[10]=logits. 47func clm2(tape: *i64, vals: *i64, st: *i64, W: *i64, ids: *i64, tgt: *i64, T: i64, dm: i64, V: i64, scale: i64, lv: *i64) -> i64 { 48 st[0]=0; st[1]=0 49 let nE: i64=nfa_leaf(tape,vals,st,V,dm,W[0] as *i64,0) 50 let nX: i64=nfa_embed(tape,vals,st,nE,ids,T) 51 let nH1: i64=block(tape,vals,st,nX, W[1] as *i64,W[2] as *i64,W[3] as *i64,W[4] as *i64, dm,scale,lv,1) 52 let nH2: i64=block(tape,vals,st,nH1, W[5] as *i64,W[6] as *i64,W[7] as *i64,W[8] as *i64, dm,scale,lv,5) 53 let nWlm: i64=nfa_leaf(tape,vals,st,dm,V,W[9] as *i64,0) 54 let nHn: i64=nfa_rmsnorm_rows(tape,vals,st,nH2) 55 let nLg: i64=nfa_matmul(tape,vals,st,nHn,nWlm) 56 let nLoss: i64=nfa_softce_rows(tape,vals,st,nLg,tgt) 57 lv[0]=nE; lv[9]=nWlm; lv[10]=nLg 58 return nLoss 59} 60func step_all(tape: *i64, grads: *i64, W: *i64, WN: *i64, lv: *i64, lr: i64, clip: i64, gb: *i64) -> i64 { 61 var i: i64=0 62 while i<10 { let ar: *i64=W[i] as *i64; let cn: i64=WN[i]; let nd: i64=lv[i]; var c: i64=0; while c<cn { var g: i64=nfa_grad(tape,grads,nd,c); if g>clip{g=clip} if g<0-clip{g=0-clip} gb[c]=g; c=c+1 } nfa_sgd(ar,gb,cn,lr); i=i+1 } 63 return 0 64} 65func do_train(tape: *i64, vals: *i64, grads: *i64, st: *i64, W: *i64, WN: *i64, S: *i64, tgt: *i64, P: i64, dm: i64, V: i64, scale: i64, lv: *i64, gb: *i64, steps: i64, sdat: *i64) -> i64 { 66 var ep: i64=0 67 while ep<steps { make_stream4(S,tgt,P,sdat); let nl: i64=clm2(tape,vals,st,W,S,tgt,P-1,dm,V,scale,lv); nfa_backward(tape,vals,grads,st[0],nl); step_all(tape,grads,W,WN,lv,13107,262144,gb); ep=ep+1 } 68 return 0 69} 70func eval_ce(tape: *i64, vals: *i64, st: *i64, W: *i64, S: *i64, tgt: *i64, P: i64, dm: i64, V: i64, scale: i64, lv: *i64, N: i64, sdat: *i64) -> i64 { 71 var acc: i64=0; var e: i64=0 72 while e<N { make_stream4(S,tgt,P,sdat); let nl: i64=clm2(tape,vals,st,W,S,tgt,P-1,dm,V,scale,lv); acc=acc+nfa_val(tape,vals,nl,0); e=e+1 } 73 let mq: i64=acc/N 74 return (mq*1000)/Q16 75} 76 77func main() -> i64 { 78 g_puts("nx_nofloat_scale2 gate (R4 DEPTH test: 2-block no-float LM on the richer 16-word grammar)\n" as *u8) 79 let V: i64=16; let P: i64=16; let dm: i64=32; let scale: i64=11585 80 let tape: *i64=sys_mmap(2048*7*8) as *i64 81 let vals: *i64=sys_mmap(262144*8) as *i64 82 let grads: *i64=sys_mmap(262144*8) as *i64 83 let st: *i64=sys_mmap(2*8) as *i64 84 let nW: i64=10 85 let W: *i64=sys_mmap(nW*8) as *i64; let WN: *i64=sys_mmap(nW*8) as *i64 86 WN[0]=V*dm; var bi: i64=1; while bi<9 { WN[bi]=dm*dm; bi=bi+1 } WN[9]=dm*V 87 var wi: i64=0; while wi<nW { let a: *i64=sys_mmap(WN[wi]*8) as *i64; dini(a,WN[wi],wi+1); W[wi]=a as i64; wi=wi+1 } 88 let lv: *i64=sys_mmap(12*8) as *i64; let gbuf: *i64=sys_mmap(4096*8) as *i64 89 let S: *i64=sys_mmap(P*8) as *i64; let tgt: *i64=sys_mmap(P*8) as *i64; let sdat: *i64=sys_mmap(8) as *i64 90 91 g_puts(" 2 stacked pre-norm attention blocks; vocab=16, dm=32, SGD lr=0.2\n" as *u8) 92 sdat[0]=12345 93 do_train(tape,vals,grads,st,W,WN,S,tgt,P,dm,V,scale,lv,gbuf,18000,sdat) 94 sdat[0]=24682468 95 let ce: i64=eval_ce(tape,vals,st,W,S,tgt,P,dm,V,scale,lv,150,sdat) 96 97 g_puts(" [measure] held-out CE="); g_pn(ce); g_puts(" milli-nats uniform=2773 floor=1324 (single-block plateaued ~2135)\n") 98 99 var pass: i64=0; var total: i64=0 100 var t1: i64=0; if ce*10 <= UNIFORM_MNAT*7 { t1=1 } 101 pass=pass+g_check("T1: held-out CE << uniform (the 2-block LM learned the richer language)" as *u8, t1); total=total+1 102 var t2: i64=0; if ce <= FLOOR_MNAT+200 { t2=1 } 103 pass=pass+g_check("T2: held-out CE ~= floor (near-OPTIMAL -> depth reached it -> R4 lands)" as *u8, t2); total=total+1 104 105 var okall: i64=0; if pass==total { okall=1 } 106 g_puts("---- scale2 gate: passed "); g_pn(pass); g_puts(" / "); g_pn(total); g_puts(" ----\n") 107 // MIGRATED onto nx_gate_verdict by nx_gate_dry_apply (D001, minimal form): every check 108 // row above is untouched, so the PASS/FAIL vector cannot change; only the hand-rolled 109 // verdict emission is replaced by the ONE shared base class. Proven by nx_gate_migrate verify. 110 let ctr__dry: *i64 = gv_ctr() 111 ctr__dry[0] = pass 112 ctr__dry[1] = total 113 let rc__dry: i64 = gv_verdict("NOFLOAT-SCALE2-GATE" as *u8, ctr__dry, "depth reached the richer floor -- R4 lands at scale, pure no-float)" as *u8) 114 sys_exit(rc__dry) 115 return rc__dry 116}