nx_nofloat_landing_gate.nx source

1// nx_nofloat_landing_gate.nx -- THE MEASURED LANDING (roadmap R4 increment; finding 7 = perplexity is the 2// standard cheap held-out metric, nfs_perplexity.raw). A no-float LM is "landed" on a domain when its 3// HELD-OUT per-token cross-entropy reaches the domain's INFORMATION FLOOR (the irreducible entropy) and is far 4// below the uniform baseline. Domain = the grammar [ARTICLE,NOUN,VERB] (fresh-random members), whose intrinsic 5// per-token entropy = avg(ln2, ln3, ln3) ~= 0.964 nats (perplexity ~= 2.62): an article position has 2 equally 6// likely members, noun/verb positions have 3 -- the members are RANDOM so NO model can beat that floor; the 7// uniform-over-vocab baseline is ln(8) = 2.079 nats (perplexity 8). We train on fresh-random streams (so the 8// model learns the GRAMMAR, not sequences) and measure CE on NEVER-SEEN streams. 9// T1 held-out CE << uniform ln(8) (the model beat the baseline = it learned). 10// T2 held-out CE ~= the information floor (near-OPTIMAL = landed: it reached the domain's entropy limit). 11// Pure integer Q16. Sovereign: nx_nofloat_autograd + nx_syscalls. expect_exit: 0 12import "nx_nofloat_autograd.nx" 13import "nx_syscalls.nx" 14import "nx_gate_emit_lib.nx" 15const Q16: i64 = 65536 16// reference points in milli-nats (x1000): uniform ln(8)=2079, grammar floor avg(ln2,ln3,ln3)=964 17const UNIFORM_MNAT: i64 = 2079 18const FLOOR_MNAT: i64 = 964 19 20 21func dini(a: *i64, n: i64, sd: i64) -> i64 { var i: i64=0; while i<n { a[i]=(((i*7+sd*13+1)%11)-5)*13107; i=i+1 } return 0 } 22func lcg(st: *i64) -> i64 { st[0]=(st[0]*1103515245 + 12345) & 2147483647; return (st[0] >> 15) } 23func make_stream(S: *i64, tgt: *i64, P: i64, st: *i64) -> i64 { 24 var i: i64=0 25 while i<P { let c: i64=i%3; if c==0 { S[i]=lcg(st)%2 } if c==1 { S[i]=2+lcg(st)%3 } if c==2 { S[i]=5+lcg(st)%3 } i=i+1 } 26 i=0; while i<P-1 { tgt[i]=S[i+1]; i=i+1 } tgt[P-1]=S[0] 27 return 0 28} 29func clm_fwd(tape: *i64, vals: *i64, st: *i64, W: *i64, ids: *i64, tgt: *i64, T: i64, dm: i64, V: i64, scale: i64, leaves: *i64) -> i64 { 30 let E: *i64=W[0] as *i64; let Wq: *i64=W[1] as *i64; let Wk: *i64=W[2] as *i64; let Wv: *i64=W[3] as *i64; let Wo: *i64=W[4] as *i64; let Wlm: *i64=W[5] as *i64 31 st[0]=0; st[1]=0 32 let nE: i64=nfa_leaf(tape,vals,st,V,dm,E,0) 33 let nWq: i64=nfa_leaf(tape,vals,st,dm,dm,Wq,0) 34 let nWk: i64=nfa_leaf(tape,vals,st,dm,dm,Wk,0) 35 let nWv: i64=nfa_leaf(tape,vals,st,dm,dm,Wv,0) 36 let nWo: i64=nfa_leaf(tape,vals,st,dm,dm,Wo,0) 37 let nWlm: i64=nfa_leaf(tape,vals,st,dm,V,Wlm,0) 38 let nX: i64=nfa_embed(tape,vals,st,nE,ids,T) 39 let nXn: i64=nfa_rmsnorm_rows(tape,vals,st,nX) 40 let nQ: i64=nfa_matmul(tape,vals,st,nXn,nWq) 41 let nK: i64=nfa_matmul(tape,vals,st,nXn,nWk) 42 let nV: i64=nfa_matmul(tape,vals,st,nXn,nWv) 43 let nQr: i64=nfa_rope(tape,vals,st,nQ) 44 let nKr: i64=nfa_rope(tape,vals,st,nK) 45 let nS: i64=nfa_matmul_nt(tape,vals,st,nQr,nKr) 46 let nSs: i64=nfa_cmul(tape,vals,st,nS,scale) 47 let nA: i64=nfa_softmax_rows(tape,vals,st,nSs,1) 48 let nO: i64=nfa_matmul(tape,vals,st,nA,nV) 49 let nOp: i64=nfa_matmul(tape,vals,st,nO,nWo) 50 let nH: i64=nfa_vadd(tape,vals,st,nX,nOp) 51 let nHn: i64=nfa_rmsnorm_rows(tape,vals,st,nH) 52 let nLg: i64=nfa_matmul(tape,vals,st,nHn,nWlm) 53 let nLoss: i64=nfa_softce_rows(tape,vals,st,nLg,tgt) 54 leaves[0]=nE; leaves[1]=nWq; leaves[2]=nWk; leaves[3]=nWv; leaves[4]=nWo; leaves[5]=nWlm; leaves[6]=nLg 55 return nLoss 56} 57func step_all(tape: *i64, grads: *i64, W: *i64, WN: *i64, leaves: *i64, nW: i64, lr: i64, clip: i64, gb: *i64) -> i64 { 58 var i: i64=0 59 while i<nW { let ar: *i64=W[i] as *i64; let cn: i64=WN[i]; let nd: i64=leaves[i]; var c: i64=0; while c<cn { var g: i64=nfa_grad(tape,grads,nd,c); if g>clip{g=clip} if g<0-clip{g=0-clip} gb[c]=g; c=c+1 } nfa_sgd(ar,gb,cn,lr); i=i+1 } 60 return 0 61} 62func do_train(tape: *i64, vals: *i64, grads: *i64, st: *i64, W: *i64, WN: *i64, S: *i64, tgt: *i64, P: i64, dm: i64, V: i64, scale: i64, leaves: *i64, gb: *i64, steps: i64, sdat: *i64) -> i64 { 63 var ep: i64=0 64 while ep < steps { make_stream(S,tgt,P,sdat); let nl: i64=clm_fwd(tape,vals,st,W,S,tgt,P-1,dm,V,scale,leaves); nfa_backward(tape,vals,grads,st[0],nl); step_all(tape,grads,W,WN,leaves,6,6554,262144,gb); ep=ep+1 } 65 return 0 66} 67// held-out mean per-token CE in milli-nats: average the softce loss (mean CE/token, Q16 nats) over N fresh streams 68func eval_ce(tape: *i64, vals: *i64, st: *i64, W: *i64, S: *i64, tgt: *i64, P: i64, dm: i64, V: i64, scale: i64, leaves: *i64, N: i64, sdat: *i64) -> i64 { 69 var acc: i64=0; var e: i64=0 70 while e<N { 71 make_stream(S,tgt,P,sdat) 72 let nl: i64=clm_fwd(tape,vals,st,W,S,tgt,P-1,dm,V,scale,leaves) 73 acc = acc + nfa_val(tape,vals,nl,0) // mean CE/token for this stream (Q16 nats) 74 e=e+1 75 } 76 let mean_q16: i64 = acc / N // mean held-out CE (Q16 nats) 77 return (mean_q16 * 1000) / Q16 // -> milli-nats 78} 79 80func main() -> i64 { 81 g_puts("nx_nofloat_landing gate (MEASURED LANDING: held-out perplexity at the domain information floor)\n" as *u8) 82 let V: i64=8; let P: i64=12; let dm: i64=24; let scale: i64=13377 83 let tape: *i64 = sys_mmap(512*7*8) as *i64 84 let vals: *i64 = sys_mmap(65536*8) as *i64 85 let grads: *i64 = sys_mmap(65536*8) as *i64 86 let st: *i64 = sys_mmap(2*8) as *i64 87 let nW: i64=6 88 let W: *i64 = sys_mmap(nW*8) as *i64 89 let WN: *i64 = sys_mmap(nW*8) as *i64 90 WN[0]=V*dm; WN[1]=dm*dm; WN[2]=dm*dm; WN[3]=dm*dm; WN[4]=dm*dm; WN[5]=dm*V 91 var wi: i64=0 92 while wi<nW { let a: *i64=sys_mmap(WN[wi]*8) as *i64; dini(a,WN[wi],wi+1); W[wi]=a as i64; wi=wi+1 } 93 let leaves: *i64 = sys_mmap(8*8) as *i64 94 let gbuf: *i64 = sys_mmap(4096*8) as *i64 95 let S: *i64 = sys_mmap(P*8) as *i64; let tgt: *i64 = sys_mmap(P*8) as *i64 96 let sdat: *i64 = sys_mmap(8) as *i64 97 98 sdat[0]=12345 99 do_train(tape,vals,grads,st,W,WN,S,tgt,P,dm,V,scale,leaves,gbuf,30000,sdat) 100 sdat[0]=70707070 101 let ce: i64 = eval_ce(tape,vals,st,W,S,tgt,P,dm,V,scale,leaves,200,sdat) // held-out CE, milli-nats 102 103 g_puts(" [measure] held-out CE=" as *u8); g_pn(ce); g_puts(" milli-nats uniform-baseline ln(8)=" as *u8); g_pn(UNIFORM_MNAT); g_puts(" information floor avg(ln2,ln3,ln3)=" as *u8); g_pn(FLOOR_MNAT); g_puts(" (perplexity=exp(CE))\n" as *u8) 104 105 var pass: i64=0; var total: i64=0 106 var t1: i64=0; if ce*10 <= UNIFORM_MNAT*7 { t1=1 } // held-out CE <= 0.7x uniform = clearly beat baseline 107 pass=pass+g_check("T1: held-out CE << uniform baseline (the no-float LM LEARNED the domain)" as *u8, t1); total=total+1 108 var t2: i64=0; if ce <= FLOOR_MNAT+150 { t2=1 } // within 0.15 nats of the information floor = near-OPTIMAL 109 pass=pass+g_check("T2: held-out CE ~= the information FLOOR (near-OPTIMAL = landed at the domain entropy limit)" as *u8, t2); total=total+1 110 111 var okall: i64=0; if pass==total { okall=1 } 112 if okall==1 { 113 let logf: i64 = sys_openat_append("knowledge/status/nofloat_landing.log" as *u8, 420) 114 if logf >= 0 { let x0: i64=sys_write(logf,"NOFLOATLANDING held-out CE/perplexity at domain floor measured\n" as *u8,62); sys_close(logf) } 115 } 116 g_puts("---- landing gate: passed " as *u8); g_pn(pass); g_puts(" / " as *u8); g_pn(total); g_puts(" ----\n" as *u8) 117 if okall==1 { g_puts("verdict=GREEN (LANDED: a no-float LM near-optimal on its domain by held-out perplexity -- the affordable land)\n" as *u8); sys_exit(0); return 0 } 118 g_puts("verdict=RED\n" as *u8); sys_exit(1); return 1 119}