nx_nofloat_landing_gate.nx source

1// nx_nofloat_landing_gate.nx -- THE MEASURED LANDING (roadmap R4 increment; finding 7 = perplexity is the 2// standard cheap held-out metric, nfs_perplexity.raw). A no-float LM is "landed" on a domain when its 3// HELD-OUT per-token cross-entropy reaches the domain's INFORMATION FLOOR (the irreducible entropy) and is far 4// below the uniform baseline. Domain = the grammar [ARTICLE,NOUN,VERB] (fresh-random members), whose intrinsic 5// per-token entropy = avg(ln2, ln3, ln3) ~= 0.964 nats (perplexity ~= 2.62): an article position has 2 equally 6// likely members, noun/verb positions have 3 -- the members are RANDOM so NO model can beat that floor; the 7// uniform-over-vocab baseline is ln(8) = 2.079 nats (perplexity 8). We train on fresh-random streams (so the 8// model learns the GRAMMAR, not sequences) and measure CE on NEVER-SEEN streams. 9// T1 held-out CE << uniform ln(8) (the model beat the baseline = it learned). 10// T2 held-out CE ~= the information floor (near-OPTIMAL = landed: it reached the domain's entropy limit). 11// Pure integer Q16. Sovereign: nx_nofloat_autograd + nx_syscalls. expect_exit: 0 12import "nx_nofloat_autograd.nx" 13import "nx_syscalls.nx" 14import "nx_gate_emit_lib.nx" 15import "nx_gate_verdict.nx" 16const Q16: i64 = 65536 17// reference points in milli-nats (x1000): uniform ln(8)=2079, grammar floor avg(ln2,ln3,ln3)=964 18const UNIFORM_MNAT: i64 = 2079 19const FLOOR_MNAT: i64 = 964 20 21 22func dini(a: *i64, n: i64, sd: i64) -> i64 { var i: i64=0; while i<n { a[i]=(((i*7+sd*13+1)%11)-5)*13107; i=i+1 } return 0 } 23func lcg(st: *i64) -> i64 { st[0]=(st[0]*1103515245 + 12345) & 2147483647; return (st[0] >> 15) } 24func make_stream(S: *i64, tgt: *i64, P: i64, st: *i64) -> i64 { 25 var i: i64=0 26 while i<P { let c: i64=i%3; if c==0 { S[i]=lcg(st)%2 } if c==1 { S[i]=2+lcg(st)%3 } if c==2 { S[i]=5+lcg(st)%3 } i=i+1 } 27 i=0; while i<P-1 { tgt[i]=S[i+1]; i=i+1 } tgt[P-1]=S[0] 28 return 0 29} 30func clm_fwd(tape: *i64, vals: *i64, st: *i64, W: *i64, ids: *i64, tgt: *i64, T: i64, dm: i64, V: i64, scale: i64, leaves: *i64) -> i64 { 31 let E: *i64=W[0] as *i64; let Wq: *i64=W[1] as *i64; let Wk: *i64=W[2] as *i64; let Wv: *i64=W[3] as *i64; let Wo: *i64=W[4] as *i64; let Wlm: *i64=W[5] as *i64 32 st[0]=0; st[1]=0 33 let nE: i64=nfa_leaf(tape,vals,st,V,dm,E,0) 34 let nWq: i64=nfa_leaf(tape,vals,st,dm,dm,Wq,0) 35 let nWk: i64=nfa_leaf(tape,vals,st,dm,dm,Wk,0) 36 let nWv: i64=nfa_leaf(tape,vals,st,dm,dm,Wv,0) 37 let nWo: i64=nfa_leaf(tape,vals,st,dm,dm,Wo,0) 38 let nWlm: i64=nfa_leaf(tape,vals,st,dm,V,Wlm,0) 39 let nX: i64=nfa_embed(tape,vals,st,nE,ids,T) 40 let nXn: i64=nfa_rmsnorm_rows(tape,vals,st,nX) 41 let nQ: i64=nfa_matmul(tape,vals,st,nXn,nWq) 42 let nK: i64=nfa_matmul(tape,vals,st,nXn,nWk) 43 let nV: i64=nfa_matmul(tape,vals,st,nXn,nWv) 44 let nQr: i64=nfa_rope(tape,vals,st,nQ) 45 let nKr: i64=nfa_rope(tape,vals,st,nK) 46 let nS: i64=nfa_matmul_nt(tape,vals,st,nQr,nKr) 47 let nSs: i64=nfa_cmul(tape,vals,st,nS,scale) 48 let nA: i64=nfa_softmax_rows(tape,vals,st,nSs,1) 49 let nO: i64=nfa_matmul(tape,vals,st,nA,nV) 50 let nOp: i64=nfa_matmul(tape,vals,st,nO,nWo) 51 let nH: i64=nfa_vadd(tape,vals,st,nX,nOp) 52 let nHn: i64=nfa_rmsnorm_rows(tape,vals,st,nH) 53 let nLg: i64=nfa_matmul(tape,vals,st,nHn,nWlm) 54 let nLoss: i64=nfa_softce_rows(tape,vals,st,nLg,tgt) 55 leaves[0]=nE; leaves[1]=nWq; leaves[2]=nWk; leaves[3]=nWv; leaves[4]=nWo; leaves[5]=nWlm; leaves[6]=nLg 56 return nLoss 57} 58func step_all(tape: *i64, grads: *i64, W: *i64, WN: *i64, leaves: *i64, nW: i64, lr: i64, clip: i64, gb: *i64) -> i64 { 59 var i: i64=0 60 while i<nW { let ar: *i64=W[i] as *i64; let cn: i64=WN[i]; let nd: i64=leaves[i]; var c: i64=0; while c<cn { var g: i64=nfa_grad(tape,grads,nd,c); if g>clip{g=clip} if g<0-clip{g=0-clip} gb[c]=g; c=c+1 } nfa_sgd(ar,gb,cn,lr); i=i+1 } 61 return 0 62} 63func do_train(tape: *i64, vals: *i64, grads: *i64, st: *i64, W: *i64, WN: *i64, S: *i64, tgt: *i64, P: i64, dm: i64, V: i64, scale: i64, leaves: *i64, gb: *i64, steps: i64, sdat: *i64) -> i64 { 64 var ep: i64=0 65 while ep < steps { make_stream(S,tgt,P,sdat); let nl: i64=clm_fwd(tape,vals,st,W,S,tgt,P-1,dm,V,scale,leaves); nfa_backward(tape,vals,grads,st[0],nl); step_all(tape,grads,W,WN,leaves,6,6554,262144,gb); ep=ep+1 } 66 return 0 67} 68// held-out mean per-token CE in milli-nats: average the softce loss (mean CE/token, Q16 nats) over N fresh streams 69func eval_ce(tape: *i64, vals: *i64, st: *i64, W: *i64, S: *i64, tgt: *i64, P: i64, dm: i64, V: i64, scale: i64, leaves: *i64, N: i64, sdat: *i64) -> i64 { 70 var acc: i64=0; var e: i64=0 71 while e<N { 72 make_stream(S,tgt,P,sdat) 73 let nl: i64=clm_fwd(tape,vals,st,W,S,tgt,P-1,dm,V,scale,leaves) 74 acc = acc + nfa_val(tape,vals,nl,0) // mean CE/token for this stream (Q16 nats) 75 e=e+1 76 } 77 let mean_q16: i64 = acc / N // mean held-out CE (Q16 nats) 78 return (mean_q16 * 1000) / Q16 // -> milli-nats 79} 80 81func main() -> i64 { 82 g_puts("nx_nofloat_landing gate (MEASURED LANDING: held-out perplexity at the domain information floor)\n" as *u8) 83 let V: i64=8; let P: i64=12; let dm: i64=24; let scale: i64=13377 84 let tape: *i64 = sys_mmap(512*7*8) as *i64 85 let vals: *i64 = sys_mmap(65536*8) as *i64 86 let grads: *i64 = sys_mmap(65536*8) as *i64 87 let st: *i64 = sys_mmap(2*8) as *i64 88 let nW: i64=6 89 let W: *i64 = sys_mmap(nW*8) as *i64 90 let WN: *i64 = sys_mmap(nW*8) as *i64 91 WN[0]=V*dm; WN[1]=dm*dm; WN[2]=dm*dm; WN[3]=dm*dm; WN[4]=dm*dm; WN[5]=dm*V 92 var wi: i64=0 93 while wi<nW { let a: *i64=sys_mmap(WN[wi]*8) as *i64; dini(a,WN[wi],wi+1); W[wi]=a as i64; wi=wi+1 } 94 let leaves: *i64 = sys_mmap(8*8) as *i64 95 let gbuf: *i64 = sys_mmap(4096*8) as *i64 96 let S: *i64 = sys_mmap(P*8) as *i64; let tgt: *i64 = sys_mmap(P*8) as *i64 97 let sdat: *i64 = sys_mmap(8) as *i64 98 99 sdat[0]=12345 100 do_train(tape,vals,grads,st,W,WN,S,tgt,P,dm,V,scale,leaves,gbuf,30000,sdat) 101 sdat[0]=70707070 102 let ce: i64 = eval_ce(tape,vals,st,W,S,tgt,P,dm,V,scale,leaves,200,sdat) // held-out CE, milli-nats 103 104 g_puts(" [measure] held-out CE=" as *u8); g_pn(ce); g_puts(" milli-nats uniform-baseline ln(8)=" as *u8); g_pn(UNIFORM_MNAT); g_puts(" information floor avg(ln2,ln3,ln3)=" as *u8); g_pn(FLOOR_MNAT); g_puts(" (perplexity=exp(CE))\n" as *u8) 105 106 var pass: i64=0; var total: i64=0 107 var t1: i64=0; if ce*10 <= UNIFORM_MNAT*7 { t1=1 } // held-out CE <= 0.7x uniform = clearly beat baseline 108 pass=pass+g_check("T1: held-out CE << uniform baseline (the no-float LM LEARNED the domain)" as *u8, t1); total=total+1 109 var t2: i64=0; if ce <= FLOOR_MNAT+150 { t2=1 } // within 0.15 nats of the information floor = near-OPTIMAL 110 pass=pass+g_check("T2: held-out CE ~= the information FLOOR (near-OPTIMAL = landed at the domain entropy limit)" as *u8, t2); total=total+1 111 112 var okall: i64=0; if pass==total { okall=1 } 113 if okall==1 { 114 let logf: i64 = sys_openat_append("knowledge/status/nofloat_landing.log" as *u8, 420) 115 if logf >= 0 { let x0: i64=sys_write(logf,"NOFLOATLANDING held-out CE/perplexity at domain floor measured\n" as *u8,62); sys_close(logf) } 116 } 117 g_puts("---- landing gate: passed " as *u8); g_pn(pass); g_puts(" / " as *u8); g_pn(total); g_puts(" ----\n" as *u8) 118 // MIGRATED onto nx_gate_verdict by nx_gate_dry_apply (D001, minimal form): every check 119 // row above is untouched, so the PASS/FAIL vector cannot change; only the hand-rolled 120 // verdict emission is replaced by the ONE shared base class. Proven by nx_gate_migrate verify. 121 let ctr__dry: *i64 = gv_ctr() 122 ctr__dry[0] = pass 123 ctr__dry[1] = total 124 let rc__dry: i64 = gv_verdict("NOFLOAT-LANDING-GATE" as *u8, ctr__dry, "LANDED: a no-float LM near-optimal on its domain by held-out perplexity -- the affordable land)" as *u8) 125 sys_exit(rc__dry) 126 return rc__dry 127}