nx_nofloat_sample_gate.nx source

1// nx_nofloat_sample_gate.nx -- TEMPERATURE SAMPLING decode in pure integer Q16 (CAP-NF-SAMPLE). Real LLMs 2// GENERATE by sampling from the softmax distribution (with a temperature), not just greedy argmax. The no-float 3// stack had only argmax; this adds a CORRECT fixed-point sampler: a deterministic LCG PRNG + softmax-with- 4// temperature (scale logits by 1/T, max-subtract, fxexp, normalize) + inverse-CDF pick. All integer -> the 5// sampling is REPRODUCIBLE given a seed (the no-float determinism exceed: same seed+weights -> same text, which 6// float/GPU sampling cannot guarantee bit-for-bit). 7// 8// T1 LOW temperature concentrates on the argmax (>= 90% of draws) -- approaches greedy as T->0. 9// T2 at temperature=1 the EMPIRICAL sample frequencies match softmax(logits) (a CORRECT sampler, measured). 10// T3 REPRODUCIBLE: two runs with the same seed produce byte-identical samples (no-float determinism). 11// T4 HIGH temperature is measurably MORE UNIFORM than low (argmax-frequency drops) -- temperature has teeth. 12// 13// Evidence -> knowledge/status/nofloat_sample.log. Sovereign: nx_nofloat_autograd (nfa_fxexp/nfa_qmul) + nx_syscalls. 14// expect_exit: 0 license_tier: ORIGINAL 15import "nx_nofloat_autograd.nx" 16import "nx_syscalls.nx" 17import "nx_gate_emit_lib.nx" 18import "nx_gate_verdict.nx" 19const Q16: i64 = 65536 20 21 22func g_abs(v: i64) -> i64 { if v<0 { return 0-v } return v } 23func s_ws(fd: i64, s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(fd,s,n); return 0 } 24func s_wn(fd: i64, v: i64) -> i64 { let b: *u8=sys_mmap(28); var m: i64=v; if m<0{sys_write(fd,"-" as *u8,1);m=0-m} let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i: i64=0; while i<k{b[i]=t[k-1-i];i=i+1} sys_write(fd,b,k); return 0 } 25 26// deterministic LCG -> u in [0, Q16). state updated in place. (glibc constants; >>15 takes high bits.) 27func lcg(st: *i64) -> i64 { st[0] = (st[0]*1103515245 + 12345) & 2147483647; return (st[0] >> 15) & 65535 } 28// out = softmax(logits * invtemp) in Q16 (invtemp = 1/T, Q16). max-subtract for stability; nfa_fxexp(arg<=0). 29func softmax_temp(logits: *i64, n: i64, invtemp: i64, out: *i64) -> i64 { 30 var i: i64=0 31 while i<n { out[i] = nfa_qmul(logits[i], invtemp); i=i+1 } 32 var mx: i64=out[0]; i=1 33 while i<n { if out[i]>mx { mx=out[i] } i=i+1 } 34 var sum: i64=0; i=0 35 while i<n { let e: i64=nfa_fxexp(out[i]-mx); out[i]=e; sum=sum+e; i=i+1 } 36 if sum<=0 { sum=1 } 37 i=0 38 while i<n { out[i]=(out[i]<<16)/sum; i=i+1 } 39 return 0 40} 41// inverse-CDF sample: pick first i with cumulative prob > u. probs Q16 (~sum Q16), u in [0,Q16). 42func sample_idx(probs: *i64, n: i64, u: i64) -> i64 { 43 var c: i64=0; var i: i64=0 44 while i<n { c=c+probs[i]; if u<c { return i } i=i+1 } 45 return n-1 46} 47func argmax_of(probs: *i64, n: i64) -> i64 { var b: i64=0; var bv: i64=probs[0]; var i: i64=1; while i<n { if probs[i]>bv { bv=probs[i]; b=i } i=i+1 } return b } 48 49func main() -> i64 { 50 g_puts("nx_nofloat_sample gate (TEMPERATURE sampling decode in pure integer Q16 -- MEASURED)\n" as *u8) 51 var pass: i64=0; var total: i64=0 52 let n: i64=4 53 let logits: *i64 = sys_mmap(n*8) as *i64 54 logits[0]=32768; logits[1]=65536; logits[2]=131072; logits[3]=0 // [0.5, 1.0, 2.0, 0.0]; argmax=class 2 55 let probs: *i64 = sys_mmap(n*8) as *i64 56 let cnt: *i64 = sys_mmap(n*8) as *i64 57 let st: *i64 = sys_mmap(8) as *i64 58 59 // ---- T1: low temperature (invtemp=8.0) concentrates on argmax ---- 60 softmax_temp(logits, n, 8*Q16, probs) 61 let am: i64 = argmax_of(probs, n) 62 st[0]=12345 63 var draws: i64=4000; var hitArg: i64=0; var d: i64=0 64 while d<draws { let u: i64=lcg(st); let s: i64=sample_idx(probs,n,u); if s==am { hitArg=hitArg+1 } d=d+1 } 65 g_puts(" [measure] low-temp(T=1/8): argmax=class " as *u8); g_pn(am); g_puts(", argmax-draws=" as *u8); g_pn(hitArg); g_puts("/" as *u8); g_pn(draws); g_puts("\n" as *u8) 66 var t1: i64=0; if hitArg*100 >= draws*90 { t1=1 } 67 pass=pass+g_check("T1: low temperature concentrates on argmax (>= 90% of draws)" as *u8, t1); total=total+1 68 69 // ---- T2: temperature=1 empirical frequencies match softmax(logits) ---- 70 softmax_temp(logits, n, Q16, probs) 71 var z: i64=0; while z<n { cnt[z]=0; z=z+1 } 72 st[0]=999 73 draws=8000; d=0 74 while d<draws { let u: i64=lcg(st); let s: i64=sample_idx(probs,n,u); cnt[s]=cnt[s]+1; d=d+1 } 75 var t2: i64=1; var worst: i64=0; var i: i64=0 76 while i<n { 77 let emp: i64=(cnt[i]<<16)/draws // empirical freq in Q16 78 let df: i64=g_abs(emp-probs[i]) 79 if df>worst { worst=df } 80 if df > 4915 { t2=0 } // tolerance 0.075 (4915/65536) 81 g_puts(" class " as *u8); g_pn(i); g_puts(": softmax=" as *u8); g_pn((probs[i]*1000)>>16); g_puts("/1000 empirical=" as *u8); g_pn((emp*1000)>>16); g_puts("/1000\n" as *u8) 82 i=i+1 83 } 84 g_puts(" [measure] worst |empirical-softmax| = " as *u8); g_pn((worst*1000)>>16); g_puts("/1000 (tol=75/1000)\n" as *u8) 85 pass=pass+g_check("T2: temperature=1 empirical frequencies MATCH softmax(logits) (correct sampler)" as *u8, t2); total=total+1 86 87 // ---- T3: reproducible (same seed -> identical sample sequence) ---- 88 softmax_temp(logits, n, Q16, probs) 89 let seqA: *i64 = sys_mmap(64*8) as *i64; let seqB: *i64 = sys_mmap(64*8) as *i64 90 st[0]=2026; var k: i64=0; while k<64 { seqA[k]=sample_idx(probs,n,lcg(st)); k=k+1 } 91 st[0]=2026; k=0; while k<64 { seqB[k]=sample_idx(probs,n,lcg(st)); k=k+1 } 92 var t3: i64=1; k=0; while k<64 { if seqA[k]!=seqB[k] { t3=0 } k=k+1 } 93 pass=pass+g_check("T3: reproducible -- same seed gives byte-identical samples (no-float determinism)" as *u8, t3); total=total+1 94 95 // ---- T4: high temperature is more uniform than low (argmax-frequency drops) ---- 96 softmax_temp(logits, n, 16384, probs) // invtemp=0.25 -> T=4 (flatter) 97 let amH: i64 = argmax_of(probs, n) 98 st[0]=777; draws=4000; var hitH: i64=0; d=0 99 while d<draws { if sample_idx(probs,n,lcg(st))==amH { hitH=hitH+1 } d=d+1 } 100 g_puts(" [measure] high-temp(T=4): argmax-draws=" as *u8); g_pn(hitH); g_puts("/" as *u8); g_pn(draws); g_puts(" vs low-temp " as *u8); g_pn(hitArg); g_puts("/" as *u8); g_pn(4000); g_puts("\n" as *u8) 101 var t4: i64=0; if hitH < hitArg { t4=1 } // high temp picks argmax LESS often = more uniform 102 pass=pass+g_check("T4: high temperature is MORE UNIFORM than low (argmax-frequency drops) -- teeth" as *u8, t4); total=total+1 103 104 var okall: i64=0; if pass==total { okall=1 } 105 let logf: i64 = sys_openat_append("knowledge/status/nofloat_sample.log" as *u8, 420) 106 if logf >= 0 { 107 s_ws(logf,"NOFLOATSAMPLE T1_lowtemp=" as *u8); s_wn(logf,t1); s_ws(logf," T2_distmatch=" as *u8); s_wn(logf,t2); s_ws(logf," T3_repro=" as *u8); s_wn(logf,t3); s_ws(logf," T4_temp_teeth=" as *u8); s_wn(logf,t4) 108 if okall==1 { s_ws(logf," verdict=GREEN\n" as *u8) } else { s_ws(logf," verdict=RED\n" as *u8) } 109 sys_close(logf) 110 } 111 g_puts("---- nofloat_sample gate: passed " as *u8); g_pn(pass); g_puts(" / " as *u8); g_pn(total); g_puts(" ----\n" as *u8) 112 // MIGRATED onto nx_gate_verdict by nx_gate_dry_apply (D001, minimal form): every check 113 // row above is untouched, so the PASS/FAIL vector cannot change; only the hand-rolled 114 // verdict emission is replaced by the ONE shared base class. Proven by nx_gate_migrate verify. 115 let ctr__dry: *i64 = gv_ctr() 116 ctr__dry[0] = pass 117 ctr__dry[1] = total 118 let rc__dry: i64 = gv_verdict("NOFLOAT-SAMPLE-GATE" as *u8, ctr__dry, "a correct temperature sampler in pure integer Q16; reproducible decode)" as *u8) 119 sys_exit(rc__dry) 120 return rc__dry 121}