nx_nofloat_sample_gate.nx source

1// nx_nofloat_sample_gate.nx -- TEMPERATURE SAMPLING decode in pure integer Q16 (CAP-NF-SAMPLE). Real LLMs 2// GENERATE by sampling from the softmax distribution (with a temperature), not just greedy argmax. The no-float 3// stack had only argmax; this adds a CORRECT fixed-point sampler: a deterministic LCG PRNG + softmax-with- 4// temperature (scale logits by 1/T, max-subtract, fxexp, normalize) + inverse-CDF pick. All integer -> the 5// sampling is REPRODUCIBLE given a seed (the no-float determinism exceed: same seed+weights -> same text, which 6// float/GPU sampling cannot guarantee bit-for-bit). 7// 8// T1 LOW temperature concentrates on the argmax (>= 90% of draws) -- approaches greedy as T->0. 9// T2 at temperature=1 the EMPIRICAL sample frequencies match softmax(logits) (a CORRECT sampler, measured). 10// T3 REPRODUCIBLE: two runs with the same seed produce byte-identical samples (no-float determinism). 11// T4 HIGH temperature is measurably MORE UNIFORM than low (argmax-frequency drops) -- temperature has teeth. 12// 13// Evidence -> knowledge/status/nofloat_sample.log. Sovereign: nx_nofloat_autograd (nfa_fxexp/nfa_qmul) + nx_syscalls. 14// expect_exit: 0 license_tier: ORIGINAL 15import "nx_nofloat_autograd.nx" 16import "nx_syscalls.nx" 17import "nx_gate_emit_lib.nx" 18const Q16: i64 = 65536 19 20 21func g_abs(v: i64) -> i64 { if v<0 { return 0-v } return v } 22func s_ws(fd: i64, s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(fd,s,n); return 0 } 23func s_wn(fd: i64, v: i64) -> i64 { let b: *u8=sys_mmap(28); var m: i64=v; if m<0{sys_write(fd,"-" as *u8,1);m=0-m} let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i: i64=0; while i<k{b[i]=t[k-1-i];i=i+1} sys_write(fd,b,k); return 0 } 24 25// deterministic LCG -> u in [0, Q16). state updated in place. (glibc constants; >>15 takes high bits.) 26func lcg(st: *i64) -> i64 { st[0] = (st[0]*1103515245 + 12345) & 2147483647; return (st[0] >> 15) & 65535 } 27// out = softmax(logits * invtemp) in Q16 (invtemp = 1/T, Q16). max-subtract for stability; nfa_fxexp(arg<=0). 28func softmax_temp(logits: *i64, n: i64, invtemp: i64, out: *i64) -> i64 { 29 var i: i64=0 30 while i<n { out[i] = nfa_qmul(logits[i], invtemp); i=i+1 } 31 var mx: i64=out[0]; i=1 32 while i<n { if out[i]>mx { mx=out[i] } i=i+1 } 33 var sum: i64=0; i=0 34 while i<n { let e: i64=nfa_fxexp(out[i]-mx); out[i]=e; sum=sum+e; i=i+1 } 35 if sum<=0 { sum=1 } 36 i=0 37 while i<n { out[i]=(out[i]<<16)/sum; i=i+1 } 38 return 0 39} 40// inverse-CDF sample: pick first i with cumulative prob > u. probs Q16 (~sum Q16), u in [0,Q16). 41func sample_idx(probs: *i64, n: i64, u: i64) -> i64 { 42 var c: i64=0; var i: i64=0 43 while i<n { c=c+probs[i]; if u<c { return i } i=i+1 } 44 return n-1 45} 46func argmax_of(probs: *i64, n: i64) -> i64 { var b: i64=0; var bv: i64=probs[0]; var i: i64=1; while i<n { if probs[i]>bv { bv=probs[i]; b=i } i=i+1 } return b } 47 48func main() -> i64 { 49 g_puts("nx_nofloat_sample gate (TEMPERATURE sampling decode in pure integer Q16 -- MEASURED)\n" as *u8) 50 var pass: i64=0; var total: i64=0 51 let n: i64=4 52 let logits: *i64 = sys_mmap(n*8) as *i64 53 logits[0]=32768; logits[1]=65536; logits[2]=131072; logits[3]=0 // [0.5, 1.0, 2.0, 0.0]; argmax=class 2 54 let probs: *i64 = sys_mmap(n*8) as *i64 55 let cnt: *i64 = sys_mmap(n*8) as *i64 56 let st: *i64 = sys_mmap(8) as *i64 57 58 // ---- T1: low temperature (invtemp=8.0) concentrates on argmax ---- 59 softmax_temp(logits, n, 8*Q16, probs) 60 let am: i64 = argmax_of(probs, n) 61 st[0]=12345 62 var draws: i64=4000; var hitArg: i64=0; var d: i64=0 63 while d<draws { let u: i64=lcg(st); let s: i64=sample_idx(probs,n,u); if s==am { hitArg=hitArg+1 } d=d+1 } 64 g_puts(" [measure] low-temp(T=1/8): argmax=class " as *u8); g_pn(am); g_puts(", argmax-draws=" as *u8); g_pn(hitArg); g_puts("/" as *u8); g_pn(draws); g_puts("\n" as *u8) 65 var t1: i64=0; if hitArg*100 >= draws*90 { t1=1 } 66 pass=pass+g_check("T1: low temperature concentrates on argmax (>= 90% of draws)" as *u8, t1); total=total+1 67 68 // ---- T2: temperature=1 empirical frequencies match softmax(logits) ---- 69 softmax_temp(logits, n, Q16, probs) 70 var z: i64=0; while z<n { cnt[z]=0; z=z+1 } 71 st[0]=999 72 draws=8000; d=0 73 while d<draws { let u: i64=lcg(st); let s: i64=sample_idx(probs,n,u); cnt[s]=cnt[s]+1; d=d+1 } 74 var t2: i64=1; var worst: i64=0; var i: i64=0 75 while i<n { 76 let emp: i64=(cnt[i]<<16)/draws // empirical freq in Q16 77 let df: i64=g_abs(emp-probs[i]) 78 if df>worst { worst=df } 79 if df > 4915 { t2=0 } // tolerance 0.075 (4915/65536) 80 g_puts(" class " as *u8); g_pn(i); g_puts(": softmax=" as *u8); g_pn((probs[i]*1000)>>16); g_puts("/1000 empirical=" as *u8); g_pn((emp*1000)>>16); g_puts("/1000\n" as *u8) 81 i=i+1 82 } 83 g_puts(" [measure] worst |empirical-softmax| = " as *u8); g_pn((worst*1000)>>16); g_puts("/1000 (tol=75/1000)\n" as *u8) 84 pass=pass+g_check("T2: temperature=1 empirical frequencies MATCH softmax(logits) (correct sampler)" as *u8, t2); total=total+1 85 86 // ---- T3: reproducible (same seed -> identical sample sequence) ---- 87 softmax_temp(logits, n, Q16, probs) 88 let seqA: *i64 = sys_mmap(64*8) as *i64; let seqB: *i64 = sys_mmap(64*8) as *i64 89 st[0]=2026; var k: i64=0; while k<64 { seqA[k]=sample_idx(probs,n,lcg(st)); k=k+1 } 90 st[0]=2026; k=0; while k<64 { seqB[k]=sample_idx(probs,n,lcg(st)); k=k+1 } 91 var t3: i64=1; k=0; while k<64 { if seqA[k]!=seqB[k] { t3=0 } k=k+1 } 92 pass=pass+g_check("T3: reproducible -- same seed gives byte-identical samples (no-float determinism)" as *u8, t3); total=total+1 93 94 // ---- T4: high temperature is more uniform than low (argmax-frequency drops) ---- 95 softmax_temp(logits, n, 16384, probs) // invtemp=0.25 -> T=4 (flatter) 96 let amH: i64 = argmax_of(probs, n) 97 st[0]=777; draws=4000; var hitH: i64=0; d=0 98 while d<draws { if sample_idx(probs,n,lcg(st))==amH { hitH=hitH+1 } d=d+1 } 99 g_puts(" [measure] high-temp(T=4): argmax-draws=" as *u8); g_pn(hitH); g_puts("/" as *u8); g_pn(draws); g_puts(" vs low-temp " as *u8); g_pn(hitArg); g_puts("/" as *u8); g_pn(4000); g_puts("\n" as *u8) 100 var t4: i64=0; if hitH < hitArg { t4=1 } // high temp picks argmax LESS often = more uniform 101 pass=pass+g_check("T4: high temperature is MORE UNIFORM than low (argmax-frequency drops) -- teeth" as *u8, t4); total=total+1 102 103 var okall: i64=0; if pass==total { okall=1 } 104 let logf: i64 = sys_openat_append("knowledge/status/nofloat_sample.log" as *u8, 420) 105 if logf >= 0 { 106 s_ws(logf,"NOFLOATSAMPLE T1_lowtemp=" as *u8); s_wn(logf,t1); s_ws(logf," T2_distmatch=" as *u8); s_wn(logf,t2); s_ws(logf," T3_repro=" as *u8); s_wn(logf,t3); s_ws(logf," T4_temp_teeth=" as *u8); s_wn(logf,t4) 107 if okall==1 { s_ws(logf," verdict=GREEN\n" as *u8) } else { s_ws(logf," verdict=RED\n" as *u8) } 108 sys_close(logf) 109 } 110 g_puts("---- nofloat_sample gate: passed " as *u8); g_pn(pass); g_puts(" / " as *u8); g_pn(total); g_puts(" ----\n" as *u8) 111 if okall==1 { g_puts("verdict=GREEN (a correct temperature sampler in pure integer Q16; reproducible decode)\n" as *u8); sys_exit(0); return 0 } 112 g_puts("verdict=RED\n" as *u8); sys_exit(1); return 1 113}