nx_nofloat_attn.nx source

1// nx_nofloat_attn.nx -- SOVEREIGN NO-FLOAT ATTENTION (the transformer HEART), integer Q16, deterministic. The hard 2// part of attention is softmax (needs exp); this builds a fixed-point exp via 2^x decomposition (|x|*log2e -> integer 3// shift + quadratic 2^frac), then softmax (max-subtract for stability -> exp -> normalize) + attention (weighted sum 4// of V). KEY: even with an APPROXIMATE exp, softmax NORMALIZES (each / sum), so the weights sum to ONE exactly and 5// monotonicity holds -- all integer => bit-exact deterministic (the moat vs float/CUDA attention). 6// T1 weights non-negative + sum ~= ONE. T2 monotonic (highest logit -> highest weight, order preserved). 7// T3 (EXCEED) determinism bit-identical. T4 attention output = convex combo of V, dominated by the max-score key. 8// expect_exit: 0 Sovereign: nx_syscalls. 9import "nx_syscalls.nx" 10import "nx_g_puts_lib.nx" 11 12func g_pn(v: i64) -> i64 { let b: *u8=sys_mmap(28); var x: i64=v; if x<0{b[0]=45;sys_write(1,b,1);x=0-x} if x==0{b[0]=48;sys_write(1,b,1);return 0} var d: i64=0; var y: i64=x; while y>0{d=d+1;y=y/10} var i: i64=d-1; y=x; while i>=0{b[i]=(48+(y%10)) as u8;y=y/10;i=i-1} sys_write(1,b,d); return 0 } 13func ck(name: *u8, c: i64) -> i64 { if c==1 { g_puts(" PASS " as *u8) } else { g_puts(" FAIL " as *u8) } g_puts(name); g_puts("\n" as *u8); return c } 14 15const Q: i64 = 16 16const ONE: i64 = 65536 17const LOG2E: i64 = 94548 // 1.442695 in Q16 18const C1: i64 = 45426 // ln2 in Q16 (2^f linear term) 19const C2: i64 = 15743 // 0.2402 in Q16 (2^f quadratic term) 20func fxmul(a: i64, b: i64) -> i64 { return (a*b)>>Q } 21 22// fixed-point exp for x <= 0 (softmax subtracts the max, so inputs are <= 0). returns exp(x) in Q16, (0, ONE]. 23func exp_fx(x: i64) -> i64 { 24 var xx: i64 = x; if xx>0 { xx=0 } 25 let yabs: i64 = fxmul(0-xx, LOG2E) // |x|*log2(e) >= 0 26 let nabs: i64 = yabs >> Q // integer part (>=0) 27 let fabs: i64 = yabs - (nabs<<Q) // fractional part [0, ONE) 28 let f2: i64 = fxmul(fabs, fabs) 29 let p: i64 = ONE + fxmul(fabs, C1) + fxmul(f2, C2) // 2^fabs in [ONE, 2*ONE) 30 let inv: i64 = (ONE*ONE)/p // 2^(-fabs) in (ONE/2, ONE] 31 if nabs >= 31 { return 0 } 32 return inv >> nabs // 2^(-fabs) * 2^(-nabs) = exp(x) 33} 34// softmax over scores[0..n) -> w[0..n) in Q16 summing to ONE. 35func softmax(scores: *i64, w: *i64, n: i64) -> i64 { 36 var mx: i64 = scores[0]; var i: i64=1; while i<n { if scores[i]>mx { mx=scores[i] } i=i+1 } 37 let e: *i64 = sys_mmap(n*8) as *i64; var sum: i64=0 38 i=0; while i<n { e[i]=exp_fx(scores[i]-mx); sum=sum+e[i]; i=i+1 } 39 i=0; while i<n { w[i]=(e[i]*ONE)/sum; i=i+1 } 40 return sum 41} 42func attention(w: *i64, V: *i64, n: i64) -> i64 { var out: i64=0; var i: i64=0; while i<n { out=out+fxmul(w[i], V[i]); i=i+1 } return out } 43 44func main() -> i64 { 45 g_puts("nx_nofloat_attn (SOVEREIGN no-float attention/softmax -- the transformer heart, integer Q16, deterministic)\n" as *u8) 46 var pass: i64=0; var total: i64=0 47 g_puts(" exp check (Q16): exp(0)="); g_pn(exp_fx(0)); g_puts(" (ONE=65536) exp(-1)="); g_pn(exp_fx(0-ONE)); g_puts(" (~24109=0.368) exp(-2)="); g_pn(exp_fx(0-2*ONE)); g_puts(" (~8847=0.135)\n" as *u8) 48 49 let scores: *i64 = sys_mmap(3*8) as *i64; scores[0]=ONE; scores[1]=3*ONE; scores[2]=2*ONE // logits 1.0, 3.0, 2.0 50 let w: *i64 = sys_mmap(3*8) as *i64 51 softmax(scores, w, 3) 52 g_puts(" softmax(1,3,2): w0="); g_pn(w[0]); g_puts(" w1="); g_pn(w[1]); g_puts(" w2="); g_pn(w[2]); g_puts(" sum="); g_pn(w[0]+w[1]+w[2]); g_puts(" (ONE=65536)\n" as *u8) 53 var t1: i64=0; let sm: i64=w[0]+w[1]+w[2]; if w[0]>=0 { if w[1]>=0 { if w[2]>=0 { var d: i64=sm-ONE; if d<0 {d=0-d} if d<(ONE/50) { t1=1 } } } } 54 pass=pass+ck("T1: softmax weights non-negative and sum ~= ONE (a valid probability distribution)" as *u8, t1); total=total+1 55 56 var t2: i64=0; if w[1]>w[2] { if w[2]>w[0] { t2=1 } } 57 pass=pass+ck("T2: MONOTONIC -- highest logit (3.0) gets highest weight; order w1>w2>w0 matches scores 3>2>1" as *u8, t2); total=total+1 58 59 let w2: *i64 = sys_mmap(3*8) as *i64; softmax(scores, w2, 3) 60 var t3: i64=0; if w[0]==w2[0] { if w[1]==w2[1] { if w[2]==w2[2] { t3=1 } } } 61 pass=pass+ck("T3 (EXCEED): DETERMINISTIC -- recomputed softmax is BIT-IDENTICAL (float/CUDA attention cannot guarantee)" as *u8, t3); total=total+1 62 63 let V: *i64 = sys_mmap(3*8) as *i64; V[0]=10*ONE; V[1]=30*ONE; V[2]=20*ONE // value vectors (scalar dim) 64 let out: i64 = attention(w, V, 3) 65 g_puts(" attention(V=[10,30,20]) = "); g_pn(out); g_puts(" (~"); g_pn(out/ONE); g_puts(".x; min10/max30, pulled toward 30 = the highest-score key)\n" as *u8) 66 var t4: i64=0; if out>(10*ONE) { if out<(30*ONE) { if out>(20*ONE) { t4=1 } } } 67 pass=pass+ck("T4: attention output is a CONVEX COMBINATION of V, dominated by the max-score key (between 10/30, above the uniform mean 20)" as *u8, t4); total=total+1 68 69 g_puts(" >> the transformer HEART now runs in pure integer + deterministic. CLIMB: multi-head + Q/K/V projections (matmul we have)\n" as *u8) 70 g_puts(" + positional encoding + layernorm (integer) -> a full no-float transformer block; MEASURE accuracy vs MLPerf.\n" as *u8) 71 72 var okall: i64=0; if pass==total { okall=1 } 73 g_puts("---- nx_nofloat_attn: passed "); g_pn(pass); g_puts(" / "); g_pn(total); g_puts(" ----\n" as *u8) 74 if okall==1 { 75 let logf: i64=sys_openat_append("knowledge/status/nofloat_attn.log" as *u8, 420) 76 if logf>=0 { let z: i64=sys_write(logf,"NXNOFLOATATTN GREEN: integer fixed-point exp + softmax (sums to ONE, monotonic) + attention (convex combo); deterministic transformer heart\n" as *u8,134); sys_close(logf) } 77 g_puts("verdict=GREEN (the transformer HEART in pure integer: fixed-point exp + softmax + attention, deterministic; the rung toward a sovereign no-float LLM)\n" as *u8); sys_exit(0); return 0 78 } 79 g_puts("verdict=RED\n" as *u8); sys_exit(1); return 1 80}