nx_nofloat_transformer.nx source

1// nx_nofloat_transformer.nx -- SOVEREIGN NO-FLOAT TRANSFORMER BLOCK (the AI culmination), pure integer Q16, 2// deterministic. Pre-norm block: h = x + SelfAttn(LayerNorm(x)); out = h + FFN(LayerNorm(h)). New primitive = 3// integer LayerNorm (mean/var + Newton integer sqrt for std). Self-attention reuses fixed-point exp/softmax. 4// FFN = matvec -> ReLU -> matvec. Everything integer => bit-exact deterministic (the moat vs float transformers). 5// T1 integer sqrt correct (sqrt_q16(4)=2, sqrt_q16(9)=3). T2 LayerNorm normalizes (output mean ~= 0). 6// T3 full block runs end-to-end (output transformed vs input, bounded). T4 (EXCEED) determinism bit-identical. 7// expect_exit: 0 Sovereign: nx_syscalls. 8import "nx_syscalls.nx" 9import "nx_g_puts_lib.nx" 10 11func g_pn(v: i64) -> i64 { let b: *u8=sys_mmap(28); var x: i64=v; if x<0{b[0]=45;sys_write(1,b,1);x=0-x} if x==0{b[0]=48;sys_write(1,b,1);return 0} var d: i64=0; var y: i64=x; while y>0{d=d+1;y=y/10} var i: i64=d-1; y=x; while i>=0{b[i]=(48+(y%10)) as u8;y=y/10;i=i-1} sys_write(1,b,d); return 0 } 12func ck(name: *u8, c: i64) -> i64 { if c==1 { g_puts(" PASS " as *u8) } else { g_puts(" FAIL " as *u8) } g_puts(name); g_puts("\n" as *u8); return c } 13 14const Q: i64 = 16 15const ONE: i64 = 65536 16const D: i64 = 4 17const T: i64 = 2 18const LOG2E: i64 = 94548 19const C1: i64 = 45426 20const C2: i64 = 15743 21const EPS: i64 = 65 // ~0.001 in Q16 (layernorm epsilon) 22func fxmul(a: i64, b: i64) -> i64 { return (a*b)>>Q } 23func relu(x: i64) -> i64 { if x>0 { return x } return 0 } 24func iabs(v: i64) -> i64 { if v<0 { return 0-v } return v } 25 26func isqrt(n: i64) -> i64 { if n<2 { return n } var x: i64=n; var y: i64=(x+1)/2; while y<x { x=y; y=(x + n/x)/2 } return x } 27func sqrt_q16(y: i64) -> i64 { return isqrt(y*ONE) } // sqrt of a Q16 value, result in Q16 28 29func exp_fx(x: i64) -> i64 { 30 var xx: i64=x; if xx>0 { xx=0 } 31 let yabs: i64=fxmul(0-xx, LOG2E); let nabs: i64=yabs>>Q; let fabs: i64=yabs-(nabs<<Q) 32 let f2: i64=fxmul(fabs,fabs); let p: i64=ONE+fxmul(fabs,C1)+fxmul(f2,C2); let inv: i64=(ONE*ONE)/p 33 if nabs>=31 { return 0 } return inv>>nabs 34} 35func dotp(a: *i64, b: *i64, n: i64) -> i64 { var s: i64=0; var i: i64=0; while i<n { s=s+fxmul(a[i],b[i]); i=i+1 } return s } 36// LayerNorm of vector x[0..D) -> out[0..D): zero-mean, unit-var (then identity affine). 37func layernorm(x: *i64, out: *i64) -> i64 { 38 var sum: i64=0; var i: i64=0; while i<D { sum=sum+x[i]; i=i+1 } let mean: i64=sum/D 39 var vs: i64=0; i=0; while i<D { let d: i64=x[i]-mean; vs=vs+fxmul(d,d); i=i+1 } let var_: i64=vs/D 40 let std: i64=sqrt_q16(var_+EPS) 41 i=0; while i<D { out[i]=((x[i]-mean)*ONE)/std; i=i+1 } return mean 42} 43func matvec(W: *i64, v: *i64, out: *i64) -> i64 { var i: i64=0; while i<D { var s: i64=0; var j: i64=0; while j<D { s=s+fxmul(W[i*D+j], v[j]); j=j+1 } out[i]=s; i=i+1 } return 0 } 44 45// one pre-norm transformer block over x[T*D] -> out[T*D]. 46func block(x: *i64, out: *i64, W1: *i64, W2: *i64) -> i64 { 47 let ln: *i64 = sys_mmap(T*D*8) as *i64 48 // LayerNorm each token (index math, unambiguous) 49 var t: i64=0; while t<T { let xi: *i64 = sys_mmap(D*8) as *i64; let oi: *i64 = sys_mmap(D*8) as *i64; var k: i64=0; while k<D { xi[k]=x[t*D+k]; k=k+1 } layernorm(xi, oi); k=0; while k<D { ln[t*D+k]=oi[k]; k=k+1 } t=t+1 } 50 // self-attention over the T tokens: scores[i][j]=dot(ln_i,ln_j); softmax over j; attn_i = sum_j w_j * ln_j 51 let h: *i64 = sys_mmap(T*D*8) as *i64 52 var i: i64=0 53 while i<T { 54 let sc: *i64 = sys_mmap(T*8) as *i64; let a: *i64 = sys_mmap(T*8) as *i64; let lni: *i64 = sys_mmap(D*8) as *i64 55 var k: i64=0; while k<D { lni[k]=ln[i*D+k]; k=k+1 } 56 var j: i64=0; while j<T { let lnj: *i64=sys_mmap(D*8) as *i64; var m: i64=0; while m<D { lnj[m]=ln[j*D+m]; m=m+1 } sc[j]=dotp(lni,lnj,D); j=j+1 } 57 var mx: i64=sc[0]; j=1; while j<T { if sc[j]>mx { mx=sc[j] } j=j+1 } 58 var sume: i64=0; j=0; while j<T { a[j]=exp_fx(sc[j]-mx); sume=sume+a[j]; j=j+1 } 59 j=0; while j<T { a[j]=(a[j]*ONE)/sume; j=j+1 } 60 var d: i64=0; while d<D { var acc: i64=0; j=0; while j<T { acc=acc+fxmul(a[j], ln[j*D+d]); j=j+1 } h[i*D+d]=x[i*D+d]+acc; d=d+1 } // residual 61 i=i+1 62 } 63 // FFN with pre-norm + residual: out = h + W2*relu(W1*LN(h)) 64 i=0 65 while i<T { 66 let hi: *i64=sys_mmap(D*8) as *i64; let lnh: *i64=sys_mmap(D*8) as *i64; var k: i64=0; while k<D { hi[k]=h[i*D+k]; k=k+1 } 67 layernorm(hi, lnh) 68 let m1: *i64=sys_mmap(D*8) as *i64; matvec(W1, lnh, m1); k=0; while k<D { m1[k]=relu(m1[k]); k=k+1 } 69 let m2: *i64=sys_mmap(D*8) as *i64; matvec(W2, m1, m2) 70 k=0; while k<D { out[i*D+k]=h[i*D+k]+m2[k]; k=k+1 } 71 i=i+1 72 } 73 return 0 74} 75 76func main() -> i64 { 77 g_puts("nx_nofloat_transformer (SOVEREIGN no-float TRANSFORMER BLOCK -- integer LayerNorm+sqrt+attention+FFN, deterministic)\n" as *u8) 78 var pass: i64=0; var total: i64=0 79 80 g_puts(" integer sqrt: sqrt_q16(4.0)="); g_pn(sqrt_q16(4*ONE)); g_puts(" (want 131072=2.0) sqrt_q16(9.0)="); g_pn(sqrt_q16(9*ONE)); g_puts(" (want 196608=3.0)\n" as *u8) 81 var t1: i64=0; if sqrt_q16(4*ONE)==(2*ONE) { if sqrt_q16(9*ONE)==(3*ONE) { t1=1 } } 82 pass=pass+ck("T1: integer Newton sqrt correct (sqrt 4.0=2.0, 9.0=3.0 in Q16)" as *u8, t1); total=total+1 83 84 let tok: *i64=sys_mmap(D*8) as *i64; tok[0]=ONE; tok[1]=2*ONE; tok[2]=0; tok[3]=4*ONE 85 let lno: *i64=sys_mmap(D*8) as *i64; layernorm(tok, lno) 86 var lnsum: i64=lno[0]+lno[1]+lno[2]+lno[3] 87 g_puts(" LayerNorm([1,2,0,4]) = ["); g_pn(lno[0]); g_puts(","); g_pn(lno[1]); g_puts(","); g_pn(lno[2]); g_puts(","); g_pn(lno[3]); g_puts("] sum(mean*D)="); g_pn(lnsum); g_puts(" (~0)\n" as *u8) 88 var t2: i64=0; if iabs(lnsum)<(ONE/10) { t2=1 } 89 pass=pass+ck("T2: LayerNorm normalizes -- output is ~zero-mean (sum ~= 0)" as *u8, t2); total=total+1 90 91 // build the block input + weights 92 let x: *i64=sys_mmap(T*D*8) as *i64; x[0]=ONE; x[1]=2*ONE; x[2]=0; x[3]=ONE; x[4]=3*ONE; x[5]=0; x[6]=ONE; x[7]=2*ONE 93 let W1: *i64=sys_mmap(D*D*8) as *i64; let W2: *i64=sys_mmap(D*D*8) as *i64 94 var i: i64=0; while i<D*D { W1[i]=ONE/2; W2[i]=ONE/3; i=i+1 } // simple fixed weights 95 let out: *i64=sys_mmap(T*D*8) as *i64; block(x, out, W1, W2) 96 var changed: i64=0; var bounded: i64=1; i=0; while i<T*D { if out[i]!=x[i] { changed=1 } if iabs(out[i])>(1000*ONE) { bounded=0 } i=i+1 } 97 g_puts(" block out[0..3]=["); g_pn(out[0]); g_puts(","); g_pn(out[1]); g_puts(","); g_pn(out[2]); g_puts(","); g_pn(out[3]); g_puts("] (transformed + bounded)\n" as *u8) 98 var t3: i64=0; if changed==1 { if bounded==1 { t3=1 } } 99 pass=pass+ck("T3: full transformer block runs end-to-end (LN->attn->residual->LN->FFN->residual); output transformed + bounded" as *u8, t3); total=total+1 100 101 let out2: *i64=sys_mmap(T*D*8) as *i64; block(x, out2, W1, W2) 102 var same: i64=1; i=0; while i<T*D { if out[i]!=out2[i] { same=0 } i=i+1 } 103 var t4: i64=0; if same==1 { t4=1 } 104 pass=pass+ck("T4 (EXCEED): DETERMINISTIC -- two block runs are BIT-IDENTICAL (float transformers cannot guarantee)" as *u8, t4); total=total+1 105 106 g_puts(" >> the SOVEREIGN no-float TRANSFORMER BLOCK runs in pure integer + is deterministic. The full stack:\n" as *u8) 107 g_puts(" matmul + train(SGD) + multilayer-backprop + attention/softmax + LayerNorm + the assembled block -- all no-float.\n" as *u8) 108 g_puts(" CLIMB: multi-head + Q/K/V projections + positional encoding + stack N blocks + a tokenizer = a sovereign deterministic LLM; MEASURE vs MLPerf.\n" as *u8) 109 110 var okall: i64=0; if pass==total { okall=1 } 111 g_puts("---- nx_nofloat_transformer: passed "); g_pn(pass); g_puts(" / "); g_pn(total); g_puts(" ----\n" as *u8) 112 if okall==1 { 113 let logf: i64=sys_openat_append("knowledge/status/nofloat_transformer.log" as *u8, 420) 114 if logf>=0 { let z: i64=sys_write(logf,"NXNOFLOATTRANSFORMER GREEN: integer LayerNorm+sqrt + self-attention + residual + FFN = full transformer block, deterministic\n" as *u8,119); sys_close(logf) } 115 g_puts("verdict=GREEN (the sovereign no-float TRANSFORMER BLOCK: integer LayerNorm+attention+FFN+residual, deterministic; the AI culmination toward a sovereign LLM)\n" as *u8); sys_exit(0); return 0 116 } 117 g_puts("verdict=RED\n" as *u8); sys_exit(1); return 1 118}