nx_nofloat_llm.nx source

1// nx_nofloat_llm.nx -- CANONICAL sovereign no-float LLM library (ONE source of truth; no copy-paste debt). 2// 3// Every integer-Q16 building block for running a real GGUF transformer (Qwen2.5) in deterministic no-float: 4// - dequant of every quant the model uses (F32 / Q5_0 / Q8_0 / Q4_K / Q6_K) -> Q16 5// - by-name tensor load + window-dequant (for embeddings / LM-head rows) 6// - fixed-point transcendentals (exp, sigmoid, SiLU, sin, cos) 7// - the ONE matmul: full-precision accumulate-then-shift (the correct, non-underflowing version) 8// - RMSNorm, RoPE, attention sublayer, FFN sublayer, the lazy N-layer stack 9// 10// Gates IMPORT this instead of re-defining (CLAUDE.md #15 DRY). One canonical, correct implementation: 11// a precision/convention fix is made HERE, once, not in 25 copies. No `main` (pure library). 12// No hw writes (Rule 26). license_tier: ORIGINAL 13import "nx_syscalls.nx" 14import "nx_tier.nx" 15import "nx_le.nx" 16import "nx_tensor.nx" 17import "nx_gguf.nx" 18import "nx_gguf_load.nx" 19import "nx_thread_pool.nx" // decode-speed: banded bit-exact parallel kernels (2026-07-09) 20import "nx_vecmath.nx" 21 22const Q16: i64 = 65536 23const LOG2E: i64 = 94548 24const PC0: i64 = 65536 25const PC1: i64 = 45426 26const PC2: i64 = 15743 27const PC3: i64 = 4367 28const LN_BASE_Q16: i64 = 905421 // ln(1e6), Qwen2.5 rope base 29const HALF_PI: i64 = 102944 30const PI: i64 = 205887 31const THREE_HALF_PI: i64 = 308831 32const TWO_PI: i64 = 411775 33 34func qmul(a: i64, b: i64) -> i64 { return (a*b) >> 16 } 35func cpy(d: *i64, s: *i64, n: i64) -> i64 { var i: i64=0; while i<n { d[i]=s[i]; i=i+1 } return 0 } 36func q_i8(buf: *u8, off: i64) -> i64 { let v: i64=nx_le_read_u8(buf, off); if v>=128 { return v-256 } return v } 37 38// ---- scalar dtype decode -> Q16 (pure integer; no float touches the weights) ---- 39func f32_to_q16(bits: i64) -> i64 { let s: i64=(bits>>31)&1; let e: i64=(bits>>23)&255; let mant: i64=bits&8388607; var v: i64=0; if e==0 { v=0 } else { if e==255 { v=2147483647 } else { let m: i64=8388608+mant; let ee: i64=e-134; if ee>=0 { v=m<<ee } else { v=m>>(0-ee) } } } if s==1 { v=0-v } return v } 40func f16_to_q16(h: i64) -> i64 { let s: i64=(h>>15)&1; let e: i64=(h>>10)&31; let mant: i64=h&1023; var v: i64=0; if e==0 { v=mant>>8 } else { if e==31 { v=2147483647 } else { let m: i64=1024+mant; let ee: i64=e-9; if ee>=0 { v=m<<ee } else { v=m>>(0-ee) } } } if s==1 { v=0-v } return v } 41 42// ---- block dequants (one super-block / block each) -> Q16 ---- 43func q5_0_block(buf: *u8, off: i64, n: i64, out: *i64) -> i64 { let d: i64=f16_to_q16(nx_le_read_u16(buf, off)); let qh: i64=nx_le_read_u32(buf, off+2); let qo: i64=off+6; var j: i64=0; while j<16 { let qs: i64=nx_le_read_u8(buf, qo+j); let xh0: i64=((qh>>j)<<4)&16; let xh1: i64=(qh>>(j+12))&16; if j<n { out[j]=d*(((qs&15)|xh0)-16) } if j+16<n { out[j+16]=d*(((qs>>4)|xh1)-16) } j=j+1 } return 0 } 44func q8_0_block(buf: *u8, off: i64, n: i64, out: *i64) -> i64 { let d: i64=f16_to_q16(nx_le_read_u16(buf, off)); var j: i64=0; while j<32 { if j<n { out[j]=d*q_i8(buf, off+2+j) } j=j+1 } return 0 } 45// ⚠ Q6_K super-scale d is ~1e-5 on real trained weights -- f16_to_q16 (16 frac bits) ROUNDS IT TO 0, collapsing whole 46// super-blocks to zero (measured: ffn_down 84/896 rows survived -> the FFN down-proj died -> garbage predictions). 47// FIX 2026-07-09: decode d in Q24 (exact for every finite f16, like the Q4_K fix), compute d24*sc*(q-32) in Q24, >>8 to Q16. 48func q6k_block(buf: *u8, so: i64, n: i64, out: *i64) -> i64 { 49 let d24: i64=_gguf_f16_to_q24(nx_le_read_u16(buf, so+208)); var half: i64=0 50 while half<2 { let bql: i64=so+half*64; let bqh: i64=so+128+half*32; let bsc: i64=so+192+half*8; let yb: i64=half*128; var l: i64=0 51 while l<32 { let is: i64=l/16; let ql: i64=nx_le_read_u8(buf, bql+l); let ql3: i64=nx_le_read_u8(buf, bql+l+32); let qh: i64=nx_le_read_u8(buf, bqh+l) 52 if yb+l+0<n { out[yb+l+0]=(d24*q_i8(buf,bsc+is+0)*(((ql&15)|(((qh>>0)&3)<<4))-32))>>8 } 53 if yb+l+32<n { out[yb+l+32]=(d24*q_i8(buf,bsc+is+2)*(((ql3&15)|(((qh>>2)&3)<<4))-32))>>8 } 54 if yb+l+64<n { out[yb+l+64]=(d24*q_i8(buf,bsc+is+4)*(((ql>>4)|(((qh>>4)&3)<<4))-32))>>8 } 55 if yb+l+96<n { out[yb+l+96]=(d24*q_i8(buf,bsc+is+6)*(((ql3>>4)|(((qh>>6)&3)<<4))-32))>>8 } 56 l=l+1 } 57 half=half+1 } 58 return 0 59} 60// ---- DEBT-EATEN 2026-07-15 ("eat our debt, we want state of the art"): the FULL mainstream ggml quant 61// family. Census (nx_gguf_type_census_gate) says the fleet ships F32/Q5_0/Q8_0/Q4_K/Q6_K today -- but any 62// new pack can carry Q5_K/Q3_K/Q2_K/Q4_0/Q4_1/Q5_1/F16/BF16, and the old dispatch either SILENTLY zeroed 63// (dequant_to_q16) or WRONG-DECODED as Q4_K (dequant_row's else-branch). Now: every mainstream type 64// decodes (KAT-gated, nx_gguf_dequant_kat_gate), everything else fails LOUD (-1). IQ-family (i-quants) 65// remains NAMED-unsupported (codebook arc of its own). ---- 66// Q4_0: 18B/32 = d(f16) + 16 nibble bytes; y = d*(q-8). 67func q4_0_block(buf: *u8, off: i64, n: i64, out: *i64) -> i64 { let d: i64=f16_to_q16(nx_le_read_u16(buf, off)); let qo: i64=off+2; var j: i64=0; while j<16 { let qs: i64=nx_le_read_u8(buf, qo+j); if j<n { out[j]=d*((qs&15)-8) } if j+16<n { out[j+16]=d*((qs>>4)-8) } j=j+1 } return 0 } 68// Q4_1: 20B/32 = d(f16) + m(f16) + 16 nibble bytes; y = d*q + m. 69func q4_1_block(buf: *u8, off: i64, n: i64, out: *i64) -> i64 { let d: i64=f16_to_q16(nx_le_read_u16(buf, off)); let m: i64=f16_to_q16(nx_le_read_u16(buf, off+2)); let qo: i64=off+4; var j: i64=0; while j<16 { let qs: i64=nx_le_read_u8(buf, qo+j); if j<n { out[j]=d*(qs&15)+m } if j+16<n { out[j+16]=d*(qs>>4)+m } j=j+1 } return 0 } 70// Q5_1: 24B/32 = d(f16) + m(f16) + qh(u32) + 16 nibble bytes; y = d*q5 + m (5th bit from qh). 71func q5_1_block(buf: *u8, off: i64, n: i64, out: *i64) -> i64 { let d: i64=f16_to_q16(nx_le_read_u16(buf, off)); let m: i64=f16_to_q16(nx_le_read_u16(buf, off+2)); let qh: i64=nx_le_read_u32(buf, off+4); let qo: i64=off+8; var j: i64=0; while j<16 { let qs: i64=nx_le_read_u8(buf, qo+j); let xh0: i64=((qh>>j)<<4)&16; let xh1: i64=(qh>>(j+12))&16; if j<n { out[j]=d*((qs&15)|xh0)+m } if j+16<n { out[j+16]=d*((qs>>4)|xh1)+m } j=j+1 } return 0 } 72// Q4_K/Q5_K shared 6-bit scale/min pair unpack (the proven nx_gguf_load idiom): box[0]=sc box[1]=m. 73func qk_scm(buf: *u8, sco: i64, is: i64, box: *i64) -> i64 { 74 if is < 4 { 75 box[0] = nx_le_read_u8(buf, sco+is) & 63 76 box[1] = nx_le_read_u8(buf, sco+is+4) & 63 77 } else { 78 let k: i64 = is - 4 79 let bk: i64 = nx_le_read_u8(buf, sco+k) 80 let bk4: i64 = nx_le_read_u8(buf, sco+4+k) 81 let b8k: i64 = nx_le_read_u8(buf, sco+8+k) 82 box[0] = ((bk >> 6) << 4) | (b8k & 15) 83 box[1] = ((bk4 >> 6) << 4) | (b8k >> 4) 84 } 85 return 0 86} 87// Q5_K: 176B/256 = d(f16) dmin(f16) scales[12] qh[32] qs[128]; y = d*sc*((q4|qh<<4)) - dmin*m. Q24 math -> >>8. 88func q5k_block(buf: *u8, so: i64, n: i64, out: *i64) -> i64 { 89 let d24: i64=_gguf_f16_to_q24(nx_le_read_u16(buf, so)) 90 let dm24: i64=_gguf_f16_to_q24(nx_le_read_u16(buf, so+2)) 91 let sco: i64=so+4 92 let qho: i64=so+16 93 let qlo: i64=so+48 94 let box: *i64 = sys_mmap(16) as *i64 95 var j: i64=0 96 while j<4 { 97 qk_scm(buf, sco, 2*j, box) 98 let sc0: i64=box[0] 99 let m0: i64=box[1] 100 qk_scm(buf, sco, 2*j+1, box) 101 let sc1: i64=box[0] 102 let m1: i64=box[1] 103 let u1: i64=1<<(2*j) 104 let u2: i64=2<<(2*j) 105 var l: i64=0 106 while l<32 { 107 let ql: i64=nx_le_read_u8(buf, qlo+j*32+l) 108 let qh: i64=nx_le_read_u8(buf, qho+l) 109 var lo: i64=ql&15 110 if (qh&u1)!=0 { lo=lo+16 } 111 var hi: i64=ql>>4 112 if (qh&u2)!=0 { hi=hi+16 } 113 if j*64+l<n { out[j*64+l]=((d24*sc0*lo)-(dm24*m0))>>8 } 114 if j*64+32+l<n { out[j*64+32+l]=((d24*sc1*hi)-(dm24*m1))>>8 } 115 l=l+1 116 } 117 j=j+1 118 } 119 return 0 120} 121// Q3_K 16 x 6-bit scale (RAW 0..63; caller subtracts 32) from the 12 packed bytes at sco. 122func q3k_sc(buf: *u8, sco: i64, is: i64) -> i64 { 123 let k: i64 = is & 3 124 var lo: i64 = 0 125 var hb: i64 = 0 126 if is < 4 { lo = nx_le_read_u8(buf, sco+k) & 15; hb = nx_le_read_u8(buf, sco+8+k) & 3 } else { 127 if is < 8 { lo = nx_le_read_u8(buf, sco+4+k) & 15; hb = (nx_le_read_u8(buf, sco+8+k) >> 2) & 3 } else { 128 if is < 12 { lo = nx_le_read_u8(buf, sco+k) >> 4; hb = (nx_le_read_u8(buf, sco+8+k) >> 4) & 3 } else { 129 lo = nx_le_read_u8(buf, sco+4+k) >> 4 130 hb = (nx_le_read_u8(buf, sco+8+k) >> 6) & 3 131 } 132 } 133 } 134 return lo | (hb << 4) 135} 136// Q3_K: 110B/256 = hmask[32] qs[64] scales[12] d(f16); y = d*(sc-32)*(q3 - (hbit?0:4)). Q24 -> >>8. 137func q3k_block(buf: *u8, so: i64, n: i64, out: *i64) -> i64 { 138 let d24: i64=_gguf_f16_to_q24(nx_le_read_u16(buf, so+108)) 139 let sco: i64=so+96 140 var yi: i64=0 141 var is: i64=0 142 var mbit: i64=1 143 var half: i64=0 144 while half<2 { 145 let qb: i64=so+32+half*32 146 var shift: i64=0 147 var jj: i64=0 148 while jj<4 { 149 let dl0: i64=d24*(q3k_sc(buf, sco, is)-32) 150 is=is+1 151 var l: i64=0 152 while l<16 { 153 let qv: i64=(nx_le_read_u8(buf, qb+l)>>shift)&3 154 var t: i64=qv 155 if (nx_le_read_u8(buf, so+l)&mbit)==0 { t=t-4 } 156 if yi<n { out[yi]=(dl0*t)>>8 } 157 yi=yi+1 158 l=l+1 159 } 160 let dl1: i64=d24*(q3k_sc(buf, sco, is)-32) 161 is=is+1 162 l=0 163 while l<16 { 164 let qv2: i64=(nx_le_read_u8(buf, qb+16+l)>>shift)&3 165 var t2: i64=qv2 166 if (nx_le_read_u8(buf, so+16+l)&mbit)==0 { t2=t2-4 } 167 if yi<n { out[yi]=(dl1*t2)>>8 } 168 yi=yi+1 169 l=l+1 170 } 171 shift=shift+2 172 mbit=mbit<<1 173 jj=jj+1 174 } 175 half=half+1 176 } 177 return 0 178} 179// Q2_K: 84B/256 = scales[16] (lo4=sc hi4=m) qs[64] d(f16) dmin(f16); y = d*sc*q2 - dmin*m. Q24 -> >>8. 180func q2k_block(buf: *u8, so: i64, n: i64, out: *i64) -> i64 { 181 let d24: i64=_gguf_f16_to_q24(nx_le_read_u16(buf, so+80)) 182 let dm24: i64=_gguf_f16_to_q24(nx_le_read_u16(buf, so+82)) 183 var yi: i64=0 184 var is: i64=0 185 var half: i64=0 186 while half<2 { 187 let qb: i64=so+16+half*32 188 var shift: i64=0 189 var jj: i64=0 190 while jj<4 { 191 let sb0: i64=nx_le_read_u8(buf, so+is) 192 is=is+1 193 let dl0: i64=d24*(sb0&15) 194 let ml0: i64=dm24*(sb0>>4) 195 var l: i64=0 196 while l<16 { 197 let qv: i64=(nx_le_read_u8(buf, qb+l)>>shift)&3 198 if yi<n { out[yi]=((dl0*qv)-ml0)>>8 } 199 yi=yi+1 200 l=l+1 201 } 202 let sb1: i64=nx_le_read_u8(buf, so+is) 203 is=is+1 204 let dl1: i64=d24*(sb1&15) 205 let ml1: i64=dm24*(sb1>>4) 206 l=0 207 while l<16 { 208 let qv2: i64=(nx_le_read_u8(buf, qb+16+l)>>shift)&3 209 if yi<n { out[yi]=((dl1*qv2)-ml1)>>8 } 210 yi=yi+1 211 l=l+1 212 } 213 shift=shift+2 214 jj=jj+1 215 } 216 half=half+1 217 } 218 return 0 219} 220// canonical type stride: vb[0]=values/block vb[1]=bytes/block. 0 ok, -1 = NOT A SUPPORTED TYPE (the 221// serve's fail-fast init scans every tensor through this -- unknown types refuse to serve, never zero). 222func nf_type_stride(ty: i64, vb: *i64) -> i64 { 223 if ty==NX_GGML_TYPE_F32 { vb[0]=1; vb[1]=4; return 0 } 224 if ty==NX_GGML_TYPE_F16 { vb[0]=1; vb[1]=2; return 0 } 225 if ty==NX_GGML_TYPE_BF16 { vb[0]=1; vb[1]=2; return 0 } 226 if ty==NX_GGML_TYPE_Q4_0 { vb[0]=32; vb[1]=18; return 0 } 227 if ty==NX_GGML_TYPE_Q4_1 { vb[0]=32; vb[1]=20; return 0 } 228 if ty==NX_GGML_TYPE_Q5_0 { vb[0]=32; vb[1]=22; return 0 } 229 if ty==NX_GGML_TYPE_Q5_1 { vb[0]=32; vb[1]=24; return 0 } 230 if ty==NX_GGML_TYPE_Q8_0 { vb[0]=32; vb[1]=34; return 0 } 231 if ty==NX_GGML_TYPE_Q2_K { vb[0]=256; vb[1]=84; return 0 } 232 if ty==NX_GGML_TYPE_Q3_K { vb[0]=256; vb[1]=110; return 0 } 233 if ty==NX_GGML_TYPE_Q4_K { vb[0]=256; vb[1]=144; return 0 } 234 if ty==NX_GGML_TYPE_Q5_K { vb[0]=256; vb[1]=176; return 0 } 235 if ty==NX_GGML_TYPE_Q6_K { vb[0]=256; vb[1]=210; return 0 } 236 return 0 - 1 237} 238// byte offset of value index `voff` inside a quantized blob (canonical stride; MoE expert slices ride this). 239// voff MUST be block-aligned (expert slices are: ff*D is a 256/32 multiple). -1 on unsupported type. 240func nf_expert_byteoff(ty: i64, voff: i64) -> i64 { 241 let vb: *i64 = sys_mmap(16) as *i64 242 if nf_type_stride(ty, vb) != 0 { return 0 - 1 } 243 return (voff / vb[0]) * vb[1] 244} 245// dequant n contiguous values of a tensor (any supported quant) starting at byte `base` -> Q16. -1 if unsupported. 246func dequant_to_q16(buf: *u8, base: i64, gt: i64, n: i64, out: *i64) -> i64 { 247 if gt==NX_GGML_TYPE_F32 { var i: i64=0; while i<n { out[i]=f32_to_q16(nx_le_read_u32(buf, base+i*4)); i=i+1 } return 0 } 248 if gt==NX_GGML_TYPE_F16 { var i: i64=0; while i<n { out[i]=f16_to_q16(nx_le_read_u16(buf, base+i*2)); i=i+1 } return 0 } 249 if gt==NX_GGML_TYPE_BF16 { var i: i64=0; while i<n { out[i]=f32_to_q16(nx_le_read_u16(buf, base+i*2)<<16); i=i+1 } return 0 } 250 if gt==NX_GGML_TYPE_Q4_K { nx_gguf_dequant_q4_k_q14(buf, base, n, out); var i: i64=0; while i<n { out[i]=out[i]/256; i=i+1 } return 0 } // Q4_K dequant now emits Q24 -> Q16 (/256); was Q14<<2 251 if gt==NX_GGML_TYPE_Q6_K { var sb: i64=0; while sb*256<n { var tk: i64=n-sb*256; if tk>256 { tk=256 } q6k_block(buf, base+sb*210, tk, ((out as i64)+sb*256*8) as *i64); sb=sb+1 } return 0 } 252 if gt==NX_GGML_TYPE_Q5_K { var sb: i64=0; while sb*256<n { var tk: i64=n-sb*256; if tk>256 { tk=256 } q5k_block(buf, base+sb*176, tk, ((out as i64)+sb*256*8) as *i64); sb=sb+1 } return 0 } 253 if gt==NX_GGML_TYPE_Q3_K { var sb: i64=0; while sb*256<n { var tk: i64=n-sb*256; if tk>256 { tk=256 } q3k_block(buf, base+sb*110, tk, ((out as i64)+sb*256*8) as *i64); sb=sb+1 } return 0 } 254 if gt==NX_GGML_TYPE_Q2_K { var sb: i64=0; while sb*256<n { var tk: i64=n-sb*256; if tk>256 { tk=256 } q2k_block(buf, base+sb*84, tk, ((out as i64)+sb*256*8) as *i64); sb=sb+1 } return 0 } 255 if gt==NX_GGML_TYPE_Q5_0 { var sb: i64=0; while sb*32<n { var tk: i64=n-sb*32; if tk>32 { tk=32 } q5_0_block(buf, base+sb*22, tk, ((out as i64)+sb*32*8) as *i64); sb=sb+1 } return 0 } 256 if gt==NX_GGML_TYPE_Q5_1 { var sb: i64=0; while sb*32<n { var tk: i64=n-sb*32; if tk>32 { tk=32 } q5_1_block(buf, base+sb*24, tk, ((out as i64)+sb*32*8) as *i64); sb=sb+1 } return 0 } 257 if gt==NX_GGML_TYPE_Q4_0 { var sb: i64=0; while sb*32<n { var tk: i64=n-sb*32; if tk>32 { tk=32 } q4_0_block(buf, base+sb*18, tk, ((out as i64)+sb*32*8) as *i64); sb=sb+1 } return 0 } 258 if gt==NX_GGML_TYPE_Q4_1 { var sb: i64=0; while sb*32<n { var tk: i64=n-sb*32; if tk>32 { tk=32 } q4_1_block(buf, base+sb*20, tk, ((out as i64)+sb*32*8) as *i64); sb=sb+1 } return 0 } 259 if gt==NX_GGML_TYPE_Q8_0 { var sb: i64=0; while sb*32<n { var tk: i64=n-sb*32; if tk>32 { tk=32 } q8_0_block(buf, base+sb*34, tk, ((out as i64)+sb*32*8) as *i64); sb=sb+1 } return 0 } 260 return 0 - 1 261} 262// find tensor by name + dequant up to `want` values -> out (Q16). #values, -1 not-found, -2 unsupported. 263func load_named_q16(buf: *u8, hdr: *NxGgufHeader, name: *u8, nlen: i64, out: *i64, want: i64) -> i64 { 264 let idx: nx_int = nx_gguf_find_tensor(hdr, name, nlen) 265 if idx < 0 { return 0 - 1 } 266 let ti: *NxGgufTensorInfo = nx_gguf_tensor_at(hdr, idx) 267 let nv: i64 = nx_gguf_tensor_n_values(ti) 268 var n: i64=want; if nv<n { n=nv } 269 if dequant_to_q16(buf, hdr.data_off + ti.offset, ti.ggml_type, n, out) < 0 { return 0 - 2 } 270 return n 271} 272// window-dequant row v (ne values at v*ne) of a [ne, rows] tensor (for embeddings + LM-head rows). 273// DEBT-EATEN 2026-07-15: the old 256-block tail was Q6_K-else-ASSUME-Q4_K -- any other k-quant WRONG-DECODED 274// silently. Now every mainstream type has an explicit row; anything else returns -1 (LOUD). 275func dequant_row(buf: *u8, base: i64, gt: i64, v: i64, ne: i64, out: *i64, tmp: *i64) -> i64 { 276 let vstart: i64=v*ne 277 if gt==NX_GGML_TYPE_F32 { var k: i64=0; while k<ne { out[k]=f32_to_q16(nx_le_read_u32(buf, base+(vstart+k)*4)); k=k+1 } return 0 } 278 if gt==NX_GGML_TYPE_F16 { var k: i64=0; while k<ne { out[k]=f16_to_q16(nx_le_read_u16(buf, base+(vstart+k)*2)); k=k+1 } return 0 } 279 if gt==NX_GGML_TYPE_BF16 { var k: i64=0; while k<ne { out[k]=f32_to_q16(nx_le_read_u16(buf, base+(vstart+k)*2)<<16); k=k+1 } return 0 } 280 if gt==NX_GGML_TYPE_Q8_0 { let fb: i64=vstart/32; let oi: i64=vstart-fb*32; let nb: i64=((vstart+ne-1)/32)-fb+1; var b: i64=0; while b<nb { q8_0_block(buf, base+(fb+b)*34, 32, ((tmp as i64)+b*32*8) as *i64); b=b+1 } var k: i64=0; while k<ne { out[k]=tmp[oi+k]; k=k+1 } return 0 } 281 if gt==NX_GGML_TYPE_Q5_0 { let fb: i64=vstart/32; let oi: i64=vstart-fb*32; let nb: i64=((vstart+ne-1)/32)-fb+1; var b: i64=0; while b<nb { q5_0_block(buf, base+(fb+b)*22, 32, ((tmp as i64)+b*32*8) as *i64); b=b+1 } var k: i64=0; while k<ne { out[k]=tmp[oi+k]; k=k+1 } return 0 } 282 if gt==NX_GGML_TYPE_Q5_1 { let fb: i64=vstart/32; let oi: i64=vstart-fb*32; let nb: i64=((vstart+ne-1)/32)-fb+1; var b: i64=0; while b<nb { q5_1_block(buf, base+(fb+b)*24, 32, ((tmp as i64)+b*32*8) as *i64); b=b+1 } var k: i64=0; while k<ne { out[k]=tmp[oi+k]; k=k+1 } return 0 } 283 if gt==NX_GGML_TYPE_Q4_0 { let fb: i64=vstart/32; let oi: i64=vstart-fb*32; let nb: i64=((vstart+ne-1)/32)-fb+1; var b: i64=0; while b<nb { q4_0_block(buf, base+(fb+b)*18, 32, ((tmp as i64)+b*32*8) as *i64); b=b+1 } var k: i64=0; while k<ne { out[k]=tmp[oi+k]; k=k+1 } return 0 } 284 if gt==NX_GGML_TYPE_Q4_1 { let fb: i64=vstart/32; let oi: i64=vstart-fb*32; let nb: i64=((vstart+ne-1)/32)-fb+1; var b: i64=0; while b<nb { q4_1_block(buf, base+(fb+b)*20, 32, ((tmp as i64)+b*32*8) as *i64); b=b+1 } var k: i64=0; while k<ne { out[k]=tmp[oi+k]; k=k+1 } return 0 } 285 let fb: i64=vstart/256; let oi: i64=vstart-fb*256; let nb: i64=((vstart+ne-1)/256)-fb+1 286 var okb: i64=0 287 if gt==NX_GGML_TYPE_Q6_K { var b: i64=0; while b<nb { q6k_block(buf, base+(fb+b)*210, 256, ((tmp as i64)+b*256*8) as *i64); b=b+1 } okb=1 } 288 if gt==NX_GGML_TYPE_Q5_K { var b: i64=0; while b<nb { q5k_block(buf, base+(fb+b)*176, 256, ((tmp as i64)+b*256*8) as *i64); b=b+1 } okb=1 } 289 if gt==NX_GGML_TYPE_Q3_K { var b: i64=0; while b<nb { q3k_block(buf, base+(fb+b)*110, 256, ((tmp as i64)+b*256*8) as *i64); b=b+1 } okb=1 } 290 if gt==NX_GGML_TYPE_Q2_K { var b: i64=0; while b<nb { q2k_block(buf, base+(fb+b)*84, 256, ((tmp as i64)+b*256*8) as *i64); b=b+1 } okb=1 } 291 if gt==NX_GGML_TYPE_Q4_K { nx_gguf_dequant_q4_k_q14(buf, base+fb*144, nb*256, tmp); var z: i64=0; while z<nb*256 { tmp[z]=tmp[z]/256; z=z+1 } okb=1 } // Q24 -> Q16 (/256) 292 if okb==0 { return 0 - 1 } 293 var k: i64=0; while k<ne { out[k]=tmp[oi+k]; k=k+1 } 294 return 0 295} 296// build "blk.<L><suffix>" into out, return length; load that tensor. 297func build_name(out: *u8, L: i64, suffix: *u8) -> i64 { 298 out[0]=98 as u8; out[1]=108 as u8; out[2]=107 as u8; out[3]=46 as u8 299 var p: i64=4 300 if L==0 { out[p]=48 as u8; p=p+1 } else { let tmp: *u8=sys_mmap(16); var k: i64=0; var m: i64=L; while m>0 { tmp[k]=(48+(m%10)) as u8; m=m/10; k=k+1 } var i: i64=k-1; while i>=0 { out[p]=tmp[i]; p=p+1; i=i-1 } } 301 var j: i64=0; while suffix[j]!=(0 as u8) { out[p]=suffix[j]; p=p+1; j=j+1 } 302 out[p]=0 as u8 303 return p 304} 305func load_blk(buf: *u8, hdr: *NxGgufHeader, nm: *u8, L: i64, suffix: *u8, out: *i64, want: i64) -> i64 { let len: i64=build_name(nm, L, suffix); return load_named_q16(buf, hdr, nm, len, out, want) } 306 307// ---- fixed-point math ---- 308func isqrt(v: i64) -> i64 { return vm_isqrt(v) } 309func fx_exp(x: i64) -> i64 { var xm: i64=0-x; if x>0 { xm=0 } let ym: i64=(xm*LOG2E)>>16; let yi: i64=ym>>16; let yf: i64=ym-(yi<<16); let g: i64=Q16-yf; var t: i64=PC3; t=PC2+((g*t)>>16); t=PC1+((g*t)>>16); t=PC0+((g*t)>>16); t=t>>1; if yi>=31 { return 0 } return t>>yi } 310func sigmoid(x: i64) -> i64 { if x>=0 { let ex: i64=fx_exp(0-x); return (Q16*Q16)/(Q16+ex) } let ex: i64=fx_exp(x); let sp: i64=(Q16*Q16)/(Q16+ex); return Q16-sp } 311func silu(x: i64) -> i64 { return qmul(x, sigmoid(x)) } 312func sin_q(x: i64) -> i64 { let x2: i64=qmul(x,x); let x3: i64=qmul(x2,x); let x5: i64=qmul(x3,x2); let x7: i64=qmul(x5,x2); let x9: i64=qmul(x7,x2); return x - x3/6 + x5/120 - x7/5040 + x9/362880 } 313func cos_q(x: i64) -> i64 { let x2: i64=qmul(x,x); let x4: i64=qmul(x2,x2); let x6: i64=qmul(x4,x2); let x8: i64=qmul(x6,x2); return Q16 - x2/2 + x4/24 - x6/720 + x8/40320 } 314func reduce2pi(a: i64) -> i64 { var t: i64=a; while t<0 { t=t+TWO_PI } while t>=TWO_PI { t=t-TWO_PI } return t } 315func sin_full(a: i64) -> i64 { let t: i64=reduce2pi(a); if t<HALF_PI { return sin_q(t) } if t<PI { return sin_q(PI-t) } if t<THREE_HALF_PI { return 0-sin_q(t-PI) } return 0-sin_q(TWO_PI-t) } 316func cos_full(a: i64) -> i64 { let t: i64=reduce2pi(a); if t<HALF_PI { return cos_q(t) } if t<PI { return 0-cos_q(PI-t) } if t<THREE_HALF_PI { return 0-cos_q(t-PI) } return cos_q(TWO_PI-t) } 317// ⚠ NEOX pairing (Qwen2.5 GGUF = llama.cpp LLAMA_ROPE_TYPE_NEOX): rotate the two HALVES (x[i], x[i+half]) with 318// theta_i=freqs[i]=base^(-2i/d). NOT the adjacent (x[2i],x[2i+1]) variant -- that rotates the WRONG pairs at every 319// pos>0 -> coherent-looking token 0 (rope=identity) but incoherent multi-token. Matches nx_f32_rope_apply_cs_neox. (fix 2026-07-09) 320func rope_apply(v: *i64, hd: i64, pos: i64, freqs: *i64) -> i64 { let np: i64=hd/2; var i: i64=0; while i<np { let ang: i64=pos*freqs[i]; let c: i64=cos_full(ang); let s: i64=sin_full(ang); let a: i64=v[i]; let b: i64=v[i+np]; v[i]=qmul(a,c)-qmul(b,s); v[i+np]=qmul(a,s)+qmul(b,c); i=i+1 } return 0 } 321// per-head RoPE base^(-2i/hd) frequencies (Q16) -> freqs[hd/2]. 322func rope_freqs(freqs: *i64, hd: i64) -> i64 { let np: i64=hd/2; var i: i64=0; while i<np { freqs[i]=fx_exp(0-((i*LN_BASE_Q16)/np)); i=i+1 } return 0 } 323// ln(x) in Q16 for a positive integer x (2026-07-15 debt-eaten: makes rope base CONFIGURABLE from model 324// metadata instead of the hardcoded ln(1e6) const). ln(x)=p*ln2 + ln(m), m=x/2^p in [1,2) via the fast 325// atanh series ln(m)=2*(z+z^3/3+z^5/5+z^7/7), z=(m-1)/(m+1). MEASURED: ln(1e6)=905404 (true 905420, ~2e-4 326// rel = the Q16 fixed-point floor; negligible after rope's /np + exp), ln(1e4)=603606, ln(2)=45426, ln(1)=0. 327func fx_ln_int(x: i64) -> i64 { 328 if x < 1 { return 0 } 329 var p: i64 = 0 330 var v: i64 = x 331 while v > 1 { v = v >> 1; p = p + 1 } 332 // m in Q16 = (x << 16) >> p, in [65536, 131072) 333 let m: i64 = (x << 16) >> p 334 let num: i64 = m - Q16 335 let den: i64 = m + Q16 336 let z: i64 = (num << 16) / den 337 let z2: i64 = (z * z) >> 16 338 var term: i64 = z 339 var sum: i64 = z 340 term = (term * z2) >> 16 341 sum = sum + term / 3 342 term = (term * z2) >> 16 343 sum = sum + term / 5 344 term = (term * z2) >> 16 345 sum = sum + term / 7 346 let ln_m: i64 = 2 * sum 347 return p * 45426 + ln_m // 45426 = ln(2) in Q16 348} 349// per-head RoPE frequencies for an ARBITRARY base (base^(-2i/hd) = exp(-(i/np)*ln(base))). Additive: the 350// bare rope_freqs stays the compile-time-1e6 path (identity for Qwen); this one takes ln(base) in Q16 so a 351// model's real rope base (OLMoE 1e4, Qwen 1e6, Llama 5e5...) drives attention correctly. 352func rope_freqs_base(freqs: *i64, hd: i64, ln_base_q16: i64) -> i64 { let np: i64=hd/2; var i: i64=0; while i<np { freqs[i]=fx_exp(0-((i*ln_base_q16)/np)); i=i+1 } return 0 } 353 354// ---- THE matmul: GGUF [out,in], full-precision accumulate then ONE shift (no per-term underflow) ---- 355func mm_out_in(inp: *i64, W: *i64, dst: *i64, T: i64, in_dim: i64, out_dim: i64, rev: i64) -> i64 { 356 var t: i64=0 357 while t<T { var o: i64=0 358 while o<out_dim { var s: i64=0 359 if rev==0 { var k: i64=0; while k<in_dim { s=s+(inp[t*in_dim+k]*W[o*in_dim+k]); k=k+1 } } 360 else { var k: i64=in_dim-1; while k>=0 { s=s+(inp[t*in_dim+k]*W[o*in_dim+k]); k=k-1 } } 361 dst[t*out_dim+o]=s>>16; o=o+1 } 362 t=t+1 } 363 return 0 364} 365func rmsnorm_gamma_row(x: *i64, gamma: *i64, xoff: i64, D: i64, out: *i64, ooff: i64) -> i64 { 366 var ss: i64=0; var i: i64=0; while i<D { let v: i64=x[xoff+i]; ss=ss+((v*v)>>16); i=i+1 } 367 let ms: i64=ss/D; let sd: i64=isqrt((ms+1)<<16); if sd<=0 { return 0 } 368 i=0; while i<D { let nm: i64=(x[xoff+i]<<16)/sd; out[ooff+i]=qmul(nm, gamma[i]); i=i+1 } 369 return 0 370} 371// PRECISION LIFT (2026-07-09): the ~1587 massive activations dominate the RMS, so signal dims normalize to ~1e-3 and 372// keep only ~3 bits in Q16 (then gamma~0.08 halves it) -> logits imprecise. This emits the normed*gamma output in Q24 373// (8 more fractional bits, ~11 bits on the signal). Input x is Q16 (residual); output is Q24. Feed to mm_q24in. 374func rmsnorm_gamma_row_q24(x: *i64, gamma: *i64, xoff: i64, D: i64, out: *i64, ooff: i64) -> i64 { 375 var ss: i64=0; var i: i64=0; while i<D { let v: i64=x[xoff+i]; ss=ss+((v*v)>>16); i=i+1 } 376 let ms: i64=ss/D; let sd: i64=isqrt((ms+1)<<16); if sd<=0 { return 0 } 377 i=0; while i<D { let nm: i64=(x[xoff+i]<<24)/sd; out[ooff+i]=(nm*gamma[i])>>16; i=i+1 } 378 return 0 379} 380// matmul with a Q24 input (rmsnorm_gamma_row_q24 output) x Q16 weights -> Q16 output (>>24). Full-precision accumulate. 381func mm_q24in(inp: *i64, W: *i64, dst: *i64, T: i64, in_dim: i64, out_dim: i64, rev: i64) -> i64 { 382 var t: i64=0 383 while t<T { var o: i64=0 384 while o<out_dim { var s: i64=0 385 if rev==0 { var k: i64=0; while k<in_dim { s=s+(inp[t*in_dim+k]*W[o*in_dim+k]); k=k+1 } } 386 else { var k: i64=in_dim-1; while k>=0 { s=s+(inp[t*in_dim+k]*W[o*in_dim+k]); k=k-1 } } 387 dst[t*out_dim+o]=s>>24; o=o+1 } 388 t=t+1 } 389 return 0 390} 391// ==== DECODE-SPEED: shared thread pool + banded BIT-EXACT parallel kernels (2026-07-09) ===================== 392// Banding is by whole OUTPUT ELEMENT (matmul rows / vocab rows) or whole TENSOR (weight dequant) -> no partial 393// sum ever crosses a band -> results are BIT-IDENTICAL to the serial path (kvgen gate pins the tokens to prove 394// it). Single-submitter: every caller is a main-thread forward. Static POINTERS lazy-mmap'd (BSS-array trap). 395// R0q (2026-09-17): the decode floor was the accumulator's memory round trip -- _nfmm_i8_task feeds ONE int32x8 396// accumulator every 16 weights, so each __i16x16_madd waits on the previous store (measured 232 ms/token on the 1.5B 397// at 635 percent CPU, 13 GB/s streamed of a 40+ GB/s box). Four accumulators strided by 16 lanes give four independent 398// chains; the chunk total is the same integer sum in a different association order, so the result is bit-identical, 399// and each lane now holds a quarter of the products the NF_CHUNK_K bound was derived for. 400const NF_MADD_W: i64 = 16 // lanes one __i16x16_madd consumes 401const NF_R4_STRIDE: i64 = 64 // four madds per step, one per accumulator 402const NF_ACC_BYTES: i64 = 32 // one int32x8 accumulator 403const NF_ACC4_I64: i64 = 16 // four accumulators as i64 slots to zero (128 B) 404// R0r (2026-09-17) BATCHED PREFILL. The serve prefilled a ~500-token BRIGHT prompt token by token, so every prompt token 405// cost a full weight pass (97 ms, the engine's own /gen receipt, the same as a generated token) and prefill was half of a 406// ~98 s query. _nfmm_i8_task_m streams each weight row ONCE and dots it against M activation rows (the loop order 407// nx_nofloat_mm_i8_batch_gate proved amortises the weight read); per (row, output) it runs _nfmm_i8_task's own chunk loop, 408// so every cell is bit-identical to mm_pool_i8 on that row by construction. The M activation rows must stay 409// cache-resident while the weights stream: NF_PREFILL_M x 8960 (the ffn width) x 2 B = 573 KB, inside one core's L2, 410// so the caller cuts a prompt into blocks of NF_PREFILL_M positions. 411const NF_PREFILL_M: i64 = 32 412// R0r-c (2026-09-17): the batched kernel streams each weight row once per NF_PREFILL_M activation rows, so it is no longer 413// DRAM-bound (585 us per row on the decode shape against a 27.5 MB pass that would stream in 0.8 ms once per block) but 414// bound by the accumulator's memory round trip on every madd -- the chain R0q measured as a wash on the single-row path 415// because that path is DRAM-bound. The four-accumulator chunk loop is R0q's, bit-identical by integer associativity. 416// R0r-b (2026-09-17): the four-accumulator arm was a wash because the bound was never the chain, it was the accumulator's memory 417// round trip on EVERY madd (the builtin loads and stores it each call). __i16_dot keeps the int32x8 accumulator in a 418// register for a whole chunk and sums it once; the same integer sum in the same association order per lane, so it is 419// bit-identical to the madd loop (the gate's T9e/T9f). 1 = serve the builtin in the batched kernel, 0 = the madd loop. 420const NF_I8_DOT_DEFAULT: i64 = 1 421// R0s (2026-09-17): the instrument above measured a third of every batched matmul call in the SERIAL quantisation of the 422// activation rows (a division per element), repeated for every projection sharing the input. The rows are now quantised 423// by the pool through the one row routine (nf_quant_row) the serial control also calls, and the engine quantises each 424// layer input once (nf_quant_rows_pool + mm_pool_i8_mq). 0 = pool quantisation (served), 1 = the serial control (the gate's T10). 425const NF_QUANT_SERIAL_DEFAULT: i64 = 0 426const NF_MMARENA_SLOTS: i64 = 32 // ctx slots in g_nf_mmarena (128 B each): the ceiling on bands and on quantisation tasks 427// R0v (2026-09-17): the R0u profile put 68 percent of the batched prefill and about a quarter of every decode token in the 428// attention core -- scalar, one thread, one head after another. The heads are independent, and once a prefill block's K/V 429// rows are in the cache so are its (row, head) pairs under the causal count. 0 = pool attention (served), 1 = serial (the control). 430const NF_ATTN_SERIAL_DEFAULT: i64 = 0 431const NF_I8_R4M_DEFAULT: i64 = 0 // MEASURED 2026-09-17 on the decode shape, 32 rows: single-accumulator 451 us per row, four-accumulator 448 (1006 permil, a wash), so the batched kernel is not bound by the accumulator chain either; the simpler task stays served, the four-accumulator task is the gate's bit-identical control, and the bottleneck is the per-madd instruction count the compiler emits (R0r-b) 432const NF_MMCTX_M: i64 = 10 // band ctx slot: activation row count 433const NF_MMCTX_OUT: i64 = 11 // band ctx slot: out_dim, the dst row stride 434const NF_I8_R4_DEFAULT: i64 = 0 // MEASURED 2026-09-17 on the decode shape (1536 x 8960): incumbent 33.7 GB/s, four-accumulator 27.3 GB/s -- decode is DRAM-bound and the extra accumulator traffic costs 19 percent, so the incumbent stays the served path; the r4 task is kept as the gate's bit-identical control 435static g_nf_pool: *NxThreadPool 436static g_nf_ldarena: *u8 // 12 x 128B: load-task ctx (64B) + private name scratch (64B) 437static g_nf_mmarena: *u8 // 32 x 128B: matmul band ctx 438static g_nf_hdctx: *u8 // 32 x 128B: head band ctx 439static g_nf_hdrow: *u8 // 32 x 8KB : per-band dequant row 440static g_nf_hdtmp: *u8 // 32 x 32KB: per-band dequant window tmp 441static g_nf_hdout: *u8 // 32 x 16B : per-band (best_id, best_logit) 442static g_nf_xi8: *u8 // 4864 x 2 : quantized activation (i16 lanes) for the SIMD W8A8 path 443static g_nf_i8acc: *u8 // 32 x 64B : per-band int32x8 SIMD accumulator 444static g_nf_i8acc4: *u8 // R0q: 32 x 128B : per-band FOUR int32x8 accumulators (k-interleaved, independent chains) 445static g_nf_i8_r4: i64 // R0q: 0 = the single-accumulator incumbent (the served default, NF_I8_R4_DEFAULT), 1 = the four-accumulator task (the gate's bit-identical control arm) 446static g_nf_i8_r4m: i64 // R0r-c: 1 = four accumulators in the BATCHED kernel (served), 0 = the single-accumulator batched task (the gate's control) 447static g_nf_i8_dot: i64 // R0r-b: 1 = the batched kernel runs __i16_dot per chunk (served, NF_I8_DOT_DEFAULT), 0 = the per-madd loop (the gate's control) 448static g_nf_mm_us_quant: i64 // R0r-b profile: microseconds the LAST mm_pool_i8_m spent quantising the activation rows (single-threaded) 449static g_nf_mm_us_pool: i64 // R0r-b profile: microseconds the LAST mm_pool_i8_m spent in the pool (submit .. wait) 450static g_nf_quant_serial: i64 // R0s: 1 = quantise the rows serially (the gate's control), 0 = pool tasks (served) 451// R0s prefill phase profile, microseconds accumulated over the LAST nf_prefill_batched call (reset at entry) 452static g_nf_pf_us_norm: i64 453static g_nf_pf_us_qkv: i64 454static g_nf_pf_us_attn: i64 455static g_nf_pf_us_o: i64 456static g_nf_pf_us_gu: i64 457static g_nf_pf_us_silu: i64 458static g_nf_pf_us_down: i64 459static g_nf_pf_us_quant: i64 460static g_nf_attn_serial: i64 // R0v: 1 = serial attention (the control), 0 = pool tasks (served) 461static g_nf_attn_sc: *i64 // R0v: per-task attention scratch, NF_MMARENA_SLOTS x 2 x positions i64, grown on demand 462static g_nf_attn_sc_cap: i64 463static g_nf_xi8m: *u8 // R0r: the batched kernel's quantised activation rows (grown to the largest M x in_dim seen, never per call) 464static g_nf_xi8m_cap: i64 465static g_nf_sxm: *i64 // R0r: per-row activation scales for the batched kernel 466static g_nf_sxm_cap: i64 467func nf_pool() -> *NxThreadPool { 468 if (g_nf_pool as i64) == 0 { 469 g_nf_pool = nx_pool_new(0, 64) 470 g_nf_ldarena = sys_mmap(12*128) 471 g_nf_mmarena = sys_mmap(NF_MMARENA_SLOTS*128) 472 g_nf_hdctx = sys_mmap(32*128) 473 g_nf_hdrow = sys_mmap(32*8192) 474 g_nf_hdtmp = sys_mmap(32*32768) 475 g_nf_hdout = sys_mmap(32*16) 476 g_nf_xi8 = sys_mmap(4864*2) 477 g_nf_i8acc = sys_mmap(32*64) 478 g_nf_i8acc4 = sys_mmap(32*128) 479 g_nf_i8_r4 = NF_I8_R4_DEFAULT 480 g_nf_i8_r4m = NF_I8_R4M_DEFAULT 481 g_nf_i8_dot = NF_I8_DOT_DEFAULT 482 g_nf_quant_serial = NF_QUANT_SERIAL_DEFAULT 483 g_nf_attn_serial = NF_ATTN_SERIAL_DEFAULT 484 g_nf_xi8m_cap = 0 485 g_nf_sxm_cap = 0 486 } 487 return g_nf_pool 488} 489// int32x8 horizontal sum for the SIMD path. Since the 2026-07-10 sext compiler fix, *i32 loads SIGN-extend 490// natively (the manual branchless sign-extend this fn used to carry was removed in lockstep with the bless -- 491// keeping it would DOUBLE-extend and corrupt already-negative lanes). 492func nf_hsum_sx(acc: *u8) -> i64 { 493 let pp: *i32 = acc as *i32 494 var s: i64=0; var i: i64=0 495 while i<8 { s=s+(pp[i] as i64); i=i+1 } 496 return s 497} 498// one banded-matmul task: rows [lo,hi) of dst = inp x W^T, full accumulate, >>shift. 499func _nfmm_task(ctx_i: i64) -> i64 { 500 let c: *i64 = ctx_i as *i64 501 let inp: *i64=c[0] as *i64; let W: *i64=c[1] as *i64; let dst: *i64=c[2] as *i64 502 let T: i64=c[3]; let ind: i64=c[4]; let outd: i64=c[5]; let lo: i64=c[6]; let hi: i64=c[7]; let sh: i64=c[8] 503 var t: i64=0 504 while t<T { var o: i64=lo 505 while o<hi { var s: i64=0; var k: i64=0; while k<ind { s=s+(inp[t*ind+k]*W[o*ind+k]); k=k+1 } 506 if sh==24 { dst[t*outd+o]=s>>24 } else { dst[t*outd+o]=s>>16 } 507 o=o+1 } 508 t=t+1 } 509 return 0 510} 511// pooled matmul (forward accumulation order only): bit-identical to mm_out_in/mm_q24in with rev=0. 512func mm_pool(inp: *i64, W: *i64, dst: *i64, T: i64, in_dim: i64, out_dim: i64, shift: i64) -> i64 { 513 let p: *NxThreadPool = nf_pool() 514 var bands: i64 = p.n_workers 515 if bands > 32 { bands = 32 } 516 if bands > out_dim { bands = out_dim } 517 if bands < 1 { bands = 1 } 518 let per: i64 = (out_dim + bands - 1) / bands 519 let done0: i64 = nx_pool_n_completed(p) 520 var b: i64 = 0 521 while b < bands { 522 let c: *i64 = ((g_nf_mmarena as i64) + b*128) as *i64 523 c[0]=inp as i64; c[1]=W as i64; c[2]=dst as i64; c[3]=T; c[4]=in_dim; c[5]=out_dim; c[6]=b*per 524 var hi: i64=(b+1)*per 525 if hi>out_dim { hi=out_dim } 526 c[7]=hi; c[8]=shift 527 nx_pool_submit(p, _nfmm_task, c as i64) 528 b=b+1 529 } 530 nx_pool_wait(p, done0+bands) 531 return 0 532} 533// one weight-load task: load_blk with a PRIVATE name scratch (12 concurrent tasks must not share nm). 534func _nfld_task(ctx_i: i64) -> i64 { 535 let c: *i64 = ctx_i as *i64 536 let nm: *u8 = (ctx_i + 64) as *u8 537 load_blk(c[0] as *u8, c[1] as *NxGgufHeader, nm, c[2], c[3] as *u8, c[4] as *i64, c[5]) 538 return 0 539} 540func nf_ld1(p: *NxThreadPool, i: i64, buf: *u8, hdr: *NxGgufHeader, L: i64, sfx: *u8, out: i64, want: i64) -> i64 { 541 let c: *i64 = ((g_nf_ldarena as i64) + i*128) as *i64 542 c[0]=buf as i64; c[1]=hdr as i64; c[2]=L; c[3]=sfx as i64; c[4]=out; c[5]=want 543 nx_pool_submit(p, _nfld_task, c as i64) 544 return 0 545} 546// load one layer's 12 tensors in parallel (disjoint outputs; each task private name scratch). Bit-exact. 547func nf_load_layer_pool(buf: *u8, hdr: *NxGgufHeader, L: i64, wb: *i64, ne: i64, qd: i64, kvd: i64, fd: i64) -> i64 { 548 let p: *NxThreadPool = nf_pool() 549 let done0: i64 = nx_pool_n_completed(p) 550 nf_ld1(p, 0, buf, hdr, L, ".attn_norm.weight\x00" as *u8, wb[0], ne) 551 nf_ld1(p, 1, buf, hdr, L, ".attn_q.weight\x00" as *u8, wb[1], qd*ne) 552 nf_ld1(p, 2, buf, hdr, L, ".attn_k.weight\x00" as *u8, wb[2], kvd*ne) 553 nf_ld1(p, 3, buf, hdr, L, ".attn_v.weight\x00" as *u8, wb[3], kvd*ne) 554 nf_ld1(p, 4, buf, hdr, L, ".attn_output.weight\x00" as *u8, wb[4], ne*qd) 555 nf_ld1(p, 5, buf, hdr, L, ".attn_q.bias\x00" as *u8, wb[9], qd) 556 nf_ld1(p, 6, buf, hdr, L, ".attn_k.bias\x00" as *u8, wb[10], kvd) 557 nf_ld1(p, 7, buf, hdr, L, ".attn_v.bias\x00" as *u8, wb[11], kvd) 558 nf_ld1(p, 8, buf, hdr, L, ".ffn_norm.weight\x00" as *u8, wb[5], ne) 559 nf_ld1(p, 9, buf, hdr, L, ".ffn_gate.weight\x00" as *u8, wb[6], ne*fd) 560 nf_ld1(p, 10, buf, hdr, L, ".ffn_up.weight\x00" as *u8, wb[7], ne*fd) 561 nf_ld1(p, 11, buf, hdr, L, ".ffn_down.weight\x00" as *u8, wb[8], fd*ne) 562 nx_pool_wait(p, done0+12) 563 return 0 564} 565// one head band: scan vocab rows [lo,hi), local strict-> argmax (keeps the LOWEST id on ties, matching serial). 566func _nfhd_task(ctx_i: i64) -> i64 { 567 let c: *i64 = ctx_i as *i64 568 let buf: *u8=c[0] as *u8; let base: i64=c[1]; let ty: i64=c[2]; let normed: *i64=c[3] as *i64 569 let ne: i64=c[4]; let lo: i64=c[5]; let hi: i64=c[6] 570 let row: *i64=c[7] as *i64; let tmp: *i64=c[8] as *i64; let outp: *i64=c[9] as *i64 571 var best: i64=lo 572 var bestv: i64=0-9223372036854775807 573 var v: i64=lo 574 while v<hi { 575 dequant_row(buf, base, ty, v, ne, row, tmp) 576 var s: i64=0; var k: i64=0; while k<ne { s=s+(normed[k]*row[k]); k=k+1 } 577 let lg: i64=s>>24 578 if lg>bestv { bestv=lg; best=v } 579 v=v+1 580 } 581 outp[0]=best 582 outp[1]=bestv 583 return 0 584} 585// pooled streamed LM-head argmax. hp = [buf, oh_base, oh_ty, normed(Q24), vocab, ne, out_id_ptr, out_logit_ptr]. 586// Returns the argmax id; logits use the Q24 head convention (dot >>24). Band-ascending strict-> reduce keeps the 587// lowest id on ties = identical to the serial scan. 588func head_argmax_pool(hp: *i64) -> i64 { 589 let p: *NxThreadPool = nf_pool() 590 let vocab: i64=hp[4] 591 var bands: i64 = p.n_workers 592 if bands > 32 { bands = 32 } 593 if bands > vocab { bands = vocab } 594 if bands < 1 { bands = 1 } 595 let per: i64 = (vocab + bands - 1) / bands 596 // ROOT FIX 2026-07-15: dims-sized per-call band scratch (the fixed 8192B row slot overflows at 597 // ne>1024 into the neighbor band -- same corruption class as the i32 head-cache build; see 598 // nf_dequant_head_all_i32). Freed after the wait. ~2 syscalls vs ~650ms/token here = negligible. 599 let hne: i64 = hp[5] 600 let rstride2: i64 = hne*8 601 let tstride2: i64 = hne*8 + 4096 602 let rows2: *u8 = sys_mmap(bands*rstride2) 603 let tmps2: *u8 = sys_mmap(bands*tstride2) 604 let done0: i64 = nx_pool_n_completed(p) 605 var b: i64 = 0 606 while b < bands { 607 let c: *i64 = ((g_nf_hdctx as i64) + b*128) as *i64 608 c[0]=hp[0]; c[1]=hp[1]; c[2]=hp[2]; c[3]=hp[3]; c[4]=hp[5]; c[5]=b*per 609 var hi: i64=(b+1)*per 610 if hi>vocab { hi=vocab } 611 c[6]=hi 612 c[7]=((rows2 as i64) + b*rstride2) 613 c[8]=((tmps2 as i64) + b*tstride2) 614 c[9]=((g_nf_hdout as i64) + b*16) 615 nx_pool_submit(p, _nfhd_task, c as i64) 616 b=b+1 617 } 618 nx_pool_wait(p, done0+bands) 619 sys_munmap(rows2, bands*rstride2) 620 sys_munmap(tmps2, bands*tstride2) 621 var best: i64=0 622 var bestv: i64=0-9223372036854775807 623 var rb: i64=0 624 while rb<bands { 625 let o: *i64=((g_nf_hdout as i64) + rb*16) as *i64 626 if o[1]>bestv { bestv=o[1]; best=o[0] } 627 rb=rb+1 628 } 629 let oid: *i64=hp[6] as *i64 630 let olg: *i64=hp[7] as *i64 631 oid[0]=best 632 olg[0]=bestv 633 return best 634} 635// attention sublayer: out = x + W_o( GQA-causal-softmax( RoPE(W_q·RMSNorm x), RoPE(W_k·..), W_v·.. ) ). 636// cfg = [T, ne, n_heads, n_kv, head_dim, q_dim, kv_dim, attn_scale]. 637func attn_sublayer(x: *i64, gamma: *i64, Wq: *i64, Wk: *i64, Wv: *i64, Wo: *i64, bq: *i64, bk: *i64, bv: *i64, freqs: *i64, xn: *i64, Q: *i64, K: *i64, V: *i64, concat: *i64, sc: *i64, at: *i64, proj: *i64, out: *i64, cfg: *i64, rev: i64) -> i64 { 638 let T: i64=cfg[0]; let ne: i64=cfg[1]; let nh: i64=cfg[2]; let nkv: i64=cfg[3]; let hd: i64=cfg[4]; let qd: i64=cfg[5]; let kvd: i64=cfg[6]; let scale: i64=cfg[7] 639 var t: i64=0 640 while t<T { rmsnorm_gamma_row_q24(x, gamma, t*ne, ne, xn, t*ne); t=t+1 } 641 // pooled (bit-exact) matmuls on the forward path; rev!=0 keeps the serial reversed-accumulation path so the 642 // fwd-vs-rev determinism teeth still exercise a genuinely different reduction order. 643 if rev==0 { mm_pool(xn, Wq, Q, T, ne, qd, 24); mm_pool(xn, Wk, K, T, ne, kvd, 24); mm_pool(xn, Wv, V, T, ne, kvd, 24) } 644 else { mm_q24in(xn, Wq, Q, T, ne, qd, rev); mm_q24in(xn, Wk, K, T, ne, kvd, rev); mm_q24in(xn, Wv, V, T, ne, kvd, rev) } 645 // Qwen2 Q/K/V projection biases (added before RoPE) 646 var bt: i64=0 647 while bt<T { var bo: i64=0; while bo<qd { Q[bt*qd+bo]=Q[bt*qd+bo]+bq[bo]; bo=bo+1 } var bp: i64=0; while bp<kvd { K[bt*kvd+bp]=K[bt*kvd+bp]+bk[bp]; V[bt*kvd+bp]=V[bt*kvd+bp]+bv[bp]; bp=bp+1 } bt=bt+1 } 648 t=0 649 while t<T { var h: i64=0; while h<nh { rope_apply(((Q as i64)+(t*qd+h*hd)*8) as *i64, hd, t, freqs); h=h+1 } var hk: i64=0; while hk<nkv { rope_apply(((K as i64)+(t*kvd+hk*hd)*8) as *i64, hd, t, freqs); hk=hk+1 } t=t+1 } 650 let group: i64=nh/nkv 651 var hh: i64=0 652 while hh<nh { 653 let kv: i64=hh/group; let bq: i64=hh*hd; let bk: i64=kv*hd 654 t=0 655 while t<T { 656 let cnt: i64=t+1; var s: i64=0 657 while s<cnt { var dot: i64=0; var d: i64=0; while d<hd { dot=dot+(Q[t*qd+bq+d]*K[s*kvd+bk+d]); d=d+1 } sc[s]=qmul(dot>>16, scale); s=s+1 } 658 var mmax: i64=sc[0]; var j: i64=1; while j<cnt { if sc[j]>mmax { mmax=sc[j] } j=j+1 } 659 var sum: i64=0; j=0; while j<cnt { let e: i64=fx_exp(sc[j]-mmax); at[j]=e; sum=sum+e; j=j+1 } 660 if sum<=0 { sum=1 } 661 j=0; while j<cnt { at[j]=(at[j]<<16)/sum; j=j+1 } 662 var d2: i64=0 663 while d2<hd { var acc: i64=0; s=0; while s<cnt { acc=acc+(at[s]*V[s*kvd+bk+d2]); s=s+1 } concat[t*qd+bq+d2]=acc>>16; d2=d2+1 } 664 t=t+1 665 } 666 hh=hh+1 667 } 668 mm_out_in(concat, Wo, proj, T, qd, ne, rev) 669 var i: i64=0; while i<T*ne { out[i]=x[i]+proj[i]; i=i+1 } 670 return 0 671} 672// FFN sublayer: out = x + W_down( SiLU(W_gate·RMSNorm x) (*) W_up·RMSNorm x ). cfg = [T, ne, ffn_dim]. 673func ffn_sublayer(x: *i64, gamma: *i64, Wg: *i64, Wu: *i64, Wd: *i64, xn: *i64, gate: *i64, up: *i64, hbuf: *i64, proj: *i64, out: *i64, cfg: *i64, rev: i64) -> i64 { 674 let T: i64=cfg[0]; let ne: i64=cfg[1]; let fd: i64=cfg[2] 675 var t: i64=0 676 while t<T { rmsnorm_gamma_row_q24(x, gamma, t*ne, ne, xn, t*ne); t=t+1 } 677 mm_q24in(xn, Wg, gate, T, ne, fd, rev); mm_q24in(xn, Wu, up, T, ne, fd, rev) 678 var i: i64=0; while i<T*fd { hbuf[i]=qmul(silu(gate[i]), up[i]); i=i+1 } 679 mm_out_in(hbuf, Wd, proj, T, fd, ne, rev) 680 i=0; while i<T*ne { out[i]=x[i]+proj[i]; i=i+1 } 681 return 0 682} 683// run N transformer blocks (lazy per-layer load), x -> out. wb=9 weight buffers, sb=14 scratch (ptrs as i64). 684func run_stack(buf: *u8, hdr: *NxGgufHeader, x: *i64, out: *i64, wb: *i64, sb: *i64, nm: *u8, freqs: *i64, cfgA: *i64, cfgF: *i64, N: i64, rev: i64) -> i64 { 685 let T: i64=cfgA[0]; let ne: i64=cfgA[1]; let qd: i64=cfgA[5]; let kvd: i64=cfgA[6]; let fd: i64=cfgF[2] 686 let gA: *i64=wb[0] as *i64; let Wq: *i64=wb[1] as *i64; let Wk: *i64=wb[2] as *i64; let Wv: *i64=wb[3] as *i64; let Wo: *i64=wb[4] as *i64 687 let gF: *i64=wb[5] as *i64; let Wg: *i64=wb[6] as *i64; let Wu: *i64=wb[7] as *i64; let Wd: *i64=wb[8] as *i64 688 let bq: *i64=wb[9] as *i64; let bk: *i64=wb[10] as *i64; let bv: *i64=wb[11] as *i64 689 let xn: *i64=sb[0] as *i64; let Q: *i64=sb[1] as *i64; let K: *i64=sb[2] as *i64; let V: *i64=sb[3] as *i64; let concat: *i64=sb[4] as *i64 690 let sc: *i64=sb[5] as *i64; let at: *i64=sb[6] as *i64; let proj: *i64=sb[7] as *i64; let gate: *i64=sb[8] as *i64; let up: *i64=sb[9] as *i64 691 let hbuf: *i64=sb[10] as *i64; let hmid: *i64=sb[11] as *i64; let cur: *i64=sb[12] as *i64; let cur2: *i64=sb[13] as *i64 692 cpy(cur, x, T*ne) 693 var L: i64=0 694 while L<N { 695 load_blk(buf, hdr, nm, L, ".attn_norm.weight\x00" as *u8, gA, ne) 696 load_blk(buf, hdr, nm, L, ".attn_q.weight\x00" as *u8, Wq, qd*ne) 697 load_blk(buf, hdr, nm, L, ".attn_k.weight\x00" as *u8, Wk, kvd*ne) 698 load_blk(buf, hdr, nm, L, ".attn_v.weight\x00" as *u8, Wv, kvd*ne) 699 load_blk(buf, hdr, nm, L, ".attn_output.weight\x00" as *u8, Wo, ne*qd) 700 load_blk(buf, hdr, nm, L, ".attn_q.bias\x00" as *u8, bq, qd) 701 load_blk(buf, hdr, nm, L, ".attn_k.bias\x00" as *u8, bk, kvd) 702 load_blk(buf, hdr, nm, L, ".attn_v.bias\x00" as *u8, bv, kvd) 703 load_blk(buf, hdr, nm, L, ".ffn_norm.weight\x00" as *u8, gF, ne) 704 load_blk(buf, hdr, nm, L, ".ffn_gate.weight\x00" as *u8, Wg, ne*fd) 705 load_blk(buf, hdr, nm, L, ".ffn_up.weight\x00" as *u8, Wu, ne*fd) 706 load_blk(buf, hdr, nm, L, ".ffn_down.weight\x00" as *u8, Wd, fd*ne) 707 attn_sublayer(cur, gA, Wq, Wk, Wv, Wo, bq, bk, bv, freqs, xn, Q, K, V, concat, sc, at, proj, hmid, cfgA, rev) 708 ffn_sublayer(hmid, gF, Wg, Wu, Wd, xn, gate, up, hbuf, proj, cur2, cfgF, rev) 709 cpy(cur, cur2, T*ne) 710 L=L+1 711 } 712 cpy(out, cur, T*ne) 713 return 0 714} 715// ==== KV-CACHE decode path (2026-07-09) ==================================================================== 716// Prefill captures each layer's post-bias/post-RoPE K rows + post-bias V rows; decode_step_kv forwards ONLY the 717// new token, appending to + attending over the cache. LOSSLESS vs the uncached forward: causal masking means an 718// old token's K/V at every layer depend only on tokens 0..itself, so cached decode is BIT-IDENTICAL (gate-proven 719// by matching the uncached generation token-for-token). kvc = 2N ptrs [Kc_0, Vc_0, Kc_1, Vc_1, ...], each cache 720// MAXT*kvd i64. ADDITIVE: run_stack keeps its signature (nx_cc has no arity check -- changing it would let old 721// call sites pass garbage silently). 722func run_stack_prefill_kv(buf: *u8, hdr: *NxGgufHeader, x: *i64, out: *i64, wb: *i64, sb: *i64, nm: *u8, freqs: *i64, kvc: *i64, cfgA: *i64, cfgF: *i64, N: i64) -> i64 { 723 let T: i64=cfgA[0]; let ne: i64=cfgA[1]; let qd: i64=cfgA[5]; let kvd: i64=cfgA[6]; let fd: i64=cfgF[2] 724 let gA: *i64=wb[0] as *i64; let Wq: *i64=wb[1] as *i64; let Wk: *i64=wb[2] as *i64; let Wv: *i64=wb[3] as *i64; let Wo: *i64=wb[4] as *i64 725 let gF: *i64=wb[5] as *i64; let Wg: *i64=wb[6] as *i64; let Wu: *i64=wb[7] as *i64; let Wd: *i64=wb[8] as *i64 726 let bq: *i64=wb[9] as *i64; let bk: *i64=wb[10] as *i64; let bv: *i64=wb[11] as *i64 727 let xn: *i64=sb[0] as *i64; let Q: *i64=sb[1] as *i64; let K: *i64=sb[2] as *i64; let V: *i64=sb[3] as *i64; let concat: *i64=sb[4] as *i64 728 let sc: *i64=sb[5] as *i64; let at: *i64=sb[6] as *i64; let proj: *i64=sb[7] as *i64; let gate: *i64=sb[8] as *i64; let up: *i64=sb[9] as *i64 729 let hbuf: *i64=sb[10] as *i64; let hmid: *i64=sb[11] as *i64; let cur: *i64=sb[12] as *i64; let cur2: *i64=sb[13] as *i64 730 cpy(cur, x, T*ne) 731 var L: i64=0 732 while L<N { 733 nf_load_layer_pool(buf, hdr, L, wb, ne, qd, kvd, fd) // parallel weight load (matmuls stay serial: one-time) 734 attn_sublayer(cur, gA, Wq, Wk, Wv, Wo, bq, bk, bv, freqs, xn, Q, K, V, concat, sc, at, proj, hmid, cfgA, 0) 735 // capture this layer's K/V rows (K = post-bias post-RoPE; V = post-bias) into the layer cache 736 let Kc: *i64 = kvc[2*L] as *i64 737 let Vc: *i64 = kvc[2*L+1] as *i64 738 cpy(Kc, K, T*kvd) 739 cpy(Vc, V, T*kvd) 740 ffn_sublayer(hmid, gF, Wg, Wu, Wd, xn, gate, up, hbuf, proj, cur2, cfgF, 0) 741 cpy(cur, cur2, T*ne) 742 L=L+1 743 } 744 cpy(out, cur, T*ne) 745 return 0 746} 747// decode ONE token at absolute position `pos` against the populated caches. x1/out1 = single ne-row. Mirrors the 748// uncached math exactly: RMSNorm(Q24) -> QKV(mm_q24in) -> +bias -> RoPE(pos) -> append cache row -> GQA softmax over 749// rows 0..pos -> W_o -> residual -> ffn_sublayer(T=1). 750func decode_step_kv(buf: *u8, hdr: *NxGgufHeader, x1: *i64, out1: *i64, wb: *i64, sb: *i64, nm: *u8, freqs: *i64, kvc: *i64, pos: i64, cfgA: *i64, cfgF: *i64, N: i64) -> i64 { 751 let ne: i64=cfgA[1]; let nh: i64=cfgA[2]; let nkv: i64=cfgA[3]; let hd: i64=cfgA[4]; let qd: i64=cfgA[5]; let kvd: i64=cfgA[6]; let scale: i64=cfgA[7]; let fd: i64=cfgF[2] 752 let gA: *i64=wb[0] as *i64; let Wq: *i64=wb[1] as *i64; let Wk: *i64=wb[2] as *i64; let Wv: *i64=wb[3] as *i64; let Wo: *i64=wb[4] as *i64 753 let gF: *i64=wb[5] as *i64; let Wg: *i64=wb[6] as *i64; let Wu: *i64=wb[7] as *i64; let Wd: *i64=wb[8] as *i64 754 let bq: *i64=wb[9] as *i64; let bk: *i64=wb[10] as *i64; let bv: *i64=wb[11] as *i64 755 let xn: *i64=sb[0] as *i64; let Q: *i64=sb[1] as *i64; let K: *i64=sb[2] as *i64; let V: *i64=sb[3] as *i64; let concat: *i64=sb[4] as *i64 756 let sc: *i64=sb[5] as *i64; let at: *i64=sb[6] as *i64; let proj: *i64=sb[7] as *i64; let gate: *i64=sb[8] as *i64; let up: *i64=sb[9] as *i64 757 let hbuf: *i64=sb[10] as *i64; let hmid: *i64=sb[11] as *i64; let cur: *i64=sb[12] as *i64; let cur2: *i64=sb[13] as *i64 758 cpy(cur, x1, ne) 759 var L: i64=0 760 while L<N { 761 nf_load_layer_pool(buf, hdr, L, wb, ne, qd, kvd, fd) // all 12 tensors in parallel (disjoint outputs) 762 rmsnorm_gamma_row_q24(cur, gA, 0, ne, xn, 0) 763 mm_pool(xn, Wq, Q, 1, ne, qd, 24) 764 mm_pool(xn, Wk, K, 1, ne, kvd, 24) 765 mm_pool(xn, Wv, V, 1, ne, kvd, 24) 766 var bo: i64=0; while bo<qd { Q[bo]=Q[bo]+bq[bo]; bo=bo+1 } 767 var bp: i64=0; while bp<kvd { K[bp]=K[bp]+bk[bp]; V[bp]=V[bp]+bv[bp]; bp=bp+1 } 768 var h: i64=0; while h<nh { rope_apply(((Q as i64)+(h*hd)*8) as *i64, hd, pos, freqs); h=h+1 } 769 var hk: i64=0; while hk<nkv { rope_apply(((K as i64)+(hk*hd)*8) as *i64, hd, pos, freqs); hk=hk+1 } 770 let Kc: *i64 = kvc[2*L] as *i64 771 let Vc: *i64 = kvc[2*L+1] as *i64 772 var ci: i64=0; while ci<kvd { Kc[pos*kvd+ci]=K[ci]; Vc[pos*kvd+ci]=V[ci]; ci=ci+1 } 773 let group: i64=nh/nkv 774 let cnt: i64=pos+1 775 var hh: i64=0 776 while hh<nh { 777 let kvh: i64=hh/group; let qb: i64=hh*hd; let kb: i64=kvh*hd 778 var s: i64=0 779 while s<cnt { var dot: i64=0; var d: i64=0; while d<hd { dot=dot+(Q[qb+d]*Kc[s*kvd+kb+d]); d=d+1 } sc[s]=qmul(dot>>16, scale); s=s+1 } 780 var mmax: i64=sc[0]; var j: i64=1; while j<cnt { if sc[j]>mmax { mmax=sc[j] } j=j+1 } 781 var sum: i64=0; j=0; while j<cnt { let e: i64=fx_exp(sc[j]-mmax); at[j]=e; sum=sum+e; j=j+1 } 782 if sum<=0 { sum=1 } 783 j=0; while j<cnt { at[j]=(at[j]<<16)/sum; j=j+1 } 784 var d2: i64=0 785 while d2<hd { var acc: i64=0; s=0; while s<cnt { acc=acc+(at[s]*Vc[s*kvd+kb+d2]); s=s+1 } concat[qb+d2]=acc>>16; d2=d2+1 } 786 hh=hh+1 787 } 788 mm_pool(concat, Wo, proj, 1, qd, ne, 16) 789 var ri: i64=0; while ri<ne { hmid[ri]=cur[ri]+proj[ri]; ri=ri+1 } 790 // inlined FFN (T=1) with pooled matmuls -- bit-exact vs ffn_sublayer 791 rmsnorm_gamma_row_q24(hmid, gF, 0, ne, xn, 0) 792 mm_pool(xn, Wg, gate, 1, ne, fd, 24) 793 mm_pool(xn, Wu, up, 1, ne, fd, 24) 794 var fi: i64=0; while fi<fd { hbuf[fi]=qmul(silu(gate[fi]), up[fi]); fi=fi+1 } 795 mm_pool(hbuf, Wd, proj, 1, fd, ne, 16) 796 var fo: i64=0; while fo<ne { cur[fo]=hmid[fo]+proj[fo]; fo=fo+1 } 797 L=L+1 798 } 799 cpy(out1, cur, ne) 800 return 0 801} 802// ==== DEQUANT-ONCE WEIGHT CACHE (2026-07-09) ============================================================= 803// MEASURED (dqprobe gate): a decode_step_kv spends ~88% of its time (1213/1363 ms) RE-DEQUANTIZING the 804// frozen weights -- every layer, every token -- even though the weights never change across decode steps. 805// Dequant ONCE into a per-layer cache and reuse => decode is bit-identical (same dequant, just not repeated) 806// but ~9x faster. wcache = N pointers, each a 12-slot wb set (identical layout to nf_load_layer_pool's). 807// ADDITIVE: brand-new functions; decode_step_kv / run_stack untouched (nx_cc has no arity check, so changing 808// a proven signature would let old call sites pass garbage silently). 809func nf_alloc_layer_wb(ne: i64, qd: i64, kvd: i64, fd: i64) -> *i64 { 810 let w: *i64 = sys_mmap(12*8) as *i64 811 w[0]=sys_mmap(ne*8) as i64; w[1]=sys_mmap(qd*ne*8) as i64; w[2]=sys_mmap(kvd*ne*8) as i64; w[3]=sys_mmap(kvd*ne*8) as i64; w[4]=sys_mmap(ne*qd*8) as i64 812 w[5]=sys_mmap(ne*8) as i64; w[6]=sys_mmap(ne*fd*8) as i64; w[7]=sys_mmap(ne*fd*8) as i64; w[8]=sys_mmap(fd*ne*8) as i64 813 w[9]=sys_mmap(qd*8) as i64; w[10]=sys_mmap(kvd*8) as i64; w[11]=sys_mmap(kvd*8) as i64 814 return w 815} 816// dequant ALL N layers ONCE into wcache (wcache[L] = a fresh 12-slot wb set). Reuses the proven 817// nf_load_layer_pool dequant verbatim => bit-identical to the per-step path, done a single time. 818func nf_dequant_all_layers(buf: *u8, hdr: *NxGgufHeader, wcache: *i64, N: i64, ne: i64, qd: i64, kvd: i64, fd: i64) -> i64 { 819 var L: i64=0 820 while L<N { 821 let w: *i64 = nf_alloc_layer_wb(ne, qd, kvd, fd) 822 nf_load_layer_pool(buf, hdr, L, w, ne, qd, kvd, fd) 823 wcache[L]=w as i64 824 L=L+1 825 } 826 return 0 827} 828// decode ONE token using the pre-dequantized wcache -> NO per-step dequant. Byte-for-byte the same math as 829// decode_step_kv (same weights, same accumulation order); only the redundant re-dequant is removed. The 830// weight pointers are derived per-layer INSIDE the loop from wcache[L] (each layer has its own buffers). 831func decode_step_kv_cached(buf: *u8, hdr: *NxGgufHeader, x1: *i64, out1: *i64, wcache: *i64, sb: *i64, nm: *u8, freqs: *i64, kvc: *i64, pos: i64, cfgA: *i64, cfgF: *i64, N: i64) -> i64 { 832 let ne: i64=cfgA[1]; let nh: i64=cfgA[2]; let nkv: i64=cfgA[3]; let hd: i64=cfgA[4]; let qd: i64=cfgA[5]; let kvd: i64=cfgA[6]; let scale: i64=cfgA[7]; let fd: i64=cfgF[2] 833 let xn: *i64=sb[0] as *i64; let Q: *i64=sb[1] as *i64; let K: *i64=sb[2] as *i64; let V: *i64=sb[3] as *i64; let concat: *i64=sb[4] as *i64 834 let sc: *i64=sb[5] as *i64; let at: *i64=sb[6] as *i64; let proj: *i64=sb[7] as *i64; let gate: *i64=sb[8] as *i64; let up: *i64=sb[9] as *i64 835 let hbuf: *i64=sb[10] as *i64; let hmid: *i64=sb[11] as *i64; let cur: *i64=sb[12] as *i64 836 cpy(cur, x1, ne) 837 var L: i64=0 838 while L<N { 839 let wb: *i64 = wcache[L] as *i64 840 let gA: *i64=wb[0] as *i64; let Wq: *i64=wb[1] as *i64; let Wk: *i64=wb[2] as *i64; let Wv: *i64=wb[3] as *i64; let Wo: *i64=wb[4] as *i64 841 let gF: *i64=wb[5] as *i64; let Wg: *i64=wb[6] as *i64; let Wu: *i64=wb[7] as *i64; let Wd: *i64=wb[8] as *i64 842 let bq: *i64=wb[9] as *i64; let bk: *i64=wb[10] as *i64; let bv: *i64=wb[11] as *i64 843 rmsnorm_gamma_row_q24(cur, gA, 0, ne, xn, 0) 844 mm_pool(xn, Wq, Q, 1, ne, qd, 24) 845 mm_pool(xn, Wk, K, 1, ne, kvd, 24) 846 mm_pool(xn, Wv, V, 1, ne, kvd, 24) 847 var bo: i64=0; while bo<qd { Q[bo]=Q[bo]+bq[bo]; bo=bo+1 } 848 var bp: i64=0; while bp<kvd { K[bp]=K[bp]+bk[bp]; V[bp]=V[bp]+bv[bp]; bp=bp+1 } 849 var h: i64=0; while h<nh { rope_apply(((Q as i64)+(h*hd)*8) as *i64, hd, pos, freqs); h=h+1 } 850 var hk: i64=0; while hk<nkv { rope_apply(((K as i64)+(hk*hd)*8) as *i64, hd, pos, freqs); hk=hk+1 } 851 let Kc: *i64 = kvc[2*L] as *i64 852 let Vc: *i64 = kvc[2*L+1] as *i64 853 var ci: i64=0; while ci<kvd { Kc[pos*kvd+ci]=K[ci]; Vc[pos*kvd+ci]=V[ci]; ci=ci+1 } 854 let group: i64=nh/nkv 855 let cnt: i64=pos+1 856 var hh: i64=0 857 while hh<nh { 858 let kvh: i64=hh/group; let qb: i64=hh*hd; let kb: i64=kvh*hd 859 var s: i64=0 860 while s<cnt { var dot: i64=0; var d: i64=0; while d<hd { dot=dot+(Q[qb+d]*Kc[s*kvd+kb+d]); d=d+1 } sc[s]=qmul(dot>>16, scale); s=s+1 } 861 var mmax: i64=sc[0]; var j: i64=1; while j<cnt { if sc[j]>mmax { mmax=sc[j] } j=j+1 } 862 var sum: i64=0; j=0; while j<cnt { let e: i64=fx_exp(sc[j]-mmax); at[j]=e; sum=sum+e; j=j+1 } 863 if sum<=0 { sum=1 } 864 j=0; while j<cnt { at[j]=(at[j]<<16)/sum; j=j+1 } 865 var d2: i64=0 866 while d2<hd { var acc: i64=0; s=0; while s<cnt { acc=acc+(at[s]*Vc[s*kvd+kb+d2]); s=s+1 } concat[qb+d2]=acc>>16; d2=d2+1 } 867 hh=hh+1 868 } 869 mm_pool(concat, Wo, proj, 1, qd, ne, 16) 870 var ri: i64=0; while ri<ne { hmid[ri]=cur[ri]+proj[ri]; ri=ri+1 } 871 rmsnorm_gamma_row_q24(hmid, gF, 0, ne, xn, 0) 872 mm_pool(xn, Wg, gate, 1, ne, fd, 24) 873 mm_pool(xn, Wu, up, 1, ne, fd, 24) 874 var fi: i64=0; while fi<fd { hbuf[fi]=qmul(silu(gate[fi]), up[fi]); fi=fi+1 } 875 mm_pool(hbuf, Wd, proj, 1, fd, ne, 16) 876 var fo: i64=0; while fo<ne { cur[fo]=hmid[fo]+proj[fo]; fo=fo+1 } 877 L=L+1 878 } 879 cpy(out1, cur, ne) 880 return 0 881} 882// ==== DEQUANT-ONCE OUTPUT-HEAD CACHE ==================================================================== 883// head_argmax_pool re-dequantizes ALL `vocab` head rows every token (~650 ms of the per-token cost). Dequant 884// the whole [vocab x ne] head ONCE into an i64 cache; then argmax is a pure pooled dot -> LOSSLESS + flat. 885func _nfhdq_task(ctx_i: i64) -> i64 { 886 let c: *i64 = ctx_i as *i64 887 let buf: *u8=c[0] as *u8; let base: i64=c[1]; let ty: i64=c[2]; let cache: *i64=c[3] as *i64 888 let ne: i64=c[4]; let lo: i64=c[5]; let hi: i64=c[6]; let tmp: *i64=c[7] as *i64 889 var v: i64=lo 890 while v<hi { dequant_row(buf, base, ty, v, ne, ((cache as i64)+v*ne*8) as *i64, tmp); v=v+1 } 891 return 0 892} 893// dequant the full output head ONCE (banded over vocab). Returns the [vocab x ne] i64 cache base (0 on OOM). 894func nf_dequant_head_all(buf: *u8, oh_base: i64, oh_ty: i64, vocab: i64, ne: i64) -> *i64 { 895 let p: *NxThreadPool = nf_pool() 896 let cache: *i64 = sys_mmap(vocab*ne*8) as *i64 897 if (cache as i64)==0 { return cache } 898 var bands: i64 = p.n_workers 899 if bands > 32 { bands = 32 } 900 if bands > vocab { bands = vocab } 901 if bands < 1 { bands = 1 } 902 let per: i64 = (vocab + bands - 1) / bands 903 let done0: i64 = nx_pool_n_completed(p) 904 var b: i64 = 0 905 while b < bands { 906 let c: *i64 = ((g_nf_hdctx as i64) + b*128) as *i64 907 c[0]=buf as i64; c[1]=oh_base; c[2]=oh_ty; c[3]=cache as i64; c[4]=ne; c[5]=b*per 908 var hi: i64=(b+1)*per 909 if hi>vocab { hi=vocab } 910 c[6]=hi 911 c[7]=((g_nf_hdtmp as i64) + b*32768) 912 nx_pool_submit(p, _nfhdq_task, c as i64) 913 b=b+1 914 } 915 nx_pool_wait(p, done0+bands) 916 return cache 917} 918// one cached-head argmax band: dot normed(Q24) against pre-dequantized rows [lo,hi); strict-> keeps lowest id. 919func _nfhac_task(ctx_i: i64) -> i64 { 920 let c: *i64 = ctx_i as *i64 921 let cache: *i64=c[0] as *i64; let normed: *i64=c[1] as *i64; let ne: i64=c[2]; let lo: i64=c[3]; let hi: i64=c[4]; let outp: *i64=c[5] as *i64 922 var best: i64=lo 923 var bestv: i64=0-9223372036854775807 924 var v: i64=lo 925 while v<hi { 926 let row: *i64=((cache as i64)+v*ne*8) as *i64 927 var s: i64=0; var k: i64=0; while k<ne { s=s+(normed[k]*row[k]); k=k+1 } 928 let lg: i64=s>>24 929 if lg>bestv { bestv=lg; best=v } 930 v=v+1 931 } 932 outp[0]=best; outp[1]=bestv 933 return 0 934} 935// pooled argmax over the PRE-DEQUANTIZED head cache. hcp = [cache, normed(Q24), vocab, ne, idout, lgout]. 936// Band-ascending strict-> reduce keeps the lowest id on ties = bit-identical to head_argmax_pool. 937func head_argmax_cached(hcp: *i64) -> i64 { 938 let p: *NxThreadPool = nf_pool() 939 let cache: i64=hcp[0]; let normed: i64=hcp[1]; let vocab: i64=hcp[2]; let ne: i64=hcp[3] 940 var bands: i64 = p.n_workers 941 if bands > 32 { bands = 32 } 942 if bands > vocab { bands = vocab } 943 if bands < 1 { bands = 1 } 944 let per: i64 = (vocab + bands - 1) / bands 945 let done0: i64 = nx_pool_n_completed(p) 946 var b: i64 = 0 947 while b < bands { 948 let c: *i64 = ((g_nf_hdctx as i64) + b*128) as *i64 949 c[0]=cache; c[1]=normed; c[2]=ne; c[3]=b*per 950 var hi: i64=(b+1)*per 951 if hi>vocab { hi=vocab } 952 c[4]=hi 953 c[5]=((g_nf_hdout as i64) + b*16) 954 nx_pool_submit(p, _nfhac_task, c as i64) 955 b=b+1 956 } 957 nx_pool_wait(p, done0+bands) 958 var best: i64=0 959 var bestv: i64=0-9223372036854775807 960 var rb: i64=0 961 while rb<bands { 962 let o: *i64=((g_nf_hdout as i64) + rb*16) as *i64 963 if o[1]>bestv { bestv=o[1]; best=o[0] } 964 rb=rb+1 965 } 966 let oid: *i64=hcp[4] as *i64 967 let olg: *i64=hcp[5] as *i64 968 oid[0]=best; olg[0]=bestv 969 return best 970} 971// ==== i32 COMPACT CACHE (2026-07-09) ===================================================================== 972// MEASURED (fastgen gate): after killing re-dequant, the decode residual is MEMORY BANDWIDTH -- 2.73 GB of i64 973// weights read per token vs a 150 ms compute floor. Q16 weight values are real*65536 and |real|<32768 always, 974// so they fit EXACTLY in i32 -> halving the cache (2.73->1.43 GB) is LOSSLESS. *i32 loads SIGN-extend natively 975// since the 2026-07-10 sext compiler fix (the hot loops used to carry a manual branchless sign-extend; removed 976// in lockstep with the bless). The narrow pass counts any value that would NOT fit i32 -> nonzero = NOT lossless. 977func nf_narrow_i32(src: *i64, dst: *i32, cnt: i64) -> i64 { 978 var ovf: i64=0 979 var i: i64=0 980 while i<cnt { 981 let v: i64=src[i] 982 if v > 2147483647 { ovf=ovf+1 } 983 if v < (0-2147483648) { ovf=ovf+1 } 984 dst[i]=v as i32 985 i=i+1 986 } 987 return ovf 988} 989// one banded-matmul task with i32 weights (sign-extended on load). inp/dst stay i64. Bit-identical to _nfmm_task. 990func _nfmm_i32_task(ctx_i: i64) -> i64 { 991 let c: *i64 = ctx_i as *i64 992 let inp: *i64=c[0] as *i64; let W: *i32=c[1] as *i32; let dst: *i64=c[2] as *i64 993 let T: i64=c[3]; let ind: i64=c[4]; let outd: i64=c[5]; let lo: i64=c[6]; let hi: i64=c[7]; let sh: i64=c[8] 994 var t: i64=0 995 while t<T { var o: i64=lo 996 while o<hi { var s: i64=0; var k: i64=0 997 while k<ind { s=s+(inp[t*ind+k]*(W[o*ind+k] as i64)); k=k+1 } // *i32 sign-extends natively (2026-07-10 sext fix) 998 if sh==24 { dst[t*outd+o]=s>>24 } else { dst[t*outd+o]=s>>16 } 999 o=o+1 } 1000 t=t+1 } 1001 return 0 1002} 1003// pooled i32-weight matmul: bit-identical to mm_pool (same accumulation order, sign-extended i32 == the i64 value). 1004func mm_pool_i32(inp: *i64, W: *i32, dst: *i64, T: i64, in_dim: i64, out_dim: i64, shift: i64) -> i64 { 1005 let p: *NxThreadPool = nf_pool() 1006 var bands: i64 = p.n_workers 1007 if bands > 32 { bands = 32 } 1008 if bands > out_dim { bands = out_dim } 1009 if bands < 1 { bands = 1 } 1010 let per: i64 = (out_dim + bands - 1) / bands 1011 let done0: i64 = nx_pool_n_completed(p) 1012 var b: i64 = 0 1013 while b < bands { 1014 let c: *i64 = ((g_nf_mmarena as i64) + b*128) as *i64 1015 c[0]=inp as i64; c[1]=W as i64; c[2]=dst as i64; c[3]=T; c[4]=in_dim; c[5]=out_dim; c[6]=b*per 1016 var hi: i64=(b+1)*per 1017 if hi>out_dim { hi=out_dim } 1018 c[7]=hi; c[8]=shift 1019 nx_pool_submit(p, _nfmm_i32_task, c as i64) 1020 b=b+1 1021 } 1022 nx_pool_wait(p, done0+bands) 1023 return 0 1024} 1025// one i32 weight set: mm weights (idx 1,2,3,4,6,7,8) are i32 (4B); norms (0,5) + biases (9,10,11) stay i64. 1026func nf_alloc_layer_wb_i32(ne: i64, qd: i64, kvd: i64, fd: i64) -> *i64 { 1027 let w: *i64 = sys_mmap(12*8) as *i64 1028 w[0]=sys_mmap(ne*8) as i64; w[1]=sys_mmap(qd*ne*4) as i64; w[2]=sys_mmap(kvd*ne*4) as i64; w[3]=sys_mmap(kvd*ne*4) as i64; w[4]=sys_mmap(ne*qd*4) as i64 1029 w[5]=sys_mmap(ne*8) as i64; w[6]=sys_mmap(ne*fd*4) as i64; w[7]=sys_mmap(ne*fd*4) as i64; w[8]=sys_mmap(fd*ne*4) as i64 1030 w[9]=sys_mmap(qd*8) as i64; w[10]=sys_mmap(kvd*8) as i64; w[11]=sys_mmap(kvd*8) as i64 1031 return w 1032} 1033// dequant all N layers to a reused i64 scratch, narrow the 7 mm weights to i32 into wc32[L]. Returns total 1034// overflow count across all layers -- MUST be 0 for the i32 cache to be lossless (gate asserts it). 1035func nf_dequant_all_layers_i32(buf: *u8, hdr: *NxGgufHeader, wc32: *i64, N: i64, ne: i64, qd: i64, kvd: i64, fd: i64) -> i64 { 1036 let tmp: *i64 = nf_alloc_layer_wb(ne, qd, kvd, fd) 1037 var ovf: i64=0 1038 var L: i64=0 1039 while L<N { 1040 nf_load_layer_pool(buf, hdr, L, tmp, ne, qd, kvd, fd) 1041 let s: *i64 = nf_alloc_layer_wb_i32(ne, qd, kvd, fd) 1042 cpy(s[0] as *i64, tmp[0] as *i64, ne) 1043 ovf=ovf+nf_narrow_i32(tmp[1] as *i64, s[1] as *i32, qd*ne) 1044 ovf=ovf+nf_narrow_i32(tmp[2] as *i64, s[2] as *i32, kvd*ne) 1045 ovf=ovf+nf_narrow_i32(tmp[3] as *i64, s[3] as *i32, kvd*ne) 1046 ovf=ovf+nf_narrow_i32(tmp[4] as *i64, s[4] as *i32, ne*qd) 1047 cpy(s[5] as *i64, tmp[5] as *i64, ne) 1048 ovf=ovf+nf_narrow_i32(tmp[6] as *i64, s[6] as *i32, ne*fd) 1049 ovf=ovf+nf_narrow_i32(tmp[7] as *i64, s[7] as *i32, ne*fd) 1050 ovf=ovf+nf_narrow_i32(tmp[8] as *i64, s[8] as *i32, fd*ne) 1051 cpy(s[9] as *i64, tmp[9] as *i64, qd) 1052 cpy(s[10] as *i64, tmp[10] as *i64, kvd) 1053 cpy(s[11] as *i64, tmp[11] as *i64, kvd) 1054 wc32[L]=s as i64 1055 L=L+1 1056 } 1057 return ovf 1058} 1059// decode ONE token using the i32 compact cache. Byte-for-byte the same math as decode_step_kv_cached (the 1060// sign-extended i32 weight == the i64 value it was narrowed from), just half the weight bandwidth. 1061func decode_step_kv_cached_i32(buf: *u8, hdr: *NxGgufHeader, x1: *i64, out1: *i64, wcache: *i64, sb: *i64, nm: *u8, freqs: *i64, kvc: *i64, pos: i64, cfgA: *i64, cfgF: *i64, N: i64) -> i64 { 1062 let ne: i64=cfgA[1]; let nh: i64=cfgA[2]; let nkv: i64=cfgA[3]; let hd: i64=cfgA[4]; let qd: i64=cfgA[5]; let kvd: i64=cfgA[6]; let scale: i64=cfgA[7]; let fd: i64=cfgF[2] 1063 let xn: *i64=sb[0] as *i64; let Q: *i64=sb[1] as *i64; let K: *i64=sb[2] as *i64; let V: *i64=sb[3] as *i64; let concat: *i64=sb[4] as *i64 1064 let sc: *i64=sb[5] as *i64; let at: *i64=sb[6] as *i64; let proj: *i64=sb[7] as *i64; let gate: *i64=sb[8] as *i64; let up: *i64=sb[9] as *i64 1065 let hbuf: *i64=sb[10] as *i64; let hmid: *i64=sb[11] as *i64; let cur: *i64=sb[12] as *i64 1066 cpy(cur, x1, ne) 1067 var L: i64=0 1068 while L<N { 1069 let wb: *i64 = wcache[L] as *i64 1070 let gA: *i64=wb[0] as *i64 1071 let Wq: *i32=wb[1] as *i32; let Wk: *i32=wb[2] as *i32; let Wv: *i32=wb[3] as *i32; let Wo: *i32=wb[4] as *i32 1072 let gF: *i64=wb[5] as *i64 1073 let Wg: *i32=wb[6] as *i32; let Wu: *i32=wb[7] as *i32; let Wd: *i32=wb[8] as *i32 1074 let bq: *i64=wb[9] as *i64; let bk: *i64=wb[10] as *i64; let bv: *i64=wb[11] as *i64 1075 rmsnorm_gamma_row_q24(cur, gA, 0, ne, xn, 0) 1076 mm_pool_i32(xn, Wq, Q, 1, ne, qd, 24) 1077 mm_pool_i32(xn, Wk, K, 1, ne, kvd, 24) 1078 mm_pool_i32(xn, Wv, V, 1, ne, kvd, 24) 1079 var bo: i64=0; while bo<qd { Q[bo]=Q[bo]+bq[bo]; bo=bo+1 } 1080 var bp: i64=0; while bp<kvd { K[bp]=K[bp]+bk[bp]; V[bp]=V[bp]+bv[bp]; bp=bp+1 } 1081 var h: i64=0; while h<nh { rope_apply(((Q as i64)+(h*hd)*8) as *i64, hd, pos, freqs); h=h+1 } 1082 var hk: i64=0; while hk<nkv { rope_apply(((K as i64)+(hk*hd)*8) as *i64, hd, pos, freqs); hk=hk+1 } 1083 let Kc: *i64 = kvc[2*L] as *i64 1084 let Vc: *i64 = kvc[2*L+1] as *i64 1085 var ci: i64=0; while ci<kvd { Kc[pos*kvd+ci]=K[ci]; Vc[pos*kvd+ci]=V[ci]; ci=ci+1 } 1086 let group: i64=nh/nkv 1087 let cnt: i64=pos+1 1088 var hh: i64=0 1089 while hh<nh { 1090 let kvh: i64=hh/group; let qb: i64=hh*hd; let kb: i64=kvh*hd 1091 var s: i64=0 1092 while s<cnt { var dot: i64=0; var d: i64=0; while d<hd { dot=dot+(Q[qb+d]*Kc[s*kvd+kb+d]); d=d+1 } sc[s]=qmul(dot>>16, scale); s=s+1 } 1093 var mmax: i64=sc[0]; var j: i64=1; while j<cnt { if sc[j]>mmax { mmax=sc[j] } j=j+1 } 1094 var sum: i64=0; j=0; while j<cnt { let e: i64=fx_exp(sc[j]-mmax); at[j]=e; sum=sum+e; j=j+1 } 1095 if sum<=0 { sum=1 } 1096 j=0; while j<cnt { at[j]=(at[j]<<16)/sum; j=j+1 } 1097 var d2: i64=0 1098 while d2<hd { var acc: i64=0; s=0; while s<cnt { acc=acc+(at[s]*Vc[s*kvd+kb+d2]); s=s+1 } concat[qb+d2]=acc>>16; d2=d2+1 } 1099 hh=hh+1 1100 } 1101 mm_pool_i32(concat, Wo, proj, 1, qd, ne, 16) 1102 var ri: i64=0; while ri<ne { hmid[ri]=cur[ri]+proj[ri]; ri=ri+1 } 1103 rmsnorm_gamma_row_q24(hmid, gF, 0, ne, xn, 0) 1104 mm_pool_i32(xn, Wg, gate, 1, ne, fd, 24) 1105 mm_pool_i32(xn, Wu, up, 1, ne, fd, 24) 1106 var fi: i64=0; while fi<fd { hbuf[fi]=qmul(silu(gate[fi]), up[fi]); fi=fi+1 } 1107 mm_pool_i32(hbuf, Wd, proj, 1, fd, ne, 16) 1108 var fo: i64=0; while fo<ne { cur[fo]=hmid[fo]+proj[fo]; fo=fo+1 } 1109 L=L+1 1110 } 1111 cpy(out1, cur, ne) 1112 return 0 1113} 1114// ---- i32 output-head cache (half the head bandwidth, lossless) ---- 1115func _nfhdq_i32_task(ctx_i: i64) -> i64 { 1116 let c: *i64 = ctx_i as *i64 1117 let buf: *u8=c[0] as *u8; let base: i64=c[1]; let ty: i64=c[2]; let cache: *i32=c[3] as *i32 1118 let ne: i64=c[4]; let lo: i64=c[5]; let hi: i64=c[6]; let tmp: *i64=c[7] as *i64; let row: *i64=c[8] as *i64 1119 var v: i64=lo 1120 while v<hi { dequant_row(buf, base, ty, v, ne, row, tmp); var k: i64=0; while k<ne { cache[v*ne+k]=(row[k]) as i32; k=k+1 } v=v+1 } 1121 return 0 1122} 1123func nf_dequant_head_all_i32(buf: *u8, oh_base: i64, oh_ty: i64, vocab: i64, ne: i64) -> *i32 { 1124 let p: *NxThreadPool = nf_pool() 1125 let cache: *i32 = sys_mmap(vocab*ne*4) as *i32 1126 if (cache as i64)==0 { return cache } 1127 var bands: i64 = p.n_workers 1128 if bands > 32 { bands = 32 } 1129 if bands > vocab { bands = vocab } 1130 if bands < 1 { bands = 1 } 1131 let per: i64 = (vocab + bands - 1) / bands 1132 // ROOT FIX 2026-07-15 (the run-to-run greedy divergence, proven by exact-repeat 1.5B runs): the old 1133 // shared arena gave each band a FIXED 8192B row (1024 i64). ne=1536 (Qwen2.5-1.5B) dequants 12288B -> 1134 // band b OVERFLOWED 4KB into band b+1's row MID-COPY -> timing-dependent corrupt cache rows, frozen 1135 // for the run (within-run determinism held; across runs diverged at near-tie logits). Per-call 1136 // DIMS-SIZED scratch: row stride ne*8, tmp stride ne*8+4096 (covers every quant type's block window 1137 // at any ne). Freed after the wait (all band tasks complete by then). 1138 let rstride: i64 = ne*8 1139 let tstride: i64 = ne*8 + 4096 1140 let rows: *u8 = sys_mmap(bands*rstride) 1141 let tmps: *u8 = sys_mmap(bands*tstride) 1142 let done0: i64 = nx_pool_n_completed(p) 1143 var b: i64 = 0 1144 while b < bands { 1145 let c: *i64 = ((g_nf_hdctx as i64) + b*128) as *i64 1146 c[0]=buf as i64; c[1]=oh_base; c[2]=oh_ty; c[3]=cache as i64; c[4]=ne; c[5]=b*per 1147 var hi: i64=(b+1)*per 1148 if hi>vocab { hi=vocab } 1149 c[6]=hi 1150 c[7]=((tmps as i64) + b*tstride) 1151 c[8]=((rows as i64) + b*rstride) 1152 nx_pool_submit(p, _nfhdq_i32_task, c as i64) 1153 b=b+1 1154 } 1155 nx_pool_wait(p, done0+bands) 1156 sys_munmap(rows, bands*rstride) 1157 sys_munmap(tmps, bands*tstride) 1158 return cache 1159} 1160func _nfhac_i32_task(ctx_i: i64) -> i64 { 1161 let c: *i64 = ctx_i as *i64 1162 let cache: *i32=c[0] as *i32; let normed: *i64=c[1] as *i64; let ne: i64=c[2]; let lo: i64=c[3]; let hi: i64=c[4]; let outp: *i64=c[5] as *i64 1163 var best: i64=lo 1164 var bestv: i64=0-9223372036854775807 1165 var v: i64=lo 1166 while v<hi { 1167 var s: i64=0; var k: i64=0 1168 while k<ne { s=s+(normed[k]*(cache[v*ne+k] as i64)); k=k+1 } // *i32 sign-extends natively (2026-07-10 sext fix) 1169 let lg: i64=s>>24 1170 if lg>bestv { bestv=lg; best=v } 1171 v=v+1 1172 } 1173 outp[0]=best; outp[1]=bestv 1174 return 0 1175} 1176// one full-logits band over the i32 head cache: writes lgv[v] = dot(normed, row_v) >> 24 for rows [lo,hi). 1177// Bands write DISJOINT ranges of lgv -> deterministic regardless of thread count (same discipline as the 1178// banded matmuls). Added 2026-07-10 for SAMPLING (temperature/top-p needs ALL logits, not just the argmax). 1179func _nfhlg_task(ctx_i: i64) -> i64 { 1180 let c: *i64 = ctx_i as *i64 1181 let cache: *i32=c[0] as *i32; let normed: *i64=c[1] as *i64; let ne: i64=c[2]; let lo: i64=c[3]; let hi: i64=c[4]; let lgv: *i64=c[5] as *i64 1182 var v: i64=lo 1183 while v<hi { 1184 var s: i64=0; var k: i64=0 1185 while k<ne { s=s+(normed[k]*(cache[v*ne+k] as i64)); k=k+1 } 1186 lgv[v]=s>>24 1187 v=v+1 1188 } 1189 return 0 1190} 1191// pooled FULL-logits head over the i32 cache. hlp = [cache(i32), normed(Q24), vocab, ne, lgv(*i64 vocab)]. 1192// Same dots as head_argmax_cached_i32 (Q16 head convention), all of them kept. 1193func head_logits_cached_i32(hlp: *i64) -> i64 { 1194 let p: *NxThreadPool = nf_pool() 1195 let vocab: i64=hlp[2] 1196 var bands: i64 = p.n_workers 1197 if bands > 32 { bands = 32 } 1198 if bands > vocab { bands = vocab } 1199 if bands < 1 { bands = 1 } 1200 let per: i64 = (vocab + bands - 1) / bands 1201 let done0: i64 = nx_pool_n_completed(p) 1202 var b: i64 = 0 1203 while b < bands { 1204 let c: *i64 = ((g_nf_hdctx as i64) + b*128) as *i64 1205 c[0]=hlp[0]; c[1]=hlp[1]; c[2]=hlp[3]; c[3]=b*per 1206 var hi: i64=(b+1)*per 1207 if hi>vocab { hi=vocab } 1208 c[4]=hi 1209 c[5]=hlp[4] 1210 nx_pool_submit(p, _nfhlg_task, c as i64) 1211 b=b+1 1212 } 1213 nx_pool_wait(p, done0+bands) 1214 return 0 1215} 1216// argmax over the i32 head cache. hcp = [cache(i32), normed(Q24), vocab, ne, idout, lgout]. Bit-identical. 1217func head_argmax_cached_i32(hcp: *i64) -> i64 { 1218 let p: *NxThreadPool = nf_pool() 1219 let cache: i64=hcp[0]; let normed: i64=hcp[1]; let vocab: i64=hcp[2]; let ne: i64=hcp[3] 1220 var bands: i64 = p.n_workers 1221 if bands > 32 { bands = 32 } 1222 if bands > vocab { bands = vocab } 1223 if bands < 1 { bands = 1 } 1224 let per: i64 = (vocab + bands - 1) / bands 1225 let done0: i64 = nx_pool_n_completed(p) 1226 var b: i64 = 0 1227 while b < bands { 1228 let c: *i64 = ((g_nf_hdctx as i64) + b*128) as *i64 1229 c[0]=cache; c[1]=normed; c[2]=ne; c[3]=b*per 1230 var hi: i64=(b+1)*per 1231 if hi>vocab { hi=vocab } 1232 c[4]=hi 1233 c[5]=((g_nf_hdout as i64) + b*16) 1234 nx_pool_submit(p, _nfhac_i32_task, c as i64) 1235 b=b+1 1236 } 1237 nx_pool_wait(p, done0+bands) 1238 var best: i64=0 1239 var bestv: i64=0-9223372036854775807 1240 var rb: i64=0 1241 while rb<bands { 1242 let o: *i64=((g_nf_hdout as i64) + rb*16) as *i64 1243 if o[1]>bestv { bestv=o[1]; best=o[0] } 1244 rb=rb+1 1245 } 1246 let oid: *i64=hcp[4] as *i64 1247 let olg: *i64=hcp[5] as *i64 1248 oid[0]=best; olg[0]=bestv 1249 return best 1250} 1251// ==== W8A8 SIMD DECODE PATH (Stage 2, 2026-07-09) ======================================================== 1252// PROBE-PROVEN (nx_nofloat_simd_dot_probe): __i16x16_madd (vpmaddwd) gives a 13x matmul over the scalar i64 1253// dot, at ~0.8% per-dot quantization error. LOSSY fast path (NOT bit-exact) -> ADDITIVE alongside the lossless 1254// i32 hero; the gate MEASURES whether real tokens survive. i16 lanes are *u8 buffers with manual LE nf_pack2 1255// (nx_cc has no i16 type). int8-range values keep the int32 madd accumulator overflow-safe for our K (<=304 1256// madds * ~32k < 2.1e9). Weight quant is amortized ONCE at cache build; only the activation is quantized per matmul. 1257// 1258// W12A12 (2026-09-16, search R0l attribution): the 8-bit per-TENSOR activation scale was the engine's numeric floor -- 1259// measured on BRIGHT biology the sovereign i8 rewrites scored 330 permil against 408 for the same weights through 1260// llama.cpp, and the two engines' greedy paths flip a near-tie at token twelve and never reconverge. The i16 lanes 1261// already carry the values, so the bits were free: weights and activations now quantise to +-NF_W_QMAX / +-NF_X_QMAX 1262// (16x finer each), rounded to nearest instead of truncated, and the int32 lane accumulation is folded into an i64 1263// total every NF_CHUNK_K elements so no lane can overflow. NF_CHUNK_K IS DERIVED, NOT CHOSEN: one vpmaddwd lane sums 1264// NF_MADD_PER_LANE products per 16 elements, so a chunk of C elements puts C/8 products in a lane, each at most 1265// NF_X_QMAX*NF_W_QMAX; C/8 * 4095 * 4095 <= NF_I32_ACC_MAX gives C <= 1024.5, and 1024 is the largest multiple of 1266// 16 below it (nx_nofloat_w12a12_gate pins the arithmetic and plants 2048 as the bound-breaking neg-control). 1267const NF_W_QMAX: i64 = 4095 1268const NF_X_QMAX: i64 = 4095 1269const NF_I32_ACC_MAX: i64 = 2147483647 1270const NF_MADD_PER_LANE: i64 = 8 1271const NF_CHUNK_K: i64 = 1024 1272func nf_pack2(buf: *u8, idx: i64, val: i64) -> i64 { buf[idx*2]=(val) as u8; buf[idx*2+1]=(val>>8) as u8; return 0 } 1273// round-to-nearest division clamped to +-qmax: a truncating divide biases every value toward zero, and one bias per 1274// element over an 8960-wide row is a systematic error the dot product cannot average away 1275func nf_qround(v: i64, s: i64, qmax: i64) -> i64 { 1276 var q: i64 = 0 1277 if v >= 0 { q = (v + s/2)/s } else { q = 0 - ((0 - v + s/2)/s) } 1278 if q > qmax { q = qmax } 1279 if q < 0 - qmax { q = 0 - qmax } 1280 return q 1281} 1282func nf_quant_w_i8(Wsrc: *i64, Wi8: *u8, sw: *i64, out_dim: i64, in_dim: i64) -> i64 { 1283 var o: i64=0 1284 while o<out_dim { 1285 var rm: i64=0; var k: i64=0 1286 while k<in_dim { var a: i64=Wsrc[o*in_dim+k]; if a<0 { a=0-a } if a>rm { rm=a } k=k+1 } 1287 var s: i64=rm/NF_W_QMAX; if s<1 { s=1 } 1288 sw[o]=s 1289 let rowb: i64=(Wi8 as i64)+o*in_dim*2 1290 k=0; while k<in_dim { nf_pack2(rowb as *u8, k, nf_qround(Wsrc[o*in_dim+k], s, NF_W_QMAX)); k=k+1 } 1291 o=o+1 1292 } 1293 return 0 1294} 1295func nf_alloc_layer_wb_i8(ne: i64, qd: i64, kvd: i64, fd: i64) -> *i64 { 1296 let w: *i64 = sys_mmap(19*8) as *i64 1297 w[0]=sys_mmap(ne*8) as i64 1298 w[1]=sys_mmap(qd*ne*2) as i64; w[2]=sys_mmap(qd*8) as i64 1299 w[3]=sys_mmap(kvd*ne*2) as i64; w[4]=sys_mmap(kvd*8) as i64 1300 w[5]=sys_mmap(kvd*ne*2) as i64; w[6]=sys_mmap(kvd*8) as i64 1301 w[7]=sys_mmap(ne*qd*2) as i64; w[8]=sys_mmap(ne*8) as i64 1302 w[9]=sys_mmap(ne*8) as i64 1303 w[10]=sys_mmap(ne*fd*2) as i64; w[11]=sys_mmap(fd*8) as i64 1304 w[12]=sys_mmap(ne*fd*2) as i64; w[13]=sys_mmap(fd*8) as i64 1305 w[14]=sys_mmap(fd*ne*2) as i64; w[15]=sys_mmap(ne*8) as i64 1306 w[16]=sys_mmap(qd*8) as i64; w[17]=sys_mmap(kvd*8) as i64; w[18]=sys_mmap(kvd*8) as i64 1307 return w 1308} 1309func nf_dequant_all_layers_i8(buf: *u8, hdr: *NxGgufHeader, wc8: *i64, N: i64, ne: i64, qd: i64, kvd: i64, fd: i64) -> i64 { 1310 let tmp: *i64 = nf_alloc_layer_wb(ne, qd, kvd, fd) 1311 var L: i64=0 1312 while L<N { 1313 nf_load_layer_pool(buf, hdr, L, tmp, ne, qd, kvd, fd) 1314 let s: *i64 = nf_alloc_layer_wb_i8(ne, qd, kvd, fd) 1315 cpy(s[0] as *i64, tmp[0] as *i64, ne) 1316 nf_quant_w_i8(tmp[1] as *i64, s[1] as *u8, s[2] as *i64, qd, ne) 1317 nf_quant_w_i8(tmp[2] as *i64, s[3] as *u8, s[4] as *i64, kvd, ne) 1318 nf_quant_w_i8(tmp[3] as *i64, s[5] as *u8, s[6] as *i64, kvd, ne) 1319 nf_quant_w_i8(tmp[4] as *i64, s[7] as *u8, s[8] as *i64, ne, qd) 1320 cpy(s[9] as *i64, tmp[5] as *i64, ne) 1321 nf_quant_w_i8(tmp[6] as *i64, s[10] as *u8, s[11] as *i64, fd, ne) 1322 nf_quant_w_i8(tmp[7] as *i64, s[12] as *u8, s[13] as *i64, fd, ne) 1323 nf_quant_w_i8(tmp[8] as *i64, s[14] as *u8, s[15] as *i64, ne, fd) 1324 cpy(s[16] as *i64, tmp[9] as *i64, qd) 1325 cpy(s[17] as *i64, tmp[10] as *i64, kvd) 1326 cpy(s[18] as *i64, tmp[11] as *i64, kvd) 1327 wc8[L]=s as i64 1328 L=L+1 1329 } 1330 return 0 1331} 1332func _nfmm_i8_task(ctx_i: i64) -> i64 { 1333 let c: *i64 = ctx_i as *i64 1334 let xi8: i64=c[0]; let sx: i64=c[1]; let Wi8: i64=c[2]; let sw: *i64=c[3] as *i64; let dst: *i64=c[4] as *i64 1335 let ind: i64=c[5]; let lo: i64=c[6]; let hi: i64=c[7]; let sh: i64=c[8]; let acc: *u8=c[9] as *u8 1336 var o: i64=lo 1337 while o<hi { 1338 let wb: i64 = Wi8 + o*ind*2 1339 // chunked accumulation: the int32 lanes hold at most NF_CHUNK_K/8 products each, then fold into i64 1340 var total: i64 = 0 1341 var k0: i64 = 0 1342 while k0 < ind { 1343 let z: *i64 = acc as *i64; z[0]=0; z[1]=0; z[2]=0; z[3]=0 1344 var k1: i64 = k0 + NF_CHUNK_K 1345 if k1 > ind { k1 = ind } 1346 var k: i64 = k0 1347 while k<k1 { __i16x16_madd(acc, (xi8+k*2) as *u8, (wb+k*2) as *u8); k=k+16 } 1348 total = total + nf_hsum_sx(acc) 1349 k0 = k1 1350 } 1351 dst[o]=((sx*sw[o])*total)>>sh 1352 o=o+1 1353 } 1354 return 0 1355} 1356func _nfmm_i8_task4(ctx_i: i64) -> i64 { 1357 let c: *i64 = ctx_i as *i64 1358 let xi8: i64=c[0]; let sx: i64=c[1]; let Wi8: i64=c[2]; let sw: *i64=c[3] as *i64; let dst: *i64=c[4] as *i64 1359 let ind: i64=c[5]; let lo: i64=c[6]; let hi: i64=c[7]; let sh: i64=c[8]; let acc: *u8=c[9] as *u8 1360 let a1: *u8 = ((acc as i64) + NF_ACC_BYTES) as *u8 1361 let a2: *u8 = ((acc as i64) + 2*NF_ACC_BYTES) as *u8 1362 let a3: *u8 = ((acc as i64) + 3*NF_ACC_BYTES) as *u8 1363 var o: i64=lo 1364 while o<hi { 1365 let wb: i64 = Wi8 + o*ind*2 1366 var total: i64 = 0 1367 var k0: i64 = 0 1368 while k0 < ind { 1369 let z: *i64 = acc as *i64 1370 var zi: i64 = 0 1371 while zi < NF_ACC4_I64 { z[zi] = 0; zi = zi + 1 } 1372 var k1: i64 = k0 + NF_CHUNK_K 1373 if k1 > ind { k1 = ind } 1374 var k: i64 = k0 1375 while k + NF_R4_STRIDE <= k1 { 1376 __i16x16_madd(acc, (xi8+k*2) as *u8, (wb+k*2) as *u8) 1377 __i16x16_madd(a1, (xi8+(k+NF_MADD_W)*2) as *u8, (wb+(k+NF_MADD_W)*2) as *u8) 1378 __i16x16_madd(a2, (xi8+(k+2*NF_MADD_W)*2) as *u8, (wb+(k+2*NF_MADD_W)*2) as *u8) 1379 __i16x16_madd(a3, (xi8+(k+3*NF_MADD_W)*2) as *u8, (wb+(k+3*NF_MADD_W)*2) as *u8) 1380 k = k + NF_R4_STRIDE 1381 } 1382 while k<k1 { __i16x16_madd(acc, (xi8+k*2) as *u8, (wb+k*2) as *u8); k=k+NF_MADD_W } 1383 total = total + nf_hsum_sx(acc) + nf_hsum_sx(a1) + nf_hsum_sx(a2) + nf_hsum_sx(a3) 1384 k0 = k1 1385 } 1386 dst[o]=((sx*sw[o])*total)>>sh 1387 o=o+1 1388 } 1389 return 0 1390} 1391// pooled W8A8 SIMD matmul (T=1 decode). Quantizes the single activation row once, bands the output rows. 1392// Banded by whole output row -> thread-count-invariant. in_dim MUST be a multiple of 16 (896/128/4864 all are). 1393func mm_pool_i8(x: *i64, Wi8: *u8, sw: *i64, dst: *i64, in_dim: i64, out_dim: i64, shift: i64) -> i64 { 1394 let p: *NxThreadPool = nf_pool() 1395 var xmx: i64=0; var k: i64=0 1396 while k<in_dim { var a: i64=x[k]; if a<0 { a=0-a } if a>xmx { xmx=a } k=k+1 } 1397 var sx: i64=xmx/NF_X_QMAX; if sx<1 { sx=1 } 1398 // ROOT-FIX CLASS 2026-07-15: g_nf_xi8 was a FIXED 4864x2 arena (the 0.5B's ffn) -- in_dim=8960 1399 // (Qwen2.5-1.5B ffn) would overflow it. Per-call dims-sized; freed after the wait. 1400 let xi8: *u8 = sys_mmap(in_dim*2) 1401 k=0; while k<in_dim { nf_pack2(xi8, k, nf_qround(x[k], sx, NF_X_QMAX)); k=k+1 } 1402 var bands: i64 = p.n_workers 1403 if bands > 32 { bands = 32 } 1404 if bands > out_dim { bands = out_dim } 1405 if bands < 1 { bands = 1 } 1406 let per: i64 = (out_dim + bands - 1) / bands 1407 let done0: i64 = nx_pool_n_completed(p) 1408 var b: i64=0 1409 while b<bands { 1410 let c: *i64 = ((g_nf_mmarena as i64) + b*128) as *i64 1411 c[0]=xi8 as i64; c[1]=sx; c[2]=Wi8 as i64; c[3]=sw as i64; c[4]=dst as i64; c[5]=in_dim; c[6]=b*per 1412 var hi: i64=(b+1)*per; if hi>out_dim { hi=out_dim } 1413 c[7]=hi; c[8]=shift 1414 if g_nf_i8_r4 == 1 { c[9]=((g_nf_i8acc4 as i64)+b*128); nx_pool_submit(p, _nfmm_i8_task4, c as i64) } 1415 else { c[9]=((g_nf_i8acc as i64)+b*64); nx_pool_submit(p, _nfmm_i8_task, c as i64) } 1416 b=b+1 1417 } 1418 nx_pool_wait(p, done0+bands) 1419 sys_munmap(xi8, in_dim*2) 1420 return 0 1421} 1422// R0r: M activation rows against one weight matrix, one weight pass. See the NF_PREFILL_M note for the loop order. 1423func _nfmm_i8_task_m(ctx_i: i64) -> i64 { 1424 let c: *i64 = ctx_i as *i64 1425 let xi8: i64=c[0]; let sxm: *i64=c[1] as *i64; let Wi8: i64=c[2]; let sw: *i64=c[3] as *i64; let dst: *i64=c[4] as *i64 1426 let ind: i64=c[5]; let lo: i64=c[6]; let hi: i64=c[7]; let sh: i64=c[8]; let acc: *u8=c[9] as *u8 1427 let M: i64=c[NF_MMCTX_M]; let od: i64=c[NF_MMCTX_OUT] 1428 var o: i64=lo 1429 while o<hi { 1430 let wb: i64 = Wi8 + o*ind*2 1431 var m: i64 = 0 1432 while m<M { 1433 let xr: i64 = xi8 + m*ind*2 1434 var total: i64 = 0 1435 var k0: i64 = 0 1436 while k0 < ind { 1437 var k1: i64 = k0 + NF_CHUNK_K 1438 if k1 > ind { k1 = ind } 1439 if g_nf_i8_dot == 1 { 1440 // R0r-b: one builtin per chunk, the accumulator register-resident across it. The chunk is a positive multiple 1441 // of NF_MADD_W: in_dim is one by the madd loop's own contract and NF_CHUNK_K is 64 of them. 1442 total = total + __i16_dot((xr+k0*2) as *u8, (wb+k0*2) as *u8, k1 - k0) 1443 } else { 1444 let z: *i64 = acc as *i64; z[0]=0; z[1]=0; z[2]=0; z[3]=0 1445 var k: i64 = k0 1446 while k<k1 { __i16x16_madd(acc, (xr+k*2) as *u8, (wb+k*2) as *u8); k=k+NF_MADD_W } 1447 total = total + nf_hsum_sx(acc) 1448 } 1449 k0 = k1 1450 } 1451 dst[m*od+o]=((sxm[m]*sw[o])*total)>>sh 1452 m=m+1 1453 } 1454 o=o+1 1455 } 1456 return 0 1457} 1458// R0r-c: the batched task with four k-interleaved accumulators (R0q's loop), one weight pass per block; every cell is the 1459// same integer sum as _nfmm_i8_task_m in a different association order, so it is bit-identical (the gate's T8e). 1460func _nfmm_i8_task_m4(ctx_i: i64) -> i64 { 1461 let c: *i64 = ctx_i as *i64 1462 let xi8: i64=c[0]; let sxm: *i64=c[1] as *i64; let Wi8: i64=c[2]; let sw: *i64=c[3] as *i64; let dst: *i64=c[4] as *i64 1463 let ind: i64=c[5]; let lo: i64=c[6]; let hi: i64=c[7]; let sh: i64=c[8]; let acc: *u8=c[9] as *u8 1464 let M: i64=c[NF_MMCTX_M]; let od: i64=c[NF_MMCTX_OUT] 1465 let a1: *u8 = ((acc as i64) + NF_ACC_BYTES) as *u8 1466 let a2: *u8 = ((acc as i64) + 2*NF_ACC_BYTES) as *u8 1467 let a3: *u8 = ((acc as i64) + 3*NF_ACC_BYTES) as *u8 1468 var o: i64=lo 1469 while o<hi { 1470 let wb: i64 = Wi8 + o*ind*2 1471 var m: i64 = 0 1472 while m<M { 1473 let xr: i64 = xi8 + m*ind*2 1474 var total: i64 = 0 1475 var k0: i64 = 0 1476 while k0 < ind { 1477 let z: *i64 = acc as *i64 1478 var zi: i64 = 0 1479 while zi < NF_ACC4_I64 { z[zi] = 0; zi = zi + 1 } 1480 var k1: i64 = k0 + NF_CHUNK_K 1481 if k1 > ind { k1 = ind } 1482 var k: i64 = k0 1483 while k + NF_R4_STRIDE <= k1 { 1484 __i16x16_madd(acc, (xr+k*2) as *u8, (wb+k*2) as *u8) 1485 __i16x16_madd(a1, (xr+(k+NF_MADD_W)*2) as *u8, (wb+(k+NF_MADD_W)*2) as *u8) 1486 __i16x16_madd(a2, (xr+(k+2*NF_MADD_W)*2) as *u8, (wb+(k+2*NF_MADD_W)*2) as *u8) 1487 __i16x16_madd(a3, (xr+(k+3*NF_MADD_W)*2) as *u8, (wb+(k+3*NF_MADD_W)*2) as *u8) 1488 k = k + NF_R4_STRIDE 1489 } 1490 while k<k1 { __i16x16_madd(acc, (xr+k*2) as *u8, (wb+k*2) as *u8); k=k+NF_MADD_W } 1491 total = total + nf_hsum_sx(acc) + nf_hsum_sx(a1) + nf_hsum_sx(a2) + nf_hsum_sx(a3) 1492 k0 = k1 1493 } 1494 dst[m*od+o]=((sxm[m]*sw[o])*total)>>sh 1495 m=m+1 1496 } 1497 o=o+1 1498 } 1499 return 0 1500} 1501// R0r: the batched twin of mm_pool_i8. x holds M rows of in_dim; dst receives M rows of out_dim. Each row is quantised 1502// exactly as mm_pool_i8 quantises its single row, then the bands stream the weights once per call. 1503func mm_pool_i8_m(x: *i64, M: i64, Wi8: *u8, sw: *i64, dst: *i64, in_dim: i64, out_dim: i64, shift: i64) -> i64 { 1504 nf_quant_rows_pool(x, M, in_dim) 1505 return mm_pool_i8_mq(M, Wi8, sw, dst, in_dim, out_dim, shift) 1506} 1507// R0s: ONE row quantiser, called by the serial control and by every pool task -- the arithmetic cannot differ. 1508func nf_quant_row(xr: *i64, in_dim: i64, xq: *u8, sxm: *i64, m: i64) -> i64 { 1509 var xmx: i64=0; var k: i64=0 1510 while k<in_dim { var a: i64=xr[k]; if a<0 { a=0-a } if a>xmx { xmx=a } k=k+1 } 1511 var sx: i64=xmx/NF_X_QMAX; if sx<1 { sx=1 } 1512 sxm[m]=sx 1513 k=0; while k<in_dim { nf_pack2(xq, k, nf_qround(xr[k], sx, NF_X_QMAX)); k=k+1 } 1514 return 0 1515} 1516func _nf_quant_rows_task(ctx_i: i64) -> i64 { 1517 let c: *i64 = ctx_i as *i64 1518 let x: i64=c[0]; let xi8: i64=c[1]; let sxm: *i64=c[2] as *i64; let ind: i64=c[3]; var m: i64=c[4]; let hi: i64=c[5] 1519 while m<hi { nf_quant_row((x + m*ind*8) as *i64, ind, (xi8 + m*ind*2) as *u8, sxm, m); m=m+1 } 1520 return 0 1521} 1522// R0s: quantise M activation rows of in_dim Q16 values into g_nf_xi8m / g_nf_sxm (grown to fit), one task per row 1523// up to the arena's slots, or serially under the gate's control switch. Stamps g_nf_mm_us_quant. 1524func nf_quant_rows_pool(x: *i64, M: i64, in_dim: i64) -> i64 { 1525 let need: i64 = M*in_dim*2 1526 if need > g_nf_xi8m_cap { if g_nf_xi8m_cap > 0 { sys_munmap(g_nf_xi8m, g_nf_xi8m_cap) } g_nf_xi8m = sys_mmap(need); g_nf_xi8m_cap = need } 1527 let needs: i64 = M*8 1528 if needs > g_nf_sxm_cap { if g_nf_sxm_cap > 0 { sys_munmap(g_nf_sxm as *u8, g_nf_sxm_cap) } g_nf_sxm = sys_mmap(needs) as *i64; g_nf_sxm_cap = needs } 1529 let tq0: i64 = sys_now_us() 1530 if g_nf_quant_serial == 1 { 1531 var m: i64=0 1532 while m<M { nf_quant_row(((x as i64) + m*in_dim*8) as *i64, in_dim, ((g_nf_xi8m as i64) + m*in_dim*2) as *u8, g_nf_sxm, m); m=m+1 } 1533 } else { 1534 let p: *NxThreadPool = nf_pool() 1535 var tasks: i64 = M 1536 if tasks > NF_MMARENA_SLOTS { tasks = NF_MMARENA_SLOTS } 1537 if tasks < 1 { tasks = 1 } 1538 let per: i64 = (M + tasks - 1) / tasks 1539 let done0: i64 = nx_pool_n_completed(p) 1540 var b: i64=0 1541 while b<tasks { 1542 let c: *i64 = ((g_nf_mmarena as i64) + b*128) as *i64 1543 c[0]=x as i64; c[1]=g_nf_xi8m as i64; c[2]=g_nf_sxm as i64; c[3]=in_dim; c[4]=b*per 1544 var hi: i64=(b+1)*per; if hi>M { hi=M } 1545 c[5]=hi 1546 nx_pool_submit(p, _nf_quant_rows_task, c as i64) 1547 b=b+1 1548 } 1549 nx_pool_wait(p, done0+tasks) 1550 } 1551 g_nf_mm_us_quant = sys_now_us() - tq0 1552 return 0 1553} 1554// R0s: the matmul over rows ALREADY quantised into g_nf_xi8m / g_nf_sxm by nf_quant_rows_pool (the engine quantises a 1555// layer input once and calls this for every projection that shares it). Stamps g_nf_mm_us_pool. 1556func mm_pool_i8_mq(M: i64, Wi8: *u8, sw: *i64, dst: *i64, in_dim: i64, out_dim: i64, shift: i64) -> i64 { 1557 let p: *NxThreadPool = nf_pool() 1558 let xi8: *u8 = g_nf_xi8m 1559 let sxm: *i64 = g_nf_sxm 1560 let tq1: i64 = sys_now_us() 1561 var bands: i64 = p.n_workers 1562 if bands > 32 { bands = 32 } 1563 if bands > out_dim { bands = out_dim } 1564 if bands < 1 { bands = 1 } 1565 let per: i64 = (out_dim + bands - 1) / bands 1566 let done0: i64 = nx_pool_n_completed(p) 1567 var b: i64=0 1568 while b<bands { 1569 let c: *i64 = ((g_nf_mmarena as i64) + b*128) as *i64 1570 c[0]=xi8 as i64; c[1]=sxm as i64; c[2]=Wi8 as i64; c[3]=sw as i64; c[4]=dst as i64; c[5]=in_dim; c[6]=b*per 1571 var hi: i64=(b+1)*per; if hi>out_dim { hi=out_dim } 1572 c[7]=hi; c[8]=shift; c[NF_MMCTX_M]=M; c[NF_MMCTX_OUT]=out_dim 1573 if g_nf_i8_r4m == 1 { c[9]=((g_nf_i8acc4 as i64)+b*128); nx_pool_submit(p, _nfmm_i8_task_m4, c as i64) } 1574 else { c[9]=((g_nf_i8acc as i64)+b*64); nx_pool_submit(p, _nfmm_i8_task_m, c as i64) } 1575 b=b+1 1576 } 1577 nx_pool_wait(p, done0+bands) 1578 g_nf_mm_us_pool = sys_now_us() - tq1 1579 return 0 1580} 1581// R0r: T prompt positions [pos0, pos0+T) through the W8A8 cache with ONE pass per weight matrix per block. Row t's norm, 1582// bias, rope and attention are the per-token step's own calls at pos0+t, in order (row t attends the cache rows the 1583// earlier rows just wrote), so the cache rows and the last row's state are bit-identical to T calls of 1584// decode_step_kv_cached_i8; that sequential path stays as nx_nofloat_prefill_gate's control. xT holds T rows of ne 1585// (the dequantised embeddings); out1 receives the last row's final hidden state, exactly what the per-token step 1586// leaves in out1 for the last position. The scratch bundle is already NSV_MAXT rows deep in every slot. 1587func nf_prefill_batched(buf: *u8, hdr: *NxGgufHeader, xT: *i64, out1: *i64, wc8: *i64, sb: *i64, nm: *u8, freqs: *i64, kvc: *i64, pos0: i64, T: i64, cfgA: *i64, cfgF: *i64, N: i64) -> i64 { 1588 let ne: i64=cfgA[1]; let nh: i64=cfgA[2]; let nkv: i64=cfgA[3]; let hd: i64=cfgA[4]; let qd: i64=cfgA[5]; let kvd: i64=cfgA[6]; let fd: i64=cfgF[2] 1589 let xn: *i64=sb[0] as *i64; let Q: *i64=sb[1] as *i64; let K: *i64=sb[2] as *i64; let V: *i64=sb[3] as *i64; let concat: *i64=sb[4] as *i64 1590 let sc: *i64=sb[5] as *i64; let at: *i64=sb[6] as *i64; let proj: *i64=sb[7] as *i64; let gate: *i64=sb[8] as *i64; let up: *i64=sb[9] as *i64 1591 let hbuf: *i64=sb[10] as *i64; let hmid: *i64=sb[11] as *i64; let cur: *i64=sb[12] as *i64 1592 if T < 1 { return 0 } 1593 cpy(cur, xT, T*ne) 1594 g_nf_pf_us_norm=0; g_nf_pf_us_qkv=0; g_nf_pf_us_attn=0; g_nf_pf_us_o=0; g_nf_pf_us_gu=0; g_nf_pf_us_silu=0; g_nf_pf_us_down=0; g_nf_pf_us_quant=0 1595 var L: i64=0 1596 while L<N { 1597 let wb: *i64 = wc8[L] as *i64 1598 let gA: *i64=wb[0] as *i64 1599 let Wq: *u8=wb[1] as *u8; let swq: *i64=wb[2] as *i64 1600 let Wk: *u8=wb[3] as *u8; let swk: *i64=wb[4] as *i64 1601 let Wv: *u8=wb[5] as *u8; let swv: *i64=wb[6] as *i64 1602 let Wo: *u8=wb[7] as *u8; let swo: *i64=wb[8] as *i64 1603 let gF: *i64=wb[9] as *i64 1604 let Wg: *u8=wb[10] as *u8; let swg: *i64=wb[11] as *i64 1605 let Wu: *u8=wb[12] as *u8; let swu: *i64=wb[13] as *i64 1606 let Wd: *u8=wb[14] as *u8; let swd: *i64=wb[15] as *i64 1607 let bq: *i64=wb[16] as *i64; let bk: *i64=wb[17] as *i64; let bv: *i64=wb[18] as *i64 1608 let tp0: i64 = sys_now_us() 1609 var t: i64=0 1610 while t<T { rmsnorm_gamma_row_q24(cur, gA, t*ne, ne, xn, t*ne); t=t+1 } 1611 let tp1: i64 = sys_now_us() 1612 g_nf_pf_us_norm = g_nf_pf_us_norm + (tp1 - tp0) 1613 // R0s: the normed rows are quantised ONCE and shared by q, k and v 1614 nf_quant_rows_pool(xn, T, ne) 1615 g_nf_pf_us_quant = g_nf_pf_us_quant + g_nf_mm_us_quant 1616 mm_pool_i8_mq(T, Wq, swq, Q, ne, qd, 24) 1617 mm_pool_i8_mq(T, Wk, swk, K, ne, kvd, 24) 1618 mm_pool_i8_mq(T, Wv, swv, V, ne, kvd, 24) 1619 let tp2: i64 = sys_now_us() 1620 g_nf_pf_us_qkv = g_nf_pf_us_qkv + (tp2 - tp1) 1621 t=0 1622 while t<T { 1623 let Qt: *i64 = ((Q as i64)+t*qd*8) as *i64 1624 let Kt: *i64 = ((K as i64)+t*kvd*8) as *i64 1625 let Vt: *i64 = ((V as i64)+t*kvd*8) as *i64 1626 var bo: i64=0; while bo<qd { Qt[bo]=Qt[bo]+bq[bo]; bo=bo+1 } 1627 var bp: i64=0; while bp<kvd { Kt[bp]=Kt[bp]+bk[bp]; Vt[bp]=Vt[bp]+bv[bp]; bp=bp+1 } 1628 var h: i64=0; while h<nh { rope_apply(((Qt as i64)+(h*hd)*8) as *i64, hd, pos0+t, freqs); h=h+1 } 1629 var hk: i64=0; while hk<nkv { rope_apply(((Kt as i64)+(hk*hd)*8) as *i64, hd, pos0+t, freqs); hk=hk+1 } 1630 t=t+1 1631 } 1632 // R0v: the block's rows are biased and rotated above; their attention runs as (row, head) pairs in the pool 1633 nf_attn_kv_block(Q, K, V, kvc, L, pos0, T, cfgA, sc, at, concat) 1634 let tp3: i64 = sys_now_us() 1635 g_nf_pf_us_attn = g_nf_pf_us_attn + (tp3 - tp2) 1636 mm_pool_i8_m(concat, T, Wo, swo, proj, qd, ne, 16) 1637 g_nf_pf_us_quant = g_nf_pf_us_quant + g_nf_mm_us_quant 1638 var ri: i64=0; while ri<T*ne { hmid[ri]=cur[ri]+proj[ri]; ri=ri+1 } 1639 let tp4: i64 = sys_now_us() 1640 g_nf_pf_us_o = g_nf_pf_us_o + (tp4 - tp3) 1641 t=0 1642 while t<T { rmsnorm_gamma_row_q24(hmid, gF, t*ne, ne, xn, t*ne); t=t+1 } 1643 let tp5: i64 = sys_now_us() 1644 g_nf_pf_us_norm = g_nf_pf_us_norm + (tp5 - tp4) 1645 // R0s: quantised once, shared by gate and up 1646 nf_quant_rows_pool(xn, T, ne) 1647 g_nf_pf_us_quant = g_nf_pf_us_quant + g_nf_mm_us_quant 1648 mm_pool_i8_mq(T, Wg, swg, gate, ne, fd, 24) 1649 mm_pool_i8_mq(T, Wu, swu, up, ne, fd, 24) 1650 let tp6: i64 = sys_now_us() 1651 g_nf_pf_us_gu = g_nf_pf_us_gu + (tp6 - tp5) 1652 nf_silu_rows_pool(gate, up, hbuf, T, fd) 1653 let tp7: i64 = sys_now_us() 1654 g_nf_pf_us_silu = g_nf_pf_us_silu + (tp7 - tp6) 1655 mm_pool_i8_m(hbuf, T, Wd, swd, proj, fd, ne, 16) 1656 g_nf_pf_us_quant = g_nf_pf_us_quant + g_nf_mm_us_quant 1657 var fo: i64=0; while fo<T*ne { cur[fo]=hmid[fo]+proj[fo]; fo=fo+1 } 1658 g_nf_pf_us_down = g_nf_pf_us_down + (sys_now_us() - tp7) 1659 L=L+1 1660 } 1661 cpy(out1, ((cur as i64)+(T-1)*ne*8) as *i64, ne) 1662 return 0 1663} 1664// decode ONE token via the W8A8 SIMD weight cache. Same structure as decode_step_kv_cached; the 7 projection 1665// matmuls go through mm_pool_i8 (13x faster, ~0.8% lossy). Attention/softmax/residual stay exact i64. 1666func decode_step_kv_cached_i8(buf: *u8, hdr: *NxGgufHeader, x1: *i64, out1: *i64, wc8: *i64, sb: *i64, nm: *u8, freqs: *i64, kvc: *i64, pos: i64, cfgA: *i64, cfgF: *i64, N: i64) -> i64 { 1667 let ne: i64=cfgA[1]; let nh: i64=cfgA[2]; let nkv: i64=cfgA[3]; let hd: i64=cfgA[4]; let qd: i64=cfgA[5]; let kvd: i64=cfgA[6]; let scale: i64=cfgA[7]; let fd: i64=cfgF[2] 1668 let xn: *i64=sb[0] as *i64; let Q: *i64=sb[1] as *i64; let K: *i64=sb[2] as *i64; let V: *i64=sb[3] as *i64; let concat: *i64=sb[4] as *i64 1669 let sc: *i64=sb[5] as *i64; let at: *i64=sb[6] as *i64; let proj: *i64=sb[7] as *i64; let gate: *i64=sb[8] as *i64; let up: *i64=sb[9] as *i64 1670 let hbuf: *i64=sb[10] as *i64; let hmid: *i64=sb[11] as *i64; let cur: *i64=sb[12] as *i64 1671 cpy(cur, x1, ne) 1672 var L: i64=0 1673 while L<N { 1674 let wb: *i64 = wc8[L] as *i64 1675 let gA: *i64=wb[0] as *i64 1676 let Wq: *u8=wb[1] as *u8; let swq: *i64=wb[2] as *i64 1677 let Wk: *u8=wb[3] as *u8; let swk: *i64=wb[4] as *i64 1678 let Wv: *u8=wb[5] as *u8; let swv: *i64=wb[6] as *i64 1679 let Wo: *u8=wb[7] as *u8; let swo: *i64=wb[8] as *i64 1680 let gF: *i64=wb[9] as *i64 1681 let Wg: *u8=wb[10] as *u8; let swg: *i64=wb[11] as *i64 1682 let Wu: *u8=wb[12] as *u8; let swu: *i64=wb[13] as *i64 1683 let Wd: *u8=wb[14] as *u8; let swd: *i64=wb[15] as *i64 1684 let bq: *i64=wb[16] as *i64; let bk: *i64=wb[17] as *i64; let bv: *i64=wb[18] as *i64 1685 rmsnorm_gamma_row_q24(cur, gA, 0, ne, xn, 0) 1686 mm_pool_i8(xn, Wq, swq, Q, ne, qd, 24) 1687 mm_pool_i8(xn, Wk, swk, K, ne, kvd, 24) 1688 mm_pool_i8(xn, Wv, swv, V, ne, kvd, 24) 1689 var bo: i64=0; while bo<qd { Q[bo]=Q[bo]+bq[bo]; bo=bo+1 } 1690 var bp: i64=0; while bp<kvd { K[bp]=K[bp]+bk[bp]; V[bp]=V[bp]+bv[bp]; bp=bp+1 } 1691 var h: i64=0; while h<nh { rope_apply(((Q as i64)+(h*hd)*8) as *i64, hd, pos, freqs); h=h+1 } 1692 var hk: i64=0; while hk<nkv { rope_apply(((K as i64)+(hk*hd)*8) as *i64, hd, pos, freqs); hk=hk+1 } 1693 nf_attn_kv_core(Q, K, V, kvc, L, pos, cfgA, sc, at, concat) 1694 mm_pool_i8(concat, Wo, swo, proj, qd, ne, 16) 1695 var ri: i64=0; while ri<ne { hmid[ri]=cur[ri]+proj[ri]; ri=ri+1 } 1696 rmsnorm_gamma_row_q24(hmid, gF, 0, ne, xn, 0) 1697 mm_pool_i8(xn, Wg, swg, gate, ne, fd, 24) 1698 mm_pool_i8(xn, Wu, swu, up, ne, fd, 24) 1699 var fi: i64=0; while fi<fd { hbuf[fi]=qmul(silu(gate[fi]), up[fi]); fi=fi+1 } 1700 mm_pool_i8(hbuf, Wd, swd, proj, fd, ne, 16) 1701 var fo: i64=0; while fo<ne { cur[fo]=hmid[fo]+proj[fo]; fo=fo+1 } 1702 L=L+1 1703 } 1704 cpy(out1, cur, ne) 1705 return 0 1706} 1707 1708// ==== SHARED ATTENTION CORE (extracted 2026-09-02 for the LM4 resident-Q4_K decoder) ========================== 1709// Exactly the arithmetic decode_step_kv_cached_i8 carried inline: write K/V for pos into the cache of layer L, 1710// then per head score over 0..pos, Q16 softmax, weighted V sum -> concat. Exact i64 throughout; only the 1711// projections around it are lossy. ONE copy, so the i8 and Q4_K decoders cannot disagree on attention. 1712// R0v: ONE per-head attention routine -- the serial control and every pool task call it, so the arithmetic cannot differ. 1713// scores over cnt cached positions, fixed-point softmax, weighted V sum into the head's slice of concat. 1714func nf_attn_head(Q: *i64, Kc: *i64, Vc: *i64, concat: *i64, hh: i64, cnt: i64, hd: i64, kvd: i64, group: i64, scale: i64, sc: *i64, at: *i64) -> i64 { 1715 let kvh: i64=hh/group; let qb: i64=hh*hd; let kb: i64=kvh*hd 1716 var s: i64=0 1717 while s<cnt { var dot: i64=0; var d: i64=0; while d<hd { dot=dot+(Q[qb+d]*Kc[s*kvd+kb+d]); d=d+1 } sc[s]=qmul(dot>>16, scale); s=s+1 } 1718 var mmax: i64=sc[0]; var j: i64=1; while j<cnt { if sc[j]>mmax { mmax=sc[j] } j=j+1 } 1719 var sum: i64=0; j=0; while j<cnt { let e: i64=fx_exp(sc[j]-mmax); at[j]=e; sum=sum+e; j=j+1 } 1720 if sum<=0 { sum=1 } 1721 j=0; while j<cnt { at[j]=(at[j]<<16)/sum; j=j+1 } 1722 var d2: i64=0 1723 while d2<hd { var acc: i64=0; s=0; while s<cnt { acc=acc+(at[s]*Vc[s*kvd+kb+d2]); s=s+1 } concat[qb+d2]=acc>>16; d2=d2+1 } 1724 return 0 1725} 1726// R0v: per-task scratch -- slot b owns two runs of cntmax i64 (scores, weights) inside one arena grown on demand. 1727func nf_attn_scratch(cntmax: i64) -> i64 { 1728 let need: i64 = NF_MMARENA_SLOTS * 2 * cntmax * 8 1729 if need > g_nf_attn_sc_cap { if g_nf_attn_sc_cap > 0 { sys_munmap(g_nf_attn_sc as *u8, g_nf_attn_sc_cap) } g_nf_attn_sc = sys_mmap(need) as *i64; g_nf_attn_sc_cap = need } 1730 return 0 1731} 1732// R0v: a task owns a range of (row, head) pairs p in [c[10], c[11]): row = p / nh, head = p mod nh; row r attends to 1733// pos0 + r + 1 cached positions (the causal count), reading its Q row and writing its concat row. 1734func _nf_attn_pairs_task(ctx_i: i64) -> i64 { 1735 let c: *i64 = ctx_i as *i64 1736 let Q: i64=c[0]; let Kc: *i64=c[1] as *i64; let Vc: *i64=c[2] as *i64; let cat: i64=c[3] 1737 let nh: i64=c[4]; let pos0: i64=c[5]; let hd: i64=c[6]; let kvd: i64=c[7]; let group: i64=c[8]; let scale: i64=c[9] 1738 var p: i64=c[10]; let pe: i64=c[11]; let qd: i64=c[12]; let sc: *i64=c[13] as *i64; let at: *i64=c[14] as *i64 1739 while p<pe { 1740 let row: i64 = p / nh 1741 let hh: i64 = p - row*nh 1742 nf_attn_head((Q + row*qd*8) as *i64, Kc, Vc, (cat + row*qd*8) as *i64, hh, pos0+row+1, hd, kvd, group, scale, sc, at) 1743 p=p+1 1744 } 1745 return 0 1746} 1747// R0v: T x nh (row, head) pairs over the arena's slots; the single-token core is the T=1 case. 1748func nf_attn_pairs_pool(Q: *i64, Kc: *i64, Vc: *i64, concat: *i64, nh: i64, pos0: i64, T: i64, hd: i64, kvd: i64, group: i64, scale: i64, qd: i64) -> i64 { 1749 let p: *NxThreadPool = nf_pool() 1750 let cntmax: i64 = pos0 + T 1751 nf_attn_scratch(cntmax) 1752 let npairs: i64 = T*nh 1753 var tasks: i64 = npairs 1754 if tasks > NF_MMARENA_SLOTS { tasks = NF_MMARENA_SLOTS } 1755 if tasks < 1 { tasks = 1 } 1756 let per: i64 = (npairs + tasks - 1) / tasks 1757 let done0: i64 = nx_pool_n_completed(p) 1758 var b: i64=0 1759 while b<tasks { 1760 let c: *i64 = ((g_nf_mmarena as i64) + b*128) as *i64 1761 c[0]=Q as i64; c[1]=Kc as i64; c[2]=Vc as i64; c[3]=concat as i64; c[4]=nh; c[5]=pos0; c[6]=hd; c[7]=kvd; c[8]=group; c[9]=scale 1762 c[10]=b*per 1763 var pe: i64=(b+1)*per; if pe>npairs { pe=npairs } 1764 c[11]=pe; c[12]=qd 1765 c[13]=((g_nf_attn_sc as i64) + (b*2)*cntmax*8) 1766 c[14]=((g_nf_attn_sc as i64) + (b*2+1)*cntmax*8) 1767 nx_pool_submit(p, _nf_attn_pairs_task, c as i64) 1768 b=b+1 1769 } 1770 nx_pool_wait(p, done0+tasks) 1771 return 0 1772} 1773// attention for ONE token at position pos: write its K/V row into the layer's cache, then every head (serial control or 1774// pool tasks); the signature is unchanged so decode_step_kv_cached_i8 and the sequential prefill are untouched. 1775func nf_attn_kv_core(Q: *i64, K: *i64, V: *i64, kvc: *i64, L: i64, pos: i64, cfgA: *i64, sc: *i64, at: *i64, concat: *i64) -> i64 { 1776 let nh: i64=cfgA[2]; let nkv: i64=cfgA[3]; let hd: i64=cfgA[4]; let qd: i64=cfgA[5]; let kvd: i64=cfgA[6]; let scale: i64=cfgA[7] 1777 let Kc: *i64 = kvc[2*L] as *i64 1778 let Vc: *i64 = kvc[2*L+1] as *i64 1779 var ci: i64=0; while ci<kvd { Kc[pos*kvd+ci]=K[ci]; Vc[pos*kvd+ci]=V[ci]; ci=ci+1 } 1780 let group: i64=nh/nkv 1781 if g_nf_attn_serial == 1 { 1782 var hh: i64=0 1783 while hh<nh { nf_attn_head(Q, Kc, Vc, concat, hh, pos+1, hd, kvd, group, scale, sc, at); hh=hh+1 } 1784 return 0 1785 } 1786 return nf_attn_pairs_pool(Q, Kc, Vc, concat, nh, pos, 1, hd, kvd, group, scale, qd) 1787} 1788// R0v: attention for a prefill block of T rows at positions pos0..pos0+T-1: all K/V rows into the cache first, then every 1789// (row, head) pair -- row r reads pos0+r+1 positions, exactly what the per-token loop read when it reached r. 1790func nf_attn_kv_block(Q: *i64, K: *i64, V: *i64, kvc: *i64, L: i64, pos0: i64, T: i64, cfgA: *i64, sc: *i64, at: *i64, concat: *i64) -> i64 { 1791 let nh: i64=cfgA[2]; let nkv: i64=cfgA[3]; let hd: i64=cfgA[4]; let qd: i64=cfgA[5]; let kvd: i64=cfgA[6]; let scale: i64=cfgA[7] 1792 let Kc: *i64 = kvc[2*L] as *i64 1793 let Vc: *i64 = kvc[2*L+1] as *i64 1794 var t: i64=0 1795 while t<T { var ci: i64=0; while ci<kvd { Kc[(pos0+t)*kvd+ci]=K[t*kvd+ci]; Vc[(pos0+t)*kvd+ci]=V[t*kvd+ci]; ci=ci+1 } t=t+1 } 1796 let group: i64=nh/nkv 1797 if g_nf_attn_serial == 1 { 1798 t=0 1799 while t<T { 1800 var hh: i64=0 1801 while hh<nh { nf_attn_head(((Q as i64)+t*qd*8) as *i64, Kc, Vc, ((concat as i64)+t*qd*8) as *i64, hh, pos0+t+1, hd, kvd, group, scale, sc, at); hh=hh+1 } 1802 t=t+1 1803 } 1804 return 0 1805 } 1806 return nf_attn_pairs_pool(Q, Kc, Vc, concat, nh, pos0, T, hd, kvd, group, scale, qd) 1807} 1808// R0v: the silu/qmul rows of a prefill block as pool tasks (elementwise, so any split is bit-identical). 1809func _nf_silu_rows_task(ctx_i: i64) -> i64 { 1810 let c: *i64 = ctx_i as *i64 1811 let gate: *i64=c[0] as *i64; let up: *i64=c[1] as *i64; let hb: *i64=c[2] as *i64 1812 var i: i64=c[3]; let e: i64=c[4] 1813 while i<e { hb[i]=qmul(silu(gate[i]), up[i]); i=i+1 } 1814 return 0 1815} 1816func nf_silu_rows_pool(gate: *i64, up: *i64, hbuf: *i64, T: i64, fd: i64) -> i64 { 1817 let p: *NxThreadPool = nf_pool() 1818 let n: i64 = T*fd 1819 var tasks: i64 = T 1820 if tasks > NF_MMARENA_SLOTS { tasks = NF_MMARENA_SLOTS } 1821 if tasks < 1 { tasks = 1 } 1822 let per: i64 = (n + tasks - 1) / tasks 1823 let done0: i64 = nx_pool_n_completed(p) 1824 var b: i64=0 1825 while b<tasks { 1826 let c: *i64 = ((g_nf_mmarena as i64) + b*128) as *i64 1827 c[0]=gate as i64; c[1]=up as i64; c[2]=hbuf as i64; c[3]=b*per 1828 var e: i64=(b+1)*per; if e>n { e=n } 1829 c[4]=e 1830 nx_pool_submit(p, _nf_silu_rows_task, c as i64) 1831 b=b+1 1832 } 1833 nx_pool_wait(p, done0+tasks) 1834 return 0 1835}