nx_nofloat_llm.nx source
↩ module page · 1340 lines · 78066 B
1// nx_nofloat_llm.nx -- CANONICAL sovereign no-float LLM library (ONE source of truth; no copy-paste debt).
2//
3// Every integer-Q16 building block for running a real GGUF transformer (Qwen2.5) in deterministic no-float:
4// - dequant of every quant the model uses (F32 / Q5_0 / Q8_0 / Q4_K / Q6_K) -> Q16
5// - by-name tensor load + window-dequant (for embeddings / LM-head rows)
6// - fixed-point transcendentals (exp, sigmoid, SiLU, sin, cos)
7// - the ONE matmul: full-precision accumulate-then-shift (the correct, non-underflowing version)
8// - RMSNorm, RoPE, attention sublayer, FFN sublayer, the lazy N-layer stack
9//
10// Gates IMPORT this instead of re-defining (CLAUDE.md #15 DRY). One canonical, correct implementation:
11// a precision/convention fix is made HERE, once, not in 25 copies. No `main` (pure library).
12// No hw writes (Rule 26). license_tier: ORIGINAL
13import "nx_syscalls.nx"
14import "nx_tier.nx"
15import "nx_le.nx"
16import "nx_tensor.nx"
17import "nx_gguf.nx"
18import "nx_gguf_load.nx"
19import "nx_thread_pool.nx" // decode-speed: banded bit-exact parallel kernels (2026-07-09)
20
21const Q16: i64 = 65536
22const LOG2E: i64 = 94548
23const PC0: i64 = 65536
24const PC1: i64 = 45426
25const PC2: i64 = 15743
26const PC3: i64 = 4367
27const LN_BASE_Q16: i64 = 905421 // ln(1e6), Qwen2.5 rope base
28const HALF_PI: i64 = 102944
29const PI: i64 = 205887
30const THREE_HALF_PI: i64 = 308831
31const TWO_PI: i64 = 411775
32
33func qmul(a: i64, b: i64) -> i64 { return (a*b) >> 16 }
34func cpy(d: *i64, s: *i64, n: i64) -> i64 { var i: i64=0; while i<n { d[i]=s[i]; i=i+1 } return 0 }
35func q_i8(buf: *u8, off: i64) -> i64 { let v: i64=nx_le_read_u8(buf, off); if v>=128 { return v-256 } return v }
36
37// ---- scalar dtype decode -> Q16 (pure integer; no float touches the weights) ----
38func f32_to_q16(bits: i64) -> i64 { let s: i64=(bits>>31)&1; let e: i64=(bits>>23)&255; let mant: i64=bits&8388607; var v: i64=0; if e==0 { v=0 } else { if e==255 { v=2147483647 } else { let m: i64=8388608+mant; let ee: i64=e-134; if ee>=0 { v=m<<ee } else { v=m>>(0-ee) } } } if s==1 { v=0-v } return v }
39func f16_to_q16(h: i64) -> i64 { let s: i64=(h>>15)&1; let e: i64=(h>>10)&31; let mant: i64=h&1023; var v: i64=0; if e==0 { v=mant>>8 } else { if e==31 { v=2147483647 } else { let m: i64=1024+mant; let ee: i64=e-9; if ee>=0 { v=m<<ee } else { v=m>>(0-ee) } } } if s==1 { v=0-v } return v }
40
41// ---- block dequants (one super-block / block each) -> Q16 ----
42func q5_0_block(buf: *u8, off: i64, n: i64, out: *i64) -> i64 { let d: i64=f16_to_q16(nx_le_read_u16(buf, off)); let qh: i64=nx_le_read_u32(buf, off+2); let qo: i64=off+6; var j: i64=0; while j<16 { let qs: i64=nx_le_read_u8(buf, qo+j); let xh0: i64=((qh>>j)<<4)&16; let xh1: i64=(qh>>(j+12))&16; if j<n { out[j]=d*(((qs&15)|xh0)-16) } if j+16<n { out[j+16]=d*(((qs>>4)|xh1)-16) } j=j+1 } return 0 }
43func q8_0_block(buf: *u8, off: i64, n: i64, out: *i64) -> i64 { let d: i64=f16_to_q16(nx_le_read_u16(buf, off)); var j: i64=0; while j<32 { if j<n { out[j]=d*q_i8(buf, off+2+j) } j=j+1 } return 0 }
44// ⚠ Q6_K super-scale d is ~1e-5 on real trained weights -- f16_to_q16 (16 frac bits) ROUNDS IT TO 0, collapsing whole
45// super-blocks to zero (measured: ffn_down 84/896 rows survived -> the FFN down-proj died -> garbage predictions).
46// FIX 2026-07-09: decode d in Q24 (exact for every finite f16, like the Q4_K fix), compute d24*sc*(q-32) in Q24, >>8 to Q16.
47func q6k_block(buf: *u8, so: i64, n: i64, out: *i64) -> i64 {
48 let d24: i64=_gguf_f16_to_q24(nx_le_read_u16(buf, so+208)); var half: i64=0
49 while half<2 { let bql: i64=so+half*64; let bqh: i64=so+128+half*32; let bsc: i64=so+192+half*8; let yb: i64=half*128; var l: i64=0
50 while l<32 { let is: i64=l/16; let ql: i64=nx_le_read_u8(buf, bql+l); let ql3: i64=nx_le_read_u8(buf, bql+l+32); let qh: i64=nx_le_read_u8(buf, bqh+l)
51 if yb+l+0<n { out[yb+l+0]=(d24*q_i8(buf,bsc+is+0)*(((ql&15)|(((qh>>0)&3)<<4))-32))>>8 }
52 if yb+l+32<n { out[yb+l+32]=(d24*q_i8(buf,bsc+is+2)*(((ql3&15)|(((qh>>2)&3)<<4))-32))>>8 }
53 if yb+l+64<n { out[yb+l+64]=(d24*q_i8(buf,bsc+is+4)*(((ql>>4)|(((qh>>4)&3)<<4))-32))>>8 }
54 if yb+l+96<n { out[yb+l+96]=(d24*q_i8(buf,bsc+is+6)*(((ql3>>4)|(((qh>>6)&3)<<4))-32))>>8 }
55 l=l+1 }
56 half=half+1 }
57 return 0
58}
59// ---- DEBT-EATEN 2026-07-15 ("eat our debt, we want state of the art"): the FULL mainstream ggml quant
60// family. Census (nx_gguf_type_census_gate) says the fleet ships F32/Q5_0/Q8_0/Q4_K/Q6_K today -- but any
61// new pack can carry Q5_K/Q3_K/Q2_K/Q4_0/Q4_1/Q5_1/F16/BF16, and the old dispatch either SILENTLY zeroed
62// (dequant_to_q16) or WRONG-DECODED as Q4_K (dequant_row's else-branch). Now: every mainstream type
63// decodes (KAT-gated, nx_gguf_dequant_kat_gate), everything else fails LOUD (-1). IQ-family (i-quants)
64// remains NAMED-unsupported (codebook arc of its own). ----
65// Q4_0: 18B/32 = d(f16) + 16 nibble bytes; y = d*(q-8).
66func q4_0_block(buf: *u8, off: i64, n: i64, out: *i64) -> i64 { let d: i64=f16_to_q16(nx_le_read_u16(buf, off)); let qo: i64=off+2; var j: i64=0; while j<16 { let qs: i64=nx_le_read_u8(buf, qo+j); if j<n { out[j]=d*((qs&15)-8) } if j+16<n { out[j+16]=d*((qs>>4)-8) } j=j+1 } return 0 }
67// Q4_1: 20B/32 = d(f16) + m(f16) + 16 nibble bytes; y = d*q + m.
68func q4_1_block(buf: *u8, off: i64, n: i64, out: *i64) -> i64 { let d: i64=f16_to_q16(nx_le_read_u16(buf, off)); let m: i64=f16_to_q16(nx_le_read_u16(buf, off+2)); let qo: i64=off+4; var j: i64=0; while j<16 { let qs: i64=nx_le_read_u8(buf, qo+j); if j<n { out[j]=d*(qs&15)+m } if j+16<n { out[j+16]=d*(qs>>4)+m } j=j+1 } return 0 }
69// Q5_1: 24B/32 = d(f16) + m(f16) + qh(u32) + 16 nibble bytes; y = d*q5 + m (5th bit from qh).
70func q5_1_block(buf: *u8, off: i64, n: i64, out: *i64) -> i64 { let d: i64=f16_to_q16(nx_le_read_u16(buf, off)); let m: i64=f16_to_q16(nx_le_read_u16(buf, off+2)); let qh: i64=nx_le_read_u32(buf, off+4); let qo: i64=off+8; var j: i64=0; while j<16 { let qs: i64=nx_le_read_u8(buf, qo+j); let xh0: i64=((qh>>j)<<4)&16; let xh1: i64=(qh>>(j+12))&16; if j<n { out[j]=d*((qs&15)|xh0)+m } if j+16<n { out[j+16]=d*((qs>>4)|xh1)+m } j=j+1 } return 0 }
71// Q4_K/Q5_K shared 6-bit scale/min pair unpack (the proven nx_gguf_load idiom): box[0]=sc box[1]=m.
72func qk_scm(buf: *u8, sco: i64, is: i64, box: *i64) -> i64 {
73 if is < 4 {
74 box[0] = nx_le_read_u8(buf, sco+is) & 63
75 box[1] = nx_le_read_u8(buf, sco+is+4) & 63
76 } else {
77 let k: i64 = is - 4
78 let bk: i64 = nx_le_read_u8(buf, sco+k)
79 let bk4: i64 = nx_le_read_u8(buf, sco+4+k)
80 let b8k: i64 = nx_le_read_u8(buf, sco+8+k)
81 box[0] = ((bk >> 6) << 4) | (b8k & 15)
82 box[1] = ((bk4 >> 6) << 4) | (b8k >> 4)
83 }
84 return 0
85}
86// Q5_K: 176B/256 = d(f16) dmin(f16) scales[12] qh[32] qs[128]; y = d*sc*((q4|qh<<4)) - dmin*m. Q24 math -> >>8.
87func q5k_block(buf: *u8, so: i64, n: i64, out: *i64) -> i64 {
88 let d24: i64=_gguf_f16_to_q24(nx_le_read_u16(buf, so))
89 let dm24: i64=_gguf_f16_to_q24(nx_le_read_u16(buf, so+2))
90 let sco: i64=so+4
91 let qho: i64=so+16
92 let qlo: i64=so+48
93 let box: *i64 = sys_mmap(16) as *i64
94 var j: i64=0
95 while j<4 {
96 qk_scm(buf, sco, 2*j, box)
97 let sc0: i64=box[0]
98 let m0: i64=box[1]
99 qk_scm(buf, sco, 2*j+1, box)
100 let sc1: i64=box[0]
101 let m1: i64=box[1]
102 let u1: i64=1<<(2*j)
103 let u2: i64=2<<(2*j)
104 var l: i64=0
105 while l<32 {
106 let ql: i64=nx_le_read_u8(buf, qlo+j*32+l)
107 let qh: i64=nx_le_read_u8(buf, qho+l)
108 var lo: i64=ql&15
109 if (qh&u1)!=0 { lo=lo+16 }
110 var hi: i64=ql>>4
111 if (qh&u2)!=0 { hi=hi+16 }
112 if j*64+l<n { out[j*64+l]=((d24*sc0*lo)-(dm24*m0))>>8 }
113 if j*64+32+l<n { out[j*64+32+l]=((d24*sc1*hi)-(dm24*m1))>>8 }
114 l=l+1
115 }
116 j=j+1
117 }
118 return 0
119}
120// Q3_K 16 x 6-bit scale (RAW 0..63; caller subtracts 32) from the 12 packed bytes at sco.
121func q3k_sc(buf: *u8, sco: i64, is: i64) -> i64 {
122 let k: i64 = is & 3
123 var lo: i64 = 0
124 var hb: i64 = 0
125 if is < 4 { lo = nx_le_read_u8(buf, sco+k) & 15; hb = nx_le_read_u8(buf, sco+8+k) & 3 } else {
126 if is < 8 { lo = nx_le_read_u8(buf, sco+4+k) & 15; hb = (nx_le_read_u8(buf, sco+8+k) >> 2) & 3 } else {
127 if is < 12 { lo = nx_le_read_u8(buf, sco+k) >> 4; hb = (nx_le_read_u8(buf, sco+8+k) >> 4) & 3 } else {
128 lo = nx_le_read_u8(buf, sco+4+k) >> 4
129 hb = (nx_le_read_u8(buf, sco+8+k) >> 6) & 3
130 }
131 }
132 }
133 return lo | (hb << 4)
134}
135// Q3_K: 110B/256 = hmask[32] qs[64] scales[12] d(f16); y = d*(sc-32)*(q3 - (hbit?0:4)). Q24 -> >>8.
136func q3k_block(buf: *u8, so: i64, n: i64, out: *i64) -> i64 {
137 let d24: i64=_gguf_f16_to_q24(nx_le_read_u16(buf, so+108))
138 let sco: i64=so+96
139 var yi: i64=0
140 var is: i64=0
141 var mbit: i64=1
142 var half: i64=0
143 while half<2 {
144 let qb: i64=so+32+half*32
145 var shift: i64=0
146 var jj: i64=0
147 while jj<4 {
148 let dl0: i64=d24*(q3k_sc(buf, sco, is)-32)
149 is=is+1
150 var l: i64=0
151 while l<16 {
152 let qv: i64=(nx_le_read_u8(buf, qb+l)>>shift)&3
153 var t: i64=qv
154 if (nx_le_read_u8(buf, so+l)&mbit)==0 { t=t-4 }
155 if yi<n { out[yi]=(dl0*t)>>8 }
156 yi=yi+1
157 l=l+1
158 }
159 let dl1: i64=d24*(q3k_sc(buf, sco, is)-32)
160 is=is+1
161 l=0
162 while l<16 {
163 let qv2: i64=(nx_le_read_u8(buf, qb+16+l)>>shift)&3
164 var t2: i64=qv2
165 if (nx_le_read_u8(buf, so+16+l)&mbit)==0 { t2=t2-4 }
166 if yi<n { out[yi]=(dl1*t2)>>8 }
167 yi=yi+1
168 l=l+1
169 }
170 shift=shift+2
171 mbit=mbit<<1
172 jj=jj+1
173 }
174 half=half+1
175 }
176 return 0
177}
178// Q2_K: 84B/256 = scales[16] (lo4=sc hi4=m) qs[64] d(f16) dmin(f16); y = d*sc*q2 - dmin*m. Q24 -> >>8.
179func q2k_block(buf: *u8, so: i64, n: i64, out: *i64) -> i64 {
180 let d24: i64=_gguf_f16_to_q24(nx_le_read_u16(buf, so+80))
181 let dm24: i64=_gguf_f16_to_q24(nx_le_read_u16(buf, so+82))
182 var yi: i64=0
183 var is: i64=0
184 var half: i64=0
185 while half<2 {
186 let qb: i64=so+16+half*32
187 var shift: i64=0
188 var jj: i64=0
189 while jj<4 {
190 let sb0: i64=nx_le_read_u8(buf, so+is)
191 is=is+1
192 let dl0: i64=d24*(sb0&15)
193 let ml0: i64=dm24*(sb0>>4)
194 var l: i64=0
195 while l<16 {
196 let qv: i64=(nx_le_read_u8(buf, qb+l)>>shift)&3
197 if yi<n { out[yi]=((dl0*qv)-ml0)>>8 }
198 yi=yi+1
199 l=l+1
200 }
201 let sb1: i64=nx_le_read_u8(buf, so+is)
202 is=is+1
203 let dl1: i64=d24*(sb1&15)
204 let ml1: i64=dm24*(sb1>>4)
205 l=0
206 while l<16 {
207 let qv2: i64=(nx_le_read_u8(buf, qb+16+l)>>shift)&3
208 if yi<n { out[yi]=((dl1*qv2)-ml1)>>8 }
209 yi=yi+1
210 l=l+1
211 }
212 shift=shift+2
213 jj=jj+1
214 }
215 half=half+1
216 }
217 return 0
218}
219// canonical type stride: vb[0]=values/block vb[1]=bytes/block. 0 ok, -1 = NOT A SUPPORTED TYPE (the
220// serve's fail-fast init scans every tensor through this -- unknown types refuse to serve, never zero).
221func nf_type_stride(ty: i64, vb: *i64) -> i64 {
222 if ty==NX_GGML_TYPE_F32 { vb[0]=1; vb[1]=4; return 0 }
223 if ty==NX_GGML_TYPE_F16 { vb[0]=1; vb[1]=2; return 0 }
224 if ty==NX_GGML_TYPE_BF16 { vb[0]=1; vb[1]=2; return 0 }
225 if ty==NX_GGML_TYPE_Q4_0 { vb[0]=32; vb[1]=18; return 0 }
226 if ty==NX_GGML_TYPE_Q4_1 { vb[0]=32; vb[1]=20; return 0 }
227 if ty==NX_GGML_TYPE_Q5_0 { vb[0]=32; vb[1]=22; return 0 }
228 if ty==NX_GGML_TYPE_Q5_1 { vb[0]=32; vb[1]=24; return 0 }
229 if ty==NX_GGML_TYPE_Q8_0 { vb[0]=32; vb[1]=34; return 0 }
230 if ty==NX_GGML_TYPE_Q2_K { vb[0]=256; vb[1]=84; return 0 }
231 if ty==NX_GGML_TYPE_Q3_K { vb[0]=256; vb[1]=110; return 0 }
232 if ty==NX_GGML_TYPE_Q4_K { vb[0]=256; vb[1]=144; return 0 }
233 if ty==NX_GGML_TYPE_Q5_K { vb[0]=256; vb[1]=176; return 0 }
234 if ty==NX_GGML_TYPE_Q6_K { vb[0]=256; vb[1]=210; return 0 }
235 return 0 - 1
236}
237// byte offset of value index `voff` inside a quantized blob (canonical stride; MoE expert slices ride this).
238// voff MUST be block-aligned (expert slices are: ff*D is a 256/32 multiple). -1 on unsupported type.
239func nf_expert_byteoff(ty: i64, voff: i64) -> i64 {
240 let vb: *i64 = sys_mmap(16) as *i64
241 if nf_type_stride(ty, vb) != 0 { return 0 - 1 }
242 return (voff / vb[0]) * vb[1]
243}
244// dequant n contiguous values of a tensor (any supported quant) starting at byte `base` -> Q16. -1 if unsupported.
245func dequant_to_q16(buf: *u8, base: i64, gt: i64, n: i64, out: *i64) -> i64 {
246 if gt==NX_GGML_TYPE_F32 { var i: i64=0; while i<n { out[i]=f32_to_q16(nx_le_read_u32(buf, base+i*4)); i=i+1 } return 0 }
247 if gt==NX_GGML_TYPE_F16 { var i: i64=0; while i<n { out[i]=f16_to_q16(nx_le_read_u16(buf, base+i*2)); i=i+1 } return 0 }
248 if gt==NX_GGML_TYPE_BF16 { var i: i64=0; while i<n { out[i]=f32_to_q16(nx_le_read_u16(buf, base+i*2)<<16); i=i+1 } return 0 }
249 if gt==NX_GGML_TYPE_Q4_K { nx_gguf_dequant_q4_k_q14(buf, base, n, out); var i: i64=0; while i<n { out[i]=out[i]/256; i=i+1 } return 0 } // Q4_K dequant now emits Q24 -> Q16 (/256); was Q14<<2
250 if gt==NX_GGML_TYPE_Q6_K { var sb: i64=0; while sb*256<n { var tk: i64=n-sb*256; if tk>256 { tk=256 } q6k_block(buf, base+sb*210, tk, ((out as i64)+sb*256*8) as *i64); sb=sb+1 } return 0 }
251 if gt==NX_GGML_TYPE_Q5_K { var sb: i64=0; while sb*256<n { var tk: i64=n-sb*256; if tk>256 { tk=256 } q5k_block(buf, base+sb*176, tk, ((out as i64)+sb*256*8) as *i64); sb=sb+1 } return 0 }
252 if gt==NX_GGML_TYPE_Q3_K { var sb: i64=0; while sb*256<n { var tk: i64=n-sb*256; if tk>256 { tk=256 } q3k_block(buf, base+sb*110, tk, ((out as i64)+sb*256*8) as *i64); sb=sb+1 } return 0 }
253 if gt==NX_GGML_TYPE_Q2_K { var sb: i64=0; while sb*256<n { var tk: i64=n-sb*256; if tk>256 { tk=256 } q2k_block(buf, base+sb*84, tk, ((out as i64)+sb*256*8) as *i64); sb=sb+1 } return 0 }
254 if gt==NX_GGML_TYPE_Q5_0 { var sb: i64=0; while sb*32<n { var tk: i64=n-sb*32; if tk>32 { tk=32 } q5_0_block(buf, base+sb*22, tk, ((out as i64)+sb*32*8) as *i64); sb=sb+1 } return 0 }
255 if gt==NX_GGML_TYPE_Q5_1 { var sb: i64=0; while sb*32<n { var tk: i64=n-sb*32; if tk>32 { tk=32 } q5_1_block(buf, base+sb*24, tk, ((out as i64)+sb*32*8) as *i64); sb=sb+1 } return 0 }
256 if gt==NX_GGML_TYPE_Q4_0 { var sb: i64=0; while sb*32<n { var tk: i64=n-sb*32; if tk>32 { tk=32 } q4_0_block(buf, base+sb*18, tk, ((out as i64)+sb*32*8) as *i64); sb=sb+1 } return 0 }
257 if gt==NX_GGML_TYPE_Q4_1 { var sb: i64=0; while sb*32<n { var tk: i64=n-sb*32; if tk>32 { tk=32 } q4_1_block(buf, base+sb*20, tk, ((out as i64)+sb*32*8) as *i64); sb=sb+1 } return 0 }
258 if gt==NX_GGML_TYPE_Q8_0 { var sb: i64=0; while sb*32<n { var tk: i64=n-sb*32; if tk>32 { tk=32 } q8_0_block(buf, base+sb*34, tk, ((out as i64)+sb*32*8) as *i64); sb=sb+1 } return 0 }
259 return 0 - 1
260}
261// find tensor by name + dequant up to `want` values -> out (Q16). #values, -1 not-found, -2 unsupported.
262func load_named_q16(buf: *u8, hdr: *NxGgufHeader, name: *u8, nlen: i64, out: *i64, want: i64) -> i64 {
263 let idx: nx_int = nx_gguf_find_tensor(hdr, name, nlen)
264 if idx < 0 { return 0 - 1 }
265 let ti: *NxGgufTensorInfo = nx_gguf_tensor_at(hdr, idx)
266 let nv: i64 = nx_gguf_tensor_n_values(ti)
267 var n: i64=want; if nv<n { n=nv }
268 if dequant_to_q16(buf, hdr.data_off + ti.offset, ti.ggml_type, n, out) < 0 { return 0 - 2 }
269 return n
270}
271// window-dequant row v (ne values at v*ne) of a [ne, rows] tensor (for embeddings + LM-head rows).
272// DEBT-EATEN 2026-07-15: the old 256-block tail was Q6_K-else-ASSUME-Q4_K -- any other k-quant WRONG-DECODED
273// silently. Now every mainstream type has an explicit row; anything else returns -1 (LOUD).
274func dequant_row(buf: *u8, base: i64, gt: i64, v: i64, ne: i64, out: *i64, tmp: *i64) -> i64 {
275 let vstart: i64=v*ne
276 if gt==NX_GGML_TYPE_F32 { var k: i64=0; while k<ne { out[k]=f32_to_q16(nx_le_read_u32(buf, base+(vstart+k)*4)); k=k+1 } return 0 }
277 if gt==NX_GGML_TYPE_F16 { var k: i64=0; while k<ne { out[k]=f16_to_q16(nx_le_read_u16(buf, base+(vstart+k)*2)); k=k+1 } return 0 }
278 if gt==NX_GGML_TYPE_BF16 { var k: i64=0; while k<ne { out[k]=f32_to_q16(nx_le_read_u16(buf, base+(vstart+k)*2)<<16); k=k+1 } return 0 }
279 if gt==NX_GGML_TYPE_Q8_0 { let fb: i64=vstart/32; let oi: i64=vstart-fb*32; let nb: i64=((vstart+ne-1)/32)-fb+1; var b: i64=0; while b<nb { q8_0_block(buf, base+(fb+b)*34, 32, ((tmp as i64)+b*32*8) as *i64); b=b+1 } var k: i64=0; while k<ne { out[k]=tmp[oi+k]; k=k+1 } return 0 }
280 if gt==NX_GGML_TYPE_Q5_0 { let fb: i64=vstart/32; let oi: i64=vstart-fb*32; let nb: i64=((vstart+ne-1)/32)-fb+1; var b: i64=0; while b<nb { q5_0_block(buf, base+(fb+b)*22, 32, ((tmp as i64)+b*32*8) as *i64); b=b+1 } var k: i64=0; while k<ne { out[k]=tmp[oi+k]; k=k+1 } return 0 }
281 if gt==NX_GGML_TYPE_Q5_1 { let fb: i64=vstart/32; let oi: i64=vstart-fb*32; let nb: i64=((vstart+ne-1)/32)-fb+1; var b: i64=0; while b<nb { q5_1_block(buf, base+(fb+b)*24, 32, ((tmp as i64)+b*32*8) as *i64); b=b+1 } var k: i64=0; while k<ne { out[k]=tmp[oi+k]; k=k+1 } return 0 }
282 if gt==NX_GGML_TYPE_Q4_0 { let fb: i64=vstart/32; let oi: i64=vstart-fb*32; let nb: i64=((vstart+ne-1)/32)-fb+1; var b: i64=0; while b<nb { q4_0_block(buf, base+(fb+b)*18, 32, ((tmp as i64)+b*32*8) as *i64); b=b+1 } var k: i64=0; while k<ne { out[k]=tmp[oi+k]; k=k+1 } return 0 }
283 if gt==NX_GGML_TYPE_Q4_1 { let fb: i64=vstart/32; let oi: i64=vstart-fb*32; let nb: i64=((vstart+ne-1)/32)-fb+1; var b: i64=0; while b<nb { q4_1_block(buf, base+(fb+b)*20, 32, ((tmp as i64)+b*32*8) as *i64); b=b+1 } var k: i64=0; while k<ne { out[k]=tmp[oi+k]; k=k+1 } return 0 }
284 let fb: i64=vstart/256; let oi: i64=vstart-fb*256; let nb: i64=((vstart+ne-1)/256)-fb+1
285 var okb: i64=0
286 if gt==NX_GGML_TYPE_Q6_K { var b: i64=0; while b<nb { q6k_block(buf, base+(fb+b)*210, 256, ((tmp as i64)+b*256*8) as *i64); b=b+1 } okb=1 }
287 if gt==NX_GGML_TYPE_Q5_K { var b: i64=0; while b<nb { q5k_block(buf, base+(fb+b)*176, 256, ((tmp as i64)+b*256*8) as *i64); b=b+1 } okb=1 }
288 if gt==NX_GGML_TYPE_Q3_K { var b: i64=0; while b<nb { q3k_block(buf, base+(fb+b)*110, 256, ((tmp as i64)+b*256*8) as *i64); b=b+1 } okb=1 }
289 if gt==NX_GGML_TYPE_Q2_K { var b: i64=0; while b<nb { q2k_block(buf, base+(fb+b)*84, 256, ((tmp as i64)+b*256*8) as *i64); b=b+1 } okb=1 }
290 if gt==NX_GGML_TYPE_Q4_K { nx_gguf_dequant_q4_k_q14(buf, base+fb*144, nb*256, tmp); var z: i64=0; while z<nb*256 { tmp[z]=tmp[z]/256; z=z+1 } okb=1 } // Q24 -> Q16 (/256)
291 if okb==0 { return 0 - 1 }
292 var k: i64=0; while k<ne { out[k]=tmp[oi+k]; k=k+1 }
293 return 0
294}
295// build "blk.<L><suffix>" into out, return length; load that tensor.
296func build_name(out: *u8, L: i64, suffix: *u8) -> i64 {
297 out[0]=98 as u8; out[1]=108 as u8; out[2]=107 as u8; out[3]=46 as u8
298 var p: i64=4
299 if L==0 { out[p]=48 as u8; p=p+1 } else { let tmp: *u8=sys_mmap(16); var k: i64=0; var m: i64=L; while m>0 { tmp[k]=(48+(m%10)) as u8; m=m/10; k=k+1 } var i: i64=k-1; while i>=0 { out[p]=tmp[i]; p=p+1; i=i-1 } }
300 var j: i64=0; while suffix[j]!=(0 as u8) { out[p]=suffix[j]; p=p+1; j=j+1 }
301 out[p]=0 as u8
302 return p
303}
304func load_blk(buf: *u8, hdr: *NxGgufHeader, nm: *u8, L: i64, suffix: *u8, out: *i64, want: i64) -> i64 { let len: i64=build_name(nm, L, suffix); return load_named_q16(buf, hdr, nm, len, out, want) }
305
306// ---- fixed-point math ----
307func isqrt(v: i64) -> i64 { if v<=0 { return 0 } if v<4 { return 1 } var x: i64=v; var y: i64=(x+1)>>1; var go: i64=1; while go==1 { if y<x { x=y; y=(x+v/x)>>1 } else { go=0 } } return x }
308func fx_exp(x: i64) -> i64 { var xm: i64=0-x; if x>0 { xm=0 } let ym: i64=(xm*LOG2E)>>16; let yi: i64=ym>>16; let yf: i64=ym-(yi<<16); let g: i64=Q16-yf; var t: i64=PC3; t=PC2+((g*t)>>16); t=PC1+((g*t)>>16); t=PC0+((g*t)>>16); t=t>>1; if yi>=31 { return 0 } return t>>yi }
309func sigmoid(x: i64) -> i64 { if x>=0 { let ex: i64=fx_exp(0-x); return (Q16*Q16)/(Q16+ex) } let ex: i64=fx_exp(x); let sp: i64=(Q16*Q16)/(Q16+ex); return Q16-sp }
310func silu(x: i64) -> i64 { return qmul(x, sigmoid(x)) }
311func sin_q(x: i64) -> i64 { let x2: i64=qmul(x,x); let x3: i64=qmul(x2,x); let x5: i64=qmul(x3,x2); let x7: i64=qmul(x5,x2); let x9: i64=qmul(x7,x2); return x - x3/6 + x5/120 - x7/5040 + x9/362880 }
312func cos_q(x: i64) -> i64 { let x2: i64=qmul(x,x); let x4: i64=qmul(x2,x2); let x6: i64=qmul(x4,x2); let x8: i64=qmul(x6,x2); return Q16 - x2/2 + x4/24 - x6/720 + x8/40320 }
313func reduce2pi(a: i64) -> i64 { var t: i64=a; while t<0 { t=t+TWO_PI } while t>=TWO_PI { t=t-TWO_PI } return t }
314func sin_full(a: i64) -> i64 { let t: i64=reduce2pi(a); if t<HALF_PI { return sin_q(t) } if t<PI { return sin_q(PI-t) } if t<THREE_HALF_PI { return 0-sin_q(t-PI) } return 0-sin_q(TWO_PI-t) }
315func cos_full(a: i64) -> i64 { let t: i64=reduce2pi(a); if t<HALF_PI { return cos_q(t) } if t<PI { return 0-cos_q(PI-t) } if t<THREE_HALF_PI { return 0-cos_q(t-PI) } return cos_q(TWO_PI-t) }
316// ⚠ NEOX pairing (Qwen2.5 GGUF = llama.cpp LLAMA_ROPE_TYPE_NEOX): rotate the two HALVES (x[i], x[i+half]) with
317// theta_i=freqs[i]=base^(-2i/d). NOT the adjacent (x[2i],x[2i+1]) variant -- that rotates the WRONG pairs at every
318// pos>0 -> coherent-looking token 0 (rope=identity) but incoherent multi-token. Matches nx_f32_rope_apply_cs_neox. (fix 2026-07-09)
319func rope_apply(v: *i64, hd: i64, pos: i64, freqs: *i64) -> i64 { let np: i64=hd/2; var i: i64=0; while i<np { let ang: i64=pos*freqs[i]; let c: i64=cos_full(ang); let s: i64=sin_full(ang); let a: i64=v[i]; let b: i64=v[i+np]; v[i]=qmul(a,c)-qmul(b,s); v[i+np]=qmul(a,s)+qmul(b,c); i=i+1 } return 0 }
320// per-head RoPE base^(-2i/hd) frequencies (Q16) -> freqs[hd/2].
321func rope_freqs(freqs: *i64, hd: i64) -> i64 { let np: i64=hd/2; var i: i64=0; while i<np { freqs[i]=fx_exp(0-((i*LN_BASE_Q16)/np)); i=i+1 } return 0 }
322// ln(x) in Q16 for a positive integer x (2026-07-15 debt-eaten: makes rope base CONFIGURABLE from model
323// metadata instead of the hardcoded ln(1e6) const). ln(x)=p*ln2 + ln(m), m=x/2^p in [1,2) via the fast
324// atanh series ln(m)=2*(z+z^3/3+z^5/5+z^7/7), z=(m-1)/(m+1). MEASURED: ln(1e6)=905404 (true 905420, ~2e-4
325// rel = the Q16 fixed-point floor; negligible after rope's /np + exp), ln(1e4)=603606, ln(2)=45426, ln(1)=0.
326func fx_ln_int(x: i64) -> i64 {
327 if x < 1 { return 0 }
328 var p: i64 = 0
329 var v: i64 = x
330 while v > 1 { v = v >> 1; p = p + 1 }
331 // m in Q16 = (x << 16) >> p, in [65536, 131072)
332 let m: i64 = (x << 16) >> p
333 let num: i64 = m - Q16
334 let den: i64 = m + Q16
335 let z: i64 = (num << 16) / den
336 let z2: i64 = (z * z) >> 16
337 var term: i64 = z
338 var sum: i64 = z
339 term = (term * z2) >> 16
340 sum = sum + term / 3
341 term = (term * z2) >> 16
342 sum = sum + term / 5
343 term = (term * z2) >> 16
344 sum = sum + term / 7
345 let ln_m: i64 = 2 * sum
346 return p * 45426 + ln_m // 45426 = ln(2) in Q16
347}
348// per-head RoPE frequencies for an ARBITRARY base (base^(-2i/hd) = exp(-(i/np)*ln(base))). Additive: the
349// bare rope_freqs stays the compile-time-1e6 path (identity for Qwen); this one takes ln(base) in Q16 so a
350// model's real rope base (OLMoE 1e4, Qwen 1e6, Llama 5e5...) drives attention correctly.
351func rope_freqs_base(freqs: *i64, hd: i64, ln_base_q16: i64) -> i64 { let np: i64=hd/2; var i: i64=0; while i<np { freqs[i]=fx_exp(0-((i*ln_base_q16)/np)); i=i+1 } return 0 }
352
353// ---- THE matmul: GGUF [out,in], full-precision accumulate then ONE shift (no per-term underflow) ----
354func mm_out_in(inp: *i64, W: *i64, dst: *i64, T: i64, in_dim: i64, out_dim: i64, rev: i64) -> i64 {
355 var t: i64=0
356 while t<T { var o: i64=0
357 while o<out_dim { var s: i64=0
358 if rev==0 { var k: i64=0; while k<in_dim { s=s+(inp[t*in_dim+k]*W[o*in_dim+k]); k=k+1 } }
359 else { var k: i64=in_dim-1; while k>=0 { s=s+(inp[t*in_dim+k]*W[o*in_dim+k]); k=k-1 } }
360 dst[t*out_dim+o]=s>>16; o=o+1 }
361 t=t+1 }
362 return 0
363}
364func rmsnorm_gamma_row(x: *i64, gamma: *i64, xoff: i64, D: i64, out: *i64, ooff: i64) -> i64 {
365 var ss: i64=0; var i: i64=0; while i<D { let v: i64=x[xoff+i]; ss=ss+((v*v)>>16); i=i+1 }
366 let ms: i64=ss/D; let sd: i64=isqrt((ms+1)<<16); if sd<=0 { return 0 }
367 i=0; while i<D { let nm: i64=(x[xoff+i]<<16)/sd; out[ooff+i]=qmul(nm, gamma[i]); i=i+1 }
368 return 0
369}
370// PRECISION LIFT (2026-07-09): the ~1587 massive activations dominate the RMS, so signal dims normalize to ~1e-3 and
371// keep only ~3 bits in Q16 (then gamma~0.08 halves it) -> logits imprecise. This emits the normed*gamma output in Q24
372// (8 more fractional bits, ~11 bits on the signal). Input x is Q16 (residual); output is Q24. Feed to mm_q24in.
373func rmsnorm_gamma_row_q24(x: *i64, gamma: *i64, xoff: i64, D: i64, out: *i64, ooff: i64) -> i64 {
374 var ss: i64=0; var i: i64=0; while i<D { let v: i64=x[xoff+i]; ss=ss+((v*v)>>16); i=i+1 }
375 let ms: i64=ss/D; let sd: i64=isqrt((ms+1)<<16); if sd<=0 { return 0 }
376 i=0; while i<D { let nm: i64=(x[xoff+i]<<24)/sd; out[ooff+i]=(nm*gamma[i])>>16; i=i+1 }
377 return 0
378}
379// matmul with a Q24 input (rmsnorm_gamma_row_q24 output) x Q16 weights -> Q16 output (>>24). Full-precision accumulate.
380func mm_q24in(inp: *i64, W: *i64, dst: *i64, T: i64, in_dim: i64, out_dim: i64, rev: i64) -> i64 {
381 var t: i64=0
382 while t<T { var o: i64=0
383 while o<out_dim { var s: i64=0
384 if rev==0 { var k: i64=0; while k<in_dim { s=s+(inp[t*in_dim+k]*W[o*in_dim+k]); k=k+1 } }
385 else { var k: i64=in_dim-1; while k>=0 { s=s+(inp[t*in_dim+k]*W[o*in_dim+k]); k=k-1 } }
386 dst[t*out_dim+o]=s>>24; o=o+1 }
387 t=t+1 }
388 return 0
389}
390// ==== DECODE-SPEED: shared thread pool + banded BIT-EXACT parallel kernels (2026-07-09) =====================
391// Banding is by whole OUTPUT ELEMENT (matmul rows / vocab rows) or whole TENSOR (weight dequant) -> no partial
392// sum ever crosses a band -> results are BIT-IDENTICAL to the serial path (kvgen gate pins the tokens to prove
393// it). Single-submitter: every caller is a main-thread forward. Static POINTERS lazy-mmap'd (BSS-array trap).
394static g_nf_pool: *NxThreadPool
395static g_nf_ldarena: *u8 // 12 x 128B: load-task ctx (64B) + private name scratch (64B)
396static g_nf_mmarena: *u8 // 32 x 128B: matmul band ctx
397static g_nf_hdctx: *u8 // 32 x 128B: head band ctx
398static g_nf_hdrow: *u8 // 32 x 8KB : per-band dequant row
399static g_nf_hdtmp: *u8 // 32 x 32KB: per-band dequant window tmp
400static g_nf_hdout: *u8 // 32 x 16B : per-band (best_id, best_logit)
401static g_nf_xi8: *u8 // 4864 x 2 : quantized activation (i16 lanes) for the SIMD W8A8 path
402static g_nf_i8acc: *u8 // 32 x 64B : per-band int32x8 SIMD accumulator
403func nf_pool() -> *NxThreadPool {
404 if (g_nf_pool as i64) == 0 {
405 g_nf_pool = nx_pool_new(0, 64)
406 g_nf_ldarena = sys_mmap(12*128)
407 g_nf_mmarena = sys_mmap(32*128)
408 g_nf_hdctx = sys_mmap(32*128)
409 g_nf_hdrow = sys_mmap(32*8192)
410 g_nf_hdtmp = sys_mmap(32*32768)
411 g_nf_hdout = sys_mmap(32*16)
412 g_nf_xi8 = sys_mmap(4864*2)
413 g_nf_i8acc = sys_mmap(32*64)
414 }
415 return g_nf_pool
416}
417// int32x8 horizontal sum for the SIMD path. Since the 2026-07-10 sext compiler fix, *i32 loads SIGN-extend
418// natively (the manual branchless sign-extend this fn used to carry was removed in lockstep with the bless --
419// keeping it would DOUBLE-extend and corrupt already-negative lanes).
420func nf_hsum_sx(acc: *u8) -> i64 {
421 let pp: *i32 = acc as *i32
422 var s: i64=0; var i: i64=0
423 while i<8 { s=s+(pp[i] as i64); i=i+1 }
424 return s
425}
426// one banded-matmul task: rows [lo,hi) of dst = inp x W^T, full accumulate, >>shift.
427func _nfmm_task(ctx_i: i64) -> i64 {
428 let c: *i64 = ctx_i as *i64
429 let inp: *i64=c[0] as *i64; let W: *i64=c[1] as *i64; let dst: *i64=c[2] as *i64
430 let T: i64=c[3]; let ind: i64=c[4]; let outd: i64=c[5]; let lo: i64=c[6]; let hi: i64=c[7]; let sh: i64=c[8]
431 var t: i64=0
432 while t<T { var o: i64=lo
433 while o<hi { var s: i64=0; var k: i64=0; while k<ind { s=s+(inp[t*ind+k]*W[o*ind+k]); k=k+1 }
434 if sh==24 { dst[t*outd+o]=s>>24 } else { dst[t*outd+o]=s>>16 }
435 o=o+1 }
436 t=t+1 }
437 return 0
438}
439// pooled matmul (forward accumulation order only): bit-identical to mm_out_in/mm_q24in with rev=0.
440func mm_pool(inp: *i64, W: *i64, dst: *i64, T: i64, in_dim: i64, out_dim: i64, shift: i64) -> i64 {
441 let p: *NxThreadPool = nf_pool()
442 var bands: i64 = p.n_workers
443 if bands > 32 { bands = 32 }
444 if bands > out_dim { bands = out_dim }
445 if bands < 1 { bands = 1 }
446 let per: i64 = (out_dim + bands - 1) / bands
447 let done0: i64 = nx_pool_n_completed(p)
448 var b: i64 = 0
449 while b < bands {
450 let c: *i64 = ((g_nf_mmarena as i64) + b*128) as *i64
451 c[0]=inp as i64; c[1]=W as i64; c[2]=dst as i64; c[3]=T; c[4]=in_dim; c[5]=out_dim; c[6]=b*per
452 var hi: i64=(b+1)*per
453 if hi>out_dim { hi=out_dim }
454 c[7]=hi; c[8]=shift
455 nx_pool_submit(p, _nfmm_task, c as i64)
456 b=b+1
457 }
458 nx_pool_wait(p, done0+bands)
459 return 0
460}
461// one weight-load task: load_blk with a PRIVATE name scratch (12 concurrent tasks must not share nm).
462func _nfld_task(ctx_i: i64) -> i64 {
463 let c: *i64 = ctx_i as *i64
464 let nm: *u8 = (ctx_i + 64) as *u8
465 load_blk(c[0] as *u8, c[1] as *NxGgufHeader, nm, c[2], c[3] as *u8, c[4] as *i64, c[5])
466 return 0
467}
468func nf_ld1(p: *NxThreadPool, i: i64, buf: *u8, hdr: *NxGgufHeader, L: i64, sfx: *u8, out: i64, want: i64) -> i64 {
469 let c: *i64 = ((g_nf_ldarena as i64) + i*128) as *i64
470 c[0]=buf as i64; c[1]=hdr as i64; c[2]=L; c[3]=sfx as i64; c[4]=out; c[5]=want
471 nx_pool_submit(p, _nfld_task, c as i64)
472 return 0
473}
474// load one layer's 12 tensors in parallel (disjoint outputs; each task private name scratch). Bit-exact.
475func nf_load_layer_pool(buf: *u8, hdr: *NxGgufHeader, L: i64, wb: *i64, ne: i64, qd: i64, kvd: i64, fd: i64) -> i64 {
476 let p: *NxThreadPool = nf_pool()
477 let done0: i64 = nx_pool_n_completed(p)
478 nf_ld1(p, 0, buf, hdr, L, ".attn_norm.weight\x00" as *u8, wb[0], ne)
479 nf_ld1(p, 1, buf, hdr, L, ".attn_q.weight\x00" as *u8, wb[1], qd*ne)
480 nf_ld1(p, 2, buf, hdr, L, ".attn_k.weight\x00" as *u8, wb[2], kvd*ne)
481 nf_ld1(p, 3, buf, hdr, L, ".attn_v.weight\x00" as *u8, wb[3], kvd*ne)
482 nf_ld1(p, 4, buf, hdr, L, ".attn_output.weight\x00" as *u8, wb[4], ne*qd)
483 nf_ld1(p, 5, buf, hdr, L, ".attn_q.bias\x00" as *u8, wb[9], qd)
484 nf_ld1(p, 6, buf, hdr, L, ".attn_k.bias\x00" as *u8, wb[10], kvd)
485 nf_ld1(p, 7, buf, hdr, L, ".attn_v.bias\x00" as *u8, wb[11], kvd)
486 nf_ld1(p, 8, buf, hdr, L, ".ffn_norm.weight\x00" as *u8, wb[5], ne)
487 nf_ld1(p, 9, buf, hdr, L, ".ffn_gate.weight\x00" as *u8, wb[6], ne*fd)
488 nf_ld1(p, 10, buf, hdr, L, ".ffn_up.weight\x00" as *u8, wb[7], ne*fd)
489 nf_ld1(p, 11, buf, hdr, L, ".ffn_down.weight\x00" as *u8, wb[8], fd*ne)
490 nx_pool_wait(p, done0+12)
491 return 0
492}
493// one head band: scan vocab rows [lo,hi), local strict-> argmax (keeps the LOWEST id on ties, matching serial).
494func _nfhd_task(ctx_i: i64) -> i64 {
495 let c: *i64 = ctx_i as *i64
496 let buf: *u8=c[0] as *u8; let base: i64=c[1]; let ty: i64=c[2]; let normed: *i64=c[3] as *i64
497 let ne: i64=c[4]; let lo: i64=c[5]; let hi: i64=c[6]
498 let row: *i64=c[7] as *i64; let tmp: *i64=c[8] as *i64; let outp: *i64=c[9] as *i64
499 var best: i64=lo
500 var bestv: i64=0-9223372036854775807
501 var v: i64=lo
502 while v<hi {
503 dequant_row(buf, base, ty, v, ne, row, tmp)
504 var s: i64=0; var k: i64=0; while k<ne { s=s+(normed[k]*row[k]); k=k+1 }
505 let lg: i64=s>>24
506 if lg>bestv { bestv=lg; best=v }
507 v=v+1
508 }
509 outp[0]=best
510 outp[1]=bestv
511 return 0
512}
513// pooled streamed LM-head argmax. hp = [buf, oh_base, oh_ty, normed(Q24), vocab, ne, out_id_ptr, out_logit_ptr].
514// Returns the argmax id; logits use the Q24 head convention (dot >>24). Band-ascending strict-> reduce keeps the
515// lowest id on ties = identical to the serial scan.
516func head_argmax_pool(hp: *i64) -> i64 {
517 let p: *NxThreadPool = nf_pool()
518 let vocab: i64=hp[4]
519 var bands: i64 = p.n_workers
520 if bands > 32 { bands = 32 }
521 if bands > vocab { bands = vocab }
522 if bands < 1 { bands = 1 }
523 let per: i64 = (vocab + bands - 1) / bands
524 // ROOT FIX 2026-07-15: dims-sized per-call band scratch (the fixed 8192B row slot overflows at
525 // ne>1024 into the neighbor band -- same corruption class as the i32 head-cache build; see
526 // nf_dequant_head_all_i32). Freed after the wait. ~2 syscalls vs ~650ms/token here = negligible.
527 let hne: i64 = hp[5]
528 let rstride2: i64 = hne*8
529 let tstride2: i64 = hne*8 + 4096
530 let rows2: *u8 = sys_mmap(bands*rstride2)
531 let tmps2: *u8 = sys_mmap(bands*tstride2)
532 let done0: i64 = nx_pool_n_completed(p)
533 var b: i64 = 0
534 while b < bands {
535 let c: *i64 = ((g_nf_hdctx as i64) + b*128) as *i64
536 c[0]=hp[0]; c[1]=hp[1]; c[2]=hp[2]; c[3]=hp[3]; c[4]=hp[5]; c[5]=b*per
537 var hi: i64=(b+1)*per
538 if hi>vocab { hi=vocab }
539 c[6]=hi
540 c[7]=((rows2 as i64) + b*rstride2)
541 c[8]=((tmps2 as i64) + b*tstride2)
542 c[9]=((g_nf_hdout as i64) + b*16)
543 nx_pool_submit(p, _nfhd_task, c as i64)
544 b=b+1
545 }
546 nx_pool_wait(p, done0+bands)
547 sys_munmap(rows2, bands*rstride2)
548 sys_munmap(tmps2, bands*tstride2)
549 var best: i64=0
550 var bestv: i64=0-9223372036854775807
551 var rb: i64=0
552 while rb<bands {
553 let o: *i64=((g_nf_hdout as i64) + rb*16) as *i64
554 if o[1]>bestv { bestv=o[1]; best=o[0] }
555 rb=rb+1
556 }
557 let oid: *i64=hp[6] as *i64
558 let olg: *i64=hp[7] as *i64
559 oid[0]=best
560 olg[0]=bestv
561 return best
562}
563// attention sublayer: out = x + W_o( GQA-causal-softmax( RoPE(W_q·RMSNorm x), RoPE(W_k·..), W_v·.. ) ).
564// cfg = [T, ne, n_heads, n_kv, head_dim, q_dim, kv_dim, attn_scale].
565func attn_sublayer(x: *i64, gamma: *i64, Wq: *i64, Wk: *i64, Wv: *i64, Wo: *i64, bq: *i64, bk: *i64, bv: *i64, freqs: *i64, xn: *i64, Q: *i64, K: *i64, V: *i64, concat: *i64, sc: *i64, at: *i64, proj: *i64, out: *i64, cfg: *i64, rev: i64) -> i64 {
566 let T: i64=cfg[0]; let ne: i64=cfg[1]; let nh: i64=cfg[2]; let nkv: i64=cfg[3]; let hd: i64=cfg[4]; let qd: i64=cfg[5]; let kvd: i64=cfg[6]; let scale: i64=cfg[7]
567 var t: i64=0
568 while t<T { rmsnorm_gamma_row_q24(x, gamma, t*ne, ne, xn, t*ne); t=t+1 }
569 // pooled (bit-exact) matmuls on the forward path; rev!=0 keeps the serial reversed-accumulation path so the
570 // fwd-vs-rev determinism teeth still exercise a genuinely different reduction order.
571 if rev==0 { mm_pool(xn, Wq, Q, T, ne, qd, 24); mm_pool(xn, Wk, K, T, ne, kvd, 24); mm_pool(xn, Wv, V, T, ne, kvd, 24) }
572 else { mm_q24in(xn, Wq, Q, T, ne, qd, rev); mm_q24in(xn, Wk, K, T, ne, kvd, rev); mm_q24in(xn, Wv, V, T, ne, kvd, rev) }
573 // Qwen2 Q/K/V projection biases (added before RoPE)
574 var bt: i64=0
575 while bt<T { var bo: i64=0; while bo<qd { Q[bt*qd+bo]=Q[bt*qd+bo]+bq[bo]; bo=bo+1 } var bp: i64=0; while bp<kvd { K[bt*kvd+bp]=K[bt*kvd+bp]+bk[bp]; V[bt*kvd+bp]=V[bt*kvd+bp]+bv[bp]; bp=bp+1 } bt=bt+1 }
576 t=0
577 while t<T { var h: i64=0; while h<nh { rope_apply(((Q as i64)+(t*qd+h*hd)*8) as *i64, hd, t, freqs); h=h+1 } var hk: i64=0; while hk<nkv { rope_apply(((K as i64)+(t*kvd+hk*hd)*8) as *i64, hd, t, freqs); hk=hk+1 } t=t+1 }
578 let group: i64=nh/nkv
579 var hh: i64=0
580 while hh<nh {
581 let kv: i64=hh/group; let bq: i64=hh*hd; let bk: i64=kv*hd
582 t=0
583 while t<T {
584 let cnt: i64=t+1; var s: i64=0
585 while s<cnt { var dot: i64=0; var d: i64=0; while d<hd { dot=dot+(Q[t*qd+bq+d]*K[s*kvd+bk+d]); d=d+1 } sc[s]=qmul(dot>>16, scale); s=s+1 }
586 var mmax: i64=sc[0]; var j: i64=1; while j<cnt { if sc[j]>mmax { mmax=sc[j] } j=j+1 }
587 var sum: i64=0; j=0; while j<cnt { let e: i64=fx_exp(sc[j]-mmax); at[j]=e; sum=sum+e; j=j+1 }
588 if sum<=0 { sum=1 }
589 j=0; while j<cnt { at[j]=(at[j]<<16)/sum; j=j+1 }
590 var d2: i64=0
591 while d2<hd { var acc: i64=0; s=0; while s<cnt { acc=acc+(at[s]*V[s*kvd+bk+d2]); s=s+1 } concat[t*qd+bq+d2]=acc>>16; d2=d2+1 }
592 t=t+1
593 }
594 hh=hh+1
595 }
596 mm_out_in(concat, Wo, proj, T, qd, ne, rev)
597 var i: i64=0; while i<T*ne { out[i]=x[i]+proj[i]; i=i+1 }
598 return 0
599}
600// FFN sublayer: out = x + W_down( SiLU(W_gate·RMSNorm x) (*) W_up·RMSNorm x ). cfg = [T, ne, ffn_dim].
601func ffn_sublayer(x: *i64, gamma: *i64, Wg: *i64, Wu: *i64, Wd: *i64, xn: *i64, gate: *i64, up: *i64, hbuf: *i64, proj: *i64, out: *i64, cfg: *i64, rev: i64) -> i64 {
602 let T: i64=cfg[0]; let ne: i64=cfg[1]; let fd: i64=cfg[2]
603 var t: i64=0
604 while t<T { rmsnorm_gamma_row_q24(x, gamma, t*ne, ne, xn, t*ne); t=t+1 }
605 mm_q24in(xn, Wg, gate, T, ne, fd, rev); mm_q24in(xn, Wu, up, T, ne, fd, rev)
606 var i: i64=0; while i<T*fd { hbuf[i]=qmul(silu(gate[i]), up[i]); i=i+1 }
607 mm_out_in(hbuf, Wd, proj, T, fd, ne, rev)
608 i=0; while i<T*ne { out[i]=x[i]+proj[i]; i=i+1 }
609 return 0
610}
611// run N transformer blocks (lazy per-layer load), x -> out. wb=9 weight buffers, sb=14 scratch (ptrs as i64).
612func run_stack(buf: *u8, hdr: *NxGgufHeader, x: *i64, out: *i64, wb: *i64, sb: *i64, nm: *u8, freqs: *i64, cfgA: *i64, cfgF: *i64, N: i64, rev: i64) -> i64 {
613 let T: i64=cfgA[0]; let ne: i64=cfgA[1]; let qd: i64=cfgA[5]; let kvd: i64=cfgA[6]; let fd: i64=cfgF[2]
614 let gA: *i64=wb[0] as *i64; let Wq: *i64=wb[1] as *i64; let Wk: *i64=wb[2] as *i64; let Wv: *i64=wb[3] as *i64; let Wo: *i64=wb[4] as *i64
615 let gF: *i64=wb[5] as *i64; let Wg: *i64=wb[6] as *i64; let Wu: *i64=wb[7] as *i64; let Wd: *i64=wb[8] as *i64
616 let bq: *i64=wb[9] as *i64; let bk: *i64=wb[10] as *i64; let bv: *i64=wb[11] as *i64
617 let xn: *i64=sb[0] as *i64; let Q: *i64=sb[1] as *i64; let K: *i64=sb[2] as *i64; let V: *i64=sb[3] as *i64; let concat: *i64=sb[4] as *i64
618 let sc: *i64=sb[5] as *i64; let at: *i64=sb[6] as *i64; let proj: *i64=sb[7] as *i64; let gate: *i64=sb[8] as *i64; let up: *i64=sb[9] as *i64
619 let hbuf: *i64=sb[10] as *i64; let hmid: *i64=sb[11] as *i64; let cur: *i64=sb[12] as *i64; let cur2: *i64=sb[13] as *i64
620 cpy(cur, x, T*ne)
621 var L: i64=0
622 while L<N {
623 load_blk(buf, hdr, nm, L, ".attn_norm.weight\x00" as *u8, gA, ne)
624 load_blk(buf, hdr, nm, L, ".attn_q.weight\x00" as *u8, Wq, qd*ne)
625 load_blk(buf, hdr, nm, L, ".attn_k.weight\x00" as *u8, Wk, kvd*ne)
626 load_blk(buf, hdr, nm, L, ".attn_v.weight\x00" as *u8, Wv, kvd*ne)
627 load_blk(buf, hdr, nm, L, ".attn_output.weight\x00" as *u8, Wo, ne*qd)
628 load_blk(buf, hdr, nm, L, ".attn_q.bias\x00" as *u8, bq, qd)
629 load_blk(buf, hdr, nm, L, ".attn_k.bias\x00" as *u8, bk, kvd)
630 load_blk(buf, hdr, nm, L, ".attn_v.bias\x00" as *u8, bv, kvd)
631 load_blk(buf, hdr, nm, L, ".ffn_norm.weight\x00" as *u8, gF, ne)
632 load_blk(buf, hdr, nm, L, ".ffn_gate.weight\x00" as *u8, Wg, ne*fd)
633 load_blk(buf, hdr, nm, L, ".ffn_up.weight\x00" as *u8, Wu, ne*fd)
634 load_blk(buf, hdr, nm, L, ".ffn_down.weight\x00" as *u8, Wd, fd*ne)
635 attn_sublayer(cur, gA, Wq, Wk, Wv, Wo, bq, bk, bv, freqs, xn, Q, K, V, concat, sc, at, proj, hmid, cfgA, rev)
636 ffn_sublayer(hmid, gF, Wg, Wu, Wd, xn, gate, up, hbuf, proj, cur2, cfgF, rev)
637 cpy(cur, cur2, T*ne)
638 L=L+1
639 }
640 cpy(out, cur, T*ne)
641 return 0
642}
643// ==== KV-CACHE decode path (2026-07-09) ====================================================================
644// Prefill captures each layer's post-bias/post-RoPE K rows + post-bias V rows; decode_step_kv forwards ONLY the
645// new token, appending to + attending over the cache. LOSSLESS vs the uncached forward: causal masking means an
646// old token's K/V at every layer depend only on tokens 0..itself, so cached decode is BIT-IDENTICAL (gate-proven
647// by matching the uncached generation token-for-token). kvc = 2N ptrs [Kc_0, Vc_0, Kc_1, Vc_1, ...], each cache
648// MAXT*kvd i64. ADDITIVE: run_stack keeps its signature (nx_cc has no arity check -- changing it would let old
649// call sites pass garbage silently).
650func run_stack_prefill_kv(buf: *u8, hdr: *NxGgufHeader, x: *i64, out: *i64, wb: *i64, sb: *i64, nm: *u8, freqs: *i64, kvc: *i64, cfgA: *i64, cfgF: *i64, N: i64) -> i64 {
651 let T: i64=cfgA[0]; let ne: i64=cfgA[1]; let qd: i64=cfgA[5]; let kvd: i64=cfgA[6]; let fd: i64=cfgF[2]
652 let gA: *i64=wb[0] as *i64; let Wq: *i64=wb[1] as *i64; let Wk: *i64=wb[2] as *i64; let Wv: *i64=wb[3] as *i64; let Wo: *i64=wb[4] as *i64
653 let gF: *i64=wb[5] as *i64; let Wg: *i64=wb[6] as *i64; let Wu: *i64=wb[7] as *i64; let Wd: *i64=wb[8] as *i64
654 let bq: *i64=wb[9] as *i64; let bk: *i64=wb[10] as *i64; let bv: *i64=wb[11] as *i64
655 let xn: *i64=sb[0] as *i64; let Q: *i64=sb[1] as *i64; let K: *i64=sb[2] as *i64; let V: *i64=sb[3] as *i64; let concat: *i64=sb[4] as *i64
656 let sc: *i64=sb[5] as *i64; let at: *i64=sb[6] as *i64; let proj: *i64=sb[7] as *i64; let gate: *i64=sb[8] as *i64; let up: *i64=sb[9] as *i64
657 let hbuf: *i64=sb[10] as *i64; let hmid: *i64=sb[11] as *i64; let cur: *i64=sb[12] as *i64; let cur2: *i64=sb[13] as *i64
658 cpy(cur, x, T*ne)
659 var L: i64=0
660 while L<N {
661 nf_load_layer_pool(buf, hdr, L, wb, ne, qd, kvd, fd) // parallel weight load (matmuls stay serial: one-time)
662 attn_sublayer(cur, gA, Wq, Wk, Wv, Wo, bq, bk, bv, freqs, xn, Q, K, V, concat, sc, at, proj, hmid, cfgA, 0)
663 // capture this layer's K/V rows (K = post-bias post-RoPE; V = post-bias) into the layer cache
664 let Kc: *i64 = kvc[2*L] as *i64
665 let Vc: *i64 = kvc[2*L+1] as *i64
666 cpy(Kc, K, T*kvd)
667 cpy(Vc, V, T*kvd)
668 ffn_sublayer(hmid, gF, Wg, Wu, Wd, xn, gate, up, hbuf, proj, cur2, cfgF, 0)
669 cpy(cur, cur2, T*ne)
670 L=L+1
671 }
672 cpy(out, cur, T*ne)
673 return 0
674}
675// decode ONE token at absolute position `pos` against the populated caches. x1/out1 = single ne-row. Mirrors the
676// uncached math exactly: RMSNorm(Q24) -> QKV(mm_q24in) -> +bias -> RoPE(pos) -> append cache row -> GQA softmax over
677// rows 0..pos -> W_o -> residual -> ffn_sublayer(T=1).
678func decode_step_kv(buf: *u8, hdr: *NxGgufHeader, x1: *i64, out1: *i64, wb: *i64, sb: *i64, nm: *u8, freqs: *i64, kvc: *i64, pos: i64, cfgA: *i64, cfgF: *i64, N: i64) -> i64 {
679 let ne: i64=cfgA[1]; let nh: i64=cfgA[2]; let nkv: i64=cfgA[3]; let hd: i64=cfgA[4]; let qd: i64=cfgA[5]; let kvd: i64=cfgA[6]; let scale: i64=cfgA[7]; let fd: i64=cfgF[2]
680 let gA: *i64=wb[0] as *i64; let Wq: *i64=wb[1] as *i64; let Wk: *i64=wb[2] as *i64; let Wv: *i64=wb[3] as *i64; let Wo: *i64=wb[4] as *i64
681 let gF: *i64=wb[5] as *i64; let Wg: *i64=wb[6] as *i64; let Wu: *i64=wb[7] as *i64; let Wd: *i64=wb[8] as *i64
682 let bq: *i64=wb[9] as *i64; let bk: *i64=wb[10] as *i64; let bv: *i64=wb[11] as *i64
683 let xn: *i64=sb[0] as *i64; let Q: *i64=sb[1] as *i64; let K: *i64=sb[2] as *i64; let V: *i64=sb[3] as *i64; let concat: *i64=sb[4] as *i64
684 let sc: *i64=sb[5] as *i64; let at: *i64=sb[6] as *i64; let proj: *i64=sb[7] as *i64; let gate: *i64=sb[8] as *i64; let up: *i64=sb[9] as *i64
685 let hbuf: *i64=sb[10] as *i64; let hmid: *i64=sb[11] as *i64; let cur: *i64=sb[12] as *i64; let cur2: *i64=sb[13] as *i64
686 cpy(cur, x1, ne)
687 var L: i64=0
688 while L<N {
689 nf_load_layer_pool(buf, hdr, L, wb, ne, qd, kvd, fd) // all 12 tensors in parallel (disjoint outputs)
690 rmsnorm_gamma_row_q24(cur, gA, 0, ne, xn, 0)
691 mm_pool(xn, Wq, Q, 1, ne, qd, 24)
692 mm_pool(xn, Wk, K, 1, ne, kvd, 24)
693 mm_pool(xn, Wv, V, 1, ne, kvd, 24)
694 var bo: i64=0; while bo<qd { Q[bo]=Q[bo]+bq[bo]; bo=bo+1 }
695 var bp: i64=0; while bp<kvd { K[bp]=K[bp]+bk[bp]; V[bp]=V[bp]+bv[bp]; bp=bp+1 }
696 var h: i64=0; while h<nh { rope_apply(((Q as i64)+(h*hd)*8) as *i64, hd, pos, freqs); h=h+1 }
697 var hk: i64=0; while hk<nkv { rope_apply(((K as i64)+(hk*hd)*8) as *i64, hd, pos, freqs); hk=hk+1 }
698 let Kc: *i64 = kvc[2*L] as *i64
699 let Vc: *i64 = kvc[2*L+1] as *i64
700 var ci: i64=0; while ci<kvd { Kc[pos*kvd+ci]=K[ci]; Vc[pos*kvd+ci]=V[ci]; ci=ci+1 }
701 let group: i64=nh/nkv
702 let cnt: i64=pos+1
703 var hh: i64=0
704 while hh<nh {
705 let kvh: i64=hh/group; let qb: i64=hh*hd; let kb: i64=kvh*hd
706 var s: i64=0
707 while s<cnt { var dot: i64=0; var d: i64=0; while d<hd { dot=dot+(Q[qb+d]*Kc[s*kvd+kb+d]); d=d+1 } sc[s]=qmul(dot>>16, scale); s=s+1 }
708 var mmax: i64=sc[0]; var j: i64=1; while j<cnt { if sc[j]>mmax { mmax=sc[j] } j=j+1 }
709 var sum: i64=0; j=0; while j<cnt { let e: i64=fx_exp(sc[j]-mmax); at[j]=e; sum=sum+e; j=j+1 }
710 if sum<=0 { sum=1 }
711 j=0; while j<cnt { at[j]=(at[j]<<16)/sum; j=j+1 }
712 var d2: i64=0
713 while d2<hd { var acc: i64=0; s=0; while s<cnt { acc=acc+(at[s]*Vc[s*kvd+kb+d2]); s=s+1 } concat[qb+d2]=acc>>16; d2=d2+1 }
714 hh=hh+1
715 }
716 mm_pool(concat, Wo, proj, 1, qd, ne, 16)
717 var ri: i64=0; while ri<ne { hmid[ri]=cur[ri]+proj[ri]; ri=ri+1 }
718 // inlined FFN (T=1) with pooled matmuls -- bit-exact vs ffn_sublayer
719 rmsnorm_gamma_row_q24(hmid, gF, 0, ne, xn, 0)
720 mm_pool(xn, Wg, gate, 1, ne, fd, 24)
721 mm_pool(xn, Wu, up, 1, ne, fd, 24)
722 var fi: i64=0; while fi<fd { hbuf[fi]=qmul(silu(gate[fi]), up[fi]); fi=fi+1 }
723 mm_pool(hbuf, Wd, proj, 1, fd, ne, 16)
724 var fo: i64=0; while fo<ne { cur[fo]=hmid[fo]+proj[fo]; fo=fo+1 }
725 L=L+1
726 }
727 cpy(out1, cur, ne)
728 return 0
729}
730// ==== DEQUANT-ONCE WEIGHT CACHE (2026-07-09) =============================================================
731// MEASURED (dqprobe gate): a decode_step_kv spends ~88% of its time (1213/1363 ms) RE-DEQUANTIZING the
732// frozen weights -- every layer, every token -- even though the weights never change across decode steps.
733// Dequant ONCE into a per-layer cache and reuse => decode is bit-identical (same dequant, just not repeated)
734// but ~9x faster. wcache = N pointers, each a 12-slot wb set (identical layout to nf_load_layer_pool's).
735// ADDITIVE: brand-new functions; decode_step_kv / run_stack untouched (nx_cc has no arity check, so changing
736// a proven signature would let old call sites pass garbage silently).
737func nf_alloc_layer_wb(ne: i64, qd: i64, kvd: i64, fd: i64) -> *i64 {
738 let w: *i64 = sys_mmap(12*8) as *i64
739 w[0]=sys_mmap(ne*8) as i64; w[1]=sys_mmap(qd*ne*8) as i64; w[2]=sys_mmap(kvd*ne*8) as i64; w[3]=sys_mmap(kvd*ne*8) as i64; w[4]=sys_mmap(ne*qd*8) as i64
740 w[5]=sys_mmap(ne*8) as i64; w[6]=sys_mmap(ne*fd*8) as i64; w[7]=sys_mmap(ne*fd*8) as i64; w[8]=sys_mmap(fd*ne*8) as i64
741 w[9]=sys_mmap(qd*8) as i64; w[10]=sys_mmap(kvd*8) as i64; w[11]=sys_mmap(kvd*8) as i64
742 return w
743}
744// dequant ALL N layers ONCE into wcache (wcache[L] = a fresh 12-slot wb set). Reuses the proven
745// nf_load_layer_pool dequant verbatim => bit-identical to the per-step path, done a single time.
746func nf_dequant_all_layers(buf: *u8, hdr: *NxGgufHeader, wcache: *i64, N: i64, ne: i64, qd: i64, kvd: i64, fd: i64) -> i64 {
747 var L: i64=0
748 while L<N {
749 let w: *i64 = nf_alloc_layer_wb(ne, qd, kvd, fd)
750 nf_load_layer_pool(buf, hdr, L, w, ne, qd, kvd, fd)
751 wcache[L]=w as i64
752 L=L+1
753 }
754 return 0
755}
756// decode ONE token using the pre-dequantized wcache -> NO per-step dequant. Byte-for-byte the same math as
757// decode_step_kv (same weights, same accumulation order); only the redundant re-dequant is removed. The
758// weight pointers are derived per-layer INSIDE the loop from wcache[L] (each layer has its own buffers).
759func decode_step_kv_cached(buf: *u8, hdr: *NxGgufHeader, x1: *i64, out1: *i64, wcache: *i64, sb: *i64, nm: *u8, freqs: *i64, kvc: *i64, pos: i64, cfgA: *i64, cfgF: *i64, N: i64) -> i64 {
760 let ne: i64=cfgA[1]; let nh: i64=cfgA[2]; let nkv: i64=cfgA[3]; let hd: i64=cfgA[4]; let qd: i64=cfgA[5]; let kvd: i64=cfgA[6]; let scale: i64=cfgA[7]; let fd: i64=cfgF[2]
761 let xn: *i64=sb[0] as *i64; let Q: *i64=sb[1] as *i64; let K: *i64=sb[2] as *i64; let V: *i64=sb[3] as *i64; let concat: *i64=sb[4] as *i64
762 let sc: *i64=sb[5] as *i64; let at: *i64=sb[6] as *i64; let proj: *i64=sb[7] as *i64; let gate: *i64=sb[8] as *i64; let up: *i64=sb[9] as *i64
763 let hbuf: *i64=sb[10] as *i64; let hmid: *i64=sb[11] as *i64; let cur: *i64=sb[12] as *i64
764 cpy(cur, x1, ne)
765 var L: i64=0
766 while L<N {
767 let wb: *i64 = wcache[L] as *i64
768 let gA: *i64=wb[0] as *i64; let Wq: *i64=wb[1] as *i64; let Wk: *i64=wb[2] as *i64; let Wv: *i64=wb[3] as *i64; let Wo: *i64=wb[4] as *i64
769 let gF: *i64=wb[5] as *i64; let Wg: *i64=wb[6] as *i64; let Wu: *i64=wb[7] as *i64; let Wd: *i64=wb[8] as *i64
770 let bq: *i64=wb[9] as *i64; let bk: *i64=wb[10] as *i64; let bv: *i64=wb[11] as *i64
771 rmsnorm_gamma_row_q24(cur, gA, 0, ne, xn, 0)
772 mm_pool(xn, Wq, Q, 1, ne, qd, 24)
773 mm_pool(xn, Wk, K, 1, ne, kvd, 24)
774 mm_pool(xn, Wv, V, 1, ne, kvd, 24)
775 var bo: i64=0; while bo<qd { Q[bo]=Q[bo]+bq[bo]; bo=bo+1 }
776 var bp: i64=0; while bp<kvd { K[bp]=K[bp]+bk[bp]; V[bp]=V[bp]+bv[bp]; bp=bp+1 }
777 var h: i64=0; while h<nh { rope_apply(((Q as i64)+(h*hd)*8) as *i64, hd, pos, freqs); h=h+1 }
778 var hk: i64=0; while hk<nkv { rope_apply(((K as i64)+(hk*hd)*8) as *i64, hd, pos, freqs); hk=hk+1 }
779 let Kc: *i64 = kvc[2*L] as *i64
780 let Vc: *i64 = kvc[2*L+1] as *i64
781 var ci: i64=0; while ci<kvd { Kc[pos*kvd+ci]=K[ci]; Vc[pos*kvd+ci]=V[ci]; ci=ci+1 }
782 let group: i64=nh/nkv
783 let cnt: i64=pos+1
784 var hh: i64=0
785 while hh<nh {
786 let kvh: i64=hh/group; let qb: i64=hh*hd; let kb: i64=kvh*hd
787 var s: i64=0
788 while s<cnt { var dot: i64=0; var d: i64=0; while d<hd { dot=dot+(Q[qb+d]*Kc[s*kvd+kb+d]); d=d+1 } sc[s]=qmul(dot>>16, scale); s=s+1 }
789 var mmax: i64=sc[0]; var j: i64=1; while j<cnt { if sc[j]>mmax { mmax=sc[j] } j=j+1 }
790 var sum: i64=0; j=0; while j<cnt { let e: i64=fx_exp(sc[j]-mmax); at[j]=e; sum=sum+e; j=j+1 }
791 if sum<=0 { sum=1 }
792 j=0; while j<cnt { at[j]=(at[j]<<16)/sum; j=j+1 }
793 var d2: i64=0
794 while d2<hd { var acc: i64=0; s=0; while s<cnt { acc=acc+(at[s]*Vc[s*kvd+kb+d2]); s=s+1 } concat[qb+d2]=acc>>16; d2=d2+1 }
795 hh=hh+1
796 }
797 mm_pool(concat, Wo, proj, 1, qd, ne, 16)
798 var ri: i64=0; while ri<ne { hmid[ri]=cur[ri]+proj[ri]; ri=ri+1 }
799 rmsnorm_gamma_row_q24(hmid, gF, 0, ne, xn, 0)
800 mm_pool(xn, Wg, gate, 1, ne, fd, 24)
801 mm_pool(xn, Wu, up, 1, ne, fd, 24)
802 var fi: i64=0; while fi<fd { hbuf[fi]=qmul(silu(gate[fi]), up[fi]); fi=fi+1 }
803 mm_pool(hbuf, Wd, proj, 1, fd, ne, 16)
804 var fo: i64=0; while fo<ne { cur[fo]=hmid[fo]+proj[fo]; fo=fo+1 }
805 L=L+1
806 }
807 cpy(out1, cur, ne)
808 return 0
809}
810// ==== DEQUANT-ONCE OUTPUT-HEAD CACHE ====================================================================
811// head_argmax_pool re-dequantizes ALL `vocab` head rows every token (~650 ms of the per-token cost). Dequant
812// the whole [vocab x ne] head ONCE into an i64 cache; then argmax is a pure pooled dot -> LOSSLESS + flat.
813func _nfhdq_task(ctx_i: i64) -> i64 {
814 let c: *i64 = ctx_i as *i64
815 let buf: *u8=c[0] as *u8; let base: i64=c[1]; let ty: i64=c[2]; let cache: *i64=c[3] as *i64
816 let ne: i64=c[4]; let lo: i64=c[5]; let hi: i64=c[6]; let tmp: *i64=c[7] as *i64
817 var v: i64=lo
818 while v<hi { dequant_row(buf, base, ty, v, ne, ((cache as i64)+v*ne*8) as *i64, tmp); v=v+1 }
819 return 0
820}
821// dequant the full output head ONCE (banded over vocab). Returns the [vocab x ne] i64 cache base (0 on OOM).
822func nf_dequant_head_all(buf: *u8, oh_base: i64, oh_ty: i64, vocab: i64, ne: i64) -> *i64 {
823 let p: *NxThreadPool = nf_pool()
824 let cache: *i64 = sys_mmap(vocab*ne*8) as *i64
825 if (cache as i64)==0 { return cache }
826 var bands: i64 = p.n_workers
827 if bands > 32 { bands = 32 }
828 if bands > vocab { bands = vocab }
829 if bands < 1 { bands = 1 }
830 let per: i64 = (vocab + bands - 1) / bands
831 let done0: i64 = nx_pool_n_completed(p)
832 var b: i64 = 0
833 while b < bands {
834 let c: *i64 = ((g_nf_hdctx as i64) + b*128) as *i64
835 c[0]=buf as i64; c[1]=oh_base; c[2]=oh_ty; c[3]=cache as i64; c[4]=ne; c[5]=b*per
836 var hi: i64=(b+1)*per
837 if hi>vocab { hi=vocab }
838 c[6]=hi
839 c[7]=((g_nf_hdtmp as i64) + b*32768)
840 nx_pool_submit(p, _nfhdq_task, c as i64)
841 b=b+1
842 }
843 nx_pool_wait(p, done0+bands)
844 return cache
845}
846// one cached-head argmax band: dot normed(Q24) against pre-dequantized rows [lo,hi); strict-> keeps lowest id.
847func _nfhac_task(ctx_i: i64) -> i64 {
848 let c: *i64 = ctx_i as *i64
849 let cache: *i64=c[0] as *i64; let normed: *i64=c[1] as *i64; let ne: i64=c[2]; let lo: i64=c[3]; let hi: i64=c[4]; let outp: *i64=c[5] as *i64
850 var best: i64=lo
851 var bestv: i64=0-9223372036854775807
852 var v: i64=lo
853 while v<hi {
854 let row: *i64=((cache as i64)+v*ne*8) as *i64
855 var s: i64=0; var k: i64=0; while k<ne { s=s+(normed[k]*row[k]); k=k+1 }
856 let lg: i64=s>>24
857 if lg>bestv { bestv=lg; best=v }
858 v=v+1
859 }
860 outp[0]=best; outp[1]=bestv
861 return 0
862}
863// pooled argmax over the PRE-DEQUANTIZED head cache. hcp = [cache, normed(Q24), vocab, ne, idout, lgout].
864// Band-ascending strict-> reduce keeps the lowest id on ties = bit-identical to head_argmax_pool.
865func head_argmax_cached(hcp: *i64) -> i64 {
866 let p: *NxThreadPool = nf_pool()
867 let cache: i64=hcp[0]; let normed: i64=hcp[1]; let vocab: i64=hcp[2]; let ne: i64=hcp[3]
868 var bands: i64 = p.n_workers
869 if bands > 32 { bands = 32 }
870 if bands > vocab { bands = vocab }
871 if bands < 1 { bands = 1 }
872 let per: i64 = (vocab + bands - 1) / bands
873 let done0: i64 = nx_pool_n_completed(p)
874 var b: i64 = 0
875 while b < bands {
876 let c: *i64 = ((g_nf_hdctx as i64) + b*128) as *i64
877 c[0]=cache; c[1]=normed; c[2]=ne; c[3]=b*per
878 var hi: i64=(b+1)*per
879 if hi>vocab { hi=vocab }
880 c[4]=hi
881 c[5]=((g_nf_hdout as i64) + b*16)
882 nx_pool_submit(p, _nfhac_task, c as i64)
883 b=b+1
884 }
885 nx_pool_wait(p, done0+bands)
886 var best: i64=0
887 var bestv: i64=0-9223372036854775807
888 var rb: i64=0
889 while rb<bands {
890 let o: *i64=((g_nf_hdout as i64) + rb*16) as *i64
891 if o[1]>bestv { bestv=o[1]; best=o[0] }
892 rb=rb+1
893 }
894 let oid: *i64=hcp[4] as *i64
895 let olg: *i64=hcp[5] as *i64
896 oid[0]=best; olg[0]=bestv
897 return best
898}
899// ==== i32 COMPACT CACHE (2026-07-09) =====================================================================
900// MEASURED (fastgen gate): after killing re-dequant, the decode residual is MEMORY BANDWIDTH -- 2.73 GB of i64
901// weights read per token vs a 150 ms compute floor. Q16 weight values are real*65536 and |real|<32768 always,
902// so they fit EXACTLY in i32 -> halving the cache (2.73->1.43 GB) is LOSSLESS. *i32 loads SIGN-extend natively
903// since the 2026-07-10 sext compiler fix (the hot loops used to carry a manual branchless sign-extend; removed
904// in lockstep with the bless). The narrow pass counts any value that would NOT fit i32 -> nonzero = NOT lossless.
905func nf_narrow_i32(src: *i64, dst: *i32, cnt: i64) -> i64 {
906 var ovf: i64=0
907 var i: i64=0
908 while i<cnt {
909 let v: i64=src[i]
910 if v > 2147483647 { ovf=ovf+1 }
911 if v < (0-2147483648) { ovf=ovf+1 }
912 dst[i]=v as i32
913 i=i+1
914 }
915 return ovf
916}
917// one banded-matmul task with i32 weights (sign-extended on load). inp/dst stay i64. Bit-identical to _nfmm_task.
918func _nfmm_i32_task(ctx_i: i64) -> i64 {
919 let c: *i64 = ctx_i as *i64
920 let inp: *i64=c[0] as *i64; let W: *i32=c[1] as *i32; let dst: *i64=c[2] as *i64
921 let T: i64=c[3]; let ind: i64=c[4]; let outd: i64=c[5]; let lo: i64=c[6]; let hi: i64=c[7]; let sh: i64=c[8]
922 var t: i64=0
923 while t<T { var o: i64=lo
924 while o<hi { var s: i64=0; var k: i64=0
925 while k<ind { s=s+(inp[t*ind+k]*(W[o*ind+k] as i64)); k=k+1 } // *i32 sign-extends natively (2026-07-10 sext fix)
926 if sh==24 { dst[t*outd+o]=s>>24 } else { dst[t*outd+o]=s>>16 }
927 o=o+1 }
928 t=t+1 }
929 return 0
930}
931// pooled i32-weight matmul: bit-identical to mm_pool (same accumulation order, sign-extended i32 == the i64 value).
932func mm_pool_i32(inp: *i64, W: *i32, dst: *i64, T: i64, in_dim: i64, out_dim: i64, shift: i64) -> i64 {
933 let p: *NxThreadPool = nf_pool()
934 var bands: i64 = p.n_workers
935 if bands > 32 { bands = 32 }
936 if bands > out_dim { bands = out_dim }
937 if bands < 1 { bands = 1 }
938 let per: i64 = (out_dim + bands - 1) / bands
939 let done0: i64 = nx_pool_n_completed(p)
940 var b: i64 = 0
941 while b < bands {
942 let c: *i64 = ((g_nf_mmarena as i64) + b*128) as *i64
943 c[0]=inp as i64; c[1]=W as i64; c[2]=dst as i64; c[3]=T; c[4]=in_dim; c[5]=out_dim; c[6]=b*per
944 var hi: i64=(b+1)*per
945 if hi>out_dim { hi=out_dim }
946 c[7]=hi; c[8]=shift
947 nx_pool_submit(p, _nfmm_i32_task, c as i64)
948 b=b+1
949 }
950 nx_pool_wait(p, done0+bands)
951 return 0
952}
953// one i32 weight set: mm weights (idx 1,2,3,4,6,7,8) are i32 (4B); norms (0,5) + biases (9,10,11) stay i64.
954func nf_alloc_layer_wb_i32(ne: i64, qd: i64, kvd: i64, fd: i64) -> *i64 {
955 let w: *i64 = sys_mmap(12*8) as *i64
956 w[0]=sys_mmap(ne*8) as i64; w[1]=sys_mmap(qd*ne*4) as i64; w[2]=sys_mmap(kvd*ne*4) as i64; w[3]=sys_mmap(kvd*ne*4) as i64; w[4]=sys_mmap(ne*qd*4) as i64
957 w[5]=sys_mmap(ne*8) as i64; w[6]=sys_mmap(ne*fd*4) as i64; w[7]=sys_mmap(ne*fd*4) as i64; w[8]=sys_mmap(fd*ne*4) as i64
958 w[9]=sys_mmap(qd*8) as i64; w[10]=sys_mmap(kvd*8) as i64; w[11]=sys_mmap(kvd*8) as i64
959 return w
960}
961// dequant all N layers to a reused i64 scratch, narrow the 7 mm weights to i32 into wc32[L]. Returns total
962// overflow count across all layers -- MUST be 0 for the i32 cache to be lossless (gate asserts it).
963func nf_dequant_all_layers_i32(buf: *u8, hdr: *NxGgufHeader, wc32: *i64, N: i64, ne: i64, qd: i64, kvd: i64, fd: i64) -> i64 {
964 let tmp: *i64 = nf_alloc_layer_wb(ne, qd, kvd, fd)
965 var ovf: i64=0
966 var L: i64=0
967 while L<N {
968 nf_load_layer_pool(buf, hdr, L, tmp, ne, qd, kvd, fd)
969 let s: *i64 = nf_alloc_layer_wb_i32(ne, qd, kvd, fd)
970 cpy(s[0] as *i64, tmp[0] as *i64, ne)
971 ovf=ovf+nf_narrow_i32(tmp[1] as *i64, s[1] as *i32, qd*ne)
972 ovf=ovf+nf_narrow_i32(tmp[2] as *i64, s[2] as *i32, kvd*ne)
973 ovf=ovf+nf_narrow_i32(tmp[3] as *i64, s[3] as *i32, kvd*ne)
974 ovf=ovf+nf_narrow_i32(tmp[4] as *i64, s[4] as *i32, ne*qd)
975 cpy(s[5] as *i64, tmp[5] as *i64, ne)
976 ovf=ovf+nf_narrow_i32(tmp[6] as *i64, s[6] as *i32, ne*fd)
977 ovf=ovf+nf_narrow_i32(tmp[7] as *i64, s[7] as *i32, ne*fd)
978 ovf=ovf+nf_narrow_i32(tmp[8] as *i64, s[8] as *i32, fd*ne)
979 cpy(s[9] as *i64, tmp[9] as *i64, qd)
980 cpy(s[10] as *i64, tmp[10] as *i64, kvd)
981 cpy(s[11] as *i64, tmp[11] as *i64, kvd)
982 wc32[L]=s as i64
983 L=L+1
984 }
985 return ovf
986}
987// decode ONE token using the i32 compact cache. Byte-for-byte the same math as decode_step_kv_cached (the
988// sign-extended i32 weight == the i64 value it was narrowed from), just half the weight bandwidth.
989func decode_step_kv_cached_i32(buf: *u8, hdr: *NxGgufHeader, x1: *i64, out1: *i64, wcache: *i64, sb: *i64, nm: *u8, freqs: *i64, kvc: *i64, pos: i64, cfgA: *i64, cfgF: *i64, N: i64) -> i64 {
990 let ne: i64=cfgA[1]; let nh: i64=cfgA[2]; let nkv: i64=cfgA[3]; let hd: i64=cfgA[4]; let qd: i64=cfgA[5]; let kvd: i64=cfgA[6]; let scale: i64=cfgA[7]; let fd: i64=cfgF[2]
991 let xn: *i64=sb[0] as *i64; let Q: *i64=sb[1] as *i64; let K: *i64=sb[2] as *i64; let V: *i64=sb[3] as *i64; let concat: *i64=sb[4] as *i64
992 let sc: *i64=sb[5] as *i64; let at: *i64=sb[6] as *i64; let proj: *i64=sb[7] as *i64; let gate: *i64=sb[8] as *i64; let up: *i64=sb[9] as *i64
993 let hbuf: *i64=sb[10] as *i64; let hmid: *i64=sb[11] as *i64; let cur: *i64=sb[12] as *i64
994 cpy(cur, x1, ne)
995 var L: i64=0
996 while L<N {
997 let wb: *i64 = wcache[L] as *i64
998 let gA: *i64=wb[0] as *i64
999 let Wq: *i32=wb[1] as *i32; let Wk: *i32=wb[2] as *i32; let Wv: *i32=wb[3] as *i32; let Wo: *i32=wb[4] as *i32
1000 let gF: *i64=wb[5] as *i64
1001 let Wg: *i32=wb[6] as *i32; let Wu: *i32=wb[7] as *i32; let Wd: *i32=wb[8] as *i32
1002 let bq: *i64=wb[9] as *i64; let bk: *i64=wb[10] as *i64; let bv: *i64=wb[11] as *i64
1003 rmsnorm_gamma_row_q24(cur, gA, 0, ne, xn, 0)
1004 mm_pool_i32(xn, Wq, Q, 1, ne, qd, 24)
1005 mm_pool_i32(xn, Wk, K, 1, ne, kvd, 24)
1006 mm_pool_i32(xn, Wv, V, 1, ne, kvd, 24)
1007 var bo: i64=0; while bo<qd { Q[bo]=Q[bo]+bq[bo]; bo=bo+1 }
1008 var bp: i64=0; while bp<kvd { K[bp]=K[bp]+bk[bp]; V[bp]=V[bp]+bv[bp]; bp=bp+1 }
1009 var h: i64=0; while h<nh { rope_apply(((Q as i64)+(h*hd)*8) as *i64, hd, pos, freqs); h=h+1 }
1010 var hk: i64=0; while hk<nkv { rope_apply(((K as i64)+(hk*hd)*8) as *i64, hd, pos, freqs); hk=hk+1 }
1011 let Kc: *i64 = kvc[2*L] as *i64
1012 let Vc: *i64 = kvc[2*L+1] as *i64
1013 var ci: i64=0; while ci<kvd { Kc[pos*kvd+ci]=K[ci]; Vc[pos*kvd+ci]=V[ci]; ci=ci+1 }
1014 let group: i64=nh/nkv
1015 let cnt: i64=pos+1
1016 var hh: i64=0
1017 while hh<nh {
1018 let kvh: i64=hh/group; let qb: i64=hh*hd; let kb: i64=kvh*hd
1019 var s: i64=0
1020 while s<cnt { var dot: i64=0; var d: i64=0; while d<hd { dot=dot+(Q[qb+d]*Kc[s*kvd+kb+d]); d=d+1 } sc[s]=qmul(dot>>16, scale); s=s+1 }
1021 var mmax: i64=sc[0]; var j: i64=1; while j<cnt { if sc[j]>mmax { mmax=sc[j] } j=j+1 }
1022 var sum: i64=0; j=0; while j<cnt { let e: i64=fx_exp(sc[j]-mmax); at[j]=e; sum=sum+e; j=j+1 }
1023 if sum<=0 { sum=1 }
1024 j=0; while j<cnt { at[j]=(at[j]<<16)/sum; j=j+1 }
1025 var d2: i64=0
1026 while d2<hd { var acc: i64=0; s=0; while s<cnt { acc=acc+(at[s]*Vc[s*kvd+kb+d2]); s=s+1 } concat[qb+d2]=acc>>16; d2=d2+1 }
1027 hh=hh+1
1028 }
1029 mm_pool_i32(concat, Wo, proj, 1, qd, ne, 16)
1030 var ri: i64=0; while ri<ne { hmid[ri]=cur[ri]+proj[ri]; ri=ri+1 }
1031 rmsnorm_gamma_row_q24(hmid, gF, 0, ne, xn, 0)
1032 mm_pool_i32(xn, Wg, gate, 1, ne, fd, 24)
1033 mm_pool_i32(xn, Wu, up, 1, ne, fd, 24)
1034 var fi: i64=0; while fi<fd { hbuf[fi]=qmul(silu(gate[fi]), up[fi]); fi=fi+1 }
1035 mm_pool_i32(hbuf, Wd, proj, 1, fd, ne, 16)
1036 var fo: i64=0; while fo<ne { cur[fo]=hmid[fo]+proj[fo]; fo=fo+1 }
1037 L=L+1
1038 }
1039 cpy(out1, cur, ne)
1040 return 0
1041}
1042// ---- i32 output-head cache (half the head bandwidth, lossless) ----
1043func _nfhdq_i32_task(ctx_i: i64) -> i64 {
1044 let c: *i64 = ctx_i as *i64
1045 let buf: *u8=c[0] as *u8; let base: i64=c[1]; let ty: i64=c[2]; let cache: *i32=c[3] as *i32
1046 let ne: i64=c[4]; let lo: i64=c[5]; let hi: i64=c[6]; let tmp: *i64=c[7] as *i64; let row: *i64=c[8] as *i64
1047 var v: i64=lo
1048 while v<hi { dequant_row(buf, base, ty, v, ne, row, tmp); var k: i64=0; while k<ne { cache[v*ne+k]=(row[k]) as i32; k=k+1 } v=v+1 }
1049 return 0
1050}
1051func nf_dequant_head_all_i32(buf: *u8, oh_base: i64, oh_ty: i64, vocab: i64, ne: i64) -> *i32 {
1052 let p: *NxThreadPool = nf_pool()
1053 let cache: *i32 = sys_mmap(vocab*ne*4) as *i32
1054 if (cache as i64)==0 { return cache }
1055 var bands: i64 = p.n_workers
1056 if bands > 32 { bands = 32 }
1057 if bands > vocab { bands = vocab }
1058 if bands < 1 { bands = 1 }
1059 let per: i64 = (vocab + bands - 1) / bands
1060 // ROOT FIX 2026-07-15 (the run-to-run greedy divergence, proven by exact-repeat 1.5B runs): the old
1061 // shared arena gave each band a FIXED 8192B row (1024 i64). ne=1536 (Qwen2.5-1.5B) dequants 12288B ->
1062 // band b OVERFLOWED 4KB into band b+1's row MID-COPY -> timing-dependent corrupt cache rows, frozen
1063 // for the run (within-run determinism held; across runs diverged at near-tie logits). Per-call
1064 // DIMS-SIZED scratch: row stride ne*8, tmp stride ne*8+4096 (covers every quant type's block window
1065 // at any ne). Freed after the wait (all band tasks complete by then).
1066 let rstride: i64 = ne*8
1067 let tstride: i64 = ne*8 + 4096
1068 let rows: *u8 = sys_mmap(bands*rstride)
1069 let tmps: *u8 = sys_mmap(bands*tstride)
1070 let done0: i64 = nx_pool_n_completed(p)
1071 var b: i64 = 0
1072 while b < bands {
1073 let c: *i64 = ((g_nf_hdctx as i64) + b*128) as *i64
1074 c[0]=buf as i64; c[1]=oh_base; c[2]=oh_ty; c[3]=cache as i64; c[4]=ne; c[5]=b*per
1075 var hi: i64=(b+1)*per
1076 if hi>vocab { hi=vocab }
1077 c[6]=hi
1078 c[7]=((tmps as i64) + b*tstride)
1079 c[8]=((rows as i64) + b*rstride)
1080 nx_pool_submit(p, _nfhdq_i32_task, c as i64)
1081 b=b+1
1082 }
1083 nx_pool_wait(p, done0+bands)
1084 sys_munmap(rows, bands*rstride)
1085 sys_munmap(tmps, bands*tstride)
1086 return cache
1087}
1088func _nfhac_i32_task(ctx_i: i64) -> i64 {
1089 let c: *i64 = ctx_i as *i64
1090 let cache: *i32=c[0] as *i32; let normed: *i64=c[1] as *i64; let ne: i64=c[2]; let lo: i64=c[3]; let hi: i64=c[4]; let outp: *i64=c[5] as *i64
1091 var best: i64=lo
1092 var bestv: i64=0-9223372036854775807
1093 var v: i64=lo
1094 while v<hi {
1095 var s: i64=0; var k: i64=0
1096 while k<ne { s=s+(normed[k]*(cache[v*ne+k] as i64)); k=k+1 } // *i32 sign-extends natively (2026-07-10 sext fix)
1097 let lg: i64=s>>24
1098 if lg>bestv { bestv=lg; best=v }
1099 v=v+1
1100 }
1101 outp[0]=best; outp[1]=bestv
1102 return 0
1103}
1104// one full-logits band over the i32 head cache: writes lgv[v] = dot(normed, row_v) >> 24 for rows [lo,hi).
1105// Bands write DISJOINT ranges of lgv -> deterministic regardless of thread count (same discipline as the
1106// banded matmuls). Added 2026-07-10 for SAMPLING (temperature/top-p needs ALL logits, not just the argmax).
1107func _nfhlg_task(ctx_i: i64) -> i64 {
1108 let c: *i64 = ctx_i as *i64
1109 let cache: *i32=c[0] as *i32; let normed: *i64=c[1] as *i64; let ne: i64=c[2]; let lo: i64=c[3]; let hi: i64=c[4]; let lgv: *i64=c[5] as *i64
1110 var v: i64=lo
1111 while v<hi {
1112 var s: i64=0; var k: i64=0
1113 while k<ne { s=s+(normed[k]*(cache[v*ne+k] as i64)); k=k+1 }
1114 lgv[v]=s>>24
1115 v=v+1
1116 }
1117 return 0
1118}
1119// pooled FULL-logits head over the i32 cache. hlp = [cache(i32), normed(Q24), vocab, ne, lgv(*i64 vocab)].
1120// Same dots as head_argmax_cached_i32 (Q16 head convention), all of them kept.
1121func head_logits_cached_i32(hlp: *i64) -> i64 {
1122 let p: *NxThreadPool = nf_pool()
1123 let vocab: i64=hlp[2]
1124 var bands: i64 = p.n_workers
1125 if bands > 32 { bands = 32 }
1126 if bands > vocab { bands = vocab }
1127 if bands < 1 { bands = 1 }
1128 let per: i64 = (vocab + bands - 1) / bands
1129 let done0: i64 = nx_pool_n_completed(p)
1130 var b: i64 = 0
1131 while b < bands {
1132 let c: *i64 = ((g_nf_hdctx as i64) + b*128) as *i64
1133 c[0]=hlp[0]; c[1]=hlp[1]; c[2]=hlp[3]; c[3]=b*per
1134 var hi: i64=(b+1)*per
1135 if hi>vocab { hi=vocab }
1136 c[4]=hi
1137 c[5]=hlp[4]
1138 nx_pool_submit(p, _nfhlg_task, c as i64)
1139 b=b+1
1140 }
1141 nx_pool_wait(p, done0+bands)
1142 return 0
1143}
1144// argmax over the i32 head cache. hcp = [cache(i32), normed(Q24), vocab, ne, idout, lgout]. Bit-identical.
1145func head_argmax_cached_i32(hcp: *i64) -> i64 {
1146 let p: *NxThreadPool = nf_pool()
1147 let cache: i64=hcp[0]; let normed: i64=hcp[1]; let vocab: i64=hcp[2]; let ne: i64=hcp[3]
1148 var bands: i64 = p.n_workers
1149 if bands > 32 { bands = 32 }
1150 if bands > vocab { bands = vocab }
1151 if bands < 1 { bands = 1 }
1152 let per: i64 = (vocab + bands - 1) / bands
1153 let done0: i64 = nx_pool_n_completed(p)
1154 var b: i64 = 0
1155 while b < bands {
1156 let c: *i64 = ((g_nf_hdctx as i64) + b*128) as *i64
1157 c[0]=cache; c[1]=normed; c[2]=ne; c[3]=b*per
1158 var hi: i64=(b+1)*per
1159 if hi>vocab { hi=vocab }
1160 c[4]=hi
1161 c[5]=((g_nf_hdout as i64) + b*16)
1162 nx_pool_submit(p, _nfhac_i32_task, c as i64)
1163 b=b+1
1164 }
1165 nx_pool_wait(p, done0+bands)
1166 var best: i64=0
1167 var bestv: i64=0-9223372036854775807
1168 var rb: i64=0
1169 while rb<bands {
1170 let o: *i64=((g_nf_hdout as i64) + rb*16) as *i64
1171 if o[1]>bestv { bestv=o[1]; best=o[0] }
1172 rb=rb+1
1173 }
1174 let oid: *i64=hcp[4] as *i64
1175 let olg: *i64=hcp[5] as *i64
1176 oid[0]=best; olg[0]=bestv
1177 return best
1178}
1179// ==== W8A8 SIMD DECODE PATH (Stage 2, 2026-07-09) ========================================================
1180// PROBE-PROVEN (nx_nofloat_simd_dot_probe): __i16x16_madd (vpmaddwd) gives a 13x matmul over the scalar i64
1181// dot, at ~0.8% per-dot quantization error. LOSSY fast path (NOT bit-exact) -> ADDITIVE alongside the lossless
1182// i32 hero; the gate MEASURES whether real tokens survive. i16 lanes are *u8 buffers with manual LE nf_pack2
1183// (nx_cc has no i16 type). int8-range values keep the int32 madd accumulator overflow-safe for our K (<=304
1184// madds * ~32k < 2.1e9). Weight quant is amortized ONCE at cache build; only the activation is quantized per matmul.
1185func nf_pack2(buf: *u8, idx: i64, val: i64) -> i64 { buf[idx*2]=(val) as u8; buf[idx*2+1]=(val>>8) as u8; return 0 }
1186func nf_quant_w_i8(Wsrc: *i64, Wi8: *u8, sw: *i64, out_dim: i64, in_dim: i64) -> i64 {
1187 var o: i64=0
1188 while o<out_dim {
1189 var rm: i64=0; var k: i64=0
1190 while k<in_dim { var a: i64=Wsrc[o*in_dim+k]; if a<0 { a=0-a } if a>rm { rm=a } k=k+1 }
1191 var s: i64=rm/127; if s<1 { s=1 }
1192 sw[o]=s
1193 let rowb: i64=(Wi8 as i64)+o*in_dim*2
1194 k=0; while k<in_dim { nf_pack2(rowb as *u8, k, Wsrc[o*in_dim+k]/s); k=k+1 }
1195 o=o+1
1196 }
1197 return 0
1198}
1199func nf_alloc_layer_wb_i8(ne: i64, qd: i64, kvd: i64, fd: i64) -> *i64 {
1200 let w: *i64 = sys_mmap(19*8) as *i64
1201 w[0]=sys_mmap(ne*8) as i64
1202 w[1]=sys_mmap(qd*ne*2) as i64; w[2]=sys_mmap(qd*8) as i64
1203 w[3]=sys_mmap(kvd*ne*2) as i64; w[4]=sys_mmap(kvd*8) as i64
1204 w[5]=sys_mmap(kvd*ne*2) as i64; w[6]=sys_mmap(kvd*8) as i64
1205 w[7]=sys_mmap(ne*qd*2) as i64; w[8]=sys_mmap(ne*8) as i64
1206 w[9]=sys_mmap(ne*8) as i64
1207 w[10]=sys_mmap(ne*fd*2) as i64; w[11]=sys_mmap(fd*8) as i64
1208 w[12]=sys_mmap(ne*fd*2) as i64; w[13]=sys_mmap(fd*8) as i64
1209 w[14]=sys_mmap(fd*ne*2) as i64; w[15]=sys_mmap(ne*8) as i64
1210 w[16]=sys_mmap(qd*8) as i64; w[17]=sys_mmap(kvd*8) as i64; w[18]=sys_mmap(kvd*8) as i64
1211 return w
1212}
1213func nf_dequant_all_layers_i8(buf: *u8, hdr: *NxGgufHeader, wc8: *i64, N: i64, ne: i64, qd: i64, kvd: i64, fd: i64) -> i64 {
1214 let tmp: *i64 = nf_alloc_layer_wb(ne, qd, kvd, fd)
1215 var L: i64=0
1216 while L<N {
1217 nf_load_layer_pool(buf, hdr, L, tmp, ne, qd, kvd, fd)
1218 let s: *i64 = nf_alloc_layer_wb_i8(ne, qd, kvd, fd)
1219 cpy(s[0] as *i64, tmp[0] as *i64, ne)
1220 nf_quant_w_i8(tmp[1] as *i64, s[1] as *u8, s[2] as *i64, qd, ne)
1221 nf_quant_w_i8(tmp[2] as *i64, s[3] as *u8, s[4] as *i64, kvd, ne)
1222 nf_quant_w_i8(tmp[3] as *i64, s[5] as *u8, s[6] as *i64, kvd, ne)
1223 nf_quant_w_i8(tmp[4] as *i64, s[7] as *u8, s[8] as *i64, ne, qd)
1224 cpy(s[9] as *i64, tmp[5] as *i64, ne)
1225 nf_quant_w_i8(tmp[6] as *i64, s[10] as *u8, s[11] as *i64, fd, ne)
1226 nf_quant_w_i8(tmp[7] as *i64, s[12] as *u8, s[13] as *i64, fd, ne)
1227 nf_quant_w_i8(tmp[8] as *i64, s[14] as *u8, s[15] as *i64, ne, fd)
1228 cpy(s[16] as *i64, tmp[9] as *i64, qd)
1229 cpy(s[17] as *i64, tmp[10] as *i64, kvd)
1230 cpy(s[18] as *i64, tmp[11] as *i64, kvd)
1231 wc8[L]=s as i64
1232 L=L+1
1233 }
1234 return 0
1235}
1236func _nfmm_i8_task(ctx_i: i64) -> i64 {
1237 let c: *i64 = ctx_i as *i64
1238 let xi8: i64=c[0]; let sx: i64=c[1]; let Wi8: i64=c[2]; let sw: *i64=c[3] as *i64; let dst: *i64=c[4] as *i64
1239 let ind: i64=c[5]; let lo: i64=c[6]; let hi: i64=c[7]; let sh: i64=c[8]; let acc: *u8=c[9] as *u8
1240 var o: i64=lo
1241 while o<hi {
1242 let z: *i64 = acc as *i64; z[0]=0; z[1]=0; z[2]=0; z[3]=0
1243 let wb: i64 = Wi8 + o*ind*2
1244 var k: i64=0
1245 while k<ind { __i16x16_madd(acc, (xi8+k*2) as *u8, (wb+k*2) as *u8); k=k+16 }
1246 dst[o]=((sx*sw[o])*nf_hsum_sx(acc))>>sh
1247 o=o+1
1248 }
1249 return 0
1250}
1251// pooled W8A8 SIMD matmul (T=1 decode). Quantizes the single activation row once, bands the output rows.
1252// Banded by whole output row -> thread-count-invariant. in_dim MUST be a multiple of 16 (896/128/4864 all are).
1253func mm_pool_i8(x: *i64, Wi8: *u8, sw: *i64, dst: *i64, in_dim: i64, out_dim: i64, shift: i64) -> i64 {
1254 let p: *NxThreadPool = nf_pool()
1255 var xmx: i64=0; var k: i64=0
1256 while k<in_dim { var a: i64=x[k]; if a<0 { a=0-a } if a>xmx { xmx=a } k=k+1 }
1257 var sx: i64=xmx/127; if sx<1 { sx=1 }
1258 // ROOT-FIX CLASS 2026-07-15: g_nf_xi8 was a FIXED 4864x2 arena (the 0.5B's ffn) -- in_dim=8960
1259 // (Qwen2.5-1.5B ffn) would overflow it. Per-call dims-sized; freed after the wait.
1260 let xi8: *u8 = sys_mmap(in_dim*2)
1261 k=0; while k<in_dim { nf_pack2(xi8, k, x[k]/sx); k=k+1 }
1262 var bands: i64 = p.n_workers
1263 if bands > 32 { bands = 32 }
1264 if bands > out_dim { bands = out_dim }
1265 if bands < 1 { bands = 1 }
1266 let per: i64 = (out_dim + bands - 1) / bands
1267 let done0: i64 = nx_pool_n_completed(p)
1268 var b: i64=0
1269 while b<bands {
1270 let c: *i64 = ((g_nf_mmarena as i64) + b*128) as *i64
1271 c[0]=xi8 as i64; c[1]=sx; c[2]=Wi8 as i64; c[3]=sw as i64; c[4]=dst as i64; c[5]=in_dim; c[6]=b*per
1272 var hi: i64=(b+1)*per; if hi>out_dim { hi=out_dim }
1273 c[7]=hi; c[8]=shift; c[9]=((g_nf_i8acc as i64)+b*64)
1274 nx_pool_submit(p, _nfmm_i8_task, c as i64)
1275 b=b+1
1276 }
1277 nx_pool_wait(p, done0+bands)
1278 sys_munmap(xi8, in_dim*2)
1279 return 0
1280}
1281// decode ONE token via the W8A8 SIMD weight cache. Same structure as decode_step_kv_cached; the 7 projection
1282// matmuls go through mm_pool_i8 (13x faster, ~0.8% lossy). Attention/softmax/residual stay exact i64.
1283func decode_step_kv_cached_i8(buf: *u8, hdr: *NxGgufHeader, x1: *i64, out1: *i64, wc8: *i64, sb: *i64, nm: *u8, freqs: *i64, kvc: *i64, pos: i64, cfgA: *i64, cfgF: *i64, N: i64) -> i64 {
1284 let ne: i64=cfgA[1]; let nh: i64=cfgA[2]; let nkv: i64=cfgA[3]; let hd: i64=cfgA[4]; let qd: i64=cfgA[5]; let kvd: i64=cfgA[6]; let scale: i64=cfgA[7]; let fd: i64=cfgF[2]
1285 let xn: *i64=sb[0] as *i64; let Q: *i64=sb[1] as *i64; let K: *i64=sb[2] as *i64; let V: *i64=sb[3] as *i64; let concat: *i64=sb[4] as *i64
1286 let sc: *i64=sb[5] as *i64; let at: *i64=sb[6] as *i64; let proj: *i64=sb[7] as *i64; let gate: *i64=sb[8] as *i64; let up: *i64=sb[9] as *i64
1287 let hbuf: *i64=sb[10] as *i64; let hmid: *i64=sb[11] as *i64; let cur: *i64=sb[12] as *i64
1288 cpy(cur, x1, ne)
1289 var L: i64=0
1290 while L<N {
1291 let wb: *i64 = wc8[L] as *i64
1292 let gA: *i64=wb[0] as *i64
1293 let Wq: *u8=wb[1] as *u8; let swq: *i64=wb[2] as *i64
1294 let Wk: *u8=wb[3] as *u8; let swk: *i64=wb[4] as *i64
1295 let Wv: *u8=wb[5] as *u8; let swv: *i64=wb[6] as *i64
1296 let Wo: *u8=wb[7] as *u8; let swo: *i64=wb[8] as *i64
1297 let gF: *i64=wb[9] as *i64
1298 let Wg: *u8=wb[10] as *u8; let swg: *i64=wb[11] as *i64
1299 let Wu: *u8=wb[12] as *u8; let swu: *i64=wb[13] as *i64
1300 let Wd: *u8=wb[14] as *u8; let swd: *i64=wb[15] as *i64
1301 let bq: *i64=wb[16] as *i64; let bk: *i64=wb[17] as *i64; let bv: *i64=wb[18] as *i64
1302 rmsnorm_gamma_row_q24(cur, gA, 0, ne, xn, 0)
1303 mm_pool_i8(xn, Wq, swq, Q, ne, qd, 24)
1304 mm_pool_i8(xn, Wk, swk, K, ne, kvd, 24)
1305 mm_pool_i8(xn, Wv, swv, V, ne, kvd, 24)
1306 var bo: i64=0; while bo<qd { Q[bo]=Q[bo]+bq[bo]; bo=bo+1 }
1307 var bp: i64=0; while bp<kvd { K[bp]=K[bp]+bk[bp]; V[bp]=V[bp]+bv[bp]; bp=bp+1 }
1308 var h: i64=0; while h<nh { rope_apply(((Q as i64)+(h*hd)*8) as *i64, hd, pos, freqs); h=h+1 }
1309 var hk: i64=0; while hk<nkv { rope_apply(((K as i64)+(hk*hd)*8) as *i64, hd, pos, freqs); hk=hk+1 }
1310 let Kc: *i64 = kvc[2*L] as *i64
1311 let Vc: *i64 = kvc[2*L+1] as *i64
1312 var ci: i64=0; while ci<kvd { Kc[pos*kvd+ci]=K[ci]; Vc[pos*kvd+ci]=V[ci]; ci=ci+1 }
1313 let group: i64=nh/nkv
1314 let cnt: i64=pos+1
1315 var hh: i64=0
1316 while hh<nh {
1317 let kvh: i64=hh/group; let qb: i64=hh*hd; let kb: i64=kvh*hd
1318 var s: i64=0
1319 while s<cnt { var dot: i64=0; var d: i64=0; while d<hd { dot=dot+(Q[qb+d]*Kc[s*kvd+kb+d]); d=d+1 } sc[s]=qmul(dot>>16, scale); s=s+1 }
1320 var mmax: i64=sc[0]; var j: i64=1; while j<cnt { if sc[j]>mmax { mmax=sc[j] } j=j+1 }
1321 var sum: i64=0; j=0; while j<cnt { let e: i64=fx_exp(sc[j]-mmax); at[j]=e; sum=sum+e; j=j+1 }
1322 if sum<=0 { sum=1 }
1323 j=0; while j<cnt { at[j]=(at[j]<<16)/sum; j=j+1 }
1324 var d2: i64=0
1325 while d2<hd { var acc: i64=0; s=0; while s<cnt { acc=acc+(at[s]*Vc[s*kvd+kb+d2]); s=s+1 } concat[qb+d2]=acc>>16; d2=d2+1 }
1326 hh=hh+1
1327 }
1328 mm_pool_i8(concat, Wo, swo, proj, qd, ne, 16)
1329 var ri: i64=0; while ri<ne { hmid[ri]=cur[ri]+proj[ri]; ri=ri+1 }
1330 rmsnorm_gamma_row_q24(hmid, gF, 0, ne, xn, 0)
1331 mm_pool_i8(xn, Wg, swg, gate, ne, fd, 24)
1332 mm_pool_i8(xn, Wu, swu, up, ne, fd, 24)
1333 var fi: i64=0; while fi<fd { hbuf[fi]=qmul(silu(gate[fi]), up[fi]); fi=fi+1 }
1334 mm_pool_i8(hbuf, Wd, swd, proj, fd, ne, 16)
1335 var fo: i64=0; while fo<ne { cur[fo]=hmid[fo]+proj[fo]; fo=fo+1 }
1336 L=L+1
1337 }
1338 cpy(out1, cur, ne)
1339 return 0
1340}