nx_nofloat_q4k.nx source
↩ module page · 406 lines · 22824 B
1// nx_nofloat_q4k.nx -- LM4 RESIDENT-QUANT DECODE (2026-09-02): the 7B fits because the weights STAY Q4_K.
2// MEASURED (nx_nofloat_llm.nx nf_alloc_layer_wb_i8): the "i8" layer cache stores 2 B/weight (i16 lanes for
3// __i16x16_madd), so Qwen2.5-Coder-7B = 6.5e9 layer params = 13 GB of anonymous memory before the i32 head
4// (2.2 GB), the copied model buffer (4.7 GB via sys_read_file) and the MAXT scratch -- it cannot fit a 16 GB
5// WSL VM and no cache hygiene changes that. This lib keeps every Q4_K projection IN THE FILE MAP (0.5625
6// B/weight, file-backed, evictable) and dots it with the estate's fused SIMD kernel (nx_q4k_dot_simd_lib,
7// bit-exact vs nx_q4k_dot_row_col), so a Q4_K layer costs 0 anonymous bytes. A tensor of any other quant
8// type (Q4_K_M puts Q6_K on some attn_v/ffn_down rows) falls back PER TENSOR to the incumbent i16 cache
9// (nf_quant_w_i8 + mm_pool_i8) -- counted and announced, never silent.
10// NUMERIC CONTRACT (why the shift arithmetic is what it is): the serve keeps activations in Q16; its i8
11// path computes dst = (sx*sw*sum(xq*wq)) >> shift with W in Q16 (shift 24 for a Q24 rmsnorm input, 16 for a
12// Q16 input). nx_q4k_dot_simd returns sum over the row of W_Q24 * col_i16 -- the weight carries 8 more
13// fractional bits -- so dst = ((dot >> NQ_PRE_SHIFT) * sx) >> (shift + NQ_W_EXTRA_BITS - NQ_PRE_SHIFT).
14// The activation is packed to i16 under a dynamic per-call scale sx = ceil(max|x| / 32767) (15 significant
15// bits) where the i8 path packs to 7 bits: this route is STRICTLY MORE EXACT on the activation side and
16// lossless on the weight side (the Q4_K bytes ARE the model). The pre-shift is overflow headroom: |dot|
17// reaches ~1e16 on an 18944-wide ffn_down row and sx reaches ~2^19, so the product is taken after >>16.
18// Duplicates deliberately NOT created: attention/softmax/residual come from nf_attn_kv_core (shared with the
19// i8 decoder), the fallback matvec IS mm_pool_i8, the per-row quantiser IS nf_quant_w_i8.
20// license_tier: ORIGINAL No hw writes (Rule 26).
21import "nx_syscalls.nx"
22import "nx_tier.nx"
23import "nx_le.nx"
24import "nx_tensor.nx"
25import "nx_gguf.nx"
26import "nx_gguf_load.nx"
27import "nx_thread_pool.nx"
28import "nx_nofloat_llm.nx"
29import "nx_q4k_dot_simd_lib.nx"
30import "nx_q4k_dot_simd2_lib.nx" // LM4c: the compiler-emitted unpack (__q4k_unpack32s); qpk+qhi form its 128-byte qbuf
31import "nx_clock.nx" // nx_clock_monotonic_ns for the per-route profile accumulators
32
33const NQ_KIND_Q4K: i64 = 0
34const NQ_KIND_I16: i64 = 1
35const NQ_KIND_Q8_0: i64 = 2 // search R0s (2026-09-17): Q8_0 blocks dotted in place from the file map (block-native)
36const NQ_Q8_BLOCK_VALS: i64 = 32 // Q8_0 block width
37const NQ_Q8_BLOCK_BYTES: i64 = 34 // one f16 scale (2 bytes) + 32 int8 codes
38const NQ_Q8_SCALE_BYTES: i64 = 2
39const NQ_SLOT_WORDS: i64 = 8 // kind, base_off, in_dim, out_dim, row_stride, w16, sw, n_blocks
40const NQ_S_KIND: i64 = 0
41const NQ_S_OFF: i64 = 1
42const NQ_S_IN: i64 = 2
43const NQ_S_OUT: i64 = 3
44const NQ_S_RSTRIDE: i64 = 4
45const NQ_S_W16: i64 = 5
46const NQ_S_SW: i64 = 6
47const NQ_S_NBLK: i64 = 7
48const NQ_NPROJ: i64 = 7 // q k v o gate up down
49const NQ_P_Q: i64 = 0
50const NQ_P_K: i64 = 1
51const NQ_P_V: i64 = 2
52const NQ_P_O: i64 = 3
53const NQ_P_GATE: i64 = 4
54const NQ_P_UP: i64 = 5
55const NQ_P_DOWN: i64 = 6
56const NQ_L_GA: i64 = 56 // NQ_NPROJ * NQ_SLOT_WORDS: attn_norm gamma (Q16 i64, ne)
57const NQ_L_GF: i64 = 57 // ffn_norm gamma
58const NQ_L_BQ: i64 = 58 // q/k/v biases (Q16 i64)
59const NQ_L_BK: i64 = 59
60const NQ_L_BV: i64 = 60
61const NQ_LAYER_WORDS: i64 = 64
62const NQ_SUPER_VALS: i64 = 256 // Q4_K super-block width
63const NQ_SUPER_BYTES: i64 = 144 // Q4_K super-block bytes
64const NQ_SUBBLOCKS: i64 = 8 // sub-blocks per super-block (sc_pre rows)
65const NQ_ACT_I16_MAX: i64 = 32767
66const NQ_W_EXTRA_BITS: i64 = 8 // W_Q24 vs the serve's Q16 weight convention
67const NQ_PRE_SHIFT: i64 = 16 // overflow headroom before multiplying by sx
68const NQ_SHIFT_Q24: i64 = 24 // input in Q24 (post-rmsnorm) -> Q16 out
69const NQ_SHIFT_Q16: i64 = 16 // input in Q16 (attention concat, silu*up) -> Q16 out
70const NQ_BANDS_MAX: i64 = 32
71const NQ_BAND_BYTES: i64 = 320 // ctx 128 + qpk 64 + qhi 64 + acc 32 + spare 32
72const NQ_CTX_BYTES: i64 = 128
73const NQ_QPK_BYTES: i64 = 64
74const NQ_ACC_OFF: i64 = 256
75const NQ_ST_WORDS: i64 = 8 // stats: [0]=inplace tensors [1]=fallback tensors [2]=fallback anon bytes [3]=layers [4]=inplace file bytes
76
77static g_nq_buf: *u8 // the model file map every in-place slot reads through
78static g_nq_colq: *i64 // scaled activation (units of sx), max in_dim i64
79static g_nq_col16: *u8 // the same packed to i16 lanes
80static g_nq_scpre: *i64 // per sub-block activation sums (activation-only precompute)
81static g_nq_bands: *u8 // NQ_BANDS_MAX x NQ_BAND_BYTES: per-band task ctx + kernel scratch
82static g_nq_maxin: i64
83// PROFILE ACCUMULATORS (2026-09-02): the serve-level ms/token did not move with a 3.94x faster kernel, so the
84// decode must say where its time goes instead of the seat guessing. Two clock reads per matvec (vDSO-class cost)
85// keep cumulative ns and call counts per route; /health publishes them, a caller diffs across one generation.
86static g_nq_prof: *i64 // [0]=q4k gemm ns [1]=q4k gemm calls [2]=i16 fallback ns [3]=i16 calls
87const NQ_PROF_WORDS: i64 = 6
88const NQ_PROF_Q4K_NS: i64 = 0
89const NQ_PROF_Q4K_CALLS: i64 = 1
90const NQ_PROF_I16_NS: i64 = 2
91const NQ_PROF_I16_CALLS: i64 = 3
92const NQ_PROF_Q8_NS: i64 = 4
93const NQ_PROF_Q8_CALLS: i64 = 5
94func nq_prof() -> *i64 { if (g_nq_prof as i64) == 0 { g_nq_prof = sys_mmap(NQ_PROF_WORDS*8) as *i64 } return g_nq_prof }
95func nq_prof_get(k: i64) -> i64 { let p: *i64 = nq_prof(); return p[k] }
96
97// ---- pure decision + arithmetic (gate-tested on every branch) ----
98// which route a projection takes: in place only for Q4_K whose width is whole super-blocks; everything else i16.
99func nq_kind_for(ggml_type: i64, in_dim: i64) -> i64 {
100 if ggml_type == NX_GGML_TYPE_Q4_K { if (in_dim % NQ_SUPER_VALS) == 0 { return NQ_KIND_Q4K } }
101 if ggml_type == NX_GGML_TYPE_Q8_0 { if (in_dim % NQ_Q8_BLOCK_VALS) == 0 { return NQ_KIND_Q8_0 } }
102 return NQ_KIND_I16
103}
104func nq_post_shift(shift: i64) -> i64 { return shift + NQ_W_EXTRA_BITS - NQ_PRE_SHIFT }
105func nq_scale_out(dot: i64, sx: i64, post: i64) -> i64 { return ((dot >> NQ_PRE_SHIFT) * sx) >> post }
106// dynamic activation scale: the largest |x| must land on <= 32767 after division (ceil, never floor -- a floor
107// lets max|x|/sx exceed the i16 lane and wrap).
108func nq_act_scale(xmx: i64) -> i64 { var sx: i64 = (xmx + NQ_ACT_I16_MAX - 1) / NQ_ACT_I16_MAX; if sx < 1 { sx = 1 } return sx }
109func nq_slot(lay: *i64, p: i64) -> *i64 { return ((lay as i64) + p*NQ_SLOT_WORDS*8) as *i64 }
110
111func nq_bind_buf(buf: *u8) -> i64 { g_nq_buf = buf; return 0 }
112// scratch sized to the widest projection input; idempotent for any maxin at or under the current size.
113func nq_arena(maxin: i64) -> i64 {
114 if (g_nq_colq as i64) != 0 { if maxin <= g_nq_maxin { return 0 } }
115 g_nq_maxin = maxin
116 g_nq_colq = sys_mmap(maxin*8) as *i64
117 g_nq_col16 = sys_mmap(maxin*2)
118 g_nq_scpre = sys_mmap((maxin/NQ_SUPER_VALS + 1)*NQ_SUBBLOCKS*8) as *i64
119 if (g_nq_bands as i64) == 0 { g_nq_bands = sys_mmap(NQ_BANDS_MAX*NQ_BAND_BYTES) }
120 return 0
121}
122
123// an i16 fallback slot from an already-dequantised Q16 matrix (out_dim x in_dim, row-major).
124func nq_slot_from_q16(Wq16: *i64, slot: *i64, in_dim: i64, out_dim: i64) -> i64 {
125 let w16: *u8 = sys_mmap(out_dim*in_dim*2)
126 let sw: *i64 = sys_mmap(out_dim*8) as *i64
127 nf_quant_w_i8(Wq16, w16, sw, out_dim, in_dim)
128 slot[NQ_S_KIND]=NQ_KIND_I16; slot[NQ_S_OFF]=0; slot[NQ_S_IN]=in_dim; slot[NQ_S_OUT]=out_dim
129 slot[NQ_S_RSTRIDE]=0; slot[NQ_S_W16]=w16 as i64; slot[NQ_S_SW]=sw as i64; slot[NQ_S_NBLK]=0
130 return 0
131}
132
133// resolve ONE projection tensor of layer L into a slot. -1 when the tensor is missing or its dims disagree with
134// the arch config (fail fast at init -- never serve a garbage decode).
135func nq_slot_fill(buf: *u8, hdr: *NxGgufHeader, nm: *u8, L: i64, sfx: *u8, slot: *i64, in_dim: i64, out_dim: i64, tmp: *i64, st: *i64) -> i64 {
136 let len: i64 = build_name(nm, L, sfx)
137 let idx: nx_int = nx_gguf_find_tensor(hdr, nm, len)
138 if idx < 0 { return 0 - 1 }
139 let ti: *NxGgufTensorInfo = nx_gguf_tensor_at(hdr, idx)
140 if ti.dim_0 != in_dim { return 0 - 1 }
141 if ti.dim_1 != out_dim { return 0 - 1 }
142 slot[NQ_S_IN]=in_dim; slot[NQ_S_OUT]=out_dim
143 if nq_kind_for(ti.ggml_type, in_dim) == NQ_KIND_Q4K {
144 slot[NQ_S_KIND]=NQ_KIND_Q4K
145 slot[NQ_S_OFF]=hdr.data_off + ti.offset
146 slot[NQ_S_NBLK]=in_dim/NQ_SUPER_VALS
147 slot[NQ_S_RSTRIDE]=slot[NQ_S_NBLK]*NQ_SUPER_BYTES
148 slot[NQ_S_W16]=0; slot[NQ_S_SW]=0
149 st[0]=st[0]+1; st[4]=st[4]+slot[NQ_S_RSTRIDE]*out_dim
150 return 0
151 }
152 if nq_kind_for(ti.ggml_type, in_dim) == NQ_KIND_Q8_0 {
153 slot[NQ_S_KIND]=NQ_KIND_Q8_0
154 slot[NQ_S_OFF]=hdr.data_off + ti.offset
155 slot[NQ_S_NBLK]=in_dim/NQ_Q8_BLOCK_VALS
156 slot[NQ_S_RSTRIDE]=slot[NQ_S_NBLK]*NQ_Q8_BLOCK_BYTES
157 slot[NQ_S_W16]=0; slot[NQ_S_SW]=0
158 st[5]=st[5]+1; st[6]=st[6]+slot[NQ_S_RSTRIDE]*out_dim
159 return 0
160 }
161 if load_named_q16(buf, hdr, nm, len, tmp, out_dim*in_dim) < 1 { return 0 - 1 }
162 nq_slot_from_q16(tmp, slot, in_dim, out_dim)
163 st[1]=st[1]+1; st[2]=st[2]+out_dim*in_dim*2+out_dim*8
164 return 0
165}
166
167func nq_load_vec(buf: *u8, hdr: *NxGgufHeader, nm: *u8, L: i64, sfx: *u8, n: i64) -> i64 {
168 let v: *i64 = sys_mmap(n*8) as *i64
169 load_blk(buf, hdr, nm, L, sfx, v, n)
170 return v as i64
171}
172
173// build every layer's slot table: wq[L] -> NQ_LAYER_WORDS. st gets the census (see NQ_ST_WORDS). -1 = fail fast.
174func nq_build_layers(buf: *u8, hdr: *NxGgufHeader, wq: *i64, N: i64, ne: i64, qd: i64, kvd: i64, fd: i64, nm: *u8, st: *i64) -> i64 {
175 var maxin: i64 = ne; if fd > maxin { maxin = fd } if qd > maxin { maxin = qd }
176 nq_arena(maxin)
177 nq_bind_buf(buf)
178 var maxw: i64 = ne*fd; if qd*ne > maxw { maxw = qd*ne }
179 let tmp: *i64 = sys_mmap(maxw*8) as *i64
180 var z: i64=0; while z<NQ_ST_WORDS { st[z]=0; z=z+1 }
181 var L: i64=0
182 while L<N {
183 let lay: *i64 = sys_mmap(NQ_LAYER_WORDS*8) as *i64
184 if nq_slot_fill(buf, hdr, nm, L, ".attn_q.weight\x00" as *u8, nq_slot(lay, NQ_P_Q), ne, qd, tmp, st) != 0 { return 0 - 1 }
185 if nq_slot_fill(buf, hdr, nm, L, ".attn_k.weight\x00" as *u8, nq_slot(lay, NQ_P_K), ne, kvd, tmp, st) != 0 { return 0 - 1 }
186 if nq_slot_fill(buf, hdr, nm, L, ".attn_v.weight\x00" as *u8, nq_slot(lay, NQ_P_V), ne, kvd, tmp, st) != 0 { return 0 - 1 }
187 if nq_slot_fill(buf, hdr, nm, L, ".attn_output.weight\x00" as *u8, nq_slot(lay, NQ_P_O), qd, ne, tmp, st) != 0 { return 0 - 1 }
188 if nq_slot_fill(buf, hdr, nm, L, ".ffn_gate.weight\x00" as *u8, nq_slot(lay, NQ_P_GATE), ne, fd, tmp, st) != 0 { return 0 - 1 }
189 if nq_slot_fill(buf, hdr, nm, L, ".ffn_up.weight\x00" as *u8, nq_slot(lay, NQ_P_UP), ne, fd, tmp, st) != 0 { return 0 - 1 }
190 if nq_slot_fill(buf, hdr, nm, L, ".ffn_down.weight\x00" as *u8, nq_slot(lay, NQ_P_DOWN), fd, ne, tmp, st) != 0 { return 0 - 1 }
191 lay[NQ_L_GA]=nq_load_vec(buf, hdr, nm, L, ".attn_norm.weight\x00" as *u8, ne)
192 lay[NQ_L_GF]=nq_load_vec(buf, hdr, nm, L, ".ffn_norm.weight\x00" as *u8, ne)
193 lay[NQ_L_BQ]=nq_load_vec(buf, hdr, nm, L, ".attn_q.bias\x00" as *u8, qd)
194 lay[NQ_L_BK]=nq_load_vec(buf, hdr, nm, L, ".attn_k.bias\x00" as *u8, kvd)
195 lay[NQ_L_BV]=nq_load_vec(buf, hdr, nm, L, ".attn_v.bias\x00" as *u8, kvd)
196 wq[L]=lay as i64
197 L=L+1
198 }
199 sys_munmap(tmp as *u8, maxw*8)
200 st[3]=N
201 return 0
202}
203
204// one band of output rows through the fused Q4_K SIMD dot; kernel scratch lives beside the ctx in the band arena.
205func _nq_mm_task(ctx_i: i64) -> i64 {
206 let c: *i64 = ctx_i as *i64
207 let buf: *u8 = c[0] as *u8; let off: i64 = c[1]; let rstride: i64 = c[2]; let nblk: i64 = c[3]
208 let col16: *i64 = c[4] as *i64; let scpre: *i64 = c[5] as *i64; let dst: *i64 = c[6] as *i64
209 let lo: i64 = c[7]; let hi: i64 = c[8]; let sx: i64 = c[9]; let post: i64 = c[10]
210 // qbuf = the qpk+qhi pair (contiguous 128 bytes at ctx+NQ_CTX_BYTES): lo lanes then hi lanes, the layout
211 // __q4k_unpack32s writes and __i16x16_madd reads. NQ_QPK_BYTES still names the split for readers.
212 let qbuf: *i64 = (ctx_i + NQ_CTX_BYTES) as *i64
213 let acc: *i64 = (ctx_i + NQ_ACC_OFF) as *i64
214 var o: i64 = lo
215 while o < hi {
216 let dot: i64 = nx_q4k_dot_simd2(buf, off + o*rstride, nblk, col16, qbuf, acc, scpre)
217 dst[o] = nq_scale_out(dot, sx, post)
218 o = o + 1
219 }
220 return 0
221}
222
223// ---- Q8_0 BLOCK-NATIVE ROUTE (search R0s, 2026-09-17) ----------------------------------------------------------
224// The model's own Q8_0 blocks (one f16 scale + 32 int8 codes, 1.0625 B/weight) are dotted IN PLACE from the file map:
225// no re-quantisation of a row to 12 bits under one row scale (the i8 cache), no 2 B/weight i16 lanes in anonymous
226// memory. Each block's dot is the exact int32 dot of its 32 codes with the i16 activation lanes, scaled by the
227// block's own f16 scale in Q24 -- the same W_Q24 x col_i16 contract as the Q4_K dot, so nq_scale_out and both shift
228// regimes are shared unchanged. The 32 codes are sign-extended into 64 bytes of band scratch and dotted by the exact
229// __i16_dot; a vectorised unpack is the throughput half (R0s-b), this is the numerics half the qgap probe measures.
230// Overflow: |dot| <= 127*32767*32 and a Q8_0 scale in Q24 is well under 2^24, so a block product stays under 2^60
231// and an 18944-wide row (592 blocks) stays inside i64; the caller applies the same >>NQ_PRE_SHIFT before * sx.
232func nq8_dot_row_scalar(buf: *u8, off: i64, nblk: i64, col16: *u8, w16: *u8) -> i64 {
233 var acc: i64 = 0
234 var b: i64 = 0
235 while b < nblk {
236 let base: i64 = off + b*NQ_Q8_BLOCK_BYTES
237 let d24: i64 = _gguf_f16_to_q24(nx_le_read_u16(buf, base))
238 var i: i64 = 0
239 while i < NQ_Q8_BLOCK_VALS {
240 var q: i64 = (buf[base + NQ_Q8_SCALE_BYTES + i] as i64) & 0xff
241 if q >= 128 { q = q - 256 }
242 nf_pack2(w16, i, q)
243 i = i + 1
244 }
245 let dot: i64 = __i16_dot(((col16 as i64) + b*NQ_Q8_BLOCK_VALS*2) as *u8, w16, NQ_Q8_BLOCK_VALS)
246 acc = acc + dot*d24
247 b = b + 1
248 }
249 return acc
250}
251
252// R0s-b (2026-09-17): the block dot through the compiler builtin -- vpmovsxbw sign-extends the 32 Q8_0 codes in
253// two 16-lane halves, vpmaddwd multiplies against the i16 activation and the lanes fold to one exact i64; the
254// per-block f16 scale still multiplies OUTSIDE the dot, so this is bit-identical to nq8_dot_row_scalar by
255// contract (the gate proves it against both the i64 reference and the scalar twin). w16 stays in the signature
256// so the band task and the scalar twin share one shape; the builtin needs no scratch.
257func nq8_dot_row(buf: *u8, off: i64, nblk: i64, col16: *u8, w16: *u8) -> i64 {
258 var acc: i64 = 0
259 var b: i64 = 0
260 while b < nblk {
261 let base: i64 = off + b*NQ_Q8_BLOCK_BYTES
262 let d24: i64 = _gguf_f16_to_q24(nx_le_read_u16(buf, base))
263 let dot: i64 = __q8blk_i16dot(((buf as i64) + base + NQ_Q8_SCALE_BYTES) as *u8, ((col16 as i64) + b*NQ_Q8_BLOCK_VALS*2) as *u8)
264 acc = acc + dot*d24
265 b = b + 1
266 }
267 return acc
268}
269
270// one band of output rows through the Q8_0 block dot; the sign-extended block lives in the band's qbuf region.
271func _nq8_mm_task(ctx_i: i64) -> i64 {
272 let c: *i64 = ctx_i as *i64
273 let buf: *u8 = c[0] as *u8; let off: i64 = c[1]; let rstride: i64 = c[2]; let nblk: i64 = c[3]
274 let col16: *u8 = c[4] as *u8; let dst: *i64 = c[6] as *i64
275 let lo: i64 = c[7]; let hi: i64 = c[8]; let sx: i64 = c[9]; let post: i64 = c[10]
276 let w16: *u8 = (ctx_i + NQ_CTX_BYTES) as *u8
277 var o: i64 = lo
278 while o < hi {
279 let dot: i64 = nq8_dot_row(buf, off + o*rstride, nblk, col16, w16)
280 dst[o] = nq_scale_out(dot, sx, post)
281 o = o + 1
282 }
283 return 0
284}
285
286// one projection matvec over Q8_0 slots: the activation packs to i16 under the dynamic per-call scale (the Q4_K
287// route's contract: strictly more exact than a per-block int8 activation), then output-row bands run the block dot.
288func mq8_gemm(x: *i64, slot: *i64, dst: *i64, shift: i64) -> i64 {
289 let ind: i64 = slot[NQ_S_IN]; let outd: i64 = slot[NQ_S_OUT]
290 let p: *NxThreadPool = nf_pool()
291 var xmx: i64 = 0; var k: i64 = 0
292 while k < ind { var a: i64 = x[k]; if a < 0 { a = 0 - a } if a > xmx { xmx = a } k = k + 1 }
293 let sx: i64 = nq_act_scale(xmx)
294 k = 0; while k < ind { nf_pack2(g_nq_col16, k, x[k]/sx); k = k + 1 }
295 let nblk: i64 = slot[NQ_S_NBLK]
296 var bands: i64 = p.n_workers
297 if bands > NQ_BANDS_MAX { bands = NQ_BANDS_MAX }
298 if bands > outd { bands = outd }
299 if bands < 1 { bands = 1 }
300 let per: i64 = (outd + bands - 1) / bands
301 let done0: i64 = nx_pool_n_completed(p)
302 let post: i64 = nq_post_shift(shift)
303 var b: i64 = 0
304 while b < bands {
305 let ci: i64 = (g_nq_bands as i64) + b*NQ_BAND_BYTES
306 let c: *i64 = ci as *i64
307 c[0]=g_nq_buf as i64; c[1]=slot[NQ_S_OFF]; c[2]=slot[NQ_S_RSTRIDE]; c[3]=nblk
308 c[4]=g_nq_col16 as i64; c[5]=0; c[6]=dst as i64; c[7]=b*per
309 var hi: i64 = (b+1)*per; if hi > outd { hi = outd }
310 c[8]=hi; c[9]=sx; c[10]=post
311 nx_pool_submit(p, _nq8_mm_task, ci)
312 b = b + 1
313 }
314 nx_pool_wait(p, done0+bands)
315 return 0
316}
317
318// one projection matvec: x (in_dim; Q24 or Q16 per `shift`) x slot -> dst (out_dim, Q16). The slot kind decides
319// the route; the i16 route IS mm_pool_i8 (one fallback ruler, not a second one).
320func nq_mm(x: *i64, slot: *i64, dst: *i64, shift: i64) -> i64 {
321 let p: *i64 = nq_prof()
322 let t0: i64 = nx_clock_monotonic_ns()
323 if slot[NQ_S_KIND] == NQ_KIND_I16 {
324 mm_pool_i8(x, slot[NQ_S_W16] as *u8, slot[NQ_S_SW] as *i64, dst, slot[NQ_S_IN], slot[NQ_S_OUT], shift)
325 p[NQ_PROF_I16_NS] = p[NQ_PROF_I16_NS] + (nx_clock_monotonic_ns() - t0); p[NQ_PROF_I16_CALLS] = p[NQ_PROF_I16_CALLS] + 1
326 return 0
327 }
328 if slot[NQ_S_KIND] == NQ_KIND_Q8_0 {
329 mq8_gemm(x, slot, dst, shift)
330 p[NQ_PROF_Q8_NS] = p[NQ_PROF_Q8_NS] + (nx_clock_monotonic_ns() - t0); p[NQ_PROF_Q8_CALLS] = p[NQ_PROF_Q8_CALLS] + 1
331 return 0
332 }
333 mq_fused_dequant_gemm(x, slot, dst, shift)
334 p[NQ_PROF_Q4K_NS] = p[NQ_PROF_Q4K_NS] + (nx_clock_monotonic_ns() - t0); p[NQ_PROF_Q4K_CALLS] = p[NQ_PROF_Q4K_CALLS] + 1
335 return 0
336}
337
338// THE FUSED DEQUANT GEMM (llm LM4 / engineshift ES8 contract mq_fused_dequant_gemm): the Q4_K weights are never
339// expanded -- each output row is one fused SIMD dot straight off the packed super-blocks in the file map, scaled
340// into the serve's Q16 convention. Pooled by output-row bands (thread-count-invariant partition).
341func mq_fused_dequant_gemm(x: *i64, slot: *i64, dst: *i64, shift: i64) -> i64 {
342 let ind: i64 = slot[NQ_S_IN]; let outd: i64 = slot[NQ_S_OUT]
343 let p: *NxThreadPool = nf_pool()
344 var xmx: i64 = 0; var k: i64 = 0
345 while k < ind { var a: i64 = x[k]; if a < 0 { a = 0 - a } if a > xmx { xmx = a } k = k + 1 }
346 let sx: i64 = nq_act_scale(xmx)
347 k = 0; while k < ind { g_nq_colq[k] = x[k]/sx; nf_pack2(g_nq_col16, k, g_nq_colq[k]); k = k + 1 }
348 let nblk: i64 = slot[NQ_S_NBLK]
349 nx_q4k_sc_precompute(g_nq_colq, nblk, g_nq_scpre)
350 var bands: i64 = p.n_workers
351 if bands > NQ_BANDS_MAX { bands = NQ_BANDS_MAX }
352 if bands > outd { bands = outd }
353 if bands < 1 { bands = 1 }
354 let per: i64 = (outd + bands - 1) / bands
355 let done0: i64 = nx_pool_n_completed(p)
356 let post: i64 = nq_post_shift(shift)
357 var b: i64 = 0
358 while b < bands {
359 let ci: i64 = (g_nq_bands as i64) + b*NQ_BAND_BYTES
360 let c: *i64 = ci as *i64
361 c[0]=g_nq_buf as i64; c[1]=slot[NQ_S_OFF]; c[2]=slot[NQ_S_RSTRIDE]; c[3]=nblk
362 c[4]=g_nq_col16 as i64; c[5]=g_nq_scpre as i64; c[6]=dst as i64; c[7]=b*per
363 var hi: i64 = (b+1)*per; if hi > outd { hi = outd }
364 c[8]=hi; c[9]=sx; c[10]=post
365 nx_pool_submit(p, _nq_mm_task, ci)
366 b = b + 1
367 }
368 nx_pool_wait(p, done0+bands)
369 return 0
370}
371
372// decode ONE token at pos through the resident-Q4_K slot tables. Structure = decode_step_kv_cached_i8; the seven
373// projections route through nq_mm, everything else is the shared exact-i64 core.
374func decode_step_kv_q4k(x1: *i64, out1: *i64, wq: *i64, sb: *i64, freqs: *i64, kvc: *i64, pos: i64, cfgA: *i64, cfgF: *i64, N: i64) -> i64 {
375 let ne: i64=cfgA[1]; let nh: i64=cfgA[2]; let nkv: i64=cfgA[3]; let hd: i64=cfgA[4]; let qd: i64=cfgA[5]; let kvd: i64=cfgA[6]; let fd: i64=cfgF[2]
376 let xn: *i64=sb[0] as *i64; let Q: *i64=sb[1] as *i64; let K: *i64=sb[2] as *i64; let V: *i64=sb[3] as *i64; let concat: *i64=sb[4] as *i64
377 let sc: *i64=sb[5] as *i64; let at: *i64=sb[6] as *i64; let proj: *i64=sb[7] as *i64; let gate: *i64=sb[8] as *i64; let up: *i64=sb[9] as *i64
378 let hbuf: *i64=sb[10] as *i64; let hmid: *i64=sb[11] as *i64; let cur: *i64=sb[12] as *i64
379 cpy(cur, x1, ne)
380 var L: i64=0
381 while L<N {
382 let lay: *i64 = wq[L] as *i64
383 let gA: *i64 = lay[NQ_L_GA] as *i64; let gF: *i64 = lay[NQ_L_GF] as *i64
384 let bq: *i64 = lay[NQ_L_BQ] as *i64; let bk: *i64 = lay[NQ_L_BK] as *i64; let bv: *i64 = lay[NQ_L_BV] as *i64
385 rmsnorm_gamma_row_q24(cur, gA, 0, ne, xn, 0)
386 nq_mm(xn, nq_slot(lay, NQ_P_Q), Q, NQ_SHIFT_Q24)
387 nq_mm(xn, nq_slot(lay, NQ_P_K), K, NQ_SHIFT_Q24)
388 nq_mm(xn, nq_slot(lay, NQ_P_V), V, NQ_SHIFT_Q24)
389 var bo: i64=0; while bo<qd { Q[bo]=Q[bo]+bq[bo]; bo=bo+1 }
390 var bp: i64=0; while bp<kvd { K[bp]=K[bp]+bk[bp]; V[bp]=V[bp]+bv[bp]; bp=bp+1 }
391 var h: i64=0; while h<nh { rope_apply(((Q as i64)+(h*hd)*8) as *i64, hd, pos, freqs); h=h+1 }
392 var hk: i64=0; while hk<nkv { rope_apply(((K as i64)+(hk*hd)*8) as *i64, hd, pos, freqs); hk=hk+1 }
393 nf_attn_kv_core(Q, K, V, kvc, L, pos, cfgA, sc, at, concat)
394 nq_mm(concat, nq_slot(lay, NQ_P_O), proj, NQ_SHIFT_Q16)
395 var ri: i64=0; while ri<ne { hmid[ri]=cur[ri]+proj[ri]; ri=ri+1 }
396 rmsnorm_gamma_row_q24(hmid, gF, 0, ne, xn, 0)
397 nq_mm(xn, nq_slot(lay, NQ_P_GATE), gate, NQ_SHIFT_Q24)
398 nq_mm(xn, nq_slot(lay, NQ_P_UP), up, NQ_SHIFT_Q24)
399 var fi: i64=0; while fi<fd { hbuf[fi]=qmul(silu(gate[fi]), up[fi]); fi=fi+1 }
400 nq_mm(hbuf, nq_slot(lay, NQ_P_DOWN), proj, NQ_SHIFT_Q16)
401 var fo: i64=0; while fo<ne { cur[fo]=hmid[fo]+proj[fo]; fo=fo+1 }
402 L=L+1
403 }
404 cpy(out1, cur, ne)
405 return 0
406}