nx_f32_lazy_weight.nx source
↩ module page · 886 lines · 41152 B
1// nx_f32_lazy_weight.nx -- format-tagged weight tensor + matmul dispatcher.
2//
3// The foundation brick for lazy-Q4_K integration into the forward
4// path. Current model bindings (NxF32LlamaLayer) hold raw *i64
5// pointers to eagerly-dequanted f32 storage. Lazy variants hold
6// either f32 storage OR a (GGUF buffer + offset) pair for on-the-
7// fly Q4_K dequant during matmul.
8//
9// A future v4 forward + binder will use NxF32LazyWeight throughout.
10// This brick provides the type + dispatcher; integration with the
11// per-layer struct + forward is a separate brick to keep scope tight.
12//
13// genealogy_id: tagged_union_dispatch + ggml_format_taxonomy
14// lineage_id: substrate_f32_lazy_weight_v1
15
16import "nx_syscalls.nx"
17import "nx_tier.nx"
18import "nx_f32_matmul.nx"
19import "nx_f32_matmul_t.nx"
20import "nx_f32_q4k_matmul.nx"
21import "nx_q5_0_to_f32.nx"
22import "nx_q8_0_to_f32.nx"
23import "nx_le.nx"
24import "nx_pteam.nx"
25
26const NX_LW_DTYPE_F32: nx_int = 0
27const NX_LW_DTYPE_Q4_K: nx_int = 1
28const NX_LW_DTYPE_Q5_0: nx_int = 2 // 2026-07-08: keep Q5_0 quantized (3MB vs 34.9MB F32), fused dequant-dot
29const NX_LW_DTYPE_Q8_0: nx_int = 3 // 2026-07-08: keep Q8_0 quantized; SIMD __f32_i8dot32 dequant-dot = 10.2x (lm_head)
30
31const NX_LW_OK: nx_int = 0
32const NX_LW_ERR_BAD_TYPE: nx_int = 1
33const NX_LW_ERR_NULL: nx_int = 2
34const NX_LW_ERR_INNER: nx_int = 3 // inner matmul verdict != OK (e.g. Q4_K ERR_ALIGN -- was silently OK before 2026-07-07)
35const NX_LW_N_VERDICTS: nx_int = 4
36
37func nx_lw_verdict_is_valid(v: nx_int) -> nx_int {
38 if v < 0 { return 0 }
39 if v >= NX_LW_N_VERDICTS { return 0 }
40 return 1
41}
42
43struct NxF32LazyWeight {
44 dtype_tag: nx_int, // NX_LW_DTYPE_F32 or NX_LW_DTYPE_Q4_K
45 rows: nx_int, // outer dim
46 cols: nx_int, // inner dim
47 storage: *i64, // F32: raw f32 bits buffer (else 0)
48 q4k_bytes: *u8, // Q4_K: pointer into GGUF buffer (else 0)
49 q4k_offset: i64, // Q4_K: byte offset of tensor data (else 0)
50 pk_cache: i64, // weight cache base (0 = none). FORMAT FOLLOWS DTYPE: F32 -> packed f32;
51 // Q4_K -> Q24 integers (4 B/value either way, one budget). 2026-08-01.
52 pk_state: i64 // 0 = not tried, 1 = cached, -1 = refused (over budget)
53}
54
55const NX_LW_BYTES: nx_int = 64 // 8 fields * 8
56
57func nx_f32_lazy_weight_new_f32(storage: *i64, rows: nx_int, cols: nx_int) -> *NxF32LazyWeight {
58 let lw: *NxF32LazyWeight = sys_mmap(NX_LW_BYTES) as *NxF32LazyWeight
59 lw.dtype_tag = NX_LW_DTYPE_F32
60 lw.rows = rows
61 lw.cols = cols
62 lw.storage = storage
63 lw.q4k_bytes = 0 as *u8
64 lw.q4k_offset = 0
65 lw.pk_cache = 0
66 lw.pk_state = 0
67 return lw
68}
69
70func nx_f32_lazy_weight_new_q4k(buf: *u8, offset: i64,
71 rows: nx_int, cols: nx_int) -> *NxF32LazyWeight {
72 let lw: *NxF32LazyWeight = sys_mmap(NX_LW_BYTES) as *NxF32LazyWeight
73 lw.dtype_tag = NX_LW_DTYPE_Q4_K
74 lw.rows = rows
75 lw.cols = cols
76 lw.storage = 0 as *i64
77 lw.q4k_bytes = buf
78 lw.q4k_offset = offset
79 lw.pk_cache = 0
80 lw.pk_state = 0
81 return lw
82}
83
84// Q5_0: keep quantized (reuse q4k_bytes/q4k_offset as the quantized-
85// byte pointer+offset). The dispatcher's Q5_0 branch runs a fused
86// dequant-dot that reads the 22-byte/32-val blocks (~0.69 B/val) instead
87// of the 8 B/val F32 the eager path materialized.
88func nx_f32_lazy_weight_new_q5_0(buf: *u8, offset: i64,
89 rows: nx_int, cols: nx_int) -> *NxF32LazyWeight {
90 let lw: *NxF32LazyWeight = sys_mmap(NX_LW_BYTES) as *NxF32LazyWeight
91 lw.dtype_tag = NX_LW_DTYPE_Q5_0
92 lw.rows = rows
93 lw.cols = cols
94 lw.storage = 0 as *i64
95 lw.q4k_bytes = buf
96 lw.q4k_offset = offset
97 lw.pk_cache = 0
98 lw.pk_state = 0
99 return lw
100}
101
102func nx_f32_lazy_weight_new_q8_0(buf: *u8, offset: i64,
103 rows: nx_int, cols: nx_int) -> *NxF32LazyWeight {
104 let lw: *NxF32LazyWeight = sys_mmap(NX_LW_BYTES) as *NxF32LazyWeight
105 lw.dtype_tag = NX_LW_DTYPE_Q8_0
106 lw.rows = rows
107 lw.cols = cols
108 lw.storage = 0 as *i64
109 lw.q4k_bytes = buf
110 lw.q4k_offset = offset
111 lw.pk_cache = 0
112 lw.pk_state = 0
113 return lw
114}
115
116// Process-wide shared thread pool for the Q4_K matmul (spawn-once,
117// hw-sized). This is the "pool (fork-once) is the fix" wiring from
118// nx_f32_q4k_matmul, landed 2026-07-07: every dispatcher caller
119// (block_v4's 7 matmuls/layer, the probes) gets the column-band
120// multicore path with zero call-site changes. Lazy-static-pointer
121// pattern (nx_laziststatic_probe-proven). SINGLE-SUBMITTER: the
122// forward drives matmuls from one thread; pool workers never
123// re-enter this dispatcher.
124static g_lw_pool: *NxThreadPool
125
126// native: cap the block-op pool below the over-subscription CLIFF (measured
127// 2026-07-13: fork-join >16 workers on this 20-logical/10-phys box cliffs to
128// 0.8x; the matmul runs on pteam-12, so the block-op pool is a SECOND team ->
129// keep it small). WSL2 (native=0) keeps 0 => hw_worker_count, UNCHANGED.
130const NX_LW_NATIVE_POOL_W: i64 = 8
131
132func nx_lw_shared_pool() -> *NxThreadPool {
133 if (g_lw_pool as i64) == 0 {
134 var nw: i64 = 0
135 if nx_pool_is_native() != 0 { nw = NX_LW_NATIVE_POOL_W }
136 g_lw_pool = nx_pool_new(nw, 0)
137 }
138 return g_lw_pool
139}
140
141// ===== Packed dequant-once cache (2026-07-08) =====================
142//
143// MEASURED FINDING behind this design (nx_q4k_matmul_x4_gate ladder,
144// m=1 decode): the per-token cost is DOMINATED by re-dequanting the
145// same weights every call -- SIMD-ing only the dot lost to the
146// pooled scalar path twice (0.81x byte-wise, 0.84x pairwise stores).
147// The lever is to stop re-dequanting: on first use a Q4_K weight
148// dequants ONCE (pool-threaded) into a packed 4-byte f32 cache
149// [cols rows x rows values]; every later matmul is then pure
150// contiguous packed dots (__f32x4_dot), which is where the SIMD
151// lever actually pays.
152//
153// BUDGETED, not blanket: packed f32 is ~7x the Q4_K bytes (a 7B
154// model would want ~14GB -- the recorded 15GB-budget constraint).
155// Config hierarchy: callers override via nx_lw_set_cache_budget
156// (env/svc-config tiers wire in there); the code-tier default below
157// is the last-resort fallback. Over budget -> pk_state=-1 and the
158// weight streams via the pooled scalar path exactly as before.
159// Single-submitter contract as the shared pool (the forward drives
160// from one thread).
161
162// u2605DEFAULT FLIPPED TO 0 = STREAM 4-BIT (2026-08-01), MEASURED, not reasoned.
163// The 4 GiB default enabled the packed-f32 dequant-once cache on every model small enough to fit it, which is
164// every model these seats actually run. That cache is ~7x the Q4_K bytes, so the decode hot loop read PACKED F32
165// and the banked int4 roofline (221ms/tok f32 vs 27ms/tok int4) described a path nobody took. The cache was
166// introduced 2026-07-08 on a real measurement -- re-dequant dominated and SIMD-ing the dot alone LOST (0.81x) --
167// but that measurement predates the SIMD dequant-dot now in fq4m_rows, and it has now been RE-MEASURED head to
168// head on the same host, same model, model loaded once, via nx_embed_bench:
169// ms/token m=8 m=16 m=32 m=64 m=125
170// cached 282 418 412 326 496
171// streaming 178 217 204 225 361 <- STREAMING WINS AT EVERY POINT, 1.6x..2.0x
172// Deltas of 100-200ms against a rep-to-rep spread of ~50-100ms, same direction at all five m. So bytes-moved
173// DOES dominate here and the cache was buying dequant-compute nobody needed any more.
174// u2605A MEASURED TRADE-OFF EXPIRES WHEN EITHER SIDE OF THE TRADE CHANGES -- re-measure a policy when you speed up
175// the thing it was avoiding.
176// Callers that genuinely want the cache (repeated matmuls against one weight, tiny k, huge m) can still opt in
177// via nx_lw_set_cache_budget; this only changes the DEFAULT from cache-on to stream-on.
178const NX_LW_CACHE_BUDGET_DEFAULT: i64 = 0 // 0 = stream quantized weights (re-measured 2026-08-01)
179
180static g_lw_cache_budget: i64
181static g_lw_cache_used: i64
182
183func nx_lw_set_cache_budget(bytes: i64) -> i64 {
184 g_lw_cache_budget = bytes
185 return 0
186}
187
188func nx_lw_cache_used() -> i64 {
189 return g_lw_cache_used
190}
191
192func _lw_budget() -> i64 {
193 if g_lw_cache_budget == 0 { g_lw_cache_budget = NX_LW_CACHE_BUDGET_DEFAULT }
194 return g_lw_cache_budget
195}
196
197// Pool task: dequant weight rows [jlo,jhi) straight into the packed
198// cache. Reuses NxFq4mCtx: a_ptr=cache base, b_ptr=gguf bytes,
199// b_off=tensor offset, k=row length, jlo/jhi=row band.
200func _lw_fill_task(ctx_i: i64) -> i64 {
201 let cx: *NxFq4mCtx = ctx_i as *NxFq4mCtx
202 let bpr: i64 = (cx.k / NX_GL_Q4_K_VPB) * NX_GL_Q4_K_BPB
203 var j: i64 = cx.jlo
204 while j < cx.jhi {
205 let row_off: i64 = cx.b_off + j * bpr
206 let dst: *u8 = (cx.a_ptr + j * cx.k * 4) as *u8
207 nx_q4k_to_f32_packed(cx.b_ptr as *u8, row_off, cx.k, dst)
208 j = j + 1
209 }
210 return 0
211}
212
213// Pool task: PACK F32 weight rows [jlo,jhi) (i64-slot f32) -> the
214// contiguous 4-byte packed cache. Just a narrowing copy (the low 32
215// bits of each slot ARE the f32); no dequant. b_ptr = W.storage.
216// This is what lets F32 weights ride the same __f32x4_dot cached path
217// as Q4_K (2026-07-08: F32 was the matmul majority, running scalar).
218func _lw_fill_f32_task(ctx_i: i64) -> i64 {
219 let cx: *NxFq4mCtx = ctx_i as *NxFq4mCtx
220 let src: *i64 = cx.b_ptr as *i64
221 let dstbase: i64 = cx.a_ptr
222 let k: i64 = cx.k
223 var j: i64 = cx.jlo
224 while j < cx.jhi {
225 let so: i64 = j * k
226 let dst: *u8 = (dstbase + j * k * 4) as *u8
227 var l: i64 = 0
228 while l < k {
229 let bits: i64 = src[so + l]
230 dst[l * 4 + 0] = bits as u8
231 dst[l * 4 + 1] = (bits >> 8) as u8
232 dst[l * 4 + 2] = (bits >> 16) as u8
233 dst[l * 4 + 3] = (bits >> 24) as u8
234 l = l + 1
235 }
236 j = j + 1
237 }
238 return 0
239}
240
241// First-use fill: allocate + pool-threaded fill of all n rows. For
242// Q4_K, dequant (nx_q4k_to_f32_packed); for F32, narrowing pack.
243// Returns 1 on success (pk_state=1), 0 on refusal (pk_state=-1).
244func _lw_try_fill(W: *NxF32LazyWeight, k: nx_int, n: nx_int) -> i64 {
245 let need: i64 = (n as i64) * (k as i64) * 4
246 let used: i64 = g_lw_cache_used
247 if used + need > _lw_budget() {
248 W.pk_state = 0 - 1
249 return 0
250 }
251 let cache: *u8 = sys_mmap(need)
252 let pool: *NxThreadPool = nx_lw_shared_pool()
253 var bands: i64 = pool.n_workers
254 if bands > n { bands = n }
255 if bands < 1 { bands = 1 }
256 let ctxs: *u8 = sys_mmap(bands * NX_FQ4M_CTX_BYTES)
257 let rpb: i64 = (n + bands - 1) / bands
258 let done_before: i64 = nx_pool_n_completed(pool)
259 var b: i64 = 0
260 while b < bands {
261 let cx: *NxFq4mCtx = ((ctxs as i64) + b * NX_FQ4M_CTX_BYTES) as *NxFq4mCtx
262 cx.a_ptr = cache as i64
263 cx.c_ptr = 0
264 cx.m = 0
265 cx.k = k
266 cx.n = n
267 cx.jlo = b * rpb
268 var jhi: i64 = (b + 1) * rpb
269 if jhi > n { jhi = n }
270 cx.jhi = jhi
271 if W.dtype_tag == NX_LW_DTYPE_F32 {
272 cx.b_ptr = W.storage as i64
273 cx.b_off = 0
274 nx_pool_submit(pool, _lw_fill_f32_task, cx as i64)
275 } else {
276 cx.b_ptr = W.q4k_bytes as i64
277 cx.b_off = W.q4k_offset
278 // DOMAIN-UNIFIED CACHE 2026-08-01: fill Q24 integers (fq4m_fill_q24), not dequantised
279 // f32. Same footprint (4 B/value, budget accounting unchanged); the cached dot then runs
280 // the SAME Q24 x Q20 -> Q44 accumulate as the fused streamed path, so LWC-8's
281 // cached==streamed holds BIT-FOR-BIT by construction -- exact integer math has no
282 // reassociation to diverge. This is the "make the cache hold Q4_K and fuse both paths"
283 // contract resolution, not a tolerance.
284 nx_pool_submit(pool, _nx_fq4m_task_fill_q24, cx as i64)
285 }
286 b = b + 1
287 }
288 let wv: i64 = nx_pool_wait(pool, done_before + bands)
289 sys_munmap(ctxs, bands * NX_FQ4M_CTX_BYTES)
290 if wv != 0 {
291 sys_munmap(cache, need)
292 W.pk_state = 0 - 1
293 return 0
294 }
295 W.pk_cache = cache as i64
296 W.pk_state = 1
297 g_lw_cache_used = used + need
298 return 1
299}
300
301// VECTOR-ACCUMULATOR range dot (2026-07-08): 8-wide __f32x8_fma across
302// the whole reduction with ONE __f32x8_hsum at the end. MEASURED 2.72x
303// over the __f32x4_dot loop (which hsummed every 4 = ~scalar,
304// nx_f32x8_range_gate 2/2). `acc` = caller-owned 32-byte (8 f32)
305// scratch, re-zeroed per call. Scalar tail for count % 8 (LLM dims are
306// all /8 so the tail is empty there, but kept for generality).
307func _lw_dot_x8(a: *u8, b: *u8, count: i64, acc: *u8) -> i64 {
308 let az: *i64 = acc as *i64
309 az[0] = 0; az[1] = 0; az[2] = 0; az[3] = 0
310 let ab: i64 = a as i64
311 let bb: i64 = b as i64
312 let n8: i64 = (count / 8) * 8
313 var l: i64 = 0
314 while l < n8 {
315 __f32x8_fma(acc, (ab + l * 4) as *u8, (bb + l * 4) as *u8)
316 l = l + 8
317 }
318 var s: i64 = __f32x8_hsum(acc)
319 while l < count {
320 let av: i64 = (a[l*4] as i64) | ((a[l*4+1] as i64) << 8) | ((a[l*4+2] as i64) << 16) | ((a[l*4+3] as i64) << 24)
321 let bv: i64 = (b[l*4] as i64) | ((b[l*4+1] as i64) << 8) | ((b[l*4+2] as i64) << 16) | ((b[l*4+3] as i64) << 24)
322 s = __f32_add(s, __f32_mul(av, bv))
323 l = l + 1
324 }
325 return s
326}
327
328// Pool task: pure packed-dot bands over the cache (no dequant).
329// NxFq4mCtx: a_ptr=packed A, b_ptr=cache base, c_ptr=C.
330func _lw_dot_task(ctx_i: i64) -> i64 {
331 let cx: *NxFq4mCtx = ctx_i as *NxFq4mCtx
332 let pab: i64 = cx.a_ptr
333 let wcb: i64 = cx.b_ptr
334 let C: *i64 = cx.c_ptr as *i64
335 let m: i64 = cx.m
336 let k: i64 = cx.k
337 let n: i64 = cx.n
338 let acc: *u8 = sys_mmap(32)
339 var j: i64 = cx.jlo
340 while j < cx.jhi {
341 let wbase: i64 = wcb + j * k * 4
342 var i: i64 = 0
343 while i < m {
344 let abase: i64 = pab + i * k * 4
345 C[i * n + j] = _lw_dot_x8(abase as *u8, wbase as *u8, k, acc)
346 i = i + 1
347 }
348 j = j + 1
349 }
350 sys_munmap(acc, 32)
351 return 0
352}
353
354// Cached matmul: pack A once, banded dots over the cache. THE CACHE FORMAT IS DECIDED BY THE
355// DTYPE, never by the cache itself: F32 weights cache packed f32 and dot in f32 (x8 range-dot);
356// Q4_K weights cache Q24 integers and dot in the SAME Q24 x Q20 -> Q44 domain as the fused
357// streamed path. One domain per dtype on BOTH sides of the cache -- the LWC-8 lesson made
358// structural (a quantisation change must apply to both sides of a cache, or neither).
359func _lw_cached_matmul(W: *NxF32LazyWeight, A: *i64, C: *i64,
360 m: nx_int, k: nx_int, n: nx_int) -> nx_int {
361 let pool: *NxThreadPool = nx_lw_shared_pool()
362 if W.dtype_tag == NX_LW_DTYPE_Q4_K {
363 // Q20-pack the activations -- the IDENTICAL pack the fused streamed path performs
364 // (_fq4m_pack_q10), so cached and streamed consume byte-identical inputs.
365 let pq: *i64 = sys_mmap(m * k * 8) as *i64
366 _fq4m_pack_q10(A, m * k, pq)
367 var qbands: i64 = pool.n_workers
368 if qbands > n { qbands = n }
369 if qbands < 1 { qbands = 1 }
370 let qctxs: *u8 = sys_mmap(qbands * NX_FQ4M_CTX_BYTES)
371 let qcpb: i64 = (n + qbands - 1) / qbands
372 let qdone: i64 = nx_pool_n_completed(pool)
373 var qb: i64 = 0
374 while qb < qbands {
375 let qcx: *NxFq4mCtx = ((qctxs as i64) + qb * NX_FQ4M_CTX_BYTES) as *NxFq4mCtx
376 qcx.a_ptr = pq as i64
377 qcx.b_ptr = W.pk_cache
378 qcx.b_off = 0
379 qcx.c_ptr = C as i64
380 qcx.m = m
381 qcx.k = k
382 qcx.n = n
383 qcx.jlo = qb * qcpb
384 var qjhi: i64 = (qb + 1) * qcpb
385 if qjhi > n { qjhi = n }
386 qcx.jhi = qjhi
387 nx_pool_submit(pool, _nx_fq4m_task_cached_q24, qcx as i64)
388 qb = qb + 1
389 }
390 let qwv: i64 = nx_pool_wait(pool, qdone + qbands)
391 sys_munmap(qctxs, qbands * NX_FQ4M_CTX_BYTES)
392 sys_munmap(pq as *u8, m * k * 8)
393 if qwv != 0 { return NX_LW_ERR_INNER }
394 return NX_LW_OK
395 }
396 let pa: *u8 = sys_mmap(m * k * 4)
397 _fq4m_pack_a(A, m * k, pa)
398 var bands: i64 = pool.n_workers
399 if bands > n { bands = n }
400 if bands < 1 { bands = 1 }
401 let ctxs: *u8 = sys_mmap(bands * NX_FQ4M_CTX_BYTES)
402 let cpb: i64 = (n + bands - 1) / bands
403 let done_before: i64 = nx_pool_n_completed(pool)
404 var b: i64 = 0
405 while b < bands {
406 let cx: *NxFq4mCtx = ((ctxs as i64) + b * NX_FQ4M_CTX_BYTES) as *NxFq4mCtx
407 cx.a_ptr = pa as i64
408 cx.b_ptr = W.pk_cache
409 cx.b_off = 0
410 cx.c_ptr = C as i64
411 cx.m = m
412 cx.k = k
413 cx.n = n
414 cx.jlo = b * cpb
415 var jhi: i64 = (b + 1) * cpb
416 if jhi > n { jhi = n }
417 cx.jhi = jhi
418 nx_pool_submit(pool, _lw_dot_task, cx as i64)
419 b = b + 1
420 }
421 let wv: i64 = nx_pool_wait(pool, done_before + bands)
422 sys_munmap(ctxs, bands * NX_FQ4M_CTX_BYTES)
423 sys_munmap(pa, m * k * 4)
424 if wv != 0 { return NX_LW_ERR_INNER }
425 return NX_LW_OK
426}
427
428// ===== Q5_0 FULLY-SIMD dequant-dot (2026-07-08) — WINS 1.10× =====
429// __q5_unpack32 (SSE nibble+qh-bit-spread -> 32 int8) then __f32_i8dot32
430// (SSE convert+dot), x d. Reads ~11.6x fewer bytes (0.69 vs 8 B/val) on the
431// 79%-Q5_0 weights + cheap SIMD dequant -> nx_q5_0_threaded_gate flips 0.48x
432// (scalar) -> 1.10x (SIMD, bit-exact). qh at block+2, qs at block+6 are
433// contiguous so qhqs=block+2. Apk = A packed 4-byte once per matmul.
434static g_q5_consts: i64
435func _lw_q5_consts() -> *u8 {
436 if g_q5_consts == 0 {
437 let cc: *u8 = sys_mmap(80)
438 var i: i64 = 0
439 while i < 16 { cc[i] = 0x0F as u8; i = i + 1 }
440 i = 0
441 while i < 8 { cc[16+i] = 0 as u8; i = i + 1 }
442 while i < 16 { cc[16+i] = 1 as u8; i = i + 1 }
443 i = 0
444 while i < 8 { cc[32+i] = 2 as u8; i = i + 1 }
445 while i < 16 { cc[32+i] = 3 as u8; i = i + 1 }
446 i = 0
447 while i < 8 { cc[48+i] = (1 << i) as u8; cc[48+8+i] = (1 << i) as u8; i = i + 1 }
448 i = 0
449 while i < 16 { cc[64+i] = 0x10 as u8; i = i + 1 }
450 g_q5_consts = cc as i64
451 }
452 return g_q5_consts as *u8
453}
454
455func _lw_q5_0_dot(qbuf: *u8, qoff: i64, Apk: *u8, k: i64, i8scr: *u8, consts: *u8) -> i64 {
456 let nblk: i64 = k / NX_Q5_0_VPB
457 let apb: i64 = Apk as i64
458 let qpb: i64 = qbuf as i64
459 var acc: i64 = 0
460 var b: i64 = 0
461 while b < nblk {
462 let boff: i64 = qoff + b * NX_Q5_0_BPB
463 let d_f32: i64 = nx_f16_to_f32(nx_le_read_u16(qbuf, boff + 0))
464 __q5_unpack32((qpb + boff + 2) as *u8, i8scr, consts)
465 let raw: i64 = __f32_i8dot32(i8scr, (apb + b * NX_Q5_0_VPB * 4) as *u8)
466 acc = __f32_add(acc, __f32_mul(d_f32, raw))
467 b = b + 1
468 }
469 return acc
470}
471
472func _lw_q5_0_task(ctx_i: i64) -> i64 {
473 let cx: *NxFq4mCtx = ctx_i as *NxFq4mCtx
474 let i8scr: *u8 = sys_mmap(NX_Q5_0_VPB)
475 let consts: *u8 = _lw_q5_consts()
476 let bpr: i64 = (cx.k / NX_Q5_0_VPB) * NX_Q5_0_BPB
477 let qbuf: *u8 = cx.b_ptr as *u8
478 let base: i64 = cx.b_off
479 let C: *i64 = cx.c_ptr as *i64
480 var j: i64 = cx.jlo
481 while j < cx.jhi {
482 var i: i64 = 0
483 while i < cx.m {
484 let Arow: *u8 = ((cx.a_ptr) + i * cx.k * 4) as *u8
485 C[i * cx.n + j] = _lw_q5_0_dot(qbuf, base + j * bpr, Arow, cx.k, i8scr, consts)
486 i = i + 1
487 }
488 j = j + 1
489 }
490 sys_munmap(i8scr, NX_Q5_0_VPB)
491 return 0
492}
493
494func _lw_q5_0_matmul(W: *NxF32LazyWeight, A: *i64, C: *i64,
495 m: i64, k: i64, n: i64) -> nx_int {
496 let pool: *NxThreadPool = nx_lw_shared_pool()
497 let Apk: *u8 = sys_mmap(m * k * 4)
498 _fq4m_pack_a(A, m * k, Apk)
499 var bands: i64 = pool.n_workers
500 if bands > n { bands = n }
501 if bands < 1 { bands = 1 }
502 let ctxs: *u8 = sys_mmap(bands * NX_FQ4M_CTX_BYTES)
503 let cpb: i64 = (n + bands - 1) / bands
504 let done_before: i64 = nx_pool_n_completed(pool)
505 var b: i64 = 0
506 while b < bands {
507 let cx: *NxFq4mCtx = ((ctxs as i64) + b * NX_FQ4M_CTX_BYTES) as *NxFq4mCtx
508 cx.a_ptr = Apk as i64
509 cx.b_ptr = W.q4k_bytes as i64
510 cx.b_off = W.q4k_offset
511 cx.c_ptr = C as i64
512 cx.m = m
513 cx.k = k
514 cx.n = n
515 cx.jlo = b * cpb
516 var jhi: i64 = (b + 1) * cpb
517 if jhi > n { jhi = n }
518 cx.jhi = jhi
519 nx_pool_submit(pool, _lw_q5_0_task, cx as i64)
520 b = b + 1
521 }
522 let wv: i64 = nx_pool_wait(pool, done_before + bands)
523 sys_munmap(ctxs, bands * NX_FQ4M_CTX_BYTES)
524 sys_munmap(Apk, m * k * 4)
525 if wv != 0 { return NX_LW_ERR_INNER }
526 return NX_LW_OK
527}
528
529// ===== Q8_0 SIMD dequant-dot (2026-07-08) — PROVEN 10.2x =========
530// Q8_0 = 34-byte/32-val blocks (f16 d + 32 int8), NO nibble unpack, so the
531// blessed __f32_i8dot32 intrinsic does convert+dot in SSE directly:
532// block = d * i8dot32(int8_block, A_block). Apk = A packed to contiguous
533// 4-byte f32 once per matmul. Reads ~1 B/val vs 8 B/val F32 AND cheap SIMD
534// dequant -> nx_q8_0_simd_gate measured 10.2x over the F32 matmul at lm_head
535// scale. lm_head is Q8_0 (136M vals, 143ms/token) -> this is its win.
536// Register-acc row dot: per-block __f32_i8dot32a (AVX2 dot + hsum-in-register)
537// scaled by d, accumulated in a scalar register. MEASURED-FINAL 2026-07-10:
538// the deferred-hsum factoring (per-block __f32_i8fma32 into a MEMORY acc, one
539// __f32x8_hsum) was SLOWER (MATMUL 41->51ms) -- factoring the fma per-block
540// forces 28 acc load+stores/output through memory, costing as much as the 28
541// hsums it removed. The real deferred-hsum win needs a MONOLITHIC row kernel
542// (register acc across all blocks) -- a bigger intrinsic (block loop + f16
543// decode in asm), the next arc. __f32_i8fma32 stays BLESSED for it.
544// Register-acc per-block dot (i8dot32a). MEASURED-FINAL 2026-07-10: THREE
545// dot-kernel factorings all failed to move the forward matmul, proving the
546// gap is NOT the dot kernel: (1) AVX2 dot-width i8dot32a = 0× (hsum, not
547// width); (2) per-block deferred-hsum i8fma32 = SLOWER (memory acc round-
548// trip); (3) MONOLITHIC __f32_q8row_dot (register acc + F16C + 1 hsum) = ~6%
549// SLOWER (loop-branch + vcvtph2ps latency offset the hsum saving). The real
550// 2.7× gap vs gcc-in-WSL2 (25.5 vs 9.5 GB/s) is THREADING + MEMORY ACCESS
551// (OpenMP schedule + prefetch), not the kernel -- the next arc, different
552// layer. __f32_i8dot32a/i8fma32/q8row_dot all stay BLESSED. i8dot32a wins.
553func _lw_q8_0_dot(qbuf: *u8, qoff: i64, Apk: *u8, k: i64) -> i64 {
554 let nblk: i64 = k / NX_Q8_0_VPB
555 let qpb: i64 = qbuf as i64
556 let apb: i64 = Apk as i64
557 var acc: i64 = 0
558 var b: i64 = 0
559 while b < nblk {
560 let boff: i64 = qoff + b * NX_Q8_0_BPB
561 let d_f32: i64 = nx_f16_to_f32(nx_le_read_u16(qbuf, boff))
562 let raw: i64 = __f32_i8dot32a((qpb + boff + 2) as *u8, (apb + b * NX_Q8_0_VPB * 4) as *u8)
563 acc = __f32_add(acc, __f32_mul(d_f32, raw))
564 b = b + 1
565 }
566 return acc
567}
568
569func _lw_q8_0_task(ctx_i: i64) -> i64 {
570 let cx: *NxFq4mCtx = ctx_i as *NxFq4mCtx
571 let bpr: i64 = (cx.k / NX_Q8_0_VPB) * NX_Q8_0_BPB
572 let qbuf: *u8 = cx.b_ptr as *u8
573 let base: i64 = cx.b_off
574 let C: *i64 = cx.c_ptr as *i64
575 var j: i64 = cx.jlo
576 while j < cx.jhi {
577 var i: i64 = 0
578 while i < cx.m {
579 let Arow: *u8 = ((cx.a_ptr) + i * cx.k * 4) as *u8
580 C[i * cx.n + j] = _lw_q8_0_dot(qbuf, base + j * bpr, Arow, cx.k)
581 i = i + 1
582 }
583 j = j + 1
584 }
585 return 0
586}
587
588// SINGLE-THREADED Q8_0 matmul (2026-07-10). MEASURED (nx_q8_matmul_micro):
589// the shared pool costs ~1.5ms/call of dispatch/sync at m=1 (a 128-wide
590// matmul took LONGER than a 4864-wide one), so 168 tiny per-token matmuls
591// pay ~200ms of pure overhead -- the real decode bottleneck (NOT the kernel,
592// NOT cold streaming which is only 1.7x). For small work the dispatch tax
593// dwarfs the 14x compute speedup, so run inline: no pool, no per-call mmap
594// (a lazy-static Apk scratch), no ctx setup. The pool stays for big matmuls
595// (gate/up/down/lm_head) where 14x compute beats the tax, and is UNTOUCHED
596// for every other user.
597static g_q8st_apk: i64 // reusable packed-A scratch ptr
598static g_q8st_cap: i64 // its capacity in bytes
599func _q8st_apk(need: i64) -> *u8 {
600 if g_q8st_cap < need {
601 g_q8st_apk = sys_mmap(need) as i64 // grow (old leaks; rare, bounded by max k)
602 g_q8st_cap = need
603 }
604 return g_q8st_apk as *u8
605}
606
607func _lw_q8_0_matmul_st(W: *NxF32LazyWeight, A: *i64, C: *i64,
608 m: i64, k: i64, n: i64) -> nx_int {
609 let Apk: *u8 = _q8st_apk(m * k * 4)
610 _fq4m_pack_a(A, m * k, Apk)
611 let bpr: i64 = (k / NX_Q8_0_VPB) * NX_Q8_0_BPB
612 let qbuf: *u8 = W.q4k_bytes
613 let base: i64 = W.q4k_offset
614 let apb: i64 = Apk as i64
615 var j: i64 = 0
616 while j < n {
617 var i: i64 = 0
618 while i < m {
619 C[i * n + j] = _lw_q8_0_dot(qbuf, base + j * bpr, (apb + i * k * 4) as *u8, k)
620 i = i + 1
621 }
622 j = j + 1
623 }
624 return NX_LW_OK
625}
626
627// Route small work to the single-threaded path. Threshold = total MACs
628// (m*k*n). HISTORY: under the OLD yield-spin pool (~1.5ms dispatch tax)
629// small matmuls won 34x on ST -> threshold 1.2M. After the futex
630// spin-then-block pool landed (2026-07-10) the mix A/B FLIPPED: all-pool
631// beats threshold-routing 1.5x (back-to-back dispatches land in the
632// workers' spin window = near-zero latency; ST gaps let workers sleep).
633// Threshold now 0 = everything pooled. ST path + micro retained as the
634// dispatch-regression diagnostic (if pool kvproj per-call creeps back
635// toward ms, the tax returned).
636const NX_Q8_ST_MAC_MAX: i64 = 0
637
638// STATIC ctx-band scratch (2026-07-10): the pool path used to sys_mmap+
639// sys_munmap Apk AND ctxs EVERY matmul -- 4 syscalls/matmul, and munmap
640// forces a cross-core TLB SHOOTDOWN (IPI to all workers) since the buffers
641// are touched by the pool threads. 168 matmuls/token x that = a big chunk
642// of the ~319us/matmul pool overhead that capped scaling at 4.25x (vs gcc
643// 8.8x). Reuse a static ctxs (like the ST path's static Apk); single-
644// submitter invariant makes it safe (one matmul at a time drives the pool).
645static g_q8mt_ctxs: i64
646static g_q8mt_ctxs_cap: i64
647func _q8mt_ctxs(need: i64) -> *u8 {
648 if g_q8mt_ctxs_cap < need { g_q8mt_ctxs = sys_mmap(need) as i64; g_q8mt_ctxs_cap = need }
649 return g_q8mt_ctxs as *u8
650}
651
652func _lw_q8_0_matmul_pool_force(W: *NxF32LazyWeight, A: *i64, C: *i64,
653 m: i64, k: i64, n: i64) -> nx_int {
654 let pool: *NxThreadPool = nx_lw_shared_pool()
655 let Apk: *u8 = _q8st_apk(m * k * 4) // static, no per-matmul mmap
656 _fq4m_pack_a(A, m * k, Apk)
657 var bands: i64 = pool.n_workers
658 if bands > n { bands = n }
659 if bands < 1 { bands = 1 }
660 let ctxs: *u8 = _q8mt_ctxs(bands * NX_FQ4M_CTX_BYTES) // static, no mmap/munmap
661 let cpb: i64 = (n + bands - 1) / bands
662 let done_before: i64 = nx_pool_n_completed(pool)
663 var b: i64 = 0
664 while b < bands {
665 let cx: *NxFq4mCtx = ((ctxs as i64) + b * NX_FQ4M_CTX_BYTES) as *NxFq4mCtx
666 cx.a_ptr = Apk as i64
667 cx.b_ptr = W.q4k_bytes as i64
668 cx.b_off = W.q4k_offset
669 cx.c_ptr = C as i64
670 cx.m = m
671 cx.k = k
672 cx.n = n
673 cx.jlo = b * cpb
674 var jhi: i64 = (b + 1) * cpb
675 if jhi > n { jhi = n }
676 cx.jhi = jhi
677 nx_pool_submit(pool, _lw_q8_0_task, cx as i64)
678 b = b + 1
679 }
680 let wv2: i64 = nx_pool_wait(pool, done_before + bands)
681 // (no munmap -- Apk/ctxs are static, reused; kills the per-matmul TLB shootdown)
682 if wv2 != 0 { return NX_LW_ERR_INNER }
683 return NX_LW_OK
684}
685
686// ===== PTEAM (fork-join barrier) Q8_0 matmul (2026-07-10) =========
687// The channel pool scales ~4.5x/14thr (gcc 8.8x) because workers pick tasks
688// off the MPMC channel via contended CAS -> serialized start. nx_pteam gives
689// each worker a FIXED band read from a shared descriptor -> simultaneous
690// start. ONE shared ctx, band-indexed (cpb stored in the .jlo field).
691static g_q8_pteam: i64
692func _q8_pteam() -> *NxPTeam {
693 if g_q8_pteam == 0 { g_q8_pteam = nx_pteam_new(12) as i64 } // native sweet spot (best-of-3 matmul: 4w=2.6x 8w=4.5x 12w=5.8x 14w=6.1x; 16w CLIFFS to 0.8x — over-subscribe on 20-logical/10-phys)
694 return g_q8_pteam as *NxPTeam
695}
696func _lw_q8_0_pteam_band(band: i64, ctx: i64) -> i64 {
697 let cx: *NxFq4mCtx = ctx as *NxFq4mCtx
698 let cpb: i64 = cx.jlo // cpb stashed in jlo
699 var jlo: i64 = band * cpb
700 var jhi: i64 = (band + 1) * cpb
701 if jhi > cx.n { jhi = cx.n }
702 let bpr: i64 = (cx.k / NX_Q8_0_VPB) * NX_Q8_0_BPB
703 let qbuf: *u8 = cx.b_ptr as *u8
704 let base: i64 = cx.b_off
705 let C: *i64 = cx.c_ptr as *i64
706 var j: i64 = jlo
707 while j < jhi {
708 var i: i64 = 0
709 while i < cx.m {
710 let Arow: *u8 = ((cx.a_ptr) + i * cx.k * 4) as *u8
711 C[i * cx.n + j] = _lw_q8_0_dot(qbuf, base + j * bpr, Arow, cx.k)
712 i = i + 1
713 }
714 j = j + 1
715 }
716 return 0
717}
718func _lw_q8_0_matmul_pteam(W: *NxF32LazyWeight, A: *i64, C: *i64,
719 m: i64, k: i64, n: i64) -> nx_int {
720 let team: *NxPTeam = _q8_pteam()
721 let nw: i64 = team.n_workers
722 let Apk: *u8 = _q8st_apk(m * k * 4)
723 _fq4m_pack_a(A, m * k, Apk)
724 let cx: *NxFq4mCtx = _q8mt_ctxs(NX_FQ4M_CTX_BYTES) as *NxFq4mCtx
725 cx.a_ptr = Apk as i64
726 cx.b_ptr = W.q4k_bytes as i64
727 cx.b_off = W.q4k_offset
728 cx.c_ptr = C as i64
729 cx.m = m
730 cx.k = k
731 cx.n = n
732 cx.jlo = (n + nw - 1) / nw // cpb
733 if nx_pteam_run(team, _lw_q8_0_pteam_band, cx as i64) != 0 { return NX_LW_ERR_INNER }
734 return NX_LW_OK
735}
736
737func _lw_q8_0_matmul(W: *NxF32LazyWeight, A: *i64, C: *i64,
738 m: i64, k: i64, n: i64) -> nx_int {
739 // NATIVE (Windows PE): the channel pool's WakeByAddress dispatch is too slow
740 // for the many small matmuls (measured slower than serial) -- use the fixed-band
741 // fork-join pteam (spin-detected generation, done-flags, no channel/wake), which
742 // nx_natbw_mt proved SCALES native (4.5x aggregate bw).
743 if nx_pool_is_native() != 0 { return _lw_q8_0_matmul_pteam(W, A, C, m, k, n) }
744 if m * k * n <= NX_Q8_ST_MAC_MAX { return _lw_q8_0_matmul_st(W, A, C, m, k, n) }
745 // MEASURED 2026-07-10: the naive fork-join _lw_q8_0_matmul_pteam scales
746 // WORSE (2.23x) than the channel pool (3.71x) -- refutes "channel CAS is
747 // the bottleneck". gcc OpenMP's 8.8x comes from a TUNED barrier (tree/
748 // affinity/backoff), not naive fork-join. Pool wins; pteam stays built +
749 // gate-passing (1000x bit-exact) but UNWIRED.
750 return _lw_q8_0_matmul_pool_force(W, A, C, m, k, n)
751}
752
753// Dispatched matmul: A [m, k] f32 @ W (lazy) -> C [m, n] f32.
754// k must equal W.rows, n must equal W.cols.
755
756func nx_f32_lazy_matmul(A: *i64, W: *NxF32LazyWeight, C: *i64,
757 m: nx_int, k: nx_int, n: nx_int) -> nx_int {
758 if A == (0 as *i64) { return NX_LW_ERR_NULL }
759 if W == (0 as *NxF32LazyWeight) { return NX_LW_ERR_NULL }
760 if C == (0 as *i64) { return NX_LW_ERR_NULL }
761 if W.dtype_tag == NX_LW_DTYPE_F32 {
762 // F32 weights: SCALAR threaded matmul_t. MEASURED-FINAL 2026-07-08
763 // (nx_f32_ffn_path_gate, host-noise-immune ratio): the packed
764 // range-dot cache is only 1.15x here, NOT worth +1.2GB. ROOT (the
765 // reframing insight): m=1 decode matmul is MEMORY-BANDWIDTH-BOUND --
766 // each weight value is read exactly ONCE, so SIMD *compute* can't
767 // help (the 2.72x single-dot gate was L1-resident = misleading).
768 // The real decode lever is FEWER BYTES read (keep weights QUANTIZED
769 // 4-bit + fast SIMD dequant-dot), or batch tokens (m>1 = compute-
770 // bound), NOT f32 packing. Cache infra stays for the m>1/prefill or
771 // dequant-dot future; NOT wired for F32 decode.
772 nx_f32_matmul_t_pool(nx_lw_shared_pool(), A, W.storage, C, m, k, n)
773 return NX_LW_OK
774 }
775 if W.dtype_tag == NX_LW_DTYPE_Q4_K {
776 // Guard the cacheable-shape contract before touching the
777 // cache (k must be super-block aligned; the streaming path
778 // would reject it anyway).
779 if k - (k / NX_GL_Q4_K_VPB) * NX_GL_Q4_K_VPB != 0 { return NX_LW_ERR_INNER }
780 if W.pk_state == 0 {
781 _lw_try_fill(W, k, n)
782 }
783 if W.pk_state == 1 {
784 return _lw_cached_matmul(W, A, C, m, k, n)
785 }
786 // Over budget (pk_state = -1): stream via the pooled
787 // scalar-dot path, exactly the pre-cache behavior.
788 let pool: *NxThreadPool = nx_lw_shared_pool()
789 // u2605u2605u2605u2605u2605 ADOPTION FIX 2026-07-31: dispatch the PACKED-SIMD path, not the scalar one.
790 // WHAT WAS WRONG: this called nx_f32_q4k_matmul_pool -- the SCALAR inner dot
791 // (acc = __f32_add(acc, __f32_mul(...)) one MAC at a time). The packed-SIMD sibling
792 // nx_f32_q4k_matmul_pool_x4 has existed, gated (nx_q4k_matmul_x4_gate), with __f32x4_dot
793 // MEASURED AT 5.0x OVER SCALAR -- and its own header calls itself "the forward's path".
794 // It was never dispatched. That is the ecosystem's #1 recurring law, on its highest-value
795 // surface: A CAPABILITY THAT BEATS THE BASELINE IN A GATE BUT ISN'T WIRED AT THE LIVE
796 // CHOKEPOINT *IS* THE BASELINE.
797 // WHY IT MATTERS FOR PREFILL SPECIFICALLY: fq4m_rows already dequants each weight row ONCE
798 // and reuses it across all m rows, so the WEIGHT traffic already amortises with batch size.
799 // What did NOT amortise was the per-token SCALAR dot -- per-token cost = dequant/m + k*FMA,
800 // which converges to a CONSTANT as m grows. That is exactly the measured FLAT curve
801 // (m=8 606ms/tok, m=32 673, m=64 567, m=125 593; debt 1785453966) and why prefill was
802 // 1.7x SLOWER per token than decode instead of faster, inverting the public 2026 SOTA
803 // regime split (prefill compute-bound, decode memory-bound). x4 attacks the term that
804 // actually dominates the batched path.
805 // SIGNATURE-IDENTICAL drop-in; same 256-alignment guard (checked above), same
806 // delta-wait/single-submitter pool contract.
807 // u26a0NUMERIC CONTRACT, DECLARED NOT HIDDEN: x4 accumulates in 4-lane chunks summed
808 // left-to-right, a DIFFERENT rounding ORDER than scalar. It is bit-exact vs scalar only in
809 // the exact-f32 regime (|sums| < 2^24); pool-x4 vs serial-x4 is bit-exact on ANY data
810 // (identical per-cell order; banding never splits a cell). This is ordinary SIMD
811 // reassociation, the same trade every fast GEMM makes -- but it IS a behaviour change and
812 // must be stated, not slipped in.
813 // u26d4 FUSED FLIP ATTEMPTED 2026-08-01 AND **REVERTED BY ITS OWN GATE**. Left here as the record.
814 // Switching this line to nx_f32_q4k_matmul_pool_fused turned nx_lw_cache_gate's
815 // "LWC 8 BIG-EXACT" RED. That tooth asserts l_same(BCc, BCs) -- the CACHED path and the
816 // STREAMED path must return IDENTICAL results. The fused route is integer/Q20 while
817 // _lw_cached_matmul stays f32, so the two stopped agreeing.
818 // u2605u2605u2605u2605u2605 THAT IS NOT A FUSSY GATE, IT IS A REAL DEFECT: it would make the model's output depend
819 // on whether a weight happened to be CACHED -- i.e. on MEMORY PRESSURE. Same prompt, same
820 // weights, different numbers depending on cache state. A quantisation change must apply to
821 // BOTH paths or NEITHER; it cannot be introduced on one side of a cache.
822 // u26a0And it is NOT fixable by widening precision: Q20 already measures 0.0032% on real weights,
823 // yet BIG-EXACT demands BIT equality, which no activation-quantised route can ever give.
824 // The gate was NOT weakened to let the change through -- resolving this is a contract decision
825 // (make the cache hold Q4_K and fuse both paths, or accept a declared tolerance), not a tuning
826 // knob. Everything else from this rung stays shipped: the fused kernel, its gates, the leak
827 // fix, and the Q10/Q20 conversions.
828 // ---- original flip rationale, retained because the MEASUREMENTS remain valid ----
829 // SOTA FLIP 2026-08-01: dispatch the FUSED INTEGER GEMM, not the packed-f32 one.
830 // The x4 flip above moved scalar -> SIMD and bought +23%. It could not do better, because the
831 // ceiling was never the dot: it was the DEQUANT. nx_q4k_to_f32_packed materialises k f32
832 // values per output column and EVERY f32 kernel pays that traffic regardless of lane width --
833 // which is why x8 (wider accumulate) came in 2% SLOWER, not faster. The fused route never
834 // materialises f32 at all: dequant happens inside the dot, in integer.
835 // MEASURED, same binary / same shape / same thread (nx_q4k_fused_vs_x4_gate):
836 // m=1 decode x4 15016us -> fused 1379us (>=9.9x across 5 runs; headline is the MIN)
837 // m=8 prefill x4 18771us -> fused 5907us (3.2x; m-blocking holds in the batched regime)
838 // and BIT-EXACT vs x4 at both shapes in the exact-integer regime.
839 // u26a0NUMERIC CONTRACT, DECLARED NOT HIDDEN: unlike x4/x8 this is NOT a reassociation -- it is
840 // ACTIVATION QUANTISATION (f32 -> Q20 fixed point, Q24xQ20 = Q44 accumulate). Authorised by
841 // nx_q4k_fused_fidelity_gate on REAL trained blk.0.attn_q weights, with activations
842 // deliberately built NOT to be fixed-point-representable: max relative deviation
843 // 34 per-1048576 = 0.0032%. Q10 activations measured 1.37% and were REFUSED for this flip;
844 // widening to Q20 cost nothing (same i64 multiply) and bought ~430x fidelity.
845 // u2705 i8-SIMD at the decode shape (m=1). BIT-EXACT vs x4 and 8.06x faster
846 // (nx_q4k_fused_vs_x4_gate T9/T10). Unlike the fused-integer and Q24-cache attempts, this route
847 // keeps ACTIVATIONS IN f32 -- only the WEIGHT side is int8 -- so it does not diverge from the
848 // packed-f32 cached path and does not break the LWC-8 cache-transparency contract.
849 // m>1 still goes to x4: the m-blocked i8 kernel is not written yet, and claiming a batched win
850 // before measuring it is the x8 mistake.
851 // ✅ RESOLVED 2026-08-01 -- CONTRACT DECISION (a) LANDED: the cache now holds Q24 integers
852 // (fq4m_fill_q24) and _lw_cached_matmul runs the SAME Q24 x Q20 -> Q44 accumulate as the
853 // fused route, so cached==streamed holds BIT-FOR-BIT by construction (exact integer
854 // arithmetic has no reassociation) and the LWC-8 objection above is DISSOLVED, not waived.
855 // The WHOLE Q4_K family now lives in ONE numeric domain:
856 // fill=Q24 cached=Q24xQ20 streamed=fused Q24xQ20 (all m)
857 // Absolute semantics shift by the DECLARED activation-quantisation error: Q20 measured
858 // 0.0032% max deviation on real trained weights (nx_q4k_fused_fidelity_gate) and the fused
859 // kernel passes nx_q4k_ggml_kat's 2% band vs ggml-true on real Q4_K_M model bytes -- the
860 // instrument this kernel itself names as the authorisation bar.
861 // ⚠ i8simd (m=1, 8.06x, f32 activations) is UNWIRED here, NOT deleted: keeping it dispatched
862 // would put the streamed decode path in the f32 domain while the cache sits in Q24 --
863 // recreating the exact cross-domain LWC-8 split this change removes. It stays built and
864 // gate-covered (nx_q4k_fused_vs_x4_gate T9/T10), the pteam idiom. Fused m=1 measured 1379us
865 // vs x4 15016us (>=9.9x) -- faster than i8simd's 8.06x on the same baseline -- so decode
866 // LOSES NOTHING by the unification; prefill gains 3.2x it never had.
867 let iv: nx_int = nx_f32_q4k_matmul_pool_fused(pool, A, W.q4k_bytes,
868 W.q4k_offset, C, m, k, n)
869 // Surface the inner verdict (pre-2026-07-07 this branch
870 // returned OK even on ERR_ALIGN, leaving C silently zero).
871 if iv != NX_FQ4M_OK { return NX_LW_ERR_INNER }
872 return NX_LW_OK
873 }
874 if W.dtype_tag == NX_LW_DTYPE_Q5_0 {
875 // Fused Q5_0 dequant-dot: read the quantized bytes (~11.6x less than
876 // the F32 the eager path made). k must be 32-aligned (Q5_0 VPB).
877 if k - (k / NX_Q5_0_VPB) * NX_Q5_0_VPB != 0 { return NX_LW_ERR_INNER }
878 return _lw_q5_0_matmul(W, A, C, m, k, n)
879 }
880 if W.dtype_tag == NX_LW_DTYPE_Q8_0 {
881 // SIMD Q8_0 dequant-dot (__f32_i8dot32) -- PROVEN 10.2x. k 32-aligned.
882 if k - (k / NX_Q8_0_VPB) * NX_Q8_0_VPB != 0 { return NX_LW_ERR_INNER }
883 return _lw_q8_0_matmul(W, A, C, m, k, n)
884 }
885 return NX_LW_ERR_BAD_TYPE
886}