`nx_qmatvec.nx` source

1// nx_qmatvec.nx -- the FIRST real kernel toward beating ggml/llama.cpp: a sovereign block-
2// quantized DOT PRODUCT (the hot path of LLM decode), built bits-up. The roofline proved decode
3// is MEMORY-bound, so the lever is MOVING LESS DATA -- this kernel does exactly that, and the
4// ENGINEER verifies it 1:1 against the exact dot while the quality (error) is measured vs the
5// Council's floor. Integer-only by design (no fp dequant) -- a real efficiency edge for the
6// edge/energy north-star.
7//
8// Scheme (ggml Q4_0/Q8_0 structure, integer scales): weights in blocks of B=32; each block has
9// ONE integer scale S = max|w| in the block; each weight becomes a signed code q in [-QMAX,QMAX]
10// (4-bit: -7..7, or 8-bit: -127..127). Dequant w' = q*S/QMAX. The block scale FACTORS OUT of the
11// inner loop (ggml's key trick): block_dot = (S/QMAX) * sum_i q_i*x_i, so the hot loop is pure
12// integer multiply-accumulate of small codes, and S applies once per block.
13//   bytes moved (what DRAM must deliver, the memory-bound cost):
14//     Q4: 4 bits/weight + 2 bytes scale per 32 = 4.5 bits/weight
15//     Q8: 8 bits/weight + 2 bytes scale per 32 = 8.5 bits/weight   vs int16 ref = 16 bits/weight
16// license_tier: ORIGINAL  Refs: ggml Q4_0/Q8_0 block quantization; the memory-bound roofline.
17
18import "nx_syscalls.nx"
19
20const QMV_B:     i64 = 32     // block size (ggml Q4_0/Q8_0)
21const QMV_Q4MAX: i64 = 7      // 4-bit signed symmetric range -7..7
22const QMV_Q8MAX: i64 = 127    // 8-bit signed symmetric range -127..127
23
24func qmv_abs(v: i64) -> i64 { if v < 0 { return 0 - v } return v }
25
26// block scale S = max|w| over [off, off+len)  (0 if the block is all zero).
27func qmv_block_scale(w: *i64, off: i64, len: i64) -> i64 {
28    var s: i64 = 0; var i: i64 = 0
29    while i < len { let a: i64 = qmv_abs(w[off + i]); if a > s { s = a } i = i + 1 }
30    return s
31}
32
33// quantize one weight to a signed code in [-qmax, qmax], round-to-nearest, scale S. S>0 assumed.
34func qmv_code(wv: i64, s: i64, qmax: i64) -> i64 {
35    if s <= 0 { return 0 }
36    let half: i64 = s / 2
37    var q: i64 = 0
38    if wv >= 0 { q = (wv * qmax + half) / s } else { q = 0 - (((0 - wv) * qmax + half) / s) }
39    if q > qmax { q = qmax }
40    if q < (0 - qmax) { q = 0 - qmax }
41    return q
42}
43
44// THE KERNEL: block-quantized dot product of weights w (quantized on the fly to `qmax`) with
45// activations x, length n. Returns the dequantized dot (scale factored out of the inner loop,
46// divided once at the end). This is the memory-bound hot path -- in deployment w is PRE-quantized
47// and only the small codes + scales are loaded from DRAM (the data-movement win).
48func qmv_dot_q(w: *i64, x: *i64, n: i64, qmax: i64) -> i64 {
49    var acc: i64 = 0; var off: i64 = 0
50    while off < n {
51        var len: i64 = QMV_B; if off + len > n { len = n - off }
52        let s: i64 = qmv_block_scale(w, off, len)
53        var bdot: i64 = 0; var i: i64 = 0
54        while i < len { bdot = bdot + qmv_code(w[off + i], s, qmax) * x[off + i]; i = i + 1 }
55        acc = acc + s * bdot                  // scale applied once per block (factored out)
56        off = off + len
57    }
58    return acc / qmax
59}
60
61// the EXACT reference dot (the Engineer's oracle -- verify the kernel, don't trust it).
62func qmv_dot_exact(w: *i64, x: *i64, n: i64) -> i64 {
63    var acc: i64 = 0; var i: i64 = 0
64    while i < n { acc = acc + w[i] * x[i]; i = i + 1 }
65    return acc
66}
67
68// bytes the kernel must move from DRAM (the memory-bound cost), per format. bits/8 packed.
69func qmv_bytes_q(n: i64, bits: i64) -> i64 {
70    let nb: i64 = (n + QMV_B - 1) / QMV_B            // number of blocks
71    return (n * bits) / 8 + nb * 2                   // packed codes + 2-byte int scale per block
72}
73func qmv_bytes_ref16(n: i64) -> i64 { return n * 2 }  // int16 (fp16-equivalent) baseline
74
75// relative error of the quantized dot vs exact, in PERMIL (parts per thousand). The quality metric
76// the Council floor judges. exact==0 -> 0 (avoid div by zero; caller should use a correlated vector).
77func qmv_rel_error_permil(approx: i64, exact: i64) -> i64 {
78    if exact == 0 { return 0 }
79    var d: i64 = approx - exact; if d < 0 { d = 0 - d }
80    var e: i64 = exact; if e < 0 { e = 0 - e }
81    return (d * 1000) / e
82}
nx_qmatvec.nx source

`nx_qmatvec.nx` source
