code wiki / _hdl_build / nx_kquant.nx
nx_kquant.nx source
↩ module page · 62 lines · 3006 B
1// nx_kquant.nx -- the QUALITY lever for quantization (quality is slightly more valuable than speed):
2// ggml's K-quants beat naive Q4_0 mainly by FINER-GRAINED SCALES -- a big block shares ONE scale,
3// so a single outlier weight inflates the step and wrecks the precision of every other weight in
4// the block. Smaller sub-blocks ISOLATE outliers: only the sub-block holding the outlier pays, the
5// rest stay sharp. This module makes the block granularity a PARAMETER, so the team can trade a
6// little more scale storage for materially better quality -- another point on the quality/data
7// frontier the Council balances. (Real NN weights are concentrated with occasional outliers, the
8// exact case where this wins.) license_tier: ORIGINAL Refs: ggml Q4_K/Q6_K K-quants.
9
10import "nx_qmatvec.nx" // qmv_block_scale, qmv_code, qmv_abs, QMV_Q4MAX/Q8MAX
11import "nx_qlayer.nx" // qlayer_l2_relerr_permil, qlayer_isqrt, qlayer_matvec_exact
12
13func kq_blocks_per_row(c: i64, blk: i64) -> i64 { return (c + blk - 1) / blk }
14
15// quantize an R x C matrix with a PARAMETERIZED block size `blk` (the granularity knob).
16func kq_layer_quantize(w: *i64, r: i64, c: i64, qmax: i64, blk: i64, codes: *i64, scales: *i64) -> i64 {
17 let nbpr: i64 = kq_blocks_per_row(c, blk)
18 var row: i64 = 0
19 while row < r {
20 let base: i64 = row * c
21 var bk: i64 = 0
22 while bk < nbpr {
23 let off: i64 = bk * blk
24 var len: i64 = blk; if off + len > c { len = c - off }
25 let s: i64 = qmv_block_scale(w, base + off, len)
26 scales[row * nbpr + bk] = s
27 var i: i64 = 0
28 while i < len { codes[base + off + i] = qmv_code(w[base + off + i], s, qmax); i = i + 1 }
29 bk = bk + 1
30 }
31 row = row + 1
32 }
33 return 0
34}
35
36// hot matvec with parameterized block size (scale factored out of the inner loop).
37func kq_layer_matvec(codes: *i64, scales: *i64, x: *i64, r: i64, c: i64, qmax: i64, blk: i64, y: *i64) -> i64 {
38 let nbpr: i64 = kq_blocks_per_row(c, blk)
39 var row: i64 = 0
40 while row < r {
41 let base: i64 = row * c
42 var acc: i64 = 0; var bk: i64 = 0
43 while bk < nbpr {
44 let off: i64 = bk * blk
45 var len: i64 = blk; if off + len > c { len = c - off }
46 let s: i64 = scales[row * nbpr + bk]
47 var bdot: i64 = 0; var i: i64 = 0
48 while i < len { bdot = bdot + codes[base + off + i] * x[off + i]; i = i + 1 }
49 acc = acc + s * bdot
50 bk = bk + 1
51 }
52 y[row] = acc / qmax
53 row = row + 1
54 }
55 return 0
56}
57
58// DRAM bytes at this granularity: packed codes + one int16 scale per block. Finer blk -> more bytes.
59func kq_bytes(r: i64, c: i64, bits: i64, blk: i64) -> i64 { return (r * c * bits) / 8 + (r * kq_blocks_per_row(c, blk)) * 2 }
60
61// effective bits/weight x100 at this granularity (bits + scale_bits/blk).
62func kq_bits_x100(bits: i64, blk: i64) -> i64 { return bits * 100 + (16 * 100) / blk }