nx_q4k_to_f32.nx source

1// nx_q4k_to_f32.nx -- bits-up Q4_K dequantization producing IEEE 754 2// binary32 outputs. 3// 4// L4.5 / L8 brick. Composes the L4 bits-up f32 substrate 5// (mul + sub + cvt) with the GGML Q4_K block layout to produce 6// dequantized values in genuine IEEE 754 form -- bit-comparable 7// against ggml's reference dequant on the same input bytes. 8// 9// Same algorithm as nx_gguf_dequant_q4_k (Q10 output) and 10// nx_gguf_dequant_q4_k_q14 (Q14 output), but the multiplication + 11// subtraction chain uses bits-up f32 ops shipped earlier in this 12// L4 tower. No libm, no compiler-builtin lowerings. 13// 14// Per-value formula: 15// d = f16_to_f32(super_block_d) (super-scale) 16// dmin = f16_to_f32(super_block_dmin) (super-min) 17// sc_j = unpacked 6-bit sub-block j scale (i32->f32 widened) 18// m_j = unpacked 6-bit sub-block j min 19// q4 = 4-bit nibble (0..15) (i32->f32 widened) 20// value_f32 = (d * sc_j) * q4 - (dmin * m_j) 21// 22// genealogy_id: ggml_q4k_block_canon + ieee754_binary32_dequant 23// lineage_id: substrate_q4k_to_f32_v1 24 25import "nx_syscalls.nx" 26import "nx_tier.nx" 27import "nx_le.nx" 28import "nx_gguf.nx" 29import "nx_gguf_load.nx" 30import "nx_f32.nx" 31import "nx_f32_cvt.nx" 32 33// Q4_K dequant Q10 / Q14 use the same scale/min layout; we re-use 34// the constants from nx_gguf_load.nx (NX_GL_Q4_K_VPB = 256, BPB = 144). 35 36const NX_Q4KF_OK: nx_int = 0 37const NX_Q4KF_ERR_BAD_BUF: nx_int = 1 38const NX_Q4KF_N_VERDICTS: nx_int = 2 39 40func nx_q4kf_verdict_is_valid(v: nx_int) -> nx_int { 41 if v < 0 { return 0 } 42 if v >= NX_Q4KF_N_VERDICTS { return 0 } 43 return 1 44} 45 46// Per-block dequant. Reads 144 bytes starting at super_off. 47// Writes 256 IEEE 754 binary32 raw bit patterns to out_f32[0..256]. 48// 49// Special case: caller controls n_values for partial trailing block. 50 51func nx_q4k_block_to_f32(buf: *u8, super_off: i64, 52 n_values: i64, 53 out_f32: *i64) -> nx_int { 54 if buf == (0 as *u8) { return NX_Q4KF_ERR_BAD_BUF } 55 56 // Read super-scales (f16) and widen to f32. 57 let d_raw: i64 = nx_le_read_u16(buf, super_off) 58 let dmin_raw: i64 = nx_le_read_u16(buf, super_off + 2) 59 let d_f32: i64 = nx_f16_to_f32(d_raw) 60 let dmin_f32: i64 = nx_f16_to_f32(dmin_raw) 61 62 let scales_off: i64 = super_off + 4 63 let qs_off: i64 = super_off + 16 64 65 // ggml dequantize_row_q4_K layout (clean-room, verified bit-exact vs 66 // ggml-quants.c on the real blk.0.attn_q super-block 0): 67 // process 32-byte GROUPS g=0..3; group g feeds sub-block 2g (low 68 // nibbles, 32 outputs in order) and sub-block 2g+1 (high nibbles). 69 var g: nx_int = 0 70 while g < 4 { 71 let is0: nx_int = g + g // 2g 72 let is1: nx_int = is0 + 1 // 2g+1 73 74 // Unpack 6-bit (sc,m) for both sub-blocks of this group. 75 var sc0: i64 = 0 76 var m0: i64 = 0 77 var sc1: i64 = 0 78 var m1s: i64 = 0 79 // sub-block is0 80 if is0 < 4 { 81 sc0 = nx_le_read_u8(buf, scales_off + is0) & 0x3F 82 m0 = nx_le_read_u8(buf, scales_off + is0 + 4) & 0x3F 83 } else { 84 let k0: nx_int = is0 - 4 85 let b_k: i64 = nx_le_read_u8(buf, scales_off + k0) 86 let b_k4: i64 = nx_le_read_u8(buf, scales_off + 4 + k0) 87 let b_8k: i64 = nx_le_read_u8(buf, scales_off + 8 + k0) 88 sc0 = ((b_k >> 6) << 4) | (b_8k & 0x0F) 89 m0 = ((b_k4 >> 6) << 4) | (b_8k >> 4) 90 } 91 // sub-block is1 92 if is1 < 4 { 93 sc1 = nx_le_read_u8(buf, scales_off + is1) & 0x3F 94 m1s = nx_le_read_u8(buf, scales_off + is1 + 4) & 0x3F 95 } else { 96 let k1: nx_int = is1 - 4 97 let c_k: i64 = nx_le_read_u8(buf, scales_off + k1) 98 let c_k4: i64 = nx_le_read_u8(buf, scales_off + 4 + k1) 99 let c_8k: i64 = nx_le_read_u8(buf, scales_off + 8 + k1) 100 sc1 = ((c_k >> 6) << 4) | (c_8k & 0x0F) 101 m1s = ((c_k4 >> 6) << 4) | (c_8k >> 4) 102 } 103 104 // Widen to f32 -- exact for [0, 63]. 105 let sc0_f32: i64 = nx_i32_to_f32(sc0) 106 let m0_f32: i64 = nx_i32_to_f32(m0) 107 let sc1_f32: i64 = nx_i32_to_f32(sc1) 108 let m1_f32c: i64 = nx_i32_to_f32(m1s) 109 110 // d1 = d * sc; neg_m = -(dmin * m). SSE mulss INLINE; bit-identical IEEE f32. 111 let d1_f32: i64 = __f32_mul(d_f32, sc0_f32) 112 let mm0_f32: i64 = __f32_mul(dmin_f32, m0_f32) 113 let neg_m0: i64 = __f32_mul(mm0_f32, __f32_from_i64(0 - 1)) 114 let d2_f32: i64 = __f32_mul(d_f32, sc1_f32) 115 let mm1_f32: i64 = __f32_mul(dmin_f32, m1_f32c) 116 let neg_m1: i64 = __f32_mul(mm1_f32, __f32_from_i64(0 - 1)) 117 118 let grp_off: i64 = qs_off + g * 32 119 120 var l: nx_int = 0 121 while l < 32 { 122 let byte_v: i64 = nx_le_read_u8(buf, grp_off + l) 123 let q_lo: i64 = byte_v & 0x0F 124 let q_hi: i64 = byte_v >> 4 125 let out_lo: i64 = is0 * 32 + l 126 let out_hi: i64 = is1 * 32 + l 127 if out_lo < n_values { 128 out_f32[out_lo] = __f32_add(__f32_mul(d1_f32, nx_i32_to_f32(q_lo)), neg_m0) 129 } 130 if out_hi < n_values { 131 out_f32[out_hi] = __f32_add(__f32_mul(d2_f32, nx_i32_to_f32(q_hi)), neg_m1) 132 } 133 l = l + 1 134 } 135 136 g = g + 1 137 } 138 return NX_Q4KF_OK 139} 140 141// Multi-block dequant. Walks n_super_blocks of 144 bytes each, 142// emitting n_super_blocks * 256 f32 values. 143 144func nx_q4k_to_f32(buf: *u8, base_off: i64, n_values: i64, 145 out_f32: *i64) -> nx_int { 146 let n_super: i64 = (n_values + NX_GL_Q4_K_VPB - 1) / NX_GL_Q4_K_VPB 147 var sb: i64 = 0 148 while sb < n_super { 149 let super_off: i64 = base_off + sb * NX_GL_Q4_K_BPB 150 let block_n: i64 = n_values - sb * NX_GL_Q4_K_VPB 151 var take: i64 = block_n 152 if take > NX_GL_Q4_K_VPB { take = NX_GL_Q4_K_VPB } 153 nx_q4k_block_to_f32(buf, super_off, take, 154 (out_f32 as i64 + sb * NX_GL_Q4_K_VPB * 8) as *i64) 155 sb = sb + 1 156 } 157 return NX_Q4KF_OK 158} 159 160// ===== PACKED variants (4-byte contiguous f32, 2026-07-08) ======== 161// 162// Same math, bit-identical values -- but stored as CONTIGUOUS 4-byte 163// IEEE 754 words instead of one-value-per-i64-slot. This is the 164// layout __f32x4_dot / __f32x8_dot consume (4 MACs per mulps), so 165// the SIMD matmul path dequants straight into packed form: half the 166// store traffic, no repack pass. Kept textually parallel to the 167// slot variant above (two-oracle property; the packed gate compares 168// them in the exact-f32 regime). 169 170func _q4kp_st4(out: *u8, idx: i64, bits: i64) -> i64 { 171 out[idx * 4 + 0] = bits as u8 172 out[idx * 4 + 1] = (bits >> 8) as u8 173 out[idx * 4 + 2] = (bits >> 16) as u8 174 out[idx * 4 + 3] = (bits >> 24) as u8 175 return 0 176} 177 178func nx_q4k_block_to_f32_packed(buf: *u8, super_off: i64, 179 n_values: i64, 180 out_p: *u8) -> nx_int { 181 if buf == (0 as *u8) { return NX_Q4KF_ERR_BAD_BUF } 182 183 let d_raw: i64 = nx_le_read_u16(buf, super_off) 184 let dmin_raw: i64 = nx_le_read_u16(buf, super_off + 2) 185 let d_f32: i64 = nx_f16_to_f32(d_raw) 186 let dmin_f32: i64 = nx_f16_to_f32(dmin_raw) 187 188 let scales_off: i64 = super_off + 4 189 let qs_off: i64 = super_off + 16 190 191 var g: nx_int = 0 192 while g < 4 { 193 let is0: nx_int = g + g 194 let is1: nx_int = is0 + 1 195 196 var sc0: i64 = 0 197 var m0: i64 = 0 198 var sc1: i64 = 0 199 var m1s: i64 = 0 200 if is0 < 4 { 201 sc0 = nx_le_read_u8(buf, scales_off + is0) & 0x3F 202 m0 = nx_le_read_u8(buf, scales_off + is0 + 4) & 0x3F 203 } else { 204 let k0: nx_int = is0 - 4 205 let b_k: i64 = nx_le_read_u8(buf, scales_off + k0) 206 let b_k4: i64 = nx_le_read_u8(buf, scales_off + 4 + k0) 207 let b_8k: i64 = nx_le_read_u8(buf, scales_off + 8 + k0) 208 sc0 = ((b_k >> 6) << 4) | (b_8k & 0x0F) 209 m0 = ((b_k4 >> 6) << 4) | (b_8k >> 4) 210 } 211 if is1 < 4 { 212 sc1 = nx_le_read_u8(buf, scales_off + is1) & 0x3F 213 m1s = nx_le_read_u8(buf, scales_off + is1 + 4) & 0x3F 214 } else { 215 let k1: nx_int = is1 - 4 216 let c_k: i64 = nx_le_read_u8(buf, scales_off + k1) 217 let c_k4: i64 = nx_le_read_u8(buf, scales_off + 4 + k1) 218 let c_8k: i64 = nx_le_read_u8(buf, scales_off + 8 + k1) 219 sc1 = ((c_k >> 6) << 4) | (c_8k & 0x0F) 220 m1s = ((c_k4 >> 6) << 4) | (c_8k >> 4) 221 } 222 223 let sc0_f32: i64 = nx_i32_to_f32(sc0) 224 let m0_f32: i64 = nx_i32_to_f32(m0) 225 let sc1_f32: i64 = nx_i32_to_f32(sc1) 226 let m1_f32c: i64 = nx_i32_to_f32(m1s) 227 228 let d1_f32: i64 = __f32_mul(d_f32, sc0_f32) 229 let mm0_f32: i64 = __f32_mul(dmin_f32, m0_f32) 230 let neg_m0: i64 = __f32_mul(mm0_f32, __f32_from_i64(0 - 1)) 231 let d2_f32: i64 = __f32_mul(d_f32, sc1_f32) 232 let mm1_f32: i64 = __f32_mul(dmin_f32, m1_f32c) 233 let neg_m1: i64 = __f32_mul(mm1_f32, __f32_from_i64(0 - 1)) 234 235 let grp_off: i64 = qs_off + g * 32 236 237 // PAIRWISE stores (2026-07-08): each two ADJACENT 4-byte f32 238 // outputs fuse into ONE aligned 8-byte store (pair-start 239 // indices are always even: is*32 + even l), replacing 8 240 // byte-stores + shifts per pair. This is the fix for the 241 // measured m=1 regression where byte-wise packing cost more 242 // than mulps saved. Values remain bit-identical; only the 243 // store form changes. LE layout: low word = lower index. 244 let opb: i64 = out_p as i64 245 var l: nx_int = 0 246 while l < 32 { 247 let byte_a: i64 = nx_le_read_u8(buf, grp_off + l) 248 let byte_b: i64 = nx_le_read_u8(buf, grp_off + l + 1) 249 let out_lo: i64 = is0 * 32 + l 250 let out_hi: i64 = is1 * 32 + l 251 if out_lo + 1 < n_values { 252 let vlo_a: i64 = __f32_add(__f32_mul(d1_f32, nx_i32_to_f32(byte_a & 0x0F)), neg_m0) 253 let vlo_b: i64 = __f32_add(__f32_mul(d1_f32, nx_i32_to_f32(byte_b & 0x0F)), neg_m0) 254 let w_lo: i64 = (vlo_a & 0xFFFFFFFF) | (vlo_b << 32) 255 let p_lo: *i64 = (opb + out_lo * 4) as *i64 256 p_lo[0] = w_lo 257 } else { 258 if out_lo < n_values { 259 let vlo_t: i64 = __f32_add(__f32_mul(d1_f32, nx_i32_to_f32(byte_a & 0x0F)), neg_m0) 260 _q4kp_st4(out_p, out_lo, vlo_t) 261 } 262 } 263 if out_hi + 1 < n_values { 264 let vhi_a: i64 = __f32_add(__f32_mul(d2_f32, nx_i32_to_f32(byte_a >> 4)), neg_m1) 265 let vhi_b: i64 = __f32_add(__f32_mul(d2_f32, nx_i32_to_f32(byte_b >> 4)), neg_m1) 266 let w_hi: i64 = (vhi_a & 0xFFFFFFFF) | (vhi_b << 32) 267 let p_hi: *i64 = (opb + out_hi * 4) as *i64 268 p_hi[0] = w_hi 269 } else { 270 if out_hi < n_values { 271 let vhi_t: i64 = __f32_add(__f32_mul(d2_f32, nx_i32_to_f32(byte_a >> 4)), neg_m1) 272 _q4kp_st4(out_p, out_hi, vhi_t) 273 } 274 } 275 l = l + 2 276 } 277 278 g = g + 1 279 } 280 return NX_Q4KF_OK 281} 282 283func nx_q4k_to_f32_packed(buf: *u8, base_off: i64, n_values: i64, 284 out_p: *u8) -> nx_int { 285 let n_super: i64 = (n_values + NX_GL_Q4_K_VPB - 1) / NX_GL_Q4_K_VPB 286 var sb: i64 = 0 287 while sb < n_super { 288 let super_off: i64 = base_off + sb * NX_GL_Q4_K_BPB 289 let block_n: i64 = n_values - sb * NX_GL_Q4_K_VPB 290 var take: i64 = block_n 291 if take > NX_GL_Q4_K_VPB { take = NX_GL_Q4_K_VPB } 292 nx_q4k_block_to_f32_packed(buf, super_off, take, 293 ((out_p as i64) + sb * NX_GL_Q4_K_VPB * 4) as *u8) 294 sb = sb + 1 295 } 296 return NX_Q4KF_OK 297}