nx_gguf_load.nx source

1// nx_gguf_load.nx -- GGUF tensor lookup + dequantize composer. 2// 3// L4 brick that bridges nx_gguf_parse (header + tensor_info walker, 4// SHIPPED) into NxTensor (L1 container, SHIPPED) by: 5// 6// 1. Finding a tensor by name in a parsed header 7// 2. Computing the data byte-size of any GGML-format tensor 8// 3. Dequantizing F32 / F16 / Q8_0 tensor data into Q10 i64 NxTensor 9// 10// This is the brick that lets nx_llm_run.nx leave its v1 "API- 11// composition proof" scaffold behind and become a real runner sourcing 12// weights from a real Llama / Mistral / Qwen / Z-Image GGUF file. 13// 14// Bits-up composition (every primitive cited already SHIPPED canonical): 15// nx_le.nx -- IEEE-LE binary readers 16// nx_gguf.nx -- header + tensor_info parser 17// nx_tensor.nx -- NxTensor + nx_t_alloc 18// nx_loop.nx -- bounded-loop discipline 19// 20// Format spec sources (public-domain documentation; no copied code): 21// GGUF v3 format -- gguf.md in ggml repo (Gerganov 2024) 22// ggml block_q8_0 layout -- ggml docs 23// IEEE 754 binary32 / binary16 -- IEEE standard 24// 25// genealogy_id: gguf_format_gerganov_2024 + ieee754_binary32_binary16 26// lineage_id: substrate_gguf_loader_v1_f32_f16_q8 27 28// nx_safety_envelope: 29// intended_use: "Load a GGUF tensor by name into an 30// NxTensor; dequantize F32 / F16 / Q8_0 31// formats to Q10 fixed-point i64 storage" 32// sil_target: SIL2 33// asil_target: QM 34// dal_target: DAL C 35// evidence: [gguf_v3_spec_public, q8_0_layout_public, 36// ieee754_standard, clean_room_implementation] 37// hazard_register: [bug-tape-oob-on-malformed-gguf, 38// bug-tape-overflow-on-extreme-f32-exponents, 39// bug-tape-nan-inf-leakage-into-q10] 40// residual_risk: "Adversarial GGUF input must be bounded 41// by caller; this brick assumes parse-clean 42// header from nx_gguf_parse" 43// verdict: NOT_YET_EVALUATED 44 45import "nx_syscalls.nx" 46import "nx_tier.nx" 47import "nx_loop.nx" 48import "nx_le.nx" 49import "nx_tensor.nx" 50import "nx_gguf.nx" 51const NX_MAGIC_8388607: i64 = 8388607 52const NX_MAGIC_8192: i64 = 8192 53const NX_MAGIC_1024: i64 = 1024 54const NX_MAGIC_2047: i64 = 2047 55const NX_MAGIC_16384: i64 = 16384 56const NX_MAGIC_32767: i64 = 32767 57const NX_MAGIC_32752: i64 = 32752 58 59// ===== Sealed-enum: GgufLoadVerdict =============================== 60 61const NX_GL_OK: nx_int = 0 62const NX_GL_ERR_NOT_FOUND: nx_int = 1 63const NX_GL_ERR_BAD_TYPE: nx_int = 2 64const NX_GL_ERR_BAD_SHAPE: nx_int = 3 65const NX_GL_ERR_OOB: nx_int = 4 66const NX_GL_ERR_OOM: nx_int = 5 67const NX_GL_ERR_OVERFLOW: nx_int = 6 68const NX_GL_N_VERDICTS: nx_int = 7 69 70func nx_gl_verdict_is_valid(v: nx_int) -> nx_int { 71 if v < 0 { return 0 } 72 if v >= NX_GL_N_VERDICTS { return 0 } 73 return 1 74} 75 76// ===== Block-size constants (ggml format spec) ==================== 77// 78// Each ggml_type defines values_per_block + bytes_per_block. These 79// are PUBLIC FORMAT INVARIANTS, not implementation choices. 80// 81// F32 -- block of 1, bytes = 4 82// F16 -- block of 1, bytes = 2 83// Q8_0 -- block of 32, bytes = 34 (2-byte f16 scale + 32 int8) 84// Q4_0 -- block of 32, bytes = 18 (2-byte f16 scale + 16 nibble bytes) 85// 86// Q4_K / Q5_K / Q6_K / Q8_K super-blocks are queued for the next 87// brick; their 144/176/210/256-byte super-block layouts deserve their 88// own focused arc. 89 90const NX_GL_Q8_0_VPB: nx_int = 32 91const NX_GL_Q8_0_BPB: nx_int = 34 92const NX_GL_Q4_0_VPB: nx_int = 32 93const NX_GL_Q4_0_BPB: nx_int = 18 94const NX_GL_Q4_K_VPB: nx_int = 256 95const NX_GL_Q4_K_BPB: nx_int = 144 96 97// Q10 scale (substrate-wide convention). 98const NX_GL_Q10: nx_int = 1024 99 100// Q14 scale -- 16x finer fractional resolution than Q10. 101// 102// Added 2026-05-19 in response to live-fire finding on real 103// Qwen2.5-0.5B-Instruct Q4_K_M: typical trained-model super-scales 104// `d_real ~ 0.001-0.01` round to 0 or 1 in Q10 (precision floor at 105// 1/1024 = ~9.8e-4). Q14's floor is 1/16384 = ~6.1e-5 which 106// represents real weight magnitudes correctly without underflow. 107// 108// See feedback-q10-precision-floor-collapses-real-model-weights.md. 109const NX_GL_Q14: nx_int = 16384 110 111// Saturating clamp for Q10/Q14 conversion of ±Inf and overflow. 112// 113// 2^60 fits in i64 and leaves room for multiplies in caller paths. 114const NX_GL_Q10_INF_SAT: i64 = 0x1000000000000000 115 116// ===== Public: tensor name equals ================================ 117// 118// Compare the bytes of a NxGgufTensorInfo.name (pointer INTO the 119// GGUF file buffer; not null-terminated) against a caller-supplied 120// name buffer of explicit length. 121 122func nx_gguf_name_equals(ti: *NxGgufTensorInfo, 123 name: *u8, name_len: nx_int) -> nx_int { 124 if ti.name_len != name_len { return 0 } 125 var i: nx_int = 0 126 var iter: nx_int = 0 127 var verdict: nx_int = NX_LOOP_RUNNING 128 let BUDGET: nx_int = name_len 129 while verdict == NX_LOOP_RUNNING && iter < BUDGET { 130 if ti.name[i] != name[i] { return 0 } 131 i = i + 1 132 iter = iter + 1 133 } 134 return 1 135} 136 137// ===== Public: find tensor by name =============================== 138// 139// Linear scan of the parsed tensor_info table. Returns the index in 140// [0, hdr.n_tensors) or -1 if not found. Llama-7B has ~290 tensors; 141// linear search is fine for v1. A hash-index lookup is queued if 142// scan time becomes a measurable bottleneck. 143 144func nx_gguf_find_tensor(hdr: *NxGgufHeader, 145 name: *u8, name_len: nx_int) -> nx_int { 146 var i: nx_int = 0 147 var iter: nx_int = 0 148 var verdict: nx_int = NX_LOOP_RUNNING 149 let BUDGET: nx_int = hdr.n_tensors 150 while verdict == NX_LOOP_RUNNING && iter < BUDGET { 151 let ti: *NxGgufTensorInfo = nx_gguf_tensor_at(hdr, i) 152 if nx_gguf_name_equals(ti, name, name_len) == 1 { return i } 153 i = i + 1 154 iter = iter + 1 155 } 156 return 0 - 1 157} 158 159// ===== Public: format-spec helpers =============================== 160 161func nx_gguf_ggml_values_per_block(ggml_type: i64) -> nx_int { 162 if ggml_type == NX_GGML_TYPE_F32 { return 1 } 163 if ggml_type == NX_GGML_TYPE_F16 { return 1 } 164 if ggml_type == NX_GGML_TYPE_Q8_0 { return NX_GL_Q8_0_VPB } 165 if ggml_type == NX_GGML_TYPE_Q4_0 { return NX_GL_Q4_0_VPB } 166 if ggml_type == NX_GGML_TYPE_Q4_K { return NX_GL_Q4_K_VPB } 167 return 0 168} 169 170func nx_gguf_ggml_bytes_per_block(ggml_type: i64) -> nx_int { 171 if ggml_type == NX_GGML_TYPE_F32 { return 4 } 172 if ggml_type == NX_GGML_TYPE_F16 { return 2 } 173 if ggml_type == NX_GGML_TYPE_Q8_0 { return NX_GL_Q8_0_BPB } 174 if ggml_type == NX_GGML_TYPE_Q4_0 { return NX_GL_Q4_0_BPB } 175 if ggml_type == NX_GGML_TYPE_Q4_K { return NX_GL_Q4_K_BPB } 176 return 0 177} 178 179// Total element count for a tensor: product of dims (NX_GGUF_MAX_DIMS 180// cap is 4; unused dims are stored as 1 by the parser? -- actually 181// the parser stores 0 for unused. Treat 0 as "no further dims". 182 183func nx_gguf_tensor_n_values(ti: *NxGgufTensorInfo) -> i64 { 184 var n: i64 = 1 185 if ti.n_dims >= 1 { n = n * ti.dim_0 } 186 if ti.n_dims >= 2 { n = n * ti.dim_1 } 187 if ti.n_dims >= 3 { n = n * ti.dim_2 } 188 if ti.n_dims >= 4 { n = n * ti.dim_3 } 189 return n 190} 191 192// Byte size of a tensor's storage in the GGUF data section. 193 194func nx_gguf_tensor_data_bytes(ti: *NxGgufTensorInfo) -> i64 { 195 let nv: i64 = nx_gguf_tensor_n_values(ti) 196 let vpb: nx_int = nx_gguf_ggml_values_per_block(ti.ggml_type) 197 let bpb: nx_int = nx_gguf_ggml_bytes_per_block(ti.ggml_type) 198 if vpb <= 0 { return 0 - 1 } 199 // n_blocks = ceil(nv / vpb). ggml mandates dim alignment to vpb 200 // (this is a documented spec invariant) so the divide is exact in 201 // a clean file; we still ceil to be defensive. 202 let n_blocks: i64 = (nv + vpb - 1) / vpb 203 return n_blocks * bpb 204} 205 206// ===== IEEE 754 binary32 -> Q10 (clean-room) ====================== 207// 208// Decode IEEE 754 binary32 bit pattern (read from u32 LE) into a 209// Q10 i64. All bit twiddling, no FP hardware required. 210// 211// bit 31: sign 212// bits 30..23: exponent (biased by 127) 213// bits 22..0: mantissa (implicit leading 1 for normal) 214// 215// Strategy: 216// value_q10 = (-1)^sign * (Q10 + m_frac_in_Q10) << (e - 127) when e >= 127 217// = (-1)^sign * (Q10 + m_frac_in_Q10) >> (127 - e) when e < 127 218// m_frac_in_Q10 = m >> 13 (23-bit mantissa shifted to fit Q10) 219// 220// Special cases: 221// e == 0: return 0 (zero or subnormal -- subnormals 222// are < 2^-126 ~ 1e-38 == 0 in Q10) 223// e == 255: return ±SAT (Inf), or 0 (NaN) 224// normal too big: saturate 225// normal too small: round to 0 226// 227// Accuracy: ~1 LSB in Q10 (i.e., ~1/1024 absolute error on values of 228// order 1). For model weights this is well below quantization noise. 229 230func _gguf_f32_to_q10(raw: i64) -> i64 { 231 let sign: i64 = (raw >> 31) & 1 232 let exponent: i64 = (raw >> 23) & 255 233 let mantissa: i64 = raw & NX_MAGIC_8388607 // (1 << 23) - 1 234 235 // Zero / subnormal -- round to 0. 236 if exponent == 0 { return 0 } 237 238 // Inf / NaN. 239 if exponent == 255 { 240 if mantissa == 0 { 241 if sign == 1 { return 0 - NX_GL_Q10_INF_SAT } 242 return NX_GL_Q10_INF_SAT 243 } 244 // NaN -- safe choice is 0. 245 return 0 246 } 247 248 // Normal. 249 let shift_amount: i64 = exponent - 127 250 let m_q10: i64 = mantissa / NX_MAGIC_8192 // mantissa >> 13 251 let base: i64 = NX_GL_Q10 + m_q10 // NX_MAGIC_1024..NX_MAGIC_2047 252 253 // Range guards. 254 if shift_amount > 50 { 255 if sign == 1 { return 0 - NX_GL_Q10_INF_SAT } 256 return NX_GL_Q10_INF_SAT 257 } 258 if shift_amount < 0 - 20 { return 0 } 259 260 var result: i64 = 0 261 if shift_amount >= 0 { 262 result = base << shift_amount 263 } else { 264 let neg_sa: i64 = 0 - shift_amount 265 result = base >> neg_sa 266 } 267 if sign == 1 { return 0 - result } 268 return result 269} 270 271// ===== IEEE 754 binary32 -> Q14 ================================== 272// 273// 4 more fractional bits than _gguf_f32_to_q10, for the activation path 274// where Q10 quantization gets amplified by SwiGLU's nonlinearity in the 275// FFN (measured worst 3.3%; Q14 tightens it). Same strategy; the 23-bit 276// f32 mantissa keeps its top 14 bits (>> 9) instead of top 10 (>> 13). 277func _gguf_f32_to_q14(raw: i64) -> i64 { 278 let sign: i64 = (raw >> 31) & 1 279 let exponent: i64 = (raw >> 23) & 255 280 let mantissa: i64 = raw & NX_MAGIC_8388607 281 if exponent == 0 { return 0 } 282 if exponent == 255 { 283 if mantissa == 0 { 284 if sign == 1 { return 0 - NX_GL_Q10_INF_SAT } 285 return NX_GL_Q10_INF_SAT 286 } 287 return 0 288 } 289 let shift_amount: i64 = exponent - 127 290 let m_q14: i64 = mantissa / 512 // mantissa >> 9 (top 14 bits) 291 let base: i64 = NX_GL_Q14 + m_q14 // NX_MAGIC_16384..NX_MAGIC_32767 292 if shift_amount > 46 { 293 if sign == 1 { return 0 - NX_GL_Q10_INF_SAT } 294 return NX_GL_Q10_INF_SAT 295 } 296 if shift_amount < 0 - 24 { return 0 } 297 var result: i64 = 0 298 if shift_amount >= 0 { 299 result = base << shift_amount 300 } else { 301 let neg_sa: i64 = 0 - shift_amount 302 result = base >> neg_sa 303 } 304 if sign == 1 { return 0 - result } 305 return result 306} 307 308// ===== IEEE 754 binary16 -> Q10 =================================== 309// 310// bit 15: sign 311// bits 14..10: exponent (biased by 15) 312// bits 9..0: mantissa (implicit leading 1 for normal) 313// 314// Same strategy as binary32; the mantissa is already 10 bits so it 315// is itself the Q10 fractional component. 316 317func _gguf_f16_to_q10(raw: i64) -> i64 { 318 let sign: i64 = (raw >> 15) & 1 319 let exponent: i64 = (raw >> 10) & 31 320 let mantissa: i64 = raw & 1023 // (1 << 10) - 1 321 322 if exponent == 0 { return 0 } 323 if exponent == 31 { 324 if mantissa == 0 { 325 if sign == 1 { return 0 - NX_GL_Q10_INF_SAT } 326 return NX_GL_Q10_INF_SAT 327 } 328 return 0 329 } 330 331 let shift_amount: i64 = exponent - 15 332 let base: i64 = NX_GL_Q10 + mantissa // NX_MAGIC_1024..NX_MAGIC_2047 333 334 if shift_amount > 50 { 335 if sign == 1 { return 0 - NX_GL_Q10_INF_SAT } 336 return NX_GL_Q10_INF_SAT 337 } 338 if shift_amount < 0 - 20 { return 0 } 339 340 var result: i64 = 0 341 if shift_amount >= 0 { 342 result = base << shift_amount 343 } else { 344 let neg_sa: i64 = 0 - shift_amount 345 result = base >> neg_sa 346 } 347 if sign == 1 { return 0 - result } 348 return result 349} 350 351// ===== IEEE 754 binary16 -> Q14 (precision-lifted) ================ 352// 353// Identical structure to _gguf_f16_to_q10 but emits Q14 (multiply by 354// 16384 instead of 1024). Adds 4 bits of fractional precision so 355// small super-scales like d_real ~ 0.001 land at d_q14 = 16 instead 356// of collapsing to 0 or 1 in Q10. 357// 358// The 10-bit f16 mantissa shifts left by 4 to fill the Q14 fractional 359// field directly (was: mantissa fed in as-is for Q10 since 10 bits 360// already matched 10 frac bits). 361// 362// Max representable: ~2^60 same as Q10 path (saturation guard). 363// Min representable: 1/16384 ~= 6.1e-5 (16x better than Q10's 364// 1/1024 ~= 9.8e-4). 365 366func _gguf_f16_to_q14(raw: i64) -> i64 { 367 let sign: i64 = (raw >> 15) & 1 368 let exponent: i64 = (raw >> 10) & 31 369 let mantissa: i64 = raw & 1023 370 371 if exponent == 0 { return 0 } 372 if exponent == 31 { 373 if mantissa == 0 { 374 if sign == 1 { return 0 - NX_GL_Q10_INF_SAT } 375 return NX_GL_Q10_INF_SAT 376 } 377 return 0 378 } 379 380 let shift_amount: i64 = exponent - 15 381 let base: i64 = NX_GL_Q14 + mantissa * 16 // NX_MAGIC_16384..NX_MAGIC_32752 382 383 if shift_amount > 46 { 384 if sign == 1 { return 0 - NX_GL_Q10_INF_SAT } 385 return NX_GL_Q10_INF_SAT 386 } 387 if shift_amount < 0 - 24 { return 0 } 388 389 var result: i64 = 0 390 if shift_amount >= 0 { 391 result = base << shift_amount 392 } else { 393 let neg_sa: i64 = 0 - shift_amount 394 result = base >> neg_sa 395 } 396 if sign == 1 { return 0 - result } 397 return result 398} 399 400// ===== IEEE 754 binary16 -> Q24 (EXACT for every finite f16) ====== 401// 402// Real GGUF Q4_K super-scales are ~1e-4, and each dequant value 403// d*sc*q - dmin*m is a small difference of near-equal terms. Any lossy 404// super-scale decode is therefore wrong on real weights: Q10 collapses 405// tiny d to 0 (whole tensor -> 0), and Q14 truncates asymmetrically and 406// FLIPS THE SIGN via catastrophic cancellation. Q24 represents every 407// finite f16 EXACTLY -- for a normal f16 the value * 2^24 is a pure left 408// shift with no bits discarded, and a subnormal's value * 2^24 is just 409// its mantissa -- which is exactly what the cancellation demands. This 410// is the canonical super-scale decode for Q4_K dequant; it retires the 411// Q10/Q14 super-scale decoders (see nx_q4k_to_f32 / dequant_iter / matmul). 412const NX_GL_Q24: i64 = 16777216 // 2^24 413 414func _gguf_f16_to_q24(raw: i64) -> i64 { 415 let sign: i64 = (raw >> 15) & 1 416 let exponent: i64 = (raw >> 10) & 31 417 let mantissa: i64 = raw & 1023 418 419 if exponent == 31 { 420 if mantissa == 0 { 421 if sign == 1 { return 0 - NX_GL_Q10_INF_SAT } 422 return NX_GL_Q10_INF_SAT 423 } 424 return 0 // NaN 425 } 426 427 var result: i64 = 0 428 if exponent == 0 { 429 result = mantissa // subnormal: value * 2^24 == mantissa (exact) 430 } else { 431 result = (NX_GL_Q10 + mantissa) << (exponent - 1) // normal: (NX_MAGIC_1024+mant) << (exp-1), exact 432 } 433 if sign == 1 { return 0 - result } 434 return result 435} 436 437// ===== Dequantize: F32 (per-element) ============================= 438 439func nx_gguf_dequant_f32(buf: *u8, base_off: i64, n_values: i64, 440 out_q10: *i64) -> nx_int { 441 var i: i64 = 0 442 var iter: nx_int = 0 443 var verdict: nx_int = NX_LOOP_RUNNING 444 let BUDGET: nx_int = n_values 445 while verdict == NX_LOOP_RUNNING && iter < BUDGET { 446 let raw: i64 = nx_le_read_u32(buf, base_off + i * 4) 447 out_q10[i] = _gguf_f32_to_q10(raw) 448 i = i + 1 449 iter = iter + 1 450 } 451 return NX_GL_OK 452} 453 454// ===== Dequantize: F16 (per-element) ============================= 455 456func nx_gguf_dequant_f16(buf: *u8, base_off: i64, n_values: i64, 457 out_q10: *i64) -> nx_int { 458 var i: i64 = 0 459 var iter: nx_int = 0 460 var verdict: nx_int = NX_LOOP_RUNNING 461 let BUDGET: nx_int = n_values 462 while verdict == NX_LOOP_RUNNING && iter < BUDGET { 463 let raw: i64 = nx_le_read_u16(buf, base_off + i * 2) 464 out_q10[i] = _gguf_f16_to_q10(raw) 465 i = i + 1 466 iter = iter + 1 467 } 468 return NX_GL_OK 469} 470 471// ===== Dequantize: Q8_0 (block-32, f16 scale + 32 int8) =========== 472// 473// Per block of 32 values: 474// bytes [0..2) = f16 scale d 475// bytes [2..34) = 32 signed int8 values qs[0..32) 476// 477// Dequantized value[i] = d * qs[i] 478// 479// In Q10: 480// scale_q10 = f16_to_q10(d) 481// out_q10[i] = scale_q10 * qs[i] -- already correctly scaled 482 483func nx_gguf_dequant_q8_0(buf: *u8, base_off: i64, n_values: i64, 484 out_q10: *i64) -> nx_int { 485 // n_blocks = ceil(n_values / 32). ggml mandates exact divisibility 486 // for tensors using block formats; we still ceil defensively. 487 let n_blocks: i64 = (n_values + NX_GL_Q8_0_VPB - 1) / NX_GL_Q8_0_VPB 488 489 var b: i64 = 0 490 var b_iter: nx_int = 0 491 var b_verdict: nx_int = NX_LOOP_RUNNING 492 let B_BUDGET: nx_int = n_blocks 493 while b_verdict == NX_LOOP_RUNNING && b_iter < B_BUDGET { 494 let block_off: i64 = base_off + b * NX_GL_Q8_0_BPB 495 496 // f16 scale at byte offset 0. 497 let scale_raw: i64 = nx_le_read_u16(buf, block_off) 498 let scale_q10: i64 = _gguf_f16_to_q10(scale_raw) 499 500 // 32 signed int8 values at byte offset 2. 501 var j: nx_int = 0 502 var j_iter: nx_int = 0 503 var j_verdict: nx_int = NX_LOOP_RUNNING 504 let J_BUDGET: nx_int = NX_GL_Q8_0_VPB 505 while j_verdict == NX_LOOP_RUNNING && j_iter < J_BUDGET { 506 let global_i: i64 = b * NX_GL_Q8_0_VPB + j 507 if global_i >= n_values { 508 // Past the logical end -- skip leftover block tail. 509 j = j + 1 510 j_iter = j_iter + 1 511 } else { 512 let raw_u8: i64 = nx_le_read_u8(buf, block_off + 2 + j) 513 var signed_v: i64 = raw_u8 514 if signed_v > 127 { signed_v = signed_v - 256 } 515 out_q10[global_i] = scale_q10 * signed_v 516 j = j + 1 517 j_iter = j_iter + 1 518 } 519 } 520 521 b = b + 1 522 b_iter = b_iter + 1 523 } 524 return NX_GL_OK 525} 526 527// ===== Dequantize: Q4_K (super-block 256, f16 d + f16 dmin + 528// ===== 12 packed scale/min bytes + 128 4-bit values) === 529// 530// Format (per ggml q4_K public spec, clean-room): 531// bytes [0..2) f16 d (super-scale) 532// bytes [2..4) f16 dmin (super-min) 533// bytes [4..16) 12 packed bytes encoding 8 6-bit scales + 8 6-bit mins 534// bytes [16..144) 128 bytes of 4-bit nibbles (256 values; low nibble 535// first for each byte) 536// 537// 12-byte scale/min packing (q[0..11] := scales[]): 538// For sub-block j in 0..3 (the "low" half): 539// sc[j] = q[j] & 0x3F (low 6 bits of byte j) 540// m[j] = q[j+4] & 0x3F (low 6 bits of byte j+4) 541// For sub-block j in 4..7 (the "high" half), k = j-4: 542// sc[j] = ((q[k] >> 6) << 4) | (q[8+k] & 0x0F) 543// m[j] = ((q[4+k] >> 6) << 4) | (q[8+k] >> 4) 544// 545// Dequant formula (per ggml's dequantize_row_q4_K, public algorithm -- 546// clean-room, verified BIT-EXACT vs ggml-quants.c on the real 547// blk.0.attn_q super-block 0 of a Qwen3-4B Q4_K_M model): 548// d_real = f16_to_real(d) 549// dmin_real = f16_to_real(dmin) 550// Process 32-byte GROUPS g=0..3; group g feeds sub-block 2g (low 551// nibbles) and sub-block 2g+1 (high nibbles), 32 outputs each IN ORDER: 552// d1 = d_real * sc[2g] ; m1 = dmin_real * m[2g] 553// d2 = d_real * sc[2g+1] ; m2 = dmin_real * m[2g+1] 554// for l in 0..32: out[(2g)*32 + l] = d1 * (qs[g*32+l] & 0x0F) - m1 555// for l in 0..32: out[(2g+1)*32 + l] = d2 * (qs[g*32+l] >> 4) - m2 556// 557// OUTPUT UNIT: Q24 (was Q10). Real trained-model super-scales are ~1e-4 558// and each value d*sc*q - dmin*m is a small difference of near-equal 559// terms; Q10/Q14 super-scale decode underflows/flips-sign on real 560// weights (see _gguf_f16_to_q24 note + nx_q4k_q24_gate). Q24 represents 561// every finite f16 super-scale EXACTLY so the cancellation is exact. 562// d_q24 = f16_to_q24(d); dmin_q24 = f16_to_q24(dmin) 563// value_q24 = d_q24 * sc[is] * q4 - dmin_q24 * m[is] 564// (No divide needed -- sc, m, q4 are small integers.) 565 566func nx_gguf_dequant_q4_k(buf: *u8, base_off: i64, n_values: i64, 567 out_q24: *i64) -> nx_int { 568 let n_super: i64 = (n_values + NX_GL_Q4_K_VPB - 1) / NX_GL_Q4_K_VPB 569 570 var sb: i64 = 0 571 var sb_iter: nx_int = 0 572 var sb_verdict: nx_int = NX_LOOP_RUNNING 573 let SB_BUDGET: nx_int = n_super 574 while sb_verdict == NX_LOOP_RUNNING && sb_iter < SB_BUDGET { 575 let super_off: i64 = base_off + sb * NX_GL_Q4_K_BPB 576 let d_raw: i64 = nx_le_read_u16(buf, super_off) 577 let dmin_raw: i64 = nx_le_read_u16(buf, super_off + 2) 578 let d_q24: i64 = _gguf_f16_to_q24(d_raw) 579 let dmin_q24: i64 = _gguf_f16_to_q24(dmin_raw) 580 let scales_off: i64 = super_off + 4 581 let qs_off: i64 = super_off + 16 582 583 var g: nx_int = 0 584 var g_iter: nx_int = 0 585 var g_verdict: nx_int = NX_LOOP_RUNNING 586 let G_BUDGET: nx_int = 4 587 while g_verdict == NX_LOOP_RUNNING && g_iter < G_BUDGET { 588 let is0: nx_int = g + g // 2g 589 let is1: nx_int = is0 + 1 // 2g+1 590 var sc0: i64 = 0 591 var m0: i64 = 0 592 var sc1: i64 = 0 593 var m1s: i64 = 0 594 if is0 < 4 { 595 sc0 = nx_le_read_u8(buf, scales_off + is0) & 0x3F 596 m0 = nx_le_read_u8(buf, scales_off + is0 + 4) & 0x3F 597 } else { 598 let k0: nx_int = is0 - 4 599 let b_k: i64 = nx_le_read_u8(buf, scales_off + k0) 600 let b_k4: i64 = nx_le_read_u8(buf, scales_off + 4 + k0) 601 let b_8k: i64 = nx_le_read_u8(buf, scales_off + 8 + k0) 602 sc0 = ((b_k >> 6) << 4) | (b_8k & 0x0F) 603 m0 = ((b_k4 >> 6) << 4) | (b_8k >> 4) 604 } 605 if is1 < 4 { 606 sc1 = nx_le_read_u8(buf, scales_off + is1) & 0x3F 607 m1s = nx_le_read_u8(buf, scales_off + is1 + 4) & 0x3F 608 } else { 609 let k1: nx_int = is1 - 4 610 let c_k: i64 = nx_le_read_u8(buf, scales_off + k1) 611 let c_k4: i64 = nx_le_read_u8(buf, scales_off + 4 + k1) 612 let c_8k: i64 = nx_le_read_u8(buf, scales_off + 8 + k1) 613 sc1 = ((c_k >> 6) << 4) | (c_8k & 0x0F) 614 m1s = ((c_k4 >> 6) << 4) | (c_8k >> 4) 615 } 616 617 let d1_q24: i64 = d_q24 * sc0 618 let m0_q24: i64 = dmin_q24 * m0 619 let d2_q24: i64 = d_q24 * sc1 620 let m1_q24: i64 = dmin_q24 * m1s 621 let grp_off: i64 = qs_off + g * 32 622 623 var l: nx_int = 0 624 var l_iter: nx_int = 0 625 var l_verdict: nx_int = NX_LOOP_RUNNING 626 let L_BUDGET: nx_int = 32 627 while l_verdict == NX_LOOP_RUNNING && l_iter < L_BUDGET { 628 let byte_v: i64 = nx_le_read_u8(buf, grp_off + l) 629 let q_lo: i64 = byte_v & 0x0F 630 let q_hi: i64 = byte_v >> 4 631 let out_lo: i64 = sb * NX_GL_Q4_K_VPB + is0 * 32 + l 632 let out_hi: i64 = sb * NX_GL_Q4_K_VPB + is1 * 32 + l 633 if out_lo < n_values { 634 out_q24[out_lo] = d1_q24 * q_lo - m0_q24 635 } 636 if out_hi < n_values { 637 out_q24[out_hi] = d2_q24 * q_hi - m1_q24 638 } 639 l = l + 1 640 l_iter = l_iter + 1 641 } 642 643 g = g + 1 644 g_iter = g_iter + 1 645 } 646 647 sb = sb + 1 648 sb_iter = sb_iter + 1 649 } 650 return NX_GL_OK 651} 652 653// ===== Dequantize: Q4_K -> Q24 (was "_q14"; superseded by Q24) ===== 654// 655// HISTORY: this used to emit Q14 (via _gguf_f16_to_q14) as a partial 656// fix for Q10 super-scale underflow on trained models. Q14 still 657// truncates the super-scale asymmetrically and FLIPS THE SIGN via 658// catastrophic cancellation on real weights. It now emits Q24 (via the 659// EXACT _gguf_f16_to_q24) -- identical output to nx_gguf_dequant_q4_k. 660// The name is retained so existing callers (nx_q4k_real_gemm) still 661// link; those callers have been updated to interpret the output as Q24. 662// Also carries the ggml 32-byte-group layout fix (group g -> sub-blocks 663// 2g low / 2g+1 high). Prefer nx_gguf_dequant_q4_k for new code. 664// 665// See feedback-q10-precision-floor-collapses-real-model-weights.md 666// for the live-fire discovery on Qwen2.5-0.5B Q4_K_M (2026-05-19). 667 668func nx_gguf_dequant_q4_k_q14(buf: *u8, base_off: i64, n_values: i64, 669 out_q24: *i64) -> nx_int { 670 // Delegate to the canonical Q24 dequant (single source of truth). 671 return nx_gguf_dequant_q4_k(buf, base_off, n_values, out_q24) 672} 673 674// ===== Public: load tensor by name ================================ 675// 676// Composes find + alloc + dequant into one call. Caller pre-knows 677// nothing about block layout or dtype; this routes by ggml_type. 678// 679// Output tensor is always NX_DT_I64 fixed-point in v1. UNIT BY TYPE: 680// F32/F16/Q8_0 paths emit Q10; the Q4_K path emits Q24 (the canonical 681// dequant was moved to Q24 to survive real trained-model super-scale 682// cancellation -- see nx_gguf_dequant_q4_k). A caller mixing Q4_K 683// tensors with Q10 tensors must rescale (Q24 -> Q10 = >>14 round-half). 684// When the compiler ships native F32 / F16 / BF16 dtypes, this function 685// gets an optional dtype-preserving fast path; the i64 fixed-point path 686// stays as the canonical "works everywhere" path. 687 688func nx_gguf_load_tensor(buf: *u8, hdr: *NxGgufHeader, 689 name: *u8, name_len: nx_int, 690 out_err: *i64) -> *NxTensor { 691 let idx: nx_int = nx_gguf_find_tensor(hdr, name, name_len) 692 if idx < 0 { 693 out_err[0] = NX_GL_ERR_NOT_FOUND 694 return 0 as *NxTensor 695 } 696 let ti: *NxGgufTensorInfo = nx_gguf_tensor_at(hdr, idx) 697 698 let nv: i64 = nx_gguf_tensor_n_values(ti) 699 if nv <= 0 { 700 out_err[0] = NX_GL_ERR_BAD_SHAPE 701 return 0 as *NxTensor 702 } 703 704 // Allocate NxTensor with the GGUF shape, dtype = I64 Q10. 705 let shape: *i64 = sys_mmap(4 * 8) as *i64 706 shape[0] = ti.dim_0 707 shape[1] = ti.dim_1 708 shape[2] = ti.dim_2 709 shape[3] = ti.dim_3 710 let err: *i64 = sys_mmap(8) as *i64 711 err[0] = 0 712 let t: *NxTensor = nx_t_alloc(NX_DT_I64, shape, ti.n_dims, err) 713 if err[0] != 0 { 714 out_err[0] = NX_GL_ERR_OOM 715 return 0 as *NxTensor 716 } 717 718 let storage: *i64 = t.storage as *i64 719 let data_off: i64 = hdr.data_off + ti.offset 720 721 var verdict: nx_int = NX_GL_OK 722 if ti.ggml_type == NX_GGML_TYPE_F32 { 723 verdict = nx_gguf_dequant_f32(buf, data_off, nv, storage) 724 } else { 725 if ti.ggml_type == NX_GGML_TYPE_F16 { 726 verdict = nx_gguf_dequant_f16(buf, data_off, nv, storage) 727 } else { 728 if ti.ggml_type == NX_GGML_TYPE_Q8_0 { 729 verdict = nx_gguf_dequant_q8_0(buf, data_off, nv, storage) 730 } else { 731 if ti.ggml_type == NX_GGML_TYPE_Q4_K { 732 verdict = nx_gguf_dequant_q4_k(buf, data_off, nv, storage) 733 } else { 734 out_err[0] = NX_GL_ERR_BAD_TYPE 735 return 0 as *NxTensor 736 } 737 } 738 } 739 } 740 if verdict != NX_GL_OK { 741 out_err[0] = verdict 742 return 0 as *NxTensor 743 } 744 745 out_err[0] = NX_GL_OK 746 return t 747}