code wiki / (root) / nx_q4k_fused_vs_x4_gate.nx
`nx_q4k_fused_vs_x4_gate.nx` source

↩ module page · 237 lines · 13318 B
1// nx_q4k_fused_vs_x4_gate.nx -- THE HEAD-TO-HEAD the SOTA decision actually rests on:
2// the FUSED INTEGER dequant-dot (nx_q4k_dot_row_col) against the LIVE forward kernel
3// (fq4m_rows_x4 / nx_f32_q4k_matmul_x4), in ONE binary, ONE shape, ONE thread.
4//
5// WHY THIS GATE EXISTS. nx_q4k_speed_bench reports the fused integer route beating f32 by ~20-48x, but
6// its f32 leg is SINGLE-THREADED SCALAR EMULATED f32 -- which is NOT the live path. The live path is
7// nx_f32_q4k_matmul_pool_x4: hardware __f32x4_dot AND multicore. Quoting that 20-48x as the shipped win
8// would repeat the exact "10.25x vs SERIAL scalar" error already caught once in this lane. A speedup is
9// only real against WHAT IS ACTUALLY DISPATCHED.
10//
11// THREADING IS DELIBERATELY EXCLUDED, AND THAT IS SOUND. Both routes band-parallelise over the SAME axis
12// (output columns j) with identical banding, so a pool multiplies both sides equally and the
13// single-thread ratio carries over. It is also CONSERVATIVE for the fused side: fused reads only the
14// packed Q4_K bytes while x4 materialises k f32 values per column, so under real multicore memory
15// pressure fused should scale strictly BETTER, never worse. Any win measured here is a LOWER BOUND.
16//
17// NUMERIC REGIME (why bit-exact comparison is legitimate here, and why that is NOT a general claim).
18// Weights are synthetic Q4_K super-blocks with d=1.0, dmin=0, all scales=1, mins=0, so every dequantised
19// weight is an exact integer 0..15. The activation column holds small exact integers. Then:
20//   x4    computes an exact-integer f32 sum (|dot| <= 15*3*1024 = 46080 << 2^24, so f32 add is EXACT).
21//   fused computes d1=2^24, m1=0, so v = 2^24*q4 (Q24); v*col_q10 is Q34; >>24 recovers Q10 exactly.
22// Both therefore represent the SAME integer and must agree BIT-EXACTLY. On REAL model weights they will
23// NOT -- fused is activation quantisation (W4A-fixed), a genuine precision change. This gate proves the
24// kernels compute the same function; it does NOT license dispatch on real weights without a separate
25// fidelity tooth (nx_q4k_ggml_kat's 2% band is the existing instrument for that).
26//
27// genealogy_id: nx_q4k_x8_gate (data construction, exact regime, refutation-tooth discipline)
28import "nx_f32_q4k_matmul.nx"
29import "nx_q4k_matmul.nx"
30import "nx_dequant_iter.nx"
31import "nx_f32_cvt.nx"
32import "nx_fmt.nx"
33import "nx_gate_verdict.nx"
34
35const FVX_M: i64 = 1        // decode shape -- the dominant serving case and where fused should be strongest
36const FVX_K: i64 = 1024
37const FVX_N: i64 = 512
38
39func fv_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
40func fv_num(v: i64) -> i64 {
  let b: *u8 = sys_mmap(32); var m: i64 = v
  if m == 0 { b[0] = 48 as u8; sys_write(1, b, 1); return 0 }
  if m < 0 { fv_puts("-" as *u8); m = 0 - m }
  let t: *u8 = sys_mmap(32); var k: i64 = 0
  while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
  var i: i64 = 0
  while i < k { b[i] = t[k - 1 - i]; i = i + 1 }
  sys_write(1, b, k); return 0
49}
50func fv_lcg(s: i64) -> i64 { var v: i64 = s * 1103515245 + 12345; v = v & 2147483647; return v }
51
52// Dense exact-regime Q4_K super-block: d=1.0 (f16 0x3C00), dmin=0, sc=1, m=0.
53func fv_block(buf: *u8, off: i64, seed: i64) -> i64 {
  buf[off + 0] = 0x00 as u8
  buf[off + 1] = 0x3C as u8
  buf[off + 2] = 0 as u8
  buf[off + 3] = 0 as u8
  var i: i64 = 0
  while i < 4 {
      buf[off + 4 + i]  = 0x01 as u8
      buf[off + 8 + i]  = 0x00 as u8
      buf[off + 12 + i] = 0x01 as u8
      i = i + 1
  }
  var s: i64 = seed
  var z: i64 = 0
  while z < 128 { s = fv_lcg(s); buf[off + 16 + z] = (s & 255) as u8; z = z + 1 }
  return 0
69}
70func fv_fill_w(buf: *u8, n_rows: i64, k: i64, seed: i64) -> i64 {
  let bpr: i64 = (k / 256) * 144
  var s: i64 = seed
  var r: i64 = 0
  while r < n_rows {
      var b: i64 = 0
      while b < k / 256 { s = fv_lcg(s); fv_block(buf, r * bpr + b * 144, s); b = b + 1 }
      r = r + 1
  }
  return 0
80}
81
82func main(argc: i64, argv: *i64) -> i64 {
  fv_puts("=== nx_q4k_fused_vs_x4_gate: FUSED INTEGER dot vs the LIVE x4 forward kernel ===\n" as *u8)
  var pass: i64 = 0
  var fail: i64 = 0
86
  let bpr: i64 = (FVX_K / 256) * 144
  let n_blocks: i64 = FVX_K / 256
  let W: *u8 = sys_mmap(FVX_N * bpr)
  fv_fill_w(W, FVX_N, FVX_K, 99)
91
  // ONE activation vector, expressed in BOTH domains from the same integers.
  let A:      *i64 = sys_mmap(FVX_K * 8) as *i64      // f32 bits, for x4
  let colq10: *i64 = sys_mmap(FVX_K * 8) as *i64      // Q10 fixed point, for fused
  var i: i64 = 0
  while i < FVX_K {
      let v: i64 = (i - (i / 7) * 7) - 3              // exact small ints in [-3, 3]
      A[i] = nx_i32_to_f32(v)
      colq10[i] = v * 1024
      i = i + 1
  }
102
  let C4: *i64 = sys_mmap(FVX_M * FVX_N * 8) as *i64
  let CF: *i64 = sys_mmap(FVX_M * FVX_N * 8) as *i64
  let it: *NxQ4KBlockIter = nx_q4k_iter_alloc()
106
  // ---- T1: the live x4 kernel runs ----
  let r4: nx_int = nx_f32_q4k_matmul_x4(A, W, 0, C4, FVX_M, FVX_K, FVX_N)
  if r4 == NX_FQ4M_OK { pass = pass + 1; fv_puts("  T1 x4 (live kernel) OK\n" as *u8) } else { fail = fail + 1; fv_puts("  T1 x4 rc FAIL\n" as *u8) }
110
  // ---- T2: fused produces the SAME values (exact regime) ----
  var j: i64 = 0
  while j < FVX_N {
      let dot_q34: i64 = nx_q4k_dot_row_col(W, j * bpr, n_blocks, colq10, it)
      CF[j] = nx_q10_to_f32(nx_q4km_q20_to_q10(dot_q34))
      j = j + 1
  }
  var mism: i64 = 0
  var jj: i64 = 0
  while jj < FVX_N { if CF[jj] != C4[jj] { mism = mism + 1 } jj = jj + 1 }
  if mism == 0 { pass = pass + 1; fv_puts("  T2 fused == x4 BIT-EXACT over all columns\n" as *u8) } else { fail = fail + 1; fv_puts("  T2 MISMATCH columns=" as *u8); fv_num(mism); fv_puts("\n" as *u8) }
122
  // ---- T3: NEGATIVE CONTROL -- this comparison must be ABLE to fail. ----
  // Perturb one activation in the Q10 domain only; the two routes must then DISAGREE.
  // Without this, T2 passing could mean "compared nothing".
  colq10[0] = colq10[0] + 1024
  let dot_bad: i64 = nx_q4k_dot_row_col(W, 0, n_blocks, colq10, it)
  let cf_bad: i64 = nx_q10_to_f32(nx_q4km_q20_to_q10(dot_bad))
  colq10[0] = colq10[0] - 1024
  if cf_bad != C4[0] { pass = pass + 1; fv_puts("  T3 neg-control: perturbed input DOES change the verdict\n" as *u8) } else { fail = fail + 1; fv_puts("  T3 neg-control DEAD -- T2 proves nothing\n" as *u8) }
131
  // ---- T5: nx_f32_to_q10 IN ITS REAL ROLE -- the adoption blocker. ----
  // The forward carries activations as f32 bits; the fused dot needs a Q10 column. Until now this
  // gate hand-built the Q10 column as v*1024, which tests the KERNEL but not the CONVERSION the
  // forward would actually have to perform. Rebuild the column with nx_f32_to_q10(A[i]) -- the real
  // path -- and require the fused result to STILL match x4 bit-exactly. A round-trip self-test would
  // not catch a conversion that is self-consistent but disagrees with the f32 kernel.
  let colq10b: *i64 = sys_mmap(FVX_K * 8) as *i64
  var ci: i64 = 0
  while ci < FVX_K { colq10b[ci] = nx_f32_to_q10(A[ci]); ci = ci + 1 }
  var cvt_bad: i64 = 0
  var cj: i64 = 0
  while cj < FVX_K { if colq10b[cj] != colq10[cj] { cvt_bad = cvt_bad + 1 } cj = cj + 1 }
  if cvt_bad == 0 { pass = pass + 1; fv_puts("  T5a nx_f32_to_q10 reproduces the Q10 column exactly\n" as *u8) } else { fail = fail + 1; fv_puts("  T5a nx_f32_to_q10 WRONG on " as *u8); fv_num(cvt_bad); fv_puts(" of K entries\n" as *u8) }
145
  var mism2: i64 = 0
  var jb: i64 = 0
  while jb < FVX_N {
      let db: i64 = nx_q4k_dot_row_col(W, jb * bpr, n_blocks, colq10b, it)
      if nx_q10_to_f32(nx_q4km_q20_to_q10(db)) != C4[jb] { mism2 = mism2 + 1 }
      jb = jb + 1
  }
  if mism2 == 0 { pass = pass + 1; fv_puts("  T5b fused-with-CONVERTED-column == x4 BIT-EXACT (adoption path is sound)\n" as *u8) } else { fail = fail + 1; fv_puts("  T5b converted-column MISMATCH columns=" as *u8); fv_num(mism2); fv_puts("\n" as *u8) }
154
  // ---- T4: THE MEASUREMENT. Same shape, same thread, same data. ----
  let t0: i64 = sys_now_us()
  nx_f32_q4k_matmul_x4(A, W, 0, C4, FVX_M, FVX_K, FVX_N)
  let t1: i64 = sys_now_us()
  var jf: i64 = 0
  while jf < FVX_N {
      CF[jf] = nx_q10_to_f32(nx_q4km_q20_to_q10(nx_q4k_dot_row_col(W, jf * bpr, n_blocks, colq10, it)))
      jf = jf + 1
  }
  let t2: i64 = sys_now_us()
  let us4: i64 = t1 - t0
  let usf: i64 = t2 - t1
  fv_puts("  x4_us=" as *u8); fv_num(us4)
  fv_puts("  fused_us=" as *u8); fv_num(usf); fv_puts("\n" as *u8)
  if usf > 0 { fv_puts("  fused_vs_x4_x100=" as *u8); fv_num((us4 * 100) / usf); fv_puts("  (>100 means fused is faster)\n" as *u8) }
170
  // THE REFUTATION TOOTH, written BEFORE the run: if fused is not faster than the kernel actually
  // dispatched today, the fused route does NOT justify the numeric change it costs, and must not be
  // wired into the forward. A negative result here is a real result -- exactly as it was for x8.
  if usf < us4 { pass = pass + 1; fv_puts("  T4 fused FASTER than the live x4 kernel\n" as *u8) } else { fail = fail + 1; fv_puts("  T4 fused NOT faster than live x4 -- DO NOT DISPATCH; the 20-48x was vs scalar-emulated f32, not vs the live path\n" as *u8) }
175
  // ---- T6/T7: THE m-BLOCKED FUSED GEMM -- the whole point of the rung. ----
  // nx_q4k_dot_row_col takes ONE column, so looping it m times would re-traverse and re-dequantise
  // the weight row m times, throwing away the amortisation fq4m_rows already has. fq4m_rows_fused
  // dequantises each weight element ONCE and accumulates into all m accumulators. The claim is
  // therefore NOT just "fused is fast at m=1" -- it is that fused stays ahead as m grows, which is
  // the compute-bound prefill regime. Test BOTH shapes; a win at m=1 alone would not establish it.
  let MB: i64 = 8
  let A8:  *i64 = sys_mmap(MB * FVX_K * 8) as *i64
  let C48: *i64 = sys_mmap(MB * FVX_N * 8) as *i64
  let CF8: *i64 = sys_mmap(MB * FVX_N * 8) as *i64
  var ri: i64 = 0
  while ri < MB {
      var ck: i64 = 0
      while ck < FVX_K {
          let vv: i64 = ((ri + ck) - ((ri + ck) / 7) * 7) - 3
          A8[ri * FVX_K + ck] = nx_i32_to_f32(vv)
          ck = ck + 1
      }
      ri = ri + 1
  }
196
  // T6: m=1 GEMM entry point agrees with x4 (the dispatcher-flip surface, not just the raw dot).
  let rf1: nx_int = nx_f32_q4k_matmul_fused(A, W, 0, CF, FVX_M, FVX_K, FVX_N)
  nx_f32_q4k_matmul_x4(A, W, 0, C4, FVX_M, FVX_K, FVX_N)
  var mm1: i64 = 0
  var q1: i64 = 0
  while q1 < FVX_N { if CF[q1] != C4[q1] { mm1 = mm1 + 1 } q1 = q1 + 1 }
  if rf1 == NX_FQ4M_OK { if mm1 == 0 { pass = pass + 1; fv_puts("  T6 fused GEMM m=1 == x4 BIT-EXACT\n" as *u8) } else { fail = fail + 1; fv_puts("  T6 fused GEMM m=1 MISMATCH cols=" as *u8); fv_num(mm1); fv_puts("\n" as *u8) } } else { fail = fail + 1; fv_puts("  T6 fused GEMM m=1 rc FAIL\n" as *u8) }
204
  // T7: m=8 correctness -- this is where a naive per-column fused loop would still be CORRECT but
  // slow, so correctness alone does not prove the m-blocking; T8 times it.
  let rf8: nx_int = nx_f32_q4k_matmul_fused(A8, W, 0, CF8, MB, FVX_K, FVX_N)
  nx_f32_q4k_matmul_x4(A8, W, 0, C48, MB, FVX_K, FVX_N)
  var mm8: i64 = 0
  var q8: i64 = 0
  while q8 < MB * FVX_N { if CF8[q8] != C48[q8] { mm8 = mm8 + 1 } q8 = q8 + 1 }
  if rf8 == NX_FQ4M_OK { if mm8 == 0 { pass = pass + 1; fv_puts("  T7 fused GEMM m=8 == x4 BIT-EXACT over all m*n cells\n" as *u8) } else { fail = fail + 1; fv_puts("  T7 fused GEMM m=8 MISMATCH cells=" as *u8); fv_num(mm8); fv_puts("\n" as *u8) } } else { fail = fail + 1; fv_puts("  T7 fused GEMM m=8 rc FAIL\n" as *u8) }
213
  // T8: THE PREFILL MEASUREMENT. Per-token cost is the honest unit here: an m x k x n GEMM does
  // 2*k*n FLOPs per token regardless of m, so what must not happen is fused LOSING its lead as m grows.
  let p0: i64 = sys_now_us()
  nx_f32_q4k_matmul_x4(A8, W, 0, C48, MB, FVX_K, FVX_N)
  let p1: i64 = sys_now_us()
  nx_f32_q4k_matmul_fused(A8, W, 0, CF8, MB, FVX_K, FVX_N)
  let p2: i64 = sys_now_us()
  let x4_8: i64 = p1 - p0
  let fu_8: i64 = p2 - p1
  fv_puts("  m=8  x4_us=" as *u8); fv_num(x4_8)
  fv_puts("  fused_us=" as *u8); fv_num(fu_8); fv_puts("\n" as *u8)
  if fu_8 > 0 { fv_puts("  m8_fused_vs_x4_x100=" as *u8); fv_num((x4_8 * 100) / fu_8); fv_puts("\n" as *u8) }
  if fu_8 < x4_8 { pass = pass + 1; fv_puts("  T8 fused still FASTER at m=8 (m-blocking holds in the batched regime)\n" as *u8) } else { fail = fail + 1; fv_puts("  T8 fused LOST its lead at m=8 -- m-blocking did NOT hold; do not claim the prefill regime\n" as *u8) }
227
  // D001: inherit nx_gate_verdict so nx_gate_green can judge this gate from outside and flake/erosion
  // stay visible, instead of rolling our own GREEN/RED line that nothing downstream can read.
  let ctr: *i64 = gv_ctr()
  ctr[0] = pass
  ctr[1] = pass + fail
  let rc: i64 = gv_verdict("Q4K-FUSED-VS-X4-GATE" as *u8, ctr,
                           "fused integer GEMM beats the live x4 kernel and matches it bit-exactly" as *u8)
  sys_exit(rc)
  return rc
237}
nx_q4k_fused_vs_x4_gate.nx source

`nx_q4k_fused_vs_x4_gate.nx` source
