code wiki / (root) / nx_nofloat_w12a12_gate.nx
`nx_nofloat_w12a12_gate.nx` source

↩ module page · 420 lines · 25794 B
1// nx_nofloat_w12a12_gate.nx -- the referee for the W12A12 decode matmul (search R0l attribution, 2026-09-16).
2//
3// SUBJECT: mm_pool_i8 in nx_nofloat_llm (the pooled SIMD matmul every decode projection goes through), driven
4// IN-PROCESS on synthetic data whose shape is the real one (in_dim 8960 = the Qwen2.5-1.5B ffn_down, the widest
5// projection, so the chunk bound is exercised at its limit) with activation OUTLIER channels planted, because a
6// per-tensor scale is exactly the quantiser an outlier defeats.
7// TEETH: the chunk bound is arithmetic (NF_CHUNK_K/8 * QMAX * QMAX <= i32 max) and 2048 is the planted bound-
8// breaker; the kernel's max relative error against the exact i64 dot is BELOW the old 8-bit per-tensor quantiser's
9// error on the same data (the old quantiser is re-implemented here as the CONTROL, so the improvement is measured,
10// never asserted); the kernel is exact on a row the quantisers cannot lose (all values on the grid); and the
11// chunked fold reproduces the unchunked sum on a short row (K < NF_CHUNK_K, one chunk).
12// license_tier: ORIGINAL   No hw writes (Rule 26).   expect_exit: 0
13import "nx_syscalls.nx"
14import "nx_nofloat_llm.nx"
15import "nx_gate_verdict.nx"
16
17const WG_IN: i64 = 8960
18const WG_OUT: i64 = 64
19const WG_SHORT_IN: i64 = 896
20const WG_SEED: i64 = 20260916
21const WG_LCG_A: i64 = 6364136223846793005
22const WG_LCG_C: i64 = 1442695040888963407
23const WG_MASK63: i64 = 0x7FFFFFFFFFFFFFFF
24const WG_X_BODY: i64 = 65536        // typical Q16 activation magnitude (about 1.0)
25const WG_X_OUTLIER: i64 = 4194304   // 64x the body: the outlier channel every Qwen-class model carries
26const WG_OUTLIER_EVERY: i64 = 512
27const WG_W_MAG: i64 = 4096
28const WG_SHIFT: i64 = 16
29const WG_OLD_QMAX: i64 = 127
30const WG_PERMIL: i64 = 1000
31const WG_BREAKER_CHUNK: i64 = 2048
32// on-grid rows: x = j * WG_GRID_SX with |j| <= QMAX so max|x| = QMAX * WG_GRID_SX and the activation scale is EXACTLY
33// WG_GRID_SX (no floor slack, no clamp), and every weight row carries one WG_GRID_WMAX so its scale is exactly 1
34const WG_GRID_SX: i64 = 16
35const WG_GRID_SPAN: i64 = 8191   // 2*QMAX+1 values, j in [-4095, 4095]
36const WG_GRID_WMAX: i64 = 4095
37const WG_TAIL_IN: i64 = 912       // R0q: 14 x 64 + 16, a width whose last madd goes through the tail loop
38const WG_TIMING_REPS: i64 = 40    // R0q: repeats per task for the printed throughput values
39const WG_OUT_BIG: i64 = 1536      // R0q: the ffn down projection's output rows (27.5 MB of i16 weights per call)
40const WG_BIG_REPS: i64 = 6        // R0q: repeats on the big shape
41const WG_BYTE_SPAN: i64 = 256     // one byte of pseudo-random weight fill
42const WG_BATCH_M: i64 = 32        // R0r: activation rows per batched call (the engine's block)
43const WG_TAIL_M: i64 = 5          // R0r: an odd row count on the tail-width fixture
44const WG_DOT_N: i64 = 1024        // R0r-b: one full chunk of lanes for the builtin KAT
45const WG_DOT_HALF: i64 = 512      // R0r-b: centres the ramp so half the lanes are negative
46const WG_DOT_PLANT: i64 = 777     // R0r-b: the lane the neg-control raises by one
47const WG_DOT_REAL_N: i64 = 512    // R0r-b: lanes of real weight rows in the KAT (64 products per int32 lane, a 2^30 bound)
48const WG_I16_MOD: i64 = 65536
49const WG_I16_HALF: i64 = 32768
50const WG_BYTE: i64 = 256
51// R0r-b: i16 lanes written and read back as bytes, and the exact scalar reference the builtin must equal.
52func wg_put_i16(p: *u8, i: i64, v: i64) -> i64 { var u: i64 = v; if u < 0 { u = u + WG_I16_MOD } p[i*2] = (u % WG_BYTE) as u8; p[i*2+1] = ((u / WG_BYTE) % WG_BYTE) as u8; return 0 }
53func wg_get_i16(p: *u8, i: i64) -> i64 { var u: i64 = (p[i*2] as i64) + (p[i*2+1] as i64) * WG_BYTE; if u >= WG_I16_HALF { u = u - WG_I16_MOD } return u }
54func wg_dot_ref(a: *u8, b: *u8, n: i64) -> i64 { var s: i64 = 0; var i: i64 = 0; while i < n { s = s + wg_get_i16(a, i) * wg_get_i16(b, i); i = i + 1 } return s }
55
56func wg_lcg(st: *i64) -> i64 { st[0] = (st[0]*WG_LCG_A + WG_LCG_C) & WG_MASK63; return st[0] }
57func wg_abs(v: i64) -> i64 { if v < 0 { return 0 - v } return v }
58func wg_fill_x(x: *i64, n: i64, st: *i64) -> i64 {
  var k: i64 = 0
  while k < n {
      var mag: i64 = WG_X_BODY
      if (k % WG_OUTLIER_EVERY) == 0 { mag = WG_X_OUTLIER }
      let r: i64 = wg_lcg(st) % (2*mag+1)
      x[k] = r - mag
      k = k + 1
  }
  return 0
68}
69func wg_fill_w(w: *i64, n: i64, st: *i64) -> i64 { var k: i64 = 0; while k < n { let r: i64 = wg_lcg(st) % (2*WG_W_MAG+1); w[k] = r - WG_W_MAG; k = k + 1 } return 0 }
70// the exact reference: i64 dot, then the same >> shift the kernel applies
71func wg_exact(x: *i64, w: *i64, ind: i64, out: i64, dst: *i64) -> i64 {
  var o: i64 = 0
  while o < out { var s: i64 = 0; var k: i64 = 0; while k < ind { s = s + x[k]*w[o*ind+k]; k = k + 1 } dst[o] = s >> WG_SHIFT; o = o + 1 }
  return 0
75}
76// THE CONTROL: the pre-2026-09-16 quantiser, 8-bit per-tensor activation and 8-bit per-row weight, truncating
77func wg_old_path(x: *i64, w: *i64, ind: i64, out: i64, dst: *i64) -> i64 {
  var xmx: i64 = 0; var k: i64 = 0
  while k < ind { if wg_abs(x[k]) > xmx { xmx = wg_abs(x[k]) } k = k + 1 }
  var sx: i64 = xmx / WG_OLD_QMAX; if sx < 1 { sx = 1 }
  var o: i64 = 0
  while o < out {
      var rm: i64 = 0; k = 0
      while k < ind { if wg_abs(w[o*ind+k]) > rm { rm = wg_abs(w[o*ind+k]) } k = k + 1 }
      var sw: i64 = rm / WG_OLD_QMAX; if sw < 1 { sw = 1 }
      var s: i64 = 0; k = 0
      while k < ind { s = s + (x[k]/sx) * (w[o*ind+k]/sw); k = k + 1 }
      dst[o] = ((sx*sw)*s) >> WG_SHIFT
      o = o + 1
  }
  return 0
92}
93// max relative error in permil of `got` against `ref` over `n` rows (rows with |ref| below WG_PERMIL are skipped so a
94// near-zero reference cannot manufacture an infinite ratio; the count of scored rows is returned through `scored`)
95func wg_max_err_permil(ref: *i64, got: *i64, n: i64, scored: *i64) -> i64 {
  var worst: i64 = 0; var o: i64 = 0; var c: i64 = 0
  while o < n {
      let r: i64 = wg_abs(ref[o])
      if r >= WG_PERMIL {
          let e: i64 = (wg_abs(got[o] - ref[o]) * WG_PERMIL) / r
          if e > worst { worst = e }
          c = c + 1
      }
      o = o + 1
  }
  scored[0] = c
  return worst
108}
109func wg_quant_weights(w: *i64, ind: i64, out: i64, wi8: *u8, sw: *i64) -> i64 { return nf_quant_w_i8(w, wi8, sw, out, ind) }
110
111func main(argc: i64, argv: *i64) -> i64 {
  let ctr: *i64 = gv_ctr()
  gv_head("nx_nofloat_w12a12_gate -- the 12-bit decode matmul beats the 8-bit per-tensor quantiser it replaces, and its chunk bound is arithmetic" as *u8)
  // ---- T1 the chunk bound ----
  let lane_products: i64 = NF_CHUNK_K / NF_MADD_PER_LANE
  let lane_max: i64 = lane_products * NF_X_QMAX * NF_W_QMAX
  gv_check("T1 NF_CHUNK_K/8 * QMAX * QMAX fits an int32 lane" as *u8, (lane_max <= NF_I32_ACC_MAX) as i64, ctr)
  gv_check("T1a NF_CHUNK_K is a multiple of 16 (one madd)" as *u8, ((NF_CHUNK_K % 16) == 0) as i64, ctr)
  let breaker: i64 = (WG_BREAKER_CHUNK / NF_MADD_PER_LANE) * NF_X_QMAX * NF_W_QMAX
  gv_check("T1b neg-control a 2048 chunk would overflow the lane" as *u8, (breaker > NF_I32_ACC_MAX) as i64, ctr)
  gv_check("T1c the derived bound is the largest 16-multiple: NF_CHUNK_K + 16 overflows" as *u8, (((NF_CHUNK_K + 16) / NF_MADD_PER_LANE) * NF_X_QMAX * NF_W_QMAX > NF_I32_ACC_MAX) as i64, ctr)
  gv_kv("lane_max_at_chunk" as *u8, lane_max)
  gv_kv("i32_acc_max" as *u8, NF_I32_ACC_MAX)
  // ---- T2 rounding ----
  gv_check_eq("T2 round-to-nearest, positive (7/2 -> 4)" as *u8, nf_qround(7, 2, NF_X_QMAX), 4, ctr)
  gv_check_eq("T2a round-to-nearest, negative (-7/2 -> -4)" as *u8, nf_qround(0 - 7, 2, NF_X_QMAX), 0 - 4, ctr)
  gv_check_eq("T2b clamp at +qmax" as *u8, nf_qround(99999999, 1, NF_X_QMAX), NF_X_QMAX, ctr)
  gv_check_eq("T2c clamp at -qmax" as *u8, nf_qround(0 - 99999999, 1, NF_X_QMAX), 0 - NF_X_QMAX, ctr)
  // ---- T3 the kernel vs the exact dot vs the old quantiser, real widths, planted outliers ----
  let st: *i64 = sys_mmap(8) as *i64; st[0] = WG_SEED
  let x: *i64 = sys_mmap(WG_IN*8) as *i64
  let w: *i64 = sys_mmap(WG_IN*WG_OUT*8) as *i64
  wg_fill_x(x, WG_IN, st)
  wg_fill_w(w, WG_IN*WG_OUT, st)
  let exact: *i64 = sys_mmap(WG_OUT*8) as *i64
  let old: *i64 = sys_mmap(WG_OUT*8) as *i64
  let got: *i64 = sys_mmap(WG_OUT*8) as *i64
  wg_exact(x, w, WG_IN, WG_OUT, exact)
  wg_old_path(x, w, WG_IN, WG_OUT, old)
  let wi8: *u8 = sys_mmap(WG_IN*WG_OUT*2)
  let sw: *i64 = sys_mmap(WG_OUT*8) as *i64
  wg_quant_weights(w, WG_IN, WG_OUT, wi8, sw)
  mm_pool_i8(x, wi8, sw, got, WG_IN, WG_OUT, WG_SHIFT)
  let sc: *i64 = sys_mmap(16) as *i64
  let e_old: i64 = wg_max_err_permil(exact, old, WG_OUT, sc)
  let rows_old: i64 = sc[0]
  let e_new: i64 = wg_max_err_permil(exact, got, WG_OUT, sc)
  let rows_new: i64 = sc[0]
  gv_check("T3 the fixture reached the condition: every output row scored (|exact| above the permil floor)" as *u8, ((rows_old == WG_OUT) as i64) * ((rows_new == WG_OUT) as i64), ctr)
  gv_check("T3a the planted outliers made the OLD 8-bit per-tensor path lose at least 10 permil somewhere" as *u8, (e_old >= 10) as i64, ctr)
  gv_check("T3b W12A12 max relative error is BELOW the old path's" as *u8, (e_new < e_old) as i64, ctr)
  // the grid is 32x finer (4095 vs 127 levels) so per-element rounding error falls 32x; the dot product's error
  // falls less where the old errors happened to cancel, hence a conservative 8x bound rather than the theoretical 32x
  gv_check("T3c W12A12 max relative error is at least 8x below the old path's (32x finer grid, 4x margin)" as *u8, (e_new * 8 < e_old) as i64, ctr)
  gv_kv("old_w8a8_max_err_permil" as *u8, e_old)
  gv_kv("new_w12a12_max_err_permil" as *u8, e_new)
  // ---- T4 an on-grid row is EXACT through the kernel (nothing for either quantiser to lose) ----
  var k: i64 = 0
  while k < WG_IN { x[k] = ((wg_lcg(st) % WG_GRID_SPAN) - NF_X_QMAX) * WG_GRID_SX; k = k + 1 }
  x[0] = NF_X_QMAX * WG_GRID_SX
  k = 0
  while k < WG_IN*WG_OUT { w[k] = (wg_lcg(st) % WG_GRID_SPAN) - NF_X_QMAX; k = k + 1 }
  var o2: i64 = 0
  while o2 < WG_OUT { w[o2*WG_IN] = WG_GRID_WMAX; o2 = o2 + 1 }
  wg_exact(x, w, WG_IN, WG_OUT, exact)
  wg_quant_weights(w, WG_IN, WG_OUT, wi8, sw)
  mm_pool_i8(x, wi8, sw, got, WG_IN, WG_OUT, WG_SHIFT)
  var same: i64 = 1; var o: i64 = 0
  while o < WG_OUT { if got[o] != exact[o] { same = 0 } o = o + 1 }
  gv_check("T4 an on-grid row is bit-exact through the chunked kernel (8960 wide, 9 chunks)" as *u8, same, ctr)
  // ---- T5 a short row (one chunk) agrees with the exact dot the same way ----
  let xs: *i64 = sys_mmap(WG_SHORT_IN*8) as *i64
  let ws: *i64 = sys_mmap(WG_SHORT_IN*WG_OUT*8) as *i64
  k = 0; while k < WG_SHORT_IN { xs[k] = ((wg_lcg(st) % WG_GRID_SPAN) - NF_X_QMAX) * WG_GRID_SX; k = k + 1 }
  xs[0] = NF_X_QMAX * WG_GRID_SX
  k = 0; while k < WG_SHORT_IN*WG_OUT { ws[k] = (wg_lcg(st) % WG_GRID_SPAN) - NF_X_QMAX; k = k + 1 }
  o = 0
  while o < WG_OUT { ws[o*WG_SHORT_IN] = WG_GRID_WMAX; o = o + 1 }
  wg_exact(xs, ws, WG_SHORT_IN, WG_OUT, exact)
  let wi8s: *u8 = sys_mmap(WG_SHORT_IN*WG_OUT*2)
  wg_quant_weights(ws, WG_SHORT_IN, WG_OUT, wi8s, sw)
  mm_pool_i8(xs, wi8s, sw, got, WG_SHORT_IN, WG_OUT, WG_SHIFT)
  same = 1; o = 0
  while o < WG_OUT { if got[o] != exact[o] { same = 0 } o = o + 1 }
  gv_check("T5 a one-chunk row (896 wide) is bit-exact on the grid" as *u8, same, ctr)
  // ---- T6 R0q: the four-accumulator task is bit-identical to the single-accumulator incumbent ----
  wg_fill_x(x, WG_IN, st)
  wg_fill_w(w, WG_IN*WG_OUT, st)
  wg_quant_weights(w, WG_IN, WG_OUT, wi8, sw)
  g_nf_i8_r4 = 0
  mm_pool_i8(x, wi8, sw, old, WG_IN, WG_OUT, WG_SHIFT)
  g_nf_i8_r4 = 1
  mm_pool_i8(x, wi8, sw, got, WG_IN, WG_OUT, WG_SHIFT)
  same = 1; o = 0
  while o < WG_OUT { if got[o] != old[o] { same = 0 } o = o + 1 }
  gv_check("T6 R0q four-accumulator task bit-identical to the incumbent (8960 wide, planted outliers, 9 chunks)" as *u8, same, ctr)
  // T6a a width that is a multiple of 16 but not of 64 exercises the tail madd (912 = 14 x 64 + 16)
  let xt: *i64 = sys_mmap(WG_TAIL_IN*8) as *i64
  let wt: *i64 = sys_mmap(WG_TAIL_IN*WG_OUT*8) as *i64
  wg_fill_x(xt, WG_TAIL_IN, st)
  wg_fill_w(wt, WG_TAIL_IN*WG_OUT, st)
  let wi8t: *u8 = sys_mmap(WG_TAIL_IN*WG_OUT*2)
  wg_quant_weights(wt, WG_TAIL_IN, WG_OUT, wi8t, sw)
  g_nf_i8_r4 = 0
  mm_pool_i8(xt, wi8t, sw, old, WG_TAIL_IN, WG_OUT, WG_SHIFT)
  g_nf_i8_r4 = 1
  mm_pool_i8(xt, wi8t, sw, got, WG_TAIL_IN, WG_OUT, WG_SHIFT)
  same = 1; o = 0
  while o < WG_OUT { if got[o] != old[o] { same = 0 } o = o + 1 }
  gv_check("T6a bit-identical on a 912-wide row (fourteen 64-steps plus one tail madd)" as *u8, same, ctr)
  // T6b the fixture reached the condition: the tail loop ran (912 mod 64 = 16 = one madd)
  gv_check_eq("T6b the tail width is exactly one madd" as *u8, WG_TAIL_IN % NF_R4_STRIDE, NF_MADD_W, ctr)
  // ---- T7 throughput of the two tasks on the real ffn width, printed as values (a timing is never a tooth) ----
  g_nf_i8_r4 = 0
  let t0: i64 = sys_now_us()
  var rep: i64 = 0
  while rep < WG_TIMING_REPS { mm_pool_i8(x, wi8, sw, old, WG_IN, WG_OUT, WG_SHIFT); rep = rep + 1 }
  let t1: i64 = sys_now_us()
  g_nf_i8_r4 = 1
  rep = 0
  while rep < WG_TIMING_REPS { mm_pool_i8(x, wi8, sw, got, WG_IN, WG_OUT, WG_SHIFT); rep = rep + 1 }
  let t2: i64 = sys_now_us()
  gv_kv("incumbent_us_per_call" as *u8, (t1 - t0) / WG_TIMING_REPS)
  gv_kv("r4_us_per_call" as *u8, (t2 - t1) / WG_TIMING_REPS)
  var spd: i64 = 0
  if t2 > t1 { spd = ((t1 - t0) * WG_PERMIL) / (t2 - t1) }
  gv_kv("r4_speedup_permil" as *u8, spd)
  gv_check("T7 the served default is the single-accumulator incumbent (the four-accumulator task measured slower on the decode shape)" as *u8, (NF_I8_R4_DEFAULT == 0) as i64, ctr)
  // ---- T7b the decode SHAPE: 1536 output rows x 8960 (the ffn down projection), 27.5 MB of weights per call, so the
  // per-call time is the kernel's own streaming rate and not the pool's dispatch cost on a 64-row fixture
  let wbig: *u8 = sys_mmap(WG_IN*WG_OUT_BIG*2)
  let swbig: *i64 = sys_mmap(WG_OUT_BIG*8) as *i64
  let gotbig: *i64 = sys_mmap(WG_OUT_BIG*8) as *i64
  var fb: i64 = 0
  while fb < WG_IN*WG_OUT_BIG*2 { wbig[fb] = (wg_lcg(st) % WG_BYTE_SPAN) as u8; fb = fb + 1 }
  fb = 0
  while fb < WG_OUT_BIG { swbig[fb] = 1; fb = fb + 1 }
  g_nf_i8_r4 = 0
  let tb0: i64 = sys_now_us()
  rep = 0
  while rep < WG_BIG_REPS { mm_pool_i8(x, wbig, swbig, gotbig, WG_IN, WG_OUT_BIG, WG_SHIFT); rep = rep + 1 }
  let tb1: i64 = sys_now_us()
  g_nf_i8_r4 = 1
  rep = 0
  while rep < WG_BIG_REPS { mm_pool_i8(x, wbig, swbig, gotbig, WG_IN, WG_OUT_BIG, WG_SHIFT); rep = rep + 1 }
  let tb2: i64 = sys_now_us()
  let usb_old: i64 = (tb1 - tb0) / WG_BIG_REPS
  let usb_r4: i64 = (tb2 - tb1) / WG_BIG_REPS
  gv_kv("big_incumbent_us_per_call" as *u8, usb_old)
  gv_kv("big_r4_us_per_call" as *u8, usb_r4)
  if usb_old > 0 { gv_kv("big_incumbent_mb_per_s" as *u8, (WG_IN*WG_OUT_BIG*2) / usb_old) }
  if usb_r4 > 0 { gv_kv("big_r4_mb_per_s" as *u8, (WG_IN*WG_OUT_BIG*2) / usb_r4) }
  // ---- T8 R0r: the batched kernel (M rows, one weight pass) is bit-identical to M single calls, cell for cell ----
  wg_quant_weights(w, WG_IN, WG_OUT, wi8, sw)
  let xm: *i64 = sys_mmap(WG_BATCH_M*WG_IN*8) as *i64
  wg_fill_x(xm, WG_BATCH_M*WG_IN, st)
  let gotm: *i64 = sys_mmap(WG_BATCH_M*WG_OUT*8) as *i64
  mm_pool_i8_m(xm, WG_BATCH_M, wi8, sw, gotm, WG_IN, WG_OUT, WG_SHIFT)
  same = 1
  var cells: i64 = 0
  var mrow: i64 = 0
  while mrow < WG_BATCH_M {
      mm_pool_i8(((xm as i64)+mrow*WG_IN*8) as *i64, wi8, sw, old, WG_IN, WG_OUT, WG_SHIFT)
      o = 0
      while o < WG_OUT { if gotm[mrow*WG_OUT+o] != old[o] { same = 0 } cells = cells + 1; o = o + 1 }
      mrow = mrow + 1
  }
  gv_kv("batch_cells_compared" as *u8, cells)
  gv_check("T8 R0r batched kernel bit-identical to M single calls (8960 wide, 32 rows, 9 chunks)" as *u8, same, ctr)
  // T8a an odd row count on the tail width (912 = fourteen 64-steps plus one madd)
  wg_quant_weights(wt, WG_TAIL_IN, WG_OUT, wi8t, sw)
  let xmt: *i64 = sys_mmap(WG_TAIL_M*WG_TAIL_IN*8) as *i64
  wg_fill_x(xmt, WG_TAIL_M*WG_TAIL_IN, st)
  let gotmt: *i64 = sys_mmap(WG_TAIL_M*WG_OUT*8) as *i64
  mm_pool_i8_m(xmt, WG_TAIL_M, wi8t, sw, gotmt, WG_TAIL_IN, WG_OUT, WG_SHIFT)
  same = 1
  mrow = 0
  while mrow < WG_TAIL_M {
      mm_pool_i8(((xmt as i64)+mrow*WG_TAIL_IN*8) as *i64, wi8t, sw, old, WG_TAIL_IN, WG_OUT, WG_SHIFT)
      o = 0
      while o < WG_OUT { if gotmt[mrow*WG_OUT+o] != old[o] { same = 0 } o = o + 1 }
      mrow = mrow + 1
  }
  gv_check("T8a batched kernel bit-identical on the 912-wide row with 5 activation rows" as *u8, same, ctr)
  // T8b neg-control: the comparator fires on one planted cell
  gotmt[WG_OUT+1] = gotmt[WG_OUT+1] + 1
  var planted: i64 = 1
  mrow = 0
  while mrow < WG_TAIL_M {
      mm_pool_i8(((xmt as i64)+mrow*WG_TAIL_IN*8) as *i64, wi8t, sw, old, WG_TAIL_IN, WG_OUT, WG_SHIFT)
      o = 0
      while o < WG_OUT { if gotmt[mrow*WG_OUT+o] != old[o] { planted = 0 } o = o + 1 }
      mrow = mrow + 1
  }
  gv_check("neg-control-T8b the cell comparator reads a planted +1 as a difference" as *u8, (planted == 0) as i64, ctr)
  // T8c the gate's row count is the engine's block (the fixture measures the shape the serve runs)
  gv_check_eq("T8c the gate batch equals the engine's prefill block" as *u8, WG_BATCH_M, NF_PREFILL_M, ctr)
  // ---- T8d the decode SHAPE per prompt row: 32 single calls against one 32-row call (values, never a tooth) ----
  let gotbigm: *i64 = sys_mmap(WG_BATCH_M*WG_OUT_BIG*8) as *i64
  let tc0: i64 = sys_now_us()
  mrow = 0
  while mrow < WG_BATCH_M { mm_pool_i8(((xm as i64)+mrow*WG_IN*8) as *i64, wbig, swbig, gotbig, WG_IN, WG_OUT_BIG, WG_SHIFT); mrow = mrow + 1 }
  let tc1: i64 = sys_now_us()
  mm_pool_i8_m(xm, WG_BATCH_M, wbig, swbig, gotbigm, WG_IN, WG_OUT_BIG, WG_SHIFT)
  let tc2: i64 = sys_now_us()
  gv_kv("batch_single_us_per_row" as *u8, (tc1 - tc0) / WG_BATCH_M)
  gv_kv("batch_m_us_per_row" as *u8, (tc2 - tc1) / WG_BATCH_M)
  var bspd: i64 = 0
  if tc2 > tc1 { bspd = ((tc1 - tc0) * WG_PERMIL) / (tc2 - tc1) }
  gv_kv("batch_speedup_permil" as *u8, bspd)
  // ---- T8e R0r-c: the four-accumulator batched task is bit-identical to the single-accumulator batched task ----
  let gotm4: *i64 = sys_mmap(WG_BATCH_M*WG_OUT*8) as *i64
  g_nf_i8_r4m = 0
  mm_pool_i8_m(xm, WG_BATCH_M, wi8, sw, gotm, WG_IN, WG_OUT, WG_SHIFT)
  g_nf_i8_r4m = 1
  mm_pool_i8_m(xm, WG_BATCH_M, wi8, sw, gotm4, WG_IN, WG_OUT, WG_SHIFT)
  same = 1
  var cm: i64 = 0
  while cm < WG_BATCH_M*WG_OUT { if gotm4[cm] != gotm[cm] { same = 0 } cm = cm + 1 }
  gv_check("T8e R0r-c four-accumulator batched task bit-identical to the single-accumulator batched task (32 rows, 8960 wide)" as *u8, same, ctr)
  let gotmt4: *i64 = sys_mmap(WG_TAIL_M*WG_OUT*8) as *i64
  g_nf_i8_r4m = 0
  mm_pool_i8_m(xmt, WG_TAIL_M, wi8t, sw, gotmt, WG_TAIL_IN, WG_OUT, WG_SHIFT)
  g_nf_i8_r4m = 1
  mm_pool_i8_m(xmt, WG_TAIL_M, wi8t, sw, gotmt4, WG_TAIL_IN, WG_OUT, WG_SHIFT)
  same = 1; cm = 0
  while cm < WG_TAIL_M*WG_OUT { if gotmt4[cm] != gotmt[cm] { same = 0 } cm = cm + 1 }
  gv_check("T8e2 bit-identical on the 912-wide row with 5 rows (the tail madd)" as *u8, same, ctr)
  gv_check("T8e3 the served default is the single-accumulator batched task (the four-accumulator task measured a wash on the decode shape, 451 vs 448 us per row)" as *u8, (NF_I8_R4M_DEFAULT == 0) as i64, ctr)
  // ---- T8f the decode SHAPE per prompt row, both batched tasks (values, never a tooth) ----
  g_nf_i8_r4m = 0
  let td0: i64 = sys_now_us()
  mm_pool_i8_m(xm, WG_BATCH_M, wbig, swbig, gotbigm, WG_IN, WG_OUT_BIG, WG_SHIFT)
  let td1: i64 = sys_now_us()
  g_nf_i8_r4m = 1
  mm_pool_i8_m(xm, WG_BATCH_M, wbig, swbig, gotbigm, WG_IN, WG_OUT_BIG, WG_SHIFT)
  let td2: i64 = sys_now_us()
  gv_kv("batch_m1_us_per_row" as *u8, (td1 - td0) / WG_BATCH_M)
  gv_kv("batch_m4_us_per_row" as *u8, (td2 - td1) / WG_BATCH_M)
  var m4spd: i64 = 0
  if td2 > td1 { m4spd = ((td1 - td0) * WG_PERMIL) / (td2 - td1) }
  gv_kv("batch_m4_speedup_permil" as *u8, m4spd)
  // ---- T9 R0r-b: __i16_dot, the compiler's whole-chunk integer dot with the accumulator in a register ----
  let da: *u8 = sys_mmap(WG_DOT_N*2)
  let db: *u8 = sys_mmap(WG_DOT_N*2)
  var di: i64 = 0
  while di < WG_DOT_N { wg_put_i16(da, di, di + 1 - WG_DOT_HALF); wg_put_i16(db, di, WG_DOT_HALF - di); di = di + 1 }
  let dref: i64 = wg_dot_ref(da, db, WG_DOT_N)
  let dgot: i64 = __i16_dot(da, db, WG_DOT_N)
  gv_check_eq("T9 R0r-b __i16_dot equals the exact scalar reference on a 1024-lane signed ramp" as *u8, dgot, dref, ctr)
  gv_check("T9a the reference is not zero (a vacuous fixture cannot pass)" as *u8, (dref != 0) as i64, ctr)
  gv_check_eq("T9b __i16_dot on the smallest legal call, one 16-lane trip" as *u8, __i16_dot(da, db, NF_MADD_W), wg_dot_ref(da, db, NF_MADD_W), ctr)
  wg_put_i16(da, WG_DOT_PLANT, wg_get_i16(da, WG_DOT_PLANT) + 1)
  gv_check_eq("neg-control-T9c a planted +1 in one lane of a moves the dot by exactly that lane of b" as *u8, __i16_dot(da, db, WG_DOT_N) - dgot, wg_get_i16(db, WG_DOT_PLANT), ctr)
  let wrow1: *u8 = ((wi8 as i64) + WG_IN*2) as *u8
  gv_check_eq("T9d __i16_dot over two real i16 weight rows equals the scalar reference" as *u8, __i16_dot(wi8, wrow1, WG_DOT_REAL_N), wg_dot_ref(wi8, wrow1, WG_DOT_REAL_N), ctr)
  let gotmd: *i64 = sys_mmap(WG_BATCH_M*WG_OUT*8) as *i64
  g_nf_i8_dot = 0
  mm_pool_i8_m(xm, WG_BATCH_M, wi8, sw, gotm, WG_IN, WG_OUT, WG_SHIFT)
  g_nf_i8_dot = 1
  mm_pool_i8_m(xm, WG_BATCH_M, wi8, sw, gotmd, WG_IN, WG_OUT, WG_SHIFT)
  same = 1; cm = 0
  while cm < WG_BATCH_M*WG_OUT { if gotmd[cm] != gotm[cm] { same = 0 } cm = cm + 1 }
  gv_check("T9e R0r-b the builtin batched kernel is bit-identical to the madd-loop batched kernel (32 rows, 8960 wide)" as *u8, same, ctr)
  let gotmtd: *i64 = sys_mmap(WG_TAIL_M*WG_OUT*8) as *i64
  g_nf_i8_dot = 0
  mm_pool_i8_m(xmt, WG_TAIL_M, wi8t, sw, gotmt, WG_TAIL_IN, WG_OUT, WG_SHIFT)
  g_nf_i8_dot = 1
  mm_pool_i8_m(xmt, WG_TAIL_M, wi8t, sw, gotmtd, WG_TAIL_IN, WG_OUT, WG_SHIFT)
  same = 1; cm = 0
  while cm < WG_TAIL_M*WG_OUT { if gotmtd[cm] != gotmt[cm] { same = 0 } cm = cm + 1 }
  gv_check("T9f bit-identical on the 912-wide row with 5 rows (one 912-lane call per cell)" as *u8, same, ctr)
  gv_check("T9g the served default is the builtin batched kernel" as *u8, (NF_I8_DOT_DEFAULT == 1) as i64, ctr)
  // ---- T10 R0s: the pool quantiser is byte-identical to the serial quantiser it replaces ----
  g_nf_quant_serial = 1
  nf_quant_rows_pool(xm, WG_BATCH_M, WG_IN)
  let qs_us: i64 = g_nf_mm_us_quant
  let qbytes: i64 = WG_BATCH_M*WG_IN*2
  let qcopy: *u8 = sys_mmap(qbytes)
  var qi: i64 = 0
  while qi < qbytes { qcopy[qi] = g_nf_xi8m[qi]; qi = qi + 1 }
  let scopy: *i64 = sys_mmap(WG_BATCH_M*8) as *i64
  qi = 0; while qi < WG_BATCH_M { scopy[qi] = g_nf_sxm[qi]; qi = qi + 1 }
  g_nf_quant_serial = 0
  nf_quant_rows_pool(xm, WG_BATCH_M, WG_IN)
  let qp_us: i64 = g_nf_mm_us_quant
  var qsame: i64 = 1
  qi = 0; while qi < qbytes { if qcopy[qi] != g_nf_xi8m[qi] { qsame = 0 } qi = qi + 1 }
  var ssame: i64 = 1
  qi = 0; while qi < WG_BATCH_M { if scopy[qi] != g_nf_sxm[qi] { ssame = 0 } qi = qi + 1 }
  gv_check("T10 R0s pool-quantised rows byte-identical to the serial quantiser (32 rows, 8960 wide)" as *u8, qsame, ctr)
  gv_check("T10a the per-row scales are identical" as *u8, ssame, ctr)
  qcopy[WG_DOT_PLANT] = ((qcopy[WG_DOT_PLANT] as i64) + 1) as u8
  var qdiff: i64 = 0
  qi = 0; while qi < qbytes { if qcopy[qi] != g_nf_xi8m[qi] { qdiff = qdiff + 1 } qi = qi + 1 }
  gv_check_eq("neg-control-T10b the byte comparator reads one planted byte as exactly one difference" as *u8, qdiff, 1, ctr)
  gv_check("T10c the served default is the pool quantiser" as *u8, (NF_QUANT_SERIAL_DEFAULT == 0) as i64, ctr)
  gv_kv("quant_serial_us" as *u8, qs_us)
  gv_kv("quant_pool_us" as *u8, qp_us)
  // ---- the decode SHAPE per prompt row, madd loop vs builtin (values, never a tooth: a loaded box makes timing flaky) ----
  g_nf_i8_dot = 0
  let te0: i64 = sys_now_us()
  mm_pool_i8_m(xm, WG_BATCH_M, wbig, swbig, gotbigm, WG_IN, WG_OUT_BIG, WG_SHIFT)
  let te1: i64 = sys_now_us()
  g_nf_i8_dot = 1
  mm_pool_i8_m(xm, WG_BATCH_M, wbig, swbig, gotbigm, WG_IN, WG_OUT_BIG, WG_SHIFT)
  let te2: i64 = sys_now_us()
  gv_kv("batch_madd_us_per_row" as *u8, (te1 - te0) / WG_BATCH_M)
  gv_kv("batch_dot_us_per_row" as *u8, (te2 - te1) / WG_BATCH_M)
  var dspd: i64 = 0
  if te2 > te1 { dspd = ((te1 - te0) * WG_PERMIL) / (te2 - te1) }
  gv_kv("batch_dot_speedup_permil" as *u8, dspd)
  // the split of the LAST (builtin) call: the single-threaded quantisation of 32 rows vs the pool's matmul
  gv_kv("batch_dot_quant_us" as *u8, g_nf_mm_us_quant)
  gv_kv("batch_dot_pool_us" as *u8, g_nf_mm_us_pool)
  gv_kv("batch_dot_pool_workers" as *u8, nf_pool().n_workers)
  let rc: i64 = gv_verdict("NOFLOAT-W12A12-GATE" as *u8, ctr, "the chunk bound is arithmetic, the 12-bit kernel beats the 8-bit control on planted outliers and is exact on the grid" as *u8)
  sys_exit_group(rc)   // exit_group, not the raw thread exit: the pool workers parked in futex wait must die with the verdict
  return rc
420}
nx_nofloat_w12a12_gate.nx source

`nx_nofloat_w12a12_gate.nx` source
