nx_roofline_gate.nx source

1// nx_roofline_gate.nx -- IS DECODE COMPUTE-BOUND OR MEMORY-BANDWIDTH-BOUND? 2// 3// WHY (2026-07-31): an open sev-9 calls the sovereign embedder '~1000x too slow' at 400ms/token and declares 4// the path infeasible. That was read as a CODE defect. But a batch=1 decode step must read EVERY weight once, 5// so the floor is bytes/bandwidth and nothing else: 0.5B params x 4 bytes (f32) = 2.0 GB READ PER TOKEN. 6// At a few GB/s that is hundreds of ms/token BY ARITHMETIC, with a perfect kernel. If measured is near the 7// floor then the code is not slow, it is AT ROOFLINE, and the only levers are FEWER BYTES (4-bit weights) or 8// MORE WORK PER BYTE (batch m>1). Tuning the dot product would buy nothing. 9// 10// THE CODEBASE ALREADY REASONED THIS OUT, INDEPENDENTLY: nx_f32_lazy_weight.nx:706 states 'the 2.72x 11// single-dot gate was L1-resident = misleading. The real decode lever is FEWER BYTES read (keep weights 12// QUANTIZED 4-bit + fast SIMD dequant-dot), or batch tokens (m>1 = compute-bound), NOT f32 packing.' 13// AND IT MATCHES 2026 EXTERNAL SOTA: Neural Speed reports >90pct of MEMORY BANDWIDTH utilisation on INT4 14// GEMV (the good engines are already AT the bandwidth bound, so bandwidth IS the ceiling), and 15// Graviton3/Llama3-8B-4bit measures 45.5 tok/s at batch=1 vs 184.8 tok/s at batch=8 -- a ~4x win from 16// BATCHING ALONE, which is exactly the signature of a bandwidth-bound decode turning compute-bound. 17// 18// WHY THIS ORGAN EXISTS RATHER THAN A NOTE: a small model will never re-derive this. The insight has to live 19// in an INSTRUMENT that fires for whoever comes next, or it is rediscovered by hand every time -- which is 20// how 400ms/token got recorded as a defect instead of a roofline. 21// 22// nx_roofline_gate -- selftest teeth only 23// nx_roofline_gate est <Mparams> [ms] -- floors for f32/int8/int4; with [ms], the roofline fraction 24// license_tier: ORIGINAL No hw writes (Rule 26). expect_exit: 0 25import "nx_syscalls.nx" 26 27const RF_BUFMB: i64 = 64 28const RF_MB: i64 = 1048576 29const RF_STRIDE: i64 = 64 30const RF_NEAR_PERMIL: i64 = 700 31 32func rf_puts(s: *u8) { 33 var n: i64 = 0 34 while s[n] != (0 as u8) { n = n + 1 } 35 sys_write(1, s, n) 36} 37 38func rf_puti(x: i64) { 39 var buf: *u8 = sys_mmap(64) as *u8 40 var v: i64 = x 41 var neg: i64 = 0 42 if v < 0 { 43 neg = 1 44 v = 0 - v 45 } 46 var i: i64 = 40 47 if v == 0 { 48 i = i - 1 49 buf[i] = 48 as u8 50 } 51 while v > 0 { 52 let d: i64 = v - (v / 10) * 10 53 i = i - 1 54 buf[i] = (d + 48) as u8 55 v = v / 10 56 } 57 if neg == 1 { 58 i = i - 1 59 buf[i] = 45 as u8 60 } 61 sys_write(1, ((buf as i64) + i) as *u8, 40 - i) 62} 63 64func rf_atoi(s: *u8) -> i64 { 65 var r: i64 = 0 66 var i: i64 = 0 67 while s[i] != (0 as u8) { 68 let c: i64 = s[i] as i64 69 if c >= 48 { 70 if c <= 57 { r = r * 10 + (c - 48) } 71 } 72 i = i + 1 73 } 74 return r 75} 76 77// PURE: floor in MICROseconds to stream `bytes` at `mb_per_s` megabytes/sec. 78// Separated from the measurement so the gate can prove the arithmetic without touching a clock. 79func rf_floor_us(bytes: i64, mb_per_s: i64) -> i64 { 80 if mb_per_s <= 0 { return 0 - 1 } 81 return bytes / mb_per_s 82} 83 84// PURE: bytes read per decoded token = params * bytes_per_weight. batch=1 reads every weight once. 85func rf_bytes_per_token(mparams: i64, bits: i64) -> i64 { 86 return mparams * 1000000 * bits / 8 87} 88 89// MEASURE achievable sequential read bandwidth, in MB/s. 90// The buffer is FAULTED IN FIRST: timing a cold mmap measures page faults, not bandwidth -- that mistake 91// would report a bandwidth far below the truth and make every roofline look falsely close. 92func rf_measure_mb_s() -> i64 { 93 let n: i64 = RF_BUFMB * RF_MB 94 let b: *u8 = sys_mmap(n) 95 if b as i64 == 0 { return 0 - 1 } 96 var i: i64 = 0 97 while i < n { 98 b[i] = (i - (i / 251) * 251) as u8 99 i = i + RF_STRIDE 100 } 101 let t0: i64 = sys_now_us() 102 var acc: i64 = 0 103 var p: i64 = 0 104 while p < n { 105 acc = acc + (b[p] as i64) 106 p = p + RF_STRIDE 107 } 108 let dt: i64 = sys_now_us() - t0 109 if acc == 0 - 1 { rf_puts("" as *u8) } 110 if dt <= 0 { return 0 - 1 } 111 return (RF_BUFMB * 1000000) / dt 112} 113 114func rf_row(label: *u8, mparams: i64, bits: i64, mb_s: i64, measured_ms: i64) { 115 let by: i64 = rf_bytes_per_token(mparams, bits) 116 let us: i64 = rf_floor_us(by, mb_s) 117 rf_puts(" " as *u8) 118 rf_puts(label) 119 rf_puts(" bytes/token=" as *u8); rf_puti(by) 120 rf_puts(" floor_ms=" as *u8); rf_puti(us / 1000) 121 if measured_ms > 0 { 122 let floor_ms: i64 = us / 1000 123 if floor_ms > 0 { 124 rf_puts(" measured/floor=" as *u8); rf_puti(measured_ms * 100 / floor_ms) 125 rf_puts("x" as *u8) 126 } 127 } 128 rf_puts("\n" as *u8) 129} 130 131func rf_t(name: *u8, cond: i64, ctr: *i64) { 132 if cond == 1 { 133 rf_puts(" ok " as *u8) 134 ctr[0] = ctr[0] + 1 135 } else { 136 rf_puts(" FAIL " as *u8) 137 } 138 rf_puts(name) 139 rf_puts("\n" as *u8) 140 ctr[1] = ctr[1] + 1 141} 142 143func main(argc: i64, argv: *i64) -> i64 { 144 var ctr: *i64 = sys_mmap(64) as *i64 145 ctr[0] = 0 146 ctr[1] = 0 147 148 rf_puts("=== nx_roofline_gate -- decode floor = bytes/bandwidth, measured on THIS host ===\n" as *u8) 149 150 rf_t("T1 f32 0.5B reads 2.0GB per token (params x 4 bytes)" as *u8, 151 rf_bytes_per_token(500, 32) == 2000000000, ctr) 152 rf_t("T2 int4 reads 8x FEWER bytes than f32 (the whole lever)" as *u8, 153 rf_bytes_per_token(500, 32) == rf_bytes_per_token(500, 4) * 8, ctr) 154 rf_t("T3 floor scales with bytes: 2GB at 1000MB/s = 2000ms" as *u8, 155 rf_floor_us(2000000000, 1000) == 2000000, ctr) 156 rf_t("T4 FAIL-CLOSED: zero bandwidth cannot produce a floor" as *u8, 157 rf_floor_us(2000000000, 0) == 0 - 1, ctr) 158 rf_t("T5 NEG-CONTROL: int4 floor is NOT equal to f32 floor (a flat model would hide the lever)" as *u8, 159 rf_floor_us(rf_bytes_per_token(500, 4), 1000) != rf_floor_us(rf_bytes_per_token(500, 32), 1000), ctr) 160 161 let mb_s: i64 = rf_measure_mb_s() 162 rf_t("T6 LIVE: sequential read bandwidth measured and plausible (>100 MB/s)" as *u8, mb_s > 100, ctr) 163 164 rf_puts("\nMEASURED sequential read bandwidth = " as *u8); rf_puti(mb_s) 165 rf_puts(" MB/s (buffer faulted in first, so this is bandwidth, not page-fault cost)\n" as *u8) 166 167 var mp: i64 = 500 168 var measured_ms: i64 = 0 169 if argc > 2 { mp = rf_atoi(argv[2] as *u8) } 170 if argc > 3 { measured_ms = rf_atoi(argv[3] as *u8) } 171 172 rf_puts("\n-- DECODE FLOOR at " as *u8); rf_puti(mp) 173 rf_puts("M params, batch=1 (every weight read once per token) --\n" as *u8) 174 rf_row("f32 " as *u8, mp, 32, mb_s, measured_ms) 175 rf_row("int8" as *u8, mp, 8, mb_s, measured_ms) 176 rf_row("int4" as *u8, mp, 4, mb_s, measured_ms) 177 178 rf_puts("\nREAD THIS BEFORE OPTIMISING A KERNEL: if measured/floor is near 1x on the f32 row, the code is\n" as *u8) 179 rf_puts(" AT ROOFLINE and a faster dot product buys NOTHING. The levers are FEWER BYTES (int4 row above)\n" as *u8) 180 rf_puts(" or MORE WORK PER BYTE (batch m>1, which turns a bandwidth-bound decode compute-bound --\n" as *u8) 181 rf_puts(" externally: Graviton3 Llama3-8B-4bit 45.5 tok/s at batch=1 vs 184.8 at batch=8).\n" as *u8) 182 rf_puts("envelope: bandwidth is ONE sample of a strided sequential read on a shared NAS -- a FLOOR estimate,\n" as *u8) 183 rf_puts(" not a spec sheet; re-run under quiet load before quoting it. One window is not a rate.\n" as *u8) 184 185 rf_puts("\nROOFLINE-GATE " as *u8) 186 rf_puti(ctr[0]) 187 rf_puts("/" as *u8) 188 rf_puti(ctr[1]) 189 if ctr[0] == ctr[1] { 190 rf_puts(" GREEN\n" as *u8) 191 return 0 192 } 193 rf_puts(" RED\n" as *u8) 194 return 1 195}