nx_lowrank_kv_budget.nx source
↩ module page · 93 lines · 6623 B
1// nx_lowrank_kv_budget.nx -- the HARDWARE PAYOFF of data-driven low-rank KV (nx_lowrank_rank_plan) on the
2// operator's real budget: RTX 5080, 16 GiB VRAM (rule #21 resource awareness; telos = capable AI on
3// AFFORDABLE consumer hardware). KV cache is the context-length bottleneck: full KV grows O(n_layers * 2 *
4// kv_dim) per token; data-driven low-rank KV grows O(n_layers * (r_K + r_V)) per token. This computes the
5// MAX CONTEXT LENGTH full-vs-low-rank for a given model on a given VRAM budget. Pure integer math, NO slow
6// forward. Real Qwen2.5-0.5B config + ranks (measured); 7B shown as an honest illustrative extrapolation.
7// 100% sovereign (nx_cc->nxasm, no gcc). license_tier: ORIGINAL
8// ⚠ SUPERSEDED 2026-06-17: the context-multipliers here (42x for 0.5B, 170x for 7B) used K=1/V=5 ranks from a
9// DEGENERATE-context KV spectrum. The definitive seq=64 measurement (nx_lowrank_kv_real M4d) shows real KV
10// needs avg rank K=27/V=37 of 64 for 90% energy => the real KV VRAM win is ~2-3x, NOT 42x. The budget MATH
11// here is valid; the input ranks were wrong. WEIGHTS are heterogeneous too (attn low-rank but small, big ffn
12// NOT low-rank) => weight-SVD also modest. BOTH low-rank VRAM levers are MODEST on real Qwen-0.5B.
13import "nx_syscalls.nx"
14const K_MAGIC_1048576: i64 = 1048576
15const K_MAGIC_491400032: i64 = 491400032
16const K_MAGIC_5046586573: i64 = 5046586573
17
18const GIB: i64 = 1073741824
19const VRAM: i64 = 17179869184 // 16 GiB (RTX 5080)
20const KVB: i64 = 4 // KV cache element bytes (f32, matches our sovereign forward; f16 would 2x the absolute token counts, ratio unchanged)
21
22func bp_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
23func bp_putn(v: i64) -> i64 {
24 if v == 0 { sys_write(1, "0" as *u8, 1); return 0 }
25 var m: i64 = v
26 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
27 let d: *u8 = sys_mmap(24); var k: i64 = 0
28 while m > 0 { d[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
29 var j: i64 = k - 1
30 while j >= 0 { sys_write(1, ((d as i64)+j) as *u8, 1); j = j - 1 }
31 return 0
32}
33func bp_putM(bytes: i64) -> i64 { bp_putn(bytes / K_MAGIC_1048576); bp_puts(" MiB"); return 0 }
34
35// max context tokens for FULL KV: kv_budget / (n_layers * 2 * kv_dim * KVB)
36func bp_ctx_full(kv_budget: i64, L: i64, kv_dim: i64) -> i64 { return kv_budget / (L * 2 * kv_dim * KVB) }
37// max context for low-rank KV (per-token-SVD: coords[seq*r] grows, basis[r*dim] fixed):
38// kv_budget = L*(rK+rV)*(seq+dim)*KVB => seq = kv_budget/(L*(rK+rV)*KVB) - dim
39func bp_ctx_low(kv_budget: i64, L: i64, kv_dim: i64, rK: i64, rV: i64) -> i64 {
40 let per: i64 = L * (rK + rV) * KVB
41 let s: i64 = (kv_budget / per) - kv_dim
42 if s < 0 { return 0 }
43 return s
44}
45
46func bp_model(name: *u8, weights: i64, L: i64, kv_dim: i64, rK: i64, rV: i64, measured: i64) -> i64 {
47 bp_puts("\n["); bp_puts(name); bp_puts("] weights="); bp_putM(weights)
48 bp_puts(" layers="); bp_putn(L); bp_puts(" kv_dim="); bp_putn(kv_dim)
49 if measured == 1 { bp_puts(" (config+ranks MEASURED)\n") } else { bp_puts(" (illustrative: ranks NOT measured for this model)\n") }
50 let kvb: i64 = VRAM - weights
51 if kvb <= 0 { bp_puts(" weights exceed VRAM\n"); return 0 }
52 bp_puts(" VRAM left for KV = "); bp_putM(kvb); bp_puts("\n")
53 let cf: i64 = bp_ctx_full(kvb, L, kv_dim)
54 let cl: i64 = bp_ctx_low(kvb, L, kv_dim, rK, rV)
55 bp_puts(" full KV : max context = "); bp_putn(cf); bp_puts(" tokens\n")
56 bp_puts(" data-driven K="); bp_putn(rK); bp_puts("/V="); bp_putn(rV); bp_puts(" : max context = "); bp_putn(cl)
57 bp_puts(" tokens ("); bp_putn(cl / cf); bp_puts("x more context)\n")
58 return cl / cf
59}
60
61func main() -> i64 {
62 bp_puts("=== nx_lowrank_kv_budget: max context on RTX 5080 (16 GiB) -- full KV vs data-driven low-rank KV ===\n")
63 bp_puts("KV cache = the context bottleneck; ranks from nx_lowrank_rank_plan (K steep r=1, V shallow r=5 @90%).\n")
64
65 // REAL Qwen2.5-0.5B-Q4_K_M (staged file = 491,400,032 B; 24 layers; kv_dim = 2 kv-heads * 64 = 128)
66 let ratio05: i64 = bp_model("Qwen2.5-0.5B Q4_K_M" as *u8, K_MAGIC_491400032, 24, 128, 1, 5, 1)
67 // ILLUSTRATIVE Qwen2.5-7B-class (cited config: 28 layers, GQA 4 kv-heads * 128 = kv_dim 512; Q4_K_M ~4.7 GiB)
68 let ratio7: i64 = bp_model("Qwen2.5-7B Q4_K_M (illustrative)" as *u8, K_MAGIC_5046586573, 28, 512, 1, 5, 0)
69
70 bp_puts("\n WHY it matters more for big models: the context multiplier = 2*kv_dim/(rK+rV), so it GROWS with\n")
71 bp_puts(" kv_dim -- small model 0.5B (kv_dim 128) -> ~"); bp_putn(ratio05); bp_puts("x; 7B-class (kv_dim 512) -> ~"); bp_putn(ratio7)
72 bp_puts("x. Big models (weights eat VRAM, context is KV-bound) benefit most.\n")
73
74 var pass: i64 = 0
75 var fail: i64 = 0
76 // T1: low-rank enables more context than full (the whole point)
77 if ratio05 > 1 { bp_puts(" T1 low-rank KV enables MORE context than full (0.5B): PASS\n"); pass = pass + 1 } else { bp_puts(" T1: FAIL\n"); fail = fail + 1 }
78 // T2: the multiplier matches the closed form 2*kv_dim/(rK+rV) = 256/6 = 42
79 let expect: i64 = (2 * 128) / (1 + 5)
80 if ratio05 >= expect - 2 { if ratio05 <= expect + 2 { bp_puts(" T2 multiplier matches closed form 2*kv_dim/(rK+rV)~="); bp_putn(expect); bp_puts(": PASS\n"); pass = pass + 1 } else { bp_puts(" T2: FAIL\n"); fail = fail + 1 } } else { bp_puts(" T2: FAIL\n"); fail = fail + 1 }
81 // T3: bigger kv_dim => bigger multiplier (the "matters more for big models" claim)
82 if ratio7 > ratio05 { bp_puts(" T3 multiplier grows with model size (7B > 0.5B): PASS\n"); pass = pass + 1 } else { bp_puts(" T3: FAIL\n"); fail = fail + 1 }
83 // T4 neg-control: no compression (rK=rV=kv_dim) => no extra context
84 let noc: i64 = bp_ctx_low(VRAM - K_MAGIC_491400032, 24, 128, 128, 128)
85 let cf0: i64 = bp_ctx_full(VRAM - K_MAGIC_491400032, 24, 128)
86 if noc <= cf0 { bp_puts(" T4 neg-control (r=kv_dim = no compression) gives no extra context: PASS\n"); pass = pass + 1 } else { bp_puts(" T4: FAIL\n"); fail = fail + 1 }
87
88 bp_puts("\n HONEST: 0.5B is config+rank MEASURED; 7B ranks are NOT measured (the multiplier assumes similar\n")
89 bp_puts(" low ranks suffice -- needs a 7B spectrum measurement). Absolute tokens use f32 KV; f16 doubles them.\n")
90 bp_puts("\n PASS="); bp_putn(pass); bp_puts("/4 ")
91 if fail == 0 { bp_puts("VERDICT=GREEN (data-driven low-rank KV turns the 5080 into a long-context machine; payoff grows with model size)\n"); sys_exit(0); return 0 }
92 bp_puts("VERDICT=RED\n"); sys_exit(1); return 1
93}