nx_bench_scale_dist_emit_test.nx source
↩ module page · 197 lines · 7880 B
1// nx_bench_scale_dist_emit_test.nx -- distribution measurement at 3 scales.
2//
3// Composes [[nx_bench_stats]] over multiple scale points to produce a
4// real variance-vs-load curve. 30 total independent runs:
5// * 10 runs of v2 LLM 10-iter -> distribution A
6// * 10 runs of v2 LLM 30-iter -> distribution B
7// * 10 runs of v2 LLM 100-iter -> distribution C
8//
9// Emits all three distributions side-by-side to stdout so the
10// operator sees how min/max/p50/p95/stddev scale with workload.
11//
12// Substrate-honest claim this measurement supports:
13// * If p50 scales linearly with iter count and stddev scales
14// roughly linearly too -> substrate is well-behaved.
15// * If stddev grows faster than mean -> there's a load-dependent
16// variance bug.
17// * If p95/p50 ratio stays roughly constant across scales ->
18// no tail-degradation under load.
19
20import "nx_syscalls.nx"
21import "nx_tier.nx"
22import "nx_clock.nx"
23import "nx_actor.nx"
24import "nx_message.nx"
25import "nx_session.nx"
26import "nx_bench_companion.nx"
27import "nx_bench_companion_timed.nx"
28import "nx_bench_stats.nx"
29import "nx_gguf_fixture_tiny.nx"
30import "nx_actor_role_llm_v2.nx"
31
32func _emit_dec_i64(fd: i64, n: i64) -> i64 {
33 let scratch: *u8 = sys_mmap(32)
34 var v: i64 = n
35 var neg: nx_int = 0
36 if v < 0 { neg = 1; v = 0 - v }
37 var k: i64 = 0
38 if v == 0 { scratch[0] = 0x30 as u8; k = 1 }
39 while v > 0 {
40 scratch[k] = (0x30 + (v - (v / 10) * 10)) as u8
41 v = v / 10
42 k = k + 1
43 }
44 let rev: *u8 = sys_mmap(48)
45 var ro: i64 = 0
46 if neg == 1 { rev[0] = 0x2D as u8; ro = 1 }
47 var j: i64 = 0
48 while j < k { rev[ro + j] = scratch[k - 1 - j]; j = j + 1 }
49 sys_write(fd, rev, ro + k)
50 return 0
51}
52
53func _emit_str(fd: i64, s: *u8, n: i64) -> i64 { sys_write(fd, s, n); return 0 }
54func _emit_tab(fd: i64) -> i64 { let t: *u8 = sys_mmap(1); t[0] = 0x09 as u8; sys_write(fd, t, 1); return 0 }
55func _emit_nl(fd: i64) -> i64 { let n: *u8 = sys_mmap(1); n[0] = 0x0A as u8; sys_write(fd, n, 1); return 0 }
56
57// ===== One run = one wall-clock sample =====
58func _drive_one_run(prng_seed: i64, n_iters: nx_int, now: nx_size) -> i64 {
59 let fix: *NxGgufFixtureBundle = nx_gft_build_tiny_llama(prng_seed)
60 if nx_gft_is_built(fix) != 1 { return 0 - 1 }
61 let s: *NxSession = nx_session_new_default(now)
62 if nx_session_is_ready(s) != 1 { return 0 - 1 }
63 let LLM: nx_int = 7001
64 nx_session_spawn_actor(s, LLM, 1, 80, 0, now)
65 let prompt: *u8 = sys_mmap(1); prompt[0] = 0x61
66 let ctx: *NxLlmV2ActorCtx = nx_lv_actor_new(
67 fix.spec, fix.gguf_buf, fix.hdr, fix.bpe, prompt, 1,
68 1024, 4, fix.prng_state, 10000, 724)
69 if (ctx as i64) == 0 { return 0 - 1 }
70 let start: i64 = nx_clock_monotonic_ns()
71 var iter: nx_int = 0
72 var tick: nx_size = now + 100
73 while iter < n_iters {
74 nx_lv_actor_step(ctx, s.scheduler, s.bus, LLM, tick); tick = tick + 50
75 nx_lv_actor_step(ctx, s.scheduler, s.bus, LLM, tick); tick = tick + 50
76 if ctx.runner_result < 0 { return 0 - 1 }
77 nx_lv_actor_step(ctx, s.scheduler, s.bus, LLM, tick); tick = tick + 50
78 if iter < (n_iters - 1) {
79 prompt[0] = nx_gft_vocab_byte(fix, ctx.runner_result) as u8
80 nx_lv_actor_reset_for_next_token(ctx, s.scheduler, LLM, prompt, 1)
81 }
82 iter = iter + 1
83 }
84 let end: i64 = nx_clock_monotonic_ns()
85 return (end - start) / 1000
86}
87
88// ===== Collect N samples at given scale =====
89func _collect_dist(samples: *i64, n_runs: nx_int, n_iters: nx_int,
90 seed_base: i64, now: nx_size) -> nx_int {
91 var i: nx_int = 0
92 while i < n_runs {
93 let elapsed: i64 = _drive_one_run(seed_base + (i as i64), n_iters, now)
94 if elapsed < 0 { return 0 - 1 }
95 samples[i] = elapsed
96 i = i + 1
97 }
98 return n_runs
99}
100
101// ===== Emit distribution header + summary row =====
102func _emit_dist_row(label: *u8, label_len: nx_int,
103 samples: *i64, stats: *NxBenchStats,
104 n_runs: nx_int) -> i64 {
105 _emit_str(1, label, label_len as i64)
106 _emit_tab(1)
107 _emit_dec_i64(1, n_runs as i64)
108 _emit_tab(1)
109 _emit_dec_i64(1, nx_bst_min(stats))
110 _emit_tab(1)
111 _emit_dec_i64(1, nx_bst_max(stats))
112 _emit_tab(1)
113 _emit_dec_i64(1, nx_bst_mean(stats))
114 _emit_tab(1)
115 _emit_dec_i64(1, nx_bst_p50(stats))
116 _emit_tab(1)
117 _emit_dec_i64(1, nx_bst_p95(stats))
118 _emit_tab(1)
119 _emit_dec_i64(1, nx_bst_stddev(stats))
120 _emit_nl(1)
121 return 0
122}
123
124func main() -> i64 {
125 let now: nx_size = 1000000
126 let N_RUNS: nx_int = 10
127
128 // ===== Distribution at 10 iter =====
129 let s10_buf: *u8 = sys_mmap(N_RUNS * 8)
130 let s10: *i64 = s10_buf as *i64
131 if _collect_dist(s10, N_RUNS, 10, 0xC100, now) < 0 { return 1 }
132 let st10: *NxBenchStats = nx_bst_new()
133 nx_bench_stats_compute(s10, N_RUNS, st10)
134
135 // ===== Distribution at 30 iter =====
136 let s30_buf: *u8 = sys_mmap(N_RUNS * 8)
137 let s30: *i64 = s30_buf as *i64
138 if _collect_dist(s30, N_RUNS, 30, 0xC300, now) < 0 { return 2 }
139 let st30: *NxBenchStats = nx_bst_new()
140 nx_bench_stats_compute(s30, N_RUNS, st30)
141
142 // ===== Distribution at 100 iter =====
143 let s100_buf: *u8 = sys_mmap(N_RUNS * 8)
144 let s100: *i64 = s100_buf as *i64
145 if _collect_dist(s100, N_RUNS, 100, 0xCA00, now) < 0 { return 3 }
146 let st100: *NxBenchStats = nx_bst_new()
147 nx_bench_stats_compute(s100, N_RUNS, st100)
148
149 // ===== Emit TSV-style header: label N min max mean p50 p95 stddev =====
150 let hdr: *u8 = sys_mmap(64)
151 hdr[0]=0x6C as u8; hdr[1]=0x61 as u8; hdr[2]=0x62 as u8; hdr[3]=0x65 as u8
152 hdr[4]=0x6C as u8; hdr[5]=0x09 as u8; hdr[6]=0x4E as u8; hdr[7]=0x09 as u8
153 hdr[8]=0x6D as u8; hdr[9]=0x69 as u8; hdr[10]=0x6E as u8; hdr[11]=0x09 as u8
154 hdr[12]=0x6D as u8; hdr[13]=0x61 as u8; hdr[14]=0x78 as u8; hdr[15]=0x09 as u8
155 hdr[16]=0x6D as u8; hdr[17]=0x65 as u8; hdr[18]=0x61 as u8; hdr[19]=0x6E as u8
156 hdr[20]=0x09 as u8; hdr[21]=0x70 as u8; hdr[22]=0x35 as u8; hdr[23]=0x30 as u8
157 hdr[24]=0x09 as u8; hdr[25]=0x70 as u8; hdr[26]=0x39 as u8; hdr[27]=0x35 as u8
158 hdr[28]=0x09 as u8; hdr[29]=0x73 as u8; hdr[30]=0x74 as u8; hdr[31]=0x64 as u8
159 hdr[32]=0x64 as u8; hdr[33]=0x65 as u8; hdr[34]=0x76 as u8; hdr[35]=0x0A as u8
160 sys_write(1, hdr, 36)
161
162 // ===== Labels =====
163 let l10: *u8 = sys_mmap(16)
164 l10[0]=0x76 as u8; l10[1]=0x32 as u8; l10[2]=0x5F as u8; l10[3]=0x31 as u8
165 l10[4]=0x30 as u8; l10[5]=0x69 as u8; l10[6]=0x74 as u8; l10[7]=0x65 as u8
166 l10[8]=0x72 as u8
167 _emit_dist_row(l10, 9, s10, st10, N_RUNS)
168
169 let l30: *u8 = sys_mmap(16)
170 l30[0]=0x76 as u8; l30[1]=0x32 as u8; l30[2]=0x5F as u8; l30[3]=0x33 as u8
171 l30[4]=0x30 as u8; l30[5]=0x69 as u8; l30[6]=0x74 as u8; l30[7]=0x65 as u8
172 l30[8]=0x72 as u8
173 _emit_dist_row(l30, 9, s30, st30, N_RUNS)
174
175 let l100: *u8 = sys_mmap(16)
176 l100[0]=0x76 as u8; l100[1]=0x32 as u8; l100[2]=0x5F as u8; l100[3]=0x31 as u8
177 l100[4]=0x30 as u8; l100[5]=0x30 as u8; l100[6]=0x69 as u8; l100[7]=0x74 as u8
178 l100[8]=0x65 as u8; l100[9]=0x72 as u8
179 _emit_dist_row(l100, 10, s100, st100, N_RUNS)
180
181 // ===== Substrate invariants =====
182 if nx_bst_n(st10) != N_RUNS { return 100 }
183 if nx_bst_n(st30) != N_RUNS { return 101 }
184 if nx_bst_n(st100) != N_RUNS { return 102 }
185 // p50 must increase with scale
186 if nx_bst_p50(st30) <= nx_bst_p50(st10) { return 103 }
187 if nx_bst_p50(st100) <= nx_bst_p50(st30) { return 104 }
188 // p95 must be >= p50 at every scale
189 if nx_bst_p95(st10) < nx_bst_p50(st10) { return 105 }
190 if nx_bst_p95(st30) < nx_bst_p50(st30) { return 106 }
191 if nx_bst_p95(st100) < nx_bst_p50(st100) { return 107 }
192 // mean and min must be positive
193 if nx_bst_min(st10) <= 0 { return 108 }
194 if nx_bst_mean(st100) <= 0 { return 109 }
195
196 return 0
197}