nx_bench_dist_emit_test.nx source
↩ module page · 204 lines · 7206 B
1// nx_bench_dist_emit_test.nx -- 10-run distribution measurement.
2//
3// Runs the v2 LLM 10-iter scenario TEN INDEPENDENT TIMES, captures
4// wall-clock for each, computes min/max/mean/p50/p95/stddev across
5// the 10 samples, and emits the distribution summary to stdout.
6//
7// This is the substrate-honest version of "single-point benchmark":
8// every serious comparison reports distributions because variance
9// matters. A p50 + p95 + stddev tells the operator far more than a
10// single wall-clock number.
11//
12// Output (stdout):
13// nx_bench_dist_v2_10iter N=10 min=... max=... mean=... p50=... p95=... stddev=...
14// plus raw 10 wall-clock samples line so operators can re-analyze
15//
16// HONEST SCOPE: qemu-riscv64 emulated time. Variance under qemu is
17// dominated by interpreter scheduling artifacts; native hardware will
18// show different absolute numbers + variance shape. The SHAPE of
19// the distribution (tight p50-p95 spread vs long-tail outliers) is
20// the substrate-honest signal.
21
22import "nx_syscalls.nx"
23import "nx_tier.nx"
24import "nx_clock.nx"
25import "nx_actor.nx"
26import "nx_message.nx"
27import "nx_session.nx"
28import "nx_bench_companion.nx"
29import "nx_bench_companion_timed.nx"
30import "nx_bench_stats.nx"
31import "nx_gguf_fixture_tiny.nx"
32import "nx_actor_role_llm_v2.nx"
33
34func _emit_dec_i64(fd: i64, n: i64) -> i64 {
35 let scratch: *u8 = sys_mmap(32)
36 var v: i64 = n
37 var neg: nx_int = 0
38 if v < 0 { neg = 1; v = 0 - v }
39 var k: i64 = 0
40 if v == 0 { scratch[0] = 0x30 as u8; k = 1 }
41 while v > 0 {
42 scratch[k] = (0x30 + (v - (v / 10) * 10)) as u8
43 v = v / 10
44 k = k + 1
45 }
46 let rev: *u8 = sys_mmap(48)
47 var ro: i64 = 0
48 if neg == 1 { rev[0] = 0x2D as u8; ro = 1 }
49 var j: i64 = 0
50 while j < k { rev[ro + j] = scratch[k - 1 - j]; j = j + 1 }
51 sys_write(fd, rev, ro + k)
52 return 0
53}
54
55func _emit_str(fd: i64, s: *u8, n: i64) -> i64 {
56 sys_write(fd, s, n)
57 return 0
58}
59
60func _emit_nl(fd: i64) -> i64 {
61 let nl: *u8 = sys_mmap(1)
62 nl[0] = 0x0A as u8
63 sys_write(fd, nl, 1)
64 return 0
65}
66
67func _emit_tab(fd: i64) -> i64 {
68 let t: *u8 = sys_mmap(1)
69 t[0] = 0x09 as u8
70 sys_write(fd, t, 1)
71 return 0
72}
73
74// ===== One scenario run = one wall-clock sample =================
75
76func _drive_one_run(prng_seed: i64, n_iters: nx_int, now: nx_size) -> i64 {
77 let fix: *NxGgufFixtureBundle = nx_gft_build_tiny_llama(prng_seed)
78 if nx_gft_is_built(fix) != 1 { return 0 - 1 }
79 let s: *NxSession = nx_session_new_default(now)
80 if nx_session_is_ready(s) != 1 { return 0 - 1 }
81 let LLM: nx_int = 7001
82 nx_session_spawn_actor(s, LLM, 1, 80, 0, now)
83 let prompt: *u8 = sys_mmap(1); prompt[0] = 0x61
84 let ctx: *NxLlmV2ActorCtx = nx_lv_actor_new(
85 fix.spec, fix.gguf_buf, fix.hdr, fix.bpe, prompt, 1,
86 1024, 4, fix.prng_state, 10000, 724)
87 if (ctx as i64) == 0 { return 0 - 1 }
88
89 let start: i64 = nx_clock_monotonic_ns()
90 var iter: nx_int = 0
91 var tick: nx_size = now + 100
92 while iter < n_iters {
93 nx_lv_actor_step(ctx, s.scheduler, s.bus, LLM, tick); tick = tick + 50
94 nx_lv_actor_step(ctx, s.scheduler, s.bus, LLM, tick); tick = tick + 50
95 if ctx.runner_result < 0 { return 0 - 1 }
96 nx_lv_actor_step(ctx, s.scheduler, s.bus, LLM, tick); tick = tick + 50
97 if iter < (n_iters - 1) {
98 prompt[0] = nx_gft_vocab_byte(fix, ctx.runner_result) as u8
99 nx_lv_actor_reset_for_next_token(ctx, s.scheduler, LLM, prompt, 1)
100 }
101 iter = iter + 1
102 }
103 let end: i64 = nx_clock_monotonic_ns()
104 return (end - start) / 1000 // microseconds
105}
106
107func main() -> i64 {
108 let now: nx_size = 1000000
109 let N_RUNS: nx_int = 10
110 let N_ITERS_PER_RUN: nx_int = 10
111
112 // ===== Collect N_RUNS wall-clock samples =====
113 let samples_buf: *u8 = sys_mmap(N_RUNS * 8)
114 let samples: *i64 = samples_buf as *i64
115 var run: nx_int = 0
116 while run < N_RUNS {
117 let seed: i64 = 0xB000 + (run as i64)
118 let elapsed_us: i64 = _drive_one_run(seed, N_ITERS_PER_RUN, now)
119 if elapsed_us < 0 { return 10 + run }
120 samples[run] = elapsed_us
121 run = run + 1
122 }
123
124 // ===== Compute stats =====
125 let stats: *NxBenchStats = nx_bst_new()
126 if nx_bench_stats_compute(samples, N_RUNS, stats) != NX_BST_OK { return 20 }
127
128 // ===== Emit raw samples line =====
129 let raw_lbl: *u8 = sys_mmap(12)
130 raw_lbl[0]=0x72 as u8; raw_lbl[1]=0x61 as u8; raw_lbl[2]=0x77 as u8
131 raw_lbl[3]=0x5F as u8; raw_lbl[4]=0x73 as u8; raw_lbl[5]=0x61 as u8
132 raw_lbl[6]=0x6D as u8; raw_lbl[7]=0x70 as u8; raw_lbl[8]=0x6C as u8
133 raw_lbl[9]=0x65 as u8; raw_lbl[10]=0x73 as u8; raw_lbl[11]=0x09 as u8
134 _emit_str(1, raw_lbl, 12)
135 var ri: nx_int = 0
136 while ri < N_RUNS {
137 _emit_dec_i64(1, samples[ri])
138 if ri < (N_RUNS - 1) { _emit_tab(1) }
139 ri = ri + 1
140 }
141 _emit_nl(1)
142
143 // ===== Emit stats summary lines =====
144 let n_lbl: *u8 = sys_mmap(3)
145 n_lbl[0]=0x6E as u8; n_lbl[1]=0x3D as u8; n_lbl[2]=0x09 as u8 // "n=\t"
146 _emit_str(1, n_lbl, 3)
147 _emit_dec_i64(1, nx_bst_n(stats) as i64)
148 _emit_nl(1)
149
150 let min_lbl: *u8 = sys_mmap(5)
151 min_lbl[0]=0x6D as u8; min_lbl[1]=0x69 as u8; min_lbl[2]=0x6E as u8
152 min_lbl[3]=0x3D as u8; min_lbl[4]=0x09 as u8
153 _emit_str(1, min_lbl, 5)
154 _emit_dec_i64(1, nx_bst_min(stats))
155 _emit_nl(1)
156
157 let max_lbl: *u8 = sys_mmap(5)
158 max_lbl[0]=0x6D as u8; max_lbl[1]=0x61 as u8; max_lbl[2]=0x78 as u8
159 max_lbl[3]=0x3D as u8; max_lbl[4]=0x09 as u8
160 _emit_str(1, max_lbl, 5)
161 _emit_dec_i64(1, nx_bst_max(stats))
162 _emit_nl(1)
163
164 let mean_lbl: *u8 = sys_mmap(6)
165 mean_lbl[0]=0x6D as u8; mean_lbl[1]=0x65 as u8; mean_lbl[2]=0x61 as u8
166 mean_lbl[3]=0x6E as u8; mean_lbl[4]=0x3D as u8; mean_lbl[5]=0x09 as u8
167 _emit_str(1, mean_lbl, 6)
168 _emit_dec_i64(1, nx_bst_mean(stats))
169 _emit_nl(1)
170
171 let p50_lbl: *u8 = sys_mmap(5)
172 p50_lbl[0]=0x70 as u8; p50_lbl[1]=0x35 as u8; p50_lbl[2]=0x30 as u8
173 p50_lbl[3]=0x3D as u8; p50_lbl[4]=0x09 as u8
174 _emit_str(1, p50_lbl, 5)
175 _emit_dec_i64(1, nx_bst_p50(stats))
176 _emit_nl(1)
177
178 let p95_lbl: *u8 = sys_mmap(5)
179 p95_lbl[0]=0x70 as u8; p95_lbl[1]=0x39 as u8; p95_lbl[2]=0x35 as u8
180 p95_lbl[3]=0x3D as u8; p95_lbl[4]=0x09 as u8
181 _emit_str(1, p95_lbl, 5)
182 _emit_dec_i64(1, nx_bst_p95(stats))
183 _emit_nl(1)
184
185 let sd_lbl: *u8 = sys_mmap(8)
186 sd_lbl[0]=0x73 as u8; sd_lbl[1]=0x74 as u8; sd_lbl[2]=0x64 as u8
187 sd_lbl[3]=0x64 as u8; sd_lbl[4]=0x65 as u8; sd_lbl[5]=0x76 as u8
188 sd_lbl[6]=0x3D as u8; sd_lbl[7]=0x09 as u8
189 _emit_str(1, sd_lbl, 8)
190 _emit_dec_i64(1, nx_bst_stddev(stats))
191 _emit_nl(1)
192
193 // ===== Substrate invariants =====
194 if nx_bst_n(stats) != N_RUNS { return 100 }
195 if nx_bst_min(stats) <= 0 { return 101 }
196 if nx_bst_max(stats) < nx_bst_min(stats) { return 102 }
197 if nx_bst_p50(stats) < nx_bst_min(stats) { return 103 }
198 if nx_bst_p50(stats) > nx_bst_max(stats) { return 104 }
199 if nx_bst_p95(stats) < nx_bst_p50(stats) { return 105 }
200 if nx_bst_p95(stats) > nx_bst_max(stats) { return 106 }
201 if nx_bst_stddev(stats) < 0 { return 107 }
202
203 return 0
204}