nx_bench_v1_vs_v2_test.nx source
↩ module page · 191 lines · 7485 B
1// nx_bench_v1_vs_v2_test.nx -- first paired substrate measurement.
2//
3// Per CARDINAL [[feedback-no-strawman-perf-comparisons]]: no perf
4// ratio without paired measurement. This smoke is the smallest
5// honest paired comparison:
6//
7// ARM A: nx_actor_role_llm (manual 4-vocab embed+matmul+sample
8// pipeline, single token)
9// ARM B: nx_actor_role_llm_v2 (real Llama-class transformer
10// forward via nx_llm_generate_one_v2, single token)
11//
12// Both arms drive the SAME prompt + SAME PRNG seed through the same
13// session shape (nx_session_new_default), then capture the substrate
14// counters into NxBenchReport. The delta tells the operator:
15//
16// * How many actor steps each arm consumed
17// * How much cumulative runtime each arm reported
18// * How many messages each arm produced through the bus
19//
20// HONEST SCOPE: this is NOT a wall-clock benchmark. qemu-riscv64 is
21// single-threaded; wall-clock comparisons are meaningless without
22// physical-hardware runs (Phase E queued). What this primitive
23// proves: the bench-capture substrate produces DETERMINISTIC paired
24// counters, so when Phase E + a llama.cpp comparison rig land, the
25// substrate already speaks the right language.
26
27import "nx_syscalls.nx"
28import "nx_tier.nx"
29import "nx_tensor.nx"
30import "nx_actor.nx"
31import "nx_message.nx"
32import "nx_session.nx"
33import "nx_bench_companion.nx"
34import "nx_gguf_fixture_tiny.nx"
35import "nx_actor_role_llm.nx"
36import "nx_actor_role_llm_v2.nx"
37
38func _fill_q10(t: *NxTensor, base: i64, stride: i64) -> i64 {
39 let p: *i64 = t.storage as *i64
40 var i: nx_int = 0
41 while i < t.numel {
42 p[i] = base + (i as i64) * stride
43 i = i + 1
44 }
45 return 0
46}
47
48func main() -> i64 {
49 let now: nx_size = 1000000
50
51 // ===== Shared fixture for ARM B (v2 / real transformer) =====
52 let fix: *NxGgufFixtureBundle = nx_gft_build_tiny_llama(0xc0ffee)
53 if nx_gft_is_built(fix) != 1 { return 1 }
54
55 // ===== Synthetic weights for ARM A (v1 / manual pipeline) =====
56 let shape_buf: *u8 = sys_mmap(16)
57 let shape: *i64 = shape_buf as *i64
58 shape[0] = 4
59 shape[1] = 4
60 let err_buf: *u8 = sys_mmap(8)
61 let err_p: *i64 = err_buf as *i64
62 let embed_table: *NxTensor = nx_t_alloc(NX_DT_I64, shape, 2, err_p)
63 if err_p[0] != NX_T_OK { return 2 }
64 _fill_q10(embed_table, 100, 5)
65 let out_proj: *NxTensor = nx_t_alloc(NX_DT_I64, shape, 2, err_p)
66 _fill_q10(out_proj, 40, 3)
67
68 // ===== ARM A: v1 manual LLM actor =====
69 let sA: *NxSession = nx_session_new_default(now)
70 if nx_session_is_ready(sA) != 1 { return 3 }
71 let LLM_A: nx_int = 1001
72 let LIS_A: nx_int = 1099
73 nx_session_spawn_actor(sA, LLM_A, 1, 80, 0, now)
74 nx_session_spawn_actor(sA, LIS_A, 5, 50, 0, now)
75 nx_session_subscribe(sA, LIS_A, NX_MS_KIND_LLM_TOKEN)
76
77 let ctxA: *NxLlmActorCtx = nx_lr_actor_new(embed_table, out_proj, 1, 4, 4, 1024, 2, 42)
78 if (ctxA as i64) == 0 { return 4 }
79
80 // Capture initial state of ARM A
81 let r_a_before: *NxBenchReport = nx_bc_report_new()
82 nx_bc_capture(sA, r_a_before, 11, now)
83
84 // Drive ARM A: 5 phases (INIT->EMBED->PROJ->SAMPLE->EMIT->DONE)
85 var tA: nx_size = now + 100
86 var pa: nx_int = 0
87 while pa < 5 {
88 nx_lr_actor_step(ctxA, sA.scheduler, sA.bus, LLM_A, tA)
89 tA = tA + 50
90 pa = pa + 1
91 }
92 let tokA: nx_int = nx_lr_actor_token(ctxA)
93 if tokA < 0 { return 5 }
94 if tokA >= 4 { return 6 }
95
96 // Capture final state of ARM A
97 let r_a_after: *NxBenchReport = nx_bc_report_new()
98 nx_bc_capture(sA, r_a_after, 11, tA)
99
100 // ===== ARM B: v2 runner LLM actor on real fixture =====
101 let sB: *NxSession = nx_session_new_default(now)
102 if nx_session_is_ready(sB) != 1 { return 7 }
103 let LLM_B: nx_int = 2001
104 let LIS_B: nx_int = 2099
105 nx_session_spawn_actor(sB, LLM_B, 1, 80, 0, now)
106 nx_session_spawn_actor(sB, LIS_B, 5, 50, 0, now)
107 nx_session_subscribe(sB, LIS_B, NX_MS_KIND_LLM_TOKEN)
108
109 // Use the SAME initial prompt-token-id (1 = 'b' in fixture vocab) as ARM A
110 let prompt: *u8 = sys_mmap(1)
111 prompt[0] = 0x62
112 let ctxB: *NxLlmV2ActorCtx = nx_lv_actor_new(
113 fix.spec, fix.gguf_buf, fix.hdr, fix.bpe,
114 prompt, 1,
115 1024, 4, fix.prng_state, 10000, 724)
116 if (ctxB as i64) == 0 { return 8 }
117
118 let r_b_before: *NxBenchReport = nx_bc_report_new()
119 nx_bc_capture(sB, r_b_before, 11, now)
120
121 // Drive ARM B: 3 phases (INIT->RUN->EMIT->DONE)
122 var tB: nx_size = now + 100
123 var pb: nx_int = 0
124 while pb < 3 {
125 nx_lv_actor_step(ctxB, sB.scheduler, sB.bus, LLM_B, tB)
126 tB = tB + 50
127 pb = pb + 1
128 }
129 let tokB: nx_int = nx_lv_actor_token(ctxB)
130 if tokB < 0 { return 9 }
131 if tokB >= fix.vocab_size { return 10 }
132
133 let r_b_after: *NxBenchReport = nx_bc_report_new()
134 nx_bc_capture(sB, r_b_after, 11, tB)
135
136 // ===== Paired comparison via nx_bc_diff =====
137 let delta_a: *NxBenchReport = nx_bc_report_new()
138 let delta_b: *NxBenchReport = nx_bc_report_new()
139 nx_bc_diff(r_a_before, r_a_after, delta_a)
140 nx_bc_diff(r_b_before, r_b_after, delta_b)
141
142 // ===== Substrate-counter invariants =====
143 //
144 // ARM A drives 5 actor-step calls (one per phase). Each step
145 // accumulates 1 quantum + 1 cumulative_runtime entry of 1000us.
146 // Two actors (LLM_A + LIS_A) so n_actors_delta is 0 (both already
147 // spawned before capture). Listener receives 1 LLM_TOKEN.
148 if delta_a.total_actor_steps != 5 { return 11 }
149 // Each nx_lr_actor_step calls nx_ac_step(sched, ..., 1000, now),
150 // so cumulative runtime accrues 5 * 1000 = 5000us for ARM A.
151 if delta_a.cumulative_runtime_us != 5000 { return 12 }
152 if delta_a.actors_completed != 1 { return 13 } // LLM_A completed
153 if delta_a.actors_failed != 0 { return 14 }
154 if delta_a.n_subscriptions != 0 { return 15 } // already counted in before
155
156 // ARM B drives 3 actor-step calls (INIT, RUN, EMIT). Each step
157 // accumulates 1000us.
158 if delta_b.total_actor_steps != 3 { return 16 }
159 if delta_b.cumulative_runtime_us != 3000 { return 17 }
160 if delta_b.actors_completed != 1 { return 18 }
161 if delta_b.actors_failed != 0 { return 19 }
162
163 // ===== Honest paired observation =====
164 //
165 // ARM A (v1 manual pipeline) took 5 cooperative steps + 5000us
166 // substrate runtime to produce 1 token.
167 // ARM B (v2 real transformer) took 3 cooperative steps + 3000us
168 // substrate runtime to produce 1 token.
169 //
170 // This is NOT a throughput comparison -- both arms are bounded
171 // by cooperative-step granularity, and the underlying compute
172 // cost is hidden inside ARM B's RUN phase (which invokes the full
173 // transformer forward in one step). What it DOES prove: the
174 // paired-capture substrate produces deterministic counters that
175 // a real Phase E + physical-hardware bench rig will refine into
176 // wall-clock numbers.
177 //
178 // The substrate-honest claim: B is "fewer cooperative steps per
179 // token" than A under V1 of both adapters. That is a
180 // STRUCTURAL property of the v2 monolithic-runner adapter, NOT
181 // a throughput claim.
182
183 // ARM B should report fewer steps than ARM A (3 < 5)
184 if delta_b.total_actor_steps >= delta_a.total_actor_steps { return 20 }
185
186 // Both should have produced exactly 1 token reaching the listener
187 if nx_ms_pending(sA.bus, LIS_A) != 1 { return 21 }
188 if nx_ms_pending(sB.bus, LIS_B) != 1 { return 22 }
189
190 return 0
191}