nx_benchmark_harness.nx source
↩ module page · 348 lines · 13259 B
1// nx_benchmark_harness.nx -- head-to-head perf verdict primitive.
2//
3// Substrate-level enforcer of the cardinal feedback-honest-perf-
4// verdict-no-aspirational-claims. Every NishiLang vs incumbent
5// benchmark emits a SEALED-ENUM verdict per axis: WIN / LOSE / TIE /
6// UNMEASURABLE. No "competitive with" weasel words. Every LOSE row
7// MUST be paired with a named-improvement string the caller fills in.
8//
9// IDEA-PROVENANCE (per the language-design-research discipline:
10// learn from, never copy):
11// - JMH (OpenJDK) -- multi-warmup, multi-fork, GC-quiet
12// - Criterion.rs -- bootstrap CI, outlier detection (MAD)
13// - Google Benchmark -- statistical-rigor patterns
14// - Hyperfine -- multi-trial CLI command timing
15// Every idea was re-derived from the published papers + project READMEs;
16// no source code was incorporated. See genealogy_id for paper citations.
17//
18// Six measurable axes per benchmark run:
19// 1. THROUGHPUT ops/sec or items/sec
20// 2. LATENCY_P50 median single-op time
21// 3. LATENCY_P99 99th-percentile single-op time (tail latency)
22// 4. MEMORY_PEAK peak bytes during run
23// 5. DETERMINISM same-input-same-output across N trials (Q10)
24// 6. PORTABILITY runs without modification across target archs (Q10)
25//
26// Each axis emits a per-axis sealed-enum verdict:
27// NX_BENCH_AXIS_WIN challenger >= incumbent * 1.01 (1% delta floor)
28// NX_BENCH_AXIS_TIE within 1% either direction
29// NX_BENCH_AXIS_LOSE challenger < incumbent * 0.99
30// NX_BENCH_AXIS_UNMEASURABLE measurement not available this run
31//
32// Composite verdict:
33// NX_BENCH_DECISIVE_WIN >= 4 of 6 axes WIN, zero LOSE
34// NX_BENCH_WIN majority WIN, fewer LOSE
35// NX_BENCH_TIE majority TIE or balanced WIN/LOSE
36// NX_BENCH_LOSE majority LOSE
37// NX_BENCH_DECISIVE_LOSE >= 4 of 6 axes LOSE
38// NX_BENCH_INCONCLUSIVE too many UNMEASURABLE to call
39//
40// Why 1% floor: per cardinal feedback-one-percent-minimum-delta-substrate
41// even on trivial axes substrate must score at least 1% better than
42// incumbent for WIN. Pure parity is LOSE_BY_PARITY, not TIE_WITH_HONOR.
43//
44// genealogy_id: jmh_openjdk_papers + criterion_rs_paper_2018 +
45// google_benchmark_perf_2014 + hyperfine_2020 +
46// park_miller_1988_minimal_standard
47// lineage_id: head_to_head_perf_verdict_q10
48
49// nx_safety_envelope:
50// intended_use: AUTO_APPLIED -- primitive-specific tuning queued
51// sil_target: SIL1
52// evidence: [bulk_applied_2026-05-16, see-file-comment-for-detail]
53// verdict: NOT_YET_EVALUATED
54
55import "nx_syscalls.nx"
56import "nx_tier.nx"
57
58const NX_BENCH_Q: nx_int = 1024
59
60// Per-axis verdicts
61const NX_BENCH_AXIS_UNMEASURABLE: nx_int = 0
62const NX_BENCH_AXIS_LOSE: nx_int = 1
63const NX_BENCH_AXIS_TIE: nx_int = 2
64const NX_BENCH_AXIS_WIN: nx_int = 3
65const NX_BENCH_N_AXIS_VERDICTS: nx_int = 4
66
67// Axis indices
68const NX_BENCH_AXIS_THROUGHPUT: nx_int = 0
69const NX_BENCH_AXIS_LATENCY_P50: nx_int = 1
70const NX_BENCH_AXIS_LATENCY_P99: nx_int = 2
71const NX_BENCH_AXIS_MEMORY_PEAK: nx_int = 3
72const NX_BENCH_AXIS_DETERMINISM: nx_int = 4
73const NX_BENCH_AXIS_PORTABILITY: nx_int = 5
74const NX_BENCH_N_AXES: nx_int = 6
75
76// Composite verdicts
77const NX_BENCH_INCONCLUSIVE: nx_int = 0
78const NX_BENCH_DECISIVE_LOSE: nx_int = 1
79const NX_BENCH_LOSE: nx_int = 2
80const NX_BENCH_TIE: nx_int = 3
81const NX_BENCH_WIN: nx_int = 4
82const NX_BENCH_DECISIVE_WIN: nx_int = 5
83const NX_BENCH_N_COMPOSITE_VERDICTS: nx_int = 6
84
85// 1% delta floor in Q10 = 1024 / 100 ~ 10
86const NX_BENCH_DELTA_FLOOR_Q10: nx_int = 10
87
88// Trial-count threshold to declare an axis measured (need >= this many
89// successful samples to consider it valid).
90const NX_BENCH_MIN_TRIALS: nx_int = 8
91
92struct AxisMeasurement {
93 challenger_value: nx_int, // raw measurement (units depend on axis)
94 incumbent_value: nx_int,
95 n_trials: nx_int, // 0 = NOT_RUN, < MIN_TRIALS = UNMEASURABLE
96 higher_is_better: nx_int, // 1 = higher wins; 0 = lower wins (latency)
97}
98
99struct AxisVerdict {
100 measurement: AxisMeasurement,
101 verdict: nx_int, // NX_BENCH_AXIS_*
102 delta_q10: nx_int, // signed Q10 of (challenger - incumbent) / incumbent
103}
104
105struct BenchmarkReport {
106 throughput: AxisVerdict,
107 latency_p50: AxisVerdict,
108 latency_p99: AxisVerdict,
109 memory_peak: AxisVerdict,
110 determinism: AxisVerdict,
111 portability: AxisVerdict,
112
113 n_wins: nx_int,
114 n_ties: nx_int,
115 n_losses: nx_int,
116 n_unmeasured: nx_int,
117 composite: nx_int, // NX_BENCH_*
118
119 // For LOSE / DECISIVE_LOSE composite verdicts the caller MUST
120 // populate named_improvement -- the specific concrete substrate
121 // change that would close the gap. Cardinal:
122 // feedback-honest-perf-verdict every LOSE row names an improvement.
123 named_improvement_present: nx_int, // 1 = caller set it, 0 = forgot
124 worst_axis: nx_int, // index of the lowest-scoring axis
125}
126
127// ===== Per-axis verdict computation ==================================
128
129func _bench_axis_verdict(m: *AxisMeasurement, v: *AxisVerdict) -> nx_int {
130 v.measurement = m[0]
131 if m.n_trials < NX_BENCH_MIN_TRIALS {
132 v.verdict = NX_BENCH_AXIS_UNMEASURABLE
133 v.delta_q10 = 0
134 return 0
135 }
136 if m.incumbent_value == 0 {
137 // Avoid divide-by-zero; if challenger > 0 and incumbent = 0
138 // that's a decisive WIN, otherwise UNMEASURABLE.
139 if m.challenger_value > 0 {
140 v.verdict = NX_BENCH_AXIS_WIN
141 v.delta_q10 = NX_BENCH_Q
142 } else {
143 v.verdict = NX_BENCH_AXIS_UNMEASURABLE
144 v.delta_q10 = 0
145 }
146 return 0
147 }
148 // Signed delta_q10 = (challenger - incumbent) * Q / incumbent
149 var diff: nx_int = m.challenger_value - m.incumbent_value
150 var inc: nx_int = m.incumbent_value
151 if inc < 0 { inc = -inc }
152 v.delta_q10 = (diff * NX_BENCH_Q) / inc
153
154 // For "lower is better" axes (latency, memory), flip the sign so
155 // the comparison logic below stays canonical.
156 var signed_delta: nx_int = v.delta_q10
157 if m.higher_is_better == 0 { signed_delta = -signed_delta }
158
159 // Verdict band per cardinal one-percent-minimum-delta:
160 // WIN signed_delta >= +1% (Q10 >= +10)
161 // TIE -1% < signed_delta < +1%
162 // LOSE signed_delta <= -1%
163 if signed_delta >= NX_BENCH_DELTA_FLOOR_Q10 {
164 v.verdict = NX_BENCH_AXIS_WIN
165 } else {
166 if signed_delta <= -NX_BENCH_DELTA_FLOOR_Q10 {
167 v.verdict = NX_BENCH_AXIS_LOSE
168 } else {
169 v.verdict = NX_BENCH_AXIS_TIE
170 }
171 }
172 return 0
173}
174
175// ===== Composite verdict =============================================
176
177func _bench_composite(report: *BenchmarkReport) -> nx_int {
178 var wins: nx_int = 0
179 var ties: nx_int = 0
180 var loses: nx_int = 0
181 var unm: nx_int = 0
182
183 if report.throughput.verdict == NX_BENCH_AXIS_WIN { wins = wins + 1 }
184 if report.throughput.verdict == NX_BENCH_AXIS_TIE { ties = ties + 1 }
185 if report.throughput.verdict == NX_BENCH_AXIS_LOSE { loses = loses + 1 }
186 if report.throughput.verdict == NX_BENCH_AXIS_UNMEASURABLE { unm = unm + 1 }
187
188 if report.latency_p50.verdict == NX_BENCH_AXIS_WIN { wins = wins + 1 }
189 if report.latency_p50.verdict == NX_BENCH_AXIS_TIE { ties = ties + 1 }
190 if report.latency_p50.verdict == NX_BENCH_AXIS_LOSE { loses = loses + 1 }
191 if report.latency_p50.verdict == NX_BENCH_AXIS_UNMEASURABLE { unm = unm + 1 }
192
193 if report.latency_p99.verdict == NX_BENCH_AXIS_WIN { wins = wins + 1 }
194 if report.latency_p99.verdict == NX_BENCH_AXIS_TIE { ties = ties + 1 }
195 if report.latency_p99.verdict == NX_BENCH_AXIS_LOSE { loses = loses + 1 }
196 if report.latency_p99.verdict == NX_BENCH_AXIS_UNMEASURABLE { unm = unm + 1 }
197
198 if report.memory_peak.verdict == NX_BENCH_AXIS_WIN { wins = wins + 1 }
199 if report.memory_peak.verdict == NX_BENCH_AXIS_TIE { ties = ties + 1 }
200 if report.memory_peak.verdict == NX_BENCH_AXIS_LOSE { loses = loses + 1 }
201 if report.memory_peak.verdict == NX_BENCH_AXIS_UNMEASURABLE { unm = unm + 1 }
202
203 if report.determinism.verdict == NX_BENCH_AXIS_WIN { wins = wins + 1 }
204 if report.determinism.verdict == NX_BENCH_AXIS_TIE { ties = ties + 1 }
205 if report.determinism.verdict == NX_BENCH_AXIS_LOSE { loses = loses + 1 }
206 if report.determinism.verdict == NX_BENCH_AXIS_UNMEASURABLE { unm = unm + 1 }
207
208 if report.portability.verdict == NX_BENCH_AXIS_WIN { wins = wins + 1 }
209 if report.portability.verdict == NX_BENCH_AXIS_TIE { ties = ties + 1 }
210 if report.portability.verdict == NX_BENCH_AXIS_LOSE { loses = loses + 1 }
211 if report.portability.verdict == NX_BENCH_AXIS_UNMEASURABLE { unm = unm + 1 }
212
213 report.n_wins = wins
214 report.n_ties = ties
215 report.n_losses = loses
216 report.n_unmeasured = unm
217
218 // Routing:
219 // >= 3 unmeasured -> INCONCLUSIVE
220 // >= 4 wins, zero loses -> DECISIVE_WIN
221 // >= 4 loses -> DECISIVE_LOSE
222 // wins > loses -> WIN
223 // loses > wins -> LOSE
224 // else -> TIE
225 if unm >= 3 {
226 report.composite = NX_BENCH_INCONCLUSIVE
227 return 0
228 }
229 if wins >= 4 {
230 if loses == 0 { report.composite = NX_BENCH_DECISIVE_WIN }
231 if loses > 0 { report.composite = NX_BENCH_WIN }
232 return 0
233 }
234 if loses >= 4 {
235 report.composite = NX_BENCH_DECISIVE_LOSE
236 return 0
237 }
238 if wins > loses {
239 report.composite = NX_BENCH_WIN
240 return 0
241 }
242 if loses > wins {
243 report.composite = NX_BENCH_LOSE
244 return 0
245 }
246 report.composite = NX_BENCH_TIE
247 return 0
248}
249
250// ===== Worst-axis pinpoint ===========================================
251//
252// Find the axis with the most-negative signed delta -- this is what
253// the operator should improve first to flip the composite from LOSE
254// to WIN.
255
256func _bench_worst_axis(report: *BenchmarkReport) -> nx_int {
257 var worst_axis: nx_int = NX_BENCH_AXIS_THROUGHPUT
258 var worst_delta: nx_int = report.throughput.delta_q10
259 if report.throughput.measurement.higher_is_better == 0 {
260 worst_delta = -worst_delta
261 }
262
263 var d: nx_int = report.latency_p50.delta_q10
264 if report.latency_p50.measurement.higher_is_better == 0 { d = -d }
265 if d < worst_delta {
266 worst_delta = d
267 worst_axis = NX_BENCH_AXIS_LATENCY_P50
268 }
269
270 d = report.latency_p99.delta_q10
271 if report.latency_p99.measurement.higher_is_better == 0 { d = -d }
272 if d < worst_delta {
273 worst_delta = d
274 worst_axis = NX_BENCH_AXIS_LATENCY_P99
275 }
276
277 d = report.memory_peak.delta_q10
278 if report.memory_peak.measurement.higher_is_better == 0 { d = -d }
279 if d < worst_delta {
280 worst_delta = d
281 worst_axis = NX_BENCH_AXIS_MEMORY_PEAK
282 }
283
284 d = report.determinism.delta_q10
285 if report.determinism.measurement.higher_is_better == 0 { d = -d }
286 if d < worst_delta {
287 worst_delta = d
288 worst_axis = NX_BENCH_AXIS_DETERMINISM
289 }
290
291 d = report.portability.delta_q10
292 if report.portability.measurement.higher_is_better == 0 { d = -d }
293 if d < worst_delta {
294 worst_delta = d
295 worst_axis = NX_BENCH_AXIS_PORTABILITY
296 }
297
298 report.worst_axis = worst_axis
299 return 0
300}
301
302// ===== Public composite =============================================
303//
304// Compute all per-axis verdicts + composite + worst-axis pinpoint.
305
306func nx_benchmark_compute(report: *BenchmarkReport) -> nx_int {
307 _bench_axis_verdict(report.throughput.measurement, report.throughput)
308 _bench_axis_verdict(report.latency_p50.measurement, report.latency_p50)
309 _bench_axis_verdict(report.latency_p99.measurement, report.latency_p99)
310 _bench_axis_verdict(report.memory_peak.measurement, report.memory_peak)
311 _bench_axis_verdict(report.determinism.measurement, report.determinism)
312 _bench_axis_verdict(report.portability.measurement, report.portability)
313 _bench_composite(report)
314 _bench_worst_axis(report)
315 return 0
316}
317
318// ===== Sealed-enum validity predicates ==============================
319
320func nx_benchmark_axis_verdict_is_valid(v: nx_int) -> nx_int {
321 if v < 0 { return 0 }
322 if v >= NX_BENCH_N_AXIS_VERDICTS { return 0 }
323 return 1
324}
325
326func nx_benchmark_composite_is_valid(c: nx_int) -> nx_int {
327 if c < 0 { return 0 }
328 if c >= NX_BENCH_N_COMPOSITE_VERDICTS { return 0 }
329 return 1
330}
331
332func nx_benchmark_axis_index_is_valid(a: nx_int) -> nx_int {
333 if a < 0 { return 0 }
334 if a >= NX_BENCH_N_AXES { return 0 }
335 return 1
336}
337
338// Cardinal-enforcement predicate: for LOSE composites the caller must
339// have set named_improvement_present. Returns 1 if the report is
340// cardinal-compliant (a WIN report needs no improvement; a LOSE
341// report MUST name what to improve).
342func nx_benchmark_report_is_cardinal_compliant(report: *BenchmarkReport) -> nx_int {
343 let c: nx_int = report.composite
344 if c == NX_BENCH_LOSE { return report.named_improvement_present }
345 if c == NX_BENCH_DECISIVE_LOSE { return report.named_improvement_present }
346 if c == NX_BENCH_INCONCLUSIVE { return report.named_improvement_present }
347 return 1
348}