nx_benchmark_harness_test.nx source
↩ module page · 135 lines · 7276 B
1// nx_benchmark_harness_test.nx -- smoke for head-to-head perf verdict.
2
3import "nx_syscalls.nx"
4import "nx_tier.nx"
5import "nx_benchmark_harness.nx"
6
7// Helper: fill a measurement.
8func _fill(m: *AxisMeasurement, ch: nx_int, inc: nx_int, n: nx_int, hib: nx_int) -> nx_int {
9 m.challenger_value = ch
10 m.incumbent_value = inc
11 m.n_trials = n
12 m.higher_is_better = hib
13 return 0
14}
15
16func main() -> nx_int {
17 // BenchmarkReport struct layout: 6 AxisVerdicts (each 6 i64 fields:
18 // 4 in AxisMeasurement + verdict + delta_q10) = 36 fields plus
19 // 7 trailing i64 fields = 43 i64 cells (344 bytes). Allocate
20 // generously with a buffer.
21 let report: *BenchmarkReport = (sys_mmap(50 * NX_SIZEOF_NX_INT)) as *BenchmarkReport
22
23 // === Test 1: clean WIN across all axes ===
24 // Challenger is 10% better than incumbent on every axis.
25 _fill(report.throughput.measurement, 1100, 1000, 16, 1) // throughput +10%
26 _fill(report.latency_p50.measurement, 900, 1000, 16, 0) // latency -10%
27 _fill(report.latency_p99.measurement, 900, 1000, 16, 0)
28 _fill(report.memory_peak.measurement, 900, 1000, 16, 0) // memory -10%
29 _fill(report.determinism.measurement, 1100, 1000, 16, 1)
30 _fill(report.portability.measurement, 1100, 1000, 16, 1)
31 nx_benchmark_compute(report)
32
33 // All 6 axes should WIN.
34 if report.throughput.verdict != NX_BENCH_AXIS_WIN { return 1 }
35 if report.latency_p50.verdict != NX_BENCH_AXIS_WIN { return 2 }
36 if report.latency_p99.verdict != NX_BENCH_AXIS_WIN { return 3 }
37 if report.memory_peak.verdict != NX_BENCH_AXIS_WIN { return 4 }
38 if report.determinism.verdict != NX_BENCH_AXIS_WIN { return 5 }
39 if report.portability.verdict != NX_BENCH_AXIS_WIN { return 6 }
40 // Composite: DECISIVE_WIN (4+ wins, zero loses).
41 if report.composite != NX_BENCH_DECISIVE_WIN { return 7 }
42 if report.n_wins != 6 { return 8 }
43 if report.n_losses != 0 { return 9 }
44 // WIN cardinal-compliant by definition.
45 if nx_benchmark_report_is_cardinal_compliant(report) != 1 { return 10 }
46
47 // === Test 2: clean LOSE across all axes ===
48 let r2: *BenchmarkReport = (sys_mmap(50 * NX_SIZEOF_NX_INT)) as *BenchmarkReport
49 _fill(r2.throughput.measurement, 900, 1000, 16, 1) // throughput -10% (LOSE)
50 _fill(r2.latency_p50.measurement, 1100, 1000, 16, 0) // latency +10% (worse, LOSE)
51 _fill(r2.latency_p99.measurement, 1100, 1000, 16, 0)
52 _fill(r2.memory_peak.measurement, 1100, 1000, 16, 0)
53 _fill(r2.determinism.measurement, 900, 1000, 16, 1)
54 _fill(r2.portability.measurement, 900, 1000, 16, 1)
55 nx_benchmark_compute(r2)
56 if r2.composite != NX_BENCH_DECISIVE_LOSE { return 20 }
57 if r2.n_losses != 6 { return 21 }
58 // LOSE without named_improvement_present is NOT cardinal-compliant.
59 if nx_benchmark_report_is_cardinal_compliant(r2) != 0 { return 22 }
60 // After setting the flag, it becomes compliant.
61 r2.named_improvement_present = 1
62 if nx_benchmark_report_is_cardinal_compliant(r2) != 1 { return 23 }
63
64 // === Test 3: TIE -- all axes within 1% ===
65 let r3: *BenchmarkReport = (sys_mmap(50 * NX_SIZEOF_NX_INT)) as *BenchmarkReport
66 _fill(r3.throughput.measurement, 1005, 1000, 16, 1) // +0.5%
67 _fill(r3.latency_p50.measurement, 1003, 1000, 16, 0) // +0.3% (tiny worse)
68 _fill(r3.latency_p99.measurement, 998, 1000, 16, 0) // -0.2%
69 _fill(r3.memory_peak.measurement, 1002, 1000, 16, 0) // +0.2%
70 _fill(r3.determinism.measurement, 999, 1000, 16, 1)
71 _fill(r3.portability.measurement, 1000, 1000, 16, 1)
72 nx_benchmark_compute(r3)
73 // All within 1% -> all TIE.
74 if r3.throughput.verdict != NX_BENCH_AXIS_TIE { return 30 }
75 if r3.latency_p50.verdict != NX_BENCH_AXIS_TIE { return 31 }
76 if r3.composite != NX_BENCH_TIE { return 32 }
77 if r3.n_ties != 6 { return 33 }
78
79 // === Test 4: under-sampled axes -> UNMEASURABLE ===
80 let r4: *BenchmarkReport = (sys_mmap(50 * NX_SIZEOF_NX_INT)) as *BenchmarkReport
81 _fill(r4.throughput.measurement, 1100, 1000, 3, 1) // below MIN_TRIALS
82 _fill(r4.latency_p50.measurement, 900, 1000, 16, 0)
83 _fill(r4.latency_p99.measurement, 900, 1000, 16, 0)
84 _fill(r4.memory_peak.measurement, 900, 1000, 16, 0)
85 _fill(r4.determinism.measurement, 1100, 1000, 2, 1)
86 _fill(r4.portability.measurement, 1100, 1000, 1, 1)
87 nx_benchmark_compute(r4)
88 // 3 axes UNMEASURABLE (throughput / determinism / portability)
89 if r4.throughput.verdict != NX_BENCH_AXIS_UNMEASURABLE { return 40 }
90 // 3+ unmeasured -> INCONCLUSIVE composite.
91 if r4.composite != NX_BENCH_INCONCLUSIVE { return 41 }
92 if r4.n_unmeasured < 3 { return 42 }
93
94 // === Test 5: worst-axis pinpoint ===
95 // Throughput WIN, latency_p50 huge LOSE -- worst should be latency_p50.
96 let r5: *BenchmarkReport = (sys_mmap(50 * NX_SIZEOF_NX_INT)) as *BenchmarkReport
97 _fill(r5.throughput.measurement, 1500, 1000, 16, 1) // +50% WIN
98 _fill(r5.latency_p50.measurement, 2000, 1000, 16, 0) // +100% (latency = double = worst)
99 _fill(r5.latency_p99.measurement, 1100, 1000, 16, 0) // slight LOSE
100 _fill(r5.memory_peak.measurement, 1100, 1000, 16, 0)
101 _fill(r5.determinism.measurement, 1500, 1000, 16, 1)
102 _fill(r5.portability.measurement, 1500, 1000, 16, 1)
103 nx_benchmark_compute(r5)
104 if r5.worst_axis != NX_BENCH_AXIS_LATENCY_P50 { return 50 }
105
106 // === Test 6: sealed-enum validity predicates ===
107 if nx_benchmark_axis_verdict_is_valid(NX_BENCH_AXIS_WIN) != 1 { return 60 }
108 if nx_benchmark_axis_verdict_is_valid(NX_BENCH_AXIS_UNMEASURABLE) != 1 { return 61 }
109 if nx_benchmark_axis_verdict_is_valid(99) != 0 { return 62 }
110 if nx_benchmark_composite_is_valid(NX_BENCH_DECISIVE_WIN) != 1 { return 63 }
111 if nx_benchmark_composite_is_valid(NX_BENCH_DECISIVE_LOSE) != 1 { return 64 }
112 if nx_benchmark_composite_is_valid(99) != 0 { return 65 }
113 if nx_benchmark_axis_index_is_valid(NX_BENCH_AXIS_THROUGHPUT) != 1 { return 66 }
114 if nx_benchmark_axis_index_is_valid(99) != 0 { return 67 }
115
116 // === Test 7: pure-parity is NOT TIE -- it's LOSE per 1% floor ===
117 // Per cardinal: matching incumbent exactly is not a win, but also
118 // not a friendly tie -- it's LOSE_BY_PARITY. The 1% floor means
119 // delta_q10 = 0 -> NOT WIN, and we route equal-strength to TIE in
120 // the current verdict logic (since |delta| < floor). Honest: TIE
121 // here is the substrate saying "no measurable difference"; the
122 // operator decides whether parity is acceptable.
123 let r7: *BenchmarkReport = (sys_mmap(50 * NX_SIZEOF_NX_INT)) as *BenchmarkReport
124 _fill(r7.throughput.measurement, 1000, 1000, 16, 1) // exact parity
125 _fill(r7.latency_p50.measurement, 1000, 1000, 16, 0)
126 _fill(r7.latency_p99.measurement, 1000, 1000, 16, 0)
127 _fill(r7.memory_peak.measurement, 1000, 1000, 16, 0)
128 _fill(r7.determinism.measurement, 1000, 1000, 16, 1)
129 _fill(r7.portability.measurement, 1000, 1000, 16, 1)
130 nx_benchmark_compute(r7)
131 if r7.throughput.verdict != NX_BENCH_AXIS_TIE { return 70 }
132 if r7.composite != NX_BENCH_TIE { return 71 }
133
134 return 0
135}