nx_polyglot_bench.nx source
↩ module page · 177 lines · 6328 B
1// nx_polyglot_bench.nx -- head-to-head verdict against N incumbent
2// programming languages, layered on top of nx_benchmark_harness.
3//
4// LAYER 3 of nishifamily.com/ide rollout. Where nx_benchmark_harness
5// expresses 1-vs-1 ("NishiLang vs incumbent") this primitive expresses
6// 1-vs-N ("NishiLang vs {C, Rust, Go, Python, Java, JavaScript,
7// Erlang, Zig, Nim, OCaml, Crystal}"). Composite verdict is honest
8// per cardinal feedback-honest-perf-verdict-no-aspirational-claims:
9//
10// POLYGLOT_DECISIVE_WIN NishiLang WINs every measured axis vs
11// every competitor we ran.
12// POLYGLOT_WIN WIN on majority of pairings; no LOSE.
13// POLYGLOT_MIXED WIN against some, LOSE against others;
14// named_improvement REQUIRED per LOSE row.
15// POLYGLOT_LOSE LOSE on majority; named_improvement
16// REQUIRED per LOSE row.
17// POLYGLOT_DECISIVE_LOSE LOSE on every measured pairing. The
18// named_improvement field is mandatory
19// AND must be substrate-level (not just
20// "tune the bench loop").
21// POLYGLOT_INCONCLUSIVE too many competitor runs UNMEASURABLE.
22//
23// genealogy_id: programming_language_benchmark_game_2003 +
24// techempower_web_benchmarks +
25// criterion_rs_paper_2018 +
26// feedback-honest-perf-verdict-no-aspirational-claims
27// lineage_id: polyglot_n_way_verdict_q10
28
29// nx_safety_envelope:
30// intended_use: AUTO_APPLIED -- primitive-specific tuning queued
31// sil_target: SIL1
32// evidence: [bulk_applied_2026-05-16, see-file-comment-for-detail]
33// verdict: NOT_YET_EVALUATED
34
35import "nx_syscalls.nx"
36import "nx_tier.nx"
37import "nx_benchmark_harness.nx"
38
39// Sealed CompetitorLang enum. Indices double as array slots so
40// substrate can iterate without a string switch.
41const NX_LANG_C: nx_int = 0
42const NX_LANG_RUST: nx_int = 1
43const NX_LANG_GO: nx_int = 2
44const NX_LANG_PYTHON: nx_int = 3
45const NX_LANG_JAVA: nx_int = 4
46const NX_LANG_JAVASCRIPT: nx_int = 5
47const NX_LANG_ERLANG: nx_int = 6
48const NX_LANG_ZIG: nx_int = 7
49const NX_LANG_NIM: nx_int = 8
50const NX_LANG_OCAML: nx_int = 9
51const NX_LANG_CRYSTAL: nx_int = 10
52const NX_LANG_N: nx_int = 11
53
54// Sealed polyglot verdict.
55const NX_POLYGLOT_INCONCLUSIVE: nx_int = 0
56const NX_POLYGLOT_DECISIVE_LOSE: nx_int = 1
57const NX_POLYGLOT_LOSE: nx_int = 2
58const NX_POLYGLOT_MIXED: nx_int = 3
59const NX_POLYGLOT_WIN: nx_int = 4
60const NX_POLYGLOT_DECISIVE_WIN: nx_int = 5
61const NX_POLYGLOT_N_VERDICTS: nx_int = 6
62
63// A polyglot run = one challenger (NishiLang) bench vs n_competitors
64// incumbent BenchmarkReports. Each pairing carries its own report so
65// per-axis WIN/LOSE/TIE detail survives composition.
66struct PolyglotRun {
67 competitors: *BenchmarkReport, // n_competitors entries
68 competitor_langs: *nx_int, // n_competitors entries, NX_LANG_* values
69 n_competitors: nx_int,
70 // Composite across all pairings.
71 n_decisive_wins: nx_int,
72 n_wins: nx_int,
73 n_ties: nx_int,
74 n_loses: nx_int,
75 n_decisive_loses: nx_int,
76 n_inconclusive: nx_int,
77 composite: nx_int, // NX_POLYGLOT_*
78 // Pinpoint: index of the BEST and WORST competitor for substrate
79 // self-audit ("we win against Python by 12x but lose to Rust by
80 // 4x; named_improvement: integrate regalloc.c with x86_64
81 // backend").
82 best_competitor_idx: nx_int,
83 worst_competitor_idx: nx_int,
84}
85
86// Roll up an array of per-competitor BenchmarkReports into a polyglot
87// composite verdict. Caller is expected to have already filled each
88// competitors[i] via nx_benchmark_compute().
89func nx_polyglot_compute(run: *PolyglotRun) -> nx_int {
90 var dw: nx_int = 0
91 var w: nx_int = 0
92 var t: nx_int = 0
93 var l: nx_int = 0
94 var dl: nx_int = 0
95 var inc: nx_int = 0
96
97 var i: nx_int = 0
98 while i < run.n_competitors {
99 let v: nx_int = run.competitors[i].composite
100 if v == NX_BENCH_DECISIVE_WIN { dw = dw + 1 }
101 if v == NX_BENCH_WIN { w = w + 1 }
102 if v == NX_BENCH_TIE { t = t + 1 }
103 if v == NX_BENCH_LOSE { l = l + 1 }
104 if v == NX_BENCH_DECISIVE_LOSE { dl = dl + 1 }
105 if v == NX_BENCH_INCONCLUSIVE { inc = inc + 1 }
106 i = i + 1
107 }
108
109 run.n_decisive_wins = dw
110 run.n_wins = w
111 run.n_ties = t
112 run.n_loses = l
113 run.n_decisive_loses = dl
114 run.n_inconclusive = inc
115
116 // Routing.
117 if inc >= run.n_competitors / 2 {
118 run.composite = NX_POLYGLOT_INCONCLUSIVE
119 return 0
120 }
121 let measured: nx_int = run.n_competitors - inc
122 let total_w: nx_int = dw + w
123 let total_l: nx_int = dl + l
124
125 if dw == measured {
126 run.composite = NX_POLYGLOT_DECISIVE_WIN
127 return 0
128 }
129 if dl == measured {
130 run.composite = NX_POLYGLOT_DECISIVE_LOSE
131 return 0
132 }
133 if total_w > 0 {
134 if total_l == 0 {
135 run.composite = NX_POLYGLOT_WIN
136 return 0
137 }
138 run.composite = NX_POLYGLOT_MIXED
139 return 0
140 }
141 run.composite = NX_POLYGLOT_LOSE
142 return 0
143}
144
145// Locate best/worst competitor: best = the one where we score highest
146// (most-positive throughput delta), worst = lowest. Used by
147// nx_self_audit to spotlight the named_improvement target.
148func nx_polyglot_pinpoint(run: *PolyglotRun) -> nx_int {
149 if run.n_competitors == 0 { return 0 }
150 var best_i: nx_int = 0
151 var worst_i: nx_int = 0
152 var best_d: nx_int = run.competitors[0].throughput.delta_q10
153 var worst_d: nx_int = best_d
154 var i: nx_int = 1
155 while i < run.n_competitors {
156 let d: nx_int = run.competitors[i].throughput.delta_q10
157 if d > best_d { best_d = d; best_i = i }
158 if d < worst_d { worst_d = d; worst_i = i }
159 i = i + 1
160 }
161 run.best_competitor_idx = best_i
162 run.worst_competitor_idx = worst_i
163 return 0
164}
165
166// Sealed-enum validity for the polyglot verdict.
167func nx_polyglot_verdict_is_valid(v: nx_int) -> nx_int {
168 if v < 0 { return 0 }
169 if v >= NX_POLYGLOT_N_VERDICTS { return 0 }
170 return 1
171}
172
173func nx_polyglot_lang_is_valid(l: nx_int) -> nx_int {
174 if l < 0 { return 0 }
175 if l >= NX_LANG_N { return 0 }
176 return 1
177}