nx_perf_verdict.nx source
↩ module page · 169 lines · 7354 B
1// nx_perf_verdict.nx -- substrate primitive for honest per-axis
2// head-to-head verdicts.
3//
4// Per honest-hard-perf-verdict cardinal 2026-05-14:
5// "every head-to-head bench emits sealed-enum per-axis verdict and
6// every LOSE row names a specific concrete improvement target.
7// UNMEASURABLE is OK; 'straddles or beats' / 'competitive with' are
8// REFUSED weasel words".
9//
10// Sealed enum (do not add weasel intermediates):
11// NX_VERDICT_UNMEASURABLE = 0 no comparable data yet (honest about gap)
12// NX_VERDICT_LOSE = 1 we are worse on this axis
13// NX_VERDICT_TIE = 2 parity
14// NX_VERDICT_WIN = 3 we are better
15//
16// Standard axes (use these names so scoreboards aggregate cleanly):
17// throughput, accuracy, memory_at_parity, serialized_size,
18// determinism, patent_clean, license_clean, portability
19//
20// genealogy_id: substrate_perf_verdict_2026_05_14
21// lineage_id: honest_hard_feedback
22
23// nx_safety_envelope:
24// intended_use: AUTO_APPLIED -- primitive-specific tuning queued
25// sil_target: SIL1
26// evidence: [bulk_applied_2026-05-16, see-file-comment-for-detail]
27// verdict: NOT_YET_EVALUATED
28
29import "nx_syscalls.nx"
30import "nx_runtime.nx"
31import "nx_tier.nx"
32
33const NX_VERDICT_UNMEASURABLE: nx_int = 0
34const NX_VERDICT_LOSE: nx_int = 1
35const NX_VERDICT_TIE: nx_int = 2
36const NX_VERDICT_WIN: nx_int = 3
37
38func nx_verdict_label(v: nx_int) -> *u8 {
39 if v == NX_VERDICT_WIN { return "WIN " as *u8 }
40 if v == NX_VERDICT_TIE { return "TIE " as *u8 }
41 if v == NX_VERDICT_LOSE { return "LOSE " as *u8 }
42 return "UNMEASURABLE " as *u8
43}
44
45// Emit one row: "verdict:<axis> <LABEL> nishi=<v> incumbent=<v> fix=<text>".
46// If improvement is "" the fix column is omitted.
47//
48// All field separators are double-space so the row is greppable AND
49// reads as a fixed-width report.
50func nx_verdict_emit(axis: *u8, verdict: nx_int,
51 nishi_value: nx_int, incumbent_value: nx_int,
52 improvement: *u8) {
53 print("verdict:" as *u8)
54 print(axis)
55 print(" " as *u8)
56 print(nx_verdict_label(verdict))
57 print(" nishi=" as *u8)
58 print_i64(nishi_value)
59 print(" incumbent=" as *u8)
60 print_i64(incumbent_value)
61 if improvement[0] != 0 {
62 print(" fix=" as *u8)
63 print(improvement)
64 }
65 println("" as *u8)
66}
67
68// Per 1%-minimum-delta cardinal 2026-05-14: "even if its dumb we
69// should try to be at least 1 percent better than other systems".
70// WIN floor is now 101% (we must outperform); exact parity is
71// LOSE_BY_PARITY with a 1%-improvement target named.
72//
73// Caller-supplied thresholds still respected -- caller can demand
74// 110% WIN floor / 90% LOSE ceiling etc. Defaults below assume
75// the 1%-floor doctrine.
76const NX_VERDICT_WIN_FLOOR_PCT: nx_int = 101
77const NX_VERDICT_LOSE_FLOOR_PCT: nx_int = 100
78
79// Convenience: emit a verdict computed from a ratio. WIN if
80// nishi_value >= incumbent_value * tie_floor_pct/100; LOSE if
81// nishi_value < incumbent_value * lose_ceiling_pct/100; TIE between.
82// For "lower is better" metrics (memory, error), call with swapped args.
83func nx_verdict_ratio(axis: *u8, nishi_value: nx_int, incumbent_value: nx_int,
84 tie_floor_pct: nx_int, lose_ceiling_pct: nx_int,
85 improvement_on_lose: *u8) {
86 if incumbent_value <= 0 {
87 nx_verdict_emit(axis, NX_VERDICT_UNMEASURABLE, nishi_value, incumbent_value, improvement_on_lose)
88 return
89 }
90 let pct: nx_int = (nishi_value * 100) / incumbent_value
91 var v: nx_int = NX_VERDICT_TIE
92 if pct >= tie_floor_pct { v = NX_VERDICT_WIN }
93 if pct < lose_ceiling_pct { v = NX_VERDICT_LOSE }
94 var fix: *u8 = "" as *u8
95 if v == NX_VERDICT_LOSE { fix = improvement_on_lose }
96 if v == NX_VERDICT_TIE {
97 // Pure parity counts as LOSE-by-parity per 1%-cardinal.
98 v = NX_VERDICT_LOSE
99 fix = "PARITY_NOT_ENOUGH_NEED_1PCT_IMPROVEMENT: find a code-path / cache-layout / algorithm tweak that nudges metric past 101% of incumbent" as *u8
100 }
101 nx_verdict_emit(axis, v, nishi_value, incumbent_value, fix)
102}
103
104// Convenience for "lower is better" metrics (memory, error).
105func nx_verdict_ratio_lower_better(axis: *u8, nishi_value: nx_int, incumbent_value: nx_int,
106 tie_ceiling_pct: nx_int, lose_floor_pct: nx_int,
107 improvement_on_lose: *u8) {
108 if incumbent_value <= 0 {
109 nx_verdict_emit(axis, NX_VERDICT_UNMEASURABLE, nishi_value, incumbent_value, improvement_on_lose)
110 return
111 }
112 let pct: nx_int = (nishi_value * 100) / incumbent_value
113 var v: nx_int = NX_VERDICT_TIE
114 if pct <= tie_ceiling_pct { v = NX_VERDICT_WIN }
115 if pct > lose_floor_pct { v = NX_VERDICT_LOSE }
116 var fix: *u8 = "" as *u8
117 if v == NX_VERDICT_LOSE { fix = improvement_on_lose }
118 if v == NX_VERDICT_TIE {
119 v = NX_VERDICT_LOSE
120 fix = "PARITY_NOT_ENOUGH_NEED_1PCT_IMPROVEMENT: shrink metric to 99% of incumbent (memory: tighter packing / error: denser bias table)" as *u8
121 }
122 nx_verdict_emit(axis, v, nishi_value, incumbent_value, fix)
123}
124
125// ===== Qualitative verdict (extended per user 2026-05-14) ===========
126//
127// "it needs to be the same for qualitative and quantitative where our
128// media gets better and better and our quants like functions to
129// achieve answers exceed all the previous winning functions that
130// competitive math and programming has had since the beginning"
131//
132// Rubric scores are 0..100 (use Q0 percent, never floats). Citation
133// is a URL or public-doc reference -- "based on public information
134// not anything we can be sued on" per user directive.
135//
136// Examples of qualitative axes:
137// media_quality_vs_sdcpp -- our diffusion outputs vs sdcpp at same prompt
138// function_elegance_vs_winner -- our CP-problem solution vs winner's posted
139// algorithm_optimality_vs_paper -- our approach vs published optimal
140
141func nx_verdict_emit_qualitative(axis: *u8, verdict: nx_int,
142 nishi_score: nx_int, incumbent_score: nx_int,
143 citation: *u8, justification: *u8) {
144 print("verdict_qual:" as *u8)
145 print(axis)
146 print(" " as *u8)
147 print(nx_verdict_label(verdict))
148 print(" nishi=" as *u8)
149 print_i64(nishi_score)
150 print("/100 incumbent=" as *u8)
151 print_i64(incumbent_score)
152 print("/100 cite=" as *u8)
153 print(citation)
154 print(" because=" as *u8)
155 print(justification)
156 println("" as *u8)
157}
158
159// Convenience: WIN if nishi_score > incumbent_score+margin; LOSE if
160// nishi_score < incumbent_score-margin; TIE within margin (default 5).
161func nx_verdict_qualitative_simple(axis: *u8, nishi_score: nx_int, incumbent_score: nx_int,
162 margin: nx_int, citation: *u8, justification: *u8) {
163 var v: nx_int = NX_VERDICT_TIE
164 if nishi_score > incumbent_score + margin { v = NX_VERDICT_WIN }
165 if nishi_score < incumbent_score - margin { v = NX_VERDICT_LOSE }
166 if nishi_score < 0 { v = NX_VERDICT_UNMEASURABLE }
167 if incumbent_score < 0 { v = NX_VERDICT_UNMEASURABLE }
168 nx_verdict_emit_qualitative(axis, v, nishi_score, incumbent_score, citation, justification)
169}