nx_benchmark_release_candidate_t179.nx source
↩ module page · 167 lines · 8609 B
1// nx_benchmark.nx -- TRUE competitive benchmarking, not cherry-picking (operator: a recurring flaw --
2// pick a few random features and declare "we suck" or "we exceed", ignoring the hundreds of features
3// the competitor has AND ignoring what we built). The fix is a METHODOLOGY that REFUSES to issue a
4// verdict until the comparison is actually COMPLETE:
5// 1. ENUMERATE the competitor's full known feature surface (a COUNT you cannot hide behind).
6// 2. COMPLETENESS GATE -- if the comparison covers too small a fraction of that surface, it is
7// CHERRY-PICKED and NO verdict is allowed (the exact flaw, caught).
8// 3. WEIGHTED COVERAGE both ways -- features weighted by importance, our support of theirs AND their
9// support of ours (so we neither pretend to match their breadth nor erase our unique strengths).
10// 4. HONEST VERDICT -- only when valid: AHEAD / PARITY / BEHIND, or DIFFERENTIATED when each side
11// covers little of the other (we play different games -- the honest answer that is neither
12// "we suck" nor "we exceed").
13// The Critic/Examiner ENFORCE bm_is_valid before any competitive claim is published. license_tier: ORIGINAL.
14
15import "nx_syscalls.nx"
16
17const BM_REFUSE: i64 = 0 // benchmark incomplete (cherry-picked) -> NO verdict
18const BM_AHEAD: i64 = 1
19const BM_PARITY: i64 = 2
20const BM_BEHIND: i64 = 3
21const BM_DIFFERENTIATED: i64 = 4 // each covers little of the other -> different games, not better/worse
22
23// completeness (permil): how much of the competitor's KNOWN feature surface the comparison covered.
24func bm_completeness(compared: i64, total_known: i64) -> i64 {
25 if total_known <= 0 { return 0 }
26 return (compared * 1000) / total_known
27}
28
29// is the benchmark VALID (covered enough of the surface to mean anything)? threshold e.g. 800 = 80%.
30func bm_is_valid(compared: i64, total_known: i64, threshold: i64) -> i64 {
31 if bm_completeness(compared, total_known) >= threshold { return 1 }
32 return 0
33}
34
35// WEIGHTED COVERAGE (permil): over n features each with importance[] and a support level
36// support[] (0 none, 1 partial, 2 full), the fraction of total importance that is covered.
37func bm_weighted_coverage(n: i64, importance: *i64, support: *i64) -> i64 {
38 var num: i64 = 0; var den: i64 = 0; var i: i64 = 0
39 while i < n { num = num + importance[i] * support[i]; den = den + importance[i] * 2; i = i + 1 }
40 if den <= 0 { return 0 }
41 return (num * 1000) / den
42}
43
44// the HONEST verdict -- REFUSES unless the benchmark is valid; then weighs the two-way coverage.
45// both-low coverage -> DIFFERENTIATED (different games); else AHEAD/PARITY/BEHIND by margin.
46func bm_verdict(valid: i64, our_cov_of_them: i64, their_cov_of_us: i64) -> i64 {
47 if valid == 0 { return BM_REFUSE }
48 let same_game: i64 = 600
49 if our_cov_of_them < same_game { if their_cov_of_us < same_game { return BM_DIFFERENTIATED } }
50 if our_cov_of_them > their_cov_of_us + 100 { return BM_AHEAD }
51 if their_cov_of_us > our_cov_of_them + 100 { return BM_BEHIND }
52 return BM_PARITY
53}
54
55// ---- MULTI-DIMENSIONAL honesty (operator: "to say our slow-ass generations are better because they
56// use less VRAM is bullshit"). You cannot claim BETTER by winning ONE axis (VRAM) while LOSING
57// another that matters (speed/quality). Compare across ALL relevant dimensions: ----
58
59const BM_DOMINATES: i64 = 1 // better-or-equal on ALL, better on >=1 -> a REAL exceed
60const BM_TRADEOFF: i64 = 2 // better on some, WORSE on others -> NOT better, just different costs
61const BM_DOMINATED: i64 = 3 // worse-or-equal on all -> they win
62const BM_EQUAL: i64 = 4
63
64// are we better(1)/worse(-1)/equal(0) on one dimension? higher_better=1 (speed/quality), 0 (VRAM/latency).
65func bm_dim_cmp(our_v: i64, their_v: i64, higher_better: i64) -> i64 {
66 if our_v == their_v { return 0 }
67 if higher_better == 1 { if our_v > their_v { return 1 } return 0 - 1 }
68 if our_v < their_v { return 1 }
69 return 0 - 1
70}
71
72// the multi-dimensional standing. n dims, our[]/their[] values, higher_better[] direction per dim.
73func bm_dominance(n: i64, ours: *i64, theirs: *i64, higher_better: *i64) -> i64 {
74 var wb: i64 = 0; var tb: i64 = 0; var i: i64 = 0
75 while i < n {
76 let c: i64 = bm_dim_cmp(ours[i], theirs[i], higher_better[i])
77 if c == 1 { wb = wb + 1 }
78 if c == 0 - 1 { tb = tb + 1 }
79 i = i + 1
80 }
81 if wb > 0 { if tb == 0 { return BM_DOMINATES } }
82 if tb > 0 { if wb == 0 { return BM_DOMINATED } }
83 if wb > 0 { if tb > 0 { return BM_TRADEOFF } }
84 return BM_EQUAL
85}
86
87// the multi-dim BS-guard: an "exceed/better" claim is honest ONLY if it DOMINATES (a TRADEOFF that
88// wins VRAM but loses speed is NOT "better" -- it is a tradeoff, and calling it better is bullshit).
89func bm_exceed_honest(dominance: i64) -> i64 { if dominance == BM_DOMINATES { return 1 } return 0 }
90
91// Release comparison v1: caller supplies measured evidence and explicit policy.
92// UNKNOWN is refusal to promote, not proof of regression. No population caps.
93const BM_RELEASE_PASS:i64=1
94const BM_RELEASE_FAIL:i64=2
95const BM_RELEASE_UNKNOWN:i64=3
96const BM_RELEASE_INVALID:i64=4
97const BM_I64_MAX:i64=9223372036854775807
98const BM_RATIO_SCALE:i64=1000
99
100func bm_release_merge(a:i64,b:i64)->i64 {
101 if a==BM_RELEASE_INVALID || b==BM_RELEASE_INVALID { return BM_RELEASE_INVALID }
102 if a==BM_RELEASE_FAIL || b==BM_RELEASE_FAIL { return BM_RELEASE_FAIL }
103 if a==BM_RELEASE_UNKNOWN || b==BM_RELEASE_UNKNOWN { return BM_RELEASE_UNKNOWN }
104 return BM_RELEASE_PASS
105}
106
107// flags: 0 unobserved, 1 verified, 2 contradicted; never truthy-coerce unknown.
108func bm_release_fact(fact:i64)->i64 {
109 if fact==0 { return BM_RELEASE_UNKNOWN }
110 if fact==1 { return BM_RELEASE_PASS }
111 if fact==2 { return BM_RELEASE_FAIL }
112 return BM_RELEASE_INVALID
113}
114
115func bm_release_freshness(now:i64,observed:i64,max_age:i64)->i64 {
116 if now<=0 || observed<0 || max_age<=0 { return BM_RELEASE_INVALID }
117 if observed==0 { return BM_RELEASE_UNKNOWN }
118 if observed>now { return BM_RELEASE_INVALID }
119 if now-observed>max_age { return BM_RELEASE_UNKNOWN }
120 return BM_RELEASE_PASS
121}
122
123func bm_release_trials(trials:i64,minimum:i64,duration:i64,min_duration:i64)->i64 {
124 if minimum<=0 || min_duration<=0 || trials<0 || duration<0 { return BM_RELEASE_INVALID }
125 if trials<minimum || duration<min_duration { return BM_RELEASE_UNKNOWN }
126 return BM_RELEASE_PASS
127}
128
129// Integer nanoseconds plus confidence half-widths. Conservatively compare the
130// candidate upper bound to incumbent lower bound; uncertainty is not a win.
131func bm_release_ratio_limit(value:i64,ratio:i64)->i64 {
132 if value<0 || ratio<=0 { return 0-1 }
133 let q:i64=value/BM_RATIO_SCALE
134 let r:i64=value%BM_RATIO_SCALE
135 if q>BM_I64_MAX/ratio || r>BM_I64_MAX/ratio { return 0-1 }
136 let a:i64=q*ratio
137 let b:i64=(r*ratio)/BM_RATIO_SCALE
138 if a>BM_I64_MAX-b { return 0-1 }
139 return a+b
140}
141func bm_release_metric(measured:i64,candidate:i64,incumbent:i64,c_uncertainty:i64,i_uncertainty:i64,absolute_budget:i64,relative_permil:i64)->i64 {
142 if absolute_budget<=0 || relative_permil<=0 { return BM_RELEASE_INVALID }
143 if candidate<0 || incumbent<0 || c_uncertainty<0 || i_uncertainty<0 { return BM_RELEASE_INVALID }
144 let evidence:i64=bm_release_fact(measured)
145 if evidence!=BM_RELEASE_PASS { return evidence }
146 if candidate==0 || incumbent==0 { return BM_RELEASE_INVALID }
147 if candidate>BM_I64_MAX-c_uncertainty { return BM_RELEASE_INVALID }
148 if i_uncertainty>=incumbent { return BM_RELEASE_UNKNOWN }
149 let central:i64=bm_release_ratio_limit(incumbent,relative_permil)
150 let conservative:i64=bm_release_ratio_limit(incumbent-i_uncertainty,relative_permil)
151 if central<0 || conservative<0 { return BM_RELEASE_INVALID }
152 if candidate>absolute_budget || candidate>central { return BM_RELEASE_FAIL }
153 let upper:i64=candidate+c_uncertainty
154 if upper>absolute_budget || upper>conservative { return BM_RELEASE_UNKNOWN }
155 return BM_RELEASE_PASS
156}
157
158// quality: 0 unobserved, 1 equal-or-better, 2 regressed. A permitted tradeoff
159// requires separate approval evidence; it never becomes a superiority claim.
160func bm_release_quality(quality:i64,allow_tradeoff:i64,approval:i64)->i64 {
161 if allow_tradeoff<0 || allow_tradeoff>1 { return BM_RELEASE_INVALID }
162 if quality==0 { return BM_RELEASE_UNKNOWN }
163 if quality==1 { return BM_RELEASE_PASS }
164 if quality!=2 { return BM_RELEASE_INVALID }
165 if allow_tradeoff==0 { return BM_RELEASE_FAIL }
166 return bm_release_fact(approval)
167}