nx_benchmark.nx source

1// nx_benchmark.nx -- TRUE competitive benchmarking, not cherry-picking (operator: a recurring flaw -- 2// pick a few random features and declare "we suck" or "we exceed", ignoring the hundreds of features 3// the competitor has AND ignoring what we built). The fix is a METHODOLOGY that REFUSES to issue a 4// verdict until the comparison is actually COMPLETE: 5// 1. ENUMERATE the competitor's full known feature surface (a COUNT you cannot hide behind). 6// 2. COMPLETENESS GATE -- if the comparison covers too small a fraction of that surface, it is 7// CHERRY-PICKED and NO verdict is allowed (the exact flaw, caught). 8// 3. WEIGHTED COVERAGE both ways -- features weighted by importance, our support of theirs AND their 9// support of ours (so we neither pretend to match their breadth nor erase our unique strengths). 10// 4. HONEST VERDICT -- only when valid: AHEAD / PARITY / BEHIND, or DIFFERENTIATED when each side 11// covers little of the other (we play different games -- the honest answer that is neither 12// "we suck" nor "we exceed"). 13// The Critic/Examiner ENFORCE bm_is_valid before any competitive claim is published. license_tier: ORIGINAL. 14 15import "nx_syscalls.nx" 16 17const BM_REFUSE: i64 = 0 // benchmark incomplete (cherry-picked) -> NO verdict 18const BM_AHEAD: i64 = 1 19const BM_PARITY: i64 = 2 20const BM_BEHIND: i64 = 3 21const BM_DIFFERENTIATED: i64 = 4 // each covers little of the other -> different games, not better/worse 22 23// completeness (permil): how much of the competitor's KNOWN feature surface the comparison covered. 24func bm_completeness(compared: i64, total_known: i64) -> i64 { 25 if total_known <= 0 { return 0 } 26 return (compared * 1000) / total_known 27} 28 29// is the benchmark VALID (covered enough of the surface to mean anything)? threshold e.g. 800 = 80%. 30func bm_is_valid(compared: i64, total_known: i64, threshold: i64) -> i64 { 31 if bm_completeness(compared, total_known) >= threshold { return 1 } 32 return 0 33} 34 35// WEIGHTED COVERAGE (permil): over n features each with importance[] and a support level 36// support[] (0 none, 1 partial, 2 full), the fraction of total importance that is covered. 37func bm_weighted_coverage(n: i64, importance: *i64, support: *i64) -> i64 { 38 var num: i64 = 0; var den: i64 = 0; var i: i64 = 0 39 while i < n { num = num + importance[i] * support[i]; den = den + importance[i] * 2; i = i + 1 } 40 if den <= 0 { return 0 } 41 return (num * 1000) / den 42} 43 44// the HONEST verdict -- REFUSES unless the benchmark is valid; then weighs the two-way coverage. 45// both-low coverage -> DIFFERENTIATED (different games); else AHEAD/PARITY/BEHIND by margin. 46func bm_verdict(valid: i64, our_cov_of_them: i64, their_cov_of_us: i64) -> i64 { 47 if valid == 0 { return BM_REFUSE } 48 let same_game: i64 = 600 49 if our_cov_of_them < same_game { if their_cov_of_us < same_game { return BM_DIFFERENTIATED } } 50 if our_cov_of_them > their_cov_of_us + 100 { return BM_AHEAD } 51 if their_cov_of_us > our_cov_of_them + 100 { return BM_BEHIND } 52 return BM_PARITY 53} 54 55// ---- MULTI-DIMENSIONAL honesty (operator: "to say our slow-ass generations are better because they 56// use less VRAM is bullshit"). You cannot claim BETTER by winning ONE axis (VRAM) while LOSING 57// another that matters (speed/quality). Compare across ALL relevant dimensions: ---- 58 59const BM_DOMINATES: i64 = 1 // better-or-equal on ALL, better on >=1 -> a REAL exceed 60const BM_TRADEOFF: i64 = 2 // better on some, WORSE on others -> NOT better, just different costs 61const BM_DOMINATED: i64 = 3 // worse-or-equal on all -> they win 62const BM_EQUAL: i64 = 4 63 64// are we better(1)/worse(-1)/equal(0) on one dimension? higher_better=1 (speed/quality), 0 (VRAM/latency). 65func bm_dim_cmp(our_v: i64, their_v: i64, higher_better: i64) -> i64 { 66 if our_v == their_v { return 0 } 67 if higher_better == 1 { if our_v > their_v { return 1 } return 0 - 1 } 68 if our_v < their_v { return 1 } 69 return 0 - 1 70} 71 72// the multi-dimensional standing. n dims, our[]/their[] values, higher_better[] direction per dim. 73func bm_dominance(n: i64, ours: *i64, theirs: *i64, higher_better: *i64) -> i64 { 74 var wb: i64 = 0; var tb: i64 = 0; var i: i64 = 0 75 while i < n { 76 let c: i64 = bm_dim_cmp(ours[i], theirs[i], higher_better[i]) 77 if c == 1 { wb = wb + 1 } 78 if c == 0 - 1 { tb = tb + 1 } 79 i = i + 1 80 } 81 if wb > 0 { if tb == 0 { return BM_DOMINATES } } 82 if tb > 0 { if wb == 0 { return BM_DOMINATED } } 83 if wb > 0 { if tb > 0 { return BM_TRADEOFF } } 84 return BM_EQUAL 85} 86 87// the multi-dim BS-guard: an "exceed/better" claim is honest ONLY if it DOMINATES (a TRADEOFF that 88// wins VRAM but loses speed is NOT "better" -- it is a tradeoff, and calling it better is bullshit). 89func bm_exceed_honest(dominance: i64) -> i64 { if dominance == BM_DOMINATES { return 1 } return 0 }