code wiki / _hdl_build / nx_referee.nx
nx_referee.nx source
↩ module page · 63 lines · 3229 B
1// nx_referee.nx -- the NISHI REFEREE / ADJUDICATOR (operator: "we seem to always get biased testing";
2// the racing role was scoped for running heats, but SCORING them fairly needs a separate, unbiased
3// judge). The race in nx_race_vs_claude used CLAUDE'S OWN self-assessed accuracies -- the competitor
4// graded itself, which is the exact bias to kill. The Referee enforces the bedrock of every fair judge
5// system (code-golf / ICPC / Codeforces / ML leaderboards), bits-up:
6// 1. SEPARATION -- the scorer is NOT a competitor (no self-grading)
7// 2. GROUND-TRUTH -- score against a known-correct ORACLE, not opinion
8// 3. HIDDEN HOLDOUT-- the test cases were not seen/tuned-on by the competitor (anti-overfit/Goodhart)
9// 4. DETERMINISM -- same submission -> same score, every run (no LLM-judge variance, no flaky harness)
10// 5. DECISIVENESS -- declare a winner ONLY if the margin beats a noise threshold; else TIE (no overclaim)
11// A head-to-head is VALID only if all fairness gates pass. license_tier: ORIGINAL
12// NOTE v1: core principles; to be EXTENDED by the running deep-research (contamination detection,
13// adversarial/secret test generation, special-judge checkers, statistical-significance tests).
14
15import "nx_syscalls.nx"
16
17const REF_TIE: i64 = 0
18const REF_A_WINS: i64 = 1
19const REF_B_WINS: i64 = 2
20
21// score a competitor: number of its outputs that match the ground-truth oracle over n hidden cases.
22// The Referee compares OUTPUT vs ORACLE -- it never asks the competitor how well it did.
23func ref_score(outputs: *i64, oracle: *i64, n: i64) -> i64 {
24 var c: i64 = 0; var i: i64 = 0
25 while i < n { if outputs[i] == oracle[i] { c = c + 1 } i = i + 1 }
26 return c
27}
28
29func ref_abs(x: i64) -> i64 { if x < 0 { return 0 - x } return x }
30
31// decisive only if the score margin clears a noise threshold (statistical caution on small sets).
32func ref_decisive(score_a: i64, score_b: i64, margin_threshold: i64) -> i64 {
33 if ref_abs(score_a - score_b) >= margin_threshold { return 1 }
34 return 0
35}
36
37// verdict: a winner ONLY if decisive; otherwise an honest TIE.
38func ref_verdict(score_a: i64, score_b: i64, margin_threshold: i64) -> i64 {
39 if ref_decisive(score_a, score_b, margin_threshold) == 0 { return REF_TIE }
40 if score_a > score_b { return REF_A_WINS }
41 return REF_B_WINS
42}
43
44// ===== fairness gates (the anti-bias checks) ======================================================
45
46// SEPARATION: the scorer must not be either competitor (the bug in the self-assessed race).
47func ref_no_self_grade(scorer_id: i64, competitor_a_id: i64, competitor_b_id: i64) -> i64 {
48 if scorer_id == competitor_a_id { return 0 }
49 if scorer_id == competitor_b_id { return 0 }
50 return 1
51}
52
53// HIDDEN: the competitor did not see/tune on the test cases.
54func ref_hidden(seen_by_competitor: i64) -> i64 { if seen_by_competitor == 0 { return 1 } return 0 }
55
56// a head-to-head is VALID (unbiased) only if every fairness gate passes.
57func ref_test_valid(no_self_grade: i64, hidden: i64, has_oracle: i64, deterministic: i64) -> i64 {
58 if no_self_grade != 1 { return 0 }
59 if hidden != 1 { return 0 }
60 if has_oracle != 1 { return 0 }
61 if deterministic != 1 { return 0 }
62 return 1
63}