code wiki / _hdl_build / nx_referee_v2.nx
nx_referee_v2.nx source
↩ module page · 78 lines · 4288 B
1// nx_referee_v2.nx -- the Referee, upgraded by the cited research (knowledge/research/2026-06-04-
2// objective-testing-judging.md). v1 had separation + ground-truth + determinism + decisiveness; v2 adds
3// the mechanisms the literature proves are necessary, because the DOMINANT failure is a weak test set,
4// not the scoring math:
5// R2 TPR/TNR -- test quality on TWO axes (correctness AND coverage); a single metric hides failure
6// R3 verdicts -- fixed codes (ACC/WA/TLE/RE) + all-ACC aggregation
7// R5 validity -- TASK validity + OUTCOME validity + anti-gaming isolation (no oracle access, reset)
8// R6 leakage -- a judge related to a competitor (same model / inheritance / same family) is INVALID
9// R7 holdout -- the test set must be unseen (post-cutoff), or results are contaminated
10// A head-to-head is RIGOROUS only if ALL gates pass. license_tier: ORIGINAL
11// Cites: testlib, isolate, Wasik2018, CodeContests+ (TPR/TNR), Zhu/Jin NeurIPS2025 (validity+gaming),
12// PreferenceLeakage ICLR2026, Kapoor&Narayanan 2023.
13
14import "nx_referee.nx"
15import "nx_syscalls.nx"
16
17// ---- R2: test-set quality on two orthogonal axes ------------------------------------------------
18func rf2_permil(part: i64, whole: i64) -> i64 { if whole <= 0 { return 0 } return (part * 1000) / whole }
19// TPR = correctness (are CORRECT solutions accepted?). TNR = coverage (are WRONG solutions caught?).
20func rf2_tpr(correct_accepted: i64, correct_total: i64) -> i64 { return rf2_permil(correct_accepted, correct_total) }
21func rf2_tnr(incorrect_caught: i64, incorrect_total: i64) -> i64 { return rf2_permil(incorrect_caught, incorrect_total) }
22// trustworthy ONLY if BOTH clear the floor -- a high TPR with low TNR (or vice-versa) is a broken suite.
23func rf2_testset_trustworthy(tpr: i64, tnr: i64, floor: i64) -> i64 {
24 if tpr < floor { return 0 }
25 if tnr < floor { return 0 }
26 return 1
27}
28
29// ---- R3: verdict codes + aggregation ------------------------------------------------------------
30const RF2_ACC: i64 = 0
31const RF2_WA: i64 = 1 // wrong answer
32const RF2_TLE: i64 = 2 // time-limit exceeded
33const RF2_RE: i64 = 3 // runtime error
34// aggregate: ACC iff ALL instances ACC, else the FIRST non-ACC verdict.
35func rf2_aggregate(verdicts: *i64, n: i64) -> i64 {
36 var i: i64 = 0
37 while i < n { if verdicts[i] != RF2_ACC { return verdicts[i] } i = i + 1 }
38 return RF2_ACC
39}
40
41// ---- R5: task + outcome validity, anti-gaming isolation -----------------------------------------
42// rigorous iff target-capability == task-success (task valid) AND pass == real success (outcome valid).
43func rf2_rigorous(task_valid: i64, outcome_valid: i64) -> i64 {
44 if task_valid != 1 { return 0 }
45 if outcome_valid != 1 { return 0 }
46 return 1
47}
48// anti-gaming: the competitor must have NO read/write path to the oracle and state reset per task
49// (SWE-Lancer overwrote tests; KernelBench read the answer from stale memory).
50func rf2_gaming_proof(can_read_oracle: i64, can_write_oracle: i64, state_reset: i64) -> i64 {
51 if can_read_oracle == 1 { return 0 }
52 if can_write_oracle == 1 { return 0 }
53 if state_reset != 1 { return 0 }
54 return 1
55}
56
57// ---- R6: preference-leakage -- a related judge cannot be trusted ---------------------------------
58func rf2_judge_independent(same_model: i64, inheritance: i64, same_family: i64) -> i64 {
59 if same_model == 1 { return 0 }
60 if inheritance == 1 { return 0 }
61 if same_family == 1 { return 0 }
62 return 1
63}
64
65// ---- R7: contamination / holdout ----------------------------------------------------------------
66func rf2_holdout_clean(seen_in_training: i64) -> i64 { if seen_in_training == 0 { return 1 } return 0 }
67
68// ---- the full v2 rigor gate ---------------------------------------------------------------------
69// composes v1 (no_self_grade + hidden + oracle + deterministic) with R2/R5/R6/R7.
70func rf2_rigorous_headtohead(v1_valid: i64, testset_trustworthy: i64, rigorous: i64, gaming_proof: i64, judge_independent: i64, holdout_clean: i64) -> i64 {
71 if v1_valid != 1 { return 0 }
72 if testset_trustworthy != 1 { return 0 }
73 if rigorous != 1 { return 0 }
74 if gaming_proof != 1 { return 0 }
75 if judge_independent != 1 { return 0 }
76 if holdout_clean != 1 { return 0 }
77 return 1
78}