nx_referee_v2_test.nx source

1// nx_referee_v2_test.nx -- the research-grounded gates catch the exact failures the literature documents. 2// thin suite (TPR 1000 / TNR 100): accepts correct but catches almost no wrong solutions -> NOT trustworthy 3// (mirrors CodeContests+ 4000 problems with TPR<=0.1; a single score would hide it) 4// aggregate [ACC,ACC,WA] -> WA (first non-ACC) 5// gaming: competitor can overwrite the oracle (SWE-Lancer assert 1==1) -> NOT gaming-proof 6// leakage: judge in the same model family as a competitor -> NOT independent 7// holdout: test seen in training -> NOT clean 8// the FULL rigor gate passes only when every condition holds. 9// Exit 0 on 8/8. license_tier: ORIGINAL 10 11import "nx_referee_v2.nx" 12import "nx_referee.nx" 13import "nx_syscalls.nx" 14 15func vt_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 } 16func vt_num(v: i64) -> i64 { let bb: *u8 = sys_mmap(28); var m: i64=v; if m<0 {m=0-m; sys_write(1,"-" as *u8,1)}; let t: *u8 = sys_mmap(28); var k: i64=0; if m==0 {t[0]=48;k=1}; while m>0 {t[k]=48+(m%10); m=m/10; k=k+1}; var i: i64=0; while i<k {bb[i]=t[k-1-i]; i=i+1}; sys_write(1, bb, k); return 0 } 17 18func main() -> i64 { 19 vt_puts("=== REFEREE v2: research-grounded gates (testset quality, validity, leakage, holdout) ===\n" as *u8) 20 let floor: i64 = 800 21 22 // R2 -- a THIN suite: accepts all 20 correct, but catches only 2 of 20 wrong 23 let tpr_thin: i64 = rf2_tpr(20, 20); let tnr_thin: i64 = rf2_tnr(2, 20) 24 let thin_ok: i64 = rf2_testset_trustworthy(tpr_thin, tnr_thin, floor) 25 // a GOOD suite: 19/20 correct accepted, 19/20 wrong caught 26 let tpr_good: i64 = rf2_tpr(19, 20); let tnr_good: i64 = rf2_tnr(19, 20) 27 let good_ok: i64 = rf2_testset_trustworthy(tpr_good, tnr_good, floor) 28 29 // R3 -- aggregation 30 let vs: *i64 = sys_mmap(8*8) as *i64; vs[0]=RF2_ACC; vs[1]=RF2_ACC; vs[2]=RF2_WA 31 let agg: i64 = rf2_aggregate(vs, 3) 32 33 // R5 -- anti-gaming 34 let gaming_bad: i64 = rf2_gaming_proof(0, 1, 1) // can WRITE the oracle -> not proof 35 let gaming_ok: i64 = rf2_gaming_proof(0, 0, 1) // no access + reset -> proof 36 let rigorous: i64 = rf2_rigorous(1, 1) 37 38 // R6 -- preference leakage 39 let judge_bad: i64 = rf2_judge_independent(0, 0, 1) // same family -> not independent 40 let judge_ok: i64 = rf2_judge_independent(0, 0, 0) 41 42 // R7 -- holdout 43 let hold_bad: i64 = rf2_holdout_clean(1) // seen in training 44 let hold_ok: i64 = rf2_holdout_clean(0) 45 46 // full gate: v1 valid + every v2 gate 47 let v1_valid: i64 = ref_test_valid(ref_no_self_grade(99, 1, 2), ref_hidden(0), 1, 1) 48 let full_ok: i64 = rf2_rigorous_headtohead(v1_valid, good_ok, rigorous, gaming_ok, judge_ok, hold_ok) 49 let full_bad: i64 = rf2_rigorous_headtohead(v1_valid, thin_ok, rigorous, gaming_ok, judge_ok, hold_ok) 50 51 vt_puts(" R2 thin suite TPR=" as *u8); vt_num(tpr_thin); vt_puts(" TNR=" as *u8); vt_num(tnr_thin); vt_puts(" -> trustworthy=" as *u8); vt_num(thin_ok); vt_puts(" good suite -> " as *u8); vt_num(good_ok); vt_puts("\n" as *u8) 52 vt_puts(" R3 aggregate [ACC,ACC,WA] = " as *u8); vt_num(agg); vt_puts(" (1=WA) R5 gaming(write-oracle)=" as *u8); vt_num(gaming_bad); vt_puts(" gaming(isolated)=" as *u8); vt_num(gaming_ok); vt_puts("\n" as *u8) 53 vt_puts(" R6 judge same-family independent=" as *u8); vt_num(judge_bad); vt_puts(" R7 holdout(seen)=" as *u8); vt_num(hold_bad); vt_puts("\n" as *u8) 54 vt_puts(" FULL rigor gate: good-setup=" as *u8); vt_num(full_ok); vt_puts(" thin-suite-setup=" as *u8); vt_num(full_bad); vt_puts("\n" as *u8) 55 56 let r: *i64 = sys_mmap(8*8) as *i64 57 r[0] = 0; if thin_ok == 0 { r[0] = 1 } // thin suite rejected (low TNR) 58 r[1] = 0; if good_ok == 1 { r[1] = 1 } // good suite accepted 59 r[2] = 0; if agg == RF2_WA { r[2] = 1 } // first non-ACC wins aggregation 60 r[3] = 0; if gaming_bad == 0 { if gaming_ok == 1 { r[3] = 1 } } // oracle-write caught 61 r[4] = 0; if judge_bad == 0 { if judge_ok == 1 { r[4] = 1 } } // related judge caught 62 r[5] = 0; if hold_bad == 0 { if hold_ok == 1 { r[5] = 1 } } // contamination caught 63 r[6] = 0; if full_ok == 1 { r[6] = 1 } // a fully clean setup passes 64 r[7] = 0; if full_bad == 0 { r[7] = 1 } // ...but a thin-suite setup is rejected end-to-end 65 var pass: i64 = 0; var i: i64 = 0 66 while i < 8 { pass = pass + r[i]; i = i + 1 } 67 vt_puts("----\n passed " as *u8); vt_num(pass); vt_puts("/8\n" as *u8) 68 if pass == 8 { vt_puts(" REFEREE v2: a head-to-head is RIGOROUS only with high TPR+TNR tests, task+outcome validity, an isolated un-gameable oracle, an INDEPENDENT judge, and clean holdout -- the cited bias modes are gated out.\n" as *u8); sys_exit(0); return 0 } 69 vt_puts(" FAIL\n" as *u8); sys_exit(1); return 1 70}