nx_referee_test.nx source

1// nx_referee_test.nx -- a FAIR head-to-head, adjudicated. Competitor A = the TEAM's BM25 (computed 2// live), Competitor B = a fixed "Claude" answer set. Both are scored by the Referee against a 3// GROUND-TRUTH ORACLE on a hidden query set -- neither competitor grades itself. The Referee also 4// FLAGS the earlier nx_race_vs_claude setup as INVALID, because there the competitor (Claude) supplied 5// its own accuracies (self-grading). Exit 0 on 7/7. license_tier: ORIGINAL 6 7import "nx_referee.nx" 8import "nx_library_search.nx" // ls_best_bm25 -- the team competitor, computed live 9import "nx_research_extract.nx" // re_strlen 10import "nx_syscalls.nx" 11 12func ft_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 } 13func ft_num(v: i64) -> i64 { let bb: *u8 = sys_mmap(28); var m: i64=v; if m<0 {m=0-m; sys_write(1,"-" as *u8,1)}; let t: *u8 = sys_mmap(28); var k: i64=0; if m==0 {t[0]=48;k=1}; while m>0 {t[k]=48+(m%10); m=m/10; k=k+1}; var i: i64=0; while i<k {bb[i]=t[k-1-i]; i=i+1}; sys_write(1, bb, k); return 0 } 14 15func main() -> i64 { 16 ft_puts("=== REFEREE: fair head-to-head, ground-truth oracle, scorer separated ===\n" as *u8) 17 let REFEREE: i64 = 99; let TEAM: i64 = 1; let CLAUDE: i64 = 2 18 19 // corpus + hidden query set with a GROUND-TRUTH oracle 20 let N: i64 = 4 21 let d0: *u8 = "injury injury" as *u8 22 let d1: *u8 = "injury injury injury f f f f f f f f f" as *u8 23 let d2: *u8 = "settlement settlement filler" as *u8 24 let d3: *u8 = "court court court" as *u8 25 let ptrs: *i64 = sys_mmap(8*8) as *i64; let lens: *i64 = sys_mmap(8*8) as *i64 26 ptrs[0]=d0 as i64; ptrs[1]=d1 as i64; ptrs[2]=d2 as i64; ptrs[3]=d3 as i64 27 var i: i64 = 0; while i < N { lens[i] = re_strlen(ptrs[i] as *u8); i = i + 1 } 28 29 let Q: i64 = 3 30 let oracle: *i64 = sys_mmap(8*8) as *i64; oracle[0]=0; oracle[1]=3; oracle[2]=2 // GT best doc per query 31 32 // Competitor A = TEAM BM25, computed LIVE (not self-reported) 33 let a_out: *i64 = sys_mmap(8*8) as *i64 34 let q0: *i64 = sys_mmap(4*8) as *i64; q0[0]="injury" as *u8 as i64 35 let q1: *i64 = sys_mmap(4*8) as *i64; q1[0]="court" as *u8 as i64 36 let q2: *i64 = sys_mmap(4*8) as *i64; q2[0]="settlement" as *u8 as i64 37 a_out[0]=ls_best_bm25(ptrs,lens,N,q0,1); a_out[1]=ls_best_bm25(ptrs,lens,N,q1,1); a_out[2]=ls_best_bm25(ptrs,lens,N,q2,1) 38 // Competitor B = a fixed "Claude" answer set (missed Q0) 39 let b_out: *i64 = sys_mmap(8*8) as *i64; b_out[0]=1; b_out[1]=3; b_out[2]=2 40 41 let sa: i64 = ref_score(a_out, oracle, Q) 42 let sb: i64 = ref_score(b_out, oracle, Q) 43 let sa2: i64 = ref_score(a_out, oracle, Q) // deterministic: re-score -> same 44 var deterministic: i64 = 0; if sa == sa2 { deterministic = 1 } 45 46 let fair_self: i64 = ref_no_self_grade(REFEREE, TEAM, CLAUDE) // 1 = separated (fair) 47 let biased_self: i64 = ref_no_self_grade(CLAUDE, TEAM, CLAUDE) // 0 = self-grade (the old race) 48 let valid_fair: i64 = ref_test_valid(fair_self, ref_hidden(0), 1, deterministic) 49 let valid_biased: i64 = ref_test_valid(biased_self, ref_hidden(0), 1, deterministic) 50 let vsmall: i64 = ref_verdict(sa, sb, 2) // margin 1 on 3 cases -> not decisive 51 let vscale: i64 = ref_verdict(30, 20, 2) // decisive at scale 52 53 ft_puts(" TEAM(BM25) outputs=[" as *u8); i=0; while i<Q { ft_num(a_out[i]); if i<Q-1 { ft_puts("," as *u8) } i=i+1 } ft_puts("] score=" as *u8); ft_num(sa); ft_puts("/3 vs oracle\n" as *u8) 54 ft_puts(" CLAUDE outputs=[1,3,2] score=" as *u8); ft_num(sb); ft_puts("/3 vs oracle (deterministic=" as *u8); ft_num(deterministic); ft_puts(")\n" as *u8) 55 ft_puts(" fairness: scorer-separated=" as *u8); ft_num(fair_self); ft_puts(" self-grade-detected(old race)=" as *u8); ft_num(biased_self); ft_puts(" valid(fair)=" as *u8); ft_num(valid_fair); ft_puts(" valid(self-graded)=" as *u8); ft_num(valid_biased); ft_puts("\n" as *u8) 56 ft_puts(" verdict 3v2 on 3 cases = " as *u8); ft_num(vsmall); ft_puts(" (0=TIE, too few cases) 30v20 at scale = " as *u8); ft_num(vscale); ft_puts(" (1=A_WINS, decisive)\n" as *u8) 57 58 let r: *i64 = sys_mmap(8*8) as *i64 59 r[0] = 0; if sa == 3 { r[0] = 1 } // team scored objectively vs oracle 60 r[1] = 0; if sb == 2 { r[1] = 1 } // claude scored objectively vs oracle 61 r[2] = 0; if deterministic == 1 { r[2] = 1 } // re-scoring is identical (no flaky harness) 62 r[3] = 0; if fair_self == 1 { if biased_self == 0 { r[3] = 1 } } // separation gate works 63 r[4] = 0; if valid_fair == 1 { if valid_biased == 0 { r[4] = 1 } } // self-graded race flagged INVALID 64 r[5] = 0; if vsmall == REF_TIE { r[5] = 1 } // no overclaim on a 1-point margin / 3 cases 65 r[6] = 0; if vscale == REF_A_WINS { r[6] = 1 } // decisive only at a real margin 66 var pass: i64 = 0; i = 0 67 while i < 7 { pass = pass + r[i]; i = i + 1 } 68 ft_puts("----\n passed " as *u8); ft_num(pass); ft_puts("/7\n" as *u8) 69 if pass == 7 { ft_puts(" UNBIASED: scored vs a ground-truth oracle, scorer separated from competitors, deterministic, winner only when decisive. The self-assessed race is correctly flagged INVALID -- this is how head-to-heads run now.\n" as *u8); sys_exit(0); return 0 } 70 ft_puts(" FAIL\n" as *u8); sys_exit(1); return 1 71}