`nx_qa_score.nx` source

1// nx_qa_score.nx -- KAT gate for the sovereign word-level F1 + Exact-Match scorer (the standard QA benchmark
2// metric used by DeepResearcher (arXiv 2504.03160) / SQuAD / HotpotQA). The scoring functions now live in
3// nx_qa_score_lib.nx (importable by benchmark harnesses); this file keeps the KAT + the negative control
4// (a wrong answer MUST score 0 -- liar-kill on a rigged "always-full" metric). Integer/no-float, PERMILLE.
5// license_tier: ORIGINAL  expect_exit: 0
6import "nx_qa_score_lib.nx"
7const K_MAGIC_2000: i64 = 2000
8const K_MAGIC_4000: i64 = 4000
9
10func qw(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
11func qn(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m} let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1} sys_write(1,bb,k); return 0 }
12
13func kat(pred: *u8, gold: *u8, exp_f1: i64, exp_em: i64, tot: *i64) -> i64 {
14    let f: i64=qs_f1(pred,gold); let e: i64=qs_em(pred,gold)
15    var okf: i64=0; if f==exp_f1 { okf=1 }
16    var oke: i64=0; if e==exp_em { oke=1 }
17    qw("  F1(permille)="); qn(f); qw(" EM="); qn(e); qw("   pred='"); qw(pred); qw("' gold='"); qw(gold); qw("'")
18    if okf==1 { if oke==1 { qw("   PASS"); tot[0]=tot[0]+1 } else { qw("   FAIL(EM exp "); qn(exp_em); qw(")") } }
19    else { qw("   FAIL(F1 exp "); qn(exp_f1); qw(")") }
20    tot[1]=tot[1]+1
21    qw("\n")
22    return 0
23}
24
25func main() -> i64 {
26    let tot: *i64 = sys_mmap(64) as *i64
27    tot[0]=0; tot[1]=0
28    qw("=== nx_qa_score -- sovereign word-level F1 + Exact-Match (SQuAD/DeepResearcher metric), integer permille ===\n")
29    kat("Barack Obama" as *u8, "Obama" as *u8, 666, 0, tot)          // partial: common1, np2 ng1 -> K_MAGIC_2000/3=666
30    kat("the White House" as *u8, "White House" as *u8, 1000, 1, tot) // articles dropped -> exact
31    kat("New York City" as *u8, "New York" as *u8, 800, 0, tot)      // common2, np3 ng2 -> K_MAGIC_4000/5=800
32    kat("42" as *u8, "42" as *u8, 1000, 1, tot)                       // numeric exact
33    kat("Paris" as *u8, "London" as *u8, 0, 0, tot)                  // <== NEG-CONTROL: wrong answer MUST score 0
34    let empt: *u8 = sys_mmap(8); empt[0]=(0 as u8)                    // real empty buffer ("" literal aliases the next literal in nishilang)
35    kat(empt, "Paris" as *u8, 0, 0, tot)                             // empty prediction -> 0 (no free ride)
36    qw("QA-SCORE KAT "); qn(tot[0]); qw("/"); qn(tot[1]); qw("\n")
37    if tot[0]==tot[1] { qw("GREEN -- metric verified; wrong/empty answers score 0 (liar-kill).\n"); return 0 }
38    qw("RED -- metric KAT failed\n")
39    return 1
40}
nx_qa_score.nx source

`nx_qa_score.nx` source
