nx_qa_score_lib.nx source
↩ module page · 111 lines · 4344 B
1// nx_qa_score_lib.nx -- IMPORTABLE sovereign word-level F1 + Exact-Match (SQuAD/HotpotQA/DeepResearcher metric).
2// Split out of nx_qa_score.nx (which keeps the KAT main + neg-control) so benchmark harnesses can import
3// qs_f1/qs_em without a duplicate main. Integer/no-float: F1 in PERMILLE (0..1000). Normalization = SQuAD:
4// lowercase, punctuation->space, drop articles (a/an/the), collapse ws. Proven by the nx_qa_score KAT gate.
5// license_tier: ORIGINAL
6import "nx_syscalls.nx"
7
8// SQuAD normalize into dst (lowercase; [a-z0-9] kept; else -> space). returns dst length.
9func qs_norm(src: *u8, dst: *u8) -> i64 {
10 var i: i64=0; var j: i64=0
11 while src[i]!=(0 as u8) {
12 var c: i64 = src[i] as i64
13 if c>=65 { if c<=90 { c=c+32 } }
14 var keep: i64=0
15 if c>=97 { if c<=122 { keep=1 } }
16 if c>=48 { if c<=57 { keep=1 } }
17 if keep==1 { dst[j]=(c as u8) } else { dst[j]=(32 as u8) }
18 j=j+1; i=i+1
19 }
20 dst[j]=(0 as u8)
21 return j
22}
23
24// token [off,len) in buf equals "a"/"an"/"the" ?
25func qs_is_article(buf: *u8, off: i64, len: i64) -> i64 {
26 if len==1 { if buf[off]==(97 as u8){ return 1 } }
27 if len==2 { if buf[off]==(97 as u8){ if buf[off+1]==(110 as u8){ return 1 } } }
28 if len==3 { if buf[off]==(116 as u8){ if buf[off+1]==(104 as u8){ if buf[off+2]==(101 as u8){ return 1 } } } }
29 return 0
30}
31
32// tokenize normalized buf -> toff[]/tlen[] (skip empties + articles). returns ntok. (no break: sentinel at blen)
33func qs_tok(buf: *u8, blen: i64, toff: *i64, tlen: *i64, maxn: i64) -> i64 {
34 var n: i64=0
35 var start: i64 = 0-1
36 var i: i64=0
37 while i<=blen {
38 var sp: i64=1
39 if i<blen { if buf[i]!=(32 as u8) { sp=0 } }
40 if sp==0 {
41 if start<0 { start=i }
42 } else {
43 if start>=0 {
44 let ln: i64 = i-start
45 if qs_is_article(buf,start,ln)==0 {
46 if n<maxn { toff[n]=start; tlen[n]=ln; n=n+1 }
47 }
48 start = 0-1
49 }
50 }
51 i=i+1
52 }
53 return n
54}
55
56func qs_tok_eq2(pbuf: *u8, po: i64, pl: i64, gbuf: *u8, go: i64, gl: i64) -> i64 {
57 if pl!=gl { return 0 }
58 var k: i64=0
59 while k<pl { if pbuf[po+k]!=gbuf[go+k] { return 0 } k=k+1 }
60 return 1
61}
62
63// multiset intersection count of pred vs gold tokens (each gold used once).
64func qs_common(pbuf: *u8, ptoff: *i64, ptlen: *i64, np: i64, gbuf: *u8, gtoff: *i64, gtlen: *i64, ng: i64, used: *i64) -> i64 {
65 var g: i64=0; while g<ng { used[g]=0; g=g+1 }
66 var common: i64=0
67 var p: i64=0
68 while p<np {
69 var gg: i64=0
70 var found: i64=0
71 while gg<ng {
72 if found==0 { if used[gg]==0 {
73 if qs_tok_eq2(pbuf,ptoff[p],ptlen[p], gbuf,gtoff[gg],gtlen[gg])==1 { used[gg]=1; common=common+1; found=1 }
74 } }
75 gg=gg+1
76 }
77 p=p+1
78 }
79 return common
80}
81
82// word-level F1 in PERMILLE (0..1000). both-empty -> 1000; one-empty -> 0.
83func qs_f1(pred: *u8, gold: *u8) -> i64 {
84 let pb: *u8=sys_mmap(8192); let gb: *u8=sys_mmap(8192)
85 let pl: i64=qs_norm(pred,pb); let gl: i64=qs_norm(gold,gb)
86 let ptoff: *i64=sys_mmap(8192) as *i64; let ptlen: *i64=sys_mmap(8192) as *i64
87 let gtoff: *i64=sys_mmap(8192) as *i64; let gtlen: *i64=sys_mmap(8192) as *i64
88 let np: i64=qs_tok(pb,pl,ptoff,ptlen,512)
89 let ng: i64=qs_tok(gb,gl,gtoff,gtlen,512)
90 if np==0 { if ng==0 { return 1000 } }
91 if np==0 { return 0 }
92 if ng==0 { return 0 }
93 let used: *i64=sys_mmap(8192) as *i64
94 let common: i64=qs_common(pb,ptoff,ptlen,np, gb,gtoff,gtlen,ng, used)
95 if common==0 { return 0 }
96 return (2000*common)/(np+ng)
97}
98
99// exact match (normalized token sequence identical). both-empty -> 1.
100func qs_em(pred: *u8, gold: *u8) -> i64 {
101 let pb: *u8=sys_mmap(8192); let gb: *u8=sys_mmap(8192)
102 let pl: i64=qs_norm(pred,pb); let gl: i64=qs_norm(gold,gb)
103 let ptoff: *i64=sys_mmap(8192) as *i64; let ptlen: *i64=sys_mmap(8192) as *i64
104 let gtoff: *i64=sys_mmap(8192) as *i64; let gtlen: *i64=sys_mmap(8192) as *i64
105 let np: i64=qs_tok(pb,pl,ptoff,ptlen,512)
106 let ng: i64=qs_tok(gb,gl,gtoff,gtlen,512)
107 if np!=ng { return 0 }
108 var i: i64=0
109 while i<np { if qs_tok_eq2(pb,ptoff[i],ptlen[i], gb,gtoff[i],gtlen[i])==0 { return 0 } i=i+1 }
110 return 1
111}