nx_dr_drbench_cli.nx source
↩ module page · 77 lines · 4755 B
1// nx_dr_drbench_cli.nx -- callable MCP/API surface for the DRBench+DEER scorer (DR-0).
2// Scores a deep-research report from per-item judge flags and emits the field-standard
3// scorecard JSON. Insight recall is the BINDING constraint (13-36% across the field).
4// nx_dr_drbench <insight_csv> <distractor_csv> <factuality_csv> <completeness_csv> <evidence_csv>
5// each = comma-separated 0/1 flags:
6// insight[i]=1 if groundtruth insight i is covered by the report
7// distractor[i]=1 if injected distractor i leaked into the report (bad)
8// factuality[i]=1 if cited claim i is supported
9// completeness[i]=1 if request sub-question i is answered (DEER)
10// evidence[i]=1 if claim i (cited OR uncited, back-tracked) has valid evidence (DEER)
11// e.g. nx_dr_drbench 1,1,0,0,1 0,0,1,0 1,1,1,0 1,1,1,1,0,0 1,0,1
12// -> {"insight_recall_permil":600,"distractor_avoidance_permil":750,...,"overall_permil":686}
13// license_tier: ORIGINAL No hw writes (Rule 26). expect_exit: 0
14import "nx_dr_drbench.nx"
15import "nx_itoa_lib.nx" // shared MSB-first emitter (zero-alloc)
16import "nx_syscalls.nx"
17const K_MAGIC_1024: i64 = 1024
18
19func db_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
20func db_q() -> i64 { let b: *u8 = sys_mmap(1); b[0] = 34 as u8; sys_write(1, b, 1); return 0 }
21func db_key(name: *u8) -> i64 { db_q(); db_puts(name); db_q(); db_puts(":" as *u8); return 0 }
22// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
23// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
24// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
25// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
26func db_num(v: i64) -> i64 { nxi_out(v); return 0 }
27func db_strlen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
28// parse comma-separated non-negative ints; returns count written to out[0..maxn).
29func db_parse_csv(s: *u8, out: *i64, maxn: i64) -> i64 {
30 var cnt: i64 = 0; var cur: i64 = 0; var have: i64 = 0; var i: i64 = 0
31 let n: i64 = db_strlen(s)
32 while i < n {
33 let c: i64 = s[i] as i64
34 if c >= 48 { if c <= 57 { cur = cur * 10 + (c - 48); have = 1 } }
35 if c == 44 { if cnt < maxn { out[cnt] = cur; cnt = cnt + 1 } cur = 0; have = 0 }
36 i = i + 1
37 }
38 if have == 1 { if cnt < maxn { out[cnt] = cur; cnt = cnt + 1 } }
39 return cnt
40}
41
42func main(argc: i64, argv: *i64) -> i64 {
43 if argc < 6 { db_puts("{" as *u8); db_key("error" as *u8); db_q(); db_puts("usage: nx_dr_drbench <insight_csv> <distractor_csv> <factuality_csv> <completeness_csv> <evidence_csv> (0/1 flags)" as *u8); db_q(); db_puts("}\n" as *u8); sys_exit(2); return 2 }
44
45 let ins: *i64 = sys_mmap(K_MAGIC_1024 * 8) as *i64
46 let dis: *i64 = sys_mmap(K_MAGIC_1024 * 8) as *i64
47 let fac: *i64 = sys_mmap(K_MAGIC_1024 * 8) as *i64
48 let com: *i64 = sys_mmap(K_MAGIC_1024 * 8) as *i64
49 let evd: *i64 = sys_mmap(K_MAGIC_1024 * 8) as *i64
50 let ni: i64 = db_parse_csv(argv[1] as *u8, ins, K_MAGIC_1024)
51 let nd: i64 = db_parse_csv(argv[2] as *u8, dis, K_MAGIC_1024)
52 let nf: i64 = db_parse_csv(argv[3] as *u8, fac, K_MAGIC_1024)
53 let nc: i64 = db_parse_csv(argv[4] as *u8, com, K_MAGIC_1024)
54 let ne: i64 = db_parse_csv(argv[5] as *u8, evd, K_MAGIC_1024)
55
56 let m_ins: i64 = dr_frac(ins, ni)
57 let m_dis: i64 = dr_avoidance(dis, nd)
58 let m_fac: i64 = dr_frac(fac, nf)
59 let m_com: i64 = dr_frac(com, nc)
60 let m_evd: i64 = dr_frac(evd, ne)
61 let mv: *i64 = sys_mmap(5 * 8) as *i64
62 mv[0]=m_ins; mv[1]=m_dis; mv[2]=m_fac; mv[3]=m_com; mv[4]=m_evd
63 let overall: i64 = dr_mean(mv, 5)
64
65 db_puts("{" as *u8)
66 db_key("tool" as *u8); db_q(); db_puts("nx_dr_drbench" as *u8); db_q(); db_puts("," as *u8)
67 db_key("insight_recall_permil" as *u8); db_num(m_ins); db_puts("," as *u8)
68 db_key("distractor_avoidance_permil" as *u8); db_num(m_dis); db_puts("," as *u8)
69 db_key("factuality_permil" as *u8); db_num(m_fac); db_puts("," as *u8)
70 db_key("request_completeness_permil" as *u8); db_num(m_com); db_puts("," as *u8)
71 db_key("evidence_validity_permil" as *u8); db_num(m_evd); db_puts("," as *u8)
72 db_key("overall_permil" as *u8); db_num(overall); db_puts("," as *u8)
73 db_key("binding_metric" as *u8); db_q(); db_puts("insight_recall" as *u8); db_q(); db_puts("," as *u8)
74 db_key("note" as *u8); db_q(); db_puts("DRBench recall(binding 13-36pct field)+avoidance+factuality; DEER completeness+evidence-validity; permil=x1000; sovereign integer-exact" as *u8); db_q()
75 db_puts("}\n" as *u8)
76 sys_exit(0); return 0
77}