nx_dr_drbench.nx source
↩ module page · 49 lines · 2255 B
1// nx_dr_drbench.nx -- SOVEREIGN DRBench + DEER deep-research SCORER / ruler (DR-0).
2// Measurement-first: you cannot honestly claim "exceed Google" without the number.
3// Implements the metrics the field actually scores, deterministically + integer-exact:
4// DRBench (ServiceNow, arXiv:2510.00172): Insight Recall (THE binding constraint --
5// best system 36.5%, GPT-4o 13.2%), Distractor Avoidance (>93% everywhere = easy),
6// Factuality (cited-claim support).
7// DEER (LG AI, arXiv:2512.17776): Request Completeness + Evidence Validity (the two
8// weak spots of every current agent), via back-tracking of cited AND uncited claims.
9// The SCORER is sovereign/deterministic; the MATCHING (does a report insight match a
10// groundtruth insight) is the separate judge stage (lexical/PPMI now via recall lane,
11// no-float LLM escalation later) -- this organ takes the per-item match flags, exactly
12// as nx_recall_eval takes graded gains. Imports ONLY nx_syscalls = drift-immune.
13// No hardware writes (Rule 26).
14//
15// module: nishi-core.research.dr_drbench
16// depends: nx_syscalls.nx
17// genealogy_id: drbench_2026_servicenow + deer_2026_lgai
18// NOTE: DISTINCT from nx_drbench.nx (a HotpotQA multi-hop QA harness) -- eats seq225.
19import "nx_syscalls.nx"
20
21// sum of a count/flag array
22func dr_sum(a: *i64, n: i64) -> i64 {
23 var s: i64 = 0; var i: i64 = 0
24 while i < n { s = s + a[i]; i = i + 1 }
25 return s
26}
27
28// fraction of 1s in permille (0..1000). Guard n<1 -> 0. Used for insight-recall,
29// factuality, request-completeness, evidence-validity.
30func dr_frac(a: *i64, n: i64) -> i64 {
31 if n < 1 { return 0 }
32 return (dr_sum(a, n) * 1000) / n
33}
34
35// DRBench Distractor Avoidance = 1 - (distractors included / total distractors),
36// permille. `included[i]`=1 if the report leaked distractor i (bad). Empty -> 1000
37// (nothing to leak = perfect avoidance).
38func dr_avoidance(included: *i64, n: i64) -> i64 {
39 if n < 1 { return 1000 }
40 return 1000 - (dr_sum(included, n) * 1000) / n
41}
42
43// mean of k permille metrics (the composite overall). Guard k<1 -> 0.
44func dr_mean(vals: *i64, k: i64) -> i64 {
45 if k < 1 { return 0 }
46 var s: i64 = 0; var i: i64 = 0
47 while i < k { s = s + vals[i]; i = i + 1 }
48 return s / k
49}