nx_research_judgment_bench.nx source
↩ module page · 133 lines · 9083 B
1// nx_research_judgment_bench.nx -- the first AGGREGATE honest measurement of the unified researcher's JUDGMENT.
2// On a LABELED case set, does nxr_assess's verdict correctly separate PUBLISH-worthy reports (grounded + cited +
3// numbers confirmed) from NOT-publish reports (fabricated number / uncited / dangling cite / MIS-CITED)? Reports
4// discrimination accuracy + the two error kinds, with the SAFETY-CRITICAL direction FALSE-PUBLISH (shipping a
5// defective report as PUBLISH) called out. It measures TWO policies side by side: CURRENT (nxr_assess's verdict)
6// and a PROPOSED STRICT policy (any unconfirmed numeric claim is a gap) computed from the SAME out[] slots -- so
7// the value of the fix is shown WITHOUT editing the concurrently-edited nx_research_unified (read-only import).
8// GREEN iff the PROPOSED policy has zero false-publish AND zero false-hold (fix is sound + strictly additive).
9// $0, deterministic, model-free. license_tier: ORIGINAL
10import "nx_syscalls.nx"
11import "nx_research_unified.nx"
12
13func jb_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
14func jb_putn(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)} let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1} sys_write(1,bb,k); return 0 }
15func jb_len(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
16func jb_verdict(v: i64) -> i64 {
17 if v == 0 { jb_puts("PUBLISH" as *u8) } else { if v == 1 { jb_puts("CONTINUE" as *u8) } else { jb_puts("STOP-INC" as *u8) } }
18 return 0
19}
20
21// run one labeled case. expected_pub = 1 if it SHOULD publish, 0 if it should be HELD.
22// st slots: [0]=total [1]=correct_cur [2]=falsePublish_cur [3]=falseHold_cur [4]=correct_strict
23// [5]=falsePublish_strict [6]=falseHold_strict
24func jb_case(ans: *u8, srcv: *i64, nsrc: i64, expected_pub: i64, label: *u8, st: *i64) -> i64 {
25 let out: *i64 = sys_mmap(96) as *i64
26 nxr_assess(ans, jb_len(ans), srcv, nsrc, 0, 3, 1000000, out)
27 // out: [1]=uncited [2]=dangling [3]=numeric [4]=confirmed [5]=discredited [8]=verdict
28 var cur_pub: i64 = 0
29 if out[8] == 0 { cur_pub = 1 }
30 // PROPOSED STRICT: any numeric claim NOT confirmed (discredited OR unverifiable) is a gap
31 var strict_gaps: i64 = out[1] + out[2] + (out[3] - out[4])
32 if out[0] - out[1] < 1 { strict_gaps = strict_gaps + 1 } // VACUOUS: zero grounded content sentences is not publishable
33 var strict_pub: i64 = 0
34 if strict_gaps == 0 { strict_pub = 1 }
35
36 st[0] = st[0] + 1
37 if cur_pub == expected_pub { st[1] = st[1] + 1 } else { if expected_pub == 0 { st[2] = st[2] + 1 } else { st[3] = st[3] + 1 } }
38 if strict_pub == expected_pub { st[4] = st[4] + 1 } else { if expected_pub == 0 { st[5] = st[5] + 1 } else { st[6] = st[6] + 1 } }
39
40 jb_puts(" " as *u8); jb_puts(label)
41 jb_puts(" exp-pub=" as *u8); jb_putn(expected_pub)
42 jb_puts(" cur=" as *u8); jb_putn(cur_pub); jb_puts("(" as *u8); jb_verdict(out[8]); jb_puts(")" as *u8)
43 jb_puts(" strict=" as *u8); jb_putn(strict_pub)
44 if cur_pub != strict_pub { jb_puts(" <== POLICIES DIFFER" as *u8) }
45 jb_puts("\n" as *u8)
46 return 0
47}
48
49func main() -> i64 {
50 let st: *i64 = sys_mmap(96) as *i64
51 var z: i64 = 0
52 while z < 7 { st[z] = 0; z = z + 1 }
53 jb_puts("=== nx_research_judgment_bench: verdict discrimination on a labeled set (current vs proposed-strict) ===\n\n" as *u8)
54
55 let sv: *i64 = sys_mmap(64) as *i64
56
57 // ---- PUBLISH-WORTHY (grounded, cited, numbers confirmed) : expect PUBLISH ----
58 sv[0] = ("The Eiffel Tower stands in Paris, the capital of France." as *u8) as i64
59 jb_case("The Eiffel Tower stands in Paris [1]." as *u8, sv, 1, 1, "P1 clean cited, no numbers " as *u8, st)
60
61 sv[0] = ("Human performance reaches an F1 of 86.8 percent." as *u8) as i64
62 jb_case("Human performance is 86.8 [1]." as *u8, sv, 1, 1, "P2 cited + confirmed number " as *u8, st)
63
64 sv[0] = ("The Eiffel Tower stands in Paris." as *u8) as i64
65 sv[1] = ("The tower was completed in 1889." as *u8) as i64
66 jb_case("The tower stands in Paris [1]. It opened in 1889 [2]." as *u8, sv, 2, 1, "P3 two clean cited sentences " as *u8, st)
67
68 // ---- NOT-PUBLISH (each carries exactly one defect) : expect HOLD ----
69 sv[0] = ("Human performance reaches an F1 of 86.8 percent." as *u8) as i64
70 jb_case("Human performance is 50.0 [1]." as *u8, sv, 1, 0, "N1 fabricated number (adversarial)" as *u8, st)
71
72 sv[0] = ("The Eiffel Tower stands in Paris." as *u8) as i64
73 jb_case("The tower stands in Paris [1]. It is very tall." as *u8, sv, 1, 0, "N2 uncited sentence " as *u8, st)
74
75 sv[0] = ("The Eiffel Tower stands in Paris." as *u8) as i64
76 sv[1] = ("The tower was completed in 1889." as *u8) as i64
77 jb_case("The tower stands in Paris [5]." as *u8, sv, 2, 0, "N3 dangling citation " as *u8, st)
78
79 // N4 the citation-integrity trap: the number is TRUE (in src2) but the claim cites src1, which has NO number.
80 // A naive "is the value confirmed by its cited source?" -> unverifiable (not discredited) -> current PUBLISHES it.
81 sv[0] = ("The Eiffel Tower stands in Paris, with no figures here." as *u8) as i64
82 sv[1] = ("Human performance reaches an F1 of 86.8 percent." as *u8) as i64
83 jb_case("Human performance is 86.8 [1]." as *u8, sv, 2, 0, "N4 MIS-CITE to numberless src (adv)" as *u8, st)
84
85 sv[0] = ("Human performance reaches an F1 of 86.8 percent." as *u8) as i64
86 jb_case("The score is 86.8." as *u8, sv, 1, 0, "N5 uncited number " as *u8, st)
87
88 // N6 SECOND-NUMBER LAUNDERING (adv): first number TRUE, a 2nd FABRICATED number rides the same sentence.
89 sv[0] = ("Human performance reaches an F1 of 86.8 percent." as *u8) as i64
90 jb_case("Human performance is 86.8 with a sample of 999999 [1]." as *u8, sv, 1, 0, "N6 2nd-number laundering (adv) " as *u8, st)
91
92 // M1 COMPOUND: one good claim + one fabricated claim in the same report -> must hold
93 sv[0] = ("The Eiffel Tower stands in Paris." as *u8) as i64
94 sv[1] = ("Human performance reaches an F1 of 86.8 percent." as *u8) as i64
95 jb_case("The tower stands in Paris [1]. Human performance is 50.0 [2]." as *u8, sv, 2, 0, "M1 good+fabricated compound " as *u8, st)
96
97 // C1 citation-only body (citations, no prose) = vacuous-shaped, must hold
98 sv[0] = ("The Eiffel Tower stands in Paris." as *u8) as i64
99 jb_case("[1] [1]." as *u8, sv, 1, 0, "C1 citation-only body " as *u8, st)
100
101 // H1 THOUSANDS-SEPARATOR (false-hold probe): a TRUE cited number written with a comma must still confirm.
102 sv[0] = ("The corpus contains 107785 question answer pairs." as *u8) as i64
103 jb_case("The dataset has 107,785 pairs [1]." as *u8, sv, 1, 1, "H1 comma in true number (publish) " as *u8, st)
104
105 // ---- VACUOUS (defensive-at-boundaries, rule 12): empty / whitespace body must NOT publish ----
106 let emptybuf: *u8 = sys_mmap(8); emptybuf[0] = 0 as u8
107 sv[0] = ("The Eiffel Tower stands in Paris." as *u8) as i64
108 jb_case(emptybuf, sv, 1, 0, "V1 empty body (vacuous, adversarial)" as *u8, st)
109 jb_case(" " as *u8, sv, 1, 0, "V2 whitespace-only body " as *u8, st)
110
111 // ---- hard POSITIVE guard: a real multi-claim report must still PUBLISH (fix must not over-hold) ----
112 sv[0] = ("The Eiffel Tower stands in Paris." as *u8) as i64
113 sv[1] = ("Human performance reaches an F1 of 86.8 percent." as *u8) as i64
114 sv[2] = ("A replication also reports 86.8 for humans." as *u8) as i64
115 jb_case("The tower stands in Paris [1]. Human performance is 86.8 [2]." as *u8, sv, 3, 1, "P4 multi-claim confirmed+corrob " as *u8, st)
116
117 jb_puts("\n-- CURRENT policy (nxr_assess verdict) --\n" as *u8)
118 jb_puts(" correct=" as *u8); jb_putn(st[1]); jb_puts("/" as *u8); jb_putn(st[0])
119 jb_puts(" FALSE-PUBLISH(danger)=" as *u8); jb_putn(st[2])
120 jb_puts(" false-hold=" as *u8); jb_putn(st[3]); jb_puts("\n" as *u8)
121 jb_puts("-- PROPOSED STRICT policy (unconfirmed numeric = gap) --\n" as *u8)
122 jb_puts(" correct=" as *u8); jb_putn(st[4]); jb_puts("/" as *u8); jb_putn(st[0])
123 jb_puts(" FALSE-PUBLISH(danger)=" as *u8); jb_putn(st[5])
124 jb_puts(" false-hold=" as *u8); jb_putn(st[6]); jb_puts("\n\n" as *u8)
125
126 if st[2] > 0 { jb_puts("FINDING: current policy has " as *u8); jb_putn(st[2]); jb_puts(" false-publish -- a defective report reached PUBLISH (see rows marked POLICIES DIFFER above). Fix the source verdict.\n" as *u8) }
127
128 // GREEN iff the PROPOSED policy is sound: zero false-publish AND zero false-hold (fix is safe + strictly additive)
129 if st[5] == 0 { if st[6] == 0 { jb_puts("JUDGMENT-BENCH GREEN -- proposed-strict policy: 0 false-publish, 0 false-hold (fix is sound + additive)\n" as *u8); sys_exit(0) } }
130 jb_puts("JUDGMENT-BENCH RED -- proposed policy imperfect on this set\n" as *u8)
131 sys_exit(1)
132 return 1
133}