nx_paperbench_gate.nx source
↩ module page · 194 lines · 8509 B
1// nx_paperbench_gate.nx -- KAT + neg-controls for the sovereign manuscript grader (AS-0).
2// Proves the two properties that make the composite worth trusting:
3// VOID -- a zero on an INTEGRITY axis cannot be masked by perfect scores elsewhere
4// FLOOR-CAP -- presentation cannot outrun rigour
5// plus fail-closed axis scoring, monotonicity, exact tier boundaries, and a NON-VACUITY
6// probe (a genuinely perfect manuscript CAN reach 1000 -- the ruler is not a rubber stamp).
7// DRY nx_gate_verdict lib (D001 migrate-on-touch).
8import "nx_paperbench.nx"
9import "nx_gate_verdict.nx"
10
11func main() -> i64 {
12 let ctr: *i64 = gv_ctr()
13 gv_head("nx_paperbench -- sovereign manuscript grader (AS-0 ruler)")
14
15 // canonical 7-axis profile: integrity = claim_evidence(0), controls(2), stat_honesty(6)
16 let w: *i64 = sys_mmap(7 * 8) as *i64
17 var i: i64 = 0
18 while i < 7 { w[i] = 1; i = i + 1 }
19 let mk: *i64 = sys_mmap(7 * 8) as *i64
20 i = 0
21 while i < 7 { mk[i] = 0; i = i + 1 }
22 mk[0] = 1; mk[2] = 1; mk[6] = 1
23
24 // T1 axis scoring: 3 of 4 checklist items -> 750
25 var ok1: i64 = 0
26 if pb_axis(3, 4) == 750 { ok1 = 1 }
27 gv_check("T1 pb_axis 3/4 = 750", ok1, ctr)
28
29 // T2 FAIL-CLOSED: an UNMEASURED axis (total=0) scores 0, never full marks
30 var ok2: i64 = 0
31 if pb_axis(0, 0) == 0 { if pb_axis(5, 0) == 0 { ok2 = 1 } }
32 gv_check("T2 unmeasured axis scores 0 (fails closed)", ok2, ctr)
33
34 // T3 clamp: present>total cannot exceed 1000
35 var ok3: i64 = 0
36 if pb_axis(9, 4) == 1000 { ok3 = 1 }
37 gv_check("T3 pb_axis clamps at 1000", ok3, ctr)
38
39 // T4 weighted mean honours weights
40 let a2: *i64 = sys_mmap(2 * 8) as *i64
41 let w2: *i64 = sys_mmap(2 * 8) as *i64
42 a2[0] = 800; a2[1] = 400
43 w2[0] = 1; w2[1] = 1
44 var ok4: i64 = 0
45 if pb_weighted(a2, w2, 2) == 600 {
46 w2[0] = 3; w2[1] = 1
47 if pb_weighted(a2, w2, 2) == 700 { ok4 = 1 }
48 }
49 gv_check("T4 weighted mean 600 then 700 on reweight", ok4, ctr)
50
51 // T5 ***NEG-CONTROL***: ZERO claim-evidence VOIDS the grade even when every other
52 // axis is perfect. A paper whose numbers trace to nothing is not a paper.
53 let av: *i64 = sys_mmap(7 * 8) as *i64
54 i = 0
55 while i < 7 { av[i] = 1000; i = i + 1 }
56 av[0] = 0
57 var ok5: i64 = 0
58 if pb_weighted(av, w, 7) > 800 {
59 if pb_composite(av, w, mk, 7, 250) == 0 { ok5 = 1 }
60 }
61 gv_check("T5 NEG-CONTROL zero claim-evidence voids a 6-of-7-perfect paper", ok5, ctr)
62
63 // T6 FLOOR-CAP: weak controls (200) cap a manuscript whose weighted mean is 885
64 let ac: *i64 = sys_mmap(7 * 8) as *i64
65 i = 0
66 while i < 7 { ac[i] = 1000; i = i + 1 }
67 ac[2] = 200
68 var ok6: i64 = 0
69 if pb_weighted(ac, w, 7) == 885 {
70 if pb_composite(ac, w, mk, 7, 250) == 450 { ok6 = 1 }
71 }
72 gv_check("T6 floor-cap: presentation cannot outrun rigour (885 -> 450)", ok6, ctr)
73
74 // T7 MONOTONIC: raising any axis never lowers the composite
75 let am: *i64 = sys_mmap(7 * 8) as *i64
76 i = 0
77 while i < 7 { am[i] = 500; i = i + 1 }
78 let before: i64 = pb_composite(am, w, mk, 7, 250)
79 am[4] = 900
80 let after1: i64 = pb_composite(am, w, mk, 7, 250)
81 am[2] = 800
82 let after2: i64 = pb_composite(am, w, mk, 7, 250)
83 var ok7: i64 = 0
84 if after1 >= before { if after2 >= after1 { ok7 = 1 } }
85 gv_check("T7 monotonic: improving an axis never lowers the grade", ok7, ctr)
86
87 // T8 tier boundaries are EXACT and caller-owned (thresholds are data, rule 11)
88 let bars: *i64 = sys_mmap(3 * 8) as *i64
89 bars[0] = 300; bars[1] = 600; bars[2] = 850
90 var ok8: i64 = 0
91 if pb_tier(299, bars, 3) == 0 {
92 if pb_tier(300, bars, 3) == 1 {
93 if pb_tier(599, bars, 3) == 1 {
94 if pb_tier(600, bars, 3) == 2 {
95 if pb_tier(850, bars, 3) == 3 { ok8 = 1 }
96 }
97 }
98 }
99 }
100 gv_check("T8 tier boundaries exact at 300/600/850", ok8, ctr)
101
102 // T9 publishable respects the bar; the VOIDED paper is never publishable at any bar
103 var ok9: i64 = 0
104 if pb_publishable(ac, w, mk, 7, 250, 400) == 1 {
105 if pb_publishable(ac, w, mk, 7, 250, 500) == 0 {
106 if pb_publishable(av, w, mk, 7, 250, 1) == 0 { ok9 = 1 }
107 }
108 }
109 gv_check("T9 publishable bar + voided paper never publishable", ok9, ctr)
110
111 // T10 NON-VACUITY: a genuinely perfect manuscript REACHES 1000. Without this the
112 // ruler could pass every test by always returning 0 (synthetic-pass = STUB).
113 let ap: *i64 = sys_mmap(7 * 8) as *i64
114 i = 0
115 while i < 7 { ap[i] = 1000; i = i + 1 }
116 var ok10: i64 = 0
117 if pb_composite(ap, w, mk, 7, 250) == 1000 { ok10 = 1 }
118 gv_check("T10 non-vacuity: a perfect manuscript reaches 1000", ok10, ctr)
119
120 // T11 weakest axis is NAMED (the actionable output, not just a number)
121 var ok11: i64 = 0
122 if pb_weakest(ac, 7) == 2 { ok11 = 1 }
123 gv_check("T11 weakest axis identified = the named next work", ok11, ctr)
124
125 // T12 scan primitives: marker counting + numeric-claim proxy over real text
126 let txt: *u8 = "## Limitations\nrecall 306 vs 254 over n=100 tasks\n## Limitations\n" as *u8
127 var tn: i64 = 0
128 while txt[tn] != (0 as u8) { tn = tn + 1 }
129 var ok12: i64 = 0
130 if pb_find(txt, tn, "## Limitations" as *u8) == 2 {
131 if pb_find(txt, tn, "## Threats" as *u8) == 0 {
132 if pb_digit_runs(txt, tn) == 3 { ok12 = 1 }
133 }
134 }
135 gv_check("T12 scan: marker count 2, absent marker 0, 3 numeric claims", ok12, ctr)
136
137 // T13 claim boundary: bibliography digits must NOT inflate the claim denominator.
138 // Same document, counted whole vs body-only -- the well-cited paper is not punished.
139 let doc: *u8 = "recall 306 over n=100\n## References\n[R1] arXiv:2308.07832 (2023)\n" as *u8
140 var dn: i64 = 0
141 while doc[dn] != (0 as u8) { dn = dn + 1 }
142 let boff: i64 = pb_offset(doc, dn, "## References" as *u8)
143 var ok13: i64 = 0
144 if pb_digit_runs(doc, dn) == 6 {
145 if pb_digit_runs(doc, boff) == 2 {
146 if pb_offset(doc, dn, "## Absent" as *u8) == dn { ok13 = 1 }
147 }
148 }
149 gv_check("T13 claim boundary excludes bibliography (6 whole -> 2 body)", ok13, ctr)
150
151 // T14 REGRESSION: a provenance token must not count as a claim. This is the exact
152 // defect the ruler's first real use exposed -- a fully-cited manuscript graded 500
153 // permille because "[ev:E01]" contributed a phantom second claim per citation.
154 // Here: one genuine claim ("306") cited once must score a FULL 1000, not 500.
155 let cited: *u8 = "semantic recall 306 [ev:E01] at matched precision" as *u8
156 var cn2: i64 = 0
157 while cited[cn2] != (0 as u8) { cn2 = cn2 + 1 }
158 var ok14: i64 = 0
159 if pb_digit_runs(cited, cn2) == 2 {
160 if pb_claims(cited, cn2) == 1 {
161 if pb_axis(pb_find(cited, cn2, "[ev:" as *u8), pb_claims(cited, cn2)) == 1000 { ok14 = 1 }
162 }
163 }
164 gv_check("T14 REGRESSION provenance token is not a claim (fully-cited = 1000)", ok14, ctr)
165
166 // T15 pb_claims still counts genuine UNCITED numbers -- the fix must not blind the
167 // ruler, or a paper could evade claim_evidence by never citing anything at all.
168 let mixed: *u8 = "recall 306 [ev:E01] but latency 42 was not cited" as *u8
169 var mn: i64 = 0
170 while mixed[mn] != (0 as u8) { mn = mn + 1 }
171 var ok15: i64 = 0
172 if pb_claims(mixed, mn) == 2 {
173 if pb_axis(pb_find(mixed, mn, "[ev:" as *u8), pb_claims(mixed, mn)) == 500 { ok15 = 1 }
174 }
175 gv_check("T15 uncited number still counts (fix does not blind the ruler)", ok15, ctr)
176
177 // T16 REGRESSION: verb dispatch must match the WHOLE word. "scan" and "score" share
178 // the prefix "sc"; a two-character test routed `score` into the scan branch, reading
179 // the slack argument as a file path. Invisible locally, caught by a live tools/call
180 // against the deployed binary. A prefix test is not a verb test.
181 var ok16: i64 = 0
182 if pb_is_scan("scan" as *u8) == 1 {
183 if pb_is_scan("score" as *u8) == 0 {
184 if pb_is_scan("sc" as *u8) == 0 {
185 if pb_is_scan("scanner" as *u8) == 0 { ok16 = 1 }
186 }
187 }
188 }
189 gv_check("T16 REGRESSION verb dispatch matches whole word, not prefix", ok16, ctr)
190
191 let rc: i64 = gv_verdict("PAPERBENCH", ctr, "fail-closed axes + VOID and FLOOR-CAP neg-controls + non-vacuity + scan primitives + claim boundary + provenance-token and verb-dispatch regressions")
192 sys_exit(rc)
193 return rc
194}