nx_paperbench.nx source
↩ module page · 248 lines · 9845 B
1// nx_paperbench.nx -- SOVEREIGN deterministic MANUSCRIPT GRADER (AS-0: the ruler).
2//
3// WHY THIS EXISTS FIRST: the lane already emits papers (nx_paper_gen) and indexes them
4// (nx_papers_index), but NOTHING measures whether a manuscript meets a real publication
5// standard. Measurement-first doctrine: no rung may claim "arXiv-level" until a ruler
6// exists that could have said NO. This organ is that ruler.
7//
8// GROUNDING (both machine-fetched over sovereign TLS 2026-07-25, not asserted from memory):
9// REFORMS -- Kapoor et al., arXiv:2308.07832v2, "Reporting Standards For Machine Learning
10// Based Science": 32 questions, consensus of 19 researchers across CS/stats/social/biomed.
11// Explicitly intended "for referees when reviewing papers".
12// Pineau et al., arXiv:2003.12206v4 (JMLR), "Improving Reproducibility in Machine Learning
13// Research": the NeurIPS 2019 reproducibility program + the ML Reproducibility Checklist,
14// which is the PROVENANCE of the modern NeurIPS paper checklist.
15// Shape of the composite borrows DRBench's central finding (arXiv:2510.00172): a metric is
16// only useful if a cheap axis CANNOT mask a zero on the binding axis.
17// HONEST: neurips.cc itself REJECTED our TLS handshake (verdict=5), so the NeurIPS checklist
18// was NOT machine-fetched; we cite its peer-reviewed provenance (Pineau) instead of
19// claiming to have read the conference page. Refuse > fabricate.
20//
21// THE SEVEN AXES (caller supplies each as permille; canonical order):
22// 0 claim_evidence every quantitative claim traces to a re-runnable source [INTEGRITY]
23// 1 reproducibility code + data + exact command + determinism/seed + environment
24// 2 controls negative controls / falsification tests that COULD have failed [INTEGRITY]
25// 3 limitations named scope + what this is NOT
26// 4 baseline comparison against an independent external reference
27// 5 data_provenance source named, train/test separation stated, leakage addressed
28// 6 stat_honesty n reported, no n=1 headline, uncertainty stated [INTEGRITY]
29//
30// Integer, deterministic, bit-reproducible. Imports ONLY nx_syscalls = drift-immune.
31// No hw writes (Rule 26). Thresholds are PARAMETERS, never baked (Rule 11).
32//
33// module: nishi-core.research.paperbench
34// depends: nx_syscalls.nx
35// genealogy_id: reforms_2308_07832 + pineau_2003_12206 + drbench_nongameable_composite
36import "nx_syscalls.nx"
37
38// ---------------------------------------------------------------- axis scoring
39
40// permille for one checklist axis: `present` of `total` items satisfied.
41// total<=0 -> 0. An UNMEASURED axis scores ZERO, never full marks: the ruler
42// fails CLOSED, so "we didn't check" can never be laundered into "we passed".
43func pb_axis(present: i64, total: i64) -> i64 {
44 if total <= 0 { return 0 }
45 var p: i64 = present
46 if p < 0 { p = 0 }
47 if p > total { p = total }
48 return (p * 1000) / total
49}
50
51// weighted mean over n axes (permille). sum(weights)<=0 -> 0.
52func pb_weighted(axes: *i64, weights: *i64, n: i64) -> i64 {
53 var num: i64 = 0
54 var den: i64 = 0
55 var i: i64 = 0
56 while i < n {
57 num = num + axes[i] * weights[i]
58 den = den + weights[i]
59 i = i + 1
60 }
61 if den <= 0 { return 0 }
62 return num / den
63}
64
65// the WEAKEST integrity axis (mask[i]==1). No integrity axis -> 1000 (unconstrained).
66func pb_min_masked(axes: *i64, mask: *i64, n: i64) -> i64 {
67 var m: i64 = 1000
68 var any: i64 = 0
69 var i: i64 = 0
70 while i < n {
71 if mask[i] == 1 {
72 any = 1
73 if axes[i] < m { m = axes[i] }
74 }
75 i = i + 1
76 }
77 if any == 0 { return 1000 }
78 return m
79}
80
81// index of the lowest-scoring axis = the named next piece of work.
82func pb_weakest(axes: *i64, n: i64) -> i64 {
83 var bi: i64 = 0
84 var bv: i64 = 1001
85 var i: i64 = 0
86 while i < n {
87 if axes[i] < bv { bv = axes[i]; bi = i }
88 i = i + 1
89 }
90 return bi
91}
92
93// THE COMPOSITE -- non-gameable BY CONSTRUCTION, two rules:
94// (1) VOID: a ZERO on ANY integrity axis returns 0, however good everything else is.
95// A number that traces to nothing is not a result, and polish cannot buy it back.
96// (2) FLOOR-CAP: the grade may exceed the WEAKEST integrity axis by at most `slack`.
97// Presentation can never outrun rigour -- a beautifully written paper with no
98// controls is capped near its controls score, not near its prose score.
99func pb_composite(axes: *i64, weights: *i64, mask: *i64, n: i64, slack: i64) -> i64 {
100 var i: i64 = 0
101 while i < n {
102 if mask[i] == 1 {
103 if axes[i] == 0 { return 0 }
104 }
105 i = i + 1
106 }
107 let wm: i64 = pb_weighted(axes, weights, n)
108 let flr: i64 = pb_min_masked(axes, mask, n)
109 var lim: i64 = flr + slack
110 if lim > 1000 { lim = 1000 }
111 if wm > lim { return lim }
112 return wm
113}
114
115// tier from caller-supplied ASCENDING bars. Thresholds are DATA (Rule 11): the caller
116// owns them, so a future standard change is a config edit, not a recompile.
117// returns 0..nbars (0 = below the lowest bar).
118func pb_tier(composite: i64, bars: *i64, nbars: i64) -> i64 {
119 var t: i64 = 0
120 var i: i64 = 0
121 while i < nbars {
122 if composite >= bars[i] { t = i + 1 }
123 i = i + 1
124 }
125 return t
126}
127
128// publishable iff the composite clears the bar.
129func pb_publishable(axes: *i64, weights: *i64, mask: *i64, n: i64, slack: i64, bar: i64) -> i64 {
130 if pb_composite(axes, weights, mask, n, slack) >= bar { return 1 }
131 return 0
132}
133
134// ------------------------------------------------- manuscript scan primitives
135// Mechanical, non-fragile signals over a manuscript buffer. Shared with the forge
136// and the referee so there is ONE implementation of "does this paper contain X"
137// (DRY, rule 15) rather than a second copy that can drift from the ruler.
138
139// count occurrences of an exact marker (case-sensitive: markers are headings we emit).
140func pb_find(hay: *u8, n: i64, pat: *u8) -> i64 {
141 var pl: i64 = 0
142 while pat[pl] != (0 as u8) { pl = pl + 1 }
143 if pl == 0 { return 0 }
144 var cnt: i64 = 0
145 var i: i64 = 0
146 while i + pl <= n {
147 var k: i64 = 0
148 var hit: i64 = 1
149 while k < pl {
150 if hay[i+k] != pat[k] { hit = 0; k = pl } else { k = k + 1 }
151 }
152 if hit == 1 { cnt = cnt + 1 }
153 i = i + 1
154 }
155 return cnt
156}
157
158// verb dispatch, extracted from the CLI so it is GATE-TESTABLE.
159// ⚠WHY: "scan" and "score" share the prefix "sc". The CLI originally dispatched on the
160// first TWO characters, which silently routed `score` into the scan branch, where the
161// slack argument was read as a file path. It was invisible locally (only `scan` had
162// been exercised) and was caught only by a live tools/call against the DEPLOYED binary.
163// A prefix test is not a verb test. Matches the whole word. Pinned by gate T16.
164func pb_is_scan(v: *u8) -> i64 {
165 if v[0] != (115 as u8) { return 0 }
166 if v[1] != (99 as u8) { return 0 }
167 if v[2] != (97 as u8) { return 0 }
168 if v[3] != (110 as u8) { return 0 }
169 if v[4] != (0 as u8) { return 0 }
170 return 1
171}
172
173// offset of the FIRST occurrence of a marker, or n if absent. Used to bound claim
174// counting to the BODY: a bibliography is dense with digits (years, arXiv ids, page
175// numbers) that are citations, not claims, and letting them inflate the denominator
176// would punish a well-cited paper for being well-cited.
177func pb_offset(hay: *u8, n: i64, pat: *u8) -> i64 {
178 var pl: i64 = 0
179 while pat[pl] != (0 as u8) { pl = pl + 1 }
180 if pl == 0 { return n }
181 var i: i64 = 0
182 while i + pl <= n {
183 var k: i64 = 0
184 var hit: i64 = 1
185 while k < pl {
186 if hay[i+k] != pat[k] { hit = 0; k = pl } else { k = k + 1 }
187 }
188 if hit == 1 { return i }
189 i = i + 1
190 }
191 return n
192}
193
194// count NUMERIC CLAIMS, EXCLUDING digits inside a provenance token "[ev:...]".
195// ⚠WHY THIS EXISTS: the first real use of this ruler graded a fully-cited manuscript at
196// exactly 500 permille claim_evidence -- precisely 2x too harsh -- because every expansion
197// writes "<value> [ev:E01]" and the token's OWN identifier digits were counted as a second
198// numeric claim. The ruler was punishing a paper for citing, which is exactly backwards.
199// A provenance token is metadata ABOUT a claim, never another claim. Pinned by gate T14.
200func pb_claims(buf: *u8, n: i64) -> i64 {
201 var cnt: i64 = 0
202 var i: i64 = 0
203 var inrun: i64 = 0
204 var skip: i64 = 0
205 while i < n {
206 if skip == 0 {
207 if buf[i] == (91 as u8) {
208 if i + 3 < n {
209 if buf[i+1] == (101 as u8) {
210 if buf[i+2] == (118 as u8) {
211 if buf[i+3] == (58 as u8) { skip = 1; inrun = 0 }
212 }
213 }
214 }
215 }
216 }
217 if skip == 1 {
218 if buf[i] == (93 as u8) { skip = 0; inrun = 0 }
219 } else {
220 let c: i64 = buf[i] as i64
221 var isd: i64 = 0
222 if c >= 48 { if c <= 57 { isd = 1 } }
223 if isd == 1 {
224 if inrun == 0 { cnt = cnt + 1; inrun = 1 }
225 } else { inrun = 0 }
226 }
227 i = i + 1
228 }
229 return cnt
230}
231
232// raw digit-run count, provenance-blind. Kept as the primitive pb_claims is defined
233// against, and used where the bare count is what is wanted.
234func pb_digit_runs(buf: *u8, n: i64) -> i64 {
235 var cnt: i64 = 0
236 var i: i64 = 0
237 var inrun: i64 = 0
238 while i < n {
239 let c: i64 = buf[i] as i64
240 var isd: i64 = 0
241 if c >= 48 { if c <= 57 { isd = 1 } }
242 if isd == 1 {
243 if inrun == 0 { cnt = cnt + 1; inrun = 1 }
244 } else { inrun = 0 }
245 i = i + 1
246 }
247 return cnt
248}