nx_bench_harness.nx source
↩ module page · 266 lines · 12306 B
1// nx_bench_harness.nx -- the bits-up search leaderboard engine.
2//
3// module: nishi-core.search.bench.harness
4// depends: fx.nx, syscalls.nx, nx_str.nx, nx_source_tier.nx,
5// nx_bench_metrics.nx, nx_bench_intent.nx
6// capability: CORE_COMPUTE
7// wired_status: FULLY_WIRED
8//
9// WHY (operator cardinal, 2026-05-29): the search engine must S-CLASS EXCEED
10// the incumbents -- sovereignty is never the win. This module is the measured
11// 1:1 head-to-head that proves (or refutes) the exceed:
12//
13// nx_bench_score_serp -- feed ONE engine's ranked SERP + the query's qrels,
14// get ONE NxScorecard across every proven kernel (nDCG, S-recall, Cube
15// filled/speed, primary-source precision, canonical rate, MAP, MRR,
16// altitude). Q16.16 throughout => byte-identical on every machine.
17//
18// nx_bench_nishi_rank -- re-order a SHARED document universe by the Nishi
19// research-delivery policy (primary-source-first + facet diversity). Score
20// "same documents, our ranking" vs "same documents, the incumbent's
21// ranking" and the exceed is a fair, measured re-ranking result -- the
22// retrieval set is identical, only the ordering differs.
23//
24// The four kernels this composes carry their own research citations + KATs;
25// the harness adds no new math, only the wiring + a single data-driven policy.
26
27import "fx.nx"
28import "syscalls.nx"
29import "nx_str.nx"
30import "nx_source_tier.nx"
31import "nx_bench_metrics.nx"
32import "nx_bench_intent.nx"
33import "nx_facet_classify.nx"
34
35// One leaderboard row: every metric the kernels expose, evaluated @k. All
36// ratio/gain fields are Q16.16 (FX_ONE == 1.0). n_results is a plain count.
37struct NxScorecard {
38 ndcg_q16: i64, // rank quality, tier-score as graded gain
39 s_recall_q16: i64, // fraction of the 7 facets covered in top-k
40 cube_filled_q16: i64, // Cube Test water-fill (anti-redundant coverage)
41 cube_speed_q16: i64, // rank-discounted coverage (cover intents EARLY)
42 primary_prec_q16: i64, // fraction of top-k that are first-hand
43 canonical_rate_q16: i64, // distinct canonical items / results (anti-dup)
44 map_q16: i64, // mean average precision over the gold set
45 mrr_q16: i64, // reciprocal rank of first relevant
46 altitude_q16: i64, // composite climbing score (0.5 nDCG + 0.5 prim)
47 n_results: i64,
48 reach_q16: i64, // distinct gold-universe docs reached in top-k
49 longtail_reach_q16: i64, // suppressed-stratum (LONGTAIL facet) coverage
50 offgold_q16: i64, // top-k slots spent on ad/commerce noise (lower=better)
51}
52const NX_SCORECARD_BYTES: i64 = 104 // 13 i64 fields * 8
53
54// Nishi re-rank policy knob (Rule #11 -- data-driven, not a magic number).
55// A candidate that would cover a not-yet-covered facet gets this one-time
56// bonus added to its tier gain; it is what makes the greedy maximize intent
57// coverage (S-recall / Cube) ON TOP OF strict primary-source ordering.
58// 0.5 in Q16.16: enough to float a fresh-facet editorial (0.9) above an
59// already-covered-facet primary (1.0), never enough to float derivative
60// junk (gain 0) above any genuine source.
61const NX_NISHI_DIVERSITY_BONUS: i64 = 32768
62
63// Content-accessibility weight (operator directive 2026-06-10: rank for
64// ENGAGEMENT/information density -- media-heavy + interview + info-dense
65// pages -- not first-hand alone, "like how Yandex prioritizes media sites").
66// Each candidate's nx_facet_content_value (Q16.16 in [0,1]) is fused in at
67// this weight: 0.5 lets a content-rich editorial page (0.9 gain + up to 0.5)
68// overtake a content-thin primary catalog row (1.0 gain) -- a real reorder,
69// not a tiebreak -- while derivative junk (gain 0, longtail content 0.3)
70// still can never float above genuine sources. The reach/exceed gate is the
71// regression net for this knob (proven green at this value 2026-06-10).
72const NX_NISHI_CONTENT_WEIGHT: i64 = 32768
73
74// canonical url length: everything up to a '?' (query/tracking params), with
75// a single trailing '/' dropped. So ".../diorabaird/?hl=en" and
76// ".../diorabaird" canonicalize equal, while ".../name/nm1/" and
77// ".../name/nm1/bio/" stay DISTINCT (substring matching would collide them).
78func nx_bench_canon_len(s: *u8) -> i64 {
79 var n: i64 = nx_str_len(s)
80 let q: i64 = nx_str_chr(s, 0x3F) // '?'
81 if q >= 0 { n = q }
82 if n > 0 {
83 if s[n - 1] == 0x2F { n = n - 1 } // drop one trailing '/'
84 }
85 return n
86}
87
88// exact url equality after canonicalization (query-strip + trailing-slash).
89func nx_bench_url_eq(a: *u8, b: *u8) -> i64 {
90 let la: i64 = nx_bench_canon_len(a)
91 let lb: i64 = nx_bench_canon_len(b)
92 if la != lb { return 0 }
93 var i: i64 = 0
94 while i < la {
95 if a[i] != b[i] { return 0 }
96 i = i + 1
97 }
98 return 1
99}
100
101// qrels lookup: row index whose canonical url equals `url`, or -1.
102func nx_bench_qrels_find(qr_url: **u8, qr_n: i64, url: *u8) -> i64 {
103 var i: i64 = 0
104 while i < qr_n {
105 if nx_bench_url_eq(url, qr_url[i]) == 1 { return i }
106 i = i + 1
107 }
108 return 0 - 1
109}
110
111// Score one engine's ranked SERP against the query's qrels. For each result:
112// * classify the source tier -> graded gain (zeroed if derivative)
113// * bind to its gold judgment -> facet + graded relevance (derivative-capped)
114// * track first-hand + dedup signals
115// then run every kernel @k into `out`. Returns 0.
116func nx_bench_score_serp(serp_url: **u8, serp_title: **u8, serp_n: i64,
117 qr_url: **u8, qr_facet: *i64, qr_grade: *i64,
118 qr_primary: *i64, qr_n: i64,
119 k: i64, out: *NxScorecard) -> i64 {
120 let gain: *i64 = sys_mmap(serp_n * 8) as *i64
121 let facet: *i64 = sys_mmap(serp_n * 8) as *i64
122 let grade: *i64 = sys_mmap(serp_n * 8) as *i64
123 let isprim: *i64 = sys_mmap(serp_n * 8) as *i64
124 let canon: *i64 = sys_mmap(serp_n * 8) as *i64
125 let retr: *i64 = sys_mmap(serp_n * 8) as *i64
126 let v: *NxTierVerdict = sys_mmap(NX_TIER_VERDICT_BYTES) as *NxTierVerdict
127
128 var i: i64 = 0
129 while i < serp_n {
130 nx_tier_classify(serp_url[i], serp_title[i], v)
131 let idx: i64 = nx_bench_qrels_find(qr_url, qr_n, serp_url[i])
132 if idx >= 0 {
133 // JUDGED: the gold qrels are authoritative. Graded relevance is
134 // the nDCG gain. We do NOT zero or cap "derivative" results
135 // (operator cardinal 2026-05-29: additive, no prejudging -- a
136 // commentary/gallery page can carry unique content worth surfacing;
137 // earned demotion of non-additive duplicates is the novelty
138 // kernel's job). First-hand-ness comes from the GOLD label, not the
139 // domain tier (the classifier mis-reads her OWN social as platform).
140 facet[i] = qr_facet[idx]
141 grade[i] = qr_grade[idx]
142 gain[i] = fx_from_int(qr_grade[idx])
143 isprim[i] = qr_primary[idx]
144 canon[i] = idx
145 retr[i] = idx
146 } else {
147 // UNJUDGED: not relevant to any facet (zero gain/grade); fall back
148 // to the live tier classifier for the first-hand signal; a UNIQUE
149 // negative key so it is its own canonical cluster and never matches
150 // a relevant id.
151 facet[i] = 0 - 1
152 grade[i] = 0
153 gain[i] = 0
154 isprim[i] = v.is_primary
155 canon[i] = (0 - 2) - i
156 retr[i] = (0 - 2) - i
157 }
158 i = i + 1
159 }
160
161 // The relevant set is every gold row (each qrels doc is, by construction,
162 // a relevant result for the query).
163 let rel: *i64 = sys_mmap(qr_n * 8) as *i64
164 var r: i64 = 0
165 while r < qr_n { rel[r] = r; r = r + 1 }
166
167 var clim: i64 = k
168 if clim > serp_n { clim = serp_n }
169
170 out.ndcg_q16 = nx_bench_ndcg(gain, serp_n, k)
171 out.s_recall_q16 = nx_bench_subtopic_recall(facet, grade, serp_n, k, NX_FACET_COUNT)
172 out.cube_filled_q16 = nx_bench_cube_filled(facet, grade, serp_n, k, NX_FACET_COUNT, NX_CUBE_MAX_HEIGHT)
173 out.cube_speed_q16 = nx_bench_cube_speed(facet, grade, serp_n, k, NX_FACET_COUNT, NX_CUBE_MAX_HEIGHT)
174 out.primary_prec_q16 = nx_bench_primary_precision_at_k(isprim, serp_n, k)
175 out.canonical_rate_q16 = nx_bench_canonical_rate(canon, clim)
176 out.map_q16 = nx_bench_average_precision(retr, serp_n, rel, qr_n)
177 out.mrr_q16 = nx_bench_reciprocal_rank(retr, serp_n, rel, qr_n)
178 out.altitude_q16 = nx_bench_altitude(out.ndcg_q16, out.primary_prec_q16)
179 out.n_results = serp_n
180
181 // REACH cardinals (operator: exceed by reaching the sites ad-funded engines
182 // ignore). lt_total = how many gold rows live in the suppressed LONGTAIL
183 // facet; the per-result facet[]/retr[] arrays above carry the gold binding.
184 var lt_total: i64 = 0
185 var lf: i64 = 0
186 while lf < qr_n {
187 if qr_facet[lf] == NX_FACET_LONGTAIL { lt_total = lt_total + 1 }
188 lf = lf + 1
189 }
190 out.reach_q16 = nx_bench_gold_reach(retr, serp_n, k, qr_n)
191 out.longtail_reach_q16 = nx_bench_longtail_reach(facet, retr, serp_n, k, qr_n, lt_total)
192 out.offgold_q16 = nx_bench_offgold_rate(retr, serp_n, k)
193 return 0
194}
195
196// Greedy Nishi re-rank over a shared document universe. Writes a permutation
197// of [0, pool_n) into `rank_out`, best first. Policy per result:
198// score = tier_gain + content_value*CONTENT_WEIGHT
199// + (this result covers a fresh facet ? DIVERSITY_BONUS : 0)
200// pool_facet[i] is the facet of pool row i (since SREACH R2: from
201// nx_facet_classify, NOT the gold; -1 still means unjudged). This is
202// the ranking we put up against the incumbent's captured order. Returns 0.
203func nx_bench_nishi_rank(pool_url: **u8, pool_title: **u8, pool_facet: *i64,
204 pool_n: i64, rank_out: *i64) -> i64 {
205 let placed: *i64 = sys_mmap(pool_n * 8) as *i64
206 let gainv: *i64 = sys_mmap(pool_n * 8) as *i64
207 let contv: *i64 = sys_mmap(pool_n * 8) as *i64
208 let covered: *i64 = sys_mmap(NX_FACET_COUNT * 8) as *i64
209 let v: *NxTierVerdict = sys_mmap(NX_TIER_VERDICT_BYTES) as *NxTierVerdict
210
211 var i: i64 = 0
212 while i < pool_n {
213 nx_tier_classify(pool_url[i], pool_title[i], v)
214 gainv[i] = nx_tier_gain(v)
215 // content-accessibility term, weighted (Q16.16 * Q16.16 >> 16)
216 contv[i] = (nx_facet_content_value(pool_url[i], pool_title[i], pool_facet[i]) * NX_NISHI_CONTENT_WEIGHT) >> 16
217 placed[i] = 0
218 i = i + 1
219 }
220 var f: i64 = 0
221 while f < NX_FACET_COUNT { covered[f] = 0; f = f + 1 }
222
223 var out_n: i64 = 0
224 while out_n < pool_n {
225 var best: i64 = 0 - 1
226 var best_score: i64 = 0 - 1
227 var j: i64 = 0
228 while j < pool_n {
229 if placed[j] == 0 {
230 var sc: i64 = gainv[j] + contv[j]
231 let fct: i64 = pool_facet[j]
232 if fct >= 0 {
233 if fct < NX_FACET_COUNT {
234 if covered[fct] == 0 { sc = sc + NX_NISHI_DIVERSITY_BONUS }
235 }
236 }
237 if sc > best_score {
238 best_score = sc
239 best = j
240 }
241 }
242 j = j + 1
243 }
244 placed[best] = 1
245 rank_out[out_n] = best
246 let bf: i64 = pool_facet[best]
247 if bf >= 0 {
248 if bf < NX_FACET_COUNT { covered[bf] = 1 }
249 }
250 out_n = out_n + 1
251 }
252 return 0
253}
254
255// Strict S-class EXCEED verdict: 1 iff `nishi` beats `incumbent` on the
256// research-delivery cardinals -- strictly higher ranking quality (nDCG),
257// strictly more first-hand results (primary precision), strictly higher
258// composite altitude, AND no regression on intent coverage (S-recall). These
259// are the dimensions the whole thesis rests on; ties on them are NOT a win.
260func nx_bench_exceeds(nishi: *NxScorecard, incumbent: *NxScorecard) -> i64 {
261 if nishi.ndcg_q16 <= incumbent.ndcg_q16 { return 0 }
262 if nishi.primary_prec_q16 <= incumbent.primary_prec_q16 { return 0 }
263 if nishi.altitude_q16 <= incumbent.altitude_q16 { return 0 }
264 if nishi.s_recall_q16 < incumbent.s_recall_q16 { return 0 }
265 return 1
266}