nx_bench_intent.nx source
↩ module page · 254 lines · 10597 B
1// nx_bench_intent.nx -- intent/facet-aware retrieval metrics, bits-up.
2//
3// module: nishi-core.search.bench.intent
4// depends: fx.nx (Q16.16 + fx_log2), syscalls.nx (scratch)
5// capability: CORE_COMPUTE
6// wired_status: FULLY_WIRED
7//
8// WHY (operator directive 2026-05-29): rankings must not be "ours" -- they
9// must be grounded in standard, up-to-today IR research and be AGNOSTIC
10// (a person / place / thing-in-time entity model), while still satisfying
11// the DIFFERENT intents a person query carries at once: her WORKS from her
12// own SOCIALS, what she's been IN (filmography), and WHERE/WHEN (timeline,
13// age/date). One ranking cannot over-serve one intent at the others' cost.
14//
15// Method (verified from NIST TREC Dynamic Domain Overview + primary papers):
16// * Graded relevance 1-4 (1 marginal .. 4 key result) [TREC-DD 2017]
17// * Subtopic decomposition == FACETS/INTENTS [TREC-DD]
18// * S-recall: fraction of intents covered in top-k [Zhai-Cohen-Lafferty,
19// "Beyond independent relevance", SIGIR 2003]
20// * Cube Test water-filling with per-subtopic MaxHeight cap: rewards
21// covering many intents, caps redundant hits on one [Luo, Wing,
22// Yang & Hearst, "Water filling: the cube test", CIKM 2013]
23// * Derivative commentary capped at grade <= 2 so it can never outrank a
24// primary source at equal relevance (encodes first-hand > commentary
25// inside the standard graded framework).
26//
27// Agnostic by construction: the facet ids below are a person decomposition,
28// but the kernels take (subtopic_id, grade) arrays -- swap the facet table
29// for a place or product and the same math applies. Subtopic WEIGHTS belong
30// in a config table, never hardcoded (Rule #11); these kernels take the
31// caller's grades as given.
32
33import "fx.nx"
34import "syscalls.nx"
35
36// --- agnostic person/place/thing-in-time facet ids (TREC-DD subtopics) ---
37const NX_FACET_IDENTITY: i64 = 0 // bio facts + time/age/where-now
38const NX_FACET_WORKS: i64 = 1 // filmography / what they've been in
39const NX_FACET_OFFICIAL_MEDIA: i64 = 2 // first-hand photos / film stills
40const NX_FACET_INTERVIEWS: i64 = 3 // direct first-hand statements
41const NX_FACET_PRESS: i64 = 4 // professional editorial coverage
42const NX_FACET_SELF_SOCIAL: i64 = 5 // verified own social presence
43const NX_FACET_LONGTAIL: i64 = 6 // archive / forum / foreign-locale
44const NX_FACET_COUNT: i64 = 7
45
46// TREC-DD Cube Test parameters (verified; subtopics weighted equally in
47// vanilla DD). MaxHeight caps how much relevance one subtopic can bank --
48// the diminishing-returns / anti-redundancy knob.
49const NX_CUBE_MAX_HEIGHT: i64 = 5 // TREC 2017 DD: MaxHeight = 5
50
51// Cap a derivative result's grade so commentary never outranks a primary at
52// equal relevance. min(grade, 2) when derivative; unchanged otherwise.
53func nx_bench_cap_derivative_grade(grade: i64, is_derivative: i64) -> i64 {
54 if is_derivative == 1 {
55 if grade > 2 { return 2 }
56 }
57 return grade
58}
59
60// S-recall@k: fraction of facets/intents with at least one relevant
61// (grade >= 1) result in the top-k. Directly answers "did we satisfy the
62// different intents?". Returns Q16.16 in [0, FX_ONE].
63func nx_bench_subtopic_recall(subtopic: *i64, grade: *i64, n: i64, k: i64,
64 n_subtopics: i64) -> i64 {
65 if n_subtopics <= 0 { return 0 }
66 let seen_raw: *u8 = sys_mmap(n_subtopics * 8)
67 let seen: *i64 = seen_raw as *i64
68 var s: i64 = 0
69 while s < n_subtopics { seen[s] = 0; s = s + 1 }
70 var lim: i64 = k
71 if lim > n { lim = n }
72 var i: i64 = 0
73 while i < lim {
74 let st: i64 = subtopic[i]
75 if st >= 0 {
76 if st < n_subtopics {
77 if grade[i] >= 1 { seen[st] = 1 }
78 }
79 }
80 i = i + 1
81 }
82 var covered: i64 = 0
83 s = 0
84 while s < n_subtopics {
85 if seen[s] == 1 { covered = covered + 1 }
86 s = s + 1
87 }
88 return fx_from_frac(covered, n_subtopics)
89}
90
91// Cube Test water-filling coverage (Luo et al. 2013): pour each result's
92// graded relevance into its subtopic's "cube"; a cube holds at most
93// max_height. Overflow is wasted (redundant hits on an already-satisfied
94// intent earn nothing). Returns filled mass / total capacity, Q16.16 in
95// [0, FX_ONE]. High score == broad, non-redundant intent coverage.
96func nx_bench_cube_filled(subtopic: *i64, grade: *i64, n: i64, k: i64,
97 n_subtopics: i64, max_height: i64) -> i64 {
98 if n_subtopics <= 0 { return 0 }
99 if max_height <= 0 { return 0 }
100 let h_raw: *u8 = sys_mmap(n_subtopics * 8)
101 let h: *i64 = h_raw as *i64
102 var s: i64 = 0
103 while s < n_subtopics { h[s] = 0; s = s + 1 }
104 var lim: i64 = k
105 if lim > n { lim = n }
106 var filled: i64 = 0
107 var i: i64 = 0
108 while i < lim {
109 let st: i64 = subtopic[i]
110 if st >= 0 {
111 if st < n_subtopics {
112 let room: i64 = max_height - h[st]
113 if room > 0 {
114 var g: i64 = grade[i]
115 if g > room { g = room } // water-filling cap
116 h[st] = h[st] + g
117 filled = filled + g
118 }
119 }
120 }
121 i = i + 1
122 }
123 let capacity: i64 = n_subtopics * max_height
124 return fx_from_frac(filled, capacity)
125}
126
127// Rank-discounted Cube Test ("gain info FAST"): same water-filling, but each
128// newly-filled unit is discounted by 1/log2(rank+1) so covering intents
129// EARLY scores higher. Normalized by the best case (every early rank fills a
130// fresh cube). Returns Q16.16 in [0, FX_ONE].
131func nx_bench_cube_speed(subtopic: *i64, grade: *i64, n: i64, k: i64,
132 n_subtopics: i64, max_height: i64) -> i64 {
133 if n_subtopics <= 0 { return 0 }
134 if max_height <= 0 { return 0 }
135 let h_raw: *u8 = sys_mmap(n_subtopics * 8)
136 let h: *i64 = h_raw as *i64
137 var s: i64 = 0
138 while s < n_subtopics { h[s] = 0; s = s + 1 }
139 var lim: i64 = k
140 if lim > n { lim = n }
141 var acc: i64 = 0
142 var units: i64 = 0 // total fillable units placed, for ideal calc
143 var i: i64 = 0
144 while i < lim {
145 let st: i64 = subtopic[i]
146 if st >= 0 {
147 if st < n_subtopics {
148 let room: i64 = max_height - h[st]
149 if room > 0 {
150 var g: i64 = grade[i]
151 if g > room { g = room }
152 h[st] = h[st] + g
153 let disc: i64 = fx_div(FX_ONE, fx_log2_int(i + 2))
154 acc = acc + fx_mul(fx_from_int(g), disc)
155 units = units + g
156 }
157 }
158 }
159 i = i + 1
160 }
161 if units <= 0 { return 0 }
162 // Ideal: the same number of filled units placed at the earliest ranks.
163 var ideal: i64 = 0
164 var u: i64 = 0
165 while u < units {
166 let disc: i64 = fx_div(FX_ONE, fx_log2_int(u + 2))
167 ideal = ideal + disc
168 u = u + 1
169 }
170 if ideal == 0 { return 0 }
171 return fx_div(acc, ideal)
172}
173
174// ============================================================================
175// REACH -- the operator cardinal: "S-CLASS EXCEED Google by NOT ignoring the
176// sites they do; build the index with STRONGER REACH because we are not funded
177// by advertising." These three kernels turn that thesis into measured Q16.16
178// gates. They are judged at EQUAL depth k for every engine, so the comparison
179// is fair: in the same number of slots, how much of the real relevant web does
180// each engine actually reach, and how many slots does it instead spend on the
181// ad/commerce noise an ad-funded ranking model is built to surface?
182// ============================================================================
183
184// REACH@k -- fraction of the VERIFIED-RELEVANT gold universe (qr_n docs) a SERP
185// surfaces, DISTINCT, within its own top-k. retr[i] is the gold row id of
186// result i (>=0) or a unique negative (unjudged/off-gold). Repeat surfacings of
187// one gold doc (e.g. a double Wikipedia hit) are de-duped so they never inflate
188// reach. Q16.16 in [0, FX_ONE]. Higher == reaches more of the real web.
189func nx_bench_gold_reach(retr: *i64, n: i64, k: i64, qr_n: i64) -> i64 {
190 if qr_n <= 0 { return 0 }
191 let seen: *i64 = sys_mmap(qr_n * 8) as *i64
192 var r: i64 = 0
193 while r < qr_n { seen[r] = 0; r = r + 1 }
194 var lim: i64 = k
195 if lim > n { lim = n }
196 var hit: i64 = 0
197 var i: i64 = 0
198 while i < lim {
199 let id: i64 = retr[i]
200 if id >= 0 {
201 if seen[id] == 0 { seen[id] = 1; hit = hit + 1 }
202 }
203 i = i + 1
204 }
205 return fx_from_frac(hit, qr_n)
206}
207
208// LONGTAIL REACH@k -- the suppressed-stratum coverage. Of the gold docs in the
209// LONGTAIL facet (archive / forum / foreign-locale / independent bio -- exactly
210// the pages an ad-funded engine has no incentive to surface), what fraction
211// does this SERP reach in top-k? This is the sharp edge of the no-advertising
212// thesis: mainstream SERPs spend their slots on head + commerce and score ~0
213// here. facet[i] is the gold facet of result i (or -1); retr[i] its gold id;
214// lt_total is the count of gold rows whose facet == NX_FACET_LONGTAIL. Q16.16.
215func nx_bench_longtail_reach(facet: *i64, retr: *i64, n: i64, k: i64,
216 qr_n: i64, lt_total: i64) -> i64 {
217 if lt_total <= 0 { return 0 }
218 if qr_n <= 0 { return 0 }
219 let seen: *i64 = sys_mmap(qr_n * 8) as *i64
220 var r: i64 = 0
221 while r < qr_n { seen[r] = 0; r = r + 1 }
222 var lim: i64 = k
223 if lim > n { lim = n }
224 var hit: i64 = 0
225 var i: i64 = 0
226 while i < lim {
227 let id: i64 = retr[i]
228 if facet[i] == NX_FACET_LONGTAIL {
229 if id >= 0 {
230 if seen[id] == 0 { seen[id] = 1; hit = hit + 1 }
231 }
232 }
233 i = i + 1
234 }
235 return fx_from_frac(hit, lt_total)
236}
237
238// OFF-GOLD@k -- the ad-economy waste rate. Fraction of the top-k slots spent on
239// results NOT in the verified-relevant universe (booking agents, speaker
240// bureaus, commerce platforms, bio-spam) -- the slots an ad/commerce ranking
241// model fills that a research-delivery engine does not. retr[i] < 0 marks an
242// off-gold result. LOWER is better. Q16.16 in [0, FX_ONE].
243func nx_bench_offgold_rate(retr: *i64, n: i64, k: i64) -> i64 {
244 var lim: i64 = k
245 if lim > n { lim = n }
246 if lim <= 0 { return 0 }
247 var off: i64 = 0
248 var i: i64 = 0
249 while i < lim {
250 if retr[i] < 0 { off = off + 1 }
251 i = i + 1
252 }
253 return fx_from_frac(off, lim)
254}