nx_bench_metrics.nx source
↩ module page · 207 lines · 7074 B
1// nx_bench_metrics.nx -- sovereign retrieval-quality metric kernels.
2//
3// module: nishi-core.search.bench.metrics
4// depends: fx.nx (Q16.16 fixed-point + fx_log2), syscalls.nx (scratch mmap)
5// capability: CORE_COMPUTE
6// wired_status: FULLY_WIRED
7//
8// The math behind the "Mount Everest of information delivery" leaderboard:
9// score Nishi 1:1 against the real engines on a chosen turf. Built bits-up
10// in deterministic Q16.16 fixed-point so a result is BYTE-IDENTICAL on every
11// machine -- the two-is-one parity discipline demands that an IEEE-754 port
12// could never satisfy (different libm log2 => different bits).
13//
14// All ratio/gain values are Q16.16 (FX_ONE == 1.0). Ranks + counts are plain
15// integers. Every formula cites its source; goldens are in
16// nx_bench_metrics_test.nx.
17//
18// Citations:
19// DCG/nDCG Jarvelin & Kekalainen, ACM TOIS 2002.
20// MAP/MRR/precision@k/recall@k Manning/Raghavan/Schutze, IIR 2008 ch.8.
21// primary-source precision Nishi anti-Google thesis (docs/00-VISION):
22// first-hand material (the film/photo/record itself) outranks derivative
23// commentary. Encoded as: tier score is the graded gain; derivative
24// results contribute zero gain even when topically relevant.
25
26import "fx.nx"
27import "syscalls.nx"
28
29// ===== rank-discounted gain (graded relevance) =====================
30//
31// DCG@k = sum_{i=1..k} gain[i-1] / log2(i+1) (i is 1-based rank)
32// `gains` are Q16.16 graded gains in ranked order (use source-tier score
33// as the gain to reward primary-source ordering). Returns Q16.16.
34
35func nx_bench_dcg(gains: *i64, n: i64, k: i64) -> i64 {
36 var lim: i64 = k
37 if lim > n { lim = n }
38 var acc: i64 = 0
39 var i: i64 = 0
40 while i < lim {
41 // position i (0-based) is rank i+1; discount = 1/log2((i+1)+1)
42 let disc: i64 = fx_div(FX_ONE, fx_log2_int(i + 2))
43 acc = acc + fx_mul(gains[i], disc)
44 i = i + 1
45 }
46 return acc
47}
48
49// Insertion-sort a COPY of `src` descending into fresh scratch; returns it.
50// (No short-circuit && reliance -- bounds checked explicitly.)
51func nx_bench_sort_desc(src: *i64, n: i64) -> *i64 {
52 let buf_raw: *u8 = sys_mmap(n * 8)
53 let buf: *i64 = buf_raw as *i64
54 var i: i64 = 0
55 while i < n { buf[i] = src[i]; i = i + 1 }
56 var a: i64 = 1
57 while a < n {
58 let key: i64 = buf[a]
59 var b: i64 = a - 1
60 var run: i64 = 1
61 while run == 1 {
62 run = 0
63 if b >= 0 {
64 if buf[b] < key {
65 buf[b + 1] = buf[b]
66 b = b - 1
67 run = 1
68 }
69 }
70 }
71 buf[b + 1] = key
72 a = a + 1
73 }
74 return buf
75}
76
77// nDCG@k = DCG@k(ranked) / DCG@k(ideal), in [0, FX_ONE].
78// Ideal order = the same gains sorted descending. Returns 0 when no gain
79// exists (nothing to normalize against -- not a divide-by-zero).
80func nx_bench_ndcg(gains: *i64, n: i64, k: i64) -> i64 {
81 let actual: i64 = nx_bench_dcg(gains, n, k)
82 let ideal_gains: *i64 = nx_bench_sort_desc(gains, n)
83 let ideal: i64 = nx_bench_dcg(ideal_gains, n, k)
84 if ideal == 0 { return 0 }
85 return fx_div(actual, ideal)
86}
87
88// ===== set / rank metrics over id lists ============================
89//
90// ids are i64 keys (entity hashes or url hashes -- the harness's
91// entity<->url bridge maps both into one key space).
92
93func nx_bench_contains(ids: *i64, n: i64, target: i64) -> i64 {
94 var i: i64 = 0
95 while i < n {
96 if ids[i] == target { return 1 }
97 i = i + 1
98 }
99 return 0
100}
101
102// precision@k = |retrieved[:k] ∩ relevant| / k (Q16.16)
103func nx_bench_precision_at_k(retr: *i64, nret: i64, rel: *i64, nrel: i64,
104 k: i64) -> i64 {
105 if k <= 0 { return 0 }
106 var lim: i64 = k
107 if lim > nret { lim = nret }
108 var hits: i64 = 0
109 var i: i64 = 0
110 while i < lim {
111 if nx_bench_contains(rel, nrel, retr[i]) == 1 { hits = hits + 1 }
112 i = i + 1
113 }
114 return fx_from_frac(hits, k)
115}
116
117// recall@k = |retrieved[:k] ∩ relevant| / |relevant| (Q16.16)
118func nx_bench_recall_at_k(retr: *i64, nret: i64, rel: *i64, nrel: i64,
119 k: i64) -> i64 {
120 if nrel <= 0 { return 0 }
121 var lim: i64 = k
122 if lim > nret { lim = nret }
123 var hits: i64 = 0
124 var i: i64 = 0
125 while i < lim {
126 if nx_bench_contains(rel, nrel, retr[i]) == 1 { hits = hits + 1 }
127 i = i + 1
128 }
129 return fx_from_frac(hits, nrel)
130}
131
132// reciprocal rank = 1 / (rank of first relevant); 0 if none. (Q16.16)
133func nx_bench_reciprocal_rank(retr: *i64, nret: i64, rel: *i64,
134 nrel: i64) -> i64 {
135 var i: i64 = 0
136 while i < nret {
137 if nx_bench_contains(rel, nrel, retr[i]) == 1 {
138 return fx_from_frac(1, i + 1)
139 }
140 i = i + 1
141 }
142 return 0
143}
144
145// average precision = mean of precision@i at each relevant hit. (Q16.16)
146func nx_bench_average_precision(retr: *i64, nret: i64, rel: *i64,
147 nrel: i64) -> i64 {
148 if nrel <= 0 { return 0 }
149 var hits: i64 = 0
150 var acc: i64 = 0
151 var i: i64 = 0
152 while i < nret {
153 if nx_bench_contains(rel, nrel, retr[i]) == 1 {
154 hits = hits + 1
155 acc = acc + fx_from_frac(hits, i + 1) // precision@(i+1)
156 }
157 i = i + 1
158 }
159 return acc / nrel
160}
161
162// ===== research-delivery metrics (the Nishi thesis) ================
163
164// primary-source precision@k = fraction of top-k that are FIRST-HAND.
165// is_primary[i] != 0 iff result i is a primary source (her actual film/
166// photo/record/interview) vs derivative commentary. This is the metric
167// that operationalizes "first-hand outranks reaction/listicle". (Q16.16)
168func nx_bench_primary_precision_at_k(is_primary: *i64, n: i64, k: i64) -> i64 {
169 var lim: i64 = k
170 if lim > n { lim = n }
171 if lim <= 0 { return 0 }
172 var hits: i64 = 0
173 var i: i64 = 0
174 while i < lim {
175 if is_primary[i] != 0 { hits = hits + 1 }
176 i = i + 1
177 }
178 return fx_from_frac(hits, lim)
179}
180
181// canonical rate = distinct canonical items / total results. 1.0 means no
182// near-duplicate padding; a low rate means the engine repeated the same
183// item (anti-dedup). canon[i] is the dedup-cluster id of result i. (Q16.16)
184func nx_bench_canonical_rate(canon: *i64, n: i64) -> i64 {
185 if n <= 0 { return 0 }
186 var distinct: i64 = 0
187 var i: i64 = 0
188 while i < n {
189 var seen: i64 = 0
190 var j: i64 = 0
191 while j < i {
192 if canon[j] == canon[i] { seen = 1 }
193 j = j + 1
194 }
195 if seen == 0 { distinct = distinct + 1 }
196 i = i + 1
197 }
198 return fx_from_frac(distinct, n)
199}
200
201// altitude = single climbing score in [0, FX_ONE], primary-source-weighted:
202// half overall ranking quality (nDCG, tier-as-gain), half first-hand
203// precision. Weights explicit, not buried (Rule #11).
204func nx_bench_altitude(ndcg_q16: i64, primary_prec_q16: i64) -> i64 {
205 // 0.5*ndcg + 0.5*primary = (ndcg + primary) / 2
206 return (ndcg_q16 + primary_prec_q16) / 2
207}