code wiki / _hdl_build / nx_search_relevance_bench.nx
`nx_search_relevance_bench.nx` source

↩ module page · 101 lines · 5280 B
1// nx_search_relevance_bench.nx -- MEASURED RELEVANCE for nishi search (operator 2026-07-03: "make sure
2// ... we are state of the art" -- SOTA is a measured quality, not a checklist). A judged query set over
3// the REAL library shard (dp-nishifamily.com-pub-, 596 docs): each query names the marker string its
4// correct doc must contain; we run the FULL ranked search and score:
5//   MRR@5 (mean reciprocal rank, Q10: 1024/rank, the sota_mrr.html canon)
6//   mean DCG@5 gain (Q10: 1024*1024/ilog2_1024(rank+1) -- binary judgments, the sota_ndcg.html canon)
7// Prints per-query ranks + aggregates; GREEN floor = MRR >= 768 (0.75) -- a RATCHET: raise it as the
8// engine improves, never lower it. license_tier: ORIGINAL
9import "nx_docportal_search_serve.nx"
10import "nx_itoa_lib.nx"   // shared MSB-first emitter (zero-alloc)
11const K_MAGIC_1024: i64 = 1024
12
13func rb_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
14func rb_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
15// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
16// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
17// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
18// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
19func rb_num(v: i64) -> i64 { nxi_out(v); return 0 }
20func rb_contains(hay: *u8, hn: i64, ndl: *u8) -> i64 {
21    let nl: i64 = rb_len(ndl)
22    if nl == 0 { return 0 }
23    var i: i64 = 0
24    while i + nl <= hn {
25        var m: i64 = 1
26        var j: i64 = 0
27        while j < nl { if hay[i + j] != ndl[j] { m = 0; j = nl } else { j = j + 1 } }
28        if m == 1 { return 1 }
29        i = i + 1
30    }
31    return 0
32}
33// rank (1-based) of the first top-5 result whose doc text contains `marker`; 0 = not found in top 5
34func rb_rank(dom: *u8, q: *u8, marker: *u8) -> i64 {
35    let cids: *i64 = sys_mmap(8 * 8) as *i64
36    let scores: *i64 = sys_mmap(8 * 8) as *i64
37    let n: i64 = dss_search(dom, q, rb_len(q), cids, scores, 5)
38    if n <= 0 { return 0 }
39    let prefix: *u8 = sys_mmap(512); dss_prefix(dom, prefix)
40    let h: *i64 = ss_open(prefix)
41    if (h as i64) == 0 { return 0 }
42    let key: *u8 = sys_mmap(64)
43    let dp: *i64 = sys_mmap(16) as *i64
44    let dl: *i64 = sys_mmap(16) as *i64
45    var i: i64 = 0
46    while i < n {
47        dss_mkkey(cids[i], key)
48        if ss_hget(h, key, dp, dl) == 1 {
49            if rb_contains(dp[0] as *u8, dl[0], marker) == 1 { return i + 1 }
50        }
51        i = i + 1
52    }
53    return 0
54}
55func rb_one(dom: *u8, q: *u8, marker: *u8, mrrsum: *i64, dcgsum: *i64, hits1: *i64, nq: *i64) -> i64 {
56    let r: i64 = rb_rank(dom, q, marker)
57    nq[0] = nq[0] + 1
58    rb_puts("  [" as *u8); rb_puts(q); rb_puts("] rank=" as *u8); rb_num(r)
59    if r >= 1 {
60        mrrsum[0] = mrrsum[0] + K_MAGIC_1024 / r
61        dcgsum[0] = dcgsum[0] + (K_MAGIC_1024 * K_MAGIC_1024) / ilog2_1024(r + 1)
62        if r == 1 { hits1[0] = hits1[0] + 1 }
63        rb_puts("\n" as *u8)
64    } else {
65        rb_puts("  MISS\n" as *u8)
66    }
67    return 0
68}
69
70func main() -> i64 {
71    rb_puts("=== nishi search RELEVANCE bench (judged set over the 596-doc library; MRR@5 + DCG@5, integer) ===\n" as *u8)
72    let dom: *u8 = "nishifamily.com" as *u8
73    let mrrsum: *i64 = sys_mmap(16) as *i64; mrrsum[0] = 0
74    let dcgsum: *i64 = sys_mmap(16) as *i64; dcgsum[0] = 0
75    let hits1: *i64 = sys_mmap(16) as *i64; hits1[0] = 0
76    let nq: *i64 = sys_mmap(16) as *i64; nq[0] = 0
77
78    rb_one(dom, "okapi bm25" as *u8, "Okapi BM25" as *u8, mrrsum, dcgsum, hits1, nq)
79    rb_one(dom, "flashattention memory" as *u8, "FlashAttention" as *u8, mrrsum, dcgsum, hits1, nq)
80    rb_one(dom, "meilisearch" as *u8, "Meilisearch" as *u8, mrrsum, dcgsum, hits1, nq)
81    rb_one(dom, "pagerank algorithm" as *u8, "PageRank" as *u8, mrrsum, dcgsum, hits1, nq)
82    rb_one(dom, "inverted index" as *u8, "Inverted index" as *u8, mrrsum, dcgsum, hits1, nq)
83    rb_one(dom, "web crawler" as *u8, "Web crawler" as *u8, mrrsum, dcgsum, hits1, nq)
84    rb_one(dom, "featured snippet" as *u8, "Featured snippet" as *u8, mrrsum, dcgsum, hits1, nq)
85    rb_one(dom, "knowledge panel" as *u8, "Knowledge panel" as *u8, mrrsum, dcgsum, hits1, nq)
86    rb_one(dom, "typesense" as *u8, "Typesense" as *u8, mrrsum, dcgsum, hits1, nq)
87    rb_one(dom, "duckduckgo privacy" as *u8, "DuckDuckGo" as *u8, mrrsum, dcgsum, hits1, nq)
88    rb_one(dom, "\"ranking function\"" as *u8, "Okapi BM25" as *u8, mrrsum, dcgsum, hits1, nq)
89    rb_one(dom, "stract" as *u8, "Stract" as *u8, mrrsum, dcgsum, hits1, nq)
90
91    let mrr: i64 = mrrsum[0] / nq[0]
92    let dcg: i64 = dcgsum[0] / nq[0]
93    rb_puts("----\nqueries=" as *u8); rb_num(nq[0])
94    rb_puts(" hit@1=" as *u8); rb_num(hits1[0])
95    rb_puts(" MRR@5_q10=" as *u8); rb_num(mrr)
96    rb_puts(" (" as *u8); rb_num((mrr * 1000) / K_MAGIC_1024); rb_puts("/1000)" as *u8)
97    rb_puts(" meanDCG@5_q10=" as *u8); rb_num(dcg); rb_puts("\n" as *u8)
98    if mrr >= 768 { rb_puts("RELEVANCE-BENCH GREEN (ratchet floor MRR>=0.75; raise as the engine improves)\n" as *u8); return 0 }
99    rb_puts("RELEVANCE-BENCH RED (below the ratchet floor)\n" as *u8)
100    return 1
101}
nx_search_relevance_bench.nx source

`nx_search_relevance_bench.nx` source
