code wiki / _hdl_build / nx_web_relevance_bench.nx
nx_web_relevance_bench.nx source
↩ module page · 273 lines · 16593 B
1// nx_web_relevance_bench.nx -- MEASURED RELEVANCE for the WEB engine (the SOTA RULER the operator asked for:
2// "state of the art from the first byte up" = MEASURE quality, then climb the rungs the ruler demands). A judged
3// query set driven through the LIVE serve over LOOPBACK (127.0.0.1:18456 /api/search?scope=web) -- exercises the
4// exact production stack (BM25 + R1c title + R1b proximity + R1d coordination + R1e search-page + PageRank
5// authority + host-diversity) as a small HTTP client (no in-process shard mmap -> no fork-RLIMIT wall). Judged by
6// the expected canonical HOST appearing in a result URL. Metrics (integer, Q10=v/1024):
7// hit@1, hit@10, MRR@10 (1024/rank), meanDCG@10 (1024*1024/ilog2_1024(rank+1)).
8// Output goes to BOTH stdout AND knowledge/status/webrelbench.txt (so the result survives even if the edge drops
9// the tools/call response on a >15s run -- read the file via nx_fs). RATCHET: GREEN floor MRR@10>=500; RAISE it as
10// the engine improves, NEVER lower it. license_tier: ORIGINAL
11import "nx_syscalls.nx"
12import "nx_connect.nx" // bounded connect: a raw sys_connect hangs ~127s on a black-holed host
13import "nx_intlog.nx" // ilog2_1024 -- the DCG rung's integer log2
14const WRB_MAGIC_1024: i64 = 1024
15const WRB_MAGIC_262144: i64 = 262144
16
17const WRB_PORT: i64 = 18456
18const WRB_OUTCAP: i64 = 1048576
19
20static wrb_buf: *u8 // output accumulator (flushed to stdout + file at the end)
21static wrb_bo: i64
22
23func wrb_out(s: *u8, n: i64) -> i64 { var i: i64 = 0; while i < n { wrb_buf[wrb_bo] = s[i]; wrb_bo = wrb_bo + 1; i = i + 1 } return 0 }
24func wrb_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
25func wrb_puts(s: *u8) -> i64 { wrb_out(s, wrb_slen(s)); return 0 }
26func wrb_num(v: i64) -> i64 {
27 let bb: *u8 = sys_mmap(28); var m: i64 = v
28 if m < 0 { wrb_out("-" as *u8, 1); m = 0 - m }
29 let t: *u8 = sys_mmap(28); var k: i64 = 0
30 if m == 0 { t[0] = 48 as u8; k = 1 }
31 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
32 var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 }
33 wrb_out(bb, k); return 0
34}
35func wrb_hex(v: i64) -> i64 { if v < 10 { return 48 + v } return 55 + v }
36func wrb_cat(d: *u8, o: i64, s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { d[o + i] = s[i]; i = i + 1 } return o + i }
37func wrb_catb(d: *u8, o: i64, s: *u8, n: i64) -> i64 { var i: i64 = 0; while i < n { d[o + i] = s[i]; i = i + 1 } return o + n }
38func wrb_urlenc(src: *u8, slen: i64, dst: *u8) -> i64 {
39 var o: i64 = 0; var i: i64 = 0
40 while i < slen {
41 let c: i64 = src[i] as i64
42 var keep: i64 = 0
43 if c >= 48 { if c <= 57 { keep = 1 } }
44 if c >= 65 { if c <= 90 { keep = 1 } }
45 if c >= 97 { if c <= 122 { keep = 1 } }
46 if c == 45 { keep = 1 } if c == 46 { keep = 1 } if c == 95 { keep = 1 } if c == 126 { keep = 1 }
47 if keep == 1 { dst[o] = c as u8; o = o + 1 } else {
48 dst[o] = 37 as u8; o = o + 1
49 dst[o] = wrb_hex((c >> 4) & 15) as u8; o = o + 1
50 dst[o] = wrb_hex(c & 15) as u8; o = o + 1
51 }
52 i = i + 1
53 }
54 return o
55}
56func wrb_contains(hay: *u8, hn: i64, ndl: *u8) -> i64 {
57 let nl: i64 = wrb_slen(ndl); if nl == 0 { return 0 }
58 var i: i64 = 0
59 while i + nl <= hn {
60 var m: i64 = 1; var j: i64 = 0
61 while j < nl { if hay[i + j] != ndl[j] { m = 0; j = nl } else { j = j + 1 } }
62 if m == 1 { return 1 }
63 i = i + 1
64 }
65 return 0
66}
67func wrb_qend(resp: *u8, start: i64, bn: i64) -> i64 {
68 var i: i64 = start; var e: i64 = 0 - 1
69 while i < bn { if e < 0 { if resp[i] == (34 as u8) { e = i } } i = i + 1 }
70 if e < 0 { e = bn }
71 return e
72}
73func wrb_fetch(q: *u8, resp: *u8) -> i64 {
74 let enc: *u8 = sys_mmap(wrb_slen(q) * 3 + 16)
75 let el: i64 = wrb_urlenc(q, wrb_slen(q), enc)
76 let req: *u8 = sys_mmap(el + 256)
77 var o: i64 = wrb_cat(req, 0, "GET /api/search?scope=web&q=" as *u8)
78 o = wrb_catb(req, o, enc, el)
79 o = wrb_cat(req, o, "&page=0 HTTP/1.0\r\nHost: nishifamily.com\r\nConnection: close\r\n\r\n" as *u8)
80 let fd: i64 = sys_socket(2, 1, 0); if fd < 0 { return 0 }
81 let sa: *u8 = sys_mmap(16)
82 sa[0]=2 as u8; sa[1]=0 as u8; sa[2]=((WRB_PORT/256)&255) as u8; sa[3]=(WRB_PORT&255) as u8
83 sa[4]=127 as u8; sa[5]=0 as u8; sa[6]=0 as u8; sa[7]=1 as u8
84 var z: i64=8; while z<16 { sa[z]=0 as u8; z=z+1 }
85 if nx_connect_bounded(fd, sa, 16, NX_CONN_DEFAULT_MS) < 0 { sys_close(fd); return 0 }
86 var w: i64 = 0
87 while w < o { let x: i64 = sys_write(fd, ((req as i64)+w) as *u8, o-w); if x <= 0 { w = o } else { w = w + x } }
88 var tot: i64 = 0; var go: i64 = 1
89 while go == 1 { if tot >= WRB_OUTCAP { go = 0 } else { let r: i64 = sys_read(fd, ((resp as i64)+tot) as *u8, WRB_OUTCAP-tot); if r <= 0 { go = 0 } else { tot = tot + r } } }
90 sys_close(fd)
91 var bo: i64 = 0; var j: i64 = 0
92 while j + 3 < tot { if resp[j]==(13 as u8) { if resp[j+1]==(10 as u8) { if resp[j+2]==(13 as u8) { if resp[j+3]==(10 as u8) { bo = j + 4; j = tot } } } } j = j + 1 }
93 var k: i64 = 0
94 while bo + k < tot { resp[k] = resp[bo + k]; k = k + 1 }
95 return k
96}
97// ---- INDEX FINGERPRINT (2026-08-15) ----------------------------------------------------------
98// LEARNED THE HARD WAY, THIS RULER'S OWN OUTPUT: a ranking change scored 582 -> 580 here and was
99// REVERTED on that number -- and the REVERT then measured 580 as well, identical to the thing it
100// reverted. The delta was never the change. The CORPUS GREW between the runs (crawls were ingesting),
101// so baseline and treatment had never measured the same index, and nothing in the output said so.
102// A ruler that emits two numbers taken against different corpora, with no way to tell, does not
103// measure ranking -- it measures ranking PLUS drift, and attributes all of it to whatever you edited.
104// The fingerprint is FREE and DERIVED, never chosen: every judged query already returns "total", so
105// their sum moves whenever the corpus moves. SAME fp = the two runs are comparable. DIFFERENT fp =
106// UNCOMPARABLE, and a delta across it means nothing in either direction.
107static wrb_idxfp_g: i64
108static wrb_idxq_g: i64
109// unsigned int immediately following `label`; -1 = label absent or no digits (never silently 0)
110func wrb_uint_after(hay: *u8, hn: i64, label: *u8) -> i64 {
111 let ll: i64 = wrb_slen(label)
112 if ll <= 0 { return 0 - 1 }
113 var i: i64 = 0
114 while i + ll <= hn {
115 var m: i64 = 1
116 var j: i64 = 0
117 while j < ll { if hay[i + j] != label[j] { m = 0; j = ll } else { j = j + 1 } }
118 if m == 1 {
119 var v: i64 = 0
120 var any: i64 = 0
121 var k: i64 = i + ll
122 var go: i64 = 1
123 while go == 1 {
124 if k >= hn { go = 0 } else {
125 let ch: i64 = hay[k] as i64
126 if ch >= 48 { if ch <= 57 { v = v * 10 + (ch - 48); any = 1; k = k + 1 } else { go = 0 } } else { go = 0 }
127 }
128 }
129 if any == 1 { return v }
130 return 0 - 1
131 }
132 i = i + 1
133 }
134 return 0 - 1
135}
136func wrb_rank(q: *u8, ehost: *u8) -> i64 {
137 let resp: *u8 = sys_mmap(WRB_OUTCAP + 16)
138 let bn: i64 = wrb_fetch(q, resp)
139 wrb_puts(" [" as *u8); wrb_puts(q); wrb_puts("] want=" as *u8); wrb_puts(ehost)
140 // (class is printed by the caller's per-class summary; the per-query line stays terse)
141 if bn <= 0 { wrb_puts(" MISS(no-response)\n" as *u8); return 0 }
142 // fold this query's corpus-wide match count into the run's index fingerprint (see the note above)
143 let tot: i64 = wrb_uint_after(resp, bn, "\"total\":" as *u8)
144 if tot >= 0 { wrb_idxfp_g = wrb_idxfp_g + tot; wrb_idxq_g = wrb_idxq_g + 1 }
145 let pat: *u8 = "\"host\":\"" as *u8
146 var found: i64 = 0; var idx: i64 = 0; var i: i64 = 0
147 while i + 8 <= bn {
148 var m: i64 = 1; var j: i64 = 0
149 while j < 8 { if resp[i + j] != pat[j] { m = 0; j = 8 } else { j = j + 1 } }
150 if m == 1 {
151 let hs: i64 = i + 8
152 let hz: i64 = wrb_qend(resp, hs, bn)
153 idx = idx + 1
154 if idx <= 3 { wrb_puts(" #" as *u8); wrb_num(idx); wrb_puts("=" as *u8); wrb_out(((resp as i64)+hs) as *u8, hz-hs) }
155 if found == 0 { if wrb_contains(((resp as i64)+hs) as *u8, hz-hs, ehost) == 1 { found = idx } }
156 i = hz
157 } else { i = i + 1 }
158 }
159 if found >= 1 { wrb_puts(" -> rank=" as *u8); wrb_num(found); wrb_puts("\n" as *u8) } else { wrb_puts(" -> MISS\n" as *u8) }
160 return found
161}
162// QUERY CLASSES (2026-07-25 instrument rung): a single blended MRR cannot say WHICH KIND of query a change
163// broke. Every judged query carries a class, and the summary reports per-class metrics as well as the total,
164// so a future regression is ATTRIBUTABLE ("navigational held, natural-language fell") instead of just lower.
165// Judgements are assigned from the OBJECTIVELY canonical host for the query (an official site, the subject's
166// encyclopedia entry) and NEVER from what this engine currently returns -- grading against your own output is
167// how a ruler stops measuring anything. A MISS on a correct judgement is honest signal: usually crawl coverage.
168const WRB_NCLS: i64 = 5
169const WRB_C_NAV: i64 = 0 // navigational: an official site is the unambiguous answer
170const WRB_C_ENT: i64 = 1 // entity: a person/org, canonical profile or encyclopedia entry
171const WRB_C_NL: i64 = 2 // natural language: informational questions -- the measured semantic weak spot
172const WRB_C_INTL: i64 = 3 // multilingual: non-Latin scripts (regression fence for the UTF-8 tokenizer)
173const WRB_C_TECH: i64 = 4 // long-tail technical: narrow subjects, tests depth not popularity
174func wrb_clsname(c: i64) -> *u8 {
175 if c == WRB_C_NAV { return "navigational" as *u8 }
176 if c == WRB_C_ENT { return "entity " as *u8 }
177 if c == WRB_C_NL { return "natural-lang" as *u8 }
178 if c == WRB_C_INTL { return "multilingual" as *u8 }
179 return "tech-longtai" as *u8
180}
181func wrb_one(q: *u8, ehost: *u8, cls: i64, mrr: *i64, dcg: *i64, h1: *i64, h10: *i64, nq: *i64) -> i64 {
182 let r: i64 = wrb_rank(q, ehost)
183 nq[cls] = nq[cls] + 1
184 if r >= 1 {
185 mrr[cls] = mrr[cls] + WRB_MAGIC_1024 / r
186 dcg[cls] = dcg[cls] + (WRB_MAGIC_1024 * WRB_MAGIC_1024) / ilog2_1024(r + 1)
187 h10[cls] = h10[cls] + 1
188 if r == 1 { h1[cls] = h1[cls] + 1 }
189 }
190 return 0
191}
192func main(argc: i64, argv: *i64) -> i64 {
193 wrb_buf = sys_mmap(WRB_MAGIC_262144); wrb_bo = 0
194 wrb_puts("=== nishi WEB-search RELEVANCE ruler (judged set over dp-web-pub- via live serve; hit@1/hit@10/MRR@10/DCG@10) ===\n" as *u8)
195 let mrr: *i64 = sys_mmap(8 * WRB_NCLS + 16) as *i64
196 let dcg: *i64 = sys_mmap(8 * WRB_NCLS + 16) as *i64
197 let h1: *i64 = sys_mmap(8 * WRB_NCLS + 16) as *i64
198 let h10: *i64 = sys_mmap(8 * WRB_NCLS + 16) as *i64
199 let nq: *i64 = sys_mmap(8 * WRB_NCLS + 16) as *i64
200
201 // ---- the ORIGINAL nine (kept verbatim so every pre-2026-07-25 number stays comparable) ----
202 wrb_one("diora baird" as *u8, "en.wikipedia.org" as *u8, WRB_C_ENT, mrr, dcg, h1, h10, nq)
203 wrb_one("julia jav database" as *u8, "javdatabase.com" as *u8, WRB_C_ENT, mrr, dcg, h1, h10, nq)
204 wrb_one("julia kyoka" as *u8, "javdatabase.com" as *u8, WRB_C_ENT, mrr, dcg, h1, h10, nq)
205 wrb_one("malawi household survey" as *u8, "worldbank.org" as *u8, WRB_C_TECH, mrr, dcg, h1, h10, nq)
206 wrb_one("nsf public access repository" as *u8, "nsf.gov" as *u8, WRB_C_NAV, mrr, dcg, h1, h10, nq)
207 wrb_one("arkansas legislature house vote" as *u8, "arkleg.state.ar.us" as *u8, WRB_C_NAV, mrr, dcg, h1, h10, nq)
208 wrb_one("london metropolitan university" as *u8, "londonmet.ac.uk" as *u8, WRB_C_NAV, mrr, dcg, h1, h10, nq)
209 wrb_one("shri kali ashram tantra yoga" as *u8, "shrikali.org" as *u8, WRB_C_ENT, mrr, dcg, h1, h10, nq)
210 wrb_one("how to lower blood pressure" as *u8, "wikipedia.org" as *u8, WRB_C_NL, mrr, dcg, h1, h10, nq)
211 // ---- HELD-OUT additions 2026-07-25: none of these were consulted while tuning the engine ----
212 // navigational: the official site IS the answer; no judgement call involved
213 wrb_one("world health organization" as *u8, "who.int" as *u8, WRB_C_NAV, mrr, dcg, h1, h10, nq)
214 wrb_one("european space agency" as *u8, "esa.int" as *u8, WRB_C_NAV, mrr, dcg, h1, h10, nq)
215 wrb_one("internal revenue service" as *u8, "irs.gov" as *u8, WRB_C_NAV, mrr, dcg, h1, h10, nq)
216 wrb_one("linux kernel archives" as *u8, "kernel.org" as *u8, WRB_C_NAV, mrr, dcg, h1, h10, nq)
217 wrb_one("python programming language" as *u8, "python.org" as *u8, WRB_C_NAV, mrr, dcg, h1, h10, nq)
218 wrb_one("national weather service forecast" as *u8, "weather.gov" as *u8, WRB_C_NAV, mrr, dcg, h1, h10, nq)
219 wrb_one("stanford encyclopedia of philosophy" as *u8, "plato.stanford.edu" as *u8, WRB_C_NAV, mrr, dcg, h1, h10, nq)
220 // entity
221 wrb_one("ada lovelace" as *u8, "wikipedia.org" as *u8, WRB_C_ENT, mrr, dcg, h1, h10, nq)
222 wrb_one("marie curie" as *u8, "wikipedia.org" as *u8, WRB_C_ENT, mrr, dcg, h1, h10, nq)
223 // natural language: the class the engine is MEASURED weak on -- these exist to keep that honest
224 wrb_one("what causes earthquakes" as *u8, "wikipedia.org" as *u8, WRB_C_NL, mrr, dcg, h1, h10, nq)
225 wrb_one("how do vaccines work" as *u8, "wikipedia.org" as *u8, WRB_C_NL, mrr, dcg, h1, h10, nq)
226 wrb_one("symptoms of diabetes" as *u8, "wikipedia.org" as *u8, WRB_C_NL, mrr, dcg, h1, h10, nq)
227 wrb_one("why is the sky blue" as *u8, "wikipedia.org" as *u8, WRB_C_NL, mrr, dcg, h1, h10, nq)
228 // multilingual: regression fence for the UTF-8 tokenizer (Cyrillic fold + CJK bigrams)
229 wrb_one("moskva rossiya" as *u8, "wikipedia.org" as *u8, WRB_C_INTL, mrr, dcg, h1, h10, nq)
230 wrb_one("tokyo japan capital" as *u8, "wikipedia.org" as *u8, WRB_C_INTL, mrr, dcg, h1, h10, nq)
231 // long-tail technical: depth, not popularity
232 wrb_one("bm25 ranking function" as *u8, "wikipedia.org" as *u8, WRB_C_TECH, mrr, dcg, h1, h10, nq)
233 wrb_one("common crawl web dataset" as *u8, "commoncrawl.org" as *u8, WRB_C_TECH, mrr, dcg, h1, h10, nq)
234 wrb_one("inverted index data structure" as *u8, "wikipedia.org" as *u8, WRB_C_TECH, mrr, dcg, h1, h10, nq)
235
236 // PER-CLASS breakdown first (attribution), then the blended totals (the ratchet number)
237 wrb_puts("---- per-class (a regression here names its own cause) ----\n" as *u8)
238 var tm: i64 = 0; var td: i64 = 0; var t1: i64 = 0; var t10: i64 = 0; var tn: i64 = 0
239 var c: i64 = 0
240 while c < WRB_NCLS {
241 if nq[c] > 0 {
242 wrb_puts(" " as *u8); wrb_puts(wrb_clsname(c))
243 wrb_puts(" n=" as *u8); wrb_num(nq[c])
244 wrb_puts(" hit@1=" as *u8); wrb_num(h1[c])
245 wrb_puts(" hit@10=" as *u8); wrb_num(h10[c])
246 wrb_puts(" MRR@10=" as *u8); wrb_num(mrr[c] / nq[c])
247 wrb_puts("\n" as *u8)
248 }
249 tm = tm + mrr[c]; td = td + dcg[c]; t1 = t1 + h1[c]; t10 = t10 + h10[c]; tn = tn + nq[c]
250 c = c + 1
251 }
252 let mrrv: i64 = tm / tn
253 let dcgv: i64 = td / tn
254 wrb_puts("----\nqueries=" as *u8); wrb_num(tn)
255 wrb_puts(" hit@1=" as *u8); wrb_num(t1)
256 wrb_puts(" hit@10=" as *u8); wrb_num(t10)
257 wrb_puts(" MRR@10=" as *u8); wrb_num(mrrv); wrb_puts(" (" as *u8); wrb_num((mrrv * 1000) / WRB_MAGIC_1024); wrb_puts("/1000)" as *u8)
258 wrb_puts(" meanDCG@10=" as *u8); wrb_num(dcgv); wrb_puts("\n" as *u8)
259 wrb_puts("index_fp=" as *u8); wrb_num(wrb_idxfp_g)
260 wrb_puts(" fp_queries=" as *u8); wrb_num(wrb_idxq_g)
261 wrb_puts(" -- TWO RUNS ARE COMPARABLE ONLY WHEN index_fp MATCHES. A delta measured across differing\n" as *u8)
262 wrb_puts(" index_fp is UNCOMPARABLE and means nothing in either direction: it is ranking PLUS\n" as *u8)
263 wrb_puts(" corpus drift, and attributing all of it to a code change is how a good change gets\n" as *u8)
264 wrb_puts(" reverted and a bad one gets shipped.\n" as *u8)
265 var rc: i64 = 1
266 if mrrv >= 500 { wrb_puts("WEB-RELEVANCE-BENCH GREEN (ratchet floor MRR@10>=500; RAISE as the engine improves)\n" as *u8); rc = 0 } else { wrb_puts("WEB-RELEVANCE-BENCH RED (below ratchet floor -- the ruler's whole point: it shows the gap)\n" as *u8) }
267
268 // flush to stdout AND the durable file (survives an edge-dropped tools/call response)
269 sys_write(1, wrb_buf, wrb_bo)
270 let fd: i64 = sys_openat_wr("knowledge/status/webrelbench.txt" as *u8, 420)
271 if fd >= 0 { sys_write(fd, wrb_buf, wrb_bo); sys_close(fd) }
272 return rc
273}