code wiki / _hdl_build / nx_web_relevance_bench.nx
nx_web_relevance_bench.nx source
↩ module page · 225 lines · 13861 B
1// nx_web_relevance_bench.nx -- MEASURED RELEVANCE for the WEB engine (the SOTA RULER the operator asked for:
2// "state of the art from the first byte up" = MEASURE quality, then climb the rungs the ruler demands). A judged
3// query set driven through the LIVE serve over LOOPBACK (127.0.0.1:18456 /api/search?scope=web) -- exercises the
4// exact production stack (BM25 + R1c title + R1b proximity + R1d coordination + R1e search-page + PageRank
5// authority + host-diversity) as a small HTTP client (no in-process shard mmap -> no fork-RLIMIT wall). Judged by
6// the expected canonical HOST appearing in a result URL. Metrics (integer, Q10=v/1024):
7// hit@1, hit@10, MRR@10 (1024/rank), meanDCG@10 (1024*1024/ilog2_1024(rank+1)).
8// Output goes to BOTH stdout AND knowledge/status/webrelbench.txt (so the result survives even if the edge drops
9// the tools/call response on a >15s run -- read the file via nx_fs). RATCHET: GREEN floor MRR@10>=500; RAISE it as
10// the engine improves, NEVER lower it. license_tier: ORIGINAL
11import "nx_syscalls.nx"
12import "nx_connect.nx" // bounded connect: a raw sys_connect hangs ~127s on a black-holed host
13import "nx_intlog.nx" // ilog2_1024 -- the DCG rung's integer log2
14const WRB_MAGIC_1024: i64 = 1024
15const WRB_MAGIC_262144: i64 = 262144
16
17const WRB_PORT: i64 = 18456
18const WRB_OUTCAP: i64 = 1048576
19
20static wrb_buf: *u8 // output accumulator (flushed to stdout + file at the end)
21static wrb_bo: i64
22
23func wrb_out(s: *u8, n: i64) -> i64 { var i: i64 = 0; while i < n { wrb_buf[wrb_bo] = s[i]; wrb_bo = wrb_bo + 1; i = i + 1 } return 0 }
24func wrb_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
25func wrb_puts(s: *u8) -> i64 { wrb_out(s, wrb_slen(s)); return 0 }
26func wrb_num(v: i64) -> i64 {
27 let bb: *u8 = sys_mmap(28); var m: i64 = v
28 if m < 0 { wrb_out("-" as *u8, 1); m = 0 - m }
29 let t: *u8 = sys_mmap(28); var k: i64 = 0
30 if m == 0 { t[0] = 48 as u8; k = 1 }
31 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
32 var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 }
33 wrb_out(bb, k); return 0
34}
35func wrb_hex(v: i64) -> i64 { if v < 10 { return 48 + v } return 55 + v }
36func wrb_cat(d: *u8, o: i64, s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { d[o + i] = s[i]; i = i + 1 } return o + i }
37func wrb_catb(d: *u8, o: i64, s: *u8, n: i64) -> i64 { var i: i64 = 0; while i < n { d[o + i] = s[i]; i = i + 1 } return o + n }
38func wrb_urlenc(src: *u8, slen: i64, dst: *u8) -> i64 {
39 var o: i64 = 0; var i: i64 = 0
40 while i < slen {
41 let c: i64 = src[i] as i64
42 var keep: i64 = 0
43 if c >= 48 { if c <= 57 { keep = 1 } }
44 if c >= 65 { if c <= 90 { keep = 1 } }
45 if c >= 97 { if c <= 122 { keep = 1 } }
46 if c == 45 { keep = 1 } if c == 46 { keep = 1 } if c == 95 { keep = 1 } if c == 126 { keep = 1 }
47 if keep == 1 { dst[o] = c as u8; o = o + 1 } else {
48 dst[o] = 37 as u8; o = o + 1
49 dst[o] = wrb_hex((c >> 4) & 15) as u8; o = o + 1
50 dst[o] = wrb_hex(c & 15) as u8; o = o + 1
51 }
52 i = i + 1
53 }
54 return o
55}
56func wrb_contains(hay: *u8, hn: i64, ndl: *u8) -> i64 {
57 let nl: i64 = wrb_slen(ndl); if nl == 0 { return 0 }
58 var i: i64 = 0
59 while i + nl <= hn {
60 var m: i64 = 1; var j: i64 = 0
61 while j < nl { if hay[i + j] != ndl[j] { m = 0; j = nl } else { j = j + 1 } }
62 if m == 1 { return 1 }
63 i = i + 1
64 }
65 return 0
66}
67func wrb_qend(resp: *u8, start: i64, bn: i64) -> i64 {
68 var i: i64 = start; var e: i64 = 0 - 1
69 while i < bn { if e < 0 { if resp[i] == (34 as u8) { e = i } } i = i + 1 }
70 if e < 0 { e = bn }
71 return e
72}
73func wrb_fetch(q: *u8, resp: *u8) -> i64 {
74 let enc: *u8 = sys_mmap(wrb_slen(q) * 3 + 16)
75 let el: i64 = wrb_urlenc(q, wrb_slen(q), enc)
76 let req: *u8 = sys_mmap(el + 256)
77 var o: i64 = wrb_cat(req, 0, "GET /api/search?scope=web&q=" as *u8)
78 o = wrb_catb(req, o, enc, el)
79 o = wrb_cat(req, o, "&page=0 HTTP/1.0\r\nHost: nishifamily.com\r\nConnection: close\r\n\r\n" as *u8)
80 let fd: i64 = sys_socket(2, 1, 0); if fd < 0 { return 0 }
81 let sa: *u8 = sys_mmap(16)
82 sa[0]=2 as u8; sa[1]=0 as u8; sa[2]=((WRB_PORT/256)&255) as u8; sa[3]=(WRB_PORT&255) as u8
83 sa[4]=127 as u8; sa[5]=0 as u8; sa[6]=0 as u8; sa[7]=1 as u8
84 var z: i64=8; while z<16 { sa[z]=0 as u8; z=z+1 }
85 if nx_connect_bounded(fd, sa, 16, NX_CONN_DEFAULT_MS) < 0 { sys_close(fd); return 0 }
86 var w: i64 = 0
87 while w < o { let x: i64 = sys_write(fd, ((req as i64)+w) as *u8, o-w); if x <= 0 { w = o } else { w = w + x } }
88 var tot: i64 = 0; var go: i64 = 1
89 while go == 1 { if tot >= WRB_OUTCAP { go = 0 } else { let r: i64 = sys_read(fd, ((resp as i64)+tot) as *u8, WRB_OUTCAP-tot); if r <= 0 { go = 0 } else { tot = tot + r } } }
90 sys_close(fd)
91 var bo: i64 = 0; var j: i64 = 0
92 while j + 3 < tot { if resp[j]==(13 as u8) { if resp[j+1]==(10 as u8) { if resp[j+2]==(13 as u8) { if resp[j+3]==(10 as u8) { bo = j + 4; j = tot } } } } j = j + 1 }
93 var k: i64 = 0
94 while bo + k < tot { resp[k] = resp[bo + k]; k = k + 1 }
95 return k
96}
97func wrb_rank(q: *u8, ehost: *u8) -> i64 {
98 let resp: *u8 = sys_mmap(WRB_OUTCAP + 16)
99 let bn: i64 = wrb_fetch(q, resp)
100 wrb_puts(" [" as *u8); wrb_puts(q); wrb_puts("] want=" as *u8); wrb_puts(ehost)
101 // (class is printed by the caller's per-class summary; the per-query line stays terse)
102 if bn <= 0 { wrb_puts(" MISS(no-response)\n" as *u8); return 0 }
103 let pat: *u8 = "\"host\":\"" as *u8
104 var found: i64 = 0; var idx: i64 = 0; var i: i64 = 0
105 while i + 8 <= bn {
106 var m: i64 = 1; var j: i64 = 0
107 while j < 8 { if resp[i + j] != pat[j] { m = 0; j = 8 } else { j = j + 1 } }
108 if m == 1 {
109 let hs: i64 = i + 8
110 let hz: i64 = wrb_qend(resp, hs, bn)
111 idx = idx + 1
112 if idx <= 3 { wrb_puts(" #" as *u8); wrb_num(idx); wrb_puts("=" as *u8); wrb_out(((resp as i64)+hs) as *u8, hz-hs) }
113 if found == 0 { if wrb_contains(((resp as i64)+hs) as *u8, hz-hs, ehost) == 1 { found = idx } }
114 i = hz
115 } else { i = i + 1 }
116 }
117 if found >= 1 { wrb_puts(" -> rank=" as *u8); wrb_num(found); wrb_puts("\n" as *u8) } else { wrb_puts(" -> MISS\n" as *u8) }
118 return found
119}
120// QUERY CLASSES (2026-07-25 instrument rung): a single blended MRR cannot say WHICH KIND of query a change
121// broke. Every judged query carries a class, and the summary reports per-class metrics as well as the total,
122// so a future regression is ATTRIBUTABLE ("navigational held, natural-language fell") instead of just lower.
123// Judgements are assigned from the OBJECTIVELY canonical host for the query (an official site, the subject's
124// encyclopedia entry) and NEVER from what this engine currently returns -- grading against your own output is
125// how a ruler stops measuring anything. A MISS on a correct judgement is honest signal: usually crawl coverage.
126const WRB_NCLS: i64 = 5
127const WRB_C_NAV: i64 = 0 // navigational: an official site is the unambiguous answer
128const WRB_C_ENT: i64 = 1 // entity: a person/org, canonical profile or encyclopedia entry
129const WRB_C_NL: i64 = 2 // natural language: informational questions -- the measured semantic weak spot
130const WRB_C_INTL: i64 = 3 // multilingual: non-Latin scripts (regression fence for the UTF-8 tokenizer)
131const WRB_C_TECH: i64 = 4 // long-tail technical: narrow subjects, tests depth not popularity
132func wrb_clsname(c: i64) -> *u8 {
133 if c == WRB_C_NAV { return "navigational" as *u8 }
134 if c == WRB_C_ENT { return "entity " as *u8 }
135 if c == WRB_C_NL { return "natural-lang" as *u8 }
136 if c == WRB_C_INTL { return "multilingual" as *u8 }
137 return "tech-longtai" as *u8
138}
139func wrb_one(q: *u8, ehost: *u8, cls: i64, mrr: *i64, dcg: *i64, h1: *i64, h10: *i64, nq: *i64) -> i64 {
140 let r: i64 = wrb_rank(q, ehost)
141 nq[cls] = nq[cls] + 1
142 if r >= 1 {
143 mrr[cls] = mrr[cls] + WRB_MAGIC_1024 / r
144 dcg[cls] = dcg[cls] + (WRB_MAGIC_1024 * WRB_MAGIC_1024) / ilog2_1024(r + 1)
145 h10[cls] = h10[cls] + 1
146 if r == 1 { h1[cls] = h1[cls] + 1 }
147 }
148 return 0
149}
150func main(argc: i64, argv: *i64) -> i64 {
151 wrb_buf = sys_mmap(WRB_MAGIC_262144); wrb_bo = 0
152 wrb_puts("=== nishi WEB-search RELEVANCE ruler (judged set over dp-web-pub- via live serve; hit@1/hit@10/MRR@10/DCG@10) ===\n" as *u8)
153 let mrr: *i64 = sys_mmap(8 * WRB_NCLS + 16) as *i64
154 let dcg: *i64 = sys_mmap(8 * WRB_NCLS + 16) as *i64
155 let h1: *i64 = sys_mmap(8 * WRB_NCLS + 16) as *i64
156 let h10: *i64 = sys_mmap(8 * WRB_NCLS + 16) as *i64
157 let nq: *i64 = sys_mmap(8 * WRB_NCLS + 16) as *i64
158
159 // ---- the ORIGINAL nine (kept verbatim so every pre-2026-07-25 number stays comparable) ----
160 wrb_one("diora baird" as *u8, "en.wikipedia.org" as *u8, WRB_C_ENT, mrr, dcg, h1, h10, nq)
161 wrb_one("julia jav database" as *u8, "javdatabase.com" as *u8, WRB_C_ENT, mrr, dcg, h1, h10, nq)
162 wrb_one("julia kyoka" as *u8, "javdatabase.com" as *u8, WRB_C_ENT, mrr, dcg, h1, h10, nq)
163 wrb_one("malawi household survey" as *u8, "worldbank.org" as *u8, WRB_C_TECH, mrr, dcg, h1, h10, nq)
164 wrb_one("nsf public access repository" as *u8, "nsf.gov" as *u8, WRB_C_NAV, mrr, dcg, h1, h10, nq)
165 wrb_one("arkansas legislature house vote" as *u8, "arkleg.state.ar.us" as *u8, WRB_C_NAV, mrr, dcg, h1, h10, nq)
166 wrb_one("london metropolitan university" as *u8, "londonmet.ac.uk" as *u8, WRB_C_NAV, mrr, dcg, h1, h10, nq)
167 wrb_one("shri kali ashram tantra yoga" as *u8, "shrikali.org" as *u8, WRB_C_ENT, mrr, dcg, h1, h10, nq)
168 wrb_one("how to lower blood pressure" as *u8, "wikipedia.org" as *u8, WRB_C_NL, mrr, dcg, h1, h10, nq)
169 // ---- HELD-OUT additions 2026-07-25: none of these were consulted while tuning the engine ----
170 // navigational: the official site IS the answer; no judgement call involved
171 wrb_one("world health organization" as *u8, "who.int" as *u8, WRB_C_NAV, mrr, dcg, h1, h10, nq)
172 wrb_one("european space agency" as *u8, "esa.int" as *u8, WRB_C_NAV, mrr, dcg, h1, h10, nq)
173 wrb_one("internal revenue service" as *u8, "irs.gov" as *u8, WRB_C_NAV, mrr, dcg, h1, h10, nq)
174 wrb_one("linux kernel archives" as *u8, "kernel.org" as *u8, WRB_C_NAV, mrr, dcg, h1, h10, nq)
175 wrb_one("python programming language" as *u8, "python.org" as *u8, WRB_C_NAV, mrr, dcg, h1, h10, nq)
176 wrb_one("national weather service forecast" as *u8, "weather.gov" as *u8, WRB_C_NAV, mrr, dcg, h1, h10, nq)
177 wrb_one("stanford encyclopedia of philosophy" as *u8, "plato.stanford.edu" as *u8, WRB_C_NAV, mrr, dcg, h1, h10, nq)
178 // entity
179 wrb_one("ada lovelace" as *u8, "wikipedia.org" as *u8, WRB_C_ENT, mrr, dcg, h1, h10, nq)
180 wrb_one("marie curie" as *u8, "wikipedia.org" as *u8, WRB_C_ENT, mrr, dcg, h1, h10, nq)
181 // natural language: the class the engine is MEASURED weak on -- these exist to keep that honest
182 wrb_one("what causes earthquakes" as *u8, "wikipedia.org" as *u8, WRB_C_NL, mrr, dcg, h1, h10, nq)
183 wrb_one("how do vaccines work" as *u8, "wikipedia.org" as *u8, WRB_C_NL, mrr, dcg, h1, h10, nq)
184 wrb_one("symptoms of diabetes" as *u8, "wikipedia.org" as *u8, WRB_C_NL, mrr, dcg, h1, h10, nq)
185 wrb_one("why is the sky blue" as *u8, "wikipedia.org" as *u8, WRB_C_NL, mrr, dcg, h1, h10, nq)
186 // multilingual: regression fence for the UTF-8 tokenizer (Cyrillic fold + CJK bigrams)
187 wrb_one("moskva rossiya" as *u8, "wikipedia.org" as *u8, WRB_C_INTL, mrr, dcg, h1, h10, nq)
188 wrb_one("tokyo japan capital" as *u8, "wikipedia.org" as *u8, WRB_C_INTL, mrr, dcg, h1, h10, nq)
189 // long-tail technical: depth, not popularity
190 wrb_one("bm25 ranking function" as *u8, "wikipedia.org" as *u8, WRB_C_TECH, mrr, dcg, h1, h10, nq)
191 wrb_one("common crawl web dataset" as *u8, "commoncrawl.org" as *u8, WRB_C_TECH, mrr, dcg, h1, h10, nq)
192 wrb_one("inverted index data structure" as *u8, "wikipedia.org" as *u8, WRB_C_TECH, mrr, dcg, h1, h10, nq)
193
194 // PER-CLASS breakdown first (attribution), then the blended totals (the ratchet number)
195 wrb_puts("---- per-class (a regression here names its own cause) ----\n" as *u8)
196 var tm: i64 = 0; var td: i64 = 0; var t1: i64 = 0; var t10: i64 = 0; var tn: i64 = 0
197 var c: i64 = 0
198 while c < WRB_NCLS {
199 if nq[c] > 0 {
200 wrb_puts(" " as *u8); wrb_puts(wrb_clsname(c))
201 wrb_puts(" n=" as *u8); wrb_num(nq[c])
202 wrb_puts(" hit@1=" as *u8); wrb_num(h1[c])
203 wrb_puts(" hit@10=" as *u8); wrb_num(h10[c])
204 wrb_puts(" MRR@10=" as *u8); wrb_num(mrr[c] / nq[c])
205 wrb_puts("\n" as *u8)
206 }
207 tm = tm + mrr[c]; td = td + dcg[c]; t1 = t1 + h1[c]; t10 = t10 + h10[c]; tn = tn + nq[c]
208 c = c + 1
209 }
210 let mrrv: i64 = tm / tn
211 let dcgv: i64 = td / tn
212 wrb_puts("----\nqueries=" as *u8); wrb_num(tn)
213 wrb_puts(" hit@1=" as *u8); wrb_num(t1)
214 wrb_puts(" hit@10=" as *u8); wrb_num(t10)
215 wrb_puts(" MRR@10=" as *u8); wrb_num(mrrv); wrb_puts(" (" as *u8); wrb_num((mrrv * 1000) / WRB_MAGIC_1024); wrb_puts("/1000)" as *u8)
216 wrb_puts(" meanDCG@10=" as *u8); wrb_num(dcgv); wrb_puts("\n" as *u8)
217 var rc: i64 = 1
218 if mrrv >= 500 { wrb_puts("WEB-RELEVANCE-BENCH GREEN (ratchet floor MRR@10>=500; RAISE as the engine improves)\n" as *u8); rc = 0 } else { wrb_puts("WEB-RELEVANCE-BENCH RED (below ratchet floor -- the ruler's whole point: it shows the gap)\n" as *u8) }
219
220 // flush to stdout AND the durable file (survives an edge-dropped tools/call response)
221 sys_write(1, wrb_buf, wrb_bo)
222 let fd: i64 = sys_openat_wr("knowledge/status/webrelbench.txt" as *u8, 420)
223 if fd >= 0 { sys_write(fd, wrb_buf, wrb_bo); sys_close(fd) }
224 return rc
225}