code wiki / _hdl_build / nx_recall_dense.nx
nx_recall_dense.nx source
↩ module page · 292 lines · 11942 B
1// nx_recall_dense.nx -- the SOVEREIGN dense/semantic reranker over the REAL shipped model.
2// Loads knowledge/index/semppmi_v1.bin (41MB, 102318-word PPMI, built by nx_semppmi_build,
3// probes GREEN) and reranks a candidate shortlist by INTEGER LATE-INTERACTION maxsim soft-
4// alignment: score(query,doc) = SUM over query terms q of MAX over doc terms w of cos_ppmi(q,w)
5// (exact match = identity 1000). This is the ColBERT-class late-interaction pattern in pure
6// fixed-point -- the R2 semantic reranker made REAL (replaces the fixture embeddings the
7// nx_recall_rerank gate used with the actual 102K-vocab count model).
8//
9// EVIDENCE-FORK (no navel-gazing): a CONTROLLED keyword-tie fixture -- each candidate holds
10// EXACTLY ONE query keyword, so first-stage lexical retrieval TIES all four (expected P@k = R/N,
11// a coin-flip). Each relevant doc's OTHER term is a MEASURED synonym of the missing query term
12// (won~defeated=86, city~stadium=62); each distractor's other term is MEASURED-unrelated
13// (won~purple=25, city~january=35). The dense reranker must recover the relevant docs. Every
14// deciding cosine is printed in the JSON = fully auditable, not asserted. RED + exit1 if the
15// dense P@2 fails to beat the lexical coin-flip (an inversion cannot hide).
16// This is a controlled UNIT proof of the mechanism on the real model; corpus-scale nDCG is the
17// flagged BEIR row (F236), not claimed here.
18// expect_exit: 0 license_tier: ORIGINAL
19import "nx_qabench_engine.nx"
20import "nx_estate_path.nx" // ep_anchor: the CWD must not decide this organ verdict
21const K_MAGIC_4096: i64 = 4096
22const K_MAGIC_536870912: i64 = 536870912
23const K_MAGIC_262144: i64 = 262144
24
25func pb_isqrt(v: i64) -> i64 {
26 if v <= 0 { return 0 }
27 var x: i64 = v
28 var y: i64 = (x + 1) / 2
29 while y < x { x = y; let q: i64 = v / x; y = (x + q) / 2 }
30 return x
31}
32
33// g-slots: 70 blob,71 nv,72 nt,73 vh,74 ridx,75 nrm2,76 tctx,77 tval,78 loaded
34func pb_load(g: *i64) -> i64 {
35 g[78] = 0
36 let fd: i64 = sys_openat_rd("knowledge/index/semppmi_v1.bin" as *u8)
37 if fd < 0 { return 0 }
38 let hdrb: *u8 = sys_mmap(K_MAGIC_4096)
39 var hgot: i64 = 0
40 var hr: i64 = 1
41 while hr > 0 { if hgot >= 32 { hr = 0 } else { hr = sys_read(fd, (hdrb as i64 + hgot) as *u8, 32 - hgot); if hr > 0 { hgot = hgot + hr } } }
42 if hgot < 32 { sys_close(fd); return 0 }
43 let hh: *i64 = (hdrb as i64 + 8) as *i64
44 let hnv: i64 = hh[0]
45 let hnt: i64 = hh[1]
46 let need0: i64 = 32 + (hnv*8) + ((hnv+1)*8) + (hnv*8) + (hnt*8) + (hnt*8)
47 if need0 <= 32 { sys_close(fd); return 0 }
48 if need0 > K_MAGIC_536870912 { sys_close(fd); return 0 }
49 let cap: i64 = need0 + K_MAGIC_4096
50 let blob: *u8 = sys_mmap(cap)
51 var i0: i64 = 0
52 while i0 < 32 { blob[i0] = hdrb[i0]; i0 = i0 + 1 }
53 var total: i64 = 32
54 var r: i64 = 1
55 while r > 0 {
56 let left: i64 = need0 - total
57 if left <= 0 { r = 0 } else {
58 var want: i64 = K_MAGIC_262144
59 if want > left { want = left }
60 r = sys_read(fd, (blob as i64 + total) as *u8, want)
61 if r > 0 { total = total + r }
62 }
63 }
64 sys_close(fd)
65 if total < 64 { return 0 }
66 if blob[0] != (78 as u8) { return 0 }
67 if blob[6] != (49 as u8) { return 0 }
68 let hi: *i64 = (blob as i64 + 8) as *i64
69 let nv: i64 = hi[0]
70 let nt: i64 = hi[1]
71 let need: i64 = 32 + (nv*8) + ((nv+1)*8) + (nv*8) + (nt*8) + (nt*8)
72 if total < need { return 0 }
73 g[70] = blob as i64
74 g[71] = nv
75 g[72] = nt
76 var off: i64 = 32
77 g[73] = (blob as i64) + off; off = off + nv*8
78 g[74] = (blob as i64) + off; off = off + (nv+1)*8
79 g[75] = (blob as i64) + off; off = off + nv*8
80 g[76] = (blob as i64) + off; off = off + nt*8
81 g[77] = (blob as i64) + off
82 g[78] = 1
83 return 1
84}
85
86// cosine between two PPMI rows (sorted sparse merge), permille
87func pb_cos(g: *i64, a: i64, b: i64) -> i64 {
88 let ridx: *i64 = g[74] as *i64
89 let nrm2: *i64 = g[75] as *i64
90 let tctx: *i64 = g[76] as *i64
91 let tval: *i64 = g[77] as *i64
92 var ia: i64 = ridx[a]
93 var ib: i64 = ridx[b]
94 let ea: i64 = ridx[a+1]
95 let eb: i64 = ridx[b+1]
96 var dot: i64 = 0
97 while ia < ea {
98 if ib >= eb { ia = ea } else {
99 if tctx[ia] == tctx[ib] { dot = dot + tval[ia]*tval[ib]; ia = ia + 1; ib = ib + 1 }
100 else { if tctx[ia] < tctx[ib] { ia = ia + 1 } else { ib = ib + 1 } }
101 }
102 }
103 if dot <= 0 { return 0 }
104 let d1: i64 = pb_isqrt(nrm2[a])
105 let d2: i64 = pb_isqrt(nrm2[b])
106 if d1 == 0 { return 0 }
107 if d2 == 0 { return 0 }
108 var cv: i64 = (dot*1000)/(d1*d2)
109 if cv > 1000 { cv = 1000 }
110 return cv
111}
112
113// dense cosine with identity + missing-id guard
114func pb_dcos(g: *i64, a: i64, b: i64) -> i64 {
115 if a < 0 { return 0 }
116 if b < 0 { return 0 }
117 if a == b { return 1000 }
118 return pb_cos(g, a, b)
119}
120
121// resolve a null-terminated word literal -> PPMI vocab id or -1
122func pb_wid(g: *i64, s: *u8) -> i64 {
123 var n: i64 = 0
124 while s[n] != (0 as u8) { n = n + 1 }
125 let hv: i64 = db_semhash(s, 0, n)
126 return db_bsearch_i64(g[73] as *i64, g[71], hv)
127}
128
129// maxsim of query term q over a doc's two term ids
130func pb_maxcos(g: *i64, q: i64, w0: i64, w1: i64) -> i64 {
131 var m: i64 = 0
132 let c0: i64 = pb_dcos(g, q, w0)
133 if c0 > m { m = c0 }
134 let c1: i64 = pb_dcos(g, q, w1)
135 if c1 > m { m = c1 }
136 return m
137}
138
139// order indices by score DESC (ties -> lower index). out[n].
140func order_desc(score: *i64, n: i64, out: *i64) -> i64 {
141 let used: *u8 = sys_mmap(n)
142 var i: i64 = 0
143 while i < n { used[i] = 0 as u8; i = i + 1 }
144 var r: i64 = 0
145 while r < n {
146 var best: i64 = 0 - 1
147 var j: i64 = 0
148 while j < n {
149 if used[j] == (0 as u8) {
150 if best < 0 { best = j } else { if score[j] > score[best] { best = j } }
151 }
152 j = j + 1
153 }
154 out[r] = best
155 used[best] = 1 as u8
156 r = r + 1
157 }
158 return 0
159}
160
161// precision@k (permil) for a given order + gold gains
162func p_at_k(order: *i64, gold: *i64, n: i64, k: i64) -> i64 {
163 var rel: i64 = 0
164 var i: i64 = 0
165 while i < k {
166 if i < n { if gold[order[i]] > 0 { rel = rel + 1 } }
167 i = i + 1
168 }
169 return rel * 1000 / k
170}
171
172// reciprocal rank (permil) of first relevant in the order
173func mrr_permil(order: *i64, gold: *i64, n: i64) -> i64 {
174 var i: i64 = 0
175 while i < n {
176 if gold[order[i]] > 0 { return 1000 / (i + 1) }
177 i = i + 1
178 }
179 return 0
180}
181
182// emit "k":v, helper
183func jkv(k: *u8, v: i64) -> i64 { db_w("\"" as *u8); db_w(k); db_w("\":" as *u8); db_n(v); return 0 }
184
185func main() -> i64 {
186 // ANCHOR FIRST (2026-08-04, nx_cwdguard finding): this organ reads a RELATIVE
187 // knowledge/ path, so its answer depended on where it was launched. No-op when
188 // already at the estate root, so the cron/MCP context is unchanged.
189 ep_anchor()
190 let g: *i64 = sys_mmap(128*8) as *i64
191 if pb_load(g) == 0 { db_w("{\"tool\":\"nx_recall_dense\",\"error\":\"semppmi_v1.bin load failed\"}\n" as *u8); return 1 }
192
193 // ---- query + doc fixture (keyword-tie; see header) ----
194 let qa: i64 = pb_wid(g, "won" as *u8)
195 let qb: i64 = pb_wid(g, "city" as *u8)
196 // doc term ids (2 per doc)
197 let wa: *i64 = sys_mmap(4*8) as *i64
198 let wb: *i64 = sys_mmap(4*8) as *i64
199 wa[0] = pb_wid(g, "won" as *u8); wb[0] = pb_wid(g, "stadium" as *u8) // doc0 relevant
200 wa[1] = pb_wid(g, "won" as *u8); wb[1] = pb_wid(g, "january" as *u8) // doc1 distractor
201 wa[2] = pb_wid(g, "defeated" as *u8); wb[2] = pb_wid(g, "city" as *u8) // doc2 relevant
202 wa[3] = pb_wid(g, "purple" as *u8); wb[3] = pb_wid(g, "city" as *u8) // doc3 distractor
203 let gold: *i64 = sys_mmap(4*8) as *i64
204 gold[0] = 2; gold[1] = 0; gold[2] = 2; gold[3] = 0
205
206 // ---- per-doc lexical (exact keyword overlap) + dense (maxsim soft-align) ----
207 let lex: *i64 = sys_mmap(4*8) as *i64
208 let den: *i64 = sys_mmap(4*8) as *i64
209 var R: i64 = 0
210 var lex_tied: i64 = 1
211 var i: i64 = 0
212 while i < 4 {
213 var l: i64 = 0
214 if wa[i] == qa { l = l + 1 } else { if wb[i] == qa { l = l + 1 } }
215 if wa[i] == qb { l = l + 1 } else { if wb[i] == qb { l = l + 1 } }
216 lex[i] = l
217 den[i] = pb_maxcos(g, qa, wa[i], wb[i]) + pb_maxcos(g, qb, wa[i], wb[i])
218 if gold[i] > 0 { R = R + 1 }
219 if l != lex[0] { lex_tied = 0 }
220 i = i + 1
221 }
222 let N: i64 = 4
223
224 // ---- orders ----
225 let dord: *i64 = sys_mmap(4*8) as *i64
226 let lord: *i64 = sys_mmap(4*8) as *i64
227 order_desc(den, 4, dord)
228 order_desc(lex, 4, lord)
229
230 // ---- metrics ----
231 let d_p1: i64 = p_at_k(dord, gold, 4, 1)
232 let d_p2: i64 = p_at_k(dord, gold, 4, 2)
233 let d_mrr: i64 = mrr_permil(dord, gold, 4)
234 // lexical baseline: fully tied => order-independent expected precision = R/N
235 let lex_exp: i64 = R * 1000 / N
236 let l_p1: i64 = p_at_k(lord, gold, 4, 1)
237 let l_p2: i64 = p_at_k(lord, gold, 4, 2)
238
239 // ---- verdict (evidence-fork) ----
240 var verdict_green: i64 = 0
241 if d_p2 > lex_exp { verdict_green = 1 }
242
243 // ================= JSON =================
244 db_w("{\"tool\":\"nx_recall_dense\",\"model\":{" as *u8)
245 jkv("vocab" as *u8, g[71]); db_w("," as *u8); jkv("triples" as *u8, g[72]); db_w("}," as *u8)
246 db_w("\"method\":\"integer late-interaction maxsim over real PPMI-count model (identity=1000)\"," as *u8)
247 // measured cosine matrix (query x doc-vocab) -- fully auditable
248 db_w("\"cos_permil\":{" as *u8)
249 jkv("won_defeated" as *u8, pb_dcos(g, qa, pb_wid(g,"defeated" as *u8))); db_w("," as *u8)
250 jkv("won_purple" as *u8, pb_dcos(g, qa, pb_wid(g,"purple" as *u8))); db_w("," as *u8)
251 jkv("won_stadium" as *u8, pb_dcos(g, qa, pb_wid(g,"stadium" as *u8))); db_w("," as *u8)
252 jkv("won_january" as *u8, pb_dcos(g, qa, pb_wid(g,"january" as *u8))); db_w("," as *u8)
253 jkv("city_stadium" as *u8, pb_dcos(g, qb, pb_wid(g,"stadium" as *u8))); db_w("," as *u8)
254 jkv("city_january" as *u8, pb_dcos(g, qb, pb_wid(g,"january" as *u8))); db_w("," as *u8)
255 jkv("city_defeated" as *u8, pb_dcos(g, qb, pb_wid(g,"defeated" as *u8))); db_w("," as *u8)
256 jkv("city_purple" as *u8, pb_dcos(g, qb, pb_wid(g,"purple" as *u8))); db_w("}," as *u8)
257 // per-doc scores
258 db_w("\"docs\":[" as *u8)
259 i = 0
260 while i < 4 {
261 if i > 0 { db_w("," as *u8) }
262 db_w("{" as *u8); jkv("lex" as *u8, lex[i]); db_w("," as *u8); jkv("dense" as *u8, den[i]); db_w("," as *u8); jkv("gold" as *u8, gold[i]); db_w("}" as *u8)
263 i = i + 1
264 }
265 db_w("]," as *u8)
266 // gains in each order
267 db_w("\"dense_order_gains\":[" as *u8)
268 i = 0
269 while i < 4 { if i > 0 { db_w("," as *u8) } db_n(gold[dord[i]]); i = i + 1 }
270 db_w("],\"lexical_order_gains\":[" as *u8)
271 i = 0
272 while i < 4 { if i > 0 { db_w("," as *u8) } db_n(gold[lord[i]]); i = i + 1 }
273 db_w("]," as *u8)
274 // metrics
275 db_w("\"metrics\":{" as *u8)
276 jkv("relevant" as *u8, R); db_w("," as *u8); jkv("total" as *u8, N); db_w("," as *u8)
277 jkv("lexical_tied" as *u8, lex_tied); db_w("," as *u8)
278 jkv("lexical_expected_p_at_2_permil" as *u8, lex_exp); db_w("," as *u8)
279 jkv("dense_p_at_2_permil" as *u8, d_p2); db_w("," as *u8)
280 jkv("lexical_expected_p_at_1_permil" as *u8, lex_exp); db_w("," as *u8)
281 jkv("dense_p_at_1_permil" as *u8, d_p1); db_w("," as *u8)
282 jkv("lexical_actual_p_at_2_permil" as *u8, l_p2); db_w("," as *u8)
283 jkv("dense_mrr_permil" as *u8, d_mrr); db_w("}," as *u8)
284 // verdict
285 db_w("\"verdict\":\"" as *u8)
286 if verdict_green == 1 { db_w("GREEN" as *u8) } else { db_w("RED" as *u8) }
287 db_w("\",\"note\":\"controlled unit proof on the real 41MB model; corpus-scale nDCG = BEIR row F236 (flagged, not claimed)\"}" as *u8)
288 db_w("\n" as *u8)
289
290 if verdict_green == 1 { return 0 }
291 return 1
292}