code wiki / _hdl_build / nx_memfind.nx
nx_memfind.nx source
↩ module page · 397 lines · 15065 B
1// nx_memfind.nx -- BM25 ranked retrieval over the laptop-local memory corpus.
2//
3// CONVERTED TO nx_memplane_lib 2026-08-06. It now owns only the ranking: term counting, document
4// length, the integer log, the stopword list and query derivation. The walk, name table, read, join
5// and dump list come from the lib.
6//
7// WHY BM25, AND WHAT IT REPLACED (measured -- three rankers, one test)
8// -------------------------------------------------------------------
9// Test: reconstruct the ORIGINAL description of a memory that duplicates an existing one (the store
10// law, written twice by two sessions three weeks apart) and ask whether the ranker surfaces the file
11// it duplicates.
12// rarity-weighted presence -> TARGET NOT FOUND
13// conjunction (fraction present) -> TARGET RANK 22 of 2173
14// BM25 -> TARGET RANK 1 of 2173
15//
16// Both hand-rolled attempts failed for the SAME reason, solved in retrieval decades ago:
17// * RARITY fails because this is a SINGLE-PROJECT corpus -- the words that identify a doctrine are
18// the words the whole project uses (sovereign 1190/2176, data 1138, files 808, store 697,
19// tsv 442). Dropping "common" terms deletes the topic and keeps the incidental.
20// * CONJUNCTION fails because the biggest file contains every word; a 255 KB roadmap won a query
21// about flat-file policy purely by being long.
22// ★★★★★★ BOTH ARE PRESENCE MEASURES, AND PRESENCE IN A LONG DOCUMENT IS NEARLY FREE.
23// BM25 fixes exactly that pair: term-frequency SATURATION (k1) and LENGTH NORMALISATION (b).
24// ★ WHEN A HAND-ROLLED RANKER KEEPS FAILING, THE FIELD'S BASELINE IS THE KNOWN GOOD YOU WERE
25// SUPPOSED TO MATCH FIRST.
26//
27// Integer notes: idf uses log2 rather than ln, which scales every term identically and so leaves the
28// RANKING unchanged; the BM25 idf ratio simplifies to (2N+2)/(2df+1). Fixed point 1/256 for logs.
29//
30// TWO ENTRY POINTS, ONE RANKER:
31// nx_memfind <term> ... -- a human picks the query (interactive retrieval)
32// nx_memfind --file <path> -- the query is DERIVED from a file's own name+description
33//
34// ⚠ per-term df is printed deliberately: a rewrite once dropped it silently and the gate caught it.
35// ★ A REWRITE THAT PRESERVES THE OUTPUT BUT DROPS THE EVIDENCE FOR IT IS A REGRESSION.
36//
37// usage: nx_memfind [--dir D] [--n K] [--quiet] (--file <path> | <term> [<term> ...])
38// Sovereign: imports nx_memplane_lib. license_tier: ORIGINAL
39import "nx_memplane_lib.nx"
40
41const MS_DIR: *u8 = "/mnt/c/Users/elder/.claude/projects/C--Users-elder/memory"
42const MS_FBUF: i64 = 1048576
43const MS_MSG: i64 = 65536
44const MS_MAXT: i64 = 40
45const MS_TLEN: i64 = 64
46const MS_MINW: i64 = 3
47const MS_DESC: i64 = 120
48
49func ms_termptr(terms: *u8, i: i64) -> *u8 { return ((terms as i64) + i * MS_TLEN) as *u8 }
50
51func ms_lower(c: u8) -> i64 {
52 if c >= (65 as u8) { if c <= (90 as u8) { return (c as i64) + 32 } }
53 return c as i64
54}
55
56// COUNT word-boundary occurrences. Term frequency is what BM25 saturates; a presence flag throws
57// away the signal that separates a document ABOUT a subject from one that mentions it once.
58func ms_count(buf: *u8, n: i64, pat: *u8) -> i64 {
59 let pl: i64 = mp_len(pat)
60 if pl == 0 { return 0 }
61 if pl > n { return 0 }
62 var hits: i64 = 0
63 var i: i64 = 0
64 let lim: i64 = n - pl
65 while i <= lim {
66 var j: i64 = 0
67 var ok: i64 = 1
68 while j < pl {
69 if ms_lower(buf[i + j]) != ms_lower(pat[j]) { ok = 0; j = pl } else { j = j + 1 }
70 }
71 if ok == 1 {
72 var bounded: i64 = 1
73 if i > 0 { if mp_alnum(buf[i - 1]) == 1 { bounded = 0 } }
74 if i + pl < n { if mp_alnum(buf[i + pl]) == 1 { bounded = 0 } }
75 if bounded == 1 { hits = hits + 1; i = i + pl } else { i = i + 1 }
76 } else { i = i + 1 }
77 }
78 return hits
79}
80
81// document length in word tokens, the denominator BM25 normalises against
82func ms_doclen(buf: *u8, n: i64) -> i64 {
83 var toks: i64 = 0
84 var run: i64 = 0
85 var i: i64 = 0
86 while i <= n {
87 var a: i64 = 0
88 if i < n { a = mp_alnum(buf[i]) }
89 if a == 1 { run = run + 1 } else {
90 if run >= MS_MINW { toks = toks + 1 }
91 run = 0
92 }
93 i = i + 1
94 }
95 return toks
96}
97
98// log2(v) in 1/256 fixed point, linear inside the binade
99func ms_log2fx(v: i64) -> i64 {
100 if v <= 1 { return 0 }
101 var p: i64 = 0
102 var m: i64 = v
103 while m > 1 { m = m / 2; p = p + 1 }
104 var pow: i64 = 1
105 var k: i64 = 0
106 while k < p { pow = pow * 2; k = k + 1 }
107 let frac: i64 = ((v - pow) * 256) / pow
108 return p * 256 + frac
109}
110
111func ms_isstop(w: *u8) -> i64 {
112 if mp_streq(w, "the" as *u8) == 1 { return 1 }
113 if mp_streq(w, "and" as *u8) == 1 { return 1 }
114 if mp_streq(w, "for" as *u8) == 1 { return 1 }
115 if mp_streq(w, "are" as *u8) == 1 { return 1 }
116 if mp_streq(w, "not" as *u8) == 1 { return 1 }
117 if mp_streq(w, "but" as *u8) == 1 { return 1 }
118 if mp_streq(w, "its" as *u8) == 1 { return 1 }
119 if mp_streq(w, "with" as *u8) == 1 { return 1 }
120 if mp_streq(w, "that" as *u8) == 1 { return 1 }
121 if mp_streq(w, "this" as *u8) == 1 { return 1 }
122 if mp_streq(w, "from" as *u8) == 1 { return 1 }
123 if mp_streq(w, "was" as *u8) == 1 { return 1 }
124 if mp_streq(w, "our" as *u8) == 1 { return 1 }
125 if mp_streq(w, "all" as *u8) == 1 { return 1 }
126 if mp_streq(w, "any" as *u8) == 1 { return 1 }
127 if mp_streq(w, "new" as *u8) == 1 { return 1 }
128 if mp_streq(w, "out" as *u8) == 1 { return 1 }
129 if mp_streq(w, "has" as *u8) == 1 { return 1 }
130 if mp_streq(w, "had" as *u8) == 1 { return 1 }
131 if mp_streq(w, "name" as *u8) == 1 { return 1 }
132 if mp_streq(w, "description" as *u8) == 1 { return 1 }
133 if mp_streq(w, "metadata" as *u8) == 1 { return 1 }
134 return 0
135}
136
137func ms_addterm(terms: *u8, nt: i64, src: *u8, s: i64, wl: i64) -> i64 {
138 if wl < MS_MINW { return nt }
139 if wl >= MS_TLEN { return nt }
140 if nt >= MS_MAXT { return nt }
141 let dst: *u8 = ms_termptr(terms, nt)
142 var c: i64 = 0
143 while c < wl { dst[c] = ms_lower(src[s + c]) as u8; c = c + 1 }
144 dst[c] = 0 as u8
145 if ms_isstop(dst) == 1 { return nt }
146 var t: i64 = 0
147 while t < nt {
148 if mp_streq(ms_termptr(terms, t), dst) == 1 { return nt }
149 t = t + 1
150 }
151 return nt + 1
152}
153
154func main(argc: i64, argv: *i64) -> i64 {
155 var dir: *u8 = MS_DIR
156 var topn: i64 = 10
157 var quiet: i64 = 0
158 var fromfile: *u8 = 0 as *u8
159 let terms: *u8 = sys_mmap(MS_MAXT * MS_TLEN)
160 var nt: i64 = 0
161 let msg: *u8 = sys_mmap(MS_MSG)
162
163 var ai: i64 = 1
164 while ai < argc {
165 let a: *u8 = argv[ai] as *u8
166 var used: i64 = 0
167 if mp_streq(a, "--dir" as *u8) == 1 { if ai + 1 < argc { dir = argv[ai + 1] as *u8; ai = ai + 1; used = 1 } }
168 if mp_streq(a, "--file" as *u8) == 1 { if ai + 1 < argc { fromfile = argv[ai + 1] as *u8; ai = ai + 1; used = 1 } }
169 if mp_streq(a, "--quiet" as *u8) == 1 { quiet = 1; used = 1 }
170 if mp_streq(a, "--n" as *u8) == 1 {
171 if ai + 1 < argc {
172 let s: *u8 = argv[ai + 1] as *u8
173 var v: i64 = 0
174 var k: i64 = 0
175 while s[k] != (0 as u8) {
176 if s[k] >= (48 as u8) { if s[k] <= (57 as u8) { v = v * 10 + ((s[k] as i64) - 48) } }
177 k = k + 1
178 }
179 if v > 0 { topn = v }
180 ai = ai + 1
181 used = 1
182 }
183 }
184 if used == 0 {
185 let wl: i64 = mp_len(a)
186 var s2: i64 = 0
187 var p2: i64 = 0
188 while p2 <= wl {
189 var isw: i64 = 0
190 if p2 < wl { isw = mp_alnum(a[p2]) }
191 if isw == 0 { nt = ms_addterm(terms, nt, a, s2, p2 - s2); s2 = p2 + 1 }
192 p2 = p2 + 1
193 }
194 }
195 ai = ai + 1
196 }
197
198 // ---- --file: derive the query from the author's OWN summary (filename + description) ----
199 var selfname: *u8 = 0 as *u8
200 if fromfile != (0 as *u8) {
201 let tb: *u8 = mp_base(fromfile)
202 selfname = tb
203 let tbuf: *u8 = sys_mmap(MS_FBUF)
204 let tl: i64 = mp_readf(fromfile, tbuf, MS_FBUF)
205 if tl <= 0 { return 0 }
206 var he: i64 = tl
207 if he > 1600 { he = 1600 }
208 let nl: i64 = mp_len(tb)
209 var s3: i64 = 0
210 var p3: i64 = 0
211 while p3 <= nl {
212 var w3: i64 = 0
213 if p3 < nl { w3 = mp_alnum(tb[p3]) }
214 if w3 == 0 { nt = ms_addterm(terms, nt, tb, s3, p3 - s3); s3 = p3 + 1 }
215 p3 = p3 + 1
216 }
217 let dpat: *u8 = "description:" as *u8
218 let dpl: i64 = mp_len(dpat)
219 var i4: i64 = 0
220 while i4 + dpl < he {
221 var j4: i64 = 0
222 var ok4: i64 = 1
223 while j4 < dpl { if tbuf[i4 + j4] != dpat[j4] { ok4 = 0; j4 = dpl } else { j4 = j4 + 1 } }
224 if ok4 == 1 {
225 let st: i64 = i4 + dpl
226 var q4: i64 = st
227 var s5: i64 = st
228 var go5: i64 = 1
229 while go5 == 1 {
230 if q4 >= he { go5 = 0 } else {
231 if tbuf[q4] == (10 as u8) { go5 = 0 } else {
232 if mp_alnum(tbuf[q4]) == 0 { nt = ms_addterm(terms, nt, tbuf, s5, q4 - s5); s5 = q4 + 1 }
233 q4 = q4 + 1
234 }
235 }
236 }
237 nt = ms_addterm(terms, nt, tbuf, s5, q4 - s5)
238 i4 = he
239 } else { i4 = i4 + 1 }
240 }
241 }
242
243 if nt == 0 {
244 var u: i64 = mp_cat(msg, 0, "usage: nx_memfind [--dir D] [--n K] (--file <path> | <term> ...)\n" as *u8)
245 mp_say(msg, u)
246 return 1
247 }
248
249 let names: *u8 = sys_mmap(MP_MAXF * MP_SLOT)
250 let tbl: *i64 = sys_mmap(8 * MP_HASH) as *i64
251 let dlen: *i64 = sys_mmap(8 * MP_MAXF) as *i64
252 let score: *i64 = sys_mmap(8 * MP_MAXF) as *i64
253 let tfm: *i64 = sys_mmap(8 * MP_MAXF * MS_MAXT) as *i64
254 let df: *i64 = sys_mmap(8 * MS_MAXT) as *i64
255 let fbuf: *u8 = sys_mmap(MS_FBUF)
256 let path: *u8 = sys_mmap(4096)
257 let dbuf: *u8 = sys_mmap(1024)
258
259 let cnt: i64 = mp_scan(dir, names, tbl, MP_MAXF)
260 if cnt < 0 { return 1 }
261 var i: i64 = 0
262 while i < cnt { dlen[i] = 0; score[i] = 0; i = i + 1 }
263 var t: i64 = 0
264 while t < nt { df[t] = 0; t = t + 1 }
265
266 // ---- one pass: term frequencies + document lengths ----
267 var totlen: i64 = 0
268 var used: i64 = 0
269 var f: i64 = 0
270 while f < cnt {
271 let fname: *u8 = mp_nameptr(names, f)
272 var skip: i64 = 0
273 if mp_is_dump(fname) == 1 { skip = 1 }
274 if mp_streq(fname, "MEMORY.md" as *u8) == 1 { skip = 1 }
275 if selfname != (0 as *u8) { if mp_streq(fname, selfname) == 1 { skip = 1 } }
276 if skip == 0 {
277 mp_join(path, dir, fname)
278 let total: i64 = mp_readf(path, fbuf, MS_FBUF)
279 if total > 0 {
280 dlen[f] = ms_doclen(fbuf, total)
281 totlen = totlen + dlen[f]
282 used = used + 1
283 t = 0
284 while t < nt {
285 let c2: i64 = ms_count(fbuf, total, ms_termptr(terms, t))
286 tfm[f * MS_MAXT + t] = c2
287 if c2 > 0 { df[t] = df[t] + 1 }
288 t = t + 1
289 }
290 }
291 }
292 f = f + 1
293 }
294 var avgdl: i64 = 1
295 if used > 0 { avgdl = totlen / used }
296 if avgdl < 1 { avgdl = 1 }
297
298 // ---- BM25: k1=1.2 b=0.75 -> num 2200*f, den 1000*f + 300 + 900*dl/avgdl ----
299 f = 0
300 while f < cnt {
301 if dlen[f] > 0 {
302 var s: i64 = 0
303 t = 0
304 while t < nt {
305 let fr: i64 = tfm[f * MS_MAXT + t]
306 if fr > 0 {
307 let idf: i64 = ms_log2fx((2 * cnt + 2) / (2 * df[t] + 1))
308 let den: i64 = 1000 * fr + 300 + (900 * dlen[f]) / avgdl
309 if den > 0 { s = s + (idf * (2200 * fr)) / den }
310 }
311 t = t + 1
312 }
313 score[f] = s
314 }
315 f = f + 1
316 }
317
318 if quiet == 0 {
319 var m: i64 = mp_cat(msg, 0, "nx_memfind[BM25]: " as *u8)
320 m = mp_catn(msg, m, used)
321 m = mp_cat(msg, m, " files, " as *u8)
322 m = mp_catn(msg, m, nt)
323 m = mp_cat(msg, m, " terms, avgdl=" as *u8)
324 m = mp_catn(msg, m, avgdl)
325 m = mp_cat(msg, m, " [" as *u8)
326 t = 0
327 while t < nt {
328 if t > 0 { m = mp_cat(msg, m, " " as *u8) }
329 m = mp_cat(msg, m, ms_termptr(terms, t))
330 m = mp_cat(msg, m, "/df=" as *u8)
331 m = mp_catn(msg, m, df[t])
332 t = t + 1
333 }
334 m = mp_cat(msg, m, "]\n" as *u8)
335 mp_say(msg, m)
336 }
337
338 var shown: i64 = 0
339 while shown < topn {
340 var best: i64 = 0 - 1
341 var bs: i64 = 0
342 f = 0
343 while f < cnt {
344 if score[f] > bs { bs = score[f]; best = f }
345 f = f + 1
346 }
347 if best < 0 { shown = topn } else {
348 score[best] = 0
349 let nm2: *u8 = mp_nameptr(names, best)
350 mp_join(path, dir, nm2)
351 let tl2: i64 = mp_readf(path, fbuf, MS_FBUF)
352 var s6: i64 = 0 - 1
353 let dpat2: *u8 = "description:" as *u8
354 let dpl2: i64 = mp_len(dpat2)
355 var i6: i64 = 0
356 var lim6: i64 = tl2 - dpl2
357 if lim6 > 3000 { lim6 = 3000 }
358 while i6 < lim6 {
359 var j6: i64 = 0
360 var ok6: i64 = 1
361 while j6 < dpl2 { if fbuf[i6 + j6] != dpat2[j6] { ok6 = 0; j6 = dpl2 } else { j6 = j6 + 1 } }
362 if ok6 == 1 { s6 = i6 + dpl2; i6 = lim6 } else { i6 = i6 + 1 }
363 }
364 var dl6: i64 = 0
365 if s6 >= 0 {
366 var p6: i64 = s6
367 while p6 < tl2 {
368 if dl6 >= MS_DESC { p6 = tl2 } else {
369 let c6: u8 = fbuf[p6]
370 if c6 == (10 as u8) { p6 = tl2 } else {
371 var sk: i64 = 0
372 if c6 == (34 as u8) { sk = 1 }
373 if c6 == (13 as u8) { sk = 1 }
374 if dl6 == 0 { if c6 == (32 as u8) { sk = 1 } }
375 if sk == 0 { dbuf[dl6] = c6; dl6 = dl6 + 1 }
376 p6 = p6 + 1
377 }
378 }
379 }
380 }
381 dbuf[dl6] = 0 as u8
382 var d: i64 = mp_cat(msg, 0, " " as *u8)
383 d = mp_catn(msg, d, bs / 256)
384 d = mp_cat(msg, d, " " as *u8)
385 d = mp_cat(msg, d, nm2)
386 d = mp_cat(msg, d, "\n" as *u8)
387 if dbuf[0] != (0 as u8) {
388 d = mp_cat(msg, d, " " as *u8)
389 d = mp_cat(msg, d, dbuf)
390 d = mp_cat(msg, d, "\n" as *u8)
391 }
392 mp_say(msg, d)
393 shown = shown + 1
394 }
395 }
396 return 0
397}