code wiki / _hdl_build / nx_lawsearch.nx
nx_lawsearch.nx source
↩ module page · 536 lines · 25218 B
1// nx_lawsearch.nx -- DENSE SEMANTIC RETRIEVAL over the learnings corpus (the SOTA stage-2 unlock).
2//
3// WHY THIS EXISTS. The ecosystem owns a real 41MB PPMI model (knowledge/index/semppmi_v1.bin, vocab
4// 102318, 2558014 triples) and measured dense p@1 1000 permil vs lexical 500 -- but NOTHING could
5// query it: nx_recall_dense is a fixed-fixture proof with no argv, so the single largest retrieval
6// asset we own was unreachable (debt 1785560370). This organ gives it a query interface, composing
7// the CANONICAL nx_ppmi_lib rather than hand-rolling a third copy of the loader (D001 discipline --
8// the lib header records that two copies had already silently diverged).
9//
10// Sparse BM25 cannot match a synonym: "reversible" never retrieves "undoable". Our own bench sizes
11// the gap -- BM25 ndcg@5 494 permil, fused 659, reranked 964 -- and 2026 agent-memory SOTA is
12// hybrid sparse+dense merged by RRF then cross-encoder reranked. learn_query.ps1 already ships the
13// sparse+RRF half; this is the dense view it fuses with.
14//
15// dense <query> <corpus.jsonl> <model.bin> [k] -> ranked JSON lines {score,rank,slug,text}
16// selftest <model.bin> -> gate teeth incl. a DENSE-BEATS-LEXICAL tooth
17// license_tier: ORIGINAL
18import "nx_ppmi_lib.nx"
19// TRAINED dense embeddings (embed_v1.bin). dj_* already implements the loader, mean-centering and
20// late-interaction maxsim over the trained vectors, keyed by the SAME vocabulary ppl_wid_range
21// resolves -- so this is a scorer swap, not a second implementation. Slots 80..83 are disjoint from
22// the PPMI slots 70..78 by design, which is why BOTH models can be live in one g.
23import "nx_dr_densejudge.nx"
24import "nx_sovjson_lib.nx"
25import "nx_syscalls.nx"
26import "nx_gate_verdict.nx"
27const LS_MAGIC_8190: i64 = 8190
28const LS_MAGIC_65535: i64 = 65535
29const LS_MAGIC_1024: i64 = 1024
30const LS_MAGIC_8192: i64 = 8192
31const LS_MAGIC_65536: i64 = 65536
32
33const LS_WIN: i64 = 33554432
34const LS_MAXTOK: i64 = 192
35const LS_TOPK: i64 = 64
36const LS_MODE_644: i64 = 420
37
38func ls_read(path: *u8, buf: *u8, cap: i64) -> i64 {
39 let fd: i64 = sys_openat_rd(path)
40 if fd < 0 { return 0 - 1 }
41 var n: i64 = 0
42 var go: i64 = 1
43 while go == 1 {
44 let r: i64 = sys_read(fd, ((buf as i64)+n) as *u8, cap-n)
45 if r <= 0 { go = 0 } else { n = n + r }
46 if n >= cap { go = 0 }
47 }
48 sys_close(fd)
49 return n
50}
51
52// local substring search (the lib does not export one). Defined BEFORE its callers.
53func pl_find(q: *u8, from: i64, endlim: i64, lit: *u8) -> i64 {
54 var i: i64 = from
55 while i < endlim {
56 var k: i64 = 0
57 var ok: i64 = 1
58 var go: i64 = 1
59 while go == 1 {
60 if lit[k] == (0 as u8) { go = 0 } else {
61 if i + k >= endlim { ok = 0; go = 0 } else {
62 if q[i+k] != lit[k] { ok = 0; go = 0 } else { k = k + 1 }
63 }
64 }
65 }
66 if ok == 1 { return i }
67 i = i + 1
68 }
69 return 0 - 1
70}
71
72// pull the "text":"..." payload of one JSONL record into out; returns length (JSON-unescaped)
73func ls_field(buf: *u8, s: i64, e: i64, out: *u8, cap: i64) -> i64 {
74 let p0: i64 = pl_find(buf, s, e, "\"text\":\"" as *u8)
75 if p0 < 0 { return 0 }
76 var p: i64 = p0 + 8
77 var o: i64 = 0
78 var go: i64 = 1
79 while go == 1 {
80 if p >= e { go = 0 } else {
81 let c: u8 = buf[p]
82 if c == (34 as u8) { go = 0 } else {
83 if c == (92 as u8) {
84 let d: u8 = buf[p+1]
85 var em: u8 = d
86 if d == (110 as u8) { em = 32 as u8 }
87 if d == (116 as u8) { em = 32 as u8 }
88 if o < cap - 2 { out[o] = em; o = o + 1 }
89 p = p + 2
90 } else {
91 if o < cap - 2 { out[o] = c; o = o + 1 }
92 p = p + 1
93 }
94 }
95 }
96 }
97 out[o] = 0 as u8
98 return o
99}
100
101func main(argc: i64, argv: *i64) -> i64 {
102 if argc < 3 { sj_puts("usage: nx_lawsearch {dense <query> <corpus.jsonl> <model.bin> [k] | selftest <model.bin>}\n" as *u8); sys_exit(2); return 2 }
103 let verb: *u8 = argv[1] as *u8
104 let vl: i64 = sj_vlen(verb)
105 let g: *i64 = sys_mmap(LS_MAGIC_1024) as *i64
106
107 if sj_lit_eq(verb, 0, vl, "selftest" as *u8) == 1 {
108 let ctr: *i64 = gv_ctr()
109 gv_head("nx_lawsearch selftest -- dense retrieval over the real shipped PPMI model" as *u8)
110 let okload: i64 = ppl_load(g, argv[2] as *u8)
111 gv_check("T1 the real shipped PPMI model loads (fail-closed if absent)" as *u8, okload, ctr)
112 if okload == 1 {
113 let qa: *i64 = sys_mmap(LS_MAXTOK * 8) as *i64
114 let da: *i64 = sys_mmap(LS_MAXTOK * 8) as *i64
115 let db: *i64 = sys_mmap(LS_MAXTOK * 8) as *i64
116 let q1: *u8 = "firmware must never be bricked" as *u8
117 let d1: *u8 = "hardware writes must be reversible by construction" as *u8
118 let d2: *u8 = "the memory index budget rides a silent byte cliff" as *u8
119 let nq: i64 = ppl_tokenize_ids(g, q1, sj_vlen(q1), qa, LS_MAXTOK)
120 let n1: i64 = ppl_tokenize_ids(g, d1, sj_vlen(d1), da, LS_MAXTOK)
121 let n2: i64 = ppl_tokenize_ids(g, d2, sj_vlen(d2), db, LS_MAXTOK)
122 gv_check("T2 query and docs tokenize to in-vocabulary ids" as *u8, ((nq > 0) * (n1 > 0)) * (n2 > 0), ctr)
123 let s1: i64 = ppl_maxsim_idf(g, qa, nq, da, n1)
124 let s2: i64 = ppl_maxsim_idf(g, qa, nq, db, n2)
125 var ok3: i64 = 0
126 if s1 > s2 { ok3 = 1 }
127 gv_check("T3 DENSE-BEATS-LEXICAL: the topically-related doc outscores the unrelated one WITHOUT sharing a query term" as *u8, ok3, ctr)
128 var ok4: i64 = 0
129 if s2 >= 0 { ok4 = 1 }
130 gv_check("T4 an unrelated doc still returns a defined score, never a fabricated -1" as *u8, ok4, ctr)
131 let s1b: i64 = ppl_maxsim_idf(g, qa, nq, da, n1)
132 var ok5: i64 = 0
133 if s1b == s1 { ok5 = 1 }
134 gv_check("T5 scoring is DETERMINISTIC (same inputs, same score)" as *u8, ok5, ctr)
135 }
136 if okload == 1 {
137 let gz: *i64 = sys_mmap(LS_MAXTOK * 8) as *i64
138 let junk: *u8 = "zqxjv wibblefrotz plimsollary qwertzuiop" as *u8
139 let jt: i64 = ppl_tokenize_ids(g, junk, sj_vlen(junk), gz, LS_MAXTOK)
140 var jres: i64 = 0
141 var ji: i64 = 0
142 while ji < jt { if gz[ji] >= 0 { jres = jres + 1 } ji = ji + 1 }
143 var ok6: i64 = 0
144 if jres == 0 { ok6 = 1 }
145 gv_check("T6 OOV NEG-CONTROL: pure gibberish resolves to ZERO in-vocabulary ids" as *u8, ok6, ctr)
146 var ok7: i64 = 0
147 if jt > 0 { ok7 = 1 }
148 gv_check("T7 ...yet tokenize still COUNTS those tokens -- the return value is a token count, NOT a vocab hit count" as *u8, ok7, ctr)
149 let real: *u8 = "the guard protects the operation" as *u8
150 let rt: i64 = ppl_tokenize_ids(g, real, sj_vlen(real), gz, LS_MAXTOK)
151 var rres: i64 = 0
152 var ri: i64 = 0
153 while ri < rt { if gz[ri] >= 0 { rres = rres + 1 } ri = ri + 1 }
154 var ok8: i64 = 0
155 if rres > 0 { ok8 = 1 }
156 gv_check("T8 real English DOES resolve -- so T6 measures vocabulary, not a dead tokenizer" as *u8, ok8, ctr)
157 }
158 let rc: i64 = gv_verdict("LAWSEARCH-GATE" as *u8, ctr, "dense retrieval over the real shipped model" as *u8)
159 sys_exit(rc)
160 return rc
161 }
162
163 if sj_lit_eq(verb, 0, vl, "vocab" as *u8) == 1 {
164 if argc < 4 { sj_puts("vocab <text|@corpusfile> <model.bin>\n" as *u8); sys_exit(2); return 2 }
165 if ppl_load(g, argv[3] as *u8) != 1 { sj_puts("VOCAB FAIL model-load\n" as *u8); sys_exit(1); return 1 }
166 let vb: *u8 = sys_mmap(LS_WIN)
167 var vn: i64 = 0
168 let a2: *u8 = argv[2] as *u8
169 if a2[0] == (64 as u8) {
170 vn = ls_read(((a2 as i64) + 1) as *u8, vb, LS_WIN - 8)
171 if vn <= 0 { sj_puts("VOCAB FAIL corpus-read\n" as *u8); sys_exit(1); return 1 }
172 } else {
173 let al: i64 = sj_vlen(a2)
174 var c0: i64 = 0
175 while c0 < al { vb[c0] = a2[c0]; c0 = c0 + 1 }
176 vn = al
177 }
178 // optional argv[4] = MIN WORD LENGTH. The corpus-wide 643 permil figure counts stopwords,
179 // and this model EXCLUDES them on purpose (measured: 2 of 12 common function words resolve).
180 // Counting an intentional exclusion as a miss inflates the OOV rate and would justify a
181 // model rebuild that the evidence does not actually support. Filtering by length isolates
182 // CONTENT-word coverage, which is the number that decides whether a domain model is worth it.
183 var minlen: i64 = 1
184 if argc >= 5 { let ml: i64 = sj_atoi_z(argv[4] as *u8); if ml > 0 { minlen = ml } }
185 let ids: *i64 = sys_mmap(LS_MAXTOK * 8) as *i64
186 let tb2: *u8 = sys_mmap(LS_MAGIC_8192)
187 var words: i64 = 0
188 var invocab: i64 = 0
189 var lines: i64 = 0
190 var i2: i64 = 0
191 while i2 < vn {
192 let le2: i64 = sj_le(vb, i2, vn)
193 var tl3: i64 = ls_field(vb, i2, le2, tb2, LS_MAGIC_8192)
194 if tl3 <= 0 {
195 var cc: i64 = 0
196 while cc < (le2 - i2) { if cc < LS_MAGIC_8190 { tb2[cc] = vb[i2+cc] } cc = cc + 1 }
197 tl3 = le2 - i2
198 if tl3 > LS_MAGIC_8190 { tl3 = LS_MAGIC_8190 }
199 tb2[tl3] = 0 as u8
200 }
201 if tl3 > 0 {
202 lines = lines + 1
203 // WORD BY WORD, so the length filter gates RESOLUTION and not merely the count.
204 // ppl_tokenize_ids returns a TOKEN count and writes -1 for OOV, so using its return
205 // value as an in-vocab count made pure gibberish score 1000 permil -- the neg-control
206 // passed and the instrument was lying. ppl_wid_range gives the real per-word answer.
207 var ws: i64 = 0 - 1
208 var w: i64 = 0
209 while w <= tl3 {
210 var isal: i64 = 0
211 if w < tl3 {
212 let ch: i64 = tb2[w] as i64
213 if ch >= 97 { if ch <= 122 { isal = 1 } }
214 if ch >= 65 { if ch <= 90 { isal = 1 } }
215 if ch >= 48 { if ch <= 57 { isal = 1 } }
216 }
217 if isal == 1 { if ws < 0 { ws = w } } else {
218 if ws >= 0 {
219 if (w - ws) >= minlen {
220 words = words + 1
221 if ppl_wid_range(g, tb2, ws, w) >= 0 { invocab = invocab + 1 }
222 }
223 ws = 0 - 1
224 }
225 }
226 w = w + 1
227 }
228 }
229 i2 = le2 + 1
230 }
231 let rb2: *u8 = sys_mmap(256)
232 var r2: i64 = sj_cat(rb2, 0, "VOCAB lines=" as *u8)
233 r2 = sj_catn(rb2, r2, lines)
234 r2 = sj_cat(rb2, r2, " words=" as *u8)
235 r2 = sj_catn(rb2, r2, words)
236 r2 = sj_cat(rb2, r2, " invocab=" as *u8)
237 r2 = sj_catn(rb2, r2, invocab)
238 r2 = sj_cat(rb2, r2, " coverage_permil=" as *u8)
239 var cov: i64 = 0 - 1
240 if words > 0 { cov = invocab * 1000 / words }
241 r2 = sj_catn(rb2, r2, cov)
242 rb2[r2] = 10 as u8
243 sys_write(1, rb2, r2 + 1)
244 sys_exit(0)
245 return 0
246 }
247 if sj_lit_eq(verb, 0, vl, "tdense" as *u8) == 1 {
248 if argc < 6 { sj_puts("tdense needs <query> <corpus.jsonl> <ppmi.bin> <embed.bin> [k]\n" as *u8); sys_exit(2); return 2 }
249 var tk: i64 = 8
250 if argc >= 7 { let kz2: i64 = sj_atoi_z(argv[6] as *u8); if kz2 > 0 { tk = kz2 } }
251 if tk > LS_TOPK { tk = LS_TOPK }
252 if ppl_load(g, argv[4] as *u8) != 1 { sj_puts("TDENSE FAIL ppmi-load\n" as *u8); sys_exit(1); return 1 }
253 if dj_load_embed(g, argv[5] as *u8) != 1 { sj_puts("TDENSE FAIL embed-load\n" as *u8); sys_exit(1); return 1 }
254 let mo: i64 = dj_build_mean(g)
255 let qt2: *u8 = argv[2] as *u8
256 let qa2: *i64 = sys_mmap(LS_MAXTOK * 8) as *i64
257 let nqt: i64 = ppl_tokenize_ids(g, qt2, sj_vlen(qt2), qa2, LS_MAXTOK)
258 var nq2: i64 = 0
259 var qi2: i64 = 0
260 while qi2 < nqt { if qa2[qi2] >= 0 { qa2[nq2] = qa2[qi2]; nq2 = nq2 + 1 } qi2 = qi2 + 1 }
261 if nq2 <= 0 { sj_puts("TDENSE EMPTY query has no in-vocabulary terms\n" as *u8); sys_exit(0); return 0 }
262 let bf2: *u8 = sys_mmap(LS_WIN)
263 let n2: i64 = ls_read(argv[3] as *u8, bf2, LS_WIN - 8)
264 if n2 <= 0 { sj_puts("TDENSE FAIL corpus-read\n" as *u8); sys_exit(1); return 1 }
265 let sc2: *i64 = sys_mmap((LS_TOPK + 2) * 8) as *i64
266 let so3: *i64 = sys_mmap((LS_TOPK + 2) * 8) as *i64
267 let se3: *i64 = sys_mmap((LS_TOPK + 2) * 8) as *i64
268 var cn2: i64 = 0
269 let tb4: *u8 = sys_mmap(LS_MAGIC_8192)
270 let da2: *i64 = sys_mmap(LS_MAXTOK * 8) as *i64
271 var scan2: i64 = 0
272 var i4: i64 = 0
273 while i4 < n2 {
274 let le4: i64 = sj_le(bf2, i4, n2)
275 let tl5: i64 = ls_field(bf2, i4, le4, tb4, LS_MAGIC_8192)
276 if tl5 > 0 {
277 scan2 = scan2 + 1
278 let ndt: i64 = ppl_tokenize_ids(g, tb4, tl5, da2, LS_MAXTOK)
279 var nd2: i64 = 0
280 var di2: i64 = 0
281 while di2 < ndt { if da2[di2] >= 0 { da2[nd2] = da2[di2]; nd2 = nd2 + 1 } di2 = di2 + 1 }
282 if nd2 > 0 {
283 let s2: i64 = dj_maxsim_centered(g, qa2, nq2, da2, nd2)
284 if s2 > 0 {
285 var ps: i64 = cn2
286 if ps >= LS_TOPK { ps = LS_TOPK - 1 }
287 var pl2: i64 = 0
288 var j2: i64 = ps
289 while pl2 == 0 {
290 if j2 <= 0 { pl2 = 1 } else {
291 if sc2[j2-1] < s2 { sc2[j2] = sc2[j2-1]; so3[j2] = so3[j2-1]; se3[j2] = se3[j2-1]; j2 = j2 - 1 } else { pl2 = 1 }
292 }
293 }
294 if j2 < LS_TOPK { sc2[j2] = s2; so3[j2] = i4; se3[j2] = le4; if cn2 < LS_TOPK { cn2 = cn2 + 1 } }
295 }
296 }
297 }
298 i4 = le4 + 1
299 }
300 let ob5: *u8 = sys_mmap(LS_MAGIC_65536)
301 var sh2: i64 = 0
302 var z2: i64 = 0
303 while z2 < cn2 {
304 if sh2 < tk {
305 let tl6: i64 = ls_field(bf2, so3[z2], se3[z2], tb4, LS_MAGIC_8192)
306 var o6: i64 = sj_cat(ob5, 0, "{\"rank\":" as *u8)
307 o6 = sj_catn(ob5, o6, sh2 + 1)
308 o6 = sj_cat(ob5, o6, ",\"score\":" as *u8)
309 o6 = sj_catn(ob5, o6, sc2[z2])
310 o6 = sj_cat(ob5, o6, ",\"text\":\"" as *u8)
311 var w6: i64 = 0
312 while w6 < tl6 {
313 let c6: u8 = tb4[w6]
314 if c6 == (34 as u8) { ob5[o6] = 92 as u8; o6 = o6 + 1; ob5[o6] = 34 as u8; o6 = o6 + 1 } else {
315 if c6 == (92 as u8) { ob5[o6] = 92 as u8; o6 = o6 + 1; ob5[o6] = 92 as u8; o6 = o6 + 1 } else {
316 if c6 < (32 as u8) { ob5[o6] = 32 as u8; o6 = o6 + 1 } else { ob5[o6] = c6; o6 = o6 + 1 }
317 }
318 }
319 w6 = w6 + 1
320 }
321 o6 = sj_cat(ob5, o6, "\"}" as *u8)
322 ob5[o6] = 10 as u8
323 sys_write(1, ob5, o6 + 1)
324 sh2 = sh2 + 1
325 }
326 z2 = z2 + 1
327 }
328 let sb5: *u8 = sys_mmap(256)
329 var s5: i64 = sj_cat(sb5, 0, "TDENSE OK shown=" as *u8)
330 s5 = sj_catn(sb5, s5, sh2)
331 s5 = sj_cat(sb5, s5, " scanned=" as *u8)
332 s5 = sj_catn(sb5, s5, scan2)
333 s5 = sj_cat(sb5, s5, " qterms=" as *u8)
334 s5 = sj_catn(sb5, s5, nq2)
335 s5 = sj_cat(sb5, s5, " meanok=" as *u8)
336 s5 = sj_catn(sb5, s5, mo)
337 sb5[s5] = 10 as u8
338 sys_write(1, sb5, s5 + 1)
339 sys_exit(0)
340 return 0
341 }
342 if sj_lit_eq(verb, 0, vl, "vocabmiss" as *u8) == 1 {
343 if argc < 4 { sj_puts("vocabmiss @<corpusfile> <model.bin> [limit]\n" as *u8); sys_exit(2); return 2 }
344 if ppl_load(g, argv[3] as *u8) != 1 { sj_puts("VOCABMISS FAIL model-load\n" as *u8); sys_exit(1); return 1 }
345 var lim: i64 = 60
346 if argc >= 5 { let lz: i64 = sj_atoi_z(argv[4] as *u8); if lz > 0 { lim = lz } }
347 let vb3: *u8 = sys_mmap(LS_WIN)
348 let a3: *u8 = argv[2] as *u8
349 var vn3: i64 = 0
350 if a3[0] == (64 as u8) { vn3 = ls_read(((a3 as i64) + 1) as *u8, vb3, LS_WIN - 8) } else { vn3 = 0 }
351 if vn3 <= 0 { sj_puts("VOCABMISS FAIL corpus-read\n" as *u8); sys_exit(1); return 1 }
352 let tb3: *u8 = sys_mmap(LS_MAGIC_8192)
353 let ob3: *u8 = sys_mmap(LS_MAGIC_65536)
354 var shown3: i64 = 0
355 var i3: i64 = 0
356 while i3 < vn3 {
357 let le3: i64 = sj_le(vb3, i3, vn3)
358 let tl4: i64 = ls_field(vb3, i3, le3, tb3, LS_MAGIC_8192)
359 if tl4 > 0 {
360 var s4: i64 = 0 - 1
361 var w4: i64 = 0
362 while w4 <= tl4 {
363 var alnum4: i64 = 0
364 if w4 < tl4 {
365 let c4: i64 = tb3[w4] as i64
366 if c4 >= 48 { if c4 <= 57 { alnum4 = 1 } }
367 if c4 >= 65 { if c4 <= 90 { alnum4 = 1 } }
368 if c4 >= 97 { if c4 <= 122 { alnum4 = 1 } }
369 }
370 if alnum4 == 1 { if s4 < 0 { s4 = w4 } } else {
371 if s4 >= 0 {
372 if (w4 - s4) >= 4 {
373 if ppl_wid_range(g, tb3, s4, w4) < 0 {
374 if shown3 < lim {
375 var o5: i64 = 0
376 var k5: i64 = s4
377 while k5 < w4 { ob3[o5] = tb3[k5]; o5 = o5 + 1; k5 = k5 + 1 }
378 ob3[o5] = 10 as u8
379 sys_write(1, ob3, o5 + 1)
380 shown3 = shown3 + 1
381 }
382 }
383 }
384 s4 = 0 - 1
385 }
386 }
387 w4 = w4 + 1
388 }
389 }
390 i3 = le3 + 1
391 }
392 let rb3: *u8 = sys_mmap(128)
393 var r3: i64 = sj_cat(rb3, 0, "VOCABMISS shown=" as *u8)
394 r3 = sj_catn(rb3, r3, shown3)
395 rb3[r3] = 10 as u8
396 sys_write(1, rb3, r3 + 1)
397 sys_exit(0)
398 return 0
399 }
400 if sj_lit_eq(verb, 0, vl, "pack" as *u8) == 1 {
401 if argc < 4 { sj_puts("pack needs <manifest> <outfile>\n" as *u8); sys_exit(2); return 2 }
402 let mb: *u8 = sys_mmap(LS_MAGIC_65536)
403 let mn: i64 = ls_read(argv[2] as *u8, mb, LS_MAGIC_65535)
404 if mn <= 0 { sj_puts("PACK FAIL manifest-read\n" as *u8); sys_exit(1); return 1 }
405 let ofd: i64 = sys_openat_append(argv[3] as *u8, LS_MODE_644)
406 if ofd < 0 { sj_puts("PACK FAIL out-open\n" as *u8); sys_exit(1); return 1 }
407 let pb: *u8 = sys_mmap(LS_WIN)
408 let pathb: *u8 = sys_mmap(LS_MAGIC_1024)
409 var parts: i64 = 0
410 var total: i64 = 0
411 var mi: i64 = 0
412 while mi < mn {
413 let mle: i64 = sj_le(mb, mi, mn)
414 let plen: i64 = mle - mi
415 if plen > 2 {
416 var pc: i64 = 0
417 while pc < plen { if pc < 1023 { pathb[pc] = mb[mi+pc] } pc = pc + 1 }
418 pathb[plen] = 0 as u8
419 let pn: i64 = ls_read(pathb, pb, LS_WIN - 8)
420 if pn > 0 { sys_write(ofd, pb, pn); parts = parts + 1; total = total + pn }
421 }
422 mi = mle + 1
423 }
424 sys_close(ofd)
425 let rb: *u8 = sys_mmap(256)
426 var ro: i64 = sj_cat(rb, 0, "PACK OK parts=" as *u8)
427 ro = sj_catn(rb, ro, parts)
428 ro = sj_cat(rb, ro, " bytes=" as *u8)
429 ro = sj_catn(rb, ro, total)
430 rb[ro] = 10 as u8
431 sys_write(1, rb, ro + 1)
432 sys_exit(0)
433 return 0
434 }
435 if sj_lit_eq(verb, 0, vl, "dense" as *u8) == 1 {
436 if argc < 5 { sj_puts("dense needs <query> <corpus.jsonl> <model.bin> [k]\n" as *u8); sys_exit(2); return 2 }
437 var topk: i64 = 8
438 if argc >= 6 { let kz: i64 = sj_atoi_z(argv[5] as *u8); if kz > 0 { topk = kz } }
439 if topk > LS_TOPK { topk = LS_TOPK }
440 if ppl_load(g, argv[4] as *u8) != 1 { sj_puts("LAWSEARCH FAIL model-load\n" as *u8); sys_exit(1); return 1 }
441 let qtext: *u8 = argv[2] as *u8
442 let qa: *i64 = sys_mmap(LS_MAXTOK * 8) as *i64
443 let nqtok: i64 = ppl_tokenize_ids(g, qtext, sj_vlen(qtext), qa, LS_MAXTOK)
444 // COMPACT the query to RESOLVED ids only. ppl_tokenize_ids returns a TOKEN count and writes
445 // -1 for OOV, so the old `nq <= 0` check could never fire: an entirely out-of-vocabulary
446 // query passed as "non-empty", scored 0 against every doc, and reported OK with shown=0 --
447 // a fully-OOV query and a genuinely-no-match query were indistinguishable to the caller.
448 var nq: i64 = 0
449 var qi: i64 = 0
450 while qi < nqtok { if qa[qi] >= 0 { qa[nq] = qa[qi]; nq = nq + 1 } qi = qi + 1 }
451 if nq <= 0 { sj_puts("LAWSEARCH EMPTY query has no in-vocabulary terms\n" as *u8); sys_exit(0); return 0 }
452 let buf: *u8 = sys_mmap(LS_WIN)
453 let n: i64 = ls_read(argv[3] as *u8, buf, LS_WIN - 8)
454 if n <= 0 { sj_puts("LAWSEARCH FAIL corpus-read\n" as *u8); sys_exit(1); return 1 }
455 let sc: *i64 = sys_mmap((LS_TOPK + 2) * 8) as *i64
456 let so: *i64 = sys_mmap((LS_TOPK + 2) * 8) as *i64
457 let se: *i64 = sys_mmap((LS_TOPK + 2) * 8) as *i64
458 var cnt: i64 = 0
459 let tb: *u8 = sys_mmap(LS_MAGIC_8192)
460 let da: *i64 = sys_mmap(LS_MAXTOK * 8) as *i64
461 var scanned: i64 = 0
462 var i: i64 = 0
463 while i < n {
464 let le: i64 = sj_le(buf, i, n)
465 let tl: i64 = ls_field(buf, i, le, tb, LS_MAGIC_8192)
466 if tl > 0 {
467 scanned = scanned + 1
468 let nd: i64 = ppl_tokenize_ids(g, tb, tl, da, LS_MAXTOK)
469 if nd > 0 {
470 let s: i64 = ppl_maxsim_idf(g, qa, nq, da, nd)
471 if s > 0 {
472 var pos: i64 = cnt
473 if pos >= LS_TOPK { pos = LS_TOPK - 1 }
474 var placed: i64 = 0
475 var j: i64 = pos
476 while placed == 0 {
477 if j <= 0 { placed = 1 } else {
478 if sc[j-1] < s {
479 sc[j] = sc[j-1]; so[j] = so[j-1]; se[j] = se[j-1]
480 j = j - 1
481 } else { placed = 1 }
482 }
483 }
484 if j < LS_TOPK {
485 sc[j] = s; so[j] = i; se[j] = le
486 if cnt < LS_TOPK { cnt = cnt + 1 }
487 }
488 }
489 }
490 }
491 i = le + 1
492 }
493 let ob: *u8 = sys_mmap(LS_MAGIC_65536)
494 var shown: i64 = 0
495 var z: i64 = 0
496 while z < cnt {
497 if shown < topk {
498 let tl2: i64 = ls_field(buf, so[z], se[z], tb, LS_MAGIC_8192)
499 var o: i64 = sj_cat(ob, 0, "{\"rank\":" as *u8)
500 o = sj_catn(ob, o, shown + 1)
501 o = sj_cat(ob, o, ",\"score\":" as *u8)
502 o = sj_catn(ob, o, sc[z])
503 o = sj_cat(ob, o, ",\"text\":\"" as *u8)
504 var w: i64 = 0
505 while w < tl2 {
506 let c2: u8 = tb[w]
507 if c2 == (34 as u8) { ob[o] = 92 as u8; o = o + 1; ob[o] = 34 as u8; o = o + 1 } else {
508 if c2 == (92 as u8) { ob[o] = 92 as u8; o = o + 1; ob[o] = 92 as u8; o = o + 1 } else {
509 if c2 < (32 as u8) { ob[o] = 32 as u8; o = o + 1 } else { ob[o] = c2; o = o + 1 }
510 }
511 }
512 w = w + 1
513 }
514 o = sj_cat(ob, o, "\"}" as *u8)
515 ob[o] = 10 as u8
516 sys_write(1, ob, o + 1)
517 shown = shown + 1
518 }
519 z = z + 1
520 }
521 let sb: *u8 = sys_mmap(256)
522 var so2: i64 = sj_cat(sb, 0, "LAWSEARCH OK shown=" as *u8)
523 so2 = sj_catn(sb, so2, shown)
524 so2 = sj_cat(sb, so2, " scanned=" as *u8)
525 so2 = sj_catn(sb, so2, scanned)
526 so2 = sj_cat(sb, so2, " qterms=" as *u8)
527 so2 = sj_catn(sb, so2, nq)
528 sb[so2] = 10 as u8
529 sys_write(1, sb, so2 + 1)
530 sys_exit(0)
531 return 0
532 }
533 sj_puts("LAWSEARCH unknown verb\n" as *u8)
534 sys_exit(2)
535 return 2
536}