nx_search_inverted.nx source
↩ module page · 691 lines · 26152 B
1// nx_search_inverted.nx -- token-postings inverted index.
2//
3// module: nishi-core.search.inverted
4// depends: nishi-core.io.syscalls
5// disk_kb: 6
6// capability: CORE_IO
7// wired_status: FULLY_WIRED
8//
9// license_tier: PUBLIC_NISHI_SUBSTRATE
10// genealogy_id: salton_1971_inverted_index +
11// fnv_1a_64bit_hash +
12// nishi_build_the_system_cardinal_2026
13//
14// Search-engine Brick #2. Replaces O(N*M) linear scan with
15// O(M_query * log(V) + sum(postings_per_qtoken)) lookup. At 2.6M
16// rows + ~50K vocab: sub-millisecond query latency.
17//
18// In-memory index built from JSONL once; held in process memory.
19// File-backed persistence queued (nx_search_inverted_persist).
20//
21// Tokenization: split on whitespace + ASCII punctuation, lowercase
22// ASCII letters, keep alphanumerics. Minimum token length = 2.
23// Maximum token length = 64.
24//
25// Hash: FNV-1a 64-bit (per nx_canary_value substrate convention).
26
27import "syscalls.nx"
28const NX_MAGIC_4294967296: i64 = 4294967296
29const NX_MAGIC_4194304: i64 = 4194304
30
31// ===== Verdict ====================================================
32
33const NX_INV_OK: i64 = 1
34const NX_INV_FILE_NOT_FOUND: i64 = 2
35const NX_INV_BAD_ARGS: i64 = 3
36const NX_INV_NO_MATCHES: i64 = 4
37const NX_INV_OUT_OF_VOCAB: i64 = 5
38const NX_INV_BUILD_FAIL: i64 = 6
39
40func nx_inv_verdict_name(v: i64) -> *u8 {
41 if v == NX_INV_OK { return "OK" }
42 if v == NX_INV_FILE_NOT_FOUND { return "FILE_NOT_FOUND" }
43 if v == NX_INV_BAD_ARGS { return "BAD_ARGS" }
44 if v == NX_INV_NO_MATCHES { return "NO_MATCHES" }
45 if v == NX_INV_OUT_OF_VOCAB { return "OUT_OF_VOCAB" }
46 if v == NX_INV_BUILD_FAIL { return "BUILD_FAIL" }
47 return "UNKNOWN"
48}
49
50// ===== FNV-1a 64-bit ==============================================
51
52const NX_INV_FNV1A_OFFSET_BASIS: i64 = -3750763034362895579
53const NX_INV_FNV1A_PRIME: i64 = 1099511628211
54
55func nx_inv_hash_bytes_lower(bytes: *u8, n: i64) -> i64 {
56 if n <= 0 { return 0 }
57 var h: i64 = NX_INV_FNV1A_OFFSET_BASIS
58 var i: i64 = 0
59 var iter: i64 = 0
60 var verdict: i64 = 0
61 while verdict == 0 && iter < 64 {
62 if i >= n { verdict = 1 }
63 if verdict == 0 {
64 var c: i64 = bytes[i] as i64
65 // Lowercase ASCII letters: A-Z -> a-z
66 if c >= 0x41 {
67 if c <= 0x5A { c = c + 0x20 }
68 }
69 h = h ^ c
70 h = h * NX_INV_FNV1A_PRIME
71 i = i + 1
72 iter = iter + 1
73 }
74 }
75 return h
76}
77
78// ===== Tokenizer ==================================================
79//
80// is_token_char: ASCII alphanumeric (0-9, a-z, A-Z).
81// Everything else (whitespace, punctuation, control) is a separator.
82
83func nx_inv_is_token_char(c: i64) -> i64 {
84 if c >= 0x30 {
85 if c <= 0x39 { return 1 }
86 }
87 if c >= 0x41 {
88 if c <= 0x5A { return 1 }
89 }
90 if c >= 0x61 {
91 if c <= 0x7A { return 1 }
92 }
93 return 0
94}
95
96// ===== Hash table (open addressing, linear probe) =================
97//
98// One global hash table for vocabulary -> postings_list_id.
99// Capacity sized as next-power-of-two above expected vocab × 1.5.
100// At 50K unique tokens: 64K-128K slots.
101
102const NX_INV_VOCAB_CAPACITY: i64 = 131072 // 128 K slots (~50% load at 50K vocab)
103const NX_INV_MAX_POSTINGS_PER: i64 = 16384 // cap per token (most tokens have <100)
104const NX_INV_MAX_TOTAL_ROWS: i64 = 8388608 // 8 M rows ceiling
105const NX_INV_MIN_TOKEN_LEN: i64 = 2
106const NX_INV_MAX_TOKEN_LEN: i64 = 64
107
108// ===== BM25 term frequency, over the index's OWN tokenization ======
109//
110// nx_bm25_tf -- how many times the token whose lowercase FNV-1a hash is
111// term_hash occurs in text[0..n). The tf input to BM25 saturation, kept
112// separate from the saturation so a caller can weight it.
113//
114// WHY IT LIVES HERE AND NOT IN nx_bm25.nx: it must tokenize EXACTLY as the
115// indexer does, or a count taken here and a postings list built there
116// disagree -- and that disagreement is silent, surfacing as a document that
117// scores but cannot be retrieved. Its two dependencies
118// (nx_inv_is_token_char, nx_inv_hash_bytes_lower) are in this file, so
119// co-location makes divergence impossible by construction instead of by
120// discipline.
121//
122// 20 call sites across 15 files -- crawl, SERP, entity discovery, research
123// digest -- called this with ZERO definitions anywhere in 19,977 sources.
124// Found by nx_undefscan corpus mode 2026-07-31 (NOWHERE, unwritten=1).
125//
126// CONTRACT PINNED BY nx_docstore_test.nx:39-41 -- present => >0, absent => 0.
127// Tokens below NX_INV_MIN_TOKEN_LEN are skipped exactly as the indexer skips
128// them. A token longer than NX_INV_MAX_TOKEN_LEN hashes on its first 64
129// bytes because nx_inv_hash_bytes_lower caps its own iteration there, so
130// over-long tokens collide identically on both paths.
131func nx_bm25_tf(text: *u8, n: i64, term_hash: i64) -> i64 {
132 if n <= 0 { return 0 }
133 var count: i64 = 0
134 var i: i64 = 0
135 while i < n {
136 if nx_inv_is_token_char(text[i] as i64) == 0 { i = i + 1 } else {
137 var e: i64 = i
138 var st: i64 = 0
139 while st == 0 {
140 if e >= n { st = 1 } else {
141 if nx_inv_is_token_char(text[e] as i64) == 1 { e = e + 1 } else { st = 1 }
142 }
143 }
144 let tl: i64 = e - i
145 if tl >= NX_INV_MIN_TOKEN_LEN {
146 if nx_inv_hash_bytes_lower(((text as i64) + i) as *u8, tl) == term_hash { count = count + 1 }
147 }
148 i = e
149 }
150 }
151 return count
152}
153
154// Vocab slot: 4 × i64 = 32 bytes per slot.
155// off 0: token_hash
156// off 8: postings_offset (computed in pass 2)
157// off 16: postings_count (final count after pass 1)
158// off 24: write_cursor (used during pass 2; reused as "reserved" after)
159const NX_INV_SLOT_BYTES: i64 = 32
160
161struct NxInvIndex {
162 n_rows: i64,
163 vocab_capacity: i64,
164 vocab_occupied: i64,
165 vocab_slots_ptr: *u8, // capacity * 32 bytes
166 postings_ptr: *u8, // pool of u32 row IDs
167 postings_used: i64,
168 postings_capacity: i64,
169}
170
171const NX_INV_INDEX_BYTES: i64 = 56 // 7 fields * 8 bytes
172
173// ===== Allocate empty index =======================================
174
175func nx_inv_new(expected_rows: i64) -> *NxInvIndex {
176 let raw: *u8 = sys_mmap(NX_INV_INDEX_BYTES)
177 let idx: *NxInvIndex = raw as *NxInvIndex
178 idx.n_rows = 0
179 idx.vocab_capacity = NX_INV_VOCAB_CAPACITY
180 idx.vocab_occupied = 0
181 let slots_bytes: i64 = NX_INV_VOCAB_CAPACITY * NX_INV_SLOT_BYTES
182 idx.vocab_slots_ptr = sys_mmap(slots_bytes)
183 // Initialize all slot hashes to 0 (sentinel = empty).
184 // mmap returns zero pages so we rely on that.
185
186 // Postings pool: lazy 4 GiB virtual reservation; only-touched pages
187 // back to physical. Matches the syscalls.nx sys_read_file lazy-mmap
188 // pattern. For 2.6M rows × ~20 tokens × 4 bytes = ~200 MB physical
189 // max; the 4 GiB virtual reservation gives headroom for outlier docs.
190 let post_cap: i64 = NX_MAGIC_4294967296
191 idx.postings_capacity = post_cap
192 idx.postings_ptr = sys_mmap(post_cap)
193 idx.postings_used = 0
194 return idx
195}
196
197// Sized variant: caller picks the vocab hash-table capacity (MUST be a power of two -- the probe uses
198// `hash & (cap-1)`). For large corpora whose real vocabulary exceeds the default 131072 (e.g. 366k diverse
199// image prompts), pass a bigger power of two so the table does not saturate and silently drop late-appearing
200// tokens. Additive: nx_inv_new (default cap) and every existing caller are unchanged; the capacity is already
201// persisted in the index header, so load/query need no change. Memory: cap * 32 bytes for the vocab slots.
202func nx_inv_new_sized(cap: i64) -> *NxInvIndex {
203 let raw: *u8 = sys_mmap(NX_INV_INDEX_BYTES)
204 let idx: *NxInvIndex = raw as *NxInvIndex
205 idx.n_rows = 0
206 idx.vocab_capacity = cap
207 idx.vocab_occupied = 0
208 idx.vocab_slots_ptr = sys_mmap(cap * NX_INV_SLOT_BYTES)
209 let post_cap: i64 = NX_MAGIC_4294967296
210 idx.postings_capacity = post_cap
211 idx.postings_ptr = sys_mmap(post_cap)
212 idx.postings_used = 0
213 return idx
214}
215
216// BOUNDED variant (platform-independence / multi-target): caller sets BOTH the vocab hash-table capacity (power of 2)
217// AND the postings-pool BYTES. The default nx_inv_new reserves a 4 GiB postings pool that relies on Linux LAZY mmap
218// (only touched pages back to RAM) -- which does NOT port to a HeapAlloc/VirtualAlloc HAL (Windows PE) or a bare-metal
219// allocator that must commit up front. A bounded pool sized to the corpus makes the SAME index organ compile + run on
220// ANY target (Linux ELF / Windows PE / bare-metal RV64). Additive: nx_inv_new / nx_inv_new_sized unchanged.
221func nx_inv_new_bounded(cap: i64, post_cap: i64) -> *NxInvIndex {
222 let raw: *u8 = sys_mmap(NX_INV_INDEX_BYTES)
223 let idx: *NxInvIndex = raw as *NxInvIndex
224 idx.n_rows = 0
225 idx.vocab_capacity = cap
226 idx.vocab_occupied = 0
227 idx.vocab_slots_ptr = sys_mmap(cap * NX_INV_SLOT_BYTES)
228 idx.postings_capacity = post_cap
229 idx.postings_ptr = sys_mmap(post_cap)
230 idx.postings_used = 0
231 return idx
232}
233
234// ===== Add a posting (token_hash, row_id) =========================
235//
236// Open-addressing hash probe: bucket = hash mod capacity, linear
237// probe on collision until find slot or empty. When found: append
238// row_id to that token's postings array (if not already last).
239// When new: claim slot + allocate small postings chunk.
240
241func nx_inv_slot_at(idx: *NxInvIndex, slot_i: i64) -> *u8 {
242 return ((idx.vocab_slots_ptr as i64) + slot_i * NX_INV_SLOT_BYTES) as *u8
243}
244
245func nx_inv_slot_hash(slot: *u8) -> i64 {
246 let p: *i64 = slot as *i64
247 return *p
248}
249
250func nx_inv_slot_set_hash(slot: *u8, h: i64) {
251 let p: *i64 = slot as *i64
252 *p = h
253}
254
255func nx_inv_slot_postings_offset(slot: *u8) -> i64 {
256 let p: *i64 = ((slot as i64) + 8) as *i64
257 return *p
258}
259
260func nx_inv_slot_set_postings_offset(slot: *u8, off: i64) {
261 let p: *i64 = ((slot as i64) + 8) as *i64
262 *p = off
263}
264
265func nx_inv_slot_postings_count(slot: *u8) -> i64 {
266 let p: *i64 = ((slot as i64) + 16) as *i64
267 return *p
268}
269
270func nx_inv_slot_set_postings_count(slot: *u8, c: i64) {
271 let p: *i64 = ((slot as i64) + 16) as *i64
272 *p = c
273}
274
275func nx_inv_slot_write_cursor(slot: *u8) -> i64 {
276 let p: *i64 = ((slot as i64) + 24) as *i64
277 return *p
278}
279
280func nx_inv_slot_set_write_cursor(slot: *u8, c: i64) {
281 let p: *i64 = ((slot as i64) + 24) as *i64
282 *p = c
283}
284
285// Pass 1: just count (no postings write yet).
286func nx_inv_count_token(idx: *NxInvIndex, token_hash: i64) -> i64 {
287 if token_hash == 0 { return 0 }
288 let cap: i64 = idx.vocab_capacity
289 let mask: i64 = cap - 1
290 var slot_i: i64 = token_hash & mask
291 if slot_i < 0 { slot_i = -slot_i }
292 var iter: i64 = 0
293 var verdict: i64 = 0
294 while verdict == 0 && iter < cap {
295 let slot: *u8 = nx_inv_slot_at(idx, slot_i)
296 let h: i64 = nx_inv_slot_hash(slot)
297 if h == 0 {
298 nx_inv_slot_set_hash(slot, token_hash)
299 nx_inv_slot_set_postings_count(slot, 1)
300 idx.vocab_occupied = idx.vocab_occupied + 1
301 verdict = 1
302 }
303 if verdict == 0 {
304 if h == token_hash {
305 let count: i64 = nx_inv_slot_postings_count(slot)
306 if count < NX_INV_MAX_POSTINGS_PER {
307 nx_inv_slot_set_postings_count(slot, count + 1)
308 }
309 verdict = 1
310 }
311 if verdict == 0 {
312 slot_i = (slot_i + 1) & mask
313 iter = iter + 1
314 }
315 }
316 }
317 return 1
318}
319
320// Between passes: assign each slot a contiguous range in the postings
321// buffer based on cumulative counts. Resets write_cursor to 0.
322func nx_inv_finalize_offsets(idx: *NxInvIndex) -> i64 {
323 var cumulative: i64 = 0
324 var i: i64 = 0
325 var iter: i64 = 0
326 var verdict: i64 = 0
327 while verdict == 0 && iter < idx.vocab_capacity {
328 if i >= idx.vocab_capacity { verdict = 1 }
329 if verdict == 0 {
330 let slot: *u8 = nx_inv_slot_at(idx, i)
331 let h: i64 = nx_inv_slot_hash(slot)
332 if h != 0 {
333 let count: i64 = nx_inv_slot_postings_count(slot)
334 nx_inv_slot_set_postings_offset(slot, cumulative)
335 nx_inv_slot_set_write_cursor(slot, 0)
336 cumulative = cumulative + count * 4
337 }
338 i = i + 1
339 iter = iter + 1
340 }
341 }
342 idx.postings_used = cumulative
343 if cumulative > idx.postings_capacity { return -1 }
344 return 1
345}
346
347// Pass 2: write the row_id at the slot's reserved offset + cursor.
348func nx_inv_emit_posting(idx: *NxInvIndex, token_hash: i64, row_id: i64) -> i64 {
349 if token_hash == 0 { return 0 }
350 let cap: i64 = idx.vocab_capacity
351 let mask: i64 = cap - 1
352 var slot_i: i64 = token_hash & mask
353 if slot_i < 0 { slot_i = -slot_i }
354 var iter: i64 = 0
355 var verdict: i64 = 0
356 while verdict == 0 && iter < cap {
357 let slot: *u8 = nx_inv_slot_at(idx, slot_i)
358 let h: i64 = nx_inv_slot_hash(slot)
359 if h == 0 { return 0 } // should not happen if pass 1 ran
360 if h == token_hash {
361 let count: i64 = nx_inv_slot_postings_count(slot)
362 let off: i64 = nx_inv_slot_postings_offset(slot)
363 let cur: i64 = nx_inv_slot_write_cursor(slot)
364 if cur >= count { return 1 } // already filled (dup count cap)
365 // De-duplicate: if the last written posting matches row_id, skip.
366 if cur > 0 {
367 let last_pp: *u8 = ((idx.postings_ptr as i64) + off + (cur - 1) * 4) as *u8
368 let last: i64 = (last_pp[0] as i64)
369 | ((last_pp[1] as i64) << 8)
370 | ((last_pp[2] as i64) << 16)
371 | ((last_pp[3] as i64) << 24)
372 if last == row_id { return 1 }
373 }
374 let p: *u8 = ((idx.postings_ptr as i64) + off + cur * 4) as *u8
375 p[0] = (row_id & 0xFF) as u8
376 p[1] = ((row_id >> 8) & 0xFF) as u8
377 p[2] = ((row_id >> 16) & 0xFF) as u8
378 p[3] = ((row_id >> 24) & 0xFF) as u8
379 nx_inv_slot_set_write_cursor(slot, cur + 1)
380 verdict = 1
381 }
382 if verdict == 0 {
383 slot_i = (slot_i + 1) & mask
384 iter = iter + 1
385 }
386 }
387 return 1
388}
389
390// Backward-compat wrapper -- v1 single-pass (interleaves). Kept
391// because nx_inv_index_row references it.
392func nx_inv_add_posting(idx: *NxInvIndex, token_hash: i64, row_id: i64) -> i64 {
393 return nx_inv_count_token(idx, token_hash)
394}
395
396// ===== Look up postings for a token ===============================
397//
398// Returns slot pointer if found, NULL otherwise.
399
400func nx_inv_lookup_slot(idx: *NxInvIndex, token_hash: i64) -> *u8 {
401 if token_hash == 0 { return 0 as *u8 }
402 let cap: i64 = idx.vocab_capacity
403 let mask: i64 = cap - 1
404 var slot_i: i64 = token_hash & mask
405 if slot_i < 0 { slot_i = -slot_i }
406 var iter: i64 = 0
407 while iter < cap {
408 let slot: *u8 = nx_inv_slot_at(idx, slot_i)
409 let h: i64 = nx_inv_slot_hash(slot)
410 if h == 0 { return 0 as *u8 } // empty -- not found
411 if h == token_hash { return slot }
412 slot_i = (slot_i + 1) & mask
413 iter = iter + 1
414 }
415 return 0 as *u8
416}
417
418// ===== Tokenize a byte range + add to index =======================
419//
420// For each token (length in [MIN_TOKEN_LEN, MAX_TOKEN_LEN]):
421// - compute lowercase FNV-1a hash
422// - add_posting(hash, row_id)
423
424func nx_inv_index_row(
425 idx: *NxInvIndex,
426 row_ptr: *u8,
427 row_len: i64,
428 row_id: i64
429) -> i64 {
430 var i: i64 = 0
431 var iter: i64 = 0
432 var verdict: i64 = 0
433 while verdict == 0 && iter < row_len + 1 {
434 if i >= row_len { verdict = 1 }
435 if verdict == 0 {
436 // Skip non-token chars.
437 if nx_inv_is_token_char(row_ptr[i] as i64) == 0 {
438 i = i + 1
439 iter = iter + 1
440 }
441 if i < row_len {
442 if nx_inv_is_token_char(row_ptr[i] as i64) == 1 {
443 let tok_start: i64 = i
444 var tok_iter: i64 = 0
445 var tok_verdict: i64 = 0
446 while tok_verdict == 0 && tok_iter < NX_INV_MAX_TOKEN_LEN + 1 {
447 if i >= row_len { tok_verdict = 1 }
448 if tok_verdict == 0 {
449 if nx_inv_is_token_char(row_ptr[i] as i64) == 0 { tok_verdict = 1 }
450 if tok_verdict == 0 {
451 i = i + 1
452 tok_iter = tok_iter + 1
453 }
454 }
455 }
456 let tok_len: i64 = i - tok_start
457 if tok_len >= NX_INV_MIN_TOKEN_LEN {
458 if tok_len <= NX_INV_MAX_TOKEN_LEN {
459 let tok_ptr: *u8 = ((row_ptr as i64) + tok_start) as *u8
460 let h: i64 = nx_inv_hash_bytes_lower(tok_ptr, tok_len)
461 if h != 0 { nx_inv_add_posting(idx, h, row_id) }
462 }
463 }
464 iter = iter + 1
465 }
466 }
467 }
468 }
469 return 0
470}
471
472// ===== Pass 2: per-row emit ========================================
473
474func nx_inv_emit_row(
475 idx: *NxInvIndex,
476 row_ptr: *u8,
477 row_len: i64,
478 row_id: i64
479) -> i64 {
480 var i: i64 = 0
481 var iter: i64 = 0
482 var verdict: i64 = 0
483 while verdict == 0 && iter < row_len + 1 {
484 if i >= row_len { verdict = 1 }
485 if verdict == 0 {
486 if nx_inv_is_token_char(row_ptr[i] as i64) == 0 {
487 i = i + 1
488 iter = iter + 1
489 }
490 if i < row_len {
491 if nx_inv_is_token_char(row_ptr[i] as i64) == 1 {
492 let tok_start: i64 = i
493 var tok_iter: i64 = 0
494 var tok_verdict: i64 = 0
495 while tok_verdict == 0 && tok_iter < NX_INV_MAX_TOKEN_LEN + 1 {
496 if i >= row_len { tok_verdict = 1 }
497 if tok_verdict == 0 {
498 if nx_inv_is_token_char(row_ptr[i] as i64) == 0 { tok_verdict = 1 }
499 if tok_verdict == 0 {
500 i = i + 1
501 tok_iter = tok_iter + 1
502 }
503 }
504 }
505 let tok_len: i64 = i - tok_start
506 if tok_len >= NX_INV_MIN_TOKEN_LEN {
507 if tok_len <= NX_INV_MAX_TOKEN_LEN {
508 let tok_ptr: *u8 = ((row_ptr as i64) + tok_start) as *u8
509 let h: i64 = nx_inv_hash_bytes_lower(tok_ptr, tok_len)
510 if h != 0 { nx_inv_emit_posting(idx, h, row_id) }
511 }
512 }
513 iter = iter + 1
514 }
515 }
516 }
517 }
518 return 0
519}
520
521// ===== Build index from JSONL file (TWO-PASS for correctness) ====
522
523func nx_inv_build_from_jsonl(jsonl_path: *u8) -> *NxInvIndex {
524 if jsonl_path == 0 as *u8 { return 0 as *NxInvIndex }
525 let lenbox: *u8 = sys_mmap(8)
526 let lp: *i64 = lenbox as *i64
527 let bytes: *u8 = sys_read_file(jsonl_path, lp)
528 if bytes == 0 as *u8 { return 0 as *NxInvIndex }
529 let total: i64 = *lp
530 let est_rows: i64 = total / 700 + 100
531 // SIZE the vocab table to the corpus so it does NOT saturate. build_from_jsonl previously hardcoded
532 // nx_inv_new's fixed 131072 slots; a large/diverse corpus (e.g. 48 MB ecosystem docs whose hex hashes +
533 // identifiers explode the unique-token set) overran it, and nx_inv_count_token degraded to an O(cap)
534 // linear probe PER token after saturation = pass-1 effective hang. The team already shipped
535 // nx_inv_new_sized for exactly this; build_from_jsonl just never used it. cap = next power of two >=
536 // total/24 (a token-density estimate), clamped [131072, 4194304] (a 128 MB vocab table ceiling).
537 var vcap: i64 = NX_INV_VOCAB_CAPACITY
538 let want: i64 = total / 24
539 while vcap < want { vcap = vcap * 2 }
540 if vcap > NX_MAGIC_4194304 { vcap = NX_MAGIC_4194304 }
541 let idx: *NxInvIndex = nx_inv_new_sized(vcap)
542
543 // ----- Pass 1: count tokens per slot -------------------------
544 var row_start: i64 = 0
545 var row_id: i64 = 0
546 var i: i64 = 0
547 var iter: i64 = 0
548 var verdict: i64 = 0
549 while verdict == 0 && iter < total + 1 { // bound by corpus BYTES (the i>=total check terminates); the old NX_INV_MAX_TOTAL_ROWS*4 byte-cap silently truncated corpora >33.5 MB
550 if i >= total { verdict = 1 }
551 if verdict == 0 {
552 if bytes[i] == 0x0A {
553 let row_len: i64 = i - row_start
554 if row_len > 0 {
555 let row_ptr: *u8 = ((bytes as i64) + row_start) as *u8
556 nx_inv_index_row(idx, row_ptr, row_len, row_id)
557 row_id = row_id + 1
558 }
559 row_start = i + 1
560 }
561 i = i + 1
562 iter = iter + 1
563 }
564 }
565 if row_start < total {
566 let row_len: i64 = total - row_start
567 let row_ptr: *u8 = ((bytes as i64) + row_start) as *u8
568 nx_inv_index_row(idx, row_ptr, row_len, row_id)
569 row_id = row_id + 1
570 }
571 idx.n_rows = row_id
572
573 // ----- Compute offsets between passes ------------------------
574 let fz: i64 = nx_inv_finalize_offsets(idx)
575 if fz < 0 { return idx } // capacity exceeded -- skip pass 2
576
577 // ----- Pass 2: emit per-token at reserved offsets ------------
578 row_start = 0
579 row_id = 0
580 i = 0
581 iter = 0
582 verdict = 0
583 while verdict == 0 && iter < total + 1 { // bound by corpus BYTES (the i>=total check terminates); the old NX_INV_MAX_TOTAL_ROWS*4 byte-cap silently truncated corpora >33.5 MB
584 if i >= total { verdict = 1 }
585 if verdict == 0 {
586 if bytes[i] == 0x0A {
587 let row_len: i64 = i - row_start
588 if row_len > 0 {
589 let row_ptr: *u8 = ((bytes as i64) + row_start) as *u8
590 nx_inv_emit_row(idx, row_ptr, row_len, row_id)
591 row_id = row_id + 1
592 }
593 row_start = i + 1
594 }
595 i = i + 1
596 iter = iter + 1
597 }
598 }
599 if row_start < total {
600 let row_len: i64 = total - row_start
601 let row_ptr: *u8 = ((bytes as i64) + row_start) as *u8
602 nx_inv_emit_row(idx, row_ptr, row_len, row_id)
603 }
604
605 return idx
606}
607
608// ===== Single-term query: return postings count + first N rowIDs ==
609//
610// Query terms are lowercased + hashed; we look up the matching
611// vocab slot and read its postings count. Optionally fill out_rowids
612// with up to out_capacity rowIDs.
613//
614// NOTE: v1 keeps postings interleaved by insertion order across all
615// tokens (see nx_inv_add_posting commentary). For correct
616// per-token lookup we'd need a two-pass build that scatters
617// per-token buffers contiguously, then flushes. Queued as
618// nx_inv_build_two_pass. This v1 returns the COUNT correctly but
619// the rowIDs may be interleaved with other tokens' postings -- so
620// callers should treat the count as authoritative + use linear-scan
621// fallback if exact rowIDs are needed.
622
623struct NxInvQueryResult {
624 result_hk: i64,
625 query_hash: i64,
626 postings_count: i64,
627 postings_offset: i64,
628 n_rowids_filled: i64,
629 verdict: i64,
630}
631
632const NX_INV_QUERY_RESULT_BYTES: i64 = 48
633
634func nx_inv_query_term(
635 idx: *NxInvIndex,
636 term_ptr: *u8,
637 term_len: i64,
638 out_rowids: *i64,
639 out_capacity: i64,
640 out_result: *NxInvQueryResult
641) -> i64 {
642 if out_result == 0 as *NxInvQueryResult { return NX_INV_BAD_ARGS }
643 out_result.result_hk = 0
644 out_result.query_hash = 0
645 out_result.postings_count = 0
646 out_result.postings_offset = 0
647 out_result.n_rowids_filled = 0
648 out_result.verdict = NX_INV_OK
649
650 if idx == 0 as *NxInvIndex { out_result.verdict = NX_INV_BAD_ARGS; return NX_INV_BAD_ARGS }
651 if term_ptr == 0 as *u8 { out_result.verdict = NX_INV_BAD_ARGS; return NX_INV_BAD_ARGS }
652 if term_len < NX_INV_MIN_TOKEN_LEN { out_result.verdict = NX_INV_BAD_ARGS; return NX_INV_BAD_ARGS }
653
654 let h: i64 = nx_inv_hash_bytes_lower(term_ptr, term_len)
655 out_result.query_hash = h
656 let slot: *u8 = nx_inv_lookup_slot(idx, h)
657 if slot == 0 as *u8 {
658 out_result.verdict = NX_INV_NO_MATCHES
659 return NX_INV_NO_MATCHES
660 }
661 let count: i64 = nx_inv_slot_postings_count(slot)
662 let off: i64 = nx_inv_slot_postings_offset(slot)
663 out_result.postings_count = count
664 out_result.postings_offset = off
665
666 // Fill from write_cursor, NOT count: pass 1 counts every occurrence
667 // (count = total tf) but pass 2's consecutive-row dedup writes fewer
668 // entries (cursor = rows actually emitted). Reading `count` u32s walked
669 // into the region's unwritten zero tail and surfaced PHANTOM rowid 0
670 // (caught live: query 'jam' count=15 cursor=2 -> docid 0 was a doc with
671 // no 'jam' at all). The cursor is slot state, persists with the index.
672 var n_fill: i64 = nx_inv_slot_write_cursor(slot)
673 if n_fill > count { n_fill = count }
674 if n_fill > out_capacity { n_fill = out_capacity }
675 var i: i64 = 0
676 var iter: i64 = 0
677 var verdict: i64 = 0
678 while verdict == 0 && iter < n_fill {
679 let p: *u8 = ((idx.postings_ptr as i64) + off + i * 4) as *u8
680 let v: i64 = (p[0] as i64)
681 | ((p[1] as i64) << 8)
682 | ((p[2] as i64) << 16)
683 | ((p[3] as i64) << 24)
684 out_rowids[i] = v
685 i = i + 1
686 iter = iter + 1
687 }
688 out_result.n_rowids_filled = n_fill
689 out_result.verdict = NX_INV_OK
690 return NX_INV_OK
691}