nx_search_inverted_persist.nx source
↩ module page · 62 lines · 3260 B
1// nx_search_inverted_persist.nx -- DISK PERSISTENCE for the inverted index (the S-class STORAGE rung the team
2// flagged as "queued": nx_search_inverted holds the index in memory only). This ships it: save the built index
3// to disk + load it back, so the team builds the index ONCE over the NAS corpus and queries it forever without
4// re-tokenizing 81 GB. Composes nx_search_inverted (brings syscalls.nx == nx_syscalls.nx, no clash). The design
5// is clean because vocab slots store OFFSETS into the postings pool (not pointers), so load needs NO relocation:
6// read the file as one buffer and point the index struct INTO it. license_tier: ORIGINAL
7import "nx_search_inverted.nx"
8
9const NX_INVP_MAGIC: i64 = 336659591510 // "NXINV" -- format sentinel, rejects foreign/corrupt files
10const NX_INVP_HDR_BYTES: i64 = 40 // 5 x i64: magic, n_rows, vocab_capacity, vocab_occupied, postings_used
11
12// write `n` bytes from `buf` to fd, looping over partial writes (a 4 MB vocab table won't fit one syscall reliably).
13func nx_invp_write_all(fd: i64, buf: *u8, n: i64) -> i64 {
14 var off: i64 = 0
15 while off < n {
16 let w: i64 = sys_write(fd, ((buf as i64) + off) as *u8, n - off)
17 if w <= 0 { return 0 - 1 }
18 off = off + w
19 }
20 return off
21}
22
23// SAVE: header + vocab-slots block (vocab_capacity*32) + postings block (postings_used). Returns NX_INV_OK.
24func nx_inv_save(idx: *NxInvIndex, path: *u8) -> i64 {
25 if idx == 0 as *NxInvIndex { return NX_INV_BAD_ARGS }
26 let fd: i64 = sys_openat_wr(path, 0x1a4)
27 if fd < 0 { return NX_INV_BUILD_FAIL }
28 let hdr: *i64 = sys_mmap(NX_INVP_HDR_BYTES) as *i64
29 hdr[0] = NX_INVP_MAGIC
30 hdr[1] = idx.n_rows
31 hdr[2] = idx.vocab_capacity
32 hdr[3] = idx.vocab_occupied
33 hdr[4] = idx.postings_used
34 if nx_invp_write_all(fd, hdr as *u8, NX_INVP_HDR_BYTES) < 0 { sys_close(fd); return NX_INV_BUILD_FAIL }
35 if nx_invp_write_all(fd, idx.vocab_slots_ptr, idx.vocab_capacity * NX_INV_SLOT_BYTES) < 0 { sys_close(fd); return NX_INV_BUILD_FAIL }
36 if idx.postings_used > 0 {
37 if nx_invp_write_all(fd, idx.postings_ptr, idx.postings_used) < 0 { sys_close(fd); return NX_INV_BUILD_FAIL }
38 }
39 sys_close(fd)
40 return NX_INV_OK
41}
42
43// LOAD: read the whole file; point the index struct INTO the buffer (slots use offsets -> no relocation). The
44// loaded index is query-ready (read-only; postings_capacity = postings_used, no further adds). Returns NULL on
45// missing/corrupt file.
46func nx_inv_load(path: *u8) -> *NxInvIndex {
47 let lenbox: *i64 = sys_mmap(8) as *i64
48 let buf: *u8 = sys_read_file(path, lenbox)
49 if buf == 0 as *u8 { return 0 as *NxInvIndex }
50 if lenbox[0] < NX_INVP_HDR_BYTES { return 0 as *NxInvIndex }
51 let hdr: *i64 = buf as *i64
52 if hdr[0] != NX_INVP_MAGIC { return 0 as *NxInvIndex }
53 let idx: *NxInvIndex = sys_mmap(NX_INV_INDEX_BYTES) as *NxInvIndex
54 idx.n_rows = hdr[1]
55 idx.vocab_capacity = hdr[2]
56 idx.vocab_occupied = hdr[3]
57 idx.postings_used = hdr[4]
58 idx.postings_capacity = hdr[4]
59 idx.vocab_slots_ptr = ((buf as i64) + NX_INVP_HDR_BYTES) as *u8
60 idx.postings_ptr = ((buf as i64) + NX_INVP_HDR_BYTES + idx.vocab_capacity * NX_INV_SLOT_BYTES) as *u8
61 return idx
62}