nx_lib_index.nx source
↩ module page · 142 lines · 5690 B
1// nx_lib_index.nx -- PERSISTED inverted index for the library search (the
2// perf-EXCEED over the linear scan). Wires the ecosystem's sovereign engine
3// nx_search_inverted(+_persist): build the index ONCE over the work titles
4// (docid = __works__ line), save to disk; query is O(1) hash lookup per term
5// (vs SQLite FTS5's B-tree AND the linear scan's O(n)). AND semantics: a work
6// matches only if EVERY query term hits it. Uses nishi APIs, no new ranking.
7// license_tier: ORIGINAL | genealogy_id: nishi_library_sovereign_index_2026_07_01
8import "nx_lib_store.nx"
9import "nx_search_inverted_persist.nx"
10const LIBINDEX_MAGIC_1024: i64 = 1024
11const LIBINDEX_MAGIC_1048576: i64 = 1048576
12const LIBINDEX_MAGIC_16384: i64 = 16384
13
14const LIBINDEX_PATH: *u8 = "knowledge/libindex.nxinv" as *u8
15
16// one pass over the works: emit=0 -> nx_inv_index_row (count), emit=1 -> nx_inv_emit_row.
17// docid = the __works__ line index (stable, append-only). returns doc count.
18func lidx_pass(ix: *NxInvIndex, idxbuf: *u8, inn: i64, emit: i64, h: *i64, kbuf: *u8) -> i64 {
19 let po: *i64 = sys_mmap(16) as *i64
20 let lo: *i64 = sys_mmap(16) as *i64
21 let hkbuf: *u8 = sys_mmap(LIBINDEX_MAGIC_1024)
22 var docid: i64 = 0
23 var ls: i64 = 0
24 var i: i64 = 0
25 while i <= inn {
26 var eol: i64 = 0
27 if i == inn { eol = 1 } else { if idxbuf[i] == 10 as u8 { eol = 1 } }
28 if eol == 1 {
29 if i > ls {
30 var k: i64 = 0
31 var j: i64 = ls
32 while j < i { hkbuf[k] = idxbuf[j]; k = k + 1; j = j + 1 }
33 hkbuf[k] = 0 as u8
34 if nx_lib_store_get_work_h(h, kbuf, hkbuf, po, lo) >= 0 {
35 let rec: *u8 = po[0] as *u8
36 let reclen: i64 = lo[0]
37 // index the WHOLE record (title + doi + abstract + authors) =
38 // full-text search, not just the title.
39 if emit == 1 { nx_inv_emit_row(ix, rec, reclen, docid) }
40 else { nx_inv_index_row(ix, rec, reclen, docid) }
41 }
42 docid = docid + 1
43 }
44 ls = i + 1
45 }
46 i = i + 1
47 }
48 return docid
49}
50
51// build the persisted inverted index over the whole catalog. returns doc count.
52func nx_lib_index_build() -> i64 {
53 let idxbuf: *u8 = sys_mmap(LIBINDEX_MAGIC_1048576)
54 let inn: i64 = nx_lib_store_index(idxbuf)
55 var ndocs: i64 = 0
56 var z: i64 = 0
57 while z < inn { if idxbuf[z] == 10 as u8 { ndocs = ndocs + 1 } z = z + 1 }
58 let ix: *NxInvIndex = nx_inv_new(ndocs + 8)
59 let h: *i64 = nx_lib_store_open() // cache the store once (O(1) gets, no re-mmap)
60 let kbuf: *u8 = sys_mmap(512)
61 lidx_pass(ix, idxbuf, inn, 0, h, kbuf) // pass 1: count tokens
62 nx_inv_finalize_offsets(ix)
63 lidx_pass(ix, idxbuf, inn, 1, h, kbuf) // pass 2: emit postings
64 nx_inv_save(ix, LIBINDEX_PATH)
65 return ndocs
66}
67
68// O(1) search via the persisted index: query -> JSON array of matching hks
69// (AND across terms). out = JSON; returns length.
70func nx_lib_index_search(query: *u8, qn: i64, out: *u8) -> i64 {
71 let ix: *NxInvIndex = nx_inv_load(LIBINDEX_PATH)
72
73 // docid -> hk table from __works__ (same order the index was built with)
74 let idxbuf: *u8 = sys_mmap(LIBINDEX_MAGIC_1048576)
75 let inn: i64 = nx_lib_store_index(idxbuf)
76 var ndocs: i64 = 0
77 var z: i64 = 0
78 while z < inn { if idxbuf[z] == 10 as u8 { ndocs = ndocs + 1 } z = z + 1 }
79 let hkoff: *i64 = sys_mmap(8 * (ndocs + 8)) as *i64
80 let hklen: *i64 = sys_mmap(8 * (ndocs + 8)) as *i64
81 var d2: i64 = 0
82 var ls2: i64 = 0
83 var i2: i64 = 0
84 while i2 <= inn {
85 var eol: i64 = 0
86 if i2 == inn { eol = 1 } else { if idxbuf[i2] == 10 as u8 { eol = 1 } }
87 if eol == 1 { if i2 > ls2 { hkoff[d2] = ls2; hklen[d2] = i2 - ls2; d2 = d2 + 1 } ls2 = i2 + 1 }
88 i2 = i2 + 1
89 }
90
91 let cnt: *i64 = sys_mmap(8 * (ndocs + 8)) as *i64
92 var ci: i64 = 0
93 while ci < ndocs { cnt[ci] = 0; ci = ci + 1 }
94
95 let rowids: *i64 = sys_mmap(8 * LIBINDEX_MAGIC_16384) as *i64
96 let res: *NxInvQueryResult = sys_mmap(64) as *NxInvQueryResult
97
98 var nterms: i64 = 0
99 var ts: i64 = 0
100 while ts < qn {
101 if query[ts] == 32 as u8 { ts = ts + 1 }
102 else {
103 var termend: i64 = qn
104 var te: i64 = ts
105 while te < qn { if termend == qn { if query[te] == 32 as u8 { termend = te } } te = te + 1 }
106 let termlen: i64 = termend - ts
107 let termptr: *u8 = (query as i64 + ts) as *u8
108 if ix != 0 as *NxInvIndex {
109 nx_inv_query_term(ix, termptr, termlen, rowids, LIBINDEX_MAGIC_16384, res)
110 var ri: i64 = 0
111 while ri < res.n_rowids_filled {
112 let rid: i64 = rowids[ri]
113 if rid >= 0 { if rid < ndocs { cnt[rid] = cnt[rid] + 1 } }
114 ri = ri + 1
115 }
116 }
117 nterms = nterms + 1
118 ts = termend
119 }
120 }
121
122 var o: i64 = 0
123 out[o] = 91 as u8; o = o + 1 // '['
124 var first: i64 = 1
125 if nterms > 0 {
126 var d: i64 = 0
127 while d < ndocs {
128 if cnt[d] == nterms {
129 if first == 0 { out[o] = 44 as u8; o = o + 1 }
130 first = 0
131 out[o] = 34 as u8; o = o + 1
132 var b: i64 = hkoff[d]
133 let e: i64 = hkoff[d] + hklen[d]
134 while b < e { out[o] = idxbuf[b]; o = o + 1; b = b + 1 }
135 out[o] = 34 as u8; o = o + 1
136 }
137 d = d + 1
138 }
139 }
140 out[o] = 93 as u8; o = o + 1 // ']'
141 return o
142}