nx_wiki_index_builder.nx source
↩ module page · 447 lines · 17988 B
1// nx_wiki_index_builder.nx -- wiki-specific index builder + doc store.
2//
3// COMPOSES (zero new index/tokenizer code introduced):
4// nx_search_inverted (two-pass NxInvIndex build via index_row + emit_row)
5// nx_syscalls (sys_open, sys_read_file, sys_close for doc bytes)
6//
7// COMPOSED BY:
8// wiki/nx_wiki_search_wiring (resolver swap; closes placeholder gap)
9// wiki/nx_wiki_main (startup-time index build)
10//
11// SEPARATION OF CONCERNS:
12// - nx_wiki_doc_store: pure storage; rowid -> (title, url, body) lookup
13// - nx_wiki_index_builder: orchestrates two-pass NxInvIndex build +
14// doc-store population; reads bytes via sys_read_file
15// - V1: caller supplies a list of doc paths (no dir-walk yet); V2
16// adds sys_getdents-based auto-discover
17//
18// V1 SCOPE per NISHI_SEARCH_CHARTER ยง6:
19// - Caller-driven doc list (operator picks which docs to index)
20// - Title extraction: first markdown # heading or fallback to filename
21// - URL derivation: caller supplies (typically "/wiki/<doc-name>")
22// - Body: full file bytes (no body extraction yet; V2 strips front matter)
23// - Tokenization: nx_inv_index_row + nx_inv_emit_row (standard
24// nx_search_inverted convention; lowercase + 2..64 char tokens)
25//
26// V2 SCOPE (TODO):
27// - sys_getdents directory auto-discover
28// - Front-matter parser (skip <head> from indexable body)
29// - nx_nishi_page_validator integration (skip non-conformant docs;
30// log via nx_telemetry)
31// - Incremental rebuild (content-hash diff vs prior build)
32//
33// Status: V1. 2026-05-27.
34
35import "nx_syscalls.nx"
36import "nx_search_inverted.nx"
37
38// ===== Sealed verdict surface (codes 2600-2619) =================================================
39const NX_WIB_OK: i64 = 0
40const NX_WIB_BAD_INPUT: i64 = 2600
41const NX_WIB_DOC_OVERFLOW: i64 = 2601
42const NX_WIB_BUF_OVERFLOW: i64 = 2602
43const NX_WIB_READ_FAILED: i64 = 2603
44const NX_WIB_INDEX_INIT_FAILED: i64 = 2604
45const NX_WIB_PASS1_FAILED: i64 = 2605
46const NX_WIB_FINALIZE_FAILED: i64 = 2606
47const NX_WIB_PASS2_FAILED: i64 = 2607
48const NX_WIB_DOUBLE_FINALIZE: i64 = 2608
49const NX_WIB_LOOP_BUDGET: i64 = 2609
50
51// ===== Named constants (M7) =================================================
52const NX_WIB_DEFAULT_DOCS_CAP: i64 = 1000
53const NX_WIB_DEFAULT_TITLE_POOL: i64 = 65536 // 64 KB
54const NX_WIB_DEFAULT_URL_POOL: i64 = 65536 // 64 KB
55const NX_WIB_DEFAULT_BODY_POOL: i64 = 4194304 // 4 MB; 4 KB avg per doc
56const NX_WIB_MAX_TITLE_LEN: i64 = 200
57const NX_WIB_MAX_URL_LEN: i64 = 512
58const NX_WIB_MAX_BODY_LEN: i64 = 524288 // 512 KB per doc
59const NX_WIB_LOOP_BUDGET_CAP: i64 = 10000000
60
61// ===== NxWikiDocStore: rowid-keyed parallel pools =================================================
62
63struct NxWikiDocStore {
64 // Title pool: packed UTF-8 strings; titles_offs/lens index into pool.
65 titles_pool: *u8
66 titles_pool_cap: i64
67 titles_pool_used: i64
68 titles_offs: *i64
69 titles_lens: *i64
70
71 // URL pool (same shape)
72 urls_pool: *u8
73 urls_pool_cap: i64
74 urls_pool_used: i64
75 urls_offs: *i64
76 urls_lens: *i64
77
78 // Body pool (same shape; bodies are the raw doc bytes; resolver
79 // hands these to nx_search_snippet_extract)
80 bodies_pool: *u8
81 bodies_pool_cap: i64
82 bodies_pool_used: i64
83 bodies_offs: *i64
84 bodies_lens: *i64
85
86 doc_count: i64
87 docs_cap: i64
88 valid: i64
89}
90
91func nx_wiki_doc_store_init(s: *NxWikiDocStore, docs_cap: i64,
92 titles_pool_cap: i64, urls_pool_cap: i64,
93 bodies_pool_cap: i64) -> i64 {
94 if (s as i64) == 0 { return 0 - NX_WIB_BAD_INPUT }
95 if docs_cap < 1 { return 0 - NX_WIB_BAD_INPUT }
96 if docs_cap > NX_WIB_DEFAULT_DOCS_CAP { return 0 - NX_WIB_BAD_INPUT }
97 if titles_pool_cap < 1024 { return 0 - NX_WIB_BAD_INPUT }
98 if urls_pool_cap < 1024 { return 0 - NX_WIB_BAD_INPUT }
99 if bodies_pool_cap < 1024 { return 0 - NX_WIB_BAD_INPUT }
100
101 s.titles_pool = (sys_mmap(titles_pool_cap)) as *u8
102 s.titles_pool_cap = titles_pool_cap
103 s.titles_pool_used = 0
104 s.titles_offs = (sys_mmap(docs_cap * 8)) as *i64
105 s.titles_lens = (sys_mmap(docs_cap * 8)) as *i64
106
107 s.urls_pool = (sys_mmap(urls_pool_cap)) as *u8
108 s.urls_pool_cap = urls_pool_cap
109 s.urls_pool_used = 0
110 s.urls_offs = (sys_mmap(docs_cap * 8)) as *i64
111 s.urls_lens = (sys_mmap(docs_cap * 8)) as *i64
112
113 s.bodies_pool = (sys_mmap(bodies_pool_cap)) as *u8
114 s.bodies_pool_cap = bodies_pool_cap
115 s.bodies_pool_used = 0
116 s.bodies_offs = (sys_mmap(docs_cap * 8)) as *i64
117 s.bodies_lens = (sys_mmap(docs_cap * 8)) as *i64
118
119 s.doc_count = 0
120 s.docs_cap = docs_cap
121 s.valid = 1
122 return NX_WIB_OK
123}
124
125// Reserve space in a pool; return new offset (or -error).
126func nx_wib_pool_alloc(pool: *u8, cap: i64, used_p: *i64, n: i64) -> i64 {
127 if used_p[0] + n > cap { return 0 - NX_WIB_BUF_OVERFLOW }
128 let off: i64 = used_p[0]
129 var i: i64 = 0
130 while i < n {
131 if i >= NX_WIB_LOOP_BUDGET_CAP { return 0 - NX_WIB_LOOP_BUDGET }
132 i = i + 1
133 }
134 used_p[0] = used_p[0] + n
135 return off
136}
137
138// Add one doc to the store (caller supplies title + url + body bytes).
139// Returns the rowid (>= 0) or -verdict.
140func nx_wiki_doc_store_add(s: *NxWikiDocStore,
141 title: *u8, title_n: i64,
142 url: *u8, url_n: i64,
143 body: *u8, body_n: i64) -> i64 {
144 if s.valid != 1 { return 0 - NX_WIB_BAD_INPUT }
145 if s.doc_count >= s.docs_cap { return 0 - NX_WIB_DOC_OVERFLOW }
146 if title_n < 0 { return 0 - NX_WIB_BAD_INPUT }
147 if title_n > NX_WIB_MAX_TITLE_LEN { return 0 - NX_WIB_BAD_INPUT }
148 if url_n < 0 { return 0 - NX_WIB_BAD_INPUT }
149 if url_n > NX_WIB_MAX_URL_LEN { return 0 - NX_WIB_BAD_INPUT }
150 if body_n < 0 { return 0 - NX_WIB_BAD_INPUT }
151 if body_n > NX_WIB_MAX_BODY_LEN { return 0 - NX_WIB_BAD_INPUT }
152
153 let rowid: i64 = s.doc_count
154
155 // Title
156 let used_t_p: *i64 = (sys_mmap(8)) as *i64
157 used_t_p[0] = s.titles_pool_used
158 let t_off: i64 = nx_wib_pool_alloc(s.titles_pool, s.titles_pool_cap, used_t_p, title_n)
159 if t_off < 0 { return t_off }
160 var i: i64 = 0
161 while i < title_n { s.titles_pool[t_off + i] = title[i]; i = i + 1 }
162 s.titles_pool_used = used_t_p[0]
163 s.titles_offs[rowid] = t_off
164 s.titles_lens[rowid] = title_n
165
166 // URL
167 let used_u_p: *i64 = (sys_mmap(8)) as *i64
168 used_u_p[0] = s.urls_pool_used
169 let u_off: i64 = nx_wib_pool_alloc(s.urls_pool, s.urls_pool_cap, used_u_p, url_n)
170 if u_off < 0 { return u_off }
171 i = 0
172 while i < url_n { s.urls_pool[u_off + i] = url[i]; i = i + 1 }
173 s.urls_pool_used = used_u_p[0]
174 s.urls_offs[rowid] = u_off
175 s.urls_lens[rowid] = url_n
176
177 // Body
178 let used_b_p: *i64 = (sys_mmap(8)) as *i64
179 used_b_p[0] = s.bodies_pool_used
180 let b_off: i64 = nx_wib_pool_alloc(s.bodies_pool, s.bodies_pool_cap, used_b_p, body_n)
181 if b_off < 0 { return b_off }
182 i = 0
183 while i < body_n {
184 if i >= NX_WIB_MAX_BODY_LEN { return 0 - NX_WIB_BUF_OVERFLOW }
185 s.bodies_pool[b_off + i] = body[i]
186 i = i + 1
187 }
188 s.bodies_pool_used = used_b_p[0]
189 s.bodies_offs[rowid] = b_off
190 s.bodies_lens[rowid] = body_n
191
192 s.doc_count = s.doc_count + 1
193 return rowid
194}
195
196// ===== Lookup API (for resolver) =================================================
197
198func nx_wiki_doc_store_lookup(s: *NxWikiDocStore, rowid: i64,
199 out_title_ptr: *i64, out_title_n: *i64,
200 out_url_ptr: *i64, out_url_n: *i64,
201 out_body_ptr: *i64, out_body_n: *i64) -> i64 {
202 if s.valid != 1 { return 0 - NX_WIB_BAD_INPUT }
203 if rowid < 0 { return 0 - NX_WIB_BAD_INPUT }
204 if rowid >= s.doc_count { return 0 - NX_WIB_BAD_INPUT }
205 if (out_title_ptr as i64) == 0 { return 0 - NX_WIB_BAD_INPUT }
206 if (out_title_n as i64) == 0 { return 0 - NX_WIB_BAD_INPUT }
207 if (out_url_ptr as i64) == 0 { return 0 - NX_WIB_BAD_INPUT }
208 if (out_url_n as i64) == 0 { return 0 - NX_WIB_BAD_INPUT }
209 if (out_body_ptr as i64) == 0 { return 0 - NX_WIB_BAD_INPUT }
210 if (out_body_n as i64) == 0 { return 0 - NX_WIB_BAD_INPUT }
211
212 out_title_ptr[0] = (s.titles_pool as i64) + s.titles_offs[rowid]
213 out_title_n[0] = s.titles_lens[rowid]
214 out_url_ptr[0] = (s.urls_pool as i64) + s.urls_offs[rowid]
215 out_url_n[0] = s.urls_lens[rowid]
216 out_body_ptr[0] = (s.bodies_pool as i64) + s.bodies_offs[rowid]
217 out_body_n[0] = s.bodies_lens[rowid]
218 return NX_WIB_OK
219}
220
221func nx_wiki_doc_store_count(s: *NxWikiDocStore) -> i64 {
222 if s.valid != 1 { return 0 }
223 return s.doc_count
224}
225
226// Linear-scan URL -> rowid lookup. V1 acceptable for ~100 docs;
227// V2 swaps for a hash-map (nx_kv_store-backed) when corpus > 1000.
228// Returns rowid (>= 0) on hit, -1 on miss.
229
230func nx_wiki_doc_store_find_by_url(s: *NxWikiDocStore,
231 url: *u8, url_n: i64) -> i64 {
232 if s.valid != 1 { return 0 - 1 }
233 if (url as i64) == 0 { return 0 - 1 }
234 if url_n < 1 { return 0 - 1 }
235 if url_n > NX_WIB_MAX_URL_LEN { return 0 - 1 }
236
237 var i: i64 = 0
238 while i < s.doc_count {
239 if i >= NX_WIB_LOOP_BUDGET_CAP { return 0 - 1 }
240 let stored_n: i64 = s.urls_lens[i]
241 if stored_n == url_n {
242 let stored: *u8 = ((s.urls_pool as i64) + s.urls_offs[i]) as *u8
243 // Flat byte-compare with a single equality flag.
244 // PRIOR shape used a nested-if increment
245 // `if j <= url_n { if j < url_n { j = j + 1 } }` which the
246 // native compiler MISCOMPILES (DEBT T#native-nested-if-incr,
247 // repro runtime/_hdl_build/_probe_find.nx: faithful copy
248 // returns -1, flat rewrite returns 0 on identical data) ->
249 // find_by_url always missed -> wiki render 404. Flat form is
250 // correct AND idiomatic.
251 var j: i64 = 0
252 var eq: i64 = 1
253 while j < url_n {
254 if stored[j] != url[j] { eq = 0 }
255 j = j + 1
256 }
257 if eq == 1 { return i }
258 }
259 i = i + 1
260 }
261 return 0 - 1
262}
263
264// ===== Title extraction from markdown body =================================================
265//
266// First "# " heading -> title; fallback to first 60 bytes of first
267// non-empty line; fallback to "(untitled)".
268
269func nx_wib_extract_md_title(body: *u8, body_n: i64,
270 out_off: *i64, out_len: *i64) -> i64 {
271 if (out_off as i64) == 0 { return 0 - NX_WIB_BAD_INPUT }
272 if (out_len as i64) == 0 { return 0 - NX_WIB_BAD_INPUT }
273 out_off[0] = 0 - 1
274 out_len[0] = 0
275
276 var i: i64 = 0
277 var iter: i64 = 0
278 while i < body_n {
279 if iter >= NX_WIB_LOOP_BUDGET_CAP { return 0 - NX_WIB_LOOP_BUDGET }
280 iter = iter + 1
281 // Look for "# " at line start.
282 var line_start: i64 = i
283 if (body[i] as i64) == 0x23 { // '#'
284 if i + 1 < body_n {
285 if (body[i + 1] as i64) == 0x20 { // ' '
286 let tstart: i64 = i + 2
287 // ROOT FIX (2026-06-15): scan to the END OF THE HEADING LINE and
288 // record the newline POSITION as the title end. The prior loop set
289 // tend=body_n+1 on the newline but then derived real_end=body_n,
290 // so title_n=body_n-tstart -- the ENTIRE rest of the document
291 // (capped at 200) leaked into <title>/nishi-title as raw markdown.
292 // Flat scan: heading_end = first 0x0A at/after tstart, else body_n.
293 var heading_end: i64 = body_n
294 var scan: i64 = tstart
295 var found_nl: i64 = 0
296 while scan < body_n {
297 if found_nl == 0 {
298 if (body[scan] as i64) == 0x0A { heading_end = scan; found_nl = 1 }
299 }
300 scan = scan + 1
301 }
302 // trim a trailing CR (CRLF line endings) so the title is clean text.
303 if heading_end > tstart {
304 if (body[heading_end - 1] as i64) == 0x0D { heading_end = heading_end - 1 }
305 }
306 var title_n: i64 = heading_end - tstart
307 if title_n < 0 { title_n = 0 }
308 if title_n > NX_WIB_MAX_TITLE_LEN { title_n = NX_WIB_MAX_TITLE_LEN }
309 out_off[0] = tstart
310 out_len[0] = title_n
311 return NX_WIB_OK
312 }
313 }
314 }
315 // Skip to next line.
316 while i < body_n {
317 if (body[i] as i64) == 0x0A { i = body_n + 1 }
318 if i <= body_n { if i < body_n { i = i + 1 } }
319 }
320 var i_next: i64 = i + 1
321 if i > body_n { i_next = body_n }
322 i = i_next
323 if i <= line_start { i = line_start + 1 }
324 }
325 return NX_WIB_OK // no title; caller falls back
326}
327
328// ===== NxWikiIndexBuilder: orchestrates two-pass NxInvIndex build =================================================
329
330struct NxWikiIndexBuilder {
331 store: *NxWikiDocStore,
332 inv_index: *NxInvIndex,
333 finalized: i64,
334 valid: i64,
335}
336
337func nx_wiki_index_builder_init(b: *NxWikiIndexBuilder,
338 store: *NxWikiDocStore,
339 expected_docs: i64) -> i64 {
340 if (b as i64) == 0 { return 0 - NX_WIB_BAD_INPUT }
341 if store.valid != 1 { return 0 - NX_WIB_BAD_INPUT }
342 if expected_docs < 1 { return 0 - NX_WIB_BAD_INPUT }
343 b.store = store
344 // DEBT T#parser-field-assign-imported-call: the native compiler
345 // desyncs on a DIRECT `field = imported_call(args)` assignment
346 // (minimal repro: runtime/_hdl_build/_probe_init.nx). Binding the
347 // call result to a temp first is the correct, idiomatic form and
348 // sidesteps the desync until the parser root-fix lands.
349 let inv: *NxInvIndex = nx_inv_new(expected_docs)
350 b.inv_index = inv
351 if (b.inv_index as i64) == 0 { return 0 - NX_WIB_INDEX_INIT_FAILED }
352 b.finalized = 0
353 b.valid = 1
354 return NX_WIB_OK
355}
356
357// Add a doc (PASS 1: count tokens + insert into store).
358// Returns rowid (>= 0) or -verdict.
359func nx_wiki_index_builder_add(b: *NxWikiIndexBuilder,
360 title: *u8, title_n: i64,
361 url: *u8, url_n: i64,
362 body: *u8, body_n: i64) -> i64 {
363 if b.valid != 1 { return 0 - NX_WIB_BAD_INPUT }
364 if b.finalized == 1 { return 0 - NX_WIB_DOUBLE_FINALIZE }
365
366 let rowid: i64 = nx_wiki_doc_store_add(b.store, title, title_n, url, url_n, body, body_n)
367 if rowid < 0 { return rowid }
368
369 // PASS 1: count_token for each token in body.
370 let rc: i64 = nx_inv_index_row(b.inv_index, body, body_n, rowid)
371 if rc != NX_INV_OK { return 0 - NX_WIB_PASS1_FAILED }
372 return rowid
373}
374
375// Finalize: allocate posting arrays + PASS 2 emit.
376func nx_wiki_index_builder_finalize(b: *NxWikiIndexBuilder) -> i64 {
377 if b.valid != 1 { return 0 - NX_WIB_BAD_INPUT }
378 if b.finalized == 1 { return 0 - NX_WIB_DOUBLE_FINALIZE }
379
380 let rc_fin: i64 = nx_inv_finalize_offsets(b.inv_index)
381 if rc_fin != NX_INV_OK { return 0 - NX_WIB_FINALIZE_FAILED }
382
383 // PASS 2: walk doc store again + emit_row per doc.
384 var i: i64 = 0
385 while i < b.store.doc_count {
386 if i >= NX_WIB_LOOP_BUDGET_CAP { return 0 - NX_WIB_LOOP_BUDGET }
387 let body_ptr: *u8 = ((b.store.bodies_pool as i64) + b.store.bodies_offs[i]) as *u8
388 let body_n: i64 = b.store.bodies_lens[i]
389 let rc: i64 = nx_inv_emit_row(b.inv_index, body_ptr, body_n, i)
390 if rc != 0 { return 0 - NX_WIB_PASS2_FAILED }
391 i = i + 1
392 }
393
394 b.finalized = 1
395 return NX_WIB_OK
396}
397
398// Accessor for the finalized index (handed to nx_search_flow_execute_onsite).
399func nx_wiki_index_builder_get_index(b: *NxWikiIndexBuilder) -> *NxInvIndex {
400 if b.valid != 1 { return 0 as *NxInvIndex }
401 if b.finalized == 0 { return 0 as *NxInvIndex } // not ready
402 return b.inv_index
403}
404
405func nx_wiki_index_builder_get_store(b: *NxWikiIndexBuilder) -> *NxWikiDocStore {
406 if b.valid != 1 { return 0 as *NxWikiDocStore }
407 return b.store
408}
409
410// ===== Convenience: add doc by reading bytes from a file path =================================================
411//
412// Reads file via sys_read_file_x86_64; extracts title via
413// nx_wib_extract_md_title; uses caller-supplied URL.
414
415func nx_wiki_index_builder_add_path(b: *NxWikiIndexBuilder,
416 path_z: *u8,
417 url: *u8, url_n: i64) -> i64 {
418 if b.valid != 1 { return 0 - NX_WIB_BAD_INPUT }
419 if (path_z as i64) == 0 { return 0 - NX_WIB_BAD_INPUT }
420
421 let lenbox: *u8 = sys_mmap(8)
422 let lp: *i64 = lenbox as *i64
423 lp[0] = 0
424 let body: *u8 = sys_read_file(path_z, lp)
425 if (body as i64) == 0 { return 0 - NX_WIB_READ_FAILED }
426 let body_n: i64 = lp[0]
427 if body_n < 1 { return 0 - NX_WIB_READ_FAILED }
428 if body_n > NX_WIB_MAX_BODY_LEN { return 0 - NX_WIB_BUF_OVERFLOW }
429
430 // Extract title from first '#' heading.
431 let t_off: *i64 = (sys_mmap(8)) as *i64
432 let t_len: *i64 = (sys_mmap(8)) as *i64
433 t_off[0] = 0 - 1; t_len[0] = 0
434 let rc_t: i64 = nx_wib_extract_md_title(body, body_n, t_off, t_len)
435 if rc_t != NX_WIB_OK { return rc_t }
436
437 var title_ptr: *u8 = "(untitled)" as *u8
438 var title_n: i64 = 10
439 if t_off[0] >= 0 {
440 if t_len[0] > 0 {
441 title_ptr = (body as i64 + t_off[0]) as *u8
442 title_n = t_len[0]
443 }
444 }
445
446 return nx_wiki_index_builder_add(b, title_ptr, title_n, url, url_n, body, body_n)
447}