nx_libindex_lib.nx source
↩ module page · 370 lines · 16396 B
1// nx_libindex_lib.nx -- THE LIBRARY DISCOVERY INDEX, function side (PG1, procgen.plan, 2026-08-24).
2//
3// THE DEFECT THIS CLOSES, OPERATOR-NAMED AND MEASURED: nx_capsearch -- the organ CLAUDE.md orders run BEFORE
4// BUILDING ANYTHING -- ranks REGISTERED MCP TOOLS. A library has no registration row by design (nx_catalog
5// says so itself: for a lib the adoption question is importer count, not binary presence), so every library
6// in the estate is INVISIBLE to the one instrument that exists to answer "does this already exist?".
7//
8// MEASURED CONSEQUENCE, same session: EIGHT private isqrt copies (gsplat, bodybench, gltf2mesh, gtlabels gate,
9// worldgen, worley, anatstack, part_solver) written beside each other because the discovery organ could never
10// have shown them; two FALSE ABSENT rows on one compare board; a capsearch zero read as corroboration of
11// absence. Any absence claim resting on a capsearch zero is UNPROVEN until re-checked here.
12//
13// COMPOSES THE INCUMBENT, ADDS NO SECOND RULER: the LIB test is nx_catalog_lib's cl_is_lib, the same
14// classifier the adoption ladder and the compare generator already use, so this index cannot disagree with
15// the board about what a library is. The symbol scan uses the beginning-of-line "func " rule that
16// cl_symbol_declared already applies, so a symbol listed here is one that ruler will confirm.
17//
18// THE PUBLISHED ARTIFACT IS A SOVEREIGN ROW PLANE, NOT A FILE (corrected 2026-08-24, same day: the first cut
19// wrote knowledge/status/libindex.tsv, which is exactly the third-party-format-in-the-plane shape the
20// operator retired on 2026-08-13). Rows stream to an organ-owned SCRATCH descriptor while the tree is walked,
21// then the scratch is read back whole (sys_read_file sizes from the file and cannot short-read) and seeded
22// into the store at `prefix` under the plane lock in ONE commit -- sts_seed, the same primitive nx_store_put
23// uses. Readers use sts_load_fit, which returns the whole plane or REFUSES; it never hands back a partial.
24// Row schema: name<TAB>path<TAB>title<TAB> sym1 sym2 ... (id = col 0, the N-col row-plane contract).
25//
26// SPLIT lib/program so the GATE can compose these functions in-process instead of forking a deployed binary.
27//
28// NO SILENT CAPS. The only bounded buffers are the kernel dirent block (composed from nx_fs at 65536) and a
29// filename buffer; a bind is COUNTED and forces corpus_complete=0 rather than truncating.
30//
31// THE PARTITION IS PRINTED AND MUST SUM: nx_files == libs + programs + unreadable. And a corpus in which not
32// one file resolved to lib or program is INCOMPLETE by definition -- measured on the first run, 18,882
33// unreadable files summed to a clean partition and read complete with an empty index.
34// license_tier: ORIGINAL
35
36import "nx_syscalls.nx"
37import "nx_catalog_lib.nx"
38import "nx_store_seed_lib.nx"
39
40// Composed from nx_fs.fs_ls, not chosen here: the same kernel dirent block size the estate's own ls uses.
41const LI_DIRBUF: i64 = 65536
42// POSIX NAME_MAX is 255. 512 is 2x headroom and matches nx_fs.fs_ls. A longer name is an anomaly, not a
43// tuning question, so it is counted and forces an incomplete verdict rather than being truncated.
44const LI_NAME_CAP: i64 = 512
45const LI_PATH_CAP: i64 = 1024
46
47const LI_O_DIRECTORY: i64 = 0x10000
48const LI_O_WRONLY_CREAT_TRUNC: i64 = 0x241
49const LI_MODE_644: i64 = 420
50const LI_AT_FDCWD: i64 = 0 - 100
51const LI_SYS_OPENAT: i64 = 257
52const LI_SYS_GETDENTS64: i64 = 217
53
54// dirent64 field offsets -- fixed by the kernel ABI, not chosen by us.
55const LI_DIRENT_RECLEN_OFF: i64 = 16
56const LI_DIRENT_TYPE_OFF: i64 = 18
57const LI_DIRENT_NAME_OFF: i64 = 19
58const LI_DT_DIR: i64 = 4
59
60const LI_TAB: i64 = 9
61const LI_NL: i64 = 10
62const LI_SP: i64 = 32
63const LI_DOT: i64 = 46
64const LI_SLASH: i64 = 47
65const LI_LPAREN: i64 = 40
66
67// Census slots, named so the partition can be printed and reconciled.
68const LI_C_ENTRIES: i64 = 0
69const LI_C_NX: i64 = 1
70const LI_C_LIB: i64 = 2
71const LI_C_PROG: i64 = 3
72const LI_C_UNREAD: i64 = 4
73const LI_C_SYMS: i64 = 5
74const LI_C_NAMEBIND: i64 = 6
75const LI_C_ROWS: i64 = 7 // rows the plane commit reported, or -1 if the seed refused
76const LI_C_N: i64 = 8
77
78func li_cen() -> *i64 {
79 let c: *i64 = sys_mmap(LI_C_N * 8) as *i64
80 var i: i64 = 0
81 while i < LI_C_N { c[i] = 0; i = i + 1 }
82 return c
83}
84
85func li_w(fd: i64, s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(fd, s, n); return 0 }
86func li_out(s: *u8) -> i64 { return li_w(1, s) }
87
88func li_num_fd(fd: i64, v0: i64) -> i64 {
89 let b: *u8 = sys_mmap(32)
90 var v: i64 = v0
91 var neg: i64 = 0
92 if v < 0 { neg = 1; v = 0 - v }
93 var i: i64 = 31
94 b[i] = 0 as u8
95 if v == 0 { i = i - 1; b[i] = 48 as u8 }
96 while v > 0 { i = i - 1; b[i] = ((v % 10) + 48) as u8; v = v / 10 }
97 if neg == 1 { i = i - 1; b[i] = 45 as u8 }
98 var n: i64 = 0
99 while b[i+n] != (0 as u8) { n = n + 1 }
100 sys_write(fd, b + i, n)
101 sys_munmap(b, 32)
102 return 0
103}
104func li_num(v: i64) -> i64 { return li_num_fd(1, v) }
105
106func li_byte(fd: i64, c: i64) -> i64 {
107 let b: *u8 = sys_mmap(8)
108 b[0] = c as u8
109 sys_write(fd, b, 1)
110 sys_munmap(b, 8)
111 return 0
112}
113
114func li_ends_nx(nm: *u8, nl: i64) -> i64 {
115 if nl < 4 { return 0 }
116 if nm[nl-3] != (LI_DOT as u8) { return 0 }
117 if nm[nl-2] != (110 as u8) { return 0 }
118 if nm[nl-1] != (120 as u8) { return 0 }
119 return 1
120}
121
122// The FIRST comment line is the organ's own one-line statement of purpose. Streamed with tabs scrubbed to
123// spaces so a title can never forge a column boundary in the plane row.
124func li_emit_title(fd: i64, b: *u8, n: i64) -> i64 {
125 if n < 3 { return 0 }
126 if b[0] != (LI_SLASH as u8) { return 0 }
127 if b[1] != (LI_SLASH as u8) { return 0 }
128 var i: i64 = 2
129 var go: i64 = 1
130 while go == 1 {
131 if i >= n { go = 0 }
132 if go == 1 { if b[i] == (LI_NL as u8) { go = 0 } }
133 if go == 1 {
134 var c: i64 = b[i] as i64
135 if c == LI_TAB { c = LI_SP }
136 li_byte(fd, c)
137 i = i + 1
138 }
139 }
140 return 0
141}
142
143// Beginning-of-line "func " -- the same rule cl_symbol_declared applies, so this index and that ruler cannot
144// disagree about whether a symbol is declared.
145func li_emit_syms(fd: i64, b: *u8, n: i64, cen: *i64) -> i64 {
146 var i: i64 = 0
147 while i < n {
148 var atbol: i64 = 0
149 if i == 0 { atbol = 1 }
150 if i > 0 { if b[i-1] == (LI_NL as u8) { atbol = 1 } }
151 if atbol == 1 { if i + 5 < n {
152 if b[i] == (102 as u8) { if b[i+1] == (117 as u8) { if b[i+2] == (110 as u8) {
153 if b[i+3] == (99 as u8) { if b[i+4] == (LI_SP as u8) {
154 li_byte(fd, LI_SP)
155 var k: i64 = i + 5
156 var go: i64 = 1
157 while go == 1 {
158 if k >= n { go = 0 }
159 if go == 1 {
160 let c: i64 = b[k] as i64
161 if c == LI_LPAREN { go = 0 }
162 if c == LI_SP { go = 0 }
163 if c == LI_NL { go = 0 }
164 if go == 1 { li_byte(fd, c); k = k + 1 }
165 }
166 }
167 cen[LI_C_SYMS] = cen[LI_C_SYMS] + 1
168 } } } } }
169 } }
170 i = i + 1
171 }
172 return 0
173}
174
175func li_scan_dir(root: *u8, ofd: i64, cen: *i64) -> i64 {
176 let dfd: i64 = __syscall(LI_SYS_OPENAT, LI_AT_FDCWD, root, LI_O_DIRECTORY, 0, 0, 0)
177 if dfd < 0 { return 0 - 1 }
178 let buf: *u8 = sys_mmap(LI_DIRBUF)
179 let nm: *u8 = sys_mmap(LI_NAME_CAP)
180 let path: *u8 = sys_mmap(LI_PATH_CAP)
181 let lp: *i64 = sys_mmap(16) as *i64
182 var go: i64 = 1
183 // LOOP until getdents64 returns 0. One call is not a directory listing: a single-call read silently
184 // publishes a PREFIX of a big directory as if it were the whole population.
185 while go == 1 {
186 let nread: i64 = __syscall(LI_SYS_GETDENTS64, dfd, buf, LI_DIRBUF, 0, 0, 0)
187 if nread <= 0 { go = 0 }
188 if nread > 0 {
189 var pos: i64 = 0
190 while pos < nread {
191 let reclen: i64 = (buf[pos+LI_DIRENT_RECLEN_OFF] as i64) | ((buf[pos+LI_DIRENT_RECLEN_OFF+1] as i64) << 8)
192 let dtype: i64 = buf[pos+LI_DIRENT_TYPE_OFF] as i64
193 var nl: i64 = 0
194 var bind: i64 = 0
195 while buf[pos+LI_DIRENT_NAME_OFF+nl] != (0 as u8) {
196 if nl < LI_NAME_CAP - 1 { nm[nl] = buf[pos+LI_DIRENT_NAME_OFF+nl] }
197 if nl >= LI_NAME_CAP - 1 { bind = 1 }
198 nl = nl + 1
199 }
200 if bind == 1 { cen[LI_C_NAMEBIND] = cen[LI_C_NAMEBIND] + 1; nl = LI_NAME_CAP - 1 }
201 nm[nl] = 0 as u8
202 cen[LI_C_ENTRIES] = cen[LI_C_ENTRIES] + 1
203 if dtype != LI_DT_DIR { if li_ends_nx(nm, nl) == 1 {
204 cen[LI_C_NX] = cen[LI_C_NX] + 1
205 cl_cat3(path, root, "/" as *u8, nm)
206 let isl: i64 = cl_is_lib(path)
207 if isl < 0 { cen[LI_C_UNREAD] = cen[LI_C_UNREAD] + 1 }
208 if isl == 0 { cen[LI_C_PROG] = cen[LI_C_PROG] + 1 }
209 if isl == 1 {
210 cen[LI_C_LIB] = cen[LI_C_LIB] + 1
211 let b: *u8 = cl_read_whole(path, lp)
212 if (b as i64) != 0 {
213 let n: i64 = lp[0]
214 li_w(ofd, nm)
215 li_byte(ofd, LI_TAB)
216 li_w(ofd, path)
217 li_byte(ofd, LI_TAB)
218 li_emit_title(ofd, b, n)
219 li_byte(ofd, LI_TAB)
220 li_emit_syms(ofd, b, n, cen)
221 li_byte(ofd, LI_NL)
222 sys_munmap(b, n + 8)
223 }
224 }
225 } }
226 if reclen <= 0 { pos = nread }
227 if reclen > 0 { pos = pos + reclen }
228 }
229 }
230 }
231 sys_close(dfd)
232 sys_munmap(buf, LI_DIRBUF)
233 sys_munmap(nm, LI_NAME_CAP)
234 sys_munmap(path, LI_PATH_CAP)
235 return 0
236}
237
238// Walk both roots into the scratch descriptor, then seed the plane at `prefix` in one locked commit.
239// Returns 0 GREEN, 1 INCOMPLETE (partition leaked, a name bound, a root would not open, nothing classified),
240// 2 REFUSED (scratch unopenable or the plane commit refused). The census is written into cen so a caller --
241// including the gate -- asserts on the numbers, never on printed text.
242func li_build_to(scratch: *u8, prefix: *u8, cen: *i64, quiet: i64) -> i64 {
243 let ofd: i64 = __syscall(LI_SYS_OPENAT, LI_AT_FDCWD, scratch, LI_O_WRONLY_CREAT_TRUNC, LI_MODE_644, 0, 0)
244 if ofd < 0 {
245 if quiet == 0 { li_out("NX-LIBINDEX REFUSED cannot-open-scratch " as *u8); li_out(scratch); li_out("\n" as *u8) }
246 return 2
247 }
248 // Both roots the builder itself probes. Reporting on one would measure a narrower subject than the name
249 // implies -- the exact defect class this index exists to close.
250 let r1: i64 = li_scan_dir("buildroot/runtime/_hdl_build" as *u8, ofd, cen)
251 let r2: i64 = li_scan_dir("buildroot/runtime" as *u8, ofd, cen)
252 sys_close(ofd)
253 let roots_ok: i64 = 2 + r1 + r2
254 let part: i64 = cen[LI_C_LIB] + cen[LI_C_PROG] + cen[LI_C_UNREAD]
255 var complete: i64 = 1
256 if cen[LI_C_NAMEBIND] != 0 { complete = 0 }
257 if part != cen[LI_C_NX] { complete = 0 }
258 if roots_ok != 2 { complete = 0 }
259 // MEASURED 2026-08-24 on the first run: 18,882 files ALL unreadable summed to a clean partition and this
260 // reported complete=1 with an empty index. A partition that reconciles proves nothing was LOST; it does
261 // not prove anything was CLASSIFIED. A corpus in which not one file resolved to lib or program is an
262 // environment failure wearing a green, so it is incomplete by definition.
263 if cen[LI_C_LIB] + cen[LI_C_PROG] == 0 { complete = 0 }
264
265 // ---- publish: the row plane, ONE commit under the plane lock ------------------------------------
266 // sys_read_file sizes its buffer from the scratch file and cannot short-read; sts_seed commits the
267 // whole buffer or returns -1. A refused seed is a REFUSAL, not an empty plane -- the previous plane
268 // contents are left exactly as they were.
269 let lp: *i64 = sys_mmap(16) as *i64
270 let rows_buf: *u8 = sys_read_file(scratch, lp)
271 var rows: i64 = 0 - 1
272 if (rows_buf as i64) != 0 {
273 let nbytes: i64 = lp[0]
274 let lk: i64 = sts_lock(prefix)
275 if lk >= 0 {
276 rows = sts_seed(prefix, rows_buf, nbytes)
277 sts_unlock(lk)
278 }
279 }
280 cen[LI_C_ROWS] = rows
281 var rc: i64 = 0
282 if complete == 0 { rc = 1 }
283 if rows < 0 { rc = 2 }
284 if quiet == 0 {
285 li_out("NX-LIBINDEX plane=" as *u8); li_out(prefix)
286 li_out(" entries=" as *u8); li_num(cen[LI_C_ENTRIES])
287 li_out(" nx_files=" as *u8); li_num(cen[LI_C_NX])
288 li_out(" libs=" as *u8); li_num(cen[LI_C_LIB])
289 li_out(" programs=" as *u8); li_num(cen[LI_C_PROG])
290 li_out(" unreadable=" as *u8); li_num(cen[LI_C_UNREAD])
291 li_out(" symbols=" as *u8); li_num(cen[LI_C_SYMS])
292 li_out(" roots_opened=" as *u8); li_num(roots_ok)
293 li_out(" name_binds=" as *u8); li_num(cen[LI_C_NAMEBIND])
294 li_out(" partition=" as *u8); li_num(part); li_out("/" as *u8); li_num(cen[LI_C_NX])
295 if part == cen[LI_C_NX] { li_out(" RECONCILES" as *u8) }
296 if part != cen[LI_C_NX] { li_out(" LEAK" as *u8) }
297 li_out(" corpus_complete=" as *u8); li_num(complete)
298 li_out(" plane_rows=" as *u8); li_num(rows)
299 if rows < 0 { li_out(" PLANE-SEED-REFUSED (previous plane untouched)" as *u8) }
300 li_out("\n" as *u8)
301 }
302 return rc
303}
304
305// Substring match over the plane. Deliberately NOT a scored ranker: nx_capsearch owns ranking, and a second
306// scorer beside it would be the duplicate-ruler defect this organ exists to complain about. Returns the
307// number of matching rows, or a negative refusal: -2 no plane / plane unreadable (sts_load_fit refused --
308// it never returns a partial), -3 empty plane. "No plane" and "no matches" are different facts and must
309// never share an answer.
310func li_find_in(prefix: *u8, terms: *i64, nterms: i64, quiet: i64) -> i64 {
311 let lp: *i64 = sys_mmap(16) as *i64
312 let b: *u8 = sts_load_fit(prefix, lp)
313 if (b as i64) == 0 {
314 if quiet == 0 { li_out("NX-LIBINDEX-FIND REFUSED no-plane -- run `nx_libindex build` first\n" as *u8) }
315 return 0 - 2
316 }
317 let n: i64 = lp[0]
318 if n <= 0 {
319 if quiet == 0 { li_out("NX-LIBINDEX-FIND REFUSED empty-plane\n" as *u8) }
320 return 0 - 3
321 }
322 var hits: i64 = 0
323 var rows: i64 = 0
324 var ls: i64 = 0
325 var i: i64 = 0
326 while i <= n {
327 var eol: i64 = 0
328 if i == n { eol = 1 }
329 if i < n { if b[i] == (LI_NL as u8) { eol = 1 } }
330 if eol == 1 {
331 if i > ls {
332 rows = rows + 1
333 var matched: i64 = 0
334 var t: i64 = 0
335 while t < nterms {
336 let q: *u8 = terms[t] as *u8
337 let ql: i64 = cl_slen(q)
338 var p: i64 = ls
339 var found: i64 = 0
340 while p + ql <= i {
341 var k: i64 = 0
342 var same: i64 = 1
343 while k < ql { if b[p+k] != q[k] { same = 0; k = ql } k = k + 1 }
344 if same == 1 { found = 1; p = i }
345 p = p + 1
346 }
347 if found == 1 { matched = matched + 1 }
348 t = t + 1
349 }
350 if matched > 0 {
351 hits = hits + 1
352 if quiet == 0 {
353 li_out(" [" as *u8); li_num(matched); li_out("/" as *u8); li_num(nterms); li_out("] " as *u8)
354 sys_write(1, b + ls, i - ls)
355 li_out("\n" as *u8)
356 }
357 }
358 }
359 ls = i + 1
360 }
361 i = i + 1
362 }
363 if quiet == 0 {
364 li_out("NX-LIBINDEX-FIND rows_scanned=" as *u8); li_num(rows)
365 li_out(" hits=" as *u8); li_num(hits)
366 li_out(" terms=" as *u8); li_num(nterms)
367 li_out("\n" as *u8)
368 }
369 return hits
370}