nx_wiki_content_loader.nx source
↩ module page · 861 lines · 38497 B
1// nx_wiki_content_loader.nx -- materialize charters into per-site wiki.
2//
3// V-WIKI-CONTENT-LOAD-1 per NISHI_WIKI_CONTENT_INDEX v1.0 §5 +
4// operator cardinal 2026-05-27 *"make sure all this information gets
5// into the wiki"*.
6//
7// At daemon startup, for each canonical charter listed in §1 of
8// NISHI_WIKI_CONTENT_INDEX.md:
9// 1. sys_read_file(<silicon_repo>/<charter_path>) -> bytes
10// 2. nx_wiki_index_builder_add(builder, title, "/wiki/<slug>",
11// body, body_n)
12// Result: each charter becomes a searchable + readable wiki page at
13// nishifamily.com/wiki/<slug>.
14//
15// COMPOSES (per "avoid duplicate primitives" cardinal):
16// wiki/nx_wiki_index_builder doc_store + builder for indexing
17// nx_syscalls sys_read_file for charter content
18//
19// COMPOSED BY:
20// wiki/nx_wiki_https_daemon_mv setup_site calls this after init
21//
22// SCOPE LIMIT (per M6 anti-pretend-stub):
23// V1 ships a HARDCODED charter list per NISHI_WIKI_CONTENT_INDEX
24// §1. V+1 reads the index .md file at startup + parses §1 tables
25// to discover charters dynamically (sys_getdents queued per
26// project-wiki-v1-complete-2026-05-27 honest gap). Hardcoded list
27// below MUST be kept in sync with NISHI_WIKI_CONTENT_INDEX §1
28// per its §6 update discipline.
29//
30// Operator-side: daemon must run where the silicon repo is
31// readable. Default path /mnt/c/Users/elder/nishi-silicon on WSL;
32// override via NX_WCL_SILICON_ROOT env var (V+1).
33
34import "nx_syscalls.nx"
35import "wiki/nx_wiki_index_builder.nx"
36
37// ===== Sealed verdict surface (codes 1380-1399) =================================================
38const NX_WCL_OK: i64 = 0
39const NX_WCL_BAD_INPUT: i64 = 1380
40const NX_WCL_FILE_NOT_FOUND: i64 = 1381 // silicon repo missing or path wrong
41const NX_WCL_INDEX_ADD_FAILED: i64 = 1382
42const NX_WCL_DIR_OPEN_FAILED: i64 = 1383 // V+1 auto-discover: content dir not openable
43
44// ===== V+1 auto-discovery constants (M7) =================================================
45//
46// Closes the V+1 TODO in this file's header + NISHI_WIKI_CONTENT_INDEX
47// §0 ("V+1 auto-discovers via filesystem walk"). A struct-free
48// getdents64 walk over the content dir turns every *.md file into a
49// page WITHOUT a code edit -- drop a new charter into the dir and it
50// becomes /wiki/<slug> + a search hit on next load.
51const NX_WCL_DIRBUF_BYTES: i64 = 262144 // getdents64 batch buffer (256 KiB)
52const NX_WCL_MAX_DISCOVER: i64 = 1000 // hard ceiling on pages discovered (JPL rule 2)
53const NX_WCL_MAX_NAME_LEN: i64 = 255 // single dir-entry name cap
54const NX_WCL_MAX_SLUG_LEN: i64 = 256 // derived slug cap
55const NX_WCL_MAX_URL_LEN: i64 = 512 // "/wiki/<slug>" cap
56const NX_WCL_FALLBACK_TITLE: *u8 = "(untitled)" as *u8
57const NX_WCL_FALLBACK_TITLE_N: i64 = 10
58
59// ===== Named constants (M7) =================================================
60//
61// V1: silicon repo path is hardcoded. V+1: env-var override.
62//
63// On WSL: /mnt/c/Users/elder/nishi-silicon/<charter>.md
64// On Linux: /home/<user>/nishi-silicon/<charter>.md (operator config)
65//
66// Per NISHI_SITES_INVENTORY operator-side: substrate-host machine has
67// silicon repo at /mnt/c/Users/elder/nishi-silicon by convention.
68
69// V-LANGEXT M1: parser now accepts string-literal in const decls
70// natively (no helper-fn workaround).
71// DEPLOY CONFIG (2026-06-16): content root pointed at the NAS import dir so the live wiki daemon
72// auto-discovers .md files staged there (drop a file in -> /wiki/<slug> on next (re)start). The dev
73// path was /mnt/c/Users/elder/nishi-silicon/ (Cardinal 17 env-override is the eventual clean form).
74const NX_WCL_SILICON_ROOT: *u8 = "/volume1/homes/elderwesto/nishihost/wiki-content/" as *u8
75const NX_WCL_SILICON_ROOT_N: i64 = 49 // length of NX_WCL_SILICON_ROOT
76const NX_WCL_MAX_PATH_LEN: i64 = 512
77const NX_WCL_MAX_CHARTER_BYTES: i64 = 524288 // 512KB; charters cap
78
79// ===== Path join helper =================================================
80//
81// Composes NX_WCL_SILICON_ROOT + relative_path into a buffer.
82
83func _wcl_join_path(out: *u8, out_cap: i64,
84 rel_path: *u8, rel_path_n: i64,
85 out_n: *i64) -> i64 {
86 if NX_WCL_SILICON_ROOT_N + rel_path_n + 1 > out_cap {
87 return 0 - NX_WCL_BAD_INPUT
88 }
89 var i: i64 = 0
90 while i < NX_WCL_SILICON_ROOT_N {
91 out[i] = NX_WCL_SILICON_ROOT[i]
92 i = i + 1
93 }
94 var j: i64 = 0
95 while j < rel_path_n {
96 out[NX_WCL_SILICON_ROOT_N + j] = rel_path[j]
97 j = j + 1
98 }
99 out[NX_WCL_SILICON_ROOT_N + rel_path_n] = 0 as u8
100 out_n[0] = NX_WCL_SILICON_ROOT_N + rel_path_n
101 return NX_WCL_OK
102}
103
104// ===== Single-charter load + index helper =================================================
105//
106// Reads one charter file + adds to per-site builder. Per-call:
107// 1. join NX_WCL_SILICON_ROOT + rel_path
108// 2. sys_read_file -> bytes
109// 3. nx_wiki_index_builder_add(builder, title, "/wiki/<slug>",
110// body, body_n)
111// Returns: NX_WCL_OK + non-negative rowid, OR negative verdict.
112// Charter-missing is non-fatal at the per-call level so caller can
113// log + continue.
114
115func nx_wcl_load_one(
116 builder: *NxWikiIndexBuilder,
117 rel_path: *u8, rel_path_n: i64,
118 slug: *u8, slug_n: i64,
119 title: *u8, title_n: i64
120) -> i64 {
121 if (builder as i64) == 0 { return 0 - NX_WCL_BAD_INPUT }
122 if (rel_path as i64) == 0 { return 0 - NX_WCL_BAD_INPUT }
123 if rel_path_n < 1 { return 0 - NX_WCL_BAD_INPUT }
124 if rel_path_n > NX_WCL_MAX_PATH_LEN { return 0 - NX_WCL_BAD_INPUT }
125 if (slug as i64) == 0 { return 0 - NX_WCL_BAD_INPUT }
126 if slug_n < 1 { return 0 - NX_WCL_BAD_INPUT }
127 if (title as i64) == 0 { return 0 - NX_WCL_BAD_INPUT }
128 if title_n < 1 { return 0 - NX_WCL_BAD_INPUT }
129
130 // 1. join path
131 let path_buf: *u8 = sys_mmap(NX_WCL_MAX_PATH_LEN)
132 let path_n_box: *i64 = (sys_mmap(8)) as *i64
133 path_n_box[0] = 0
134 let rc_join: i64 = _wcl_join_path(path_buf, NX_WCL_MAX_PATH_LEN,
135 rel_path, rel_path_n,
136 path_n_box)
137 if rc_join != NX_WCL_OK { return rc_join }
138
139 // 2. read file
140 let body_len_box: *i64 = (sys_mmap(8)) as *i64
141 body_len_box[0] = 0
142 let body: *u8 = sys_read_file(path_buf, body_len_box)
143 if (body as i64) == 0 { return 0 - NX_WCL_FILE_NOT_FOUND }
144 let body_n: i64 = body_len_box[0]
145 if body_n < 1 { return 0 - NX_WCL_FILE_NOT_FOUND }
146 if body_n > NX_WCL_MAX_CHARTER_BYTES { return 0 - NX_WCL_BAD_INPUT }
147
148 // 3. build the URL "/wiki/<slug>"
149 let url_prefix: *u8 = "/wiki/" as *u8
150 let url_prefix_n: i64 = 6
151 let url_total_n: i64 = url_prefix_n + slug_n
152 let url_buf: *u8 = sys_mmap(url_total_n + 1)
153 var k: i64 = 0
154 while k < url_prefix_n { url_buf[k] = url_prefix[k]; k = k + 1 }
155 var m: i64 = 0
156 while m < slug_n { url_buf[url_prefix_n + m] = slug[m]; m = m + 1 }
157
158 // 4. index it
159 let rowid: i64 = nx_wiki_index_builder_add(builder,
160 title, title_n,
161 url_buf, url_total_n,
162 body, body_n)
163 if rowid < 0 { return 0 - NX_WCL_INDEX_ADD_FAILED }
164 return NX_WCL_OK
165}
166
167// ===== V+1 AUTO-DISCOVERY: directory walk of the wiki content dir =================================================
168//
169// Closes this file's header V+1 TODO + NISHI_WIKI_CONTENT_INDEX §0/§22:
170// "V+1 reads the index .md file at startup + parses §1 tables to
171// discover charters dynamically (sys_getdents queued)". A new .md file
172// dropped into the content dir becomes a page + search hit with NO
173// code edit.
174//
175// The walk is struct-free over raw linux_dirent64 bytes
176// (dirent_reclen/dirent_type/dirent_name), mirroring
177// nx_sovereignty_audit's PROVEN shape -- per nx_nas_book_census's
178// header the NxDirent STRUCT path SIGSEGVs under nx_cc_sovereign, so
179// we read fields off the raw record.
180
181// ASCII lower of one byte (A-Z -> a-z; else passthrough).
182func nx_wcl_lc(c: i64) -> i64 {
183 if c >= 65 { if c <= 90 { return c + 32 } }
184 return c
185}
186
187// Does a NUL-terminated entry name end in ".md" (case-insensitive)?
188// Returns the name length (>0) if it is a markdown file, else 0.
189func nx_wcl_is_md(name: *u8) -> i64 {
190 var n: i64 = 0
191 while name[n] != (0 as u8) {
192 if n >= NX_WCL_MAX_NAME_LEN { return 0 }
193 n = n + 1
194 }
195 if n < 4 { return 0 } // need at least "x.md"
196 if name[n - 3] != (46 as u8) { return 0 } // '.'
197 if nx_wcl_lc(name[n - 2] as i64) != 109 { return 0 } // 'm'
198 if nx_wcl_lc(name[n - 1] as i64) != 100 { return 0 } // 'd'
199 return n
200}
201
202// Derive a /wiki slug from a markdown filename: strip the trailing
203// ".md", lowercase A-Z, and map '_' -> '-' so e.g.
204// "NISHI_WIKI_CHARTER.md" -> "nishi-wiki-charter" (the same convention
205// the hardcoded §1 manifest uses). Other bytes pass through verbatim.
206// Writes a NUL-terminated slug into `out`; returns the slug length.
207func nx_wcl_slug_from_name(name: *u8, name_n: i64, out: *u8) -> i64 {
208 let base: i64 = name_n - 3 // drop ".md"
209 var i: i64 = 0
210 var w: i64 = 0
211 while i < base {
212 if w >= NX_WCL_MAX_SLUG_LEN - 1 { i = base } else {
213 var c: i64 = name[i] as i64
214 if c == 95 { c = 45 } // '_' -> '-'
215 c = nx_wcl_lc(c)
216 out[w] = c as u8
217 w = w + 1
218 i = i + 1
219 }
220 }
221 out[w] = 0 as u8
222 return w
223}
224
225// Load ONE discovered file: build absolute path = dir + "/" + name,
226// read bytes, extract first '# ' heading as title (fallback
227// "(untitled)"), build url = "/wiki/<slug>", index it. Returns
228// NX_WCL_OK + indexes, or a negative verdict (caller logs + continues).
229func nx_wcl_discover_one(
230 builder: *NxWikiIndexBuilder,
231 dir_z: *u8, dir_n: i64,
232 name: *u8, name_n: i64
233) -> i64 {
234 if (builder as i64) == 0 { return 0 - NX_WCL_BAD_INPUT }
235 if dir_n < 1 { return 0 - NX_WCL_BAD_INPUT }
236 if name_n < 4 { return 0 - NX_WCL_BAD_INPUT }
237 if dir_n + 1 + name_n + 1 > NX_WCL_MAX_PATH_LEN { return 0 - NX_WCL_BAD_INPUT }
238
239 // 1. absolute path = dir + "/" + name (NUL-terminated)
240 let path_buf: *u8 = sys_mmap(NX_WCL_MAX_PATH_LEN)
241 var p: i64 = 0
242 var a: i64 = 0
243 while a < dir_n { path_buf[p] = dir_z[a]; p = p + 1; a = a + 1 }
244 // only inject a separator if the dir didn't already end in '/'
245 if dir_z[dir_n - 1] != (47 as u8) { path_buf[p] = 47 as u8; p = p + 1 }
246 var b: i64 = 0
247 while b < name_n { path_buf[p] = name[b]; p = p + 1; b = b + 1 }
248 path_buf[p] = 0 as u8
249
250 // 2. read the file bytes
251 let body_len_box: *i64 = (sys_mmap(8)) as *i64
252 body_len_box[0] = 0
253 let body: *u8 = sys_read_file(path_buf, body_len_box)
254 if (body as i64) == 0 { return 0 - NX_WCL_FILE_NOT_FOUND }
255 let body_n: i64 = body_len_box[0]
256 if body_n < 1 { return 0 - NX_WCL_FILE_NOT_FOUND }
257 if body_n > NX_WCL_MAX_CHARTER_BYTES { return 0 - NX_WCL_BAD_INPUT }
258
259 // 3. title = first "# " heading (reuse the index_builder extractor);
260 // fallback to "(untitled)".
261 let t_off: *i64 = (sys_mmap(8)) as *i64
262 let t_len: *i64 = (sys_mmap(8)) as *i64
263 t_off[0] = 0 - 1; t_len[0] = 0
264 let rc_t: i64 = nx_wib_extract_md_title(body, body_n, t_off, t_len)
265 var title_ptr: *u8 = NX_WCL_FALLBACK_TITLE
266 var title_n: i64 = NX_WCL_FALLBACK_TITLE_N
267 if rc_t == NX_WIB_OK {
268 if t_off[0] >= 0 { if t_len[0] > 0 {
269 title_ptr = (body as i64 + t_off[0]) as *u8
270 title_n = t_len[0]
271 } }
272 }
273
274 // 4. slug + url = "/wiki/<slug>"
275 let slug_buf: *u8 = sys_mmap(NX_WCL_MAX_SLUG_LEN)
276 let slug_n: i64 = nx_wcl_slug_from_name(name, name_n, slug_buf)
277 if slug_n < 1 { return 0 - NX_WCL_BAD_INPUT }
278 let url_buf: *u8 = sys_mmap(NX_WCL_MAX_URL_LEN)
279 let url_pfx: *u8 = "/wiki/" as *u8
280 var u: i64 = 0
281 while u < 6 { url_buf[u] = url_pfx[u]; u = u + 1 }
282 var s: i64 = 0
283 while s < slug_n { url_buf[6 + s] = slug_buf[s]; s = s + 1 }
284 let url_total_n: i64 = 6 + slug_n
285
286 // 5. index it (doc store + inverted index PASS 1)
287 let rowid: i64 = nx_wiki_index_builder_add(builder,
288 title_ptr, title_n,
289 url_buf, url_total_n,
290 body, body_n)
291 if rowid < 0 { return 0 - NX_WCL_INDEX_ADD_FAILED }
292 return NX_WCL_OK
293}
294
295// AUTO-DISCOVER every *.md page in `dir_z` and index each. This is the
296// V+1 replacement for the hardcoded manifest: it opens the directory,
297// getdents64-walks it struct-free, and for every regular-file (or
298// DT_UNKNOWN -- drvfs/9p often returns 0) entry whose name ends in
299// ".md", calls nx_wcl_discover_one. Per Cardinal 14 (graceful
300// degradation) a per-file failure is skipped, not fatal. Returns the
301// count of pages SUCCESSFULLY indexed (>= 0), or a negative verdict if
302// the directory itself can't be opened.
303func nx_wcl_discover_dir(builder: *NxWikiIndexBuilder, dir_z: *u8) -> i64 {
304 if (builder as i64) == 0 { return 0 - NX_WCL_BAD_INPUT }
305 if (dir_z as i64) == 0 { return 0 - NX_WCL_BAD_INPUT }
306 var dir_n: i64 = 0
307 while dir_z[dir_n] != (0 as u8) {
308 if dir_n >= NX_WCL_MAX_PATH_LEN { return 0 - NX_WCL_BAD_INPUT }
309 dir_n = dir_n + 1
310 }
311 if dir_n < 1 { return 0 - NX_WCL_BAD_INPUT }
312
313 let dfd: i64 = sys_openat_rd(dir_z)
314 if dfd < 0 { return 0 - NX_WCL_DIR_OPEN_FAILED }
315
316 // Count via the doc-store delta (GROUND TRUTH = docs actually added),
317 // not the per-call return value: the native compiler can desync a
318 // `let rc = <cross-fn call>` result vs an immediate `rc == const`
319 // compare (documented "field = imported_call" / return-value class
320 // in nx_wiki_index_builder.nx), which made an earlier `if rc ==
321 // NX_WCL_OK { loaded+1 }` count 0 even though all 31 docs indexed.
322 // Reading doc_store.doc_count before/after is robust AND accurate.
323 let store0: *NxWikiDocStore = nx_wiki_index_builder_get_store(builder)
324 var before: i64 = 0
325 if (store0 as i64) != 0 { before = nx_wiki_doc_store_count(store0) }
326
327 let dirbuf: *u8 = sys_mmap(NX_WCL_DIRBUF_BYTES + 64)
328 var done: i64 = 0
329 while done == 0 {
330 let nb: i64 = sys_getdents64(dfd, dirbuf, NX_WCL_DIRBUF_BYTES)
331 if nb <= 0 { done = 1 } else {
332 var off: i64 = 0
333 while off < nb {
334 let rec: *u8 = (dirbuf as i64 + off) as *u8
335 let rl: i64 = dirent_reclen(rec)
336 if rl <= 0 { off = nb } else {
337 let nm: *u8 = dirent_name(rec)
338 let dt: i64 = dirent_type(rec)
339 // skip directories (DT_DIR=4) and symlinks (DT_LNK=10);
340 // regular files (DT_REG=8) AND DT_UNKNOWN(0) -> consider.
341 var consider: i64 = 0
342 if dt == 8 { consider = 1 }
343 if dt == 0 { consider = 1 }
344 if consider == 1 {
345 let nlen: i64 = nx_wcl_is_md(nm)
346 if nlen > 0 {
347 let cur: i64 = nx_wiki_doc_store_count(store0)
348 if cur - before < NX_WCL_MAX_DISCOVER {
349 nx_wcl_discover_one(builder, dir_z, dir_n, nm, nlen)
350 }
351 }
352 }
353 off = off + rl
354 }
355 }
356 }
357 }
358 sys_close(dfd)
359 var loaded: i64 = 0
360 if (store0 as i64) != 0 { loaded = nx_wiki_doc_store_count(store0) - before }
361 return loaded
362}
363
364// ===== V+2 RECURSIVE AUTO-DISCOVERY: bounded directory work-stack =================================================
365//
366// nx_wcl_discover_dir (above) walks only the TOP level of the content
367// dir, so subdir charters (hdl/*.md, synth/*.md, ...) are missed. This
368// recursive variant walks the WHOLE tree with its OWN bounded
369// getdents64 work-stack -- the PROVEN shape copied from
370// nx_sovereignty_audit's main (struct-free dirent bytes; DT_DIR pushed
371// onto the stack; DT_REG/DT_UNKNOWN classified). Per JPL rule 2 every
372// loop + the stack has an explicit budget; a ceiling HIT is reported
373// (the caller sees a partial count, never a silent crash).
374//
375// To avoid slug COLLISIONS across subdirs (e.g. every hdl/README.md,
376// synth/README.md ... would otherwise all become "readme"), the slug is
377// PATH-AWARE: it is derived from the child path RELATIVE to the root
378// (root_len bytes are skipped), minus ".md", lowercased, with '/'->'-'
379// and '_'->'-'. So "<root>/hdl/HACKING.md" -> "hdl-hacking" while a
380// top-level "<root>/NISHI_WIKI_CHARTER.md" -> "nishi-wiki-charter"
381// (identical to the single-level walk -- top-level slugs are unchanged).
382
383// ---- recursive-walk bounded budgets (JPL rule 2) ----
384const NX_WCL_MAX_DIRS: i64 = 4096 // pending-dir stack ceiling
385const NX_WCL_ITER_CAP: i64 = 100000 // hard dir-pop ceiling (symlink-loop guard)
386const NX_WCL_DIR_ARENA: i64 = 4194304 // 4 MiB of dir-path bytes for the stack
387const NX_WCL_CHILD_CAP: i64 = 8192 // assembled child-path scratch
388
389// Derive a PATH-AWARE /wiki slug from an absolute child path: skip the
390// leading `root_len` bytes (so only the path RELATIVE to the content
391// root contributes), drop the trailing ".md", lowercase A-Z, and map
392// both '_' and '/' to '-'. Writes a NUL-terminated slug into `out`;
393// returns the slug length (0 on bad input). A leading '/' just past the
394// root (root not ending in '/') is skipped so we never emit a leading
395// '-'.
396func nx_wcl_slug_from_relpath(path: *u8, path_n: i64, root_len: i64, out: *u8) -> i64 {
397 if path_n < 4 { return 0 }
398 if root_len < 0 { return 0 }
399 if root_len >= path_n { return 0 }
400 let base: i64 = path_n - 3 // drop ".md"
401 var i: i64 = root_len
402 // skip a separator immediately after the root (root w/o trailing '/')
403 if i < base { if path[i] == (47 as u8) { i = i + 1 } }
404 var w: i64 = 0
405 while i < base {
406 if w >= NX_WCL_MAX_SLUG_LEN - 1 { i = base } else {
407 var c: i64 = path[i] as i64
408 if c == 95 { c = 45 } // '_' -> '-'
409 if c == 47 { c = 45 } // '/' -> '-'
410 c = nx_wcl_lc(c)
411 out[w] = c as u8
412 w = w + 1
413 i = i + 1
414 }
415 }
416 out[w] = 0 as u8
417 return w
418}
419
420// Index ONE discovered file given its already-assembled absolute path +
421// the root length (for the path-aware slug). Reads bytes, extracts the
422// first '# ' heading as title (fallback "(untitled)"), builds url =
423// "/wiki/<path-aware-slug>", indexes it. Returns NX_WCL_OK, or a
424// negative verdict (caller logs + continues).
425func nx_wcl_discover_one_path(
426 builder: *NxWikiIndexBuilder,
427 path_z: *u8, path_n: i64, root_len: i64
428) -> i64 {
429 if (builder as i64) == 0 { return 0 - NX_WCL_BAD_INPUT }
430 if path_n < 4 { return 0 - NX_WCL_BAD_INPUT }
431
432 // 1. read the file bytes
433 let body_len_box: *i64 = (sys_mmap(8)) as *i64
434 body_len_box[0] = 0
435 let body: *u8 = sys_read_file(path_z, body_len_box)
436 if (body as i64) == 0 { return 0 - NX_WCL_FILE_NOT_FOUND }
437 let body_n: i64 = body_len_box[0]
438 if body_n < 1 { return 0 - NX_WCL_FILE_NOT_FOUND }
439 if body_n > NX_WCL_MAX_CHARTER_BYTES { return 0 - NX_WCL_BAD_INPUT }
440
441 // 2. title = first "# " heading (reuse the index_builder extractor);
442 // fallback to "(untitled)".
443 let t_off: *i64 = (sys_mmap(8)) as *i64
444 let t_len: *i64 = (sys_mmap(8)) as *i64
445 t_off[0] = 0 - 1; t_len[0] = 0
446 let rc_t: i64 = nx_wib_extract_md_title(body, body_n, t_off, t_len)
447 var title_ptr: *u8 = NX_WCL_FALLBACK_TITLE
448 var title_n: i64 = NX_WCL_FALLBACK_TITLE_N
449 if rc_t == NX_WIB_OK {
450 if t_off[0] >= 0 { if t_len[0] > 0 {
451 title_ptr = (body as i64 + t_off[0]) as *u8
452 title_n = t_len[0]
453 } }
454 }
455
456 // 3. PATH-AWARE slug + url = "/wiki/<slug>"
457 let slug_buf: *u8 = sys_mmap(NX_WCL_MAX_SLUG_LEN)
458 let slug_n: i64 = nx_wcl_slug_from_relpath(path_z, path_n, root_len, slug_buf)
459 if slug_n < 1 { return 0 - NX_WCL_BAD_INPUT }
460 let url_buf: *u8 = sys_mmap(NX_WCL_MAX_URL_LEN)
461 let url_pfx: *u8 = "/wiki/" as *u8
462 var u: i64 = 0
463 while u < 6 { url_buf[u] = url_pfx[u]; u = u + 1 }
464 var s: i64 = 0
465 while s < slug_n { url_buf[6 + s] = slug_buf[s]; s = s + 1 }
466 let url_total_n: i64 = 6 + slug_n
467
468 // 4. index it (doc store + inverted index PASS 1)
469 let rowid: i64 = nx_wiki_index_builder_add(builder,
470 title_ptr, title_n,
471 url_buf, url_total_n,
472 body, body_n)
473 if rowid < 0 { return 0 - NX_WCL_INDEX_ADD_FAILED }
474 return NX_WCL_OK
475}
476
477// AUTO-DISCOVER every *.md page in `dir_z` AND all of its subdirectories,
478// indexing each with a PATH-AWARE slug. Bounded recursive getdents64
479// work-stack (no recursion-as-callstack; an explicit dir-offset stack in
480// an mmap arena, mirroring nx_sovereignty_audit). Returns the count of
481// pages SUCCESSFULLY indexed (>= 0, via the doc-store delta = ground
482// truth), or a negative verdict if the ROOT dir itself can't be opened.
483// Per Cardinal 14 a per-file or per-subdir failure is skipped, not fatal.
484func nx_wcl_discover_tree(builder: *NxWikiIndexBuilder, dir_z: *u8) -> i64 {
485 if (builder as i64) == 0 { return 0 - NX_WCL_BAD_INPUT }
486 if (dir_z as i64) == 0 { return 0 - NX_WCL_BAD_INPUT }
487 var root_len: i64 = 0
488 while dir_z[root_len] != (0 as u8) {
489 if root_len >= NX_WCL_MAX_PATH_LEN { return 0 - NX_WCL_BAD_INPUT }
490 root_len = root_len + 1
491 }
492 if root_len < 1 { return 0 - NX_WCL_BAD_INPUT }
493 // Probe the ROOT first: if it can't be opened, this is a hard verdict
494 // (the caller falls back to the hardcoded manifest). Subdir opens that
495 // fail later are tolerated (counted as open_fail, never fatal).
496 let probe: i64 = sys_openat_rd(dir_z)
497 if probe < 0 { return 0 - NX_WCL_DIR_OPEN_FAILED }
498 sys_close(probe)
499
500 // Ground-truth count = doc-store delta (NOT a returned status code:
501 // the native compiler can desync `let rc = <cross-fn call>` vs an
502 // immediate `rc == const` compare -- see nx_wcl_discover_dir's note).
503 let store0: *NxWikiDocStore = nx_wiki_index_builder_get_store(builder)
504 var before: i64 = 0
505 if (store0 as i64) != 0 { before = nx_wiki_doc_store_count(store0) }
506
507 // ---- bounded arenas (mmap) ----
508 let parena: *u8 = sys_mmap(NX_WCL_DIR_ARENA + 64) // dir-path bytes
509 var pused: i64 = 0
510 let stkoff: *i64 = (sys_mmap(NX_WCL_MAX_DIRS * 8)) as *i64 // stack of parena offsets
511 var sp: i64 = 0
512 let dirbuf: *u8 = sys_mmap(NX_WCL_DIRBUF_BYTES + 64)
513 let childbuf: *u8 = sys_mmap(NX_WCL_CHILD_CAP)
514
515 // seed the stack with the root dir path
516 var rseed: i64 = 0
517 while rseed < root_len { parena[pused] = dir_z[rseed]; pused = pused + 1; rseed = rseed + 1 }
518 parena[pused] = 0 as u8; pused = pused + 1
519 stkoff[sp] = 0; sp = sp + 1
520
521 var iters: i64 = 0
522 while sp > 0 {
523 if iters >= NX_WCL_ITER_CAP { sp = 0 } else {
524 iters = iters + 1
525 sp = sp - 1
526 let doff: i64 = stkoff[sp]
527 let dpath: *u8 = (parena as i64 + doff) as *u8
528 var dlen: i64 = 0
529 while dpath[dlen] != (0 as u8) { dlen = dlen + 1 }
530
531 let dfd: i64 = sys_openat_rd(dpath)
532 if dfd < 0 { } else {
533 var done: i64 = 0
534 while done == 0 {
535 let nb: i64 = sys_getdents64(dfd, dirbuf, NX_WCL_DIRBUF_BYTES)
536 if nb <= 0 { done = 1 } else {
537 var off: i64 = 0
538 while off < nb {
539 let rec: *u8 = (dirbuf as i64 + off) as *u8
540 let rl: i64 = dirent_reclen(rec)
541 if rl <= 0 { off = nb } else {
542 let nm: *u8 = dirent_name(rec)
543 let dt: i64 = dirent_type(rec)
544 // skip "." and ".."
545 var skip: i64 = 0
546 if nm[0] == (46 as u8) {
547 if nm[1] == (0 as u8) { skip = 1 }
548 else { if nm[1] == (46 as u8) { if nm[2] == (0 as u8) { skip = 1 } } }
549 }
550 if skip == 0 {
551 // assemble childpath = dpath + "/" + nm
552 var cw: i64 = 0
553 var z: i64 = 0
554 while z < dlen {
555 if cw < NX_WCL_CHILD_CAP - 2 { childbuf[cw] = dpath[z]; cw = cw + 1 }
556 z = z + 1
557 }
558 if dpath[dlen - 1] != (47 as u8) {
559 if cw < NX_WCL_CHILD_CAP - 2 { childbuf[cw] = 47 as u8; cw = cw + 1 }
560 }
561 var nl: i64 = 0
562 while nm[nl] != (0 as u8) {
563 if cw < NX_WCL_CHILD_CAP - 2 { childbuf[cw] = nm[nl]; cw = cw + 1 }
564 nl = nl + 1
565 }
566 childbuf[cw] = 0 as u8
567 let clen: i64 = cw
568
569 if dt == 4 {
570 // DT_DIR: push child onto the work-stack
571 if sp < NX_WCL_MAX_DIRS {
572 if pused + clen + 1 < NX_WCL_DIR_ARENA {
573 let cbase: i64 = pused
574 var q: i64 = 0
575 while q < clen { parena[pused] = childbuf[q]; pused = pused + 1; q = q + 1 }
576 parena[pused] = 0 as u8; pused = pused + 1
577 stkoff[sp] = cbase
578 sp = sp + 1
579 }
580 }
581 } else {
582 if dt == 10 { } else {
583 // DT_REG (8) OR DT_UNKNOWN (0; drvfs/9p) -> consider.
584 let nmlen: i64 = nx_wcl_is_md(nm)
585 if nmlen > 0 {
586 let cur: i64 = nx_wiki_doc_store_count(store0)
587 if cur - before < NX_WCL_MAX_DISCOVER {
588 nx_wcl_discover_one_path(builder, childbuf, clen, root_len)
589 }
590 }
591 }
592 }
593 }
594 off = off + rl
595 }
596 }
597 }
598 }
599 sys_close(dfd)
600 }
601 }
602 }
603 var loaded: i64 = 0
604 if (store0 as i64) != 0 { loaded = nx_wiki_doc_store_count(store0) - before }
605 return loaded
606}
607
608// ===== Bulk load: NISHI_WIKI_CONTENT_INDEX §1 manifest =================================================
609//
610// Hardcoded per V1 SCOPE LIMIT above. Each entry: rel_path + slug +
611// title (matching NISHI_WIKI_CONTENT_INDEX §1 tables).
612//
613// Returns: count of successfully loaded charters (>=0); does NOT abort
614// on per-charter miss (logs + continues). Caller decides whether
615// partial-load is fatal.
616
617func nx_wcl_load_full_manifest(builder: *NxWikiIndexBuilder) -> i64 {
618 if (builder as i64) == 0 { return 0 - NX_WCL_BAD_INPUT }
619 var loaded: i64 = 0
620 let rc: *i64 = (sys_mmap(8)) as *i64
621
622 // ===== §1a foundational cardinals + standards =====
623 rc[0] = nx_wcl_load_one(builder,
624 "NISHI_LICENSE_CONTAINMENT_CARDINAL.md" as *u8, 37,
625 "nishi-license-containment" as *u8, 25,
626 "License Containment Cardinal" as *u8, 28)
627 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
628
629 rc[0] = nx_wcl_load_one(builder,
630 "NISHI_CODE_HYGIENE_STANDARD.md" as *u8, 30,
631 "nishi-code-hygiene" as *u8, 18,
632 "Code Hygiene Standard" as *u8, 21)
633 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
634
635 rc[0] = nx_wcl_load_one(builder,
636 "NISHILANG_SUPERIORITY_BAR.md" as *u8, 28,
637 "nishi-superiority-bar" as *u8, 21,
638 "NishiLang Superiority Bar" as *u8, 25)
639 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
640
641 rc[0] = nx_wcl_load_one(builder,
642 "NISHIDOC_STANDARD.md" as *u8, 20,
643 "nishi-doc-standard" as *u8, 18,
644 "NishiDoc Standard" as *u8, 17)
645 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
646
647 rc[0] = nx_wcl_load_one(builder,
648 "NISHIBENCH_WINNER_STANDARD.md" as *u8, 29,
649 "nishi-bench-winner-standard" as *u8, 27,
650 "NishiBench WINNER Standard" as *u8, 26)
651 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
652
653 // ===== §1b hosting + provisioning =====
654 rc[0] = nx_wcl_load_one(builder,
655 "NISHI_SOVEREIGN_HOSTING_CHARTER.md" as *u8, 34,
656 "nishi-sovereign-hosting-charter" as *u8, 31,
657 "Sovereign Hosting Charter" as *u8, 25)
658 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
659
660 rc[0] = nx_wcl_load_one(builder,
661 "NISHI_SITE_PROVISIONING_CHARTER.md" as *u8, 34,
662 "nishi-site-provisioning-charter" as *u8, 31,
663 "NishiHost Provisioning Charter" as *u8, 30)
664 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
665
666 rc[0] = nx_wcl_load_one(builder,
667 "NISHI_SITES_INVENTORY.md" as *u8, 24,
668 "nishi-sites-inventory" as *u8, 21,
669 "Sites Inventory" as *u8, 15)
670 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
671
672 rc[0] = nx_wcl_load_one(builder,
673 "NISHI_ANDELINWEST_DEPLOY_PLAYBOOK.md" as *u8, 36,
674 "nishi-andelinwest-deploy-playbook" as *u8, 33,
675 "andelinwest.com Deploy Playbook" as *u8, 31)
676 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
677
678 // ===== §1c authentication + privacy =====
679 rc[0] = nx_wcl_load_one(builder,
680 "NISHI_MODERN_AUTH_CHARTER.md" as *u8, 28,
681 "nishi-modern-auth-charter" as *u8, 25,
682 "Modern Auth Charter (OPAQUE + Argon2id + BIP39)" as *u8, 47)
683 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
684
685 rc[0] = nx_wcl_load_one(builder,
686 "NISHI_CLIENT_STORE_CHARTER.md" as *u8, 29,
687 "nishi-client-store-charter" as *u8, 26,
688 "Client Store Charter" as *u8, 20)
689 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
690
691 rc[0] = nx_wcl_load_one(builder,
692 "NISHI_PERSONAL_DATA_SOVEREIGNTY_CHARTER.md" as *u8, 42,
693 "nishi-personal-data-sovereignty-charter" as *u8, 39,
694 "Personal Data Sovereignty Charter" as *u8, 33)
695 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
696
697 // ===== §1d platform extensions =====
698 rc[0] = nx_wcl_load_one(builder,
699 "NISHI_INTELLIGENT_WIKI_CHARTER.md" as *u8, 33,
700 "nishi-intelligent-wiki-charter" as *u8, 30,
701 "Intelligent Wiki Charter" as *u8, 24)
702 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
703
704 rc[0] = nx_wcl_load_one(builder,
705 "NISHI_FLUID_LEARNING_PLATFORM_CHARTER.md" as *u8, 40,
706 "nishi-fluid-learning-platform-charter" as *u8, 37,
707 "Fluid Learning Platform Charter" as *u8, 31)
708 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
709
710 rc[0] = nx_wcl_load_one(builder,
711 "NISHI_PREDICTIVE_PLANNING_CHARTER.md" as *u8, 36,
712 "nishi-predictive-planning-charter" as *u8, 33,
713 "Predictive Planning Charter" as *u8, 27)
714 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
715
716 rc[0] = nx_wcl_load_one(builder,
717 "INTEROPERABILITY_CHARTER.md" as *u8, 27,
718 "nishi-interoperability-charter" as *u8, 30,
719 "Interoperability Shim Layer Charter" as *u8, 35)
720 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
721
722 // ===== §1e research integration =====
723 rc[0] = nx_wcl_load_one(builder,
724 "NISHI_SEARCH_RESEARCH_INTEGRATION.md" as *u8, 36,
725 "nishi-search-research-integration" as *u8, 33,
726 "Search Research Integration (8 papers + LiBox 37-ref graph)" as *u8, 59)
727 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
728
729 // ===== §1f silicon arc docs =====
730 rc[0] = nx_wcl_load_one(builder,
731 "ZERO_TO_ADVANCED.md" as *u8, 19,
732 "silicon-zero-to-advanced" as *u8, 24,
733 "Silicon Zero-to-Advanced Master Plan" as *u8, 36)
734 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
735
736 rc[0] = nx_wcl_load_one(builder,
737 "WORKLOAD_TARGETS.md" as *u8, 19,
738 "silicon-workload-targets" as *u8, 24,
739 "Silicon Workload Targets" as *u8, 24)
740 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
741
742 rc[0] = nx_wcl_load_one(builder,
743 "hdl/SILICON_SOVEREIGNTY_AUDIT.md" as *u8, 32,
744 "silicon-sovereignty-audit" as *u8, 25,
745 "Silicon Sovereignty Audit" as *u8, 25)
746 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
747
748 rc[0] = nx_wcl_load_one(builder,
749 "hdl/HACKING.md" as *u8, 14,
750 "silicon-hacking" as *u8, 15,
751 "Silicon HACKING dev guide" as *u8, 25)
752 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
753
754 // ===== §1g V2.0 bootstrap arc =====
755 rc[0] = nx_wcl_load_one(builder,
756 "NISHI_V2_BOOTSTRAP_ARC_PLAN.md" as *u8, 30,
757 "nishi-v2-bootstrap-arc-plan" as *u8, 27,
758 "V2.0 Bootstrap Arc Plan" as *u8, 23)
759 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
760
761 // ===== §1h bottom-up build log (live progress; appended as the stack rises) =====
762 rc[0] = nx_wcl_load_one(builder,
763 "NISHI_STACK_STATUS.md" as *u8, 21,
764 "nishi-stack-status" as *u8, 18,
765 "Stack Status: Bottom-Up Build Log" as *u8, 33)
766 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
767
768 // ===== Index itself (so /wiki/nishi-wiki-content-index works) =====
769 rc[0] = nx_wcl_load_one(builder,
770 "NISHI_WIKI_CONTENT_INDEX.md" as *u8, 27,
771 "nishi-wiki-content-index" as *u8, 24,
772 "Wiki Content Index (this manifest)" as *u8, 34)
773 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
774
775 return loaded
776}
777
778// ===== V+1 ENTRY POINT: auto-discover with hardcoded fallback =================================================
779//
780// CONVENIENCE: auto-discover the canonical silicon content dir; if the
781// dir can't be walked (e.g. a deploy host without the repo mounted),
782// gracefully FALL BACK to the hardcoded §1 manifest so existing
783// behavior is preserved. Returns the count of pages loaded by whichever
784// path ran. The hardcoded nx_wcl_load_full_manifest stays available to
785// callers that want it explicitly. Defined AFTER nx_wcl_load_full_manifest
786// so the fallback call is a backward reference (NishiLang forbids
787// forward references).
788func nx_wcl_load_full_manifest_or_discover(builder: *NxWikiIndexBuilder) -> i64 {
789 if (builder as i64) == 0 { return 0 - NX_WCL_BAD_INPUT }
790 let n: i64 = nx_wcl_discover_dir(builder, NX_WCL_SILICON_ROOT)
791 if n > 0 { return n }
792 // discovery found nothing (or dir unopenable) -> hardcoded fallback
793 return nx_wcl_load_full_manifest(builder)
794}
795
796// ===== V+2 ENTRY POINT: RECURSIVE auto-discover with hardcoded fallback =================================================
797//
798// The canonical loader for the live internal wiki engine: recursively
799// walks the silicon content dir (top-level AND subdir charters) and
800// indexes every *.md with a path-aware slug. If the ROOT dir can't be
801// walked (deploy host without the repo mounted) it gracefully FALLS BACK
802// to the hardcoded §1 manifest so existing behavior is preserved.
803// Returns the count of pages loaded by whichever path ran. Defined AFTER
804// nx_wcl_load_full_manifest so the fallback call is a backward reference
805// (NishiLang forbids forward references).
806func nx_wcl_load_discover_tree(builder: *NxWikiIndexBuilder) -> i64 {
807 if (builder as i64) == 0 { return 0 - NX_WCL_BAD_INPUT }
808 let n: i64 = nx_wcl_discover_tree(builder, NX_WCL_SILICON_ROOT)
809 if n > 0 { return n }
810 // recursive discovery found nothing (or root unopenable) -> fallback
811 return nx_wcl_load_full_manifest(builder)
812}
813
814// ===== Per-site curated load (jasonewest.com portfolio subset) =================================================
815//
816// Per NISHI_WIKI_CONTENT_INDEX §4: jasonewest.com loads a curated
817// subset showing operator's substrate work as portfolio evidence.
818
819func nx_wcl_load_jasonewest_portfolio(builder: *NxWikiIndexBuilder) -> i64 {
820 if (builder as i64) == 0 { return 0 - NX_WCL_BAD_INPUT }
821 var loaded: i64 = 0
822 let rc: *i64 = (sys_mmap(8)) as *i64
823
824 rc[0] = nx_wcl_load_one(builder,
825 "NISHI_PERSONAL_DATA_SOVEREIGNTY_CHARTER.md" as *u8, 42,
826 "nishi-personal-data-sovereignty-charter" as *u8, 39,
827 "Personal Data Sovereignty Charter" as *u8, 33)
828 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
829
830 rc[0] = nx_wcl_load_one(builder,
831 "NISHI_MODERN_AUTH_CHARTER.md" as *u8, 28,
832 "nishi-modern-auth-charter" as *u8, 25,
833 "Modern Auth Charter" as *u8, 19)
834 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
835
836 rc[0] = nx_wcl_load_one(builder,
837 "NISHI_SOVEREIGN_HOSTING_CHARTER.md" as *u8, 34,
838 "nishi-sovereign-hosting-charter" as *u8, 31,
839 "Sovereign Hosting Charter" as *u8, 25)
840 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
841
842 rc[0] = nx_wcl_load_one(builder,
843 "NISHILANG_SUPERIORITY_BAR.md" as *u8, 28,
844 "nishi-superiority-bar" as *u8, 21,
845 "NishiLang Superiority Bar" as *u8, 25)
846 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
847
848 rc[0] = nx_wcl_load_one(builder,
849 "NISHIBENCH_WINNER_STANDARD.md" as *u8, 29,
850 "nishi-bench-winner-standard" as *u8, 27,
851 "NishiBench WINNER Standard" as *u8, 26)
852 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
853
854 rc[0] = nx_wcl_load_one(builder,
855 "ZERO_TO_ADVANCED.md" as *u8, 19,
856 "silicon-zero-to-advanced" as *u8, 24,
857 "Silicon Zero-to-Advanced Master Plan" as *u8, 36)
858 if rc[0] == NX_WCL_OK { loaded = loaded + 1 }
859
860 return loaded
861}