code wiki / _hdl_build / nx_library_sources.nx
nx_library_sources.nx source
↩ module page · 112 lines · 6670 B
1// nx_library_sources.nx -- the CONSUMED-SOURCES CATALOG (operator 2026-06-20: "i cant even see on
2// nishifamily.com/library the different research sources we have already consumed -- get them all
3// parsable/readable/searchable, ingesting automatically from the sources we have given it + future ones").
4//
5// The foundation organ only auto-ingests knowledge/library/*.txt (~28); but we have also consumed
6// knowledge/research/*.md (deep-research reports) + knowledge/fetched/*.raw (sovereign-fetched sources).
7// This organ AUTO-DISCOVERS every consumed source across those dirs (getdents64, so future sources you
8// flag by dropping them in are picked up on the next run -- automatic ingestion of the catalog) and
9// emits a single data-driven, searchable HTML catalog so EVERY consumed source is VISIBLE.
10// Output: web_assets/library-sources.html. Sovereign: imports only nx_syscalls. license_tier: ORIGINAL
11import "nx_syscalls.nx"
12import "nx_itoa_lib.nx" // shared MSB-first emitter (zero-alloc)
13
14const LS_DENT: i64 = 65536
15
16func ls_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != 0 as u8 { n = n + 1 } sys_write(1, s, n); return 0 }
17// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
18// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
19// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
20// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
21func ls_putn(v: i64) -> i64 { nxi_out(v); return 0 }
22func ls_wr(fd: i64, s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(fd, s, n); return n }
23// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
24// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
25// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
26// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
27func ls_wrn(fd: i64, v: i64) -> i64 { nxi_fd(fd, v); return 0 }
28func ls_wresc(fd: i64, s: *u8) -> i64 {
29 var i: i64 = 0
30 while s[i] != (0 as u8) {
31 let c: i64 = s[i] as i64
32 if c == 38 { sys_write(fd, "&" as *u8, 5) } else { if c == 60 { sys_write(fd, "<" as *u8, 4) } else { if c == 62 { sys_write(fd, ">" as *u8, 4) } else { sys_write(fd, (s as i64 + i) as *u8, 1) } } }
33 i = i + 1
34 }
35 return 0
36}
37func ls_ends(name: *u8, ext: *u8) -> i64 {
38 var nl: i64 = 0; while name[nl] != (0 as u8) { nl = nl + 1 }
39 var el: i64 = 0; while ext[el] != (0 as u8) { el = el + 1 }
40 if el > nl { return 0 }
41 var i: i64 = 0; while i < el { if name[nl-el+i] != ext[i] { return 0 } i = i + 1 }
42 return 1
43}
44func ls_getdents(fd: i64, buf: *u8, count: i64) -> i64 { return __syscall(SYS_GETDENTS64, fd, buf, count, 0, 0, 0) }
45// emit one <li> per file in `dir` ending in `ext`. returns the count.
46func ls_emit_dir(fd: i64, dir: *u8, ext: *u8, klass: *u8) -> i64 {
47 let dfd: i64 = sys_openat_rd(dir)
48 if dfd < 0 { return 0 }
49 let gbuf: *u8 = sys_mmap(LS_DENT)
50 var found: i64 = 0
51 var nread: i64 = ls_getdents(dfd, gbuf, LS_DENT)
52 while nread > 0 {
53 var off: i64 = 0
54 while off < nread {
55 let reclen: i64 = (gbuf[off+16] as i64) | ((gbuf[off+17] as i64) << 8)
56 if reclen <= 0 { off = nread } else {
57 let name: *u8 = ((gbuf as i64) + off + 19) as *u8
58 if ls_ends(name, ext) == 1 {
59 found = found + 1
60 ls_wr(fd, "<li class=\"src\" data-k=\"")
61 ls_wresc(fd, klass)
62 ls_wr(fd, "\"><span class=\"k\">")
63 ls_wresc(fd, klass)
64 ls_wr(fd, "</span> ")
65 ls_wresc(fd, name)
66 ls_wr(fd, "</li>\n")
67 }
68 off = off + reclen
69 }
70 }
71 nread = ls_getdents(dfd, gbuf, LS_DENT)
72 }
73 sys_close(dfd)
74 return found
75}
76
77func main() -> i64 {
78 ls_puts("=== NISHI LIBRARY -- consumed-sources catalog (auto-discovered, searchable) ===\n")
79 let fd: i64 = sys_openat_wr("web_assets/library-sources.html\x00" as *u8, 0x1a4)
80 if fd < 0 { ls_puts("FATAL: cannot open web_assets/library-sources.html\n"); sys_exit(1); return 1 }
81
82 ls_wr(fd, "<!doctype html><html><head><meta charset=\"utf-8\"><title>Nishi Library -- Consumed Sources</title>\n")
83 ls_wr(fd, "<style>body{font-family:system-ui,sans-serif;max-width:60em;margin:2em auto;padding:0 1em}.k{display:inline-block;min-width:9em;color:#557;font-size:.85em}li.src{margin:.2em 0;list-style:none}h1{margin-bottom:.2em}#q{width:100%;padding:.6em;font-size:1em;margin:.5em 0}.sec{margin-top:1.5em}</style></head><body>\n")
84 ls_wr(fd, "<h1>Nishi Library — Consumed Research Sources</h1>\n")
85 ls_wr(fd, "<p>Every source the Nishi Library has consumed, auto-discovered from the corpus. Type to filter.</p>\n")
86 ls_wr(fd, "<input id=\"q\" placeholder=\"filter sources...\" oninput=\"var v=this.value.toLowerCase();document.querySelectorAll('li.src').forEach(function(e){e.style.display=e.textContent.toLowerCase().indexOf(v)<0?'none':''})\">\n")
87
88 ls_wr(fd, "<div class=\"sec\"><h2>Research reports</h2><ul>\n")
89 let r: i64 = ls_emit_dir(fd, "knowledge/research\x00" as *u8, ".md\x00" as *u8, "research-report\x00" as *u8)
90 ls_wr(fd, "</ul></div>\n<div class=\"sec\"><h2>Foundationed docs (CID, no-link-rot)</h2><ul>\n")
91 let d: i64 = ls_emit_dir(fd, "knowledge/library\x00" as *u8, ".txt\x00" as *u8, "foundationed-doc\x00" as *u8)
92 ls_wr(fd, "</ul></div>\n<div class=\"sec\"><h2>Fetched sources (sovereign TLS fetch)</h2><ul>\n")
93 let f: i64 = ls_emit_dir(fd, "knowledge/fetched\x00" as *u8, ".raw\x00" as *u8, "fetched-source\x00" as *u8)
94 ls_wr(fd, "</ul></div>\n")
95
96 let total: i64 = r + d + f
97 ls_wr(fd, "<p class=\"sec\"><b>")
98 ls_wrn(fd, total)
99 ls_wr(fd, "</b> consumed sources: ")
100 ls_wrn(fd, r); ls_wr(fd, " research reports + ")
101 ls_wrn(fd, d); ls_wr(fd, " foundationed docs + ")
102 ls_wrn(fd, f); ls_wr(fd, " fetched sources. Auto-discovered — drop a new source in and re-run to ingest it.</p>\n")
103 ls_wr(fd, "</body></html>\n")
104 sys_close(fd)
105
106 ls_puts(" research-reports="); ls_putn(r)
107 ls_puts(" foundationed-docs="); ls_putn(d)
108 ls_puts(" fetched-sources="); ls_putn(f)
109 ls_puts(" TOTAL="); ls_putn(total); ls_puts(" -> web_assets/library-sources.html (STAGED; live via the Nishi Publisher, operator-gated)\n")
110 if total > 0 { sys_exit(0); return 0 }
111 sys_exit(1); return 1
112}