code wiki / _hdl_build / nx_wiki_answer_index.nx
nx_wiki_answer_index.nx source
↩ module page · 187 lines · 11981 B
1// nx_wiki_answer_index.nx -- INDEX DRIVER for the instant-answer search over the 19-page live wiki corpus (real-tree deepened)
2// (IMS Thrust B rung 1). Builds the "id<TAB>text" search-source TSV the reusable onsite ingestor
3// (nx_onsite_index) consumes: one row per live wiki page (13 prior content pages + 6 batch-#2 pages; the search page itself is excluded by design), id = slug, text = the page's VISIBLE text with
4// HTML scrubbed. REUSE not reinvent -- the HTML->text scrub is nx_html_to_text (strips tags, suppresses
5// <script>/<style>, collapses whitespace, decodes entities) VERBATIM; the slug->web_assets file map mirrors
6// nx_ims_corpus_refresh EXACTLY (start lives in ai_start.html, the /wiki/ econsim is econsim_wiki.html), so
7// we index the SAME bytes vpub publishes (== what is served at https://nishifamily.com/wiki/<slug>.html).
8//
9// After this writes knowledge/index/wiki_answer_src.tsv, the build step runs the EXISTING ingestor:
10// nx_onsite_index knowledge/index/wiki_answer_src.tsv https://nishifamily.com/wiki/ \
11// knowledge/index/wiki.idx knowledge/index/wiki.manifest
12// which produces the durable inverted index + manifest the answer organ + onsite client read, and the
13// committed knowledge/registry/onsite_sites.tsv row (site=wiki) points at exactly that pair.
14//
15// Hardcoded fixed 19-page job (no argv) so `nx_sov_build_run.elf nx_wiki_answer_index` builds AND runs it in
16// one shot. ADDITIVE-ONLY: writes the search-source TSV; touches no live page. license_tier: ORIGINAL
17import "nx_syscalls.nx"
18import "nx_itoa_lib.nx" // shared MSB-first emitter (zero-alloc)
19import "nx_html_to_text.nx"
20
21const WAI_SRC_OUT: *u8 = "knowledge/index/wiki_answer_src.tsv"
22const WAI_HTML_CAP: i64 = 1048576 // 1 MB per source page (>> any real page)
23const WAI_TXT_CAP: i64 = 1048576 // scrubbed-text scratch
24const WAI_BUF_CAP: i64 = 8388608 // 8 MB accumulated TSV (12 pages of text)
25const WAI_MODE: i64 = 0x1a4 // 0644
26
27func wai_w(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
28// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
29// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
30// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
31// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
32func wai_num(v: i64) -> i64 { nxi_out(v); return 0 }
33func wai_cat(dst: *u8, off: i64, s: *u8) -> i64 { var i: i64 = 0; while s[i] != 0 as u8 { dst[off + i] = s[i]; i = i + 1 } return off + i }
34
35// lowercase a byte (ASCII).
36func wai_lc(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c }
37
38// case-insensitive: does src[p..] begin with "<name" followed by a tag-boundary byte ('>', '/', or whitespace)?
39// name is a lowercase NUL-terminated tag word (e.g. "nav"). Returns 1 on match.
40func wai_is_open_tag(src: *u8, n: i64, p: i64, name: *u8) -> i64 {
41 if p >= n { return 0 }
42 if src[p] != (0x3c as u8) { return 0 } // '<'
43 var i: i64 = 0
44 while name[i] != (0 as u8) {
45 if p + 1 + i >= n { return 0 }
46 if wai_lc(src[p + 1 + i] as i64) != (name[i] as i64) { return 0 }
47 i = i + 1
48 }
49 let after: i64 = p + 1 + i
50 if after >= n { return 0 }
51 let c: i64 = src[after] as i64
52 if c == 0x3e { return 1 } // '>'
53 if c == 0x2f { return 1 } // '/' (self-close, unlikely for these)
54 if c == 0x20 { return 1 } // space
55 if c == 0x09 { return 1 } // tab
56 if c == 0x0a { return 1 } // LF
57 if c == 0x0d { return 1 } // CR
58 return 0
59}
60
61// case-insensitive: does src[p..] begin with "</name>" (optional ws before '>')? Returns 1 on match.
62func wai_is_close_tag(src: *u8, n: i64, p: i64, name: *u8) -> i64 {
63 if p + 1 >= n { return 0 }
64 if src[p] != (0x3c as u8) { return 0 } // '<'
65 if src[p + 1] != (0x2f as u8) { return 0 } // '/'
66 var i: i64 = 0
67 while name[i] != (0 as u8) {
68 if p + 2 + i >= n { return 0 }
69 if wai_lc(src[p + 2 + i] as i64) != (name[i] as i64) { return 0 }
70 i = i + 1
71 }
72 // allow optional whitespace then '>'
73 var r: i64 = p + 2 + i
74 while r < n { let c: i64 = src[r] as i64; if c == 0x20 { r = r + 1 } else { if c == 0x09 { r = r + 1 } else { if c == 0x3e { return 1 } else { return 0 } } } }
75 return 0
76}
77
78// advance past the current tag's '>' (returns index AFTER '>'); if none, returns n.
79func wai_past_gt(src: *u8, n: i64, p: i64) -> i64 {
80 var q: i64 = p
81 while q < n { if src[q] == (0x3e as u8) { return q + 1 } q = q + 1 }
82 return n
83}
84
85// strip block-level CHROME elements (nav/aside/header/footer) -- tag + inner content, with same-tag nesting --
86// from `src` into `dst`. Removing the sidebar/breadcrumb/prevnext/related/backlinks/footer is the ROOT-CAUSE fix
87// for weak relevance: those blocks list EVERY sibling page title on EVERY page, so terms like "nist"/"stem" hit
88// df=12/12 (zero IDF) and a rare body term wins instead of the on-topic page. Keeps <main>/<article>/<section>
89// prose. Returns dst length. Generic over a tag name so adding tags is one more call (no formula change).
90func wai_strip_tag(src: *u8, n: i64, dst: *u8, name: *u8) -> i64 {
91 var o: i64 = 0
92 var p: i64 = 0
93 while p < n {
94 if wai_is_open_tag(src, n, p, name) == 1 {
95 var depth: i64 = 1
96 p = wai_past_gt(src, n, p) // skip the opening tag itself
97 while depth > 0 {
98 if p >= n { depth = 0 } else {
99 if wai_is_open_tag(src, n, p, name) == 1 { depth = depth + 1; p = wai_past_gt(src, n, p) }
100 else { if wai_is_close_tag(src, n, p, name) == 1 { depth = depth - 1; p = wai_past_gt(src, n, p) }
101 else { p = p + 1 } }
102 }
103 }
104 } else {
105 dst[o] = src[p]; o = o + 1; p = p + 1
106 }
107 }
108 return o
109}
110
111// append "slug<TAB>scrubbed-text<NL>" into dst at off. Reads `file`, scrubs HTML->text, then copies the text
112// with TAB/newline/CR FLATTENED to single spaces (so the whole page body is ONE clean TSV cell). Returns the
113// new off, or -1 on read fail. token-noise (long whitespace) is already collapsed by nx_html_to_text.
114func wai_row(dst: *u8, off: i64, slug: *u8, file: *u8) -> i64 {
115 let nbox: *i64 = sys_mmap(16) as *i64
116 nbox[0] = 0
117 let html: *u8 = sys_read_file(file, nbox)
118 if (html as i64) == 0 { wai_w(" READ-FAIL "); wai_w(file); wai_w("\n"); return 0 - 1 }
119 let hlen: i64 = nbox[0]
120 if hlen < 1 { wai_w(" EMPTY "); wai_w(file); wai_w("\n"); return 0 - 1 }
121
122 // strip CHROME blocks (nav/aside/header/footer) BEFORE the HTML->text scrub: the wiki-nav sidebar + breadcrumb
123 // + prev/next + related/backlinks list every sibling page's title on every page, which floods the vocab with
124 // df=12/12 terms (zero IDF) and breaks relevance. Two ping-pong scratch buffers, one tag per pass.
125 let s1: *u8 = sys_mmap(WAI_HTML_CAP)
126 let s2: *u8 = sys_mmap(WAI_HTML_CAP)
127 var clen: i64 = wai_strip_tag(html, hlen, s1, "nav" as *u8)
128 clen = wai_strip_tag(s1, clen, s2, "aside" as *u8)
129 clen = wai_strip_tag(s2, clen, s1, "header" as *u8)
130 clen = wai_strip_tag(s1, clen, s2, "footer" as *u8)
131
132 let txt: *u8 = sys_mmap(WAI_TXT_CAP)
133 let tlen: i64 = nx_html_to_text(s2, clen, txt, WAI_TXT_CAP)
134 if tlen < 0 { wai_w(" SCRUB-FAIL "); wai_w(file); wai_w("\n"); return 0 - 1 }
135
136 var o: i64 = off
137 o = wai_cat(dst, o, slug)
138 dst[o] = 0x09 as u8; o = o + 1 // TAB id|text separator
139 // copy scrubbed text, flattening TAB(0x09)/LF(0x0a)/CR(0x0d) -> space so the row stays a single TSV cell
140 var b: i64 = 0
141 while b < tlen {
142 let c: i64 = txt[b] as i64
143 if c == 0x09 { dst[o] = 0x20 as u8 } else { if c == 0x0a { dst[o] = 0x20 as u8 } else { if c == 0x0d { dst[o] = 0x20 as u8 } else { dst[o] = txt[b] } } }
144 o = o + 1
145 b = b + 1
146 }
147 dst[o] = 0x0a as u8; o = o + 1 // row terminator
148 wai_w(" indexed "); wai_w(slug); wai_w(" htmlB="); wai_num(hlen); wai_w(" dechromedB="); wai_num(clen); wai_w(" textB="); wai_num(tlen); wai_w("\n")
149 return o
150}
151
152func main() -> i64 {
153 let buf: *u8 = sys_mmap(WAI_BUF_CAP)
154 var o: i64 = 0
155 var n: i64 = 0
156 wai_w("=== wiki-answer index: scrub 12 live wiki pages -> id<TAB>text search source ===\n")
157
158 // slug -> web_assets file (MIRRORS nx_ims_corpus_refresh: start=ai_start.html, econsim=econsim_wiki.html).
159 let r1: i64 = wai_row(buf, o, "start" as *u8, "web_assets/ai_start.html" as *u8); if r1 >= 0 { o = r1; n = n + 1 }
160 let r2: i64 = wai_row(buf, o, "charter" as *u8, "web_assets/charter.html" as *u8); if r2 >= 0 { o = r2; n = n + 1 }
161 let r3: i64 = wai_row(buf, o, "nist_stem" as *u8, "web_assets/nist_stem.html" as *u8); if r3 >= 0 { o = r3; n = n + 1 }
162 let r4: i64 = wai_row(buf, o, "roadmap" as *u8, "web_assets/roadmap.html" as *u8); if r4 >= 0 { o = r4; n = n + 1 }
163 let r5: i64 = wai_row(buf, o, "board" as *u8, "web_assets/board.html" as *u8); if r5 >= 0 { o = r5; n = n + 1 }
164 let r6: i64 = wai_row(buf, o, "devlog" as *u8, "web_assets/devlog.html" as *u8); if r6 >= 0 { o = r6; n = n + 1 }
165 let r7: i64 = wai_row(buf, o, "econsim" as *u8, "web_assets/econsim_wiki.html" as *u8); if r7 >= 0 { o = r7; n = n + 1 }
166 let r8: i64 = wai_row(buf, o, "products" as *u8, "web_assets/products.html" as *u8); if r8 >= 0 { o = r8; n = n + 1 }
167 let r9: i64 = wai_row(buf, o, "genesis_genealogy" as *u8, "web_assets/genesis_genealogy.html" as *u8); if r9 >= 0 { o = r9; n = n + 1 }
168 let r10: i64 = wai_row(buf, o, "access_wall" as *u8, "web_assets/access_wall.html" as *u8); if r10 >= 0 { o = r10; n = n + 1 }
169 let r11: i64 = wai_row(buf, o, "formats_uxf" as *u8, "web_assets/formats_uxf.html" as *u8); if r11 >= 0 { o = r11; n = n + 1 }
170 let r12: i64 = wai_row(buf, o, "media_studio" as *u8, "web_assets/media_studio.html" as *u8); if r12 >= 0 { o = r12; n = n + 1 }
171 // batch #2: the substrate hub + 6 leaves (the real-tree deepening) -> instant-answer now covers them.
172 let r13: i64 = wai_row(buf, o, "substrate" as *u8, "web_assets/substrate.html" as *u8); if r13 >= 0 { o = r13; n = n + 1 }
173 let r14: i64 = wai_row(buf, o, "nishi_os" as *u8, "web_assets/nishi_os.html" as *u8); if r14 >= 0 { o = r14; n = n + 1 }
174 let r15: i64 = wai_row(buf, o, "sovereign_email" as *u8, "web_assets/sovereign_email.html" as *u8); if r15 >= 0 { o = r15; n = n + 1 }
175 let r16: i64 = wai_row(buf, o, "simd_backend" as *u8, "web_assets/simd_backend.html" as *u8); if r16 >= 0 { o = r16; n = n + 1 }
176 let r17: i64 = wai_row(buf, o, "neural_mt" as *u8, "web_assets/neural_mt.html" as *u8); if r17 >= 0 { o = r17; n = n + 1 }
177 let r18: i64 = wai_row(buf, o, "vision_fr" as *u8, "web_assets/vision_fr.html" as *u8); if r18 >= 0 { o = r18; n = n + 1 }
178 let r19: i64 = wai_row(buf, o, "voxelworld" as *u8, "web_assets/voxelworld.html" as *u8); if r19 >= 0 { o = r19; n = n + 1 }
179
180 let fd: i64 = sys_openat_wr(WAI_SRC_OUT, WAI_MODE)
181 if fd < 0 { wai_w("FATAL: cannot open search-source for write\n"); sys_exit(2); return 2 }
182 sys_write(fd, buf, o)
183 sys_close(fd)
184 wai_w("=== wrote "); wai_w(WAI_SRC_OUT); wai_w(" rows="); wai_num(n); wai_w(" bytes="); wai_num(o); wai_w(" ===\n")
185 if n == 19 { sys_exit(0); return 0 }
186 sys_exit(1); return 1
187}