code wiki / _hdl_build / nx_corpus_ingest.nx
nx_corpus_ingest.nx source
↩ module page · 220 lines · 11715 B
1// nx_corpus_ingest.nx -- SOVEREIGN corpus ingestion: a DIRECTORY of documents -> a domain's PUBLIC seg_store
2// shard, searchable at once (ss_write_seg builds .terms per segment; the store IS the index -- no tsv, no
3// derived artifact). This is the front door for ALL THREE nishi-search fronts (operator 2026-07-03):
4// library: nx_corpus_ingest knowledge/library nishifamily.com (.txt/.md -> /doc text views)
5// site pages: nx_corpus_ingest <pagesdir> andelinwest.com / (.html -> text + url:<cid> row,
6// so the SERP result links to the REAL page instead of /doc)
7// web: the crawler drops fetched pages in a dir -> same ingest with absolute-URL prefix rows
8// Mechanics: getdents64 walk (flat dir) -> .txt/.md raw, .html/.htm via nx_html_to_text (script/style
9// suppressed, entities decoded) -> cid = the same polynomial hash nx_dp_tsv_migrate uses (content-addressed,
10// idempotent) -> skip-if-already-present (re-runs are clean) -> RAW ss_add doc:<cid> (+ url:<cid>) batched
11// into 1MB writer segments, each committed under a FRESH segid (manifest max + 1 -- never clobbers existing
12// segments). license_tier: ORIGINAL
13import "nx_docportal_search_seg.nx"
14import "nx_html_to_text.nx"
15const CI_MAGIC_1125899906842597: i64 = 1125899906842597
16const CI_MAGIC_131072: i64 = 131072
17const CI_MAGIC_1024: i64 = 1024
18const CI_MAGIC_65536: i64 = 65536
19
20const CI_DOCCAP: i64 = 900000 // per-doc byte cap (writer segments are 1MB; a doc must fit with headroom)
21const CI_MINDOC: i64 = 20 // skip empty-ish extractions
22const CI_RAWCAP: i64 = 8388608 // largest raw file we read
23
24func ci_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
25func ci_num(v: i64) -> i64 {
26 let bb: *u8 = sys_mmap(28); var m: i64 = v
27 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
28 let t: *u8 = sys_mmap(28); var k: i64 = 0
29 if m == 0 { t[0] = 48 as u8; k = 1 }
30 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
31 var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 }
32 sys_write(1, bb, k); return 0
33}
34func ci_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
35func ci_cat(d: *u8, o: i64, s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { d[o + i] = s[i]; i = i + 1 } return o + i }
36
37// stable positive content id -- SAME polynomial as nx_dp_tsv_migrate.mg_hash (idempotent across organs)
38func ci_hash(s: *u8, n: i64) -> i64 {
39 var h: i64 = CI_MAGIC_1125899906842597
40 var i: i64 = 0
41 while i < n { h = (h * 131) + (s[i] as i64); i = i + 1 }
42 if h < 0 { h = 0 - h }
43 return h & 0x7fffffffffffffff
44}
45// build "url:<cid>" (MUST match the serve side's dsv_mkurlkey)
46func ci_mkurlkey(cid: i64, out: *u8) -> i64 {
47 out[0] = 117 as u8; out[1] = 114 as u8; out[2] = 108 as u8; out[3] = 58 as u8
48 var o: i64 = 4
49 if cid == 0 { out[o] = 48 as u8; o = o + 1; out[o] = 0 as u8; return o }
50 let t: *u8 = sys_mmap(24)
51 var k: i64 = 0
52 var m: i64 = cid
53 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
54 var j: i64 = 0
55 while j < k { out[o] = t[k - 1 - j]; o = o + 1; j = j + 1 }
56 out[o] = 0 as u8
57 return o
58}
59// does name end with the null-terminated suffix? (case-sensitive; corpus files are ours)
60func ci_ends(name: *u8, suf: *u8) -> i64 {
61 let nl: i64 = ci_len(name)
62 let sl: i64 = ci_len(suf)
63 if nl < sl { return 0 }
64 var i: i64 = 0
65 while i < sl { if name[nl - sl + i] != suf[i] { return 0 } i = i + 1 }
66 return 1
67}
68func ci_contains(name: *u8, ndl: *u8) -> i64 {
69 let nl: i64 = ci_len(name)
70 let dl: i64 = ci_len(ndl)
71 if dl == 0 { return 0 }
72 var i: i64 = 0
73 while i + dl <= nl {
74 var j: i64 = 0
75 var hit: i64 = 1
76 while j < dl { if name[i + j] != ndl[j] { hit = 0; j = dl } else { j = j + 1 } }
77 if hit == 1 { return 1 }
78 i = i + 1
79 }
80 return 0
81}
82// classify: 1 = raw text (.txt/.md), 2 = html (.html/.htm), 0 = skip (.bak anywhere, everything else)
83func ci_kind(name: *u8) -> i64 {
84 if ci_contains(name, ".bak" as *u8) == 1 { return 0 }
85 if ci_ends(name, ".txt" as *u8) == 1 { return 1 }
86 if ci_ends(name, ".md" as *u8) == 1 { return 1 }
87 if ci_ends(name, ".rst" as *u8) == 1 { return 1 }
88 if ci_ends(name, ".html" as *u8) == 1 { return 2 }
89 if ci_ends(name, ".htm" as *u8) == 1 { return 2 }
90 return 0
91}
92
93// THE INGEST. Returns docs ingested (>=0), or -1 open-fail. counts[0]=ingested counts[1]=skipped-present
94// counts[2]=skipped-other counts[3]=segments-committed
95func ci_run(dir: *u8, domain: *u8, urlprefix: *u8, counts: *i64) -> i64 {
96 counts[0] = 0; counts[1] = 0; counts[2] = 0; counts[3] = 0
97 let prefix: *u8 = sys_mmap(512)
98 dss_prefix(domain, prefix)
99 // fresh segid = manifest max + 1 (NEVER clobber an existing segment)
100 let segs: *i64 = sys_mmap(256 * 8) as *i64
101 let nseg: i64 = ss_manifest_file(prefix, "manifest.txt" as *u8, segs)
102 var segid: i64 = 1
103 var si: i64 = 0
104 // ROOT FIX seq1730 (id 1785450987): segs[] holds POINTERS to seg-<id> name strings
105 // (nx_seg_store.nx:1234 stores segs[cnt] = name as i64), NOT ids -- so segs[si] + 1 produced
106 // MMAP_ADDRESS+1 (~1.4e14), the pointer-shaped poison that PINS a plane forever: every later
107 // epoch id sorts BELOW it in supersede order and its rows are silently shadowed while rc=0.
108 // Parse the DIGITS, as nx_web_shard_compact.nx:55-58 already does on this SAME array.
109 while si < nseg {
110 let sg_nm: *u8 = segs[si] as *u8
111 var sg_v: i64 = 0
112 var sg_ci: i64 = 0
113 while sg_nm[sg_ci] != (0 as u8) { let sg_c: i64 = sg_nm[sg_ci] as i64; if sg_c >= 48 { if sg_c <= 57 { sg_v = sg_v * 10 + (sg_c - 48) } } sg_ci = sg_ci + 1 }
114 if sg_v >= segid { segid = sg_v + 1 }
115 si = si + 1
116 }
117 // open the CURRENT store once for skip-if-present (0 = fresh shard, fine)
118 let h: *i64 = ss_open(prefix)
119 let pbox: *i64 = sys_mmap(16) as *i64
120 let lbox: *i64 = sys_mmap(16) as *i64
121
122 let fd: i64 = sys_openat_rd(dir)
123 if fd < 0 { ci_puts("corpus-ingest: cannot open dir " as *u8); ci_puts(dir); ci_puts("\n" as *u8); return 0 - 1 }
124 let dbuf: *u8 = sys_mmap(CI_MAGIC_131072)
125 let path: *u8 = sys_mmap(CI_MAGIC_1024)
126 let key: *u8 = sys_mmap(64)
127 let ukey: *u8 = sys_mmap(64)
128 let urlval: *u8 = sys_mmap(CI_MAGIC_1024)
129 let hout: *u8 = sys_mmap(CI_RAWCAP + CI_MAGIC_65536)
130 var w: *i64 = ss_begin()
131 var go: i64 = 1
132 while go == 1 {
133 let nr: i64 = sys_getdents64(fd, dbuf, CI_MAGIC_131072)
134 if nr <= 0 { go = 0 } else {
135 var off: i64 = 0
136 while off < nr {
137 let rec: *u8 = (dbuf as i64 + off) as *u8
138 let ty: i64 = dirent_type(rec)
139 let nm: *u8 = dirent_name(rec)
140 off = off + dirent_reclen(rec)
141 if ty != 4 {
142 let kind: i64 = ci_kind(nm)
143 if kind == 0 { counts[2] = counts[2] + 1 } else {
144 var po: i64 = ci_cat(path, 0, dir)
145 po = ci_cat(path, po, "/" as *u8)
146 po = ci_cat(path, po, nm)
147 path[po] = 0 as u8
148 let szp: *i64 = sys_mmap(16) as *i64
149 let raw: *u8 = ss_readall(path, szp)
150 var rn: i64 = szp[0]
151 if rn > CI_RAWCAP { rn = 0 } // absurd file -> skip, never OOM
152 if rn <= 0 { counts[2] = counts[2] + 1 } else {
153 var txt: *u8 = raw
154 var tn: i64 = rn
155 if kind == 2 {
156 let hr: i64 = nx_html_to_text(raw, rn, hout, CI_RAWCAP + CI_MAGIC_65536)
157 if hr <= 0 { tn = 0 } else { txt = hout; tn = hr }
158 }
159 if tn > CI_DOCCAP { tn = CI_DOCCAP }
160 if tn < CI_MINDOC { counts[2] = counts[2] + 1 } else {
161 let cid: i64 = ci_hash(txt, tn)
162 dss_mkkey(cid, key)
163 var present: i64 = 0
164 if (h as i64) != 0 { if ss_hget(h, key, pbox, lbox) == 1 { present = 1 } }
165 if present == 1 { counts[1] = counts[1] + 1 } else {
166 if ss_add(w, 1, key, txt, tn) < 0 {
167 // writer full -> commit this segment, start the next
168 if ss_commit(prefix, w, segid) != 0 { ci_puts(" COMMIT-FAIL seg=" as *u8); ci_num(segid); ci_puts("\n" as *u8) } else { counts[3] = counts[3] + 1 }
169 segid = segid + 1
170 w = ss_begin()
171 if ss_add(w, 1, key, txt, tn) < 0 { counts[2] = counts[2] + 1 } else { counts[0] = counts[0] + 1 }
172 } else { counts[0] = counts[0] + 1 }
173 // NULL POINTER = no url rows. NEVER an empty-string literal here: `"" as *u8`
174 // miscompiles to an unterminated pool pointer (the 2026-07-03 library-url bug --
175 // 596 docs got "corpus-ingest: dir=..." hrefs; same codegen-gotcha family as the
176 // inline cast-index one). Gate T8 pins the no-urlprefix path clean.
177 if (urlprefix as i64) != 0 { if ci_len(urlprefix) > 0 {
178 var uo: i64 = ci_cat(urlval, 0, urlprefix)
179 uo = ci_cat(urlval, uo, nm)
180 ci_mkurlkey(cid, ukey)
181 if ss_add(w, 1, ukey, urlval, uo) < 0 {
182 if ss_commit(prefix, w, segid) == 0 { counts[3] = counts[3] + 1 }
183 segid = segid + 1
184 w = ss_begin()
185 ss_add(w, 1, ukey, urlval, uo)
186 }
187 } }
188 }
189 }
190 }
191 }
192 }
193 }
194 }
195 }
196 sys_close(fd)
197 if w[1] > 0 {
198 if ss_commit(prefix, w, segid) == 0 { counts[3] = counts[3] + 1 } else { ci_puts(" FINAL-COMMIT-FAIL\n" as *u8) }
199 }
200 return counts[0]
201}
202
203func main(argc: i64, argv: *i64) -> i64 {
204 if argc < 3 { ci_puts("usage: nx_corpus_ingest <dir> <domain> [urlprefix]\n" as *u8); return 1 }
205 let dir: *u8 = argv[1] as *u8
206 let domain: *u8 = argv[2] as *u8
207 var urlprefix: *u8 = 0 as *u8 // null = no url rows ("" literal miscompiles -- see ci_run)
208 if argc >= 4 { urlprefix = argv[3] as *u8 }
209 let counts: *i64 = sys_mmap(64) as *i64
210 let rc: i64 = ci_run(dir, domain, urlprefix, counts)
211 if rc < 0 { return 2 }
212 ci_puts("corpus-ingest: dir=" as *u8); ci_puts(dir)
213 ci_puts(" -> shard dp-" as *u8); ci_puts(domain); ci_puts("-pub- ingested=" as *u8); ci_num(counts[0])
214 ci_puts(" skipped_present=" as *u8); ci_num(counts[1])
215 ci_puts(" skipped_other=" as *u8); ci_num(counts[2])
216 ci_puts(" segments=" as *u8); ci_num(counts[3])
217 ci_puts("\n" as *u8)
218 if counts[0] + counts[1] == 0 { ci_puts("corpus-ingest: NOTHING ingested (empty dir or all skipped)\n" as *u8); return 3 }
219 return 0
220}