code wiki / _hdl_build / nx_onsite_index.nx
nx_onsite_index.nx source
↩ module page · 194 lines · 9814 B
1// nx_onsite_index.nx -- SITE-AGNOSTIC INGESTOR for the reusable onsite search client. Turns a site's
2// search source (a 2-column "id<TAB>text" TSV -- the universal "search manifest source") into the durable
3// pair the client reads: an inverted index (nx_search_inverted) + a MANIFEST (url<TAB>title<TAB>text, one
4// line per doc, docid = line). REUSE not reinvent -- the gallery ALREADY produces id<TAB>text
5// (knowledge/status/galx_search.tsv via nx_galx_search_index); any site emits the same shape and gets BM25
6// onsite search for free. "find files and articles": id = the file/article id, url = <url_prefix><id>, text
7// = its searchable body.
8//
9// Usage: nx_onsite_index <src_id_text.tsv> <url_prefix> <idx_out> <manifest_out> [max_docs]
10// e.g. nx_onsite_index knowledge/status/galx_search.tsv /img/ knowledge/index/gallery.idx knowledge/index/gallery.manifest
11//
12// COLLECT-THEN-BUILD with DEDUP: one pass collects the UNIQUE-by-id rows (galx_search.tsv had ~38% duplicate
13// cids -- re-indexed images -- which would otherwise surface the same image many times); then a clean 2-pass
14// index build + manifest write over the deduped set keeps idx docids and manifest lines aligned by construction.
15// Pure-numeric tokens (image seeds/dims/steps) are scrubbed from the INDEXED copy (search noise); the manifest
16// keeps the original text. license_tier: ORIGINAL
17import "nx_search_inverted_persist.nx"
18
19const OI_DEFAULT_MAX: i64 = 4000000
20const OI_SCRATCH_CAP: i64 = 1048576 // per-doc scrub scratch (>= longest doc text)
21const OI_VOCAB_CAP: i64 = 1048576 // 2^20 vocab slots (power of 2): CEILING for large corpora w/o saturating
22const OI_VOCAB_MIN: i64 = 2048 // 2^11 floor: smallest vocab table (tiny corpora still get headroom)
23const OI_DEDUP_CAP: i64 = 2097152 // 2^21 id-dedup hash-set slots (power of 2)
24
25func oi_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
26func oi_num(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=(48 as u8);k=1}; while m>0{t[k]=((48+(m%10)) as u8);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(1,bb,k); return 0 }
27func oi_strlen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
28func oi_atoi(s: *u8) -> i64 { var v: i64=0; var i: i64=0; while s[i]!=(0 as u8){ if s[i]>=(48 as u8){ if s[i]<=(57 as u8){ v=v*10+((s[i]-(48 as u8)) as i64) } } i=i+1 } return v }
29
30// smallest power of two >= n (the vocab table is masked with hash&(cap-1) -> capacity MUST be a power of two).
31func oi_next_pow2(n: i64) -> i64 { var p: i64=1; while p<n { p=p*2 } return p }
32
33// auto-size the vocab table to the corpus so a 12-page wiki does not pay for a 1M-slot (32 MB) table while a
34// NAS-scale corpus still gets the full table. Vocab slots are bounded by the token count; total text bytes / 4
35// is a safe upper bound on distinct tokens (avg token ~>=4 chars incl. the separator), rounded up to the next
36// power of two and clamped to [OI_VOCAB_MIN, OI_VOCAB_CAP]. Load factor stays well under 50% by construction.
37func oi_autosize_vocab(total_txt_bytes: i64) -> i64 {
38 var want: i64 = oi_next_pow2(total_txt_bytes / 4)
39 if want < OI_VOCAB_MIN { want = OI_VOCAB_MIN }
40 if want > OI_VOCAB_CAP { want = OI_VOCAB_CAP }
41 return want
42}
43
44// scrub: copy src[0..n) -> dst, BLANKING pure-numeric alnum tokens (image seeds / dims / steps are search
45// noise that otherwise floods the vocab table). Non-token bytes copied as-is; output same length.
46func oi_scrub_numeric(src: *u8, n: i64, dst: *u8) -> i64 {
47 var i: i64 = 0
48 while i < n {
49 if nx_inv_is_token_char(src[i] as i64) == 0 {
50 dst[i] = src[i]
51 i = i + 1
52 } else {
53 let s: i64 = i
54 var alldigit: i64 = 1
55 var go: i64 = 1
56 while go == 1 {
57 if i >= n { go = 0 } else {
58 if nx_inv_is_token_char(src[i] as i64) == 0 { go = 0 } else {
59 let c: i64 = src[i] as i64
60 if c < 48 { alldigit = 0 } else { if c > 57 { alldigit = 0 } }
61 i = i + 1
62 }
63 }
64 }
65 var k: i64 = s
66 while k < i { if alldigit == 1 { dst[k] = 32 as u8 } else { dst[k] = src[k] } k = k + 1 }
67 }
68 }
69 return n
70}
71
72// id-dedup hash set (open addressing, linear probe). Returns 1 if `h` already present; else inserts + returns 0.
73// `h` must be non-zero (0 is the empty sentinel). cap is a power of two.
74func oi_dedup_seen(set: *i64, cap: i64, h: i64) -> i64 {
75 let mask: i64 = cap - 1
76 var s: i64 = h & mask
77 if s < 0 { s = 0 - s }
78 var iter: i64 = 0
79 while iter < cap {
80 let v: i64 = set[s]
81 if v == 0 { set[s] = h; return 0 }
82 if v == h { return 1 }
83 s = (s + 1) & mask
84 iter = iter + 1
85 }
86 return 1
87}
88
89func main(argc: i64, argv: *i64) -> i64 {
90 if argc<5 {
91 oi_puts("usage: nx_onsite_index <src_id_text.tsv> <url_prefix> <idx_out> <manifest_out> [max_docs]\n" as *u8)
92 sys_exit(2); return 2
93 }
94 let src_path: *u8=argv[1] as *u8
95 let url_prefix: *u8=argv[2] as *u8
96 let idx_out: *u8=argv[3] as *u8
97 let manifest_out: *u8=argv[4] as *u8
98 var max_docs: i64=OI_DEFAULT_MAX
99 if argc>=6 { let m: i64=oi_atoi(argv[5] as *u8); if m>0 { max_docs=m } }
100
101 let szp: *i64=sys_mmap(16) as *i64
102 let buf: *u8=sys_read_file(src_path, szp)
103 if buf==0 as *u8 { oi_puts("ONSITE-INDEX FAIL: cannot read source\n" as *u8); sys_exit(1); return 1 }
104 let sz: i64=szp[0]
105 let upl: i64=oi_strlen(url_prefix)
106
107 // count lines -> size the keep arrays
108 var nlines: i64=0
109 var ci: i64=0
110 while ci<sz { if buf[ci]==(10 as u8) { nlines=nlines+1 } ci=ci+1 }
111 if nlines<=0 { nlines=1 }
112 let keep_id: *i64=sys_mmap(8*(nlines+8)) as *i64 // id ptr (also the manifest title)
113 let keep_idl: *i64=sys_mmap(8*(nlines+8)) as *i64 // id len
114 let keep_tx: *i64=sys_mmap(8*(nlines+8)) as *i64 // text ptr
115 let keep_txl: *i64=sys_mmap(8*(nlines+8)) as *i64 // text len
116 let dedup: *i64=sys_mmap(8*OI_DEDUP_CAP) as *i64
117
118 // ---- PASS 0: dedup by id, collect unique rows in order (accumulate total kept text bytes for vocab sizing) ----
119 var ndocs: i64=0
120 var total_txt: i64=0
121 var ls: i64=0
122 var i: i64=0
123 while i<=sz {
124 var eol: i64=0
125 if i==sz { eol=1 } else { if buf[i]==(10 as u8) { eol=1 } }
126 if eol==1 {
127 if i>ls { if ndocs<max_docs {
128 var tab: i64=0-1
129 var k: i64=ls
130 while k<i { if buf[k]==(9 as u8) { if tab<0 { tab=k } } k=k+1 }
131 if tab>ls {
132 let idp: i64=(buf as i64)+ls
133 let idl: i64=tab-ls
134 var h: i64=nx_inv_hash_bytes_lower(idp as *u8, idl)
135 if h==0 { h=1 }
136 if oi_dedup_seen(dedup, OI_DEDUP_CAP, h)==0 {
137 keep_id[ndocs]=idp; keep_idl[ndocs]=idl
138 keep_tx[ndocs]=(buf as i64)+tab+1; keep_txl[ndocs]=i-tab-1
139 total_txt=total_txt+(i-tab-1)
140 ndocs=ndocs+1
141 }
142 }
143 } }
144 ls=i+1
145 }
146 i=i+1
147 }
148 if ndocs<=0 { oi_puts("ONSITE-INDEX FAIL: no valid id<TAB>text rows\n" as *u8); sys_exit(1); return 1 }
149
150 // ---- build the inverted index (2-pass) over the deduped set; numeric tokens scrubbed ----
151 // vocab table is auto-sized to the corpus (root-cause fix for the 33 MB / 12-page index: the 1M-slot table
152 // serialized 32 MB of empty slots; nx_inv_save writes vocab_capacity*32B regardless of occupancy).
153 let vocab_cap: i64=oi_autosize_vocab(total_txt)
154 let idx: *NxInvIndex=nx_inv_new_sized(vocab_cap)
155 let scrub: *u8=sys_mmap(OI_SCRATCH_CAP)
156 var d: i64=0
157 while d<ndocs {
158 let tl: i64=keep_txl[d]
159 if tl<=OI_SCRATCH_CAP { oi_scrub_numeric(keep_tx[d] as *u8, tl, scrub); nx_inv_index_row(idx, scrub, tl, d) } else { nx_inv_index_row(idx, keep_tx[d] as *u8, tl, d) }
160 d=d+1
161 }
162 idx.n_rows=ndocs
163 nx_inv_finalize_offsets(idx)
164 d=0
165 while d<ndocs {
166 let tl: i64=keep_txl[d]
167 if tl<=OI_SCRATCH_CAP { oi_scrub_numeric(keep_tx[d] as *u8, tl, scrub); nx_inv_emit_row(idx, scrub, tl, d) } else { nx_inv_emit_row(idx, keep_tx[d] as *u8, tl, d) }
168 d=d+1
169 }
170 let rc_save: i64=nx_inv_save(idx, idx_out)
171 if rc_save!=NX_INV_OK { oi_puts("ONSITE-INDEX FAIL: index save\n" as *u8); sys_exit(1); return 1 }
172
173 // ---- write MANIFEST in the SAME docid order: url<TAB>title<TAB>text (url=prefix+id, title=id) ----
174 let mfd: i64=sys_openat_wr(manifest_out, 0x1a4)
175 if mfd<0 { oi_puts("ONSITE-INDEX FAIL: manifest open\n" as *u8); sys_exit(1); return 1 }
176 d=0
177 while d<ndocs {
178 sys_write(mfd, url_prefix, upl)
179 sys_write(mfd, keep_id[d] as *u8, keep_idl[d])
180 sys_write(mfd, "\t" as *u8, 1)
181 sys_write(mfd, keep_id[d] as *u8, keep_idl[d])
182 sys_write(mfd, "\t" as *u8, 1)
183 sys_write(mfd, keep_tx[d] as *u8, keep_txl[d])
184 sys_write(mfd, "\n" as *u8, 1)
185 d=d+1
186 }
187 sys_close(mfd)
188
189 oi_puts("ONSITE-INDEX-OK docs=" as *u8); oi_num(ndocs)
190 oi_puts(" (deduped from " as *u8); oi_num(nlines); oi_puts(" rows) vocab_used=" as *u8); oi_num(idx.vocab_occupied)
191 oi_puts("/cap=" as *u8); oi_num(vocab_cap); oi_puts(" idxBytes=" as *u8); oi_num(NX_INVP_HDR_BYTES + vocab_cap * NX_INV_SLOT_BYTES + idx.postings_used)
192 oi_puts(" -> " as *u8); oi_puts(idx_out); oi_puts(" + " as *u8); oi_puts(manifest_out); oi_puts("\n" as *u8)
193 sys_exit(0); return 0
194}