code wiki / _hdl_build / nx_onsite_search.nx
nx_onsite_search.nx source
↩ module page · 294 lines · 14208 B
1// nx_onsite_search.nx -- REUSABLE per-site onsite search CLIENT (manifest model). ONE BM25 engine pointed at
2// ANY site via a data-driven sites registry (Cardinal 11/17). Each site = its own durable inverted index +
3// MANIFEST + base URL, so a query is SCOPED to exactly that site BY CONSTRUCTION (site A's index file
4// physically cannot return site B's docs). Generalizes nx_search_cli to the wiki/gallery/andelinwest.com and
5// any future site -- "find files and articles" on just that site. REUSE not reinvent: inverted-index shortlist
6// (nx_search_inverted) + the team's Okapi BM25 ranker (bm_*), zero new ranking math.
7//
8// Usage: nx_onsite_search <sites.tsv> <site> <term> [term ...]
9// Registry row (TAB-separated, trailing newline required): site<TAB>idx_path<TAB>manifest_path<TAB>base_url
10// Manifest line (built by nx_onsite_index, docid = line number): url<TAB>title<TAB>text
11//
12// MANIFEST MODEL (vs nx_search_cli's file-per-doc): the searchable text lives INLINE in the manifest, so a
13// 366k-image corpus is ONE file, not 366k tiny files, and BM25 scores in-memory (no per-candidate file reads).
14// Display + scoring are length-based (fields are slices, not NUL-terminated). license_tier: ORIGINAL
15import "nx_search_inverted_persist.nx"
16import "nx_bm25.nx"
17import "nx_bm25f.nx" // BM25F fielded ranking: title hits outrank body hits (no-op when title is non-textual, e.g. a cid)
18const OS_MAGIC_1000000: i64 = 1000000
19const OS_MAGIC_10000: i64 = 10000
20const OS_MAGIC_8192: i64 = 8192
21
22const OS_MAX_TERMS: i64 = 8
23const OS_MAX_X: i64 = 64 // expanded query-term cap (originals + per-site thesaurus concept expansions)
24const OS_ROWID_CAP: i64 = 16384 // matches NX_INV_MAX_POSTINGS_PER (per-term postings cap)
25const OS_TOP_K: i64 = 10
26
27func os_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
28func os_putb(p: *u8, n: i64) -> i64 { if n>0 { sys_write(1,p,n) } return 0 }
29func os_num(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=(48 as u8);k=1}; while m>0{t[k]=((48+(m%10)) as u8);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(1,bb,k); return 0 }
30func os_strlen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
31func os_streq(a: *u8, b: *u8) -> i64 { var i: i64=0; while a[i]!=(0 as u8){ if a[i]!=b[i]{return 0} i=i+1 } if b[i]!=(0 as u8){return 0} return 1 }
32func os_fold(s: *u8, n: i64) -> i64 { var i: i64=0; while i<n { let c: i64=s[i] as i64; if c>=65 { if c<=90 { s[i]=(c+32) as u8 } } i=i+1 } return 0 }
33func os_score(v: i64) -> i64 {
34 var m: i64=v
35 if m<0 { sys_write(1,"-" as *u8,1); m=0-m }
36 os_num(m/OS_MAGIC_1000000); sys_write(1,"." as *u8,1)
37 let frac: i64=(m%OS_MAGIC_1000000)/OS_MAGIC_10000
38 if frac<10 { sys_write(1,"0" as *u8,1) }
39 os_num(frac); return 0
40}
41
42// registry lookup -- NUL-terminate the row's 4 fields IN PLACE; return field pointers. 1=found,0=not. Registry must end with newline.
43func os_lookup_site(reg: *u8, n: i64, site: *u8, out_idx: *i64, out_mf: *i64, out_base: *i64) -> i64 {
44 var ls: i64=0; var i: i64=0
45 while i<=n {
46 var eol: i64=0
47 if i==n { eol=1 } else { if reg[i]==(10 as u8) { eol=1 } }
48 if eol==1 {
49 if i>ls {
50 var t1: i64=0-1; var t2: i64=0-1; var t3: i64=0-1
51 var k: i64=ls
52 while k<i {
53 if reg[k]==(9 as u8) { if t1<0 { t1=k } else { if t2<0 { t2=k } else { if t3<0 { t3=k } } } }
54 k=k+1
55 }
56 if t1>ls { if t2>t1 { if t3>t2 {
57 reg[t1]=0 as u8; reg[t2]=0 as u8; reg[t3]=0 as u8
58 if i<n { reg[i]=0 as u8 }
59 let sp: *u8=((reg as i64)+ls) as *u8
60 if os_streq(sp, site)==1 {
61 out_idx[0]=(reg as i64)+t1+1
62 out_mf[0]=(reg as i64)+t2+1
63 out_base[0]=(reg as i64)+t3+1
64 return 1
65 }
66 } } }
67 }
68 ls=i+1
69 }
70 i=i+1
71 }
72 return 0
73}
74
75// CONCEPT EXPANSION (sovereign, data-driven): if the per-site thesaurus has a line whose trigger (field before
76// the first TAB) == `term`, copy its expansion tokens (after the TAB, space-separated) into `scratch`
77// NUL-terminated and append them to xterms/xlens. Bridges how people TYPE ("kids","split") to how docs are
78// WRITTEN ("custody","divorce"). HONEST: curated concept expansion, NOT neural semantics -- a novel paraphrase
79// with no thesaurus line still misses (that needs the LLM partnership). Returns the new term count.
80func os_expand(syn: *u8, n: i64, term: *u8, tl: i64, xterms: *i64, xlens: *i64, nx0: i64, scratch: *u8, soff: *i64, scap: i64) -> i64 {
81 var nx: i64=nx0
82 var ls: i64=0; var i: i64=0
83 while i<=n {
84 var eol: i64=0
85 if i==n { eol=1 } else { if syn[i]==(10 as u8) { eol=1 } }
86 if eol==1 {
87 if i>ls {
88 var tab: i64=0-1; var k: i64=ls
89 while k<i { if syn[k]==(9 as u8) { if tab<0 { tab=k } } k=k+1 }
90 if tab>ls {
91 var eq: i64=0
92 if tab-ls==tl {
93 eq=1; var c: i64=0
94 while c<tl { if syn[ls+c]!=term[c] { eq=0; c=tl } else { c=c+1 } }
95 }
96 if eq==1 {
97 var p: i64=tab+1; var ts: i64=tab+1
98 while p<=i {
99 var sp: i64=0
100 if p==i { sp=1 } else { if syn[p]==(32 as u8) { sp=1 } }
101 if sp==1 {
102 let toklen: i64=p-ts
103 if toklen>0 { if nx<OS_MAX_X {
104 let off: i64=soff[0]
105 if off+toklen+1<=scap {
106 var b: i64=0
107 while b<toklen { scratch[off+b]=syn[ts+b]; b=b+1 }
108 scratch[off+toklen]=0 as u8
109 xterms[nx]=(scratch as i64)+off; xlens[nx]=toklen; nx=nx+1
110 soff[0]=off+toklen+1
111 }
112 } }
113 ts=p+1
114 }
115 p=p+1
116 }
117 }
118 }
119 }
120 ls=i+1
121 }
122 i=i+1
123 }
124 return nx
125}
126
127func main(argc: i64, argv: *i64) -> i64 {
128 if argc<4 {
129 os_puts("usage: nx_onsite_search <sites.tsv> <site> <term> [term ...]\n" as *u8)
130 sys_exit(2); return 2
131 }
132 let sites_tsv: *u8=argv[1] as *u8
133 let site: *u8=argv[2] as *u8
134 var nterms: i64=argc-3
135 if nterms>OS_MAX_TERMS { nterms=OS_MAX_TERMS }
136
137 let regbox: *i64=sys_mmap(16) as *i64
138 let reg: *u8=sys_read_file(sites_tsv, regbox)
139 if reg==0 as *u8 { os_puts("ONSITE-FAIL: no sites registry\n" as *u8); sys_exit(1); return 1 }
140 let oidx: *i64=sys_mmap(8) as *i64
141 let omf: *i64=sys_mmap(8) as *i64
142 let obase: *i64=sys_mmap(8) as *i64
143 if os_lookup_site(reg, regbox[0], site, oidx, omf, obase)==0 {
144 os_puts("ONSITE-FAIL: unknown site '" as *u8); os_puts(site); os_puts("' (not in registry)\n" as *u8)
145 sys_exit(1); return 1
146 }
147 let idx_path: *u8=oidx[0] as *u8
148 let manifest_path: *u8=omf[0] as *u8
149 let base_url: *u8=obase[0] as *u8
150
151 let idx: *NxInvIndex=nx_inv_load(idx_path)
152 if idx==0 as *NxInvIndex { os_puts("ONSITE-FAIL: no index for site (run nx_onsite_index first)\n" as *u8); sys_exit(1); return 1 }
153
154 // load the per-site manifest (one read) + parse url/title/text slices, docid = line number
155 let mbox: *i64=sys_mmap(16) as *i64
156 let mbuf: *u8=sys_read_file(manifest_path, mbox)
157 if mbuf==0 as *u8 { os_puts("ONSITE-FAIL: no manifest for site\n" as *u8); sys_exit(1); return 1 }
158 let mn: i64=mbox[0]
159 var nlines: i64=0
160 var ii: i64=0
161 while ii<mn { if mbuf[ii]==(10 as u8) { nlines=nlines+1 } ii=ii+1 }
162 if nlines<=0 { os_puts("ONSITE-FAIL: empty manifest\n" as *u8); sys_exit(1); return 1 }
163 let up: *i64=sys_mmap(8*(nlines+8)) as *i64
164 let ul: *i64=sys_mmap(8*(nlines+8)) as *i64
165 let tp: *i64=sys_mmap(8*(nlines+8)) as *i64
166 let tl: *i64=sys_mmap(8*(nlines+8)) as *i64
167 let xp: *i64=sys_mmap(8*(nlines+8)) as *i64
168 let xl: *i64=sys_mmap(8*(nlines+8)) as *i64
169 var ndocs: i64=0
170 var ls: i64=0
171 ii=0
172 while ii<mn {
173 if mbuf[ii]==(10 as u8) {
174 let le: i64=ii
175 var t1: i64=0-1; var t2: i64=0-1
176 var k: i64=ls
177 while k<le { if mbuf[k]==(9 as u8) { if t1<0 { t1=k } else { if t2<0 { t2=k } } } k=k+1 }
178 if t1>=ls { if t2>t1 {
179 up[ndocs]=(mbuf as i64)+ls; ul[ndocs]=t1-ls
180 tp[ndocs]=(mbuf as i64)+t1+1; tl[ndocs]=t2-t1-1
181 xp[ndocs]=(mbuf as i64)+t2+1; xl[ndocs]=le-t2-1
182 } else { up[ndocs]=(mbuf as i64)+ls; ul[ndocs]=0; tp[ndocs]=0; tl[ndocs]=0; xp[ndocs]=0; xl[ndocs]=0 } }
183 if t1<ls { up[ndocs]=(mbuf as i64)+ls; ul[ndocs]=0; tp[ndocs]=0; tl[ndocs]=0; xp[ndocs]=0; xl[ndocs]=0 }
184 ndocs=ndocs+1
185 ls=ii+1
186 }
187 ii=ii+1
188 }
189 if idx.n_rows!=ndocs { os_puts("ONSITE-FAIL: index rows != manifest lines (stale pair -- re-index)\n" as *u8); sys_exit(1); return 1 }
190
191 // load the optional per-site concept thesaurus at <idx_path>.syn (absent -> no expansion; gallery has none)
192 let synp: *u8=sys_mmap(512)
193 var spi: i64=0
194 while idx_path[spi]!=(0 as u8) { synp[spi]=idx_path[spi]; spi=spi+1 }
195 let suf: *u8=".syn" as *u8
196 var sj: i64=0
197 while suf[sj]!=(0 as u8) { synp[spi]=suf[sj]; spi=spi+1; sj=sj+1 }
198 synp[spi]=0 as u8
199 let synbox: *i64=sys_mmap(16) as *i64
200 let synbuf: *u8=sys_read_file(synp, synbox)
201 var synn: i64=0
202 if synbuf!=0 as *u8 { synn=synbox[0] }
203
204 // build the EXPANDED query: original (folded) terms + thesaurus concept expansions
205 let xterms: *i64=sys_mmap(8*OS_MAX_X) as *i64
206 let xlens: *i64=sys_mmap(8*OS_MAX_X) as *i64
207 let xscratch: *u8=sys_mmap(OS_MAGIC_8192)
208 let xsoff: *i64=sys_mmap(16) as *i64; xsoff[0]=0
209 var nx: i64=0
210 var t: i64=0
211 while t<nterms {
212 let term: *u8=argv[3+t] as *u8
213 let l: i64=os_strlen(term)
214 os_fold(term, l)
215 if nx<OS_MAX_X { xterms[nx]=term as i64; xlens[nx]=l; nx=nx+1 }
216 if synn>0 { nx=os_expand(synbuf, synn, term, l, xterms, xlens, nx, xscratch, xsoff, OS_MAGIC_8192) }
217 t=t+1
218 }
219
220 // shortlist: candidate docids = union of each (expanded) term's postings
221 let seen: *u8=sys_mmap(ndocs+8)
222 let res: *NxInvQueryResult=sys_mmap(64) as *NxInvQueryResult
223 let rowids: *i64=sys_mmap(8*OS_ROWID_CAP) as *i64
224 var qi: i64=0
225 while qi<nx {
226 nx_inv_query_term(idx, xterms[qi] as *u8, xlens[qi], rowids, OS_ROWID_CAP, res)
227 var ri: i64=0
228 while ri<res.n_rowids_filled {
229 let rid: i64=rowids[ri]
230 if rid>=0 { if rid<ndocs { seen[rid]=1 as u8 } }
231 ri=ri+1
232 }
233 qi=qi+1
234 }
235 let cand: *i64=sys_mmap(8*(ndocs+8)) as *i64
236 var ncand: i64=0; var di: i64=0
237 while di<ndocs { if seen[di]==(1 as u8) { cand[ncand]=di; ncand=ncand+1 } di=di+1 }
238
239 os_puts("ONSITE-RESULTS site=" as *u8); os_puts(site); os_puts(" matches=" as *u8); os_num(ncand); os_puts(" of docs=" as *u8); os_num(ndocs); os_puts("\n" as *u8)
240 if ncand==0 {
241 os_puts("ONSITE-OK matches=0 (no document on this site contains any query term)\n" as *u8)
242 sys_exit(0); return 0
243 }
244
245 // BM25F FIELDED ranking (nx_bm25f): score the TITLE field + BODY field with a per-field weight table, so a
246 // term in the title outranks the same term in the body. Title=cid for the gallery -> contributes 0 (no-op,
247 // graceful); title=slug/heading for andelinwest/wiki -> boosts the right page. x1e6 scale.
248 let f_tt: *i64=sys_mmap(8*(ncand+8)) as *i64 // title ptr
249 let f_tl: *i64=sys_mmap(8*(ncand+8)) as *i64 // title len
250 let f_dt: *i64=sys_mmap(8*(ncand+8)) as *i64 // title token count
251 let f_bt: *i64=sys_mmap(8*(ncand+8)) as *i64 // body ptr
252 let f_bl: *i64=sys_mmap(8*(ncand+8)) as *i64 // body len
253 let f_db: *i64=sys_mmap(8*(ncand+8)) as *i64 // body token count
254 var sum_t: i64=0; var sum_b: i64=0; var ci: i64=0
255 while ci<ncand {
256 let d: i64=cand[ci]
257 os_fold(tp[d] as *u8, tl[d]); os_fold(xp[d] as *u8, xl[d])
258 f_tt[ci]=tp[d]; f_tl[ci]=tl[d]; f_dt[ci]=bm_token_count(tp[d] as *u8, tl[d])
259 f_bt[ci]=xp[d]; f_bl[ci]=xl[d]; f_db[ci]=bm_token_count(xp[d] as *u8, xl[d])
260 sum_t=sum_t+f_dt[ci]; sum_b=sum_b+f_db[ci]
261 ci=ci+1
262 }
263 var avgt: i64=sum_t/ncand; if avgt<=0 { avgt=1 }
264 var avgb: i64=sum_b/ncand; if avgb<=0 { avgb=1 }
265 let scores: *i64=sys_mmap(8*(ncand+8)) as *i64
266 ci=0
267 while ci<ncand { scores[ci]=bmf_score(f_tt, f_tl, f_dt, f_bt, f_bl, f_db, ncand, ci, avgt, avgb, xterms, nx); ci=ci+1 }
268
269 // selection-sort desc on (scores, cand) lockstep; print top K as "score base_url+url title"
270 var a: i64=0
271 while a<ncand {
272 var best: i64=a; var b: i64=a+1
273 while b<ncand { if scores[b]>scores[best] { best=b } b=b+1 }
274 if best!=a {
275 let ts: i64=scores[a]; scores[a]=scores[best]; scores[best]=ts
276 let tc: i64=cand[a]; cand[a]=cand[best]; cand[best]=tc
277 }
278 a=a+1
279 }
280 var topk: i64=ncand
281 if topk>OS_TOP_K { topk=OS_TOP_K }
282 var r: i64=0
283 while r<topk {
284 let d: i64=cand[r]
285 os_puts(" #" as *u8); os_num(r+1)
286 os_puts(" score=" as *u8); os_score(scores[r])
287 os_puts(" " as *u8); os_puts(base_url); os_putb(up[d] as *u8, ul[d])
288 os_puts(" " as *u8); os_putb(tp[d] as *u8, tl[d])
289 os_puts("\n" as *u8)
290 r=r+1
291 }
292 os_puts("ONSITE-OK\n" as *u8)
293 sys_exit(0); return 0
294}