code wiki / _hdl_build / nx_research_search.nx
nx_research_search.nx source
↩ module page · 141 lines · 6839 B
1// nx_research_search.nx -- CROSS-SHARD QUERY over the self-growing researcher index. The grow organ builds
2// append-only shards (research_0.idx, research_1.idx, ...); this is the consumption side: load every shard,
3// query each for the argv terms, merge the candidate docs across shards, rank by #query-terms-matched, and
4// print the top-K "score shard:path". Makes the grown + deduped + mined corpus actually searchable end-to-end.
5// Sequentially loads shards 0,1,2,... until the first gap (grow numbers them contiguously). NOTE: loading ALL
6// shards in one process is fine for a handful; at large shard counts this needs sys_munmap (the keystone) or a
7// periodic shard-merge. Usage: nx_research_search <term> [term ...] expect_exit: 0 license_tier: ORIGINAL
8import "nx_search_inverted_persist.nx"
9import "nx_itoa_lib.nx" // shared MSB-first emitter (zero-alloc)
10
11const RS_TERMS: i64 = 8
12const RS_ROWIDS: i64 = 4096
13const RS_CAND: i64 = 16384
14const RS_TOPK: i64 = 15
15const RS_MAXSHARD: i64 = 4096
16
17func rs_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
18// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
19// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
20// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
21// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
22func rs_num(v: i64) -> i64 { nxi_out(v); return 0 }
23func rs_strlen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
24func rs_cat(dst: *u8, off: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8){dst[off+i]=s[i];i=i+1} return off+i }
25func rs_catnum(dst: *u8, off: i64, v: i64) -> i64 {
26 if v == 0 { dst[off]=48 as u8; return off+1 }
27 let tmp: *u8=sys_mmap(28); var m: i64=v; var k: i64=0
28 while m>0 { tmp[k]=(48+(m%10)) as u8; m=m/10; k=k+1 }
29 var i: i64=0; while i<k { dst[off+i]=tmp[k-1-i]; i=i+1 }
30 return off+k
31}
32
33// parse a docmap (one path per line) IN PLACE: \n -> NUL, record line starts. Returns ndocs.
34func rs_load_docmap(path: *u8, paths_out: *i64, cap: i64) -> i64 {
35 let lenbox: *i64 = sys_mmap(16) as *i64
36 let buf: *u8 = sys_read_file(path, lenbox)
37 if buf == 0 as *u8 { return 0 - 1 }
38 let total: i64 = lenbox[0]
39 var nd: i64 = 0; var start: i64 = 0; var i: i64 = 0
40 while i < total {
41 if buf[i] == (10 as u8) {
42 buf[i] = 0 as u8
43 if i > start { if nd < cap { paths_out[nd] = (buf as i64) + start; nd = nd + 1 } }
44 start = i + 1
45 }
46 i = i + 1
47 }
48 return nd
49}
50
51func main(argc: i64, argv: *i64) -> i64 {
52 if argc < 2 {
53 rs_puts("usage: nx_research_search <term> [term ...]\n" as *u8)
54 sys_exit(2); return 2
55 }
56 var nterms: i64 = argc - 1
57 if nterms > RS_TERMS { nterms = RS_TERMS }
58 rs_puts("=== nx_research_search: cross-shard query over the self-growing researcher index ===\n" as *u8)
59
60 // global candidate accumulator (score = #query-terms matched; pathptr into a live docmap buffer)
61 let cand_path: *i64 = sys_mmap(8*RS_CAND) as *i64
62 let cand_score: *i64 = sys_mmap(8*RS_CAND) as *i64
63 let cand_shard: *i64 = sys_mmap(8*RS_CAND) as *i64
64 var ncand: i64 = 0
65
66 let res: *NxInvQueryResult = sys_mmap(64) as *NxInvQueryResult
67 let rowids: *i64 = sys_mmap(8*RS_ROWIDS) as *i64
68 var shard: i64 = 0
69 var shards_loaded: i64 = 0
70 var total_docs: i64 = 0
71 var stop: i64 = 0
72 while stop == 0 {
73 if shard >= RS_MAXSHARD { stop = 1 }
74 else {
75 let ip: *u8 = sys_mmap(256); var ic: i64 = rs_cat(ip,0,"knowledge/index/shards/research_" as *u8); ic=rs_catnum(ip,ic,shard); ic=rs_cat(ip,ic,".idx" as *u8); ip[ic]=0 as u8
76 let idx: *NxInvIndex = nx_inv_load(ip)
77 if idx == 0 as *NxInvIndex { stop = 1 }
78 else {
79 let dp: *u8 = sys_mmap(256); var dc: i64 = rs_cat(dp,0,"knowledge/index/shards/research_" as *u8); dc=rs_catnum(dp,dc,shard); dc=rs_cat(dp,dc,".docmap" as *u8); dp[dc]=0 as u8
80 let paths: *i64 = sys_mmap(8*RS_ROWIDS) as *i64
81 let ndocs: i64 = rs_load_docmap(dp, paths, RS_ROWIDS)
82 if ndocs > 0 {
83 shards_loaded = shards_loaded + 1
84 total_docs = total_docs + ndocs
85 // per-doc term-hit counter for this shard
86 let hits: *u8 = sys_mmap(ndocs + 8)
87 var t: i64 = 0
88 while t < nterms {
89 let term: *u8 = argv[1+t] as *u8
90 nx_inv_query_term(idx, term, rs_strlen(term), rowids, RS_ROWIDS, res)
91 var ri: i64 = 0
92 while ri < res.n_rowids_filled {
93 let rid: i64 = rowids[ri]
94 if rid >= 0 { if rid < ndocs { hits[rid] = (hits[rid] as i64 + 1) as u8 } }
95 ri = ri + 1
96 }
97 t = t + 1
98 }
99 var di: i64 = 0
100 while di < ndocs {
101 let h: i64 = hits[di] as i64
102 if h > 0 { if ncand < RS_CAND { cand_path[ncand]=paths[di]; cand_score[ncand]=h; cand_shard[ncand]=shard; ncand=ncand+1 } }
103 di = di + 1
104 }
105 }
106 shard = shard + 1
107 }
108 }
109 }
110
111 rs_puts(" shards="); rs_num(shards_loaded); rs_puts(" docs="); rs_num(total_docs)
112 rs_puts(" candidates="); rs_num(ncand); rs_puts("\n" as *u8)
113 if ncand == 0 {
114 rs_puts(" no document across the grown index matches the query\n" as *u8)
115 sys_exit(0); return 0
116 }
117
118 // selection-sort candidates desc by score (terms-matched); top-K
119 var a: i64 = 0
120 while a < ncand {
121 var best: i64 = a; var b: i64 = a+1
122 while b < ncand { if cand_score[b] > cand_score[best] { best=b } b=b+1 }
123 if best != a {
124 let ts: i64=cand_score[a]; cand_score[a]=cand_score[best]; cand_score[best]=ts
125 let tp: i64=cand_path[a]; cand_path[a]=cand_path[best]; cand_path[best]=tp
126 let th: i64=cand_shard[a]; cand_shard[a]=cand_shard[best]; cand_shard[best]=th
127 }
128 a=a+1
129 }
130 var topk: i64 = ncand
131 if topk > RS_TOPK { topk = RS_TOPK }
132 rs_puts(" TOP RESULTS (score = #query-terms matched):\n" as *u8)
133 var r: i64 = 0
134 while r < topk {
135 rs_puts(" #"); rs_num(r+1); rs_puts(" score="); rs_num(cand_score[r])
136 rs_puts(" shard="); rs_num(cand_shard[r]); rs_puts(" "); rs_puts(cand_path[r] as *u8); rs_puts("\n" as *u8)
137 r = r + 1
138 }
139 rs_puts(" SEARCH-OK\n" as *u8)
140 sys_exit(0); return 0
141}