nx_search_cli.nx source

1// nx_search_cli.nx -- ASSIGNMENT E (tutored): the search engine's FRONT DOOR over the DURABLE index. The 2// harvest (nx_library_harvest_v2) proved build->persist; until now nothing user-facing QUERIED that artifact -- 3// the index was provably correct and practically dark. This is the live path: argv terms -> load 4// knowledge/index/library.idx (NO rebuild: build-once-query-forever) -> inverted-index postings shortlist the 5// candidate docids -> docmap names their files -> the Engineer-wired bm_* BM25 ranker (x1e6 fixed-point, the 6// _hdl_build nx_bm25.nx that shadows runtime's on this lane) scores them -> "score path" per hit. Distinct 7// from nx_library_search.nx (the Researcher's IN-MEMORY corpus ranker; no main, no 8// persistence) -- one capability per module. BM25 stats (avgdl/IDF) are computed over the CANDIDATE set; 9// corpus-wide stats stored at harvest = the next rung when the corpus outgrows reading candidates whole. 10// Usage: nx_search_cli.sov.elf <term> [term ...] license_tier: ORIGINAL 11import "nx_search_inverted_persist.nx" 12import "nx_itoa_lib.nx" // shared MSB-first emitter (zero-alloc) 13import "nx_bm25.nx" 14const LS_MAGIC_1000000: i64 = 1000000 15const LS_MAGIC_10000: i64 = 10000 16 17const LS_MAX_DOCS: i64 = 4096 18const LS_MAX_TERMS: i64 = 8 19const LS_ROWID_CAP: i64 = 1024 20const LS_TOP_K: i64 = 10 21 22func sc_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 23// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer 24// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the 25// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls). 26// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign. 27func sc_num(v: i64) -> i64 { nxi_out(v); return 0 } 28func sc_strlen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n } 29 30// fold A-Z -> a-z in place. The INDEX hashes lowercased tokens but the bm_* tf/df scorer is a case- 31// sensitive substring count -- so cased occurrences ("PagedAttention", "Tantivy") scored 0 and ranking 32// broke into 0.00 ties (caught by the quality gate's pagedattention row). Folding BOTH the query terms 33// and the candidate doc text at the boundary makes scoring agree with the index's own semantics. 34func sc_fold(s: *u8, n: i64) -> i64 { 35 var i: i64 = 0 36 while i < n { 37 let c: i64 = s[i] as i64 38 if c >= 65 { if c <= 90 { s[i] = (c + 32) as u8 } } 39 i = i + 1 40 } 41 return 0 42} 43 44// print an x1e6 fixed-point score as [-]int.dd 45func sc_score(v: i64) -> i64 { 46 var m: i64 = v 47 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m } 48 sc_num(m / LS_MAGIC_1000000) 49 sys_write(1, "." as *u8, 1) 50 let frac: i64 = (m % LS_MAGIC_1000000) / LS_MAGIC_10000 51 if frac < 10 { sys_write(1, "0" as *u8, 1) } 52 sc_num(frac) 53 return 0 54} 55 56// parse the docmap (one path per line, line i = docid i) IN PLACE: \n -> NUL, record line starts. 57// Returns ndocs. 58func sc_load_docmap(path: *u8, paths_out: *i64, cap: i64) -> i64 { 59 let lenbox: *i64 = sys_mmap(16) as *i64 60 let buf: *u8 = sys_read_file(path, lenbox) 61 if buf == 0 as *u8 { return 0 - 1 } 62 let total: i64 = lenbox[0] 63 var nd: i64 = 0 64 var start: i64 = 0 65 var i: i64 = 0 66 while i < total { 67 if buf[i] == (10 as u8) { 68 buf[i] = 0 as u8 69 if i > start { if nd < cap { paths_out[nd] = (buf as i64) + start; nd = nd + 1 } } 70 start = i + 1 71 } 72 i = i + 1 73 } 74 return nd 75} 76 77func main(argc: i64, argv: *i64) -> i64 { 78 if argc < 2 { 79 sc_puts("usage: nx_search_cli <term> [term ...] (queries knowledge/index/library.idx)\n" as *u8) 80 sys_exit(2); return 2 81 } 82 var nterms: i64 = argc - 1 83 if nterms > LS_MAX_TERMS { nterms = LS_MAX_TERMS } 84 85 // ---- load the DURABLE artifacts (no rebuild: this is the build-once-query-forever path) ---- 86 let idx: *NxInvIndex = nx_inv_load("knowledge/index/library.idx" as *u8) 87 if idx == 0 as *NxInvIndex { sc_puts("SEARCH-FAIL: no index (run nx_library_harvest_v2 first)\n" as *u8); sys_exit(1); return 1 } 88 let paths: *i64 = sys_mmap(8 * LS_MAX_DOCS) as *i64 89 let ndocs: i64 = sc_load_docmap("knowledge/index/library.docmap" as *u8, paths, LS_MAX_DOCS) 90 if ndocs <= 0 { sc_puts("SEARCH-FAIL: no docmap\n" as *u8); sys_exit(1); return 1 } 91 if idx.n_rows != ndocs { sc_puts("SEARCH-FAIL: index rows != docmap lines (stale pair -- re-harvest)\n" as *u8); sys_exit(1); return 1 } 92 93 // ---- candidate docids = union of each term's postings (deduped; index shortlists, BM25 ranks) ---- 94 let seen: *u8 = sys_mmap(ndocs + 8) 95 let res: *NxInvQueryResult = sys_mmap(64) as *NxInvQueryResult 96 let rowids: *i64 = sys_mmap(8 * LS_ROWID_CAP) as *i64 97 let qterms: *i64 = sys_mmap(8 * LS_MAX_TERMS) as *i64 98 let qlens: *i64 = sys_mmap(8 * LS_MAX_TERMS) as *i64 99 var t: i64 = 0 100 while t < nterms { 101 let term: *u8 = argv[1 + t] as *u8 102 qterms[t] = term as i64 103 qlens[t] = sc_strlen(term) 104 sc_fold(term, qlens[t]) 105 nx_inv_query_term(idx, term, qlens[t], rowids, LS_ROWID_CAP, res) 106 sc_puts("term '" as *u8); sc_puts(term); sc_puts("' postings=" as *u8); sc_num(res.postings_count); sc_puts("\n" as *u8) 107 var ri: i64 = 0 108 while ri < res.n_rowids_filled { 109 let rid: i64 = rowids[ri] 110 if rid >= 0 { if rid < ndocs { seen[rid] = 1 as u8 } } 111 ri = ri + 1 112 } 113 t = t + 1 114 } 115 let cand: *i64 = sys_mmap(8 * ndocs) as *i64 116 var ncand: i64 = 0 117 var di: i64 = 0 118 while di < ndocs { 119 if seen[di] == (1 as u8) { cand[ncand] = di; ncand = ncand + 1 } 120 di = di + 1 121 } 122 if ncand == 0 { 123 sc_puts("SEARCH-OK matches=0 (no document contains any query term)\n" as *u8) 124 sys_exit(0); return 0 125 } 126 127 // ---- read candidate docs whole + BM25-rank them (the Engineer-wired bm_* ranker, x1e6 scale) ---- 128 let texts: *i64 = sys_mmap(8 * ncand) as *i64 129 let tlens: *i64 = sys_mmap(8 * ncand) as *i64 130 let dls: *i64 = sys_mmap(8 * ncand) as *i64 131 let lenbox: *i64 = sys_mmap(16) as *i64 132 var total_dl: i64 = 0 133 var ci: i64 = 0 134 while ci < ncand { 135 let buf: *u8 = sys_read_file(paths[cand[ci]] as *u8, lenbox) 136 if buf == 0 as *u8 { texts[ci] = ("" as *u8) as i64; tlens[ci] = 0 } 137 else { texts[ci] = buf as i64; tlens[ci] = lenbox[0]; sc_fold(buf, lenbox[0]) } 138 dls[ci] = bm_token_count(texts[ci] as *u8, tlens[ci]) 139 total_dl = total_dl + dls[ci] 140 ci = ci + 1 141 } 142 var avgdl: i64 = total_dl / ncand 143 if avgdl <= 0 { avgdl = 1 } 144 let scores: *i64 = sys_mmap(8 * ncand) as *i64 145 ci = 0 146 while ci < ncand { 147 scores[ci] = bm_score(texts, tlens, dls, ncand, ci, avgdl, qterms, nterms) 148 ci = ci + 1 149 } 150 151 // ---- selection-sort desc on (scores, cand) in lockstep; print top K ---- 152 var a: i64 = 0 153 while a < ncand { 154 var best: i64 = a 155 var b: i64 = a + 1 156 while b < ncand { 157 if scores[b] > scores[best] { best = b } 158 b = b + 1 159 } 160 if best != a { 161 let ts: i64 = scores[a]; scores[a] = scores[best]; scores[best] = ts 162 let tc: i64 = cand[a]; cand[a] = cand[best]; cand[best] = tc 163 } 164 a = a + 1 165 } 166 var topk: i64 = ncand 167 if topk > LS_TOP_K { topk = LS_TOP_K } 168 sc_puts("SEARCH-RESULTS matches=" as *u8); sc_num(ncand) 169 sc_puts(" of docs=" as *u8); sc_num(ndocs); sc_puts("\n" as *u8) 170 var r: i64 = 0 171 while r < topk { 172 sc_puts(" #" as *u8); sc_num(r + 1) 173 sc_puts(" score=" as *u8); sc_score(scores[r]) 174 sc_puts(" " as *u8); sc_puts(paths[cand[r]] as *u8); sc_puts("\n" as *u8) 175 r = r + 1 176 } 177 sc_puts("SEARCH-OK\n" as *u8) 178 sys_exit(0); return 0 179}