nx_book_search.nx source

1// nx_book_search.nx -- SC20: sovereign RANKED search over the NAS book library (idx_book.wsl, 5761 books). 2// 3// REUSE not reinvent: composes the team's production BM25 ranker (nx_bm25: bm_idf_micro / bm_sat_milli / 4// bm_token_count / bm_df) + nx_research_extract (re_count/re_has). The existing nx_library_search is wired for 5// the researcher's [doc]-index corpus and caps at 256 docs; this drives the SAME BM25 math over the 5761-book 6// library index. TWO fixes for scale + real-world data: 7// (1) case-insensitive: the index paths are mixed-case, so we rank over a LOWERCASED copy of the index. 8// (2) O(N*nq) not O(N^2): bm_best recomputes df per-doc (fine for 256 docs, not 5761); we PRECOMPUTE idf 9// once per query term, then score each doc -> a true library-scale ranker. 10// Self-gated by KATs over the REAL index (dragonlance/byers/tolkien/witcher must rank a doc that contains the 11// term; "sanderson" has 0 matches -> must return NO result = negative control; dragonlance != byers = discrim). 12// Standalone (no server/reader edit -> no collision with the ark-beat loop). license_tier: ORIGINAL 13import "nx_research_extract.nx" // re_count / re_has 14import "nx_itoa_lib.nx" // shared MSB-first emitter (zero-alloc) 15import "nx_bm25.nx" // bm_token_count / bm_idf_micro / bm_sat_milli / bm_df 16import "nx_syscalls.nx" 17const BS_MAGIC_2000000: i64 = 2000000 18 19const BS_IDX: *u8 = "knowledge/staging/media/idx_book.wsl" 20const BS_LOG: *u8 = "knowledge/status/book_search.log" 21const BS_CAPDOCS: i64 = 8192 // > 5761 22 23func bs_p(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 24// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer 25// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the 26// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls). 27// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign. 28func bs_n(v: i64) -> i64 { nxi_out(v); return 0 } 29func bs_fp(fd: i64, s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(fd,s,n); return 0 } 30// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer 31// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the 32// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls). 33// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign. 34func bs_fn(fd: i64, v: i64) -> i64 { nxi_fd(fd, v); return 0 } 35func bs_lc(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c } 36 37func bs_read(path: *u8, buf: *u8, cap: i64) -> i64 { 38 let fd: i64 = sys_openat_rd(path) 39 if fd < 0 { return 0 - 1 } 40 var n: i64 = 0; var go: i64 = 1 41 while go == 1 { let r: i64 = sys_read(fd, (buf as i64 + n) as *u8, cap - 1 - n); if r <= 0 { go = 0 } else { n = n + r } if n >= cap - 1 { go = 0 } } 42 sys_close(fd) 43 return n 44} 45 46// ranked best doc for a single lowercased term, idf precomputed ONCE (library-scale O(N)). 47func bs_best1(ptrs: *i64, lens: *i64, dls: *i64, ndocs: i64, avgdl: i64, term: *u8) -> i64 { 48 let idf: i64 = bm_idf_micro(ndocs, bm_df(ptrs, lens, ndocs, term)) // x1e6, once 49 var best: i64 = 0 - 1; var bestscore: i64 = 0 50 var k: i64 = 0 51 while k < ndocs { 52 let tf: i64 = re_count(ptrs[k] as *u8, lens[k], term) 53 if tf > 0 { 54 let sc: i64 = (idf * bm_sat_milli(tf, dls[k], avgdl)) / 1000 55 if sc > bestscore { bestscore = sc; best = k } 56 } 57 k = k + 1 58 } 59 return best 60} 61 62func main() -> i64 { 63 bs_p("=== nx_book_search: SC20 ranked library search over idx_book.wsl (sovereign BM25) ===\n" as *u8) 64 65 let buf: *u8 = sys_mmap(BS_MAGIC_2000000) 66 let n: i64 = bs_read(BS_IDX, buf, BS_MAGIC_2000000) 67 if n <= 0 { bs_p("BOOK-SEARCH verdict=RED reason=idx-missing\n" as *u8); sys_exit(1); return 1 } 68 69 // lowercased mirror of the index (1:1 byte layout) for case-insensitive ranking 70 let lc: *u8 = sys_mmap(BS_MAGIC_2000000) 71 var i: i64 = 0 72 while i < n { lc[i] = bs_lc(buf[i] as i64) as u8; i = i + 1 } 73 74 // split into docs: one line = one book. ptrs into lc (matching), offs to recover original (display). 75 let ptrs: *i64 = sys_mmap(BS_CAPDOCS * 8) as *i64 76 let offs: *i64 = sys_mmap(BS_CAPDOCS * 8) as *i64 77 let lens: *i64 = sys_mmap(BS_CAPDOCS * 8) as *i64 78 let dls: *i64 = sys_mmap(BS_CAPDOCS * 8) as *i64 79 var ndocs: i64 = 0 80 var total: i64 = 0 81 var s: i64 = 0 82 while s < n { 83 var e: i64 = s 84 var found: i64 = 0 85 while found == 0 { if e >= n { found = 1 } else { if lc[e] == (10 as u8) { found = 1 } else { e = e + 1 } } } 86 let llen: i64 = e - s 87 if llen > 0 { if ndocs < BS_CAPDOCS { 88 offs[ndocs] = s 89 ptrs[ndocs] = (lc as i64) + s 90 lens[ndocs] = llen 91 dls[ndocs] = bm_token_count((lc as i64 + s) as *u8, llen) 92 total = total + dls[ndocs] 93 ndocs = ndocs + 1 94 } } 95 s = e + 1 96 } 97 var avgdl: i64 = 1 98 if ndocs > 0 { avgdl = total / ndocs } 99 if avgdl <= 0 { avgdl = 1 } 100 bs_p(" indexed docs=" as *u8); bs_n(ndocs); bs_p(" avgdl=" as *u8); bs_n(avgdl); bs_p("\n" as *u8) 101 102 // ---- KAT queries over the REAL corpus ---- 103 let bd: i64 = bs_best1(ptrs, lens, dls, ndocs, avgdl, "dragonlance" as *u8) 104 let bb: i64 = bs_best1(ptrs, lens, dls, ndocs, avgdl, "byers" as *u8) 105 let bt: i64 = bs_best1(ptrs, lens, dls, ndocs, avgdl, "tolkien" as *u8) 106 let bw: i64 = bs_best1(ptrs, lens, dls, ndocs, avgdl, "witcher" as *u8) 107 let bs: i64 = bs_best1(ptrs, lens, dls, ndocs, avgdl, "sanderson" as *u8) // 0 matches -> expect -1 108 109 // show top hit for each positive query (original-case path) 110 bs_p(" q=dragonlance -> " as *u8); if bd >= 0 { sys_write(1, (buf as i64 + offs[bd]) as *u8, lens[bd]) } else { bs_p("(none)" as *u8) } bs_p("\n" as *u8) 111 bs_p(" q=byers -> " as *u8); if bb >= 0 { sys_write(1, (buf as i64 + offs[bb]) as *u8, lens[bb]) } else { bs_p("(none)" as *u8) } bs_p("\n" as *u8) 112 bs_p(" q=tolkien -> " as *u8); if bt >= 0 { sys_write(1, (buf as i64 + offs[bt]) as *u8, lens[bt]) } else { bs_p("(none)" as *u8) } bs_p("\n" as *u8) 113 bs_p(" q=witcher -> " as *u8); if bw >= 0 { sys_write(1, (buf as i64 + offs[bw]) as *u8, lens[bw]) } else { bs_p("(none)" as *u8) } bs_p("\n" as *u8) 114 bs_p(" q=sanderson -> " as *u8); if bs >= 0 { sys_write(1, (buf as i64 + offs[bs]) as *u8, lens[bs]) } else { bs_p("(none = correct, 0 in corpus)" as *u8) } bs_p("\n" as *u8) 115 116 // ---- gate: each positive must rank a doc that CONTAINS the term; negative must be -1; discrim ---- 117 var pass: i64 = 0 118 if bd >= 0 { if re_has((lc as i64 + offs[bd]) as *u8, lens[bd], "dragonlance" as *u8) == 1 { pass = pass + 1 } } 119 if bb >= 0 { if re_has((lc as i64 + offs[bb]) as *u8, lens[bb], "byers" as *u8) == 1 { pass = pass + 1 } } 120 if bt >= 0 { if re_has((lc as i64 + offs[bt]) as *u8, lens[bt], "tolkien" as *u8) == 1 { pass = pass + 1 } } 121 if bw >= 0 { if re_has((lc as i64 + offs[bw]) as *u8, lens[bw], "witcher" as *u8) == 1 { pass = pass + 1 } } 122 var neg_ok: i64 = 0 123 if bs < 0 { neg_ok = 1; pass = pass + 1 } 124 var discrim: i64 = 0 125 if bd >= 0 { if bb >= 0 { if bd != bb { discrim = 1; pass = pass + 1 } } } 126 127 bs_p(" gate: pass=" as *u8); bs_n(pass); bs_p("/6 (4 positive-rank + 1 negative-control + 1 discrimination)\n" as *u8) 128 129 let lf: i64 = sys_openat_append(BS_LOG, 0x1a4) 130 if pass == 6 { 131 if lf >= 0 { bs_fp(lf, "BOOK-SEARCH verdict=GREEN docs=" as *u8); bs_fn(lf, ndocs); bs_fp(lf, " avgdl=" as *u8); bs_fn(lf, avgdl); bs_fp(lf, " pass=6/6 neg_ok=" as *u8); bs_fn(lf, neg_ok); bs_fp(lf, " discrim=" as *u8); bs_fn(lf, discrim); bs_fp(lf, " epoch=" as *u8); bs_fn(lf, sys_now_realtime_sec()); bs_fp(lf, "\n" as *u8); sys_close(lf) } 132 bs_p("BOOK-SEARCH verdict=GREEN (sovereign BM25 ranked search works over the real 5761-book library; SC20 engine)\n" as *u8) 133 sys_exit(0); return 0 134 } 135 if lf >= 0 { bs_fp(lf, "BOOK-SEARCH verdict=RED pass=" as *u8); bs_fn(lf, pass); bs_fp(lf, "/6\n" as *u8); sys_close(lf) } 136 bs_p("BOOK-SEARCH verdict=RED (a KAT failed)\n" as *u8) 137 sys_exit(1) 138 return 1 139}