nx_book_search_lib.nx source

1// nx_book_search_lib.nx -- shared SC20 library-search CORE (NO main). Wired into nx_media_server /api/search. 2// Multi-term, case-insensitive, BM25-ranked TOP-N over a newline-delimited index file (idx_book.wsl). 3// Reuses nx_bm25 (bm_idf_micro/bm_sat_milli/bm_token_count/bm_df) + nx_research_extract (re_count). Same proven 4// algorithm as the gated nx_book_search, extended to multi-term + top-N. license_tier: ORIGINAL 5import "nx_research_extract.nx" // re_count 6import "nx_bm25.nx" // bm_token_count / bm_idf_micro / bm_sat_milli / bm_df 7import "nx_syscalls.nx" 8const K_MAGIC_300000: i64 = 300000 9const K_MAGIC_1024: i64 = 1024 10 11func bks_lc(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c } 12 13// Search `idxpath` for `query`; fill res_off/res_len/res_score with the top `maxn` results (score-desc), 14// set cnt[0] = number found. RETURNS the original index buffer pointer (as i64) so the caller can emit the 15// path bytes at buf+res_off[i] for res_len[i] bytes. 16func bks_search(idxpath: *u8, query: *u8, res_off: *i64, res_len: *i64, res_score: *i64, maxn: i64, cnt: *i64) -> i64 { 17 cnt[0] = 0 18 let lenbox: *i64 = sys_mmap(8) as *i64 19 let buf: *u8 = sys_read_file(idxpath, lenbox) 20 let n: i64 = lenbox[0] 21 if n <= 0 { return buf as i64 } 22 23 // lowercased mirror for case-insensitive ranking 24 let lc: *u8 = sys_mmap(n + 16) 25 var i: i64 = 0 26 while i < n { lc[i] = bks_lc(buf[i] as i64) as u8; i = i + 1 } 27 28 // split into docs (one line = one item). CAP sized for the largest index (idx_ai ~228k). 29 let CAP: i64 = K_MAGIC_300000 30 let offs: *i64 = sys_mmap(CAP * 8) as *i64 31 let lens: *i64 = sys_mmap(CAP * 8) as *i64 32 let dls: *i64 = sys_mmap(CAP * 8) as *i64 33 let ptrs: *i64 = sys_mmap(CAP * 8) as *i64 34 var ndocs: i64 = 0; var total: i64 = 0; var s: i64 = 0 35 while s < n { 36 var e: i64 = s; var f: i64 = 0 37 while f == 0 { if e >= n { f = 1 } else { if lc[e] == (10 as u8) { f = 1 } else { e = e + 1 } } } 38 let llen: i64 = e - s 39 if llen > 0 { if ndocs < CAP { 40 offs[ndocs] = s; lens[ndocs] = llen; ptrs[ndocs] = (lc as i64) + s 41 dls[ndocs] = bm_token_count((lc as i64 + s) as *u8, llen); total = total + dls[ndocs]; ndocs = ndocs + 1 42 } } 43 s = e + 1 44 } 45 var avgdl: i64 = 1 46 if ndocs > 0 { avgdl = total / ndocs } 47 if avgdl <= 0 { avgdl = 1 } 48 49 // lowercase the query into qcopy and tokenize on spaces (NUL-separated) -> qterms[] 50 let qcopy: *u8 = sys_mmap(K_MAGIC_1024) 51 var ql: i64 = 0 52 while query[ql] != (0 as u8) { if ql < 1022 { qcopy[ql] = bks_lc(query[ql] as i64) as u8 } ql = ql + 1 } 53 qcopy[ql] = 0 as u8 54 let QMAX: i64 = 16 55 let qterms: *i64 = sys_mmap(QMAX * 8) as *i64 56 var nq: i64 = 0 57 var qi: i64 = 0 58 while qi < ql { 59 if qcopy[qi] == (32 as u8) { qi = qi + 1 } 60 else { 61 if nq < QMAX { qterms[nq] = (qcopy as i64) + qi; nq = nq + 1 } 62 var atend: i64 = 0 63 while atend == 0 { 64 if qi >= ql { atend = 1 } 65 else { if qcopy[qi] == (32 as u8) { qcopy[qi] = 0 as u8; qi = qi + 1; atend = 1 } else { qi = qi + 1 } } 66 } 67 } 68 } 69 if nq == 0 { return buf as i64 } 70 71 // precompute idf per term (once) 72 let idf: *i64 = sys_mmap(QMAX * 8) as *i64 73 var t: i64 = 0 74 while t < nq { idf[t] = bm_idf_micro(ndocs, bm_df(ptrs, lens, ndocs, qterms[t] as *u8)); t = t + 1 } 75 76 // score every doc; maintain top-`maxn` in res_* (score-descending) via insertion 77 var k: i64 = 0 78 while k < ndocs { 79 var sc: i64 = 0 80 var tt: i64 = 0 81 while tt < nq { 82 let term: *u8 = qterms[tt] as *u8 83 let tf: i64 = re_count(ptrs[k] as *u8, lens[k], term) 84 if tf > 0 { sc = sc + (idf[tt] * bm_sat_milli(tf, dls[k], avgdl)) / 1000 } 85 tt = tt + 1 86 } 87 if sc > 0 { 88 if cnt[0] < maxn { 89 let w: i64 = cnt[0] 90 res_off[w] = offs[k]; res_len[w] = lens[k]; res_score[w] = sc; cnt[0] = w + 1 91 var b: i64 = w 92 while b > 0 { 93 if res_score[b] > res_score[b-1] { 94 let ao: i64 = res_off[b]; res_off[b] = res_off[b-1]; res_off[b-1] = ao 95 let al: i64 = res_len[b]; res_len[b] = res_len[b-1]; res_len[b-1] = al 96 let asc: i64 = res_score[b]; res_score[b] = res_score[b-1]; res_score[b-1] = asc 97 b = b - 1 98 } else { b = 0 } 99 } 100 } else { 101 if sc > res_score[maxn-1] { 102 res_off[maxn-1] = offs[k]; res_len[maxn-1] = lens[k]; res_score[maxn-1] = sc 103 var b: i64 = maxn - 1 104 while b > 0 { 105 if res_score[b] > res_score[b-1] { 106 let ao: i64 = res_off[b]; res_off[b] = res_off[b-1]; res_off[b-1] = ao 107 let al: i64 = res_len[b]; res_len[b] = res_len[b-1]; res_len[b-1] = al 108 let asc: i64 = res_score[b]; res_score[b] = res_score[b-1]; res_score[b-1] = asc 109 b = b - 1 110 } else { b = 0 } 111 } 112 } 113 } 114 } 115 k = k + 1 116 } 117 return buf as i64 118}