nx_search_federated.nx source

1// nx_search_federated.nx -- FEDERATED QUERY across every Nishi index, driven by the search-sources MANIFEST. 2// This is the integration CONSUMER: it reads knowledge/index/search_sources.txt (the contract written by 3// nx_search_register) and queries EVERY listed index -- the ecosystem library index AND the researcher's growing 4// shards AND any future index -- through one interface. It has NO hardcoded index paths and NO knowledge of 5// "researcher" vs "library": it just queries whatever the manifest lists. Adding a new index source needs ZERO 6// change here = loose coupling. Merges candidates across sources, ranks by #query-terms-matched, prints top-K 7// `score [label] path`. Usage: nx_search_federated <term> [term ...] expect_exit: 0 license_tier: ORIGINAL 8import "nx_search_inverted_persist.nx" 9 10const FS_TERMS: i64 = 8 11const FS_ROWIDS: i64 = 4096 12const FS_CAND: i64 = 32768 13const FS_TOPK: i64 = 15 14const FS_SRC: i64 = 8192 15 16func fs_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 17func fs_num(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)} let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1} sys_write(1,bb,k); return 0 } 18func fs_strlen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n } 19 20// scan to next space/newline/end 21func fs_scan(buf: *u8, i: i64, end: i64) -> i64 { 22 var p: i64=i; var stop: i64=0 23 while stop==0 { if p>=end { stop=1 } else { if buf[p]==(32 as u8) { stop=1 } else { if buf[p]==(10 as u8) { stop=1 } else { p=p+1 } } } } 24 return p 25} 26 27// parse a docmap (one path/line) IN PLACE: \n -> NUL, record line starts. Returns ndocs. 28func fs_load_docmap(path: *u8, paths_out: *i64, cap: i64) -> i64 { 29 let lenbox: *i64 = sys_mmap(16) as *i64 30 let buf: *u8 = sys_read_file(path, lenbox) 31 if buf == 0 as *u8 { return 0 - 1 } 32 let total: i64 = lenbox[0] 33 var nd: i64 = 0; var start: i64 = 0; var i: i64 = 0 34 while i < total { 35 if buf[i] == (10 as u8) { 36 buf[i] = 0 as u8 37 if i > start { if nd < cap { paths_out[nd] = (buf as i64) + start; nd = nd + 1 } } 38 start = i + 1 39 } 40 i = i + 1 41 } 42 return nd 43} 44 45func main(argc: i64, argv: *i64) -> i64 { 46 if argc < 2 { fs_puts("usage: nx_search_federated <term> [term ...]\n" as *u8); sys_exit(2); return 2 } 47 var nterms: i64 = argc - 1 48 if nterms > FS_TERMS { nterms = FS_TERMS } 49 fs_puts("=== nx_search_federated: query ALL Nishi indexes via the search-sources manifest ===\n" as *u8) 50 51 // ---- read + parse the manifest (the integration contract); NUL-terminate fields IN PLACE ---- 52 let mbox: *i64 = sys_mmap(16) as *i64 53 let man: *u8 = sys_read_file("knowledge/index/search_sources.txt" as *u8, mbox) 54 if man == 0 as *u8 { fs_puts(" NO manifest (run nx_search_register first)\n" as *u8); sys_exit(1); return 1 } 55 let mlen: i64 = mbox[0] 56 let s_label: *i64 = sys_mmap(8*FS_SRC) as *i64 57 let s_idx: *i64 = sys_mmap(8*FS_SRC) as *i64 58 let s_dmap: *i64 = sys_mmap(8*FS_SRC) as *i64 59 var nsrc: i64 = 0 60 var i: i64 = 0 61 while i < mlen { 62 let ls: i64 = i 63 let e1: i64 = fs_scan(man, i, mlen) // label 64 let label_ptr: i64 = (man as i64) + ls 65 var j: i64 = e1; if j<mlen { if man[j]==(32 as u8) { man[j]=0 as u8; j=j+1 } } 66 let idx_ptr: i64 = (man as i64) + j 67 let e2: i64 = fs_scan(man, j, mlen) // idx path 68 var k: i64 = e2; if k<mlen { if man[k]==(32 as u8) { man[k]=0 as u8; k=k+1 } } 69 let dmap_ptr: i64 = (man as i64) + k 70 let e3: i64 = fs_scan(man, k, mlen) // docmap path 71 if e3<mlen { if man[e3]==(32 as u8) { man[e3]=0 as u8 } } 72 man[e1]=0 as u8 // terminate label (safe even if it was space) 73 // advance to end of line 74 var e: i64 = e3; var st: i64=0 75 while st==0 { if e>=mlen { st=1 } else { if man[e]==(10 as u8) { st=1 } else { e=e+1 } } } 76 if e<mlen { man[e]=0 as u8; e=e+1 } 77 if nsrc < FS_SRC { s_label[nsrc]=label_ptr; s_idx[nsrc]=idx_ptr; s_dmap[nsrc]=dmap_ptr; nsrc=nsrc+1 } 78 i = e 79 } 80 81 // ---- query every source; accumulate candidates (score = #query-terms matched) ---- 82 let cand_path: *i64 = sys_mmap(8*FS_CAND) as *i64 83 let cand_score: *i64 = sys_mmap(8*FS_CAND) as *i64 84 let cand_label: *i64 = sys_mmap(8*FS_CAND) as *i64 85 var ncand: i64 = 0 86 let res: *NxInvQueryResult = sys_mmap(64) as *NxInvQueryResult 87 let rowids: *i64 = sys_mmap(8*FS_ROWIDS) as *i64 88 var sources_ok: i64 = 0; var total_docs: i64 = 0 89 var si: i64 = 0 90 while si < nsrc { 91 let idx: *NxInvIndex = nx_inv_load(s_idx[si] as *u8) 92 if idx != 0 as *NxInvIndex { 93 let paths: *i64 = sys_mmap(8*FS_ROWIDS) as *i64 94 let ndocs: i64 = fs_load_docmap(s_dmap[si] as *u8, paths, FS_ROWIDS) 95 if ndocs > 0 { 96 sources_ok = sources_ok + 1 97 total_docs = total_docs + ndocs 98 let hits: *u8 = sys_mmap(ndocs + 8) 99 var t: i64 = 0 100 while t < nterms { 101 let term: *u8 = argv[1+t] as *u8 102 nx_inv_query_term(idx, term, fs_strlen(term), rowids, FS_ROWIDS, res) 103 var ri: i64 = 0 104 while ri < res.n_rowids_filled { 105 let rid: i64 = rowids[ri] 106 if rid >= 0 { if rid < ndocs { hits[rid] = (hits[rid] as i64 + 1) as u8 } } 107 ri = ri + 1 108 } 109 t = t + 1 110 } 111 var di: i64 = 0 112 while di < ndocs { 113 let h: i64 = hits[di] as i64 114 if h > 0 { if ncand < FS_CAND { cand_path[ncand]=paths[di]; cand_score[ncand]=h; cand_label[ncand]=s_label[si]; ncand=ncand+1 } } 115 di = di + 1 116 } 117 } 118 } 119 si = si + 1 120 } 121 122 fs_puts(" sources="); fs_num(sources_ok); fs_puts("/"); fs_num(nsrc) 123 fs_puts(" docs="); fs_num(total_docs); fs_puts(" candidates="); fs_num(ncand); fs_puts("\n" as *u8) 124 if ncand == 0 { fs_puts(" no match across the federated index\n" as *u8); sys_exit(0); return 0 } 125 126 // ---- rank desc by score, top-K ---- 127 var a: i64 = 0 128 while a < ncand { 129 var best: i64 = a; var b: i64 = a+1 130 while b < ncand { if cand_score[b] > cand_score[best] { best=b } b=b+1 } 131 if best != a { 132 let ts: i64=cand_score[a]; cand_score[a]=cand_score[best]; cand_score[best]=ts 133 let tp: i64=cand_path[a]; cand_path[a]=cand_path[best]; cand_path[best]=tp 134 let tl: i64=cand_label[a]; cand_label[a]=cand_label[best]; cand_label[best]=tl 135 } 136 a=a+1 137 } 138 var topk: i64 = ncand 139 if topk > FS_TOPK { topk = FS_TOPK } 140 fs_puts(" TOP RESULTS across all indexes (score = #query-terms matched):\n" as *u8) 141 var r: i64 = 0 142 while r < topk { 143 fs_puts(" #"); fs_num(r+1); fs_puts(" score="); fs_num(cand_score[r]) 144 fs_puts(" ["); fs_puts(cand_label[r] as *u8); fs_puts("] "); fs_puts(cand_path[r] as *u8); fs_puts("\n" as *u8) 145 r = r + 1 146 } 147 fs_puts(" FEDERATED-OK\n" as *u8) 148 sys_exit(0); return 0 149}