nx_corpus_measure.nx source

1// nx_corpus_measure.nx -- data-driven MEASURED index over a REAL crawled corpus (the honest "bigger index" step at 2// laptop scale; the NAS multiplies it). Reads a manifest of natively-crawled pages (nishi_fetch, off-WSL), indexes all 3// via nx_search_inverted (bounded footprint), MEASURES the real index (docs / unique-terms / postings bytes), and runs 4// a RANKED multi-term query -> top-k docs by term-match score. Add a manifest line = a bigger index, zero rebuild. 5// license_tier: ORIGINAL 6import "nx_search_inverted.nx" 7const CM_MAGIC_131072: i64 = 131072 8const CM_MAGIC_8388608: i64 = 8388608 9 10const CM_MAXDOC: i64 = 512 11 12func gw(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 13func gn(v: i64) -> i64 { if v==0 { sys_write(1,"0" as *u8,1); return 0 } var m: i64=v; if m<0 { sys_write(1,"-" as *u8,1); m=0-m } let t: *u8=sys_mmap(24); var k: i64=0; while m>0 { t[k]=(48+(m%10)) as u8; m=m/10; k=k+1 } let o: *u8=sys_mmap(24); var w: i64=0; var q: i64=k-1; while q>=0 { o[w]=t[q]; w=w+1; q=q-1 } sys_write(1,o,w); return 0 } 14func cm_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n } 15 16// term in doc? (write_cursor = valid distinct-doc postings, dedup-aware) 17func cm_has(idx: *NxInvIndex, term: *u8, tn: i64, docid: i64) -> i64 { 18 let slot: *u8 = nx_inv_lookup_slot(idx, nx_inv_hash_bytes_lower(term, tn)) 19 if (slot as i64)==0 { return 0 } 20 let wc: i64 = nx_inv_slot_write_cursor(slot); let off: i64 = nx_inv_slot_postings_offset(slot) 21 var k: i64=0 22 while k<wc { let p: *u8=((idx.postings_ptr as i64)+off+k*4) as *u8; let id: i64=(p[0] as i64)|((p[1] as i64)<<8)|((p[2] as i64)<<16)|((p[3] as i64)<<24); if id==docid { return 1 } k=k+1 } 23 return 0 24} 25 26func main() -> i64 { 27 gw("=== nx_corpus_measure: MEASURED index over a REAL natively-crawled corpus ===\n" as *u8) 28 let lb: *i64 = sys_mmap(8) as *i64 29 let man: *u8 = sys_read_file("knowledge/fetched/corpus_manifest.txt" as *u8, lb) 30 if man == 0 as *u8 { gw("no manifest\n" as *u8); sys_exit(1); return 1 } 31 let mn: i64 = lb[0] 32 33 // parse manifest -> paths[] (NUL-terminate each line in place) 34 let paths: *i64 = sys_mmap(8*CM_MAXDOC) as *i64 35 var ndoc: i64 = 0 36 var i: i64 = 0 37 while i < mn { 38 if man[i]==(10 as u8) { i=i+1 } else { 39 let st: i64=i 40 var go: i64=1 41 while go==1 { if i<mn { if man[i]==(10 as u8) { go=0 } else { i=i+1 } } else { go=0 } } 42 man[i]=0 as u8 43 if ndoc < CM_MAXDOC { paths[ndoc]=(man as i64)+st; ndoc=ndoc+1 } 44 i=i+1 45 } 46 } 47 48 // read each doc 49 let bufs: *i64 = sys_mmap(8*CM_MAXDOC) as *i64 50 let lens: *i64 = sys_mmap(8*CM_MAXDOC) as *i64 51 let lbx: *i64 = sys_mmap(8) as *i64 52 i=0 53 while i<ndoc { let b: *u8=sys_read_file(paths[i] as *u8, lbx); bufs[i]=b as i64; lens[i]=lbx[0]; i=i+1 } 54 55 // bounded index (8MB postings pool -- ample for a laptop corpus, HAL-portable footprint) 56 let idx: *NxInvIndex = nx_inv_new_bounded(CM_MAGIC_131072, CM_MAGIC_8388608) 57 i=0; while i<ndoc { nx_inv_index_row(idx, bufs[i] as *u8, lens[i], i); i=i+1 } 58 nx_inv_finalize_offsets(idx) 59 i=0; while i<ndoc { nx_inv_emit_row(idx, bufs[i] as *u8, lens[i], i); i=i+1 } 60 61 var total_bytes: i64=0; i=0; while i<ndoc { total_bytes=total_bytes+lens[i]; i=i+1 } 62 gw("\n INDEX MEASURED: docs=" as *u8); gn(ndoc) 63 gw(" corpus_bytes=" as *u8); gn(total_bytes) 64 gw(" unique_terms=" as *u8); gn(idx.vocab_occupied) 65 gw(" postings_bytes=" as *u8); gn(idx.postings_used) 66 gw(" (bounded pool 8MB -> HAL-portable)\n" as *u8) 67 68 // RANKED query: score doc = # query terms present; report top-3. 69 let qt: *i64 = sys_mmap(8*8) as *i64; let ql: *i64 = sys_mmap(8*8) as *i64 70 qt[0]="bm25" as *u8 as i64; ql[0]=4 71 qt[1]="ranking" as *u8 as i64; ql[1]=7 72 qt[2]="search" as *u8 as i64; ql[2]=6 73 let nq: i64=3 74 gw("\n RANKED query 'bm25 ranking search' (score = #terms matched):\n" as *u8) 75 // compute scores 76 let score: *i64 = sys_mmap(8*CM_MAXDOC) as *i64 77 i=0; while i<ndoc { score[i]=0; var q2: i64=0; while q2<nq { if cm_has(idx, qt[q2] as *u8, ql[q2], i)==1 { score[i]=score[i]+1 } q2=q2+1 } i=i+1 } 78 // print top-3 by repeated max-find 79 var rank: i64=0 80 while rank<3 { 81 var best: i64=0-1; var bs: i64=0-1 82 i=0; while i<ndoc { if score[i]>bs { bs=score[i]; best=i } i=i+1 } 83 if best<0 { rank=3 } else { 84 gw(" #" as *u8); gn(rank+1); gw(" score=" as *u8); gn(bs); gw(" " as *u8); gw(paths[best] as *u8); gw("\n" as *u8) 85 score[best]=0-1 86 rank=rank+1 87 } 88 } 89 90 gw("\n=== CORPUS-MEASURE done: real " as *u8); gn(ndoc); gw("-doc index built + ranked (native crawl -> our index) ===\n" as *u8) 91 sys_exit(0); return 0 92}