nx_corpus_search.nx source
↩ module page · 76 lines · 4899 B
1// nx_corpus_search.nx -- END-TO-END on REAL data: index 3 REAL web pages crawled natively by nishi_fetch.exe (off-WSL)
2// and SEARCH them with our own inverted index. Proves the engine works on real crawled web content, not fixtures:
3// native crawl (nishi_fetch, Windows PE) -> our index (nx_search_inverted) -> query -> retrieve. Reads valid postings
4// via write_cursor (dedup-aware: distinct docs per term). Docs: 0=Web_crawler 1=Inverted_index 2=PageRank.
5// license_tier: ORIGINAL
6import "nx_search_inverted.nx"
7const K_MAGIC_1048576: i64 = 1048576
8const K_MAGIC_131072: i64 = 131072
9
10func gw(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
11func gn(v: i64) -> i64 { if v==0 { sys_write(1,"0" as *u8,1); return 0 } var m: i64=v; if m<0 { sys_write(1,"-" as *u8,1); m=0-m } let t: *u8=sys_mmap(24); var k: i64=0; while m>0 { t[k]=(48+(m%10)) as u8; m=m/10; k=k+1 } let o: *u8=sys_mmap(24); var w: i64=0; var q: i64=k-1; while q>=0 { o[w]=t[q]; w=w+1; q=q-1 } sys_write(1,o,w); return 0 }
12func cs_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
13
14// does term's posting list contain docid? (reads write_cursor valid postings, not the occurrence count)
15func cs_has(idx: *NxInvIndex, term: *u8, docid: i64) -> i64 {
16 let h: i64 = nx_inv_hash_bytes_lower(term, cs_slen(term))
17 let slot: *u8 = nx_inv_lookup_slot(idx, h)
18 if (slot as i64)==0 { return 0 }
19 let wc: i64 = nx_inv_slot_write_cursor(slot)
20 let off: i64 = nx_inv_slot_postings_offset(slot)
21 var k: i64=0
22 while k<wc { let p: *u8=((idx.postings_ptr as i64)+off+k*4) as *u8; let id: i64=(p[0] as i64)|((p[1] as i64)<<8)|((p[2] as i64)<<16)|((p[3] as i64)<<24); if id==docid { return 1 } k=k+1 }
23 return 0
24}
25func cs_ndocs(idx: *NxInvIndex, term: *u8) -> i64 {
26 let h: i64 = nx_inv_hash_bytes_lower(term, cs_slen(term))
27 let slot: *u8 = nx_inv_lookup_slot(idx, h)
28 if (slot as i64)==0 { return 0 }
29 return nx_inv_slot_write_cursor(slot)
30}
31// print the docs (by label) that contain term
32func cs_report(idx: *NxInvIndex, term: *u8) -> i64 {
33 gw(" query '" as *u8); gw(term); gw("' -> " as *u8); gn(cs_ndocs(idx, term)); gw(" doc(s): " as *u8)
34 if cs_has(idx, term, 0)==1 { gw("[Web_crawler] " as *u8) }
35 if cs_has(idx, term, 1)==1 { gw("[Inverted_index] " as *u8) }
36 if cs_has(idx, term, 2)==1 { gw("[PageRank] " as *u8) }
37 gw("\n" as *u8)
38 return 0
39}
40
41func main() -> i64 {
42 gw("=== nx_corpus_search: index + search REAL natively-crawled web pages ===\n" as *u8)
43 let cap: i64 = K_MAGIC_1048576
44 let lb: *i64 = sys_mmap(8) as *i64
45 let b0: *u8 = sys_read_file("knowledge/fetched/crawl_web_crawler.txt" as *u8, lb); let n0: i64 = lb[0]
46 let b1: *u8 = sys_read_file("knowledge/fetched/crawl_inverted_index.txt" as *u8, lb); let n1: i64 = lb[0]
47 let b2: *u8 = sys_read_file("knowledge/fetched/crawl_pagerank.txt" as *u8, lb); let n2: i64 = lb[0]
48 var pass: i64=0; var tot: i64=0
49 tot=tot+1; if n0>0 { if n1>0 { if n2>0 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) } } else { gw(" [FAIL] " as *u8) } } else { gw(" [FAIL] " as *u8) }
50 gw("T1 read 3 real crawled docs (bytes " as *u8); gn(n0); gw("/" as *u8); gn(n1); gw("/" as *u8); gn(n2); gw(")\n" as *u8)
51
52 let idx: *NxInvIndex = nx_inv_new(K_MAGIC_131072)
53 nx_inv_index_row(idx, b0, n0, 0); nx_inv_index_row(idx, b1, n1, 1); nx_inv_index_row(idx, b2, n2, 2)
54 nx_inv_finalize_offsets(idx)
55 nx_inv_emit_row(idx, b0, n0, 0); nx_inv_emit_row(idx, b1, n1, 1); nx_inv_emit_row(idx, b2, n2, 2)
56
57 gw("\n -- live queries over the real index --\n" as *u8)
58 cs_report(idx, "crawler" as *u8)
59 cs_report(idx, "postings" as *u8)
60 cs_report(idx, "damping" as *u8)
61 cs_report(idx, "wikipedia" as *u8)
62 cs_report(idx, "zzznotintheindex" as *u8)
63
64 tot=tot+1; if cs_has(idx, "crawler" as *u8, 0)==1 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) }
65 gw("T2 'crawler' retrieves Web_crawler\n" as *u8)
66 tot=tot+1; if cs_has(idx, "damping" as *u8, 2)==1 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) }
67 gw("T3 'damping' retrieves PageRank (damping factor)\n" as *u8)
68 tot=tot+1; if cs_has(idx, "postings" as *u8, 1)==1 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) }
69 gw("T4 'postings' retrieves Inverted_index\n" as *u8)
70 tot=tot+1; if cs_ndocs(idx, "zzznotintheindex" as *u8)==0 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) }
71 gw("T5 absent term -> 0 docs\n" as *u8)
72
73 gw("\n=== nx_corpus_search " as *u8); gn(pass); gw("/" as *u8); gn(tot); gw(" ===\n" as *u8)
74 if pass==tot { gw("CORPUS-SEARCH GREEN -- REAL web pages crawled natively + indexed + searched by OUR engine, end-to-end\n" as *u8); sys_exit(0); return 0 }
75 gw("CORPUS-SEARCH RED\n" as *u8); sys_exit(1); return 1
76}