nx_external_corpus.nx source

1// nx_external_corpus.nx -- EXTERNAL (offsite) search corpus builder: turns the team's SOVEREIGN-FETCHED web 2// pages (knowledge/fetched/*.raw -- real pages pulled over the Nishi TLS-1.3 stack this session, NOT a browser) 3// into the same `id<TAB>text` source the reusable ingestor (nx_onsite_index) consumes. This is the OFFSITE half 4// of the charter's scope=onsite|offsite|both: ONE engine, two scopes. id = the real Wikipedia title (so the 5// result URL https://en.wikipedia.org/wiki/<id> is CORRECT + clickable), text = the page bytes with tabs/ 6// newlines normalized to spaces (HTML tags become corpus-wide low-IDF tokens BM25 down-weights, so content 7// terms still discriminate). Bounded reads (no 4GB sys_read_file reservation). license_tier: ORIGINAL 8import "nx_syscalls.nx" 9 10const EC_READ: i64 = 524288 // read up to 512KB of each fetched page 11const EC_TEXTCAP: i64 = 65536 // emit up to 64KB of normalized text per doc 12const EC_OUT: *u8 = "knowledge/index/web_src.tsv" as *u8 13 14func ec_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 } 15func ec_num(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=(48 as u8);k=1}; while m>0{t[k]=((48+(m%10)) as u8);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(1,bb,k); return 0 } 16func ec_strlen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n } 17 18// bounded read of `path` into buf (cap). returns bytes read (0 if missing). 19func ec_read_file(path: *u8, buf: *u8, cap: i64) -> i64 { 20 let fd: i64=sys_openat_rd(path) 21 if fd<0 { return 0 } 22 var total: i64=0 23 var nrd: i64=sys_read(fd, buf, cap) 24 while nrd>0 { total=total+nrd; if total>=cap { nrd=0 } else { nrd=sys_read(fd, ((buf as i64)+total) as *u8, cap-total) } } 25 sys_close(fd) 26 return total 27} 28 29// read one fetched page, normalize whitespace, emit "id<TAB>text\n" to ofd. returns 1 if emitted. 30func ec_emit(ofd: i64, id: *u8, path: *u8, rbuf: *u8, tbuf: *u8) -> i64 { 31 let n: i64=ec_read_file(path, rbuf, EC_READ) 32 if n<=0 { ec_puts(" MISS "); ec_puts(path); ec_puts("\n"); return 0 } 33 var o: i64=0; var i: i64=0 34 while i<n { 35 if o>=EC_TEXTCAP { i=n } else { 36 var c: i64=rbuf[i] as i64 37 if c==9 { c=32 } if c==10 { c=32 } if c==13 { c=32 } 38 tbuf[o]=c as u8; o=o+1; i=i+1 39 } 40 } 41 sys_write(ofd, id, ec_strlen(id)) 42 sys_write(ofd, "\t" as *u8, 1) 43 sys_write(ofd, tbuf, o) 44 sys_write(ofd, "\n" as *u8, 1) 45 ec_puts(" + "); ec_puts(id); ec_puts(" ("); ec_num(o); ec_puts(" bytes)\n") 46 return 1 47} 48 49func main() -> i64 { 50 ec_puts("=== EXTERNAL CORPUS BUILD (sovereign-fetched web pages -> id<TAB>text for the reusable engine) ===\n" as *u8) 51 let ofd: i64=sys_openat_wr(EC_OUT, 0x1a4) 52 if ofd<0 { ec_puts("EXT-CORPUS FAIL: cannot open output\n" as *u8); sys_exit(1); return 1 } 53 let rbuf: *u8=sys_mmap(EC_READ+16) 54 let tbuf: *u8=sys_mmap(EC_TEXTCAP+16) 55 var n: i64=0 56 // id = real Wikipedia title (URL https://en.wikipedia.org/wiki/<id> is correct); path = the sovereign fetch 57 n=n+ec_emit(ofd, "Apache_Lucene" as *u8, "knowledge/fetched/srch_lucene.raw" as *u8, rbuf, tbuf) 58 n=n+ec_emit(ofd, "Apache_Solr" as *u8, "knowledge/fetched/srch_solr.raw" as *u8, rbuf, tbuf) 59 n=n+ec_emit(ofd, "Elasticsearch" as *u8, "knowledge/fetched/srch_elastic.raw" as *u8, rbuf, tbuf) 60 n=n+ec_emit(ofd, "Learning_to_rank" as *u8, "knowledge/fetched/srch_ltr.raw" as *u8, rbuf, tbuf) 61 n=n+ec_emit(ofd, "Full-text_search" as *u8, "knowledge/fetched/srch_fulltext.raw" as *u8, rbuf, tbuf) 62 n=n+ec_emit(ofd, "Semantic_search" as *u8, "knowledge/fetched/sem_search.raw" as *u8, rbuf, tbuf) 63 n=n+ec_emit(ofd, "Vector_database" as *u8, "knowledge/fetched/sem_vector.raw" as *u8, rbuf, tbuf) 64 n=n+ec_emit(ofd, "Retrieval-augmented_generation" as *u8,"knowledge/fetched/sem_rag.raw" as *u8, rbuf, tbuf) 65 n=n+ec_emit(ofd, "Word_embedding" as *u8, "knowledge/fetched/sem_embed.raw" as *u8, rbuf, tbuf) 66 n=n+ec_emit(ofd, "Okapi_BM25" as *u8, "knowledge/fetched/lib_bm25.raw" as *u8, rbuf, tbuf) 67 n=n+ec_emit(ofd, "Faceted_search" as *u8, "knowledge/fetched/lib_facet.raw" as *u8, rbuf, tbuf) 68 sys_close(ofd) 69 ec_puts("EXT-CORPUS-OK pages="); ec_num(n); ec_puts(" -> "); ec_puts(EC_OUT); ec_puts("\n" as *u8) 70 if n<1 { sys_exit(1); return 1 } 71 sys_exit(0); return 0 72}