nx_web_crawl_index.nx source
↩ module page · 136 lines · 6577 B
1// nx_web_crawl_index.nx -- LIVE external crawl: fetch FRESH URLs on demand over the sovereign TLS-1.3 stack
2// (no browser/curl), normalize each page to one `id<TAB>text` line, and emit the offsite corpus the reusable
3// engine indexes. DATA-DRIVEN: the URLs come from a seedlist (knowledge/index/web_seedlist.tsv, lines
4// "id<TAB>full_url") -- point it at any URLs and it crawls + builds the searchable corpus. This is the LIVE
5// half of external search (vs nx_external_corpus which re-uses already-fetched .raw). Pipeline after this:
6// nx_onsite_index web_seed_corpus.tsv https://en.wikipedia.org/wiki/ webcrawl.idx webcrawl.manifest
7// expect_exit: 0 license_tier: ORIGINAL (fetch machinery cloned from nx_onsite_research_fetch)
8import "nx_syscalls.nx"
9import "nx_itoa_lib.nx" // shared MSB-first emitter (zero-alloc)
10import "nx_x509_trust_store.nx"
11import "nx_trust_store_load_from_certdata.nx"
12import "nx_tls13_client_validate_certificate.nx"
13import "nx_tls13_client_session_run.nx"
14import "nx_https_url_for_fetch.nx"
15import "nx_https_url_connect.nx"
16import "nx_https_get.nx"
17import "nx_https_get_complete.nx"
18import "nx_http_response_parse.nx"
19const WC_MAGIC_4194304: i64 = 4194304
20
21const WC_TEXTCAP: i64 = 65536
22const WC_SEED: *u8 = "knowledge/index/web_seedlist.tsv" as *u8
23const WC_OUT: *u8 = "knowledge/index/web_seed_corpus.tsv" as *u8
24
25func wc_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
26// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
27// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
28// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
29// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
30func wc_num(v: i64) -> i64 { nxi_out(v); return 0 }
31func wc_strlen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
32
33// offset of the path ('/' after the host) within a full URL
34func wc_path_off(url: *u8, n: i64) -> i64 {
35 var hoststart: i64=0; var found: i64=0; var j: i64=0
36 while j<n {
37 if found==0 { if url[j]==(58 as u8) { if j+2<n { if url[j+1]==(47 as u8) { if url[j+2]==(47 as u8) { hoststart=j+3; found=1 } } } } }
38 j=j+1
39 }
40 var k: i64=hoststart
41 while k<n { if url[k]==(47 as u8) { return k } k=k+1 }
42 return n
43}
44
45// fetch full_url over sovereign TLS; normalize the whole response to text; emit "id<TAB>text\n". returns HTTP status.
46func wc_fetch_emit(store: *TrustStore, full_url: *u8, id: *u8, ofd: i64, tbuf: *u8) -> i64 {
47 let ul: i64=wc_strlen(full_url)
48 let poff: i64=wc_path_off(full_url, ul)
49 let path: *u8=((full_url as i64)+poff) as *u8
50 let plen: i64=ul-poff
51 let cr: *u8=sys_mmap(32); var i: i64=0; while i<32 { cr[i]=(0xC0+i) as u8; i=i+1 }
52 let priv: *u8=sys_mmap(32); i=0; while i<32 { priv[i]=(0xA0+i) as u8; i=i+1 }
53 let url_p: *NxUrl=nx_url_new()
54 let target_raw: *u8=sys_mmap(32)
55 let target: *NxHttpsTarget=target_raw as *NxHttpsTarget
56 target.url=url_p; target.port=0
57 if nx_https_url_for_fetch(full_url, target)!=NX_HTTPS_URL_OK { return 0-41 }
58 let fd_p: *i64=sys_mmap(16) as *i64
59 if nx_https_url_connect(target, full_url, sys_now_realtime_sec(), fd_p)!=NX_HTTPS_CONNECT_OK { return 0-42 }
60 let fd: i64=*fd_p
61 let val_ctx_raw: *u8=sys_mmap(64)
62 let val_ctx: *TlsValidationContext=val_ctx_raw as *TlsValidationContext
63 val_ctx.store=store
64 val_ctx.sni_host=full_url + target.url.host_off
65 val_ctx.sni_host_len=target.url.host_len
66 val_ctx.now_epoch=sys_now_realtime_sec()
67 let sr: i64=nx_tls13_client_session_run(fd, full_url + target.url.host_off, target.url.host_len, cr, priv, val_ctx)
68 if sr<=0 { sys_close(fd); return 0-(200+(0-sr)) }
69 let session: *Tls13ClientSession=sr as *Tls13ClientSession
70 let buf: *u8=sys_mmap(WC_MAGIC_4194304)
71 let gc: i64=nx_https_get_complete(session, fd, path, plen, full_url + target.url.host_off, target.url.host_len, buf, WC_MAGIC_4194304)
72 sys_close(fd)
73 if gc<0 { return 0-(100+(0-gc)) }
74 let rs: *i64=sys_mmap(128) as *i64
75 nx_http_response_parse(buf, gc, rs)
76 let status: i64=rs[1]
77 // emit id<TAB> + normalized whole response (tabs/newlines -> spaces, capped) + \n
78 sys_write(ofd, id, wc_strlen(id))
79 sys_write(ofd, "\t" as *u8, 1)
80 var o: i64=0; i=0
81 while i<gc {
82 if o>=WC_TEXTCAP { i=gc } else {
83 var c: i64=buf[i] as i64
84 if c==9 { c=32 } if c==10 { c=32 } if c==13 { c=32 }
85 tbuf[o]=c as u8; o=o+1; i=i+1
86 }
87 }
88 sys_write(ofd, tbuf, o)
89 sys_write(ofd, "\n" as *u8, 1)
90 wc_puts(" ST="); wc_num(status); wc_puts(" GC="); wc_num(gc); wc_puts(" -> "); wc_puts(id); wc_puts("\n")
91 return status
92}
93
94func main() -> i64 {
95 wc_puts("=== LIVE WEB CRAWL+INDEX (sovereign fetch of seedlist URLs -> id<TAB>text offsite corpus) ===\n" as *u8)
96 let cpath: *u8 = "/tmp/mozilla_certdata.txt\x00"
97 let r: i64=nx_trust_store_load_from_certdata(cpath, 512, WC_MAGIC_4194304)
98 if r<=0 { wc_puts("CRAWL: certdata load failed (cp data/mozilla_certdata.txt /tmp/)\n"); return 1 }
99 let store: *TrustStore=r as *TrustStore
100 wc_puts("CA="); wc_num(trust_store_count(store)); wc_puts("\n")
101
102 let szp: *i64=sys_mmap(16) as *i64
103 let seed: *u8=sys_read_file(WC_SEED, szp)
104 if seed==0 as *u8 { wc_puts("CRAWL: no seedlist\n"); return 3 }
105 let sz: i64=szp[0]
106 let ofd: i64=sys_openat_wr(WC_OUT, 0x1a4)
107 if ofd<0 { wc_puts("CRAWL: cannot open output\n"); return 4 }
108 let tbuf: *u8=sys_mmap(WC_TEXTCAP+16)
109
110 var ok: i64=0
111 var ls: i64=0; var i: i64=0
112 while i<=sz {
113 var eol: i64=0
114 if i==sz { eol=1 } else { if seed[i]==(10 as u8) { eol=1 } }
115 if eol==1 {
116 if i>ls {
117 var tab: i64=0-1; var k: i64=ls
118 while k<i { if seed[k]==(9 as u8) { if tab<0 { tab=k } } k=k+1 }
119 if tab>ls {
120 seed[tab]=0 as u8
121 if i<sz { seed[i]=0 as u8 }
122 let id: *u8=((seed as i64)+ls) as *u8
123 let url: *u8=((seed as i64)+tab+1) as *u8
124 let st: i64=wc_fetch_emit(store, url, id, ofd, tbuf)
125 if st==200 { ok=ok+1 }
126 }
127 }
128 ls=i+1
129 }
130 i=i+1
131 }
132 sys_close(ofd)
133 wc_puts("CRAWL-OK pages_200="); wc_num(ok); wc_puts(" -> "); wc_puts(WC_OUT); wc_puts("\n")
134 if ok<1 { sys_exit(1); return 1 }
135 sys_exit(0); return 0
136}