nx_web_crawl_breadth.nx source
↩ module page · 188 lines · 9388 B
1// nx_web_crawl_breadth.nx -- BREADTH crawl: start from ONE seed page, EXTRACT its links, and fetch the linked
2// pages too (bounded) -> the external engine reaches the web by FOLLOWING links, not just a fixed seedlist.
3// Sovereign TLS-1.3 fetch (no browser). Politeness/safety BY CONSTRUCTION: same-domain article links only
4// (href="/wiki/<Title>", namespaced titles with ':' skipped), de-duplicated, hard MAX_PAGES cap (no runaway).
5// Emits one `id<TAB>text` line per page to knowledge/index/web_breadth_corpus.tsv for nx_onsite_index.
6// expect_exit: 0 license_tier: ORIGINAL (fetch machinery from nx_web_crawl_index; + link extractor + frontier)
7import "nx_syscalls.nx"
8import "nx_x509_trust_store.nx"
9import "nx_trust_store_load_from_certdata.nx"
10import "nx_tls13_client_validate_certificate.nx"
11import "nx_tls13_client_session_run.nx"
12import "nx_https_url_for_fetch.nx"
13import "nx_https_url_connect.nx"
14import "nx_https_get.nx"
15import "nx_https_get_complete.nx"
16import "nx_http_response_parse.nx"
17const CB_MAGIC_4194304: i64 = 4194304
18const CB_MAGIC_65536: i64 = 65536
19
20const CB_TEXTCAP: i64 = 65536
21const CB_MAX_PAGES: i64 = 6 // hard cap (seed + up to 5 discovered) -- no runaway
22const CB_HOST: *u8 = "en.wikipedia.org" as *u8
23const CB_PREFIX: *u8 = "https://en.wikipedia.org/wiki/" as *u8 // 30 chars
24const CB_SEED: *u8 = "https://en.wikipedia.org/wiki/Information_retrieval\x00"
25const CB_SEED_ID: *u8 = "Information_retrieval" as *u8
26const CB_OUT: *u8 = "knowledge/index/web_breadth_corpus.tsv" as *u8
27
28func cb_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
29func cb_num(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=(48 as u8);k=1}; while m>0{t[k]=((48+(m%10)) as u8);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(1,bb,k); return 0 }
30func cb_strlen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
31
32func cb_path_off(url: *u8, n: i64) -> i64 {
33 var hoststart: i64=0; var found: i64=0; var j: i64=0
34 while j<n {
35 if found==0 { if url[j]==(58 as u8) { if j+2<n { if url[j+1]==(47 as u8) { if url[j+2]==(47 as u8) { hoststart=j+3; found=1 } } } } }
36 j=j+1
37 }
38 var k: i64=hoststart
39 while k<n { if url[k]==(47 as u8) { return k } k=k+1 }
40 return n
41}
42
43// fetch full_url over sovereign TLS into a fresh 4MB buf; returns bytes (gc) via out_len, buf ptr via out_buf. status via return.
44func cb_fetch(store: *TrustStore, full_url: *u8, out_buf: *i64, out_len: *i64) -> i64 {
45 out_buf[0]=0; out_len[0]=0
46 let ul: i64=cb_strlen(full_url)
47 let poff: i64=cb_path_off(full_url, ul)
48 let path: *u8=((full_url as i64)+poff) as *u8
49 let plen: i64=ul-poff
50 let cr: *u8=sys_mmap(32); var i: i64=0; while i<32 { cr[i]=(0xC0+i) as u8; i=i+1 }
51 let priv: *u8=sys_mmap(32); i=0; while i<32 { priv[i]=(0xA0+i) as u8; i=i+1 }
52 let url_p: *NxUrl=nx_url_new()
53 let target_raw: *u8=sys_mmap(32)
54 let target: *NxHttpsTarget=target_raw as *NxHttpsTarget
55 target.url=url_p; target.port=0
56 if nx_https_url_for_fetch(full_url, target)!=NX_HTTPS_URL_OK { return 0-41 }
57 let fd_p: *i64=sys_mmap(16) as *i64
58 if nx_https_url_connect(target, full_url, sys_now_realtime_sec(), fd_p)!=NX_HTTPS_CONNECT_OK { return 0-42 }
59 let fd: i64=*fd_p
60 let val_ctx_raw: *u8=sys_mmap(64)
61 let val_ctx: *TlsValidationContext=val_ctx_raw as *TlsValidationContext
62 val_ctx.store=store
63 val_ctx.sni_host=full_url + target.url.host_off
64 val_ctx.sni_host_len=target.url.host_len
65 val_ctx.now_epoch=sys_now_realtime_sec()
66 let sr: i64=nx_tls13_client_session_run(fd, full_url + target.url.host_off, target.url.host_len, cr, priv, val_ctx)
67 if sr<=0 { sys_close(fd); return 0-(200+(0-sr)) }
68 let session: *Tls13ClientSession=sr as *Tls13ClientSession
69 let buf: *u8=sys_mmap(CB_MAGIC_4194304)
70 let gc: i64=nx_https_get_complete(session, fd, path, plen, full_url + target.url.host_off, target.url.host_len, buf, CB_MAGIC_4194304)
71 sys_close(fd)
72 if gc<0 { return 0-(100+(0-gc)) }
73 let rs: *i64=sys_mmap(128) as *i64
74 nx_http_response_parse(buf, gc, rs)
75 out_buf[0]=buf as i64; out_len[0]=gc
76 return rs[1]
77}
78
79// normalize buf[0..n) -> emit "id<TAB>text\n"
80func cb_emit(ofd: i64, id: *u8, buf: *u8, n: i64, tbuf: *u8) -> i64 {
81 sys_write(ofd, id, cb_strlen(id)); sys_write(ofd, "\t" as *u8, 1)
82 var o: i64=0; var i: i64=0
83 while i<n {
84 if o>=CB_TEXTCAP { i=n } else {
85 var c: i64=buf[i] as i64
86 if c==9 { c=32 } if c==10 { c=32 } if c==13 { c=32 }
87 tbuf[o]=c as u8; o=o+1; i=i+1
88 }
89 }
90 sys_write(ofd, tbuf, o); sys_write(ofd, "\n" as *u8, 1)
91 return 0
92}
93
94// extract up to cap unique article titles from href="/wiki/<Title>" (skip ':' namespaces, '#' anchors); store
95// NUL-terminated in tbuf, offsets in toff. returns count.
96func cb_links(html: *u8, n: i64, tbuf: *u8, toff: *i64, cap: i64, tbufcap: i64) -> i64 {
97 let pat: *u8="href=\"/wiki/" as *u8
98 let pl: i64=12
99 var count: i64=0; var used: i64=0; var i: i64=0
100 while i+pl<=n {
101 var m: i64=1; var j: i64=0
102 while j<pl { if html[i+j]!=pat[j] { m=0; j=pl } else { j=j+1 } }
103 if m==0 { i=i+1 } else {
104 var s: i64=i+pl; var e: i64=s; var colon: i64=0; var go: i64=1
105 while go==1 {
106 if e>=n { go=0 } else {
107 let c: i64=html[e] as i64
108 if c==34 { go=0 } else { if c==35 { go=0 } else { if c==60 { go=0 } else { if c==58 { colon=1 } e=e+1 } } }
109 }
110 }
111 let tl: i64=e-s
112 var keep: i64=0
113 if tl>0 { if tl<160 { if colon==0 { keep=1 } } }
114 if keep==1 {
115 // dedup against stored titles
116 var dup: i64=0; var q: i64=0
117 while q<count {
118 let qo: i64=toff[q]; let ql: i64=cb_strlen(((tbuf as i64)+qo) as *u8)
119 if ql==tl { var b: i64=0; var same: i64=1; while b<tl { if tbuf[qo+b]!=html[s+b] { same=0; b=tl } else { b=b+1 } } if same==1 { dup=1 } }
120 q=q+1
121 }
122 if dup==0 { if count<cap { if used+tl+1<=tbufcap {
123 var b: i64=0; while b<tl { tbuf[used+b]=html[s+b]; b=b+1 }
124 tbuf[used+tl]=0 as u8; toff[count]=used; used=used+tl+1; count=count+1
125 } } }
126 }
127 i=e
128 }
129 }
130 return count
131}
132
133func main() -> i64 {
134 cb_puts("=== BREADTH CRAWL (1 seed -> follow its article links -> bounded fetch -> offsite corpus) ===\n" as *u8)
135 let r: i64=nx_trust_store_load_from_certdata("/tmp/mozilla_certdata.txt\x00" as *u8, 512, CB_MAGIC_4194304)
136 if r<=0 { cb_puts("BREADTH: certdata load failed (cp data/mozilla_certdata.txt /tmp/)\n"); return 1 }
137 let store: *TrustStore=r as *TrustStore
138 cb_puts("CA="); cb_num(trust_store_count(store)); cb_puts("\n")
139
140 let ofd: i64=sys_openat_wr(CB_OUT, 0x1a4)
141 if ofd<0 { cb_puts("BREADTH: cannot open output\n"); return 4 }
142 let tbuf: *u8=sys_mmap(CB_TEXTCAP+16)
143
144 // fetch the seed (keep its buf to extract links)
145 let sbuf: *i64=sys_mmap(16) as *i64
146 let slen: *i64=sys_mmap(16) as *i64
147 let st: i64=cb_fetch(store, CB_SEED, sbuf, slen)
148 if st!=200 { cb_puts("BREADTH: seed fetch failed st="); cb_num(st); cb_puts("\n"); sys_close(ofd); return 5 }
149 cb_emit(ofd, CB_SEED_ID, sbuf[0] as *u8, slen[0], tbuf)
150 cb_puts(" seed "); cb_puts(CB_SEED_ID); cb_puts(" ("); cb_num(slen[0]); cb_puts(" bytes)\n")
151
152 // extract links from the seed
153 let titles: *u8=sys_mmap(CB_MAGIC_65536)
154 let toff: *i64=sys_mmap(8*512) as *i64
155 let nlinks: i64=cb_links(sbuf[0] as *u8, slen[0], titles, toff, 400, CB_MAGIC_65536)
156 cb_puts(" links discovered on seed: "); cb_num(nlinks); cb_puts("\n")
157
158 // fetch up to CB_MAX_PAGES-1 of the discovered links
159 let pfx: *u8 = "https://en.wikipedia.org/wiki/" as *u8 // local literal (module-const aliased to code; bug)
160 let prefix_n: i64=cb_strlen(pfx)
161 var fetched: i64=1 // seed counts as 1
162 var attempts: i64=0
163 var li: i64=0
164 while li<nlinks {
165 var stop: i64=0
166 if fetched>=CB_MAX_PAGES { stop=1 }
167 if attempts>=12 { stop=1 } // debug/safety cap so an all-fail run can't try all 225
168 if stop==1 { li=nlinks } else {
169 let title: *u8=((titles as i64)+toff[li]) as *u8
170 let tn: i64=cb_strlen(title)
171 let url: *u8=sys_mmap(512)
172 var a: i64=0; while a<prefix_n { url[a]=pfx[a]; a=a+1 }
173 var b: i64=0; while b<tn { url[prefix_n+b]=title[b]; b=b+1 }
174 url[prefix_n+tn]=0 as u8
175 let lb: *i64=sys_mmap(16) as *i64
176 let ll: *i64=sys_mmap(16) as *i64
177 let s2: i64=cb_fetch(store, url, lb, ll)
178 cb_puts(" try ["); cb_puts(url); cb_puts("] st="); cb_num(s2); cb_puts("\n")
179 attempts=attempts+1
180 if s2==200 { cb_emit(ofd, title, lb[0] as *u8, ll[0], tbuf); fetched=fetched+1 }
181 li=li+1
182 }
183 }
184 sys_close(ofd)
185 cb_puts("BREADTH-CRAWL-OK pages="); cb_num(fetched); cb_puts(" -> "); cb_puts(CB_OUT); cb_puts("\n")
186 if fetched<2 { sys_exit(1); return 1 }
187 sys_exit(0); return 0
188}