nx_commoncrawl_probe.nx source
↩ module page · 36 lines · 2444 B
1// nx_commoncrawl_probe.nx -- F5-R1: prove LIVE Common Crawl connectivity over the sovereign TLS 1.3 stack (the first
2// real step of the NAS scale-run, now that we're local). Fetches the CC-Index crawl list (collinfo.json -- small plain
3// JSON, no gz) and shows real crawl IDs (CC-MAIN-2026-...). Composes the proven nx_https_fetch_follow + sovereign trust
4// store. Honest: proves DATA ACQUISITION from Common Crawl works end-to-end over OUR stack; the full ingest
5// (gz-inflate segments -> nx_warc_index) is the next rung. (Name: nx_cc_probe is the compiler loop-detector -- distinct.)
6// license_tier: ORIGINAL
7import "nx_syscalls.nx"
8import "nx_x509_trust_store.nx"
9import "nx_trust_store_load_from_certdata.nx"
10import "nx_https_fetch_follow.nx"
11const K_MAGIC_4194304: i64 = 4194304
12
13func gw(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
14func gn(v: i64) -> i64 { if v==0 { sys_write(1,"0" as *u8,1); return 0 } var m: i64=v; if m<0 { sys_write(1,"-" as *u8,1); m=0-m } let t: *u8=sys_mmap(24); var k: i64=0; while m>0 { t[k]=(48+(m%10)) as u8; m=m/10; k=k+1 } let o: *u8=sys_mmap(24); var w: i64=0; var q: i64=k-1; while q>=0 { o[w]=t[q]; w=w+1; q=q-1 } sys_write(1,o,w); return 0 }
15
16func main() -> i64 {
17 gw("=== nx_commoncrawl_probe: LIVE Common Crawl over sovereign TLS 1.3 (F5-R1 data acquisition) ===\n" as *u8)
18 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, K_MAGIC_4194304)
19 if r <= 0 { gw("trust store load failed\n" as *u8); return 1 }
20 let store: *TrustStore = r as *TrustStore
21 let cap: i64 = K_MAGIC_4194304
22 let out: *u8 = sys_mmap(cap)
23 let st: *i64 = sys_mmap(8) as *i64
24 let url: *u8 = "https://index.commoncrawl.org/collinfo.json" as *u8
25 gw("fetch " as *u8); gw(url); gw("\n" as *u8)
26 let n: i64 = nx_https_fetch_follow(url, store, out, cap, 8, st)
27 gw("status=" as *u8); gn(st[0]); gw(" bytes=" as *u8); gn(n); gw("\n" as *u8)
28 if n > 0 {
29 gw("--- first 500 bytes (real CC crawl list) ---\n" as *u8)
30 var show: i64 = n; if show > 500 { show = 500 }
31 sys_write(1, out, show)
32 gw("\n--- end ---\n" as *u8)
33 }
34 if st[0] == 200 { gw("CC-PROBE GREEN -- live Common Crawl reached + real data pulled over OUR TLS 1.3\n" as *u8); sys_exit(0); return 0 }
35 gw("CC-PROBE: non-200 or fetch issue (status above) -- diagnose before scale\n" as *u8); sys_exit(1); return 1
36}