nx_cc_datahost_probe.nx source
↩ module page · 33 lines · 2483 B
1// nx_cc_datahost_probe.nx -- prove the CC DATA host (data.commoncrawl.org -- the bulk WARC/WET CDN, distinct from
2// index.commoncrawl.org already proven) serves us over sovereign TLS. Fetches a small RANGE of the latest crawl's
3// warc.paths.gz (the list of WARC segment paths) -> if 200/206 with bytes, the bulk-ingest path is viable from here.
4// Uses the best-effort fetch (minimal hello then Chrome-JA3 fallback) since CC is behind a CDN. license_tier: ORIGINAL
5import "nx_syscalls.nx"
6import "nx_x509_trust_store.nx"
7import "nx_trust_store_load_from_certdata.nx"
8import "nx_https_fetch_follow.nx"
9const K_MAGIC_4194304: i64 = 4194304
10const K_MAGIC_16777216: i64 = 16777216
11
12func gw(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
13func gn(v: i64) -> i64 { if v==0 { sys_write(1,"0" as *u8,1); return 0 } var m: i64=v; if m<0 { sys_write(1,"-" as *u8,1); m=0-m } let t: *u8=sys_mmap(24); var k: i64=0; while m>0 { t[k]=(48+(m%10)) as u8; m=m/10; k=k+1 } let o: *u8=sys_mmap(24); var w: i64=0; var q: i64=k-1; while q>=0 { o[w]=t[q]; w=w+1; q=q-1 } sys_write(1,o,w); return 0 }
14
15func main() -> i64 {
16 gw("=== nx_cc_datahost_probe: data.commoncrawl.org over sovereign TLS (bulk-ingest reachability) ===\n" as *u8)
17 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, K_MAGIC_4194304)
18 if r <= 0 { gw("trust store load failed\n" as *u8); return 1 }
19 let store: *TrustStore = r as *TrustStore
20 let cap: i64 = K_MAGIC_16777216
21 let out: *u8 = sys_mmap(cap)
22 let st: *i64 = sys_mmap(8) as *i64
23 // the WARC segment path list for the latest proven crawl (gzipped, ~7MB). A plain GET tests the data host.
24 let url: *u8 = "https://data.commoncrawl.org/crawl-data/CC-MAIN-2026-25/warc.paths.gz" as *u8
25 gw("fetch " as *u8); gw(url); gw("\n" as *u8)
26 let n: i64 = nx_https_fetch_follow_best(url, store, out, cap, 8, st)
27 gw("status=" as *u8); gn(st[0]); gw(" bytes=" as *u8); gn(n); gw("\n" as *u8)
28 var gz: i64 = 0
29 if n >= 2 { if out[0] == (31 as u8) { if out[1] == (139 as u8) { gz = 1 } } }
30 gw("gzip-magic(1f 8b)=" as *u8); gn(gz); gw("\n" as *u8)
31 if st[0] == 200 { if gz == 1 { gw("CC-DATAHOST GREEN -- data.commoncrawl.org serves us; bulk WARC/WET ingest is viable from here\n" as *u8); sys_exit(0); return 0 } }
32 gw("CC-DATAHOST: status/format issue above -- diagnose (CDN block? redirect? size?) before bulk ingest\n" as *u8); sys_exit(1); return 1
33}