nx_commoncrawl_query.nx source
↩ module page · 36 lines · 2637 B
1// nx_commoncrawl_query.nx -- F5-R2: query the LIVE CC-Index (CDX) for a URL -> real WARC filename/offset/length (the
2// targeted-ingest lookup = C2 planner, live). Uses the latest crawl proven by nx_commoncrawl_probe (CC-MAIN-2026-25).
3// The returned (warc-filename, offset, length) is the exact byte-range the NAS fetches + gz-inflates + indexes -- so a
4// URL's content is ingested WITHOUT downloading a whole 344 TiB segment. Composes nx_https_fetch_follow + trust store;
5// response is plain JSON (no gz). license_tier: ORIGINAL
6import "nx_syscalls.nx"
7import "nx_x509_trust_store.nx"
8import "nx_trust_store_load_from_certdata.nx"
9import "nx_https_fetch_follow.nx"
10const K_MAGIC_4194304: i64 = 4194304
11
12func gw(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
13func gn(v: i64) -> i64 { if v==0 { sys_write(1,"0" as *u8,1); return 0 } var m: i64=v; if m<0 { sys_write(1,"-" as *u8,1); m=0-m } let t: *u8=sys_mmap(24); var k: i64=0; while m>0 { t[k]=(48+(m%10)) as u8; m=m/10; k=k+1 } let o: *u8=sys_mmap(24); var w: i64=0; var q: i64=k-1; while q>=0 { o[w]=t[q]; w=w+1; q=q-1 } sys_write(1,o,w); return 0 }
14
15func main() -> i64 {
16 gw("=== nx_commoncrawl_query: LIVE CC-Index (CDX) lookup -> real WARC locations (F5-R2 targeted ingest) ===\n" as *u8)
17 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, K_MAGIC_4194304)
18 if r <= 0 { gw("trust store load failed\n" as *u8); return 1 }
19 let store: *TrustStore = r as *TrustStore
20 let cap: i64 = K_MAGIC_4194304
21 let out: *u8 = sys_mmap(cap)
22 let st: *i64 = sys_mmap(8) as *i64
23 // CC-Index query: all captures of example.com in the latest crawl, as JSON lines (url/mime/status/filename/offset/length).
24 let url: *u8 = "https://index.commoncrawl.org/CC-MAIN-2026-25-index?url=example.com&output=json" as *u8
25 gw("query " as *u8); gw(url); gw("\n" as *u8)
26 let n: i64 = nx_https_fetch_follow(url, store, out, cap, 10, st)
27 gw("status=" as *u8); gn(st[0]); gw(" bytes=" as *u8); gn(n); gw("\n" as *u8)
28 if n > 0 {
29 gw("--- first 900 bytes (real CDX records: filename=WARC segment, offset+length=byte-range to fetch) ---\n" as *u8)
30 var show: i64 = n; if show > 900 { show = 900 }
31 sys_write(1, out, show)
32 gw("\n--- end ---\n" as *u8)
33 }
34 if st[0] == 200 { gw("CC-QUERY GREEN -- targeted CC-Index lookup live; the (warc,offset,length) is the exact ingest byte-range\n" as *u8); sys_exit(0); return 0 }
35 gw("CC-QUERY: non-200/empty (URL may be absent in this crawl, or index shard 404) -- status above\n" as *u8); sys_exit(1); return 1
36}