nx_cc_record_fetch.nx source
↩ module page · 118 lines · 6733 B
1// nx_cc_record_fetch.nx -- THE CC INGEST KEYSTONE: pull ONE real Common Crawl record end-to-end over our TLS,
2// proving targeted acquisition (the byte-range, NOT a 344TiB segment). Flow: (1) CDX-query CC-MAIN-2026-25 for
3// argv[1] (default example.com) -> JSON with filename/offset/length; (2) parse those; (3) range-fetch exactly
4// that slice of the WARC from data.commoncrawl.org; (4) verify it's a gzip member (1f 8b) of ~length bytes.
5// This is the primitive the bulk index-only ingest loops over (extract text -> ss_add doc+url+locator, serve
6// LIVE url). Composes nx_commoncrawl_query's endpoint + the new nx_https_fetch_range. license_tier: ORIGINAL
7import "nx_syscalls.nx"
8import "nx_x509_trust_store.nx"
9import "nx_trust_store_load_from_certdata.nx"
10import "nx_https_fetch_follow.nx"
11const K_MAGIC_4194304: i64 = 4194304
12const K_MAGIC_1024: i64 = 1024
13const K_MAGIC_1048576: i64 = 1048576
14
15func gw(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
16func gn(v: i64) -> i64 { if v==0 { sys_write(1,"0" as *u8,1); return 0 } var m: i64=v; if m<0 { sys_write(1,"-" as *u8,1); m=0-m } let t: *u8=sys_mmap(24); var k: i64=0; while m>0 { t[k]=(48+(m%10)) as u8; m=m/10; k=k+1 } let o: *u8=sys_mmap(24); var q: i64=k-1; var x: i64=0; while q>=0 { o[x]=t[q]; x=x+1; q=q-1 } sys_write(1,o,x); return 0 }
17func slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
18// find the value of a JSON string field: "key":"<value>" -> copy value into out, return length (-1 absent)
19func json_str(hay: *u8, hn: i64, key: *u8, out: *u8, cap: i64) -> i64 {
20 let kl: i64 = slen(key)
21 var i: i64 = 0
22 while i + kl + 3 < hn {
23 // match "key":"
24 if hay[i] == (34 as u8) {
25 var m: i64 = 1
26 var j: i64 = 0
27 while j < kl { if hay[i+1+j] != key[j] { m = 0; j = kl } else { j = j + 1 } }
28 if m == 1 {
29 let after: i64 = i + 1 + kl // index of the key's closing quote
30 if hay[after] == (34 as u8) { if hay[after+1] == (58 as u8) { // '"' then ':'
31 // skip optional whitespace after the colon, then expect the value's opening quote
32 var p: i64 = after + 2
33 var stop: i64 = 0
34 while stop == 0 { if p >= hn { stop = 1 } else { if hay[p] == (32 as u8) { p = p + 1 } else { stop = 1 } } }
35 if p < hn { if hay[p] == (34 as u8) {
36 var s: i64 = p + 1
37 var o: i64 = 0
38 while s < hn { if hay[s] == (34 as u8) { s = hn } else { if o < cap-1 { out[o] = hay[s]; o = o + 1 } s = s + 1 } }
39 out[o] = 0 as u8
40 return o
41 } }
42 } }
43 }
44 }
45 i = i + 1
46 }
47 return 0 - 1
48}
49func atoin(s: *u8) -> i64 { var v: i64=0; var i: i64=0; while s[i]!=(0 as u8){ let c: i64=s[i] as i64; if c>=48 { if c<=57 { v=v*10+(c-48) } } i=i+1 } return v }
50
51func main(argc: i64, argv: *i64) -> i64 {
52 gw("=== nx_cc_record_fetch: ONE real Common Crawl record end-to-end (CDX -> range-fetch) ===\n" as *u8)
53 var qurl: *u8 = "example.com" as *u8
54 if argc >= 2 { qurl = argv[1] as *u8 }
55 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, K_MAGIC_4194304)
56 if r <= 0 { gw("trust store load failed\n" as *u8); return 1 }
57 let store: *TrustStore = r as *TrustStore
58 let cap: i64 = K_MAGIC_4194304
59 let out: *u8 = sys_mmap(cap)
60 let st: *i64 = sys_mmap(8) as *i64
61
62 // (1) CDX query for the URL in the latest crawl -> JSON with filename/offset/length
63 let cdx: *u8 = sys_mmap(K_MAGIC_1024)
64 var co: i64 = 0
65 let p1: *u8 = "https://index.commoncrawl.org/CC-MAIN-2026-25-index?url=" as *u8
66 var pi: i64 = 0; while p1[pi]!=(0 as u8){ cdx[co]=p1[pi]; co=co+1; pi=pi+1 }
67 var qi: i64 = 0; while qurl[qi]!=(0 as u8){ cdx[co]=qurl[qi]; co=co+1; qi=qi+1 }
68 let p2: *u8 = "&output=json&limit=1" as *u8
69 pi = 0; while p2[pi]!=(0 as u8){ cdx[co]=p2[pi]; co=co+1; pi=pi+1 }
70 cdx[co] = 0 as u8
71 gw("CDX "); gw(cdx); gw("\n" as *u8)
72 let n1: i64 = nx_https_fetch_follow(cdx, store, out, cap, 10, st)
73 gw(" cdx status="); gn(st[0]); gw(" bytes="); gn(n1); gw("\n" as *u8)
74 if st[0] != 200 { gw("CDX non-200 (url may be absent in this crawl)\n" as *u8); return 2 }
75
76 let fname: *u8 = sys_mmap(512)
77 let offs: *u8 = sys_mmap(64)
78 let lens: *u8 = sys_mmap(64)
79 if json_str(out, n1, "filename" as *u8, fname, 512) < 0 {
80 gw("no filename in CDX json -- raw response follows:\n" as *u8)
81 var dn: i64 = n1; if dn > 500 { dn = 500 }
82 sys_write(1, out, dn); gw("\n" as *u8)
83 return 3
84 }
85 if json_str(out, n1, "offset" as *u8, offs, 64) < 0 { gw("no offset\n" as *u8); return 3 }
86 if json_str(out, n1, "length" as *u8, lens, 64) < 0 { gw("no length\n" as *u8); return 3 }
87 let offv: i64 = atoin(offs)
88 let lenv: i64 = atoin(lens)
89 gw(" record: filename="); gw(fname); gw("\n offset="); gn(offv); gw(" length="); gn(lenv); gw("\n" as *u8)
90
91 // (2) range-fetch exactly that slice from the DATA host
92 let durl: *u8 = sys_mmap(K_MAGIC_1024)
93 var dodo: i64 = 0
94 let d1: *u8 = "https://data.commoncrawl.org/" as *u8
95 pi = 0; while d1[pi]!=(0 as u8){ durl[dodo]=d1[pi]; dodo=dodo+1; pi=pi+1 }
96 var fi: i64 = 0; while fname[fi]!=(0 as u8){ durl[dodo]=fname[fi]; dodo=dodo+1; fi=fi+1 }
97 durl[dodo] = 0 as u8
98 let rec: *u8 = sys_mmap(K_MAGIC_1048576)
99 let rst: *i64 = sys_mmap(8) as *i64
100 gw("RANGE bytes="); gn(offv); gw("-"); gn(offv+lenv-1); gw(" of "); gw(durl); gw("\n" as *u8)
101 // best-effort: try minimal hello, then Chrome-JA3 (CC is behind a CDN)
102 var nr: i64 = nx_https_fetch_range(durl, store, rec, K_MAGIC_1048576, offv, offv+lenv-1, rst, 0)
103 if nr <= 0 { nr = nx_https_fetch_range(durl, store, rec, K_MAGIC_1048576, offv, offv+lenv-1, rst, 1) }
104 if st[0] < 0 { nr = nr }
105 gw(" range status="); gn(rst[0]); gw(" bytes="); gn(nr); gw("\n" as *u8)
106 var gz: i64 = 0
107 if nr >= 2 { if rec[0] == (31 as u8) { if rec[1] == (139 as u8) { gz = 1 } } }
108 gw(" gzip-magic(1f 8b)="); gn(gz); gw(" (a CC WARC record is a gzip MEMBER)\n" as *u8)
109 var okstatus: i64 = 0
110 if rst[0] == 206 { okstatus = 1 }
111 if rst[0] == 200 { okstatus = 1 }
112 if okstatus == 1 { if gz == 1 { if nr >= lenv - 4 {
113 gw("CC-RECORD GREEN -- pulled the exact record byte-range over OUR TLS (gzip member, ~length bytes)\n" as *u8)
114 sys_exit(0); return 0
115 } } }
116 gw("CC-RECORD: status/format/size mismatch above -- diagnose\n" as *u8)
117 sys_exit(1); return 1
118}