nx_http_range_req.nx source
↩ module page · 72 lines · 4381 B
1// nx_http_range_req.nx -- ADDITIVE HTTP/1.1 Range request builder (RFC 9110 Section 14.2) -- the first piece of
2// CC-content fetch: to pull ONE Common Crawl record you request its byte-slice of the multi-GB .warc.gz via
3// "Range: bytes=START-END". Mirrors nx_http_client_build_request but adds the Range header (additive; existing builder
4// untouched). Gate verifies the exact request for the real example.com CC record (offset 176519143, length 946).
5// The follow-on wires this into the TLS fetch + gz-member inflate (nx_deflate) -> real CC record. license_tier: ORIGINAL
6import "nx_syscalls.nx"
7const K_MAGIC_4096: i64 = 4096
8const K_MAGIC_176519143: i64 = 176519143
9const K_MAGIC_176520088: i64 = 176520088
10
11func rr_app(out: *u8, o: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8){ out[o]=s[i]; o=o+1; i=i+1 } return o }
12func rr_appn(out: *u8, o: i64, s: *u8, n: i64) -> i64 { var i: i64=0; while i<n { out[o]=s[i]; o=o+1; i=i+1 } return o }
13func rr_u64(out: *u8, o: i64, v: i64) -> i64 {
14 if v==0 { out[o]=48 as u8; return o+1 }
15 let t: *u8=sys_mmap(24); var m: i64=v; var k: i64=0
16 while m>0 { t[k]=(48+(m%10)) as u8; m=m/10; k=k+1 }
17 var j: i64=k-1
18 while j>=0 { out[o]=t[j]; o=o+1; j=j-1 }
19 return o
20}
21
22// Build "GET <path> HTTP/1.1\r\nHost:..\r\n<UA>\r\nRange: bytes=S-E\r\nConnection: close\r\n\r\n". Returns length.
23func nx_http_build_range_get(path: *u8, path_len: i64, host: *u8, host_len: i64, rstart: i64, rend: i64, out: *u8) -> i64 {
24 var o: i64=0
25 o = rr_app(out, o, "GET " as *u8)
26 o = rr_appn(out, o, path, path_len)
27 o = rr_app(out, o, " HTTP/1.1\r\nHost: " as *u8)
28 o = rr_appn(out, o, host, host_len)
29 o = rr_app(out, o, "\r\nUser-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0\r\nAccept: */*\r\nAccept-Encoding: identity\r\nRange: bytes=" as *u8)
30 o = rr_u64(out, o, rstart)
31 out[o]=45 as u8; o=o+1 // '-'
32 o = rr_u64(out, o, rend)
33 o = rr_app(out, o, "\r\nConnection: close\r\n\r\n" as *u8)
34 return o
35}
36
37// substring search: is needle (NUL-term) present in hay[0..hn)?
38func rr_find(hay: *u8, hn: i64, needle: *u8) -> i64 {
39 var nn: i64=0; while needle[nn]!=(0 as u8){nn=nn+1}
40 if nn==0 { return 1 }
41 var i: i64=0
42 while i+nn<=hn { var j: i64=0; var ok: i64=1; while j<nn { if hay[i+j]!=needle[j] { ok=0; j=nn } else { j=j+1 } } if ok==1 { return 1 } i=i+1 }
43 return 0
44}
45func rr_w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
46func rr_chk(cond: i64, name: *u8, acc: *i64) -> i64 { acc[1]=acc[1]+1; if cond==1 { acc[0]=acc[0]+1; rr_w(" PASS " as *u8) } else { rr_w(" FAIL " as *u8) } rr_w(name); rr_w("\n" as *u8); return 0 }
47
48func main() -> i64 {
49 let out: *u8 = sys_mmap(K_MAGIC_4096)
50 let path: *u8 = "/crawl-data/CC-MAIN-2026-25/segments/1780687572080.85/warc/CC-MAIN-20260605214811-20260606004811-00162.warc.gz" as *u8
51 var pl: i64=0; while path[pl]!=(0 as u8){pl=pl+1}
52 let host: *u8 = "data.commoncrawl.org" as *u8
53 // real CC record: offset 176519143, length 946 -> bytes 176519143-176520088
54 let n: i64 = nx_http_build_range_get(path, pl, host, 20, K_MAGIC_176519143, K_MAGIC_176520088, out)
55 out[n]=0 as u8
56
57 rr_w("=== nx_http_range_req: RFC-9110 Range request for a REAL Common Crawl record ===\n\n" as *u8)
58 sys_write(1, out, n)
59 rr_w("\n\n" as *u8)
60
61 let acc: *i64 = sys_mmap(16) as *i64; acc[0]=0; acc[1]=0
62 rr_chk(rr_find(out, n, "GET /crawl-data/" as *u8), "starts with GET + path" as *u8, acc)
63 rr_chk(rr_find(out, n, "Host: data.commoncrawl.org" as *u8), "Host header" as *u8, acc)
64 rr_chk(rr_find(out, n, "Range: bytes=176519143-176520088" as *u8), "exact Range header (offset+len-1)" as *u8, acc)
65 rr_chk(rr_find(out, n, "Accept-Encoding: identity" as *u8), "identity encoding (gz member handled by us)" as *u8, acc)
66 rr_chk(rr_find(out, n, "Connection: close\r\n\r\n" as *u8), "Connection close + CRLFCRLF terminator" as *u8, acc)
67
68 rr_w("\n=== nx_http_range_req " as *u8)
69 let bb: *u8=sys_mmap(8); var v: i64=acc[0]; bb[0]=(48+v) as u8; sys_write(1,bb,1); rr_w("/" as *u8); bb[1]=(48+acc[1]) as u8; sys_write(1,((bb as i64)+1) as *u8,1)
70 if acc[0]==acc[1] { rr_w(" GREEN -- Range request builder ready for CC-content fetch\n" as *u8); sys_exit(0); return 0 }
71 rr_w(" RED\n" as *u8); sys_exit(1); return 1
72}