nx_warc_reader.nx source
↩ module page · 101 lines · 4912 B
1// nx_warc_reader.nx -- LIB: sovereign WARC (ISO 28500 / Common Crawl) record reader. A .warc file is a sequence of
2// records: "WARC/1.0" CRLF, then "Name: Value" header lines, a blank line, a Content-Length-delimited content block,
3// then trailing CRLFs. A 'response' record wraps a full HTTP response (status line + headers + body). This parses the
4// plain-WARC record loop from an in-memory buffer; multi-member-gzip streaming (.warc.gz = one gzip member per record)
5// composes the existing nx_deflate inflate + bytes-consumed in a follow-on (R2b). Zero-dependency, byte-exact.
6// warc_next_record(buf,n,pos,fields) fills fields[6] for the record at pos and returns the START OF THE NEXT record
7// (>=0), or 0-1 when there is no record at pos. fields: [0]=type_off [1]=type_len [2]=uri_off [3]=uri_len
8// [4]=content_off [5]=content_len. license_tier: ORIGINAL
9import "nx_syscalls.nx"
10
11func wr_starts(buf: *u8, off: i64, n: i64, lit: *u8, litlen: i64) -> i64 {
12 if off + litlen > n { return 0 }
13 var i: i64 = 0
14 while i < litlen { if (buf[off+i] as i64) != (lit[i] as i64) { return 0 } i = i + 1 }
15 return 1
16}
17func wr_lc(c: i64) -> i64 { if c >= 0x41 { if c <= 0x5a { return c + 0x20 } } return c }
18func wr_starts_ci(buf: *u8, off: i64, n: i64, lit: *u8, litlen: i64) -> i64 {
19 if off + litlen > n { return 0 }
20 var i: i64 = 0
21 while i < litlen { if wr_lc(buf[off+i] as i64) != (lit[i] as i64) { return 0 } i = i + 1 }
22 return 1
23}
24func wr_uint(buf: *u8, off: i64, n: i64, endp: *i64) -> i64 {
25 var v: i64 = 0
26 var i: i64 = off
27 var dg: i64 = 1
28 while dg == 1 {
29 if i >= n { dg = 0 } else {
30 let c: i64 = buf[i] as i64
31 if c < 0x30 { dg = 0 } else { if c > 0x39 { dg = 0 } else { v = v*10 + (c - 0x30); i = i + 1 } }
32 }
33 }
34 endp[0] = i
35 return v
36}
37func wr_find_blankline(buf: *u8, off: i64, n: i64) -> i64 {
38 var i: i64 = off
39 while i + 3 < n {
40 if (buf[i] as i64)==0x0d { if (buf[i+1] as i64)==0x0a { if (buf[i+2] as i64)==0x0d { if (buf[i+3] as i64)==0x0a { return i } } } }
41 i = i + 1
42 }
43 return 0 - 1
44}
45func wr_contains(buf: *u8, off: i64, len: i64, lit: *u8, litlen: i64) -> i64 {
46 var i: i64 = off
47 let endi: i64 = off + len - litlen
48 while i <= endi {
49 if wr_starts(buf, i, off+len, lit, litlen) == 1 { return 1 }
50 i = i + 1
51 }
52 return 0
53}
54// find header field `name` (lowercase) in header region [hstart,hend); writes out2[0]=value_off, out2[1]=value_len.
55func wr_header_val(buf: *u8, hstart: i64, hend: i64, n: i64, name: *u8, namelen: i64, out2: *i64) -> i64 {
56 var i: i64 = hstart
57 var linestart: i64 = 1
58 while i < hend {
59 if linestart == 1 {
60 if wr_starts_ci(buf, i, n, name, namelen) == 1 {
61 if (buf[i+namelen] as i64) == 0x3a {
62 var vs: i64 = i + namelen + 1
63 var sp: i64 = 1
64 while sp == 1 { if vs < hend { if (buf[vs] as i64)==0x20 { vs=vs+1 } else { sp=0 } } else { sp=0 } }
65 var ve: i64 = vs
66 var fe: i64 = 0
67 while fe == 0 { if ve < hend { if (buf[ve] as i64)==0x0d { fe=1 } else { ve=ve+1 } } else { fe=1 } }
68 out2[0] = vs
69 out2[1] = ve - vs
70 return 1
71 }
72 }
73 }
74 if (buf[i] as i64) == 0x0a { linestart = 1 } else { linestart = 0 }
75 i = i + 1
76 }
77 return 0
78}
79func warc_next_record(buf: *u8, n: i64, pos: i64, fields: *i64) -> i64 {
80 var p: i64 = pos
81 var sk: i64 = 1
82 while sk == 1 { if p < n { let c: i64 = buf[p] as i64; if c==0x0d { p=p+1 } else { if c==0x0a { p=p+1 } else { sk=0 } } } else { sk=0 } }
83 if p >= n { return 0 - 1 }
84 if wr_starts(buf, p, n, "WARC/" as *u8, 5) == 0 { return 0 - 1 }
85 let bl: i64 = wr_find_blankline(buf, p, n)
86 if bl < 0 { return 0 - 1 }
87 let hend: i64 = bl + 2
88 let b2: *i64 = sys_mmap(32) as *i64
89 if wr_header_val(buf, p, hend, n, "warc-type" as *u8, 9, b2) == 1 { fields[0]=b2[0]; fields[1]=b2[1] } else { fields[0]=0; fields[1]=0 }
90 if wr_header_val(buf, p, hend, n, "warc-target-uri" as *u8, 15, b2) == 1 { fields[2]=b2[0]; fields[3]=b2[1] } else { fields[2]=0; fields[3]=0 }
91 var clen: i64 = 0
92 if wr_header_val(buf, p, hend, n, "content-length" as *u8, 14, b2) == 1 { let ep: *i64 = sys_mmap(8) as *i64; clen = wr_uint(buf, b2[0], n, ep) }
93 fields[4] = bl + 4
94 fields[5] = clen
95 var np: i64 = bl + 4 + clen
96 var sk2: i64 = 1
97 while sk2 == 1 { if np < n { let c: i64 = buf[np] as i64; if c==0x0d { np=np+1 } else { if c==0x0a { np=np+1 } else { sk2=0 } } } else { sk2=0 } }
98 return np
99}
100// NOTE: no func main here -- this is an importable LIB (a stub main would link-clash with any importer's
101// main, the exact nx_str/nx_strfmt re-rolling root cause). The gate carries its own harness.