nx_cdx_parse.nx source
↩ module page · 51 lines · 2205 B
1// nx_cdx_parse.nx -- LIB: parse Wayback / Common-Crawl CDX index rows into temporal-index inputs. A CDX row is
2// space-delimited: "urlkey timestamp original mimetype statuscode digest length", e.g.
3// com,page3)/ 19971211090223 http://www.page3.com:80/ text/html 200 <digest> 909
4// cdx_rows splits a CDX buffer into row [start,end) spans; cdx_field extracts the i-th space-delimited field of a
5// row; cdx_ts8 converts the 14-digit YYYYMMDDhhmmss timestamp to a YYYYMMDD int for nx_temporal_index. Zero-dep,
6// byte-exact. license_tier: ORIGINAL
7import "nx_syscalls.nx"
8
9// split a CDX buffer into rows on '\n'; writes rowstarts[]/rowends[] (up to cap); returns row count.
10func cdx_rows(buf: *u8, n: i64, rowstarts: *i64, rowends: *i64, cap: i64) -> i64 {
11 var c: i64 = 0
12 var i: i64 = 0
13 var ls: i64 = 0
14 while i < n {
15 if (buf[i] as i64) == 0x0a {
16 if i > ls { if c < cap { rowstarts[c]=ls; rowends[c]=i; c=c+1 } }
17 ls = i + 1
18 }
19 i = i + 1
20 }
21 if n > ls { if c < cap { rowstarts[c]=ls; rowends[c]=n; c=c+1 } }
22 return c
23}
24// extract the idx-th (0-based) space-delimited field of row [rs,re); out2[0]=off out2[1]=len; returns 1 if found.
25func cdx_field(buf: *u8, rs: i64, re: i64, idx: i64, out2: *i64) -> i64 {
26 var i: i64 = rs
27 var f: i64 = 0
28 var done: i64 = 0
29 var res: i64 = 0
30 while done == 0 {
31 var sp: i64 = 1
32 while sp == 1 { if i < re { if (buf[i] as i64)==0x20 { i=i+1 } else { sp=0 } } else { sp=0 } }
33 if i >= re { done = 1 } else {
34 let fs: i64 = i
35 var nz: i64 = 1
36 while nz == 1 { if i < re { if (buf[i] as i64)==0x20 { nz=0 } else { i=i+1 } } else { nz=0 } }
37 if f == idx { out2[0]=fs; out2[1]=i-fs; res=1; done=1 }
38 f = f + 1
39 }
40 }
41 return res
42}
43// convert the leading up-to-8 digits of a CDX timestamp (YYYYMMDDhhmmss) to a YYYYMMDD int.
44func cdx_ts8(buf: *u8, off: i64, len: i64) -> i64 {
45 var v: i64 = 0
46 var i: i64 = 0
47 var lim: i64 = len
48 if lim > 8 { lim = 8 }
49 while i < lim { let c: i64 = buf[off+i] as i64; if c >= 0x30 { if c <= 0x39 { v = v*10 + (c-0x30) } } i = i + 1 }
50 return v
51}