nx_warc_reader_gate.nx source
↩ module page · 100 lines · 6409 B
1// nx_warc_reader_gate.nx -- GREEN/RED gate for nx_warc_reader. Builds a SYNTHETIC plain-WARC fixture in memory (a
2// warcinfo record + a 'response' record for https://www.page3.com/ whose HTTP body is HTML referencing /girl.jpg),
3// parses it with warc_next_record, and asserts: exactly 2 records, the response record's WARC-Type / Target-URI /
4// HTTP status / referenced media are extracted byte-exact, and a non-WARC buffer yields no record (NEG-CTL).
5// Sovereign, deterministic, no network. license_tier: ORIGINAL
6import "nx_warc_reader.nx"
7
8func gw(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
9func gn(v: i64) -> i64 { if v==0 { sys_write(1,"0" as *u8,1); return 0 } var m: i64=v; if m<0 { sys_write(1,"-" as *u8,1); m=0-m } let t: *u8=sys_mmap(24); var k: i64=0; while m>0 { t[k]=(48+(m%10)) as u8; m=m/10; k=k+1 } let o: *u8=sys_mmap(24); var w: i64=0; var q: i64=k-1; while q>=0 { o[w]=t[q]; w=w+1; q=q-1 } sys_write(1,o,w); return 0 }
10func ap(buf: *u8, off: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8) { buf[off+i]=s[i]; i=i+1 } return off+i }
11func apn(buf: *u8, off: i64, v: i64) -> i64 { if v==0 { buf[off]=0x30 as u8; return off+1 } var m: i64=v; let t: *u8=sys_mmap(24); var k: i64=0; while m>0 { t[k]=(48+(m%10)) as u8; m=m/10; k=k+1 } var o: i64=off; var q: i64=k-1; while q>=0 { buf[o]=t[q]; o=o+1; q=q-1 } return o }
12func apcrlf(buf: *u8, off: i64) -> i64 { buf[off]=0x0d as u8; buf[off+1]=0x0a as u8; return off+2 }
13func apbuf(dst: *u8, off: i64, src: *u8, m: i64) -> i64 { var i: i64=0; while i<m { dst[off+i]=src[i]; i=i+1 } return off+m }
14
15func main() -> i64 {
16 gw("=== nx_warc_reader_gate: sovereign WARC record parser on a synthetic fixture (no network) ===\n" as *u8)
17 let cap: i64 = 65536
18 let buf: *u8 = sys_mmap(cap)
19 let c1: *u8 = sys_mmap(cap)
20 let c2: *u8 = sys_mmap(cap)
21
22 // -- build content blocks first so Content-Length is exact --
23 var l1: i64 = 0
24 l1 = ap(c1, l1, "software: nishi-warc-fixture" as *u8); l1 = apcrlf(c1, l1)
25 l1 = ap(c1, l1, "format: WARC File Format 1.0" as *u8)
26
27 var l2: i64 = 0
28 l2 = ap(c2, l2, "HTTP/1.1 200 OK" as *u8); l2 = apcrlf(c2, l2)
29 l2 = ap(c2, l2, "Content-Type: text/html" as *u8); l2 = apcrlf(c2, l2)
30 l2 = apcrlf(c2, l2)
31 l2 = ap(c2, l2, "<html><body><h1>Page 3</h1><img src=/girl.jpg></body></html>" as *u8)
32
33 // -- assemble the WARC --
34 var o: i64 = 0
35 // record 1: warcinfo
36 o = ap(buf, o, "WARC/1.0" as *u8); o = apcrlf(buf, o)
37 o = ap(buf, o, "WARC-Type: warcinfo" as *u8); o = apcrlf(buf, o)
38 o = ap(buf, o, "WARC-Date: 2013-12-01T00:00:00Z" as *u8); o = apcrlf(buf, o)
39 o = ap(buf, o, "Content-Length: " as *u8); o = apn(buf, o, l1); o = apcrlf(buf, o)
40 o = apcrlf(buf, o)
41 o = apbuf(buf, o, c1, l1)
42 o = apcrlf(buf, o); o = apcrlf(buf, o)
43 // record 2: response
44 o = ap(buf, o, "WARC/1.0" as *u8); o = apcrlf(buf, o)
45 o = ap(buf, o, "WARC-Type: response" as *u8); o = apcrlf(buf, o)
46 o = ap(buf, o, "WARC-Target-URI: https://www.page3.com/" as *u8); o = apcrlf(buf, o)
47 o = ap(buf, o, "WARC-Date: 2013-12-01T00:00:01Z" as *u8); o = apcrlf(buf, o)
48 o = ap(buf, o, "Content-Type: application/http; msgtype=response" as *u8); o = apcrlf(buf, o)
49 o = ap(buf, o, "Content-Length: " as *u8); o = apn(buf, o, l2); o = apcrlf(buf, o)
50 o = apcrlf(buf, o)
51 o = apbuf(buf, o, c2, l2)
52 o = apcrlf(buf, o); o = apcrlf(buf, o)
53 let total: i64 = o
54
55 var pass: i64=0; var tot: i64=0
56 let f: *i64 = sys_mmap(64) as *i64
57
58 let np1: i64 = warc_next_record(buf, total, 0, f)
59 let np2: i64 = warc_next_record(buf, total, np1, f)
60 let r2_to: i64 = f[0]; let r2_tl: i64 = f[1]; let r2_uo: i64 = f[2]; let r2_ul: i64 = f[3]; let r2_co: i64 = f[4]; let r2_cl: i64 = f[5]
61 let np3: i64 = warc_next_record(buf, total, np2, f)
62
63 // T1: exactly 2 records
64 tot=tot+1; var t1: i64=0; if np1>=0 { if np2>=0 { if np3<0 { t1=1 } } }
65 if t1==1 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) }
66 gw("T1 RECORD-LOOP: parsed exactly 2 records (np1=" as *u8); gn(np1); gw(" np2=" as *u8); gn(np2); gw(" np3=" as *u8); gn(np3); gw(")\n" as *u8)
67
68 // T2: response record WARC-Type == "response"
69 tot=tot+1; var t2: i64=0; if r2_tl==8 { if wr_starts(buf, r2_to, total, "response" as *u8, 8)==1 { t2=1 } }
70 if t2==1 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) }
71 gw("T2 WARC-TYPE: response record typed 'response' (len=" as *u8); gn(r2_tl); gw(")\n" as *u8)
72
73 // T3: WARC-Target-URI == https://www.page3.com/
74 tot=tot+1; var t3: i64=0; if r2_ul==22 { if wr_starts(buf, r2_uo, total, "https://www.page3.com/" as *u8, 22)==1 { t3=1 } }
75 if t3==1 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) }
76 gw("T3 TARGET-URI: https://www.page3.com/ (len=" as *u8); gn(r2_ul); gw(")\n" as *u8)
77
78 // T4: content is an HTTP 200 response
79 tot=tot+1; var t4: i64=0; if wr_starts(buf, r2_co, total, "HTTP/1.1 200" as *u8, 12)==1 { t4=1 }
80 if t4==1 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) }
81 gw("T4 HTTP-PAYLOAD: response content opens 'HTTP/1.1 200' (clen=" as *u8); gn(r2_cl); gw(")\n" as *u8)
82
83 // T5: referenced media visible in the archived body (the media shopping-list)
84 tot=tot+1; var t5: i64=0; if wr_contains(buf, r2_co, r2_cl, "girl.jpg" as *u8, 8)==1 { t5=1 }
85 if t5==1 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) }
86 gw("T5 MEDIA-REF: /girl.jpg referenced in the archived HTML (the per-page media shopping-list)\n" as *u8)
87
88 // T6 NEG-CTL: a non-WARC buffer yields no record
89 let gb: *u8 = sys_mmap(64)
90 let glen: i64 = ap(gb, 0, "GARBAGE not a warc record" as *u8)
91 let negf: *i64 = sys_mmap(64) as *i64
92 let neg: i64 = warc_next_record(gb, glen, 0, negf)
93 tot=tot+1; var t6: i64=0; if neg<0 { t6=1 }
94 if t6==1 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) }
95 gw("T6 NEG-CTL: non-WARC buffer parses to NO record (parser doesn't hallucinate)\n" as *u8)
96
97 gw("\n=== nx_warc_reader_gate " as *u8); gn(pass); gw("/" as *u8); gn(tot); gw(" ===\n" as *u8)
98 if pass==tot { gw("WARC-READER-GATE GREEN -- sovereign WARC record ingest proven on a synthetic fixture\n" as *u8); sys_exit(0); return 0 }
99 gw("WARC-READER-GATE RED\n" as *u8); sys_exit(1); return 1
100}