code wiki / _hdl_build / nx_web_archive.nx
nx_web_archive.nx source
↩ module page · 182 lines · 10863 B
1// nx_web_archive.nx -- SOVEREIGN WARC-class archive store: the link-rot-prevention core. Preserve a captured
2// resource (URL + content-type + bytes) as a WARC record, and retrieve it by URL later -- so when the origin
3// site vanishes (abandonware, dead host), the search engine serves the PRESERVED copy instead of a 404. WARC
4// (ISO 28500) is the exact format the Internet Archive + Browsertrix use, so ours interoperates. A WARC is a
5// sequence of records: a text header block (WARC/1.0 + fields) then a Content-Length-delimited payload (so
6// BINARY media with embedded CRLF round-trips exactly). We write "resource" records (payload = the raw bytes).
7// license_tier: ORIGINAL
8import "nx_syscalls.nx"
9
10func wa_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
11func wa_puts(dst: *u8, off: i64, s: *u8) -> i64 { var i: i64=0; while s[i]!=(0 as u8){dst[off]=s[i];off=off+1;i=i+1} return off }
12func wa_putn(dst: *u8, off: i64, v: i64) -> i64 { var m: i64=v; if m==0 { dst[off]=48 as u8; return off+1 } let t: *u8=sys_mmap(24); var k: i64=0; while m>0 { t[k]=(48+(m%10)) as u8; m=m/10; k=k+1 } var j: i64=0; while j<k { dst[off]=t[k-1-j]; off=off+1; j=j+1 } return off }
13func wa_find(hay: *u8, hl: i64, needle: *u8, nl: i64, from: i64) -> i64 {
14 if nl == 0 { return from }
15 var i: i64 = from; let last: i64 = hl - nl
16 while i <= last { var j: i64=0; var m: i64=1; while j<nl { if (hay[i+j]&0xff)!=(needle[j]&0xff) { m=0; j=nl } else { j=j+1 } } if m==1 { return i } i=i+1 }
17 return 0 - 1
18}
19
20// APPEND a WARC resource record for (uri, date, ctype, payload[plen]) into buf at off. Returns the new offset.
21func wa_write_resource(buf: *u8, off: i64, uri: *u8, date: *u8, ctype: *u8, payload: *u8, plen: i64) -> i64 {
22 off = wa_puts(buf, off, "WARC/1.0\r\n" as *u8)
23 off = wa_puts(buf, off, "WARC-Type: resource\r\n" as *u8)
24 off = wa_puts(buf, off, "WARC-Target-URI: " as *u8); off = wa_puts(buf, off, uri); off = wa_puts(buf, off, "\r\n" as *u8)
25 off = wa_puts(buf, off, "WARC-Date: " as *u8); off = wa_puts(buf, off, date); off = wa_puts(buf, off, "\r\n" as *u8)
26 off = wa_puts(buf, off, "Content-Type: " as *u8); off = wa_puts(buf, off, ctype); off = wa_puts(buf, off, "\r\n" as *u8)
27 off = wa_puts(buf, off, "Content-Length: " as *u8); off = wa_putn(buf, off, plen); off = wa_puts(buf, off, "\r\n" as *u8)
28 off = wa_puts(buf, off, "\r\n" as *u8) // end of header block
29 var i: i64 = 0; while i < plen { buf[off] = payload[i]; off = off + 1; i = i + 1 } // raw payload (binary-safe)
30 off = wa_puts(buf, off, "\r\n\r\n" as *u8) // record trailer
31 return off
32}
33
34// parse the decimal after "field: " within [start,end); returns the value (0 if absent)
35func wa_field_num(warc: *u8, start: i64, end: i64, field: *u8) -> i64 {
36 let fl: i64 = wa_slen(field)
37 let p: i64 = wa_find(warc, end, field, fl, start)
38 if p < 0 { return 0 }
39 var i: i64 = p + fl
40 var v: i64 = 0
41 var go: i64 = 1
42 while go == 1 { let c: i64 = warc[i]&0xff; if c>=48 { if c<=57 { v=v*10+(c-48); i=i+1 } else { go=0 } } if c<48 { go=0 } if c>57 { go=0 } }
43 return v
44}
45// does the record header in [start,end) have WARC-Target-URI == uri (exact)?
46func wa_uri_matches(warc: *u8, start: i64, end: i64, uri: *u8, ulen: i64) -> i64 {
47 let p: i64 = wa_find(warc, end, "WARC-Target-URI: " as *u8, 17, start)
48 if p < 0 { return 0 }
49 var i: i64 = p + 17
50 var j: i64 = 0
51 var go: i64 = 1
52 while go == 1 {
53 if j >= ulen { go = 0 }
54 else { let c: i64 = warc[i]&0xff; if c == 13 { return 0 } if c != (uri[j]&0xff) { return 0 } i=i+1; j=j+1 }
55 }
56 if (warc[i]&0xff) == 13 { return 1 } // full match ended exactly at CR
57 return 0
58}
59
60// LOOKUP: find the archived resource for `uri`; set out_ptr (offset into warc) + out_len to its payload. 1/0.
61// Content-Length-delimited so binary media (with CRLF inside) is retrieved byte-exact.
62func wa_lookup(warc: *u8, warclen: i64, uri: *u8, ulen: i64, out_ptr: *i64, out_len: *i64) -> i64 {
63 var i: i64 = 0
64 var go: i64 = 1
65 var found: i64 = 0
66 while go == 1 {
67 let rs: i64 = wa_find(warc, warclen, "WARC/1.0\r\n" as *u8, 10, i)
68 if rs < 0 { go = 0 }
69 else {
70 let hdr_end: i64 = wa_find(warc, warclen, "\r\n\r\n" as *u8, 4, rs)
71 if hdr_end < 0 { go = 0 }
72 else {
73 let clen: i64 = wa_field_num(warc, rs, hdr_end, "Content-Length: " as *u8)
74 let payload: i64 = hdr_end + 4
75 if wa_uri_matches(warc, rs, hdr_end, uri, ulen) == 1 { out_ptr[0] = payload; out_len[0] = clen; found = 1 } // keep scanning -> newest wins
76 i = payload + clen + 4 // skip payload + trailer -> next record
77 }
78 }
79 }
80 if found == 0 { out_ptr[0] = 0; out_len[0] = 0 }
81 return found
82}
83
84// extract the Content-Type header value of a record header [start,end) into out; returns len (0 if absent)
85func wa_content_type(warc: *u8, start: i64, end: i64, out: *u8, cap: i64) -> i64 {
86 let p: i64 = wa_find(warc, end, "Content-Type: " as *u8, 14, start)
87 if p < 0 { out[0] = 0 as u8; return 0 }
88 var i: i64 = p + 14; var o: i64 = 0
89 var go: i64 = 1
90 while go == 1 { if i >= end { go = 0 } else { let c: i64 = warc[i]&0xff; if c == 13 { go = 0 } else { if o < (cap-1) { out[o] = c as u8; o = o + 1 } i = i + 1 } } }
91 out[o] = 0 as u8
92 return o
93}
94// RESOLVE a URL to its preserved record: payload ptr/len + content-type, in one scan. 1/0. This is the
95// serve-from-archive primitive the search engine calls when an origin has rotted.
96func wa_resolve(warc: *u8, warclen: i64, uri: *u8, ulen: i64, out_ptr: *i64, out_len: *i64, out_ctype: *u8, ctcap: i64) -> i64 {
97 var i: i64 = 0
98 var go: i64 = 1
99 var found: i64 = 0
100 while go == 1 {
101 let rs: i64 = wa_find(warc, warclen, "WARC/1.0\r\n" as *u8, 10, i)
102 if rs < 0 { go = 0 }
103 else {
104 let hdr_end: i64 = wa_find(warc, warclen, "\r\n\r\n" as *u8, 4, rs)
105 if hdr_end < 0 { go = 0 }
106 else {
107 let clen: i64 = wa_field_num(warc, rs, hdr_end, "Content-Length: " as *u8)
108 let payload: i64 = hdr_end + 4
109 if wa_uri_matches(warc, rs, hdr_end, uri, ulen) == 1 { out_ptr[0] = payload; out_len[0] = clen; wa_content_type(warc, rs, hdr_end, out_ctype, ctcap); found = 1 } // keep scanning -> newest wins
110 i = payload + clen + 4
111 }
112 }
113 }
114 if found == 1 { return 1 }
115 out_ptr[0] = 0; out_len[0] = 0; out_ctype[0] = 0 as u8
116 return 0
117}
118// count records in the archive
119func wa_count(warc: *u8, warclen: i64) -> i64 {
120 var n: i64 = 0; var i: i64 = 0
121 var go: i64 = 1
122 while go == 1 { let p: i64 = wa_find(warc, warclen, "WARC/1.0\r\n" as *u8, 10, i); if p < 0 { go = 0 } else { n = n + 1; i = p + 10 } }
123 return n
124}
125// persist / restore the whole archive (a .warc file = the buffer verbatim)
126func wa_save(path: *u8, warc: *u8, warclen: i64) -> i64 { let fd: i64 = sys_openat_wr(path, 0x1a4); if fd < 0 { return 0-1 } sys_write(fd, warc, warclen); sys_close(fd); return 0 }
127func wa_load(path: *u8, warc: *u8, cap: i64) -> i64 { let fd: i64 = sys_openat_rd(path); if fd < 0 { return 0 } let n: i64 = sys_read(fd, warc, cap); sys_close(fd); return n }
128
129// --- RATING-aware write + record enumeration (for the SFW /media + NSFW /gallery listings) ---
130// like wa_write_resource but stamps an X-Nishi-Rating header (sfw|nsfw) so the listing can segregate content.
131func wa_write_resource_rated(buf: *u8, off: i64, uri: *u8, date: *u8, ctype: *u8, rating: *u8, payload: *u8, plen: i64) -> i64 {
132 off = wa_puts(buf, off, "WARC/1.0\r\n" as *u8)
133 off = wa_puts(buf, off, "WARC-Type: resource\r\n" as *u8)
134 off = wa_puts(buf, off, "WARC-Target-URI: " as *u8); off = wa_puts(buf, off, uri); off = wa_puts(buf, off, "\r\n" as *u8)
135 off = wa_puts(buf, off, "WARC-Date: " as *u8); off = wa_puts(buf, off, date); off = wa_puts(buf, off, "\r\n" as *u8)
136 off = wa_puts(buf, off, "X-Nishi-Rating: " as *u8); off = wa_puts(buf, off, rating); off = wa_puts(buf, off, "\r\n" as *u8)
137 off = wa_puts(buf, off, "Content-Type: " as *u8); off = wa_puts(buf, off, ctype); off = wa_puts(buf, off, "\r\n" as *u8)
138 off = wa_puts(buf, off, "Content-Length: " as *u8); off = wa_putn(buf, off, plen); off = wa_puts(buf, off, "\r\n" as *u8)
139 off = wa_puts(buf, off, "\r\n" as *u8)
140 var i: i64 = 0; while i < plen { buf[off] = payload[i]; off = off + 1; i = i + 1 }
141 off = wa_puts(buf, off, "\r\n\r\n" as *u8)
142 return off
143}
144// read the value of `field` in a record header [start,end) into out (until CR); returns len (0 if absent)
145func wa_hdr_str(warc: *u8, start: i64, end: i64, field: *u8, out: *u8, cap: i64) -> i64 {
146 let fl: i64 = wa_slen(field)
147 let p: i64 = wa_find(warc, end, field, fl, start)
148 if p < 0 { out[0] = 0 as u8; return 0 }
149 var i: i64 = p + fl; var o: i64 = 0
150 var go: i64 = 1
151 while go == 1 { if i >= end { go = 0 } else { let c: i64 = warc[i]&0xff; if c == 13 { go = 0 } else { if o < (cap-1) { out[o]=c as u8; o=o+1 } i=i+1 } } }
152 out[o] = 0 as u8
153 return o
154}
155func wa_target_uri(warc: *u8, start: i64, end: i64, out: *u8, cap: i64) -> i64 { return wa_hdr_str(warc, start, end, "WARC-Target-URI: " as *u8, out, cap) }
156// rating; defaults to "sfw" if a record has no X-Nishi-Rating (older records / wa_write_resource)
157func wa_rating(warc: *u8, start: i64, end: i64, out: *u8, cap: i64) -> i64 {
158 let r: i64 = wa_hdr_str(warc, start, end, "X-Nishi-Rating: " as *u8, out, cap)
159 if r == 0 { out[0]=115 as u8; out[1]=102 as u8; out[2]=119 as u8; out[3]=0 as u8; return 3 }
160 return r
161}
162// enumerate the index-th record: target URI + content-type + rating + payload length. Returns 1, or 0 past end.
163func wa_record_at(warc: *u8, warclen: i64, index: i64, out_uri: *u8, uricap: i64, out_ct: *u8, ctcap: i64, out_rating: *u8, rcap: i64, out_len: *i64) -> i64 {
164 var i: i64 = 0; var idx: i64 = 0
165 var go: i64 = 1
166 while go == 1 {
167 let rs: i64 = wa_find(warc, warclen, "WARC/1.0\r\n" as *u8, 10, i)
168 if rs < 0 { go = 0 }
169 else {
170 let hdr_end: i64 = wa_find(warc, warclen, "\r\n\r\n" as *u8, 4, rs)
171 if hdr_end < 0 { go = 0 }
172 else {
173 let clen: i64 = wa_field_num(warc, rs, hdr_end, "Content-Length: " as *u8)
174 let payload: i64 = hdr_end + 4
175 if idx == index { wa_target_uri(warc, rs, hdr_end, out_uri, uricap); wa_content_type(warc, rs, hdr_end, out_ct, ctcap); wa_rating(warc, rs, hdr_end, out_rating, rcap); out_len[0] = clen; return 1 }
176 idx = idx + 1
177 i = payload + clen + 4
178 }
179 }
180 }
181 return 0
182}