code wiki / _hdl_build / nx_bench_warc.nx
nx_bench_warc.nx source
↩ module page · 269 lines · 12047 B
1// nx_bench_warc.nx -- BENCHMARK CORPUS CAPTURE RUNNER: a URL-list file -> render-fetched -> ONE durable WARC
2// at a CALLER-CHOSEN path -> reloaded and verified retrievable by URL -> ts'd verdict line appended to
3// knowledge/status/bench_warc.log. Composes bf_fetch_follow (render front) + nx_web_archive (store); adds NO
4// new mechanism. Exists because nx_crawl_preserve pins one output path + a 32MiB cap, so it cannot hold two
5// vertical corpora side by side (archive-as-oracle standard: every benchmark corpus is a reserved WARC).
6// usage: nx_bench_warc <urlfile> <warcpath> [max] (urlfile: one URL per line; blank/# lines skipped)
7// license_tier: ORIGINAL
8import "nx_syscalls.nx"
9import "nx_x509_trust_store.nx"
10import "nx_trust_store_load_from_certdata.nx"
11import "nx_csprng.nx"
12import "nx_browser_fetch.nx"
13import "nx_web_archive.nx"
14
15const BW_CERTCAP: i64 = 4194304
16const BW_LOGCAP: i64 = 1024
17const BW_REDIR: i64 = 5
18const BW_MODE: i64 = 420
19const BW_CONFCAP: i64 = 8192
20// LAST-RESORT code defaults ONLY (config hierarchy: knowledge/bench_warc.conf wins). Every one is a
21// REFUSAL threshold, never a silent truncation point -- exceeding any of them makes the run RED and
22// names exactly what was dropped (rule 11 no-magic-numbers + memfloor REFUSE-never-shrink).
23const BW_D_RESPCAP: i64 = 33554432
24const BW_D_WARCCAP: i64 = 805306368
25const BW_D_LISTCAP: i64 = 4194304
26const BW_D_URLCAP: i64 = 4096
27const BW_D_MAXURLS: i64 = 100000
28const BW_HDRSLACK: i64 = 4096
29
30func bw_w(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
31func bw_sl(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
32func bw_n(v: i64) -> i64 {
33 let b: *u8 = sys_mmap(24)
34 let t: *u8 = sys_mmap(24)
35 if (b as i64) == 0 { return 0 }
36 if (t as i64) == 0 { return 0 }
37 var m: i64 = v
38 var k: i64 = 0
39 if m == 0 { t[0] = 48 as u8; k = 1 }
40 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
41 var i: i64 = 0
42 while i < k { b[i] = t[k-1-i]; i = i + 1 }
43 sys_write(1, b, k)
44 return 0
45}
46func bw_catn(buf: *u8, off: i64, v: i64) -> i64 {
47 let t: *u8 = sys_mmap(24)
48 if (t as i64) == 0 { return off }
49 var m: i64 = v
50 var k: i64 = 0
51 if m == 0 { t[0] = 48 as u8; k = 1 }
52 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
53 var o: i64 = off
54 var i: i64 = 0
55 while i < k { buf[o] = t[k-1-i]; o = o + 1; i = i + 1 }
56 return o
57}
58func bw_cat(buf: *u8, off: i64, s: *u8) -> i64 {
59 var o: i64 = off
60 var i: i64 = 0
61 while s[i] != (0 as u8) { buf[o] = s[i]; o = o + 1; i = i + 1 }
62 return o
63}
64func bw_readfile(path: *u8, buf: *u8, cap: i64) -> i64 {
65 let fd: i64 = sys_openat_rd(path)
66 if fd < 0 { return 0 - 1 }
67 let n: i64 = sys_read(fd, buf, cap)
68 sys_close(fd)
69 return n
70}
71// copy line [start,end) into url (NUL-terminated); returns length, or 0 for blank/comment
72func bw_line(list: *u8, start: i64, end: i64, url: *u8, urlcap: i64) -> i64 {
73 var s: i64 = start
74 var e: i64 = end
75 while s < e { if (list[s]&0xff) <= 32 { s = s + 1 } else { s = e + 1 } }
76 if s > e { s = s - (e + 1) + start }
77 var i: i64 = start
78 var o: i64 = 0
79 var seen: i64 = 0
80 while i < end {
81 let c: i64 = list[i] & 0xff
82 if c > 32 { seen = 1 }
83 if seen == 1 { if c >= 32 { if o < (urlcap - 1) { url[o] = c as u8; o = o + 1 } } }
84 i = i + 1
85 }
86 while o > 0 { if (url[o-1]&0xff) <= 32 { o = o - 1 } else { url[o] = 0 as u8; return o } }
87 url[o] = 0 as u8
88 if o == 0 { return 0 }
89 if (url[0]&0xff) == 35 { return 0 }
90 return o
91}
92
93// read an i64 value for `key` from a key=value config buffer; returns dflt when absent/unparsable.
94func bw_conf(buf: *u8, n: i64, key: *u8, dflt: i64) -> i64 {
95 let kl: i64 = bw_sl(key)
96 var i: i64 = 0
97 while i < n {
98 var atline: i64 = 1
99 if i > 0 { if (buf[i-1]&0xff) != 10 { atline = 0 } }
100 if atline == 1 {
101 var j: i64 = 0
102 var hit: i64 = 1
103 while j < kl { if (i+j) >= n { hit = 0; j = kl } else { if (buf[i+j]&0xff) != (key[j]&0xff) { hit = 0; j = kl } else { j = j + 1 } } }
104 if hit == 1 { if (i+kl) < n { if (buf[i+kl]&0xff) == 61 {
105 var v: i64 = 0
106 var got: i64 = 0
107 var p: i64 = i + kl + 1
108 while p < n { let c: i64 = buf[p]&0xff; if c >= 48 { if c <= 57 { v = v*10 + (c-48); got = 1; p = p + 1 } else { p = n } } else { p = n } }
109 if got == 1 { return v }
110 } } }
111 }
112 i = i + 1
113 }
114 return dflt
115}
116
117func main(argc: i64, argv: *i64) -> i64 {
118 if argc < 3 { bw_w("usage: nx_bench_warc <urlfile> <warcpath> [max]\n" as *u8); return 1 }
119 let av: *i64 = ((argv as i64) + 8) as *i64
120 let urlfile: *u8 = (av[0]) as *u8
121 let warcpath: *u8 = (av[1]) as *u8
122 // LIMITS ARE DATA: config wins, code defaults are last resort, every limit REFUSES rather than truncates
123 let confb: *u8 = sys_mmap(BW_CONFCAP)
124 if (confb as i64) == 0 { return 8 }
125 var confn: i64 = bw_readfile("knowledge/bench_warc.conf" as *u8, confb, BW_CONFCAP)
126 if confn < 0 { confn = 0 }
127 let listcap: i64 = bw_conf(confb, confn, "list_cap" as *u8, BW_D_LISTCAP)
128 let warccap: i64 = bw_conf(confb, confn, "warc_cap" as *u8, BW_D_WARCCAP)
129 let urlcap: i64 = bw_conf(confb, confn, "url_cap" as *u8, BW_D_URLCAP)
130 let respcap: i64 = bw_conf(confb, confn, "resp_cap" as *u8, BW_D_RESPCAP)
131 var maxn: i64 = bw_conf(confb, confn, "max_urls" as *u8, BW_D_MAXURLS)
132 if argc >= 4 {
133 let ms: *u8 = (av[2]) as *u8
134 var v: i64 = 0
135 var i: i64 = 0
136 while ms[i] != (0 as u8) { let c: i64 = ms[i]&0xff; if c >= 48 { if c <= 57 { v = v*10 + (c-48) } } i = i + 1 }
137 if v > 0 { maxn = v }
138 }
139 let list: *u8 = sys_mmap(listcap)
140 if (list as i64) == 0 { bw_w("BENCH-WARC alloc list failed\n" as *u8); return 8 }
141 let ln: i64 = bw_readfile(urlfile, list, listcap)
142 if ln <= 0 { bw_w("BENCH-WARC cannot read urlfile\n" as *u8); return 6 }
143 // a read that exactly fills the buffer is INDISTINGUISHABLE from a truncated one => REFUSE, never guess
144 if ln >= listcap { bw_w("BENCH-WARC REFUSED: urlfile filled list_cap (" as *u8); bw_n(listcap); bw_w(" B) -- raise list_cap in knowledge/bench_warc.conf; NOT truncating silently\n" as *u8); return 5 }
145 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt" as *u8, 512, BW_CERTCAP)
146 if r <= 0 { bw_w("BENCH-WARC certdata load failed (cwd must be nishihost)\n" as *u8); return 7 }
147 let store: *TrustStore = r as *TrustStore
148 let cr: *u8 = sys_mmap(32)
149 let pk: *u8 = sys_mmap(32)
150 if (cr as i64) == 0 { return 8 }
151 if (pk as i64) == 0 { return 8 }
152 nx_csprng_fill(cr, 32)
153 nx_csprng_fill(pk, 32)
154 let now: i64 = sys_now_realtime_sec()
155 let resp: *u8 = sys_mmap(respcap)
156 let warc: *u8 = sys_mmap(warccap)
157 let url: *u8 = sys_mmap(urlcap)
158 if (resp as i64) == 0 { return 8 }
159 if (warc as i64) == 0 { return 8 }
160 if (url as i64) == 0 { return 8 }
161 var off: i64 = 0
162 var tried: i64 = 0
163 var preserved: i64 = 0
164 var urls_total: i64 = 0
165 var skipped_full: i64 = 0
166 var pos: i64 = 0
167 bw_w("== nx_bench_warc " as *u8); bw_w(urlfile); bw_w(" -> " as *u8); bw_w(warcpath); bw_w(" ==\n" as *u8)
168 while pos < ln {
169 var e: i64 = pos
170 while e < ln { if (list[e]&0xff) == 10 { e = ln + 1 } else { e = e + 1 } }
171 var lend: i64 = e
172 if e > ln { lend = e - ln - 1 + pos }
173 var stop: i64 = lend
174 if stop > ln { stop = ln }
175 var scan: i64 = pos
176 var nl: i64 = 0 - 1
177 while scan < ln { if (list[scan]&0xff) == 10 { nl = scan; scan = ln } else { scan = scan + 1 } }
178 var lineend: i64 = nl
179 if nl < 0 { lineend = ln }
180 let ul: i64 = bw_line(list, pos, lineend, url, urlcap)
181 if ul > 0 {
182 urls_total = urls_total + 1
183 if tried < maxn {
184 tried = tried + 1
185 let n: i64 = bf_fetch_follow(url, store, cr, pk, now, resp, respcap, BW_REDIR)
186 let st: i64 = bf_status(resp, n)
187 let boff: i64 = bf_body_off(resp, n)
188 let blen: i64 = n - boff
189 bw_w(" [" as *u8); bw_n(st); bw_w("] " as *u8); bw_n(blen); bw_w("B " as *u8); bw_w(url); bw_w("\n" as *u8)
190 if st == 200 { if blen > 0 {
191 // capacity REFUSAL, not silent drop: a record that will not fit is COUNTED and NAMED
192 if (off + blen + BW_HDRSLACK) >= warccap {
193 skipped_full = skipped_full + 1
194 bw_w(" !! warc_cap reached -- NOT archived: " as *u8); bw_w(url); bw_w("\n" as *u8)
195 } else {
196 let body: *u8 = ((resp as i64) + boff) as *u8
197 off = wa_write_resource(warc, off, url, "2026-08-05T00:00:00Z" as *u8, "text/html" as *u8, body, blen)
198 preserved = preserved + 1
199 }
200 } }
201 }
202 }
203 if nl < 0 { pos = ln } else { pos = nl + 1 }
204 }
205 wa_save(warcpath, warc, off)
206 let recs: i64 = wa_count(warc, off)
207 // durability + retrievability: reload from disk and look every preserved URL back up
208 let warc2: *u8 = sys_mmap(warccap)
209 if (warc2 as i64) == 0 { return 8 }
210 let n2: i64 = wa_load(warcpath, warc2, warccap)
211 let op: *i64 = sys_mmap(8) as *i64
212 let ol: *i64 = sys_mmap(8) as *i64
213 if (op as i64) == 0 { return 8 }
214 if (ol as i64) == 0 { return 8 }
215 var retrievable: i64 = 0
216 pos = 0
217 var checked: i64 = 0
218 while pos < ln {
219 var scan2: i64 = pos
220 var nl2: i64 = 0 - 1
221 while scan2 < ln { if (list[scan2]&0xff) == 10 { nl2 = scan2; scan2 = ln } else { scan2 = scan2 + 1 } }
222 var lineend2: i64 = nl2
223 if nl2 < 0 { lineend2 = ln }
224 let ul2: i64 = bw_line(list, pos, lineend2, url, urlcap)
225 if ul2 > 0 { if checked < maxn {
226 checked = checked + 1
227 if wa_lookup(warc2, n2, url, bw_sl(url), op, ol) == 1 { retrievable = retrievable + 1 }
228 } }
229 if nl2 < 0 { pos = ln } else { pos = nl2 + 1 }
230 }
231 let logb: *u8 = sys_mmap(BW_LOGCAP)
232 if (logb as i64) == 0 { return 8 }
233 var o: i64 = 0
234 o = bw_cat(logb, o, "BENCH-WARC ts=" as *u8)
235 o = bw_catn(logb, o, now)
236 o = bw_cat(logb, o, " list=" as *u8)
237 o = bw_cat(logb, o, urlfile)
238 o = bw_cat(logb, o, " warc=" as *u8)
239 o = bw_cat(logb, o, warcpath)
240 o = bw_cat(logb, o, " urls_total=" as *u8)
241 o = bw_catn(logb, o, urls_total)
242 o = bw_cat(logb, o, " tried=" as *u8)
243 o = bw_catn(logb, o, tried)
244 o = bw_cat(logb, o, " over_max=" as *u8)
245 o = bw_catn(logb, o, urls_total - tried)
246 o = bw_cat(logb, o, " skipped_warc_full=" as *u8)
247 o = bw_catn(logb, o, skipped_full)
248 o = bw_cat(logb, o, " preserved=" as *u8)
249 o = bw_catn(logb, o, preserved)
250 o = bw_cat(logb, o, " records=" as *u8)
251 o = bw_catn(logb, o, recs)
252 o = bw_cat(logb, o, " bytes=" as *u8)
253 o = bw_catn(logb, o, off)
254 o = bw_cat(logb, o, " retrievable=" as *u8)
255 o = bw_catn(logb, o, retrievable)
256 o = bw_cat(logb, o, " verdict=" as *u8)
257 // GREEN requires: every preserved record retrievable AND nothing dropped by a capacity limit AND
258 // no URL left untried by max_urls -- a limit that silently shrinks the corpus can never read GREEN.
259 var green: i64 = 0
260 if retrievable == preserved { if preserved > 0 { if skipped_full == 0 { if urls_total == tried { green = 1 } } } }
261 if green == 1 { o = bw_cat(logb, o, "GREEN" as *u8) } else { o = bw_cat(logb, o, "RED" as *u8) }
262 logb[o] = 10 as u8
263 o = o + 1
264 let lfd: i64 = sys_openat_append("knowledge/status/bench_warc.log" as *u8, BW_MODE)
265 if lfd >= 0 { sys_write(lfd, logb, o); sys_close(lfd) }
266 sys_write(1, logb, o)
267 if green == 1 { return 0 }
268 return 9
269}