nx_crawlmine.nx source
↩ module page · 307 lines · 13545 B
1// nx_crawlmine.nx -- MINE the crawl outcome log so the crawler improves against MEASURED loss.
2//
3// WHY THIS EXISTS. Until R11 the crawler dropped pages silently: a Cloudflare interstitial, a 403, a
4// TLS failure and a genuinely empty host all ended the same way -- no line, no row, nothing to count.
5// The host-health streak then retired the host, which turns OUR capability gap into PERMANENT coverage
6// loss that looks like the web's fault. Operator 2026-08-06: "make sure success isnt accidental and
7// that we have an error log we are mining and improving our crawler on."
8//
9// This is the mining half. It answers the only questions that can drive the next rung:
10// - what fraction of fetches actually succeed (the rate, not the anecdote)
11// - which failure CLASS dominates (botwall vs http vs transport) = what to build next
12// - which HOSTS we are losing, and to what = whether escalation is working per-host
13//
14// A SUCCESS ROW IS AS LOAD-BEARING AS A FAILURE ROW: without `ok` counts there is no denominator, and
15// a failure count with no denominator is not a rate -- the same trap that once reported a fixed
16// crash-loop as live. That is why the crawler logs ok too.
17//
18// 2026-08-13 WHOLE-FILE + WINDOW. v1 read the FIRST 4MiB and reported that head as the population --
19// a silent cap: once the log outgrew it every run returned the same frozen numbers (rows=50427
20// forever) and the post-R12 window was unobservable by construction. Now the 4MiB buffer is a CHUNK,
21// the file streams to EOF, and coverage (bytes, rows, epoch span) is printed so a partial read can
22// never pass as a population again. [since-epoch] restricts TALLIES to rows at/after that epoch =
23// the windowed referee for before/after claims (ok_per_day, hostcap paid-fetch vs free-skip).
24// nx_crawlmine [log-path] [since-epoch]
25// license_tier: ORIGINAL expect_exit: 0
26import "nx_syscalls.nx"
27const CM_MAGIC_86400: i64 = 86400
28
29const CM_LOG: *u8 = "knowledge/status/crawl_outcomes.log"
30const CM_CAP: i64 = 4194304 // CHUNK buffer size, not a population cap -- the read loop streams to EOF
31const CM_MAXC: i64 = 16 // distinct classes tracked
32const CM_MAXH: i64 = 256 // distinct hosts tracked
33const CM_SLOT: i64 = 64
34
35func cm_w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
36func cm_n(v: i64) -> i64 {
37 let b: *u8=sys_mmap(32); var x: i64=v; var i: i64=31
38 if x==0 { b[i]=48 as u8; i=i-1 }
39 while x>0 { b[i]=(48+x%10) as u8; x=x/10; i=i-1 }
40 sys_write(1, ((b as i64)+i+1) as *u8, 31-i); return 0
41}
42// decimal value of the digit bytes in [s,e); non-digits are skipped (fields are tab-bounded upstream)
43func cm_num(b: *u8, s: i64, e: i64) -> i64 {
44 var v: i64 = 0
45 var i: i64 = s
46 while i < e {
47 let c: i64 = b[i] as i64
48 if c >= 48 { if c <= 57 { v = v * 10 + (c - 48) } }
49 i = i + 1
50 }
51 return v
52}
53func cm_eqn(a: *u8, al: i64, b: *u8, bl: i64) -> i64 {
54 if al != bl { return 0 }
55 var k: i64 = 0
56 while k < al { if a[k] != b[k] { return 0 } k = k + 1 }
57 return 1
58}
59// find-or-add a name in a flat slot table; returns its index or -1 when the table is full
60func cm_slot(tab: *u8, n: *i64, maxn: i64, s: *u8, sl: i64) -> i64 {
61 if sl <= 0 { return 0 - 1 }
62 if sl >= CM_SLOT - 1 { return 0 - 1 }
63 var i: i64 = 0
64 while i < n[0] {
65 let p: *u8 = (tab as i64 + i * CM_SLOT) as *u8
66 var pl: i64 = 0
67 while p[pl] != (0 as u8) { pl = pl + 1 }
68 if cm_eqn(p, pl, s, sl) == 1 { return i }
69 i = i + 1
70 }
71 if n[0] >= maxn { return 0 - 1 }
72 let d: *u8 = (tab as i64 + n[0] * CM_SLOT) as *u8
73 var c: i64 = 0
74 while c < sl { d[c] = s[c]; c = c + 1 }
75 d[sl] = 0 as u8
76 let idx: i64 = n[0]
77 n[0] = n[0] + 1
78 return idx
79}
80// host = the span between "//" and the next '/'
81func cm_host(u: *u8, us: i64, ue: i64, hb: *u8) -> i64 {
82 var i: i64 = us
83 var st: i64 = 0 - 1
84 while i + 1 < ue { if u[i] == (47 as u8) { if u[i+1] == (47 as u8) { st = i + 2; i = ue } else { i = i + 1 } } else { i = i + 1 } }
85 if st < 0 { return 0 }
86 var e: i64 = st
87 while e < ue { if u[e] == (47 as u8) { e = ue } else { e = e + 1 } }
88 var he: i64 = st
89 while he < ue { if u[he] == (47 as u8) { he = ue + 1 } else { he = he + 1 } }
90 var end: i64 = st
91 var f: i64 = 0
92 while f == 0 { if end >= ue { f = 1 } else { if u[end] == (47 as u8) { f = 1 } else { end = end + 1 } } }
93 var n: i64 = 0
94 var k: i64 = st
95 while k < end { if n < CM_SLOT - 1 { hb[n] = u[k]; n = n + 1 } k = k + 1 }
96 hb[n] = 0 as u8
97 return n
98}
99// parse every complete line in buf[0,end): tally classes/hosts for rows with epoch >= since.
100// ctx = pointer bundle {ctab,ccnt,cn,htab,hbad,hall,hn,hb}; st = counters {rows_all, rows_win,
101// file_first_ep, file_last_ep, win_first_ep, win_last_ep, hostcap_paid, hostcap_free,
102// dropped_class_rows, dropped_host_rows}
103func cm_region(buf: *u8, end: i64, since: i64, ctx: *i64, st: *i64) -> i64 {
104 let ctab: *u8 = ctx[0] as *u8
105 let ccnt: *i64 = ctx[1] as *i64
106 let cn: *i64 = ctx[2] as *i64
107 let htab: *u8 = ctx[3] as *u8
108 let hbad: *i64 = ctx[4] as *i64
109 let hall: *i64 = ctx[5] as *i64
110 let hn: *i64 = ctx[6] as *i64
111 let hb: *u8 = ctx[7] as *u8
112 let hcl: *u8 = "hostcap" as *u8
113 var i: i64 = 0
114 while i < end {
115 let ls: i64 = i
116 var le: i64 = ls
117 var s: i64 = 1
118 while s == 1 { if le >= end { s = 0 } else { if buf[le] == (10 as u8) { s = 0 } else { le = le + 1 } } }
119 i = le + 1
120 if le - ls > 8 {
121 // epoch \t class \t status \t bytes \t url
122 var f1: i64 = ls
123 var g: i64 = 0
124 while g == 0 { if f1 >= le { g = 1 } else { if buf[f1] == (9 as u8) { g = 1 } else { f1 = f1 + 1 } } }
125 var f2: i64 = f1 + 1
126 g = 0
127 while g == 0 { if f2 >= le { g = 1 } else { if buf[f2] == (9 as u8) { g = 1 } else { f2 = f2 + 1 } } }
128 let cls_s: i64 = f1 + 1
129 let cls_l: i64 = f2 - cls_s
130 if cls_l > 0 {
131 let ep: i64 = cm_num(buf, ls, f1)
132 st[0] = st[0] + 1
133 if ep > 0 { if st[2] == 0 { st[2] = ep } st[3] = ep }
134 if ep >= since {
135 st[1] = st[1] + 1
136 if ep > 0 { if st[4] == 0 { st[4] = ep } st[5] = ep }
137 let ci: i64 = cm_slot(ctab, cn, CM_MAXC, (buf as i64 + cls_s) as *u8, cls_l)
138 if ci >= 0 { ccnt[ci] = ccnt[ci] + 1 } else { st[8] = st[8] + 1 }
139 // status = the 3rd field; for hostcap rows 0 = R12 selection-time FREE skip,
140 // nonzero = the pre-R12 PAID fetch the fix exists to eliminate
141 var f3: i64 = f2 + 1
142 g = 0
143 while g == 0 { if f3 >= le { g = 1 } else { if buf[f3] == (9 as u8) { g = 1 } else { f3 = f3 + 1 } } }
144 if cm_eqn((buf as i64 + cls_s) as *u8, cls_l, hcl, 7) == 1 {
145 let hst: i64 = cm_num(buf, f2 + 1, f3)
146 if hst == 0 { st[7] = st[7] + 1 } else { st[6] = st[6] + 1 }
147 }
148 // url = after the 4th tab
149 var f4: i64 = f3 + 1
150 g = 0
151 while g == 0 { if f4 >= le { g = 1 } else { if buf[f4] == (9 as u8) { g = 1 } else { f4 = f4 + 1 } } }
152 if f4 < le {
153 let hl: i64 = cm_host(buf, f4 + 1, le, hb)
154 if hl > 0 {
155 let hi: i64 = cm_slot(htab, hn, CM_MAXH, hb, hl)
156 if hi >= 0 {
157 hall[hi] = hall[hi] + 1
158 var isok: i64 = 0
159 if cls_l == 2 { if buf[cls_s] == (111 as u8) { if buf[cls_s+1] == (107 as u8) { isok = 1 } } }
160 if isok == 0 { hbad[hi] = hbad[hi] + 1 }
161 } else { st[9] = st[9] + 1 }
162 }
163 }
164 }
165 }
166 }
167 }
168 return 0
169}
170
171func main(argc: i64, argv: *i64) -> i64 {
172 var path: *u8 = CM_LOG
173 if argc >= 2 { path = argv[1] as *u8 }
174 var since: i64 = 0
175 if argc >= 3 {
176 let sa: *u8 = argv[2] as *u8
177 var sl: i64 = 0
178 while sa[sl] != (0 as u8) { sl = sl + 1 }
179 since = cm_num(sa, 0, sl)
180 }
181 let buf: *u8 = sys_mmap(CM_CAP)
182 let fd: i64 = sys_openat_rd(path)
183 if fd < 0 {
184 // ABSENT is a REAL answer, not an error: it means no crawl has run since the log shipped.
185 cm_w("CRAWLMINE: no outcome log at " as *u8); cm_w(path)
186 cm_w(" -- nothing mined yet (run nx_web_crawl_step first)\n" as *u8)
187 return 0
188 }
189 let ctab: *u8 = sys_mmap(CM_SLOT * CM_MAXC)
190 let ccnt: *i64 = sys_mmap(8 * CM_MAXC) as *i64
191 let cn: *i64 = sys_mmap(16) as *i64
192 cn[0] = 0
193 let htab: *u8 = sys_mmap(CM_SLOT * CM_MAXH)
194 let hbad: *i64 = sys_mmap(8 * CM_MAXH) as *i64
195 let hall: *i64 = sys_mmap(8 * CM_MAXH) as *i64
196 let hn: *i64 = sys_mmap(16) as *i64
197 hn[0] = 0
198 var z: i64 = 0
199 while z < CM_MAXC { ccnt[z] = 0; z = z + 1 }
200 z = 0
201 while z < CM_MAXH { hbad[z] = 0; hall[z] = 0; z = z + 1 }
202 let hb: *u8 = sys_mmap(CM_SLOT)
203 let ctx: *i64 = sys_mmap(8 * 8) as *i64
204 ctx[0] = ctab as i64
205 ctx[1] = ccnt as i64
206 ctx[2] = cn as i64
207 ctx[3] = htab as i64
208 ctx[4] = hbad as i64
209 ctx[5] = hall as i64
210 ctx[6] = hn as i64
211 ctx[7] = hb as i64
212 let st: *i64 = sys_mmap(8 * 12) as *i64
213 z = 0
214 while z < 12 { st[z] = 0; z = z + 1 }
215
216 var total: i64 = 0
217 var oversize: i64 = 0
218 var rem: i64 = 0
219 var reading: i64 = 1
220 while reading == 1 {
221 let r: i64 = sys_read(fd, ((buf as i64) + rem) as *u8, CM_CAP - rem)
222 if r <= 0 {
223 reading = 0
224 // a final unterminated line still counts -- the tail is the present, never drop it
225 if rem > 8 { cm_region(buf, rem, since, ctx, st) }
226 rem = 0
227 } else {
228 total = total + r
229 let n: i64 = rem + r
230 var ln: i64 = n - 1
231 var f: i64 = 0
232 while f == 0 { if ln < 0 { f = 1 } else { if buf[ln] == (10 as u8) { f = 1 } else { ln = ln - 1 } } }
233 if ln < 0 {
234 // a full chunk with no newline = an oversize line: drop it ANNOUNCED, never stall
235 if n >= CM_CAP { oversize = oversize + 1; rem = 0 } else { rem = n }
236 } else {
237 cm_region(buf, ln + 1, since, ctx, st)
238 var k: i64 = ln + 1
239 var d: i64 = 0
240 while k < n { buf[d] = buf[k]; d = d + 1; k = k + 1 }
241 rem = d
242 }
243 }
244 }
245 sys_close(fd)
246 if total <= 0 { cm_w("CRAWLMINE: outcome log is EMPTY\n" as *u8); return 0 }
247
248 let rows: i64 = st[1]
249 var okc: i64 = 0
250 var ci2: i64 = 0
251 while ci2 < cn[0] {
252 let p: *u8 = (ctab as i64 + ci2 * CM_SLOT) as *u8
253 if p[0] == (111 as u8) { if p[1] == (107 as u8) { if p[2] == (0 as u8) { okc = ccnt[ci2] } } }
254 ci2 = ci2 + 1
255 }
256 var permil: i64 = 0
257 if rows > 0 { permil = (okc * 1000) / rows }
258
259 cm_w("=== nx_crawlmine: crawl outcomes ===\n" as *u8)
260 cm_w("rows=" as *u8); cm_n(rows)
261 cm_w(" ok=" as *u8); cm_n(okc)
262 cm_w(" success_permil=" as *u8); cm_n(permil)
263 cm_w(" (a failure count with no denominator is not a rate -- ok rows ARE the denominator)\n" as *u8)
264 cm_w("-- by class (what to BUILD next) --\n" as *u8)
265 ci2 = 0
266 while ci2 < cn[0] {
267 cm_w(" " as *u8)
268 cm_w((ctab as i64 + ci2 * CM_SLOT) as *u8)
269 cm_w(" = " as *u8); cm_n(ccnt[ci2]); cm_w("\n" as *u8)
270 ci2 = ci2 + 1
271 }
272 cm_w("-- hosts we are LOSING (bad/total; escalation is working where bad is 0) --\n" as *u8)
273 var shown: i64 = 0
274 var hi2: i64 = 0
275 while hi2 < hn[0] {
276 if hbad[hi2] > 0 { if shown < 20 {
277 cm_w(" " as *u8)
278 cm_w((htab as i64 + hi2 * CM_SLOT) as *u8)
279 cm_w(" " as *u8); cm_n(hbad[hi2]); cm_w("/" as *u8); cm_n(hall[hi2]); cm_w("\n" as *u8)
280 shown = shown + 1
281 } }
282 hi2 = hi2 + 1
283 }
284 if shown == 0 { cm_w(" (none -- every logged host produced at least one indexed page)\n" as *u8) }
285 cm_w("-- coverage (whole file streamed; a capped read is not a population) --\n" as *u8)
286 cm_w(" bytes=" as *u8); cm_n(total)
287 cm_w(" rows_total=" as *u8); cm_n(st[0])
288 cm_w(" window_rows=" as *u8); cm_n(st[1])
289 cm_w(" since=" as *u8); cm_n(since)
290 cm_w(" oversize_dropped=" as *u8); cm_n(oversize)
291 cm_w("\n file_first_epoch=" as *u8); cm_n(st[2])
292 cm_w(" file_last_epoch=" as *u8); cm_n(st[3])
293 cm_w(" win_first=" as *u8); cm_n(st[4])
294 cm_w(" win_last=" as *u8); cm_n(st[5])
295 cm_w("\n" as *u8)
296 var span: i64 = st[5] - st[4]
297 if span < 1 { span = 1 }
298 cm_w(" ok_per_day=" as *u8); cm_n((okc * CM_MAGIC_86400) / span)
299 cm_w(" hostcap_paid_fetch=" as *u8); cm_n(st[6])
300 cm_w(" hostcap_free_skip=" as *u8); cm_n(st[7])
301 cm_w(" dropped_class_rows=" as *u8); cm_n(st[8])
302 cm_w(" dropped_host_rows=" as *u8); cm_n(st[9])
303 cm_w("\n" as *u8)
304 cm_w("HONEST: this mines DECISIONS, not truth. A botwall row means we were challenged, not that the\n" as *u8)
305 cm_w("page is worthless; a host at 0/N may simply not have been reached yet this cycle.\n" as *u8)
306 return 0
307}