nx_crawlaudit.nx source
↩ module page · 243 lines · 13493 B
1// nx_crawlaudit.nx -- INDEPENDENT validator for the crawl outcome log. The anti-navel-gazing organ.
2//
3// WHY IT EXISTS. nx_crawlmine reads OUR OWN log, so it can only ever confirm what the crawler already
4// believes. If the classifier is wrong, the miner reports the wrong thing CONFIDENTLY -- a closed loop
5// that measures itself and calls the result evidence. Operator 2026-08-06: "a secondary system to
6// validate we arent navel gazing like we have in all kinds of other systems."
7//
8// INDEPENDENCE IS THE ENTIRE POINT, so this organ deliberately shares NO decision logic with the
9// crawler. The crawler classifies a bot-wall by PHRASE MATCH ("Just a moment", "Attention Required").
10// Re-using those phrases here would be circular -- it would agree with itself by construction and
11// prove nothing. Instead this judges by TEXT YIELD: strip the HTML and measure how much prose comes
12// out. A challenge interstitial yields almost nothing; a real page yields thousands of characters.
13// Different mechanism, different failure modes, so agreement is INFORMATIVE.
14//
15// SEPARATING FLAKINESS FROM MISCLASSIFICATION. A re-fetch happens at a different time, so disagreement
16// alone proves nothing -- MEASURED on this very estate: the same v2ph url ingested 2,176 chars one
17// night and returned a 403 challenge the next morning. So each sampled url is fetched TWICE in this
18// run: if the two fetches disagree WITH EACH OTHER the HOST is unstable (not our bug); if they agree
19// with each other but disagree with what we RECORDED, that is a classifier candidate worth reading.
20// u2605A DISAGREEMENT IS A FLAG, NEVER A VERDICT -- and a validator that forgets that just relocates the
21// overconfidence it was built to catch.
22// nx_crawlaudit [log-path] [max-samples]
23// license_tier: ORIGINAL expect_exit: 0
24import "nx_research_engine.nx" // rf_init -> trust store, and the shared fetch ladder
25import "nx_html_to_text.nx" // the INDEPENDENT signal: prose yield, not phrase matching
26const CA_MAGIC_4096: i64 = 4096
27const CA_MAGIC_4090: i64 = 4090
28
29const CA_LOG: *u8 = "knowledge/status/crawl_outcomes.log"
30const CA_CAP: i64 = 4194304
31const CA_FETCH: i64 = 1048576
32const CA_TEXT: i64 = 262144
33const CA_MAXS: i64 = 8 // declared: sampled urls per run (each costs 2 live fetches)
34// DECLARED and deliberately NOT the crawler's WC_MININDEX: importing its threshold would re-couple the
35// two judgements. This is what THIS organ calls "a page with prose on it".
36const CA_TEXTMIN: i64 = 400
37const CA_MAXROWS: i64 = 65536 // declared: line-index capacity; a longer log is sampled from its newest rows
38
39func ca_w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
40func ca_n(v: i64) -> i64 {
41 let b: *u8=sys_mmap(32); var x: i64=v; var i: i64=31
42 if x<0 { x=0-x }
43 if x==0 { b[i]=48 as u8; i=i-1 }
44 while x>0 { b[i]=(48+x%10) as u8; x=x/10; i=i-1 }
45 if v<0 { b[i]=45 as u8; i=i-1 }
46 sys_write(1, ((b as i64)+i+1) as *u8, 31-i); return 0
47}
48// one INDEPENDENT observation: 1 = looks like CONTENT, 0 = looks like a WALL/empty
49func ca_observe(store: *TrustStore, url: *u8, stbox: *i64, txbox: *i64) -> i64 {
50 let out: *u8 = sys_mmap(CA_FETCH)
51 let st: *i64 = sys_mmap(16) as *i64
52 st[0] = 0
53 let n: i64 = nx_https_fetch_follow(url, store, out, CA_FETCH, 6, st)
54 stbox[0] = st[0]
55 txbox[0] = 0
56 if n <= 0 { sys_munmap(out, CA_FETCH); return 0 }
57 let txt: *u8 = sys_mmap(CA_TEXT)
58 let tn: i64 = nx_html_to_text(out, n, txt, CA_TEXT)
59 txbox[0] = tn
60 sys_munmap(out, CA_FETCH)
61 sys_munmap(txt, CA_TEXT)
62 if st[0] != 200 { return 0 }
63 if tn < CA_TEXTMIN { return 0 }
64 return 1
65}
66
67func main(argc: i64, argv: *i64) -> i64 {
68 var path: *u8 = CA_LOG
69 if argc >= 2 { path = argv[1] as *u8 }
70 var maxs: i64 = CA_MAXS
71 if argc >= 3 {
72 let a: *u8 = argv[2] as *u8
73 var v: i64 = 0; var k: i64 = 0
74 while a[k] != (0 as u8) { let c: i64 = a[k] as i64; if c>=48 { if c<=57 { v = v*10 + (c-48) } } k = k+1 }
75 if v > 0 { maxs = v }
76 }
77 // argv[3] = audit ONE class. False positives do not distribute evenly: they concentrate in
78 // the classes that DROP pages (botwall/thin/http), so an untargeted newest-first sample can
79 // miss the very class under suspicion -- which is exactly what happened on the first run.
80 var clsfilt: *u8 = 0 as *u8
81 if argc >= 4 { clsfilt = argv[3] as *u8 }
82 let buf: *u8 = sys_mmap(CA_CAP)
83 let fd: i64 = sys_openat_rd(path)
84 if fd < 0 { ca_w("CRAWLAUDIT: no outcome log -- nothing to validate (run the crawler first)\n" as *u8); return 0 }
85 // TAIL, NOT HEAD (2026-09-03). This read started at byte 0 and stopped at CA_CAP, and the report
86 // then printed "sampling newest-first". MEASURED on the live log: 129460940 bytes indexed as 50427
87 // rows -- 50427 rows of ~83 bytes IS CA_CAP, i.e. the OLDEST 3.2 percent, described as the newest.
88 // An append-only log's head is its PAST, and every classifier change ships after it, so the one
89 // slice guaranteed NOT to exercise current behaviour was the only slice ever audited. A validator
90 // built to stop navel-gazing was reading a fossil and calling it evidence.
91 let flen: i64 = sys_lseek(fd, 0, 2)
92 var startoff: i64 = 0
93 if flen > CA_CAP { startoff = flen - CA_CAP }
94 sys_lseek(fd, startoff, 0)
95 var n: i64 = 0
96 var go: i64 = 1
97 while go == 1 {
98 let r: i64 = sys_read(fd, ((buf as i64)+n) as *u8, CA_CAP-n)
99 if r <= 0 { go = 0 } else { n = n + r }
100 if n >= CA_CAP { go = 0 }
101 }
102 sys_close(fd)
103 // A seeked window almost never lands on a row boundary, so DROP the partial leading row rather
104 // than parse half a record into a class count.
105 if startoff > 0 {
106 var lead: i64 = 0
107 var q: i64 = 0
108 var fnd: i64 = 0
109 while fnd == 0 { if q >= n { fnd = 1 } else { if buf[q] == (10 as u8) { lead = q + 1; fnd = 1 } else { q = q + 1 } } }
110 if lead > 0 {
111 var mv: i64 = 0
112 while mv < n - lead { buf[mv] = buf[mv + lead]; mv = mv + 1 }
113 n = n - lead
114 }
115 }
116 // COVERAGE TRAVELS WITH THE SAMPLE: a window published without its denominator reads as a population.
117 ca_w("COVERAGE log_bytes=" as *u8); ca_n(flen); ca_w(" window_bytes=" as *u8); ca_n(n)
118 if startoff > 0 { ca_w(" scope=TAIL partial=1 (older rows NOT scanned -- a declared window, never the population)\n" as *u8) } else { ca_w(" scope=FULL partial=0\n" as *u8) }
119 if n <= 0 { ca_w("CRAWLAUDIT: outcome log EMPTY -- nothing to validate\n" as *u8); return 0 }
120
121 let store: *TrustStore = rf_init()
122 if (store as i64) == 0 { ca_w("CRAWLAUDIT: no trust store -- cannot fetch independently\n" as *u8); sys_exit(1); return 1 }
123
124 ca_w("=== nx_crawlaudit: INDEPENDENT re-check of logged crawl decisions ===\n" as *u8)
125 ca_w("signal: TEXT YIELD (>=" as *u8); ca_n(CA_TEXTMIN)
126 ca_w(" chars of stripped prose + status 200), deliberately NOT the crawler's phrase matcher\n" as *u8)
127
128 var agree: i64 = 0
129 var disagree: i64 = 0
130 var unstable: i64 = 0
131 var sampled: i64 = 0
132 // TWO-PASS, because the first cut walked BACKWARDS through the buffer and its line-start search
133 // collapsed to 0 on the first iteration -- it reported sampled=1 out of 4 requested and never
134 // reached the class under suspicion. A sampler that silently under-samples is worse than none:
135 // it produces a confident clean bill of health from one row.
136 // u2605AN AUDITOR THAT CANNOT PROVE IT SAMPLED WHAT IT CLAIMED IS ITSELF NAVEL-GAZING.
137 // Pass 1 records every line start; pass 2 walks them NEWEST-FIRST.
138 let starts: *i64 = sys_mmap(8 * CA_MAXROWS) as *i64
139 var nlines: i64 = 0
140 var p: i64 = 0
141 var at_start: i64 = 1
142 while p < n {
143 if at_start == 1 { if nlines < CA_MAXROWS { starts[nlines] = p; nlines = nlines + 1 } at_start = 0 }
144 if buf[p] == (10 as u8) { at_start = 1 }
145 p = p + 1
146 }
147 ca_w("log rows=" as *u8); ca_n(nlines)
148 if (clsfilt as i64) != 0 { ca_w(" class-filter=" as *u8); ca_w(clsfilt) }
149 ca_w(" sampling newest-first, max=" as *u8); ca_n(maxs); ca_w("\n" as *u8)
150
151 var li: i64 = nlines - 1
152 while li >= 0 {
153 if sampled >= maxs { li = 0 - 1 } else {
154 let ls: i64 = starts[li]
155 var le: i64 = ls
156 var f: i64 = 0
157 while f == 0 { if le >= n { f = 1 } else { if buf[le] == (10 as u8) { f = 1 } else { le = le + 1 } } }
158 if le > ls + 8 {
159 var t1: i64 = ls
160 var g: i64 = 0
161 while g == 0 { if t1 >= le { g = 1 } else { if buf[t1] == (9 as u8) { g = 1 } else { t1 = t1 + 1 } } }
162 var t2: i64 = t1 + 1
163 g = 0
164 while g == 0 { if t2 >= le { g = 1 } else { if buf[t2] == (9 as u8) { g = 1 } else { t2 = t2 + 1 } } }
165 var t3: i64 = t2 + 1
166 g = 0
167 while g == 0 { if t3 >= le { g = 1 } else { if buf[t3] == (9 as u8) { g = 1 } else { t3 = t3 + 1 } } }
168 var t4: i64 = t3 + 1
169 g = 0
170 while g == 0 { if t4 >= le { g = 1 } else { if buf[t4] == (9 as u8) { g = 1 } else { t4 = t4 + 1 } } }
171 if t4 < le {
172 let cls_s: i64 = t1 + 1
173 let cls_l: i64 = t2 - cls_s
174 var want: i64 = 1
175 if (clsfilt as i64) != 0 {
176 var fl: i64 = 0
177 while clsfilt[fl] != (0 as u8) { fl = fl + 1 }
178 want = 0
179 if fl == cls_l {
180 var eq: i64 = 1
181 var kk: i64 = 0
182 while kk < fl { if buf[cls_s + kk] != clsfilt[kk] { eq = 0; kk = fl } else { kk = kk + 1 } }
183 if eq == 1 { want = 1 }
184 }
185 }
186 if want == 1 {
187 var rec_content: i64 = 0
188 if cls_l == 2 { if buf[cls_s] == (111 as u8) { if buf[cls_s+1] == (107 as u8) { rec_content = 1 } } }
189 if cls_l == 3 { if buf[cls_s] == (100 as u8) { if buf[cls_s+1] == (117 as u8) { if buf[cls_s+2] == (112 as u8) { rec_content = 1 } } } }
190 let ub: *u8 = sys_mmap(CA_MAGIC_4096)
191 var uo: i64 = 0
192 var k2: i64 = t4 + 1
193 while k2 < le { if uo < CA_MAGIC_4090 { ub[uo] = buf[k2]; uo = uo + 1 } k2 = k2 + 1 }
194 ub[uo] = 0 as u8
195 if uo > 12 {
196 sampled = sampled + 1
197 let s1: *i64 = sys_mmap(16) as *i64
198 let x1: *i64 = sys_mmap(16) as *i64
199 let s2: *i64 = sys_mmap(16) as *i64
200 let x2: *i64 = sys_mmap(16) as *i64
201 let o1: i64 = ca_observe(store, ub, s1, x1)
202 let o2: i64 = ca_observe(store, ub, s2, x2)
203 ca_w(" [" as *u8); sys_write(1, ((buf as i64)+cls_s) as *u8, cls_l); ca_w("] " as *u8)
204 ca_w(ub)
205 ca_w("\n obs1 status=" as *u8); ca_n(s1[0]); ca_w(" text=" as *u8); ca_n(x1[0])
206 ca_w(" obs2 status=" as *u8); ca_n(s2[0]); ca_w(" text=" as *u8); ca_n(x2[0])
207 if o1 != o2 {
208 unstable = unstable + 1
209 ca_w(" -> HOST UNSTABLE (two fetches seconds apart disagree; NOT our classifier)\n" as *u8)
210 } else {
211 if o1 == rec_content {
212 agree = agree + 1
213 ca_w(" -> AGREES with the recorded class\n" as *u8)
214 } else {
215 disagree = disagree + 1
216 if rec_content == 0 {
217 ca_w(" -> DISAGREES: we DROPPED it, independent re-check finds REAL PROSE = FALSE-POSITIVE candidate\n" as *u8)
218 } else {
219 ca_w(" -> DISAGREES: we INDEXED it, independent re-check finds no prose = false-negative candidate\n" as *u8)
220 }
221 }
222 }
223 }
224 }
225 }
226 }
227 li = li - 1
228 }
229 }
230
231 ca_w("--- audit summary ---\n" as *u8)
232
233 ca_w("sampled=" as *u8); ca_n(sampled)
234 ca_w(" agree=" as *u8); ca_n(agree)
235 ca_w(" disagree=" as *u8); ca_n(disagree)
236 ca_w(" host_unstable=" as *u8); ca_n(unstable); ca_w("\n" as *u8)
237 if sampled == 0 { ca_w("NOTE: nothing sampled -- the log has no rows carrying a url yet.\n" as *u8) }
238 ca_w("HONEST ENVELOPE: this re-fetches LATER than the crawler did, so a disagreement is a FLAG for\n" as *u8)
239 ca_w("review, never a proven misclassification -- host_unstable exists precisely to catch the case\n" as *u8)
240 ca_w("where the web, not our code, changed. A clean audit is NOT proof the classifier is right; it\n" as *u8)
241 ca_w("is proof that ONE independent signal did not contradict it on THIS sample.\n" as *u8)
242 return 0
243}