nx_mirrorintegrity_lib.nx source
↩ module page · 666 lines · 36220 B
1// nx_mirrorintegrity_lib.nx -- IS THE EVIDENCE WE PINNED ACTUALLY COMPLETE?
2//
3// WHY THIS EXISTS. The /compare citation register mirrors every reference into knowledge/fetched/ and pins it
4// with a sha256. nx_compare_refs_gate then proves pin==filehash on every declared mirror. That proof is real
5// but it answers a NARROWER question than its name suggests:
6// A PIN PROVES THE BYTES ARE UNCHANGED SINCE THEY WERE STORED. IT CANNOT PROVE THEY WERE EVER COMPLETE.
7// A truncated body hashes perfectly and matches forever. Measured 2026-08-20: nx_research_fetch's TLS-1.2 leg
8// saved 255935 bytes of an 814052-byte PDF, twice, byte-identical, while printing status=200 and a SAVED line
9// that reads as success. That mirror would have passed the refs gate every day for the rest of its life.
10// A TRUNCATED MIRROR IS WORSE THAN NO MIRROR: IT IS A CITATION THAT READS AS VERIFIED.
11//
12// WHAT IT DOES. For every file under a directory it reads only the HEAD and the TAIL (never the whole body, so
13// a 6 MB PDF costs two reads) and asks the body to describe its own completeness:
14// PDF -- a linearized PDF DECLARES ITS OWN TOTAL LENGTH in the header (/L <n>); if that exceeds the file
15// on disk the body is provably short. Otherwise a complete PDF ends in the EOF marker.
16// HTML -- a complete document closes its html element.
17// JSON -- a complete document closes the brace or bracket it opened.
18// PNG -- a complete stream ends with the IEND chunk type.
19// JPEG -- a complete stream ends with the EOI marker FF D9.
20// Anything whose format declares no length and carries no terminator is UNKNOWN. It is NOT counted as complete.
21// AN AXIS THAT CANNOT SEE MUST ABSTAIN, NOT ACQUIT.
22//
23// THE PARTITION IS PRINTED AND MUST SUM: proven + suspect + complete + unknown == files.
24// CAP-EXACT is a SEPARATE AXIS, not a partition member, because a body can be complete AND land on a
25// power-of-two boundary by coincidence -- folding it in would silently break the reconciliation.
26//
27// license_tier: ORIGINAL LIBRARY -- no main. nx_mirrorintegrity.nx and nx_mirrorintegrity_gate.nx BOTH
28// consume it, so the census and its gate share ONE ruler by construction and cannot drift apart.
29import "nx_syscalls.nx"
30import "nx_lane_conf.nx"
31import "nx_html_to_text.nx"
32
33const MI_DEF_DIR: *u8 = "knowledge/fetched\x00"
34const MI_HEAD: i64 = 4096 // declared: bytes read from the head of every file
35const MI_TAIL: i64 = 4096 // declared: bytes read from the tail of every file
36const MI_PATHCAP: i64 = 4096
37const MI_DIRBUF: i64 = 131072
38const MI_MAXDEPTH: i64 = 8
39const MI_WORKCAP: i64 = 1048576 // worklist accumulation buffer, flushed when near full
40const MI_WORKFLUSH: i64 = 1047000 // flush threshold, leaves room for one long row
41const MI_CAP_FLOOR: i64 = 65536 // below this a power-of-two size is coincidence, not a cap
42const MI_SEEK_SET: i64 = 0
43const MI_SEEK_END: i64 = 2
44const MI_DT_DIR: i64 = 4
45
46const MI_PROVEN: i64 = 0
47const MI_SUSPECT: i64 = 1
48const MI_COMPLETE: i64 = 2
49const MI_UNKNOWN: i64 = 3
50
51func mi_w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
52func mi_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
53func mi_cat(o: *u8, at: i64, s: *u8) -> i64 { var i: i64=0; var a: i64=at; while s[i]!=(0 as u8){o[a]=s[i]; a=a+1; i=i+1} return a }
54func mi_catn(o: *u8, at: i64, v: i64) -> i64 {
55 var a: i64=at; var x: i64=v
56 if x<0 { o[a]=45 as u8; a=a+1; x=0-x }
57 let tm: *u8=sys_mmap(32); var k: i64=0
58 if x==0 { tm[0]=48 as u8; k=1 }
59 while x>0 { tm[k]=(48+x%10) as u8; x=x/10; k=k+1 }
60 var j: i64=0
61 while j<k { o[a]=tm[k-1-j]; a=a+1; j=j+1 }
62 return a
63}
64// index of needle within buf[from,end), or -1
65func mi_find(buf: *u8, from: i64, end: i64, needle: *u8) -> i64 {
66 let nl: i64=mi_slen(needle)
67 if nl==0 { return 0-1 }
68 var i: i64=from
69 var hit: i64=0-1
70 var go: i64=1
71 while go==1 {
72 if i+nl>end { go=0 } else {
73 var k: i64=0; var ok: i64=1
74 while k<nl { if buf[i+k]!=needle[k] { ok=0; k=nl } else { k=k+1 } }
75 if ok==1 { hit=i; go=0 } else { i=i+1 }
76 }
77 }
78 return hit
79}
80func mi_digits_at(buf: *u8, p: i64, end: i64) -> i64 {
81 var v: i64=0; var i: i64=p; var n: i64=0; var go: i64=1
82 while go==1 {
83 if i>=end { go=0 } else {
84 let c: i64 = buf[i] as i64
85 if c>=48 { if c<=57 { v=v*10+(c-48); n=n+1; i=i+1 } else { go=0 } } else { go=0 }
86 }
87 }
88 if n==0 { return 0-1 }
89 return v
90}
91func mi_is_ws(c: i64) -> i64 { if c==32 { return 1 } if c==10 { return 1 } if c==13 { return 1 } if c==9 { return 1 } return 0 }
92func mi_is_pow2(n: i64) -> i64 {
93 if n<MI_CAP_FLOOR { return 0 }
94 var x: i64=n
95 while x%2==0 { x=x/2 }
96 if x==1 { return 1 }
97 return 0
98}
99func mi_isdotdot(nm: *u8) -> i64 { if nm[0]==(46 as u8){ if nm[1]==(0 as u8){return 1} if nm[1]==(46 as u8){ if nm[2]==(0 as u8){return 1} } } return 0 }
100func mi_join(buf: *u8, base_n: i64, name: *u8) -> i64 { buf[base_n]=47 as u8; var o: i64=base_n+1; var i: i64=0; while name[i]!=(0 as u8){buf[o]=name[i];o=o+1;i=i+1} return o }
101
102// read up to cap bytes into buf starting at file offset off; returns bytes read
103func mi_read_at(fd: i64, off: i64, buf: *u8, cap: i64) -> i64 {
104 sys_lseek(fd, off, MI_SEEK_SET)
105 var n: i64=0
106 var go: i64=1
107 while go==1 {
108 if n>=cap { go=0 } else {
109 let r: i64=sys_read(fd, (buf as i64+n) as *u8, cap-n)
110 if r<=0 { go=0 } else { n=n+r }
111 }
112 }
113 return n
114}
115
116// ---- HTML TAIL VERDICT -- rewritten 2026-08-25, because the rule it replaces was measured at
117// 1000 PERMIL FALSE POSITIVES OVER ITS ENTIRE LIVE POPULATION.
118//
119// WHAT WAS WRONG. The HTML branch returned MI_SUSPECT for any body whose tail lacked a closing html
120// element. HTML5 makes that end tag OPTIONAL, so its absence is evidence of nothing -- and this lib had
121// ALREADY banked that lesson at 941 permil from the last time, in the header above, and then kept the
122// rule anyway. MEASURED 2026-08-25, FULL POPULATION, NO SAMPLING: all 22 rows on the
123// mirror-incompleteness worklist carried this one reason, and an INDEPENDENT control -- downloading
124// each of the 21 distinct urls fresh and asking whether the LIVE SERVER OWN COMPLETE 200 RESPONSE
125// closes its html element -- answered NO for 21 of 21. Thirteen stored bodies were the SAME BYTE COUNT
126// as the fresh download and three were LARGER than it, which refutes truncation by arithmetic. Not one
127// was short. A RULE WHOSE ENTIRE POPULATION IS FALSE POSITIVES IS NOT A DETECTOR, IT IS A WORKLIST
128// GENERATOR -- and a worklist of nothing teaches everyone to ignore the instrument.
129//
130// WHAT REPLACES IT, AND WHY IT IS NOT A DISARM. Three outcomes instead of one, strongest first, each
131// keyed on evidence the tail actually carries:
132// PROVEN -- the tail ends INSIDE an unclosed tag: scanning back from the last non-space byte, the
133// first bracket met is an opener. No server that finished writing a document leaves a
134// half-written tag. This is a NEW capability -- the old rule could only ever say SUSPECT
135// about HTML and could never PROVE anything about it at all.
136// UNKNOWN -- the last non-space byte IS a closing bracket. The document ends on a closed element and
137// merely omits an optional end tag. THE AXIS CANNOT SEE, SO IT ABSTAINS RATHER THAN
138// ACQUIT OR ACCUSE. All 21 measured controls land here.
139// SUSPECT -- the body ends mid-content, on neither a bracket nor an open tag. That is the shape of a
140// cut-off body, and it is what both gate fixtures plant.
141// THE RESIDUAL, DECLARED RATHER THAN HIDDEN: a valid HTML5 document MAY end in bare text with every
142// optional end tag omitted, and this rule calls that SUSPECT. On the measured corpus that shape does
143// not occur (21 of 21 live bodies end on a closing bracket), and the error it makes is the LOUD one --
144// a named row on a worklist a human reads -- never a silent acquittal of a void citation. Second
145// residual: the scan sees only the declared MI_TAIL window, so a trailing text node longer than that
146// window reads as SUSPECT for want of a bracket rather than because anything is missing.
147const MI_CH_GT: i64 = 62 // the byte a finished HTML element always ends on
148const MI_CH_LT: i64 = 60 // opens a tag; met first while scanning BACK, it proves a cut
149const MI_RC_HTML_MIDCONTENT: i64 = 4
150const MI_RC_HTML_OPENTAG: i64 = 10
151const MI_RC_HTML_OPTIONAL: i64 = 11
152
153func mi_html_tail_verdict(tbuf: *u8, tn: i64, rcode: *i64) -> i64 {
154 var e: i64 = tn-1
155 var done: i64 = 0
156 while done==0 {
157 if e<0 { done=1 } else { if mi_is_ws(tbuf[e] as i64)==1 { e=e-1 } else { done=1 } }
158 }
159 if e<0 { rcode[0]=MI_RC_HTML_MIDCONTENT; return MI_SUSPECT }
160 // scan BACK from the last markup-bearing byte to the first bracket of either kind. The cursor and
161 // the answer are separate variables on purpose: a loop that exits by clobbering its own cursor
162 // destroys the position it was asked to report.
163 var i: i64 = e
164 var first: i64 = 0
165 var scan: i64 = 1
166 while scan==1 {
167 if i<0 { scan=0 } else {
168 let c: i64 = tbuf[i] as i64
169 if c==MI_CH_GT { first=MI_CH_GT; scan=0 } else {
170 if c==MI_CH_LT { first=MI_CH_LT; scan=0 } else { i=i-1 }
171 }
172 }
173 }
174 if first==MI_CH_LT { rcode[0]=MI_RC_HTML_OPENTAG; return MI_PROVEN }
175 if first==MI_CH_GT { if i==e { rcode[0]=MI_RC_HTML_OPTIONAL; return MI_UNKNOWN } }
176 rcode[0]=MI_RC_HTML_MIDCONTENT
177 return MI_SUSPECT
178}
179
180// classify ONE file. reason[0] is set to a static string pointer via reasons table index in rcode[0].
181// returns one of MI_PROVEN / MI_SUSPECT / MI_COMPLETE / MI_UNKNOWN
182// rcode: 0 unreadable, 1 zero-byte, 2 pdf-declared-length-exceeds-file, 3 pdf-no-eof-marker,
183// 4 html-ends-mid-content, 5 json-unterminated, 6 png-no-iend, 7 jpeg-no-eoi, 8 ok, 9 no-format-signature,
184// 10 html-ends-inside-an-unclosed-tag (PROVEN), 11 html-end-tag-optional-cannot-judge (UNKNOWN)
185func mi_classify(path: *u8, hbuf: *u8, tbuf: *u8, rcode: *i64, sz: *i64, decl: *i64) -> i64 {
186 rcode[0]=0; sz[0]=0; decl[0]=0-1
187 let fd: i64=sys_openat_rd(path)
188 if fd<0 { rcode[0]=0; return MI_UNKNOWN }
189 let size: i64=sys_lseek(fd, 0, MI_SEEK_END)
190 sz[0]=size
191 if size<=0 { sys_close(fd); rcode[0]=1; return MI_PROVEN }
192 var hcap: i64=MI_HEAD
193 if size<hcap { hcap=size }
194 let hn: i64=mi_read_at(fd, 0, hbuf, hcap)
195 var toff: i64=size-MI_TAIL
196 if toff<0 { toff=0 }
197 var tcap: i64=size-toff
198 if tcap>MI_TAIL { tcap=MI_TAIL }
199 let tn: i64=mi_read_at(fd, toff, tbuf, tcap)
200 sys_close(fd)
201 if hn<=0 { rcode[0]=0; return MI_UNKNOWN }
202
203 // ---- PDF: the only format here that DECLARES its own total length
204 if mi_find(hbuf, 0, hn, "%PDF\x00" as *u8)==0 {
205 let lin: i64=mi_find(hbuf, 0, hn, "/Linearized\x00" as *u8)
206 if lin>=0 {
207 let lp: i64=mi_find(hbuf, lin, hn, "/L \x00" as *u8)
208 if lp>=0 {
209 let dv: i64=mi_digits_at(hbuf, lp+3, hn)
210 if dv>0 {
211 decl[0]=dv
212 if dv>size { rcode[0]=2; return MI_PROVEN }
213 }
214 }
215 }
216 if mi_find(tbuf, 0, tn, "%%EOF\x00" as *u8)>=0 { rcode[0]=8; return MI_COMPLETE }
217 rcode[0]=3
218 return MI_SUSPECT
219 }
220 // ---- PNG
221 if hn>=8 { if hbuf[1]==(80 as u8) { if hbuf[2]==(78 as u8) { if hbuf[3]==(71 as u8) {
222 if mi_find(tbuf, 0, tn, "IEND\x00" as *u8)>=0 { rcode[0]=8; return MI_COMPLETE }
223 rcode[0]=6
224 return MI_SUSPECT
225 } } } }
226 // ---- JPEG
227 if hn>=2 { if hbuf[0]==(255 as u8) { if hbuf[1]==(216 as u8) {
228 if tn>=2 { if tbuf[tn-2]==(255 as u8) { if tbuf[tn-1]==(217 as u8) { rcode[0]=8; return MI_COMPLETE } } }
229 rcode[0]=7
230 return MI_SUSPECT
231 } } }
232 // ---- HTML
233 var ishtml: i64=0
234 if mi_find(hbuf, 0, hn, "<html\x00" as *u8)>=0 { ishtml=1 }
235 if mi_find(hbuf, 0, hn, "<HTML\x00" as *u8)>=0 { ishtml=1 }
236 if ishtml==1 {
237 if mi_find(tbuf, 0, tn, "</html\x00" as *u8)>=0 { rcode[0]=8; return MI_COMPLETE }
238 if mi_find(tbuf, 0, tn, "</HTML\x00" as *u8)>=0 { rcode[0]=8; return MI_COMPLETE }
239 return mi_html_tail_verdict(tbuf, tn, rcode)
240 }
241 // ---- JSON
242 var fi: i64=0
243 while fi<hn { if mi_is_ws(hbuf[fi] as i64)==1 { fi=fi+1 } else { fi=hn+1 } }
244 if fi==hn+1 {
245 var p: i64=0
246 var q: i64=0
247 while q<hn { if mi_is_ws(hbuf[q] as i64)==1 { q=q+1 } else { p=q; q=hn } }
248 let c0: i64 = hbuf[p] as i64
249 if c0==123 { } else { if c0==91 { } else { p=0-1 } }
250 // SECOND-TOKEN CHECK, added 2026-09-04. The one-byte test above classifies ANY body whose
251 // first non-space byte is '{' or '[' as JSON. A GitHub README opening with a badge --
252 // "[](...)" -- was therefore read as a JSON array, found never to
253 // close, and published as json-unterminated against a file that is COMPLETE markdown. That
254 // is not an edge case: a README opening with a badge is the NORM on GitHub, so this misread
255 // every such mirror. A JSON value cannot begin "[!" -- after the opening bracket the next
256 // non-space byte must be able to START a value. Two bytes decide it and NO valid JSON can
257 // fail it, so the check is wrong only in the direction of declining to judge.
258 if p>=0 {
259 var s2: i64 = p+1
260 var seen: i64 = 0-1
261 var scanning: i64 = 1
262 while scanning==1 {
263 if s2>=hn { scanning=0 } else {
264 if mi_is_ws(hbuf[s2] as i64)==1 { s2=s2+1 } else { seen=hbuf[s2] as i64; scanning=0 }
265 }
266 }
267 // seen<0 means nothing follows the bracket at all -- that IS truncated JSON, so it is
268 // deliberately left classified rather than excused.
269 if seen>=0 {
270 var okv: i64 = 0
271 if c0==123 {
272 if seen==34 { okv=1 }
273 if seen==125 { okv=1 }
274 } else {
275 if seen==34 { okv=1 }
276 if seen==123 { okv=1 }
277 if seen==91 { okv=1 }
278 if seen==93 { okv=1 }
279 if seen==45 { okv=1 }
280 if seen>=48 { if seen<=57 { okv=1 } }
281 if seen==116 { okv=1 }
282 if seen==102 { okv=1 }
283 if seen==110 { okv=1 }
284 }
285 if okv==0 { p=0-1 }
286 }
287 }
288 if p>=0 {
289 var le: i64=tn-1
290 var done: i64=0
291 while done==0 { if le<0 { done=1 } else { if mi_is_ws(tbuf[le] as i64)==1 { le=le-1 } else { done=1 } } }
292 if le>=0 {
293 let cl: i64 = tbuf[le] as i64
294 if cl==125 { rcode[0]=8; return MI_COMPLETE }
295 if cl==93 { rcode[0]=8; return MI_COMPLETE }
296 }
297 rcode[0]=5
298 return MI_SUSPECT
299 }
300 }
301 rcode[0]=9
302 return MI_UNKNOWN
303}
304
305func mi_reason(rc: i64) -> *u8 {
306 if rc==0 { return "unreadable\x00" as *u8 }
307 if rc==1 { return "zero-byte-body\x00" as *u8 }
308 if rc==2 { return "pdf-declared-length-exceeds-file\x00" as *u8 }
309 if rc==3 { return "pdf-no-eof-marker\x00" as *u8 }
310 if rc==4 { return "html-ends-mid-content-not-on-a-closing-bracket\x00" as *u8 }
311 if rc==10 { return "html-ends-inside-an-unclosed-tag\x00" as *u8 }
312 if rc==11 { return "html-end-tag-optional-cannot-judge\x00" as *u8 }
313 if rc==5 { return "json-unterminated\x00" as *u8 }
314 if rc==6 { return "png-no-iend-chunk\x00" as *u8 }
315 if rc==7 { return "jpeg-no-eoi-marker\x00" as *u8 }
316 if rc==8 { return "ok\x00" as *u8 }
317 return "no-format-signature-cannot-judge\x00" as *u8
318}
319
320// ================= CONTENT AXIS -- added 2026-08-25 =================
321// A PIN PROVES THE BYTES ARE UNCHANGED. THE COMPLETENESS AXIS ABOVE PROVES THE BODY IS WHOLE.
322// NEITHER PROVES THE BODY CARRIES THE CONTENT WE WENT TO THAT URL FOR.
323//
324// MEASURED 2026-08-25 on europe.naverlabs.com/research/code/: a JS-rendered site returns a COMPLETE,
325// well-formed HTML document whose visible text is WordPress and CSS boilerplate. The catalogue we cited
326// it for is assembled in a browser and IS NOT IN THE BYTES. That mirror is COMPLETE-PROVEN on the axis
327// above, pins perfectly forever, and is worthless as evidence.
328// A MIRROR THAT IS WHOLE AND EMPTY IS A CITATION THAT READS AS VERIFIED.
329//
330// NOT A PARTITION MEMBER, for exactly the reason CAP-EXACT is not: a body can be COMPLETE and
331// content-empty AT ONCE, so folding it in would silently break the reconciliation the partition exists
332// to provide. It is a SEPARATE AXIS and is reported as one.
333//
334// THE FALSE-POSITIVE GUARD IS THE WHOLE DESIGN. This lib already shipped one completeness rule with a
335// MEASURED 941-permil false-positive rate on real data (html-no-closing-html-tag; HTML5 end tags are
336// OPTIONAL). The lesson banked from that is that a marker is evidence ONLY when the body cannot explain
337// itself otherwise, so the rule here is ordered, not incidental:
338// A BODY THAT CARRIES REAL PROSE IS OK NO MATTER WHICH MARKERS IT CONTAINS.
339// An article ABOUT Cloudflare contains the words Just a moment; a bot wall IS Just a moment. Only a
340// PROSE-POOR body is ever marker-classified. That single ordering is what stops this axis becoming the
341// saturated-vocabulary detector the estate has been bitten by three times.
342//
343// THE PROSE MEASURE COMPOSES nx_html_to_text -- the estate ONE renderer, which already suppresses script
344// and style bodies. No second tag list, no second entity table, no second whitespace policy.
345//
346// MARKER ORDER IS REMEDY ROUTING, NOT COSMETICS. A Cloudflare interstitial literally tells the reader to
347// enable JavaScript, so testing the JS-SHELL markers first would file every bot wall as a JS shell and
348// send it to the browser-render lane -- which CANNOT fix a bot wall. Challenge is therefore tested first,
349// and each class is named for the lane that can actually resolve it.
350// Named for PURPOSE, never for value (rule 11). ASCII code points the argv parser compares against:
351const MI_CH_MINUS: i64 = 45
352const MI_CH_ZERO: i64 = 48
353const MI_CH_NINE: i64 = 57
354// the counts arena shared by the completeness and content walks: MI_COUNTS_SLOTS i64 slots
355const MI_COUNTS_SLOTS: i64 = 16
356const MI_COUNTS_BYTES: i64 = 128
357// report buffer for the content-axis summary block
358const MI_AXISBUF: i64 = 4096
359const MI_CONTENT_CONF_REL: *u8 = "knowledge/mirror_content.conf\x00"
360const MI_CONTENT_CONF_ABS: *u8 = "/volume1/homes/elderwesto/nishihost/knowledge/mirror_content.conf\x00"
361const MI_PROSE_FLOOR_KEY: *u8 = "prose-floor-bytes\x00"
362// BOOTSTRAP ONLY (rule 17: argv > conf > this). The shipped bar is the conf row, which carries the
363// measured distribution it was derived from. This value is deliberately the SMALLEST useful bar so that
364// an absent conf under-reports rather than over-reports: a detector that fails toward silence is safe.
365const MI_DEF_PROSE_FLOOR: i64 = 1
366
367const MC_OK: i64 = 0
368const MC_JS_SHELL: i64 = 1
369const MC_BOT_CHALLENGE: i64 = 2
370const MC_LOGIN_WALL: i64 = 3
371const MC_HTTP_ERROR: i64 = 4
372const MC_PROSE_EMPTY: i64 = 5
373const MC_NOT_HTML: i64 = 6
374
375func mi_prose_floor() -> i64 { return lc_geti(MI_CONTENT_CONF_REL as *u8, MI_CONTENT_CONF_ABS as *u8, MI_PROSE_FLOOR_KEY as *u8, MI_DEF_PROSE_FLOOR) }
376
377// argv integer, sign-aware. A NEGATIVE floor is meaningful here: it selects calibration mode.
378func mi_atoi_arg(s: *u8) -> i64 {
379 var v: i64=0
380 var i: i64=0
381 var neg: i64=0
382 if s[0]==(MI_CH_MINUS as u8) { neg=1; i=1 }
383 while s[i]!=(0 as u8) {
384 let c: i64=s[i] as i64
385 if c>=MI_CH_ZERO { if c<=MI_CH_NINE { v=v*10+(c-MI_CH_ZERO) } }
386 i=i+1
387 }
388 if neg==1 { return 0-v }
389 return v
390}
391
392func mi_pc_lc(c: i64) -> i64 { if c>=65 { if c<=90 { return c+32 } } return c }
393
394// case-insensitive needle search over buf[from,end); -1 when absent.
395func mi_pc_find_ci(buf: *u8, from: i64, end: i64, needle: *u8) -> i64 {
396 let nl: i64=mi_slen(needle)
397 if nl==0 { return 0-1 }
398 var i: i64=from
399 var hit: i64=0-1
400 var go: i64=1
401 while go==1 {
402 if i+nl>end { go=0 } else {
403 var k: i64=0; var ok: i64=1
404 while k<nl { if mi_pc_lc(buf[i+k] as i64)!=mi_pc_lc(needle[k] as i64) { ok=0; k=nl } else { k=k+1 } }
405 if ok==1 { hit=i; go=0 } else { i=i+1 }
406 }
407 }
408 return hit
409}
410func mi_has(src: *u8, n: i64, needle: *u8) -> i64 { if mi_pc_find_ci(src, 0, n, needle)>=0 { return 1 } return 0 }
411
412// HTML is decided from the head window only, the same window the completeness axis already reads.
413func mi_is_html(src: *u8, n: i64) -> i64 {
414 var lim: i64=n
415 if lim>MI_HEAD { lim=MI_HEAD }
416 if mi_pc_find_ci(src, 0, lim, "<html\x00" as *u8)>=0 { return 1 }
417 if mi_pc_find_ci(src, 0, lim, "<!doctype html\x00" as *u8)>=0 { return 1 }
418 return 0
419}
420
421// HOW BIG THE RENDER BUFFER IS. nx_html_to_text only ever REMOVES bytes -- tags, script and style
422// bodies, collapsed whitespace -- so the input length is already an upper bound on the output, and
423// this slack is pure defensive headroom for any terminator the renderer appends. It is NOT a tuned
424// threshold: no classification outcome depends on its value.
425const MI_RENDER_SLACK: i64 = 4096
426
427// THE RENDERED TEXT IS THE PRODUCT; ITS LENGTH IS ONLY ONE VIEW OF IT. This is the primitive, and
428// mi_prose_bytes DELEGATES to it, so every existing caller of mi_prose_bytes keeps byte-identical
429// behaviour BY CONSTRUCTION while a caller that needs the text ITSELF can have it without paying to
430// render the same body twice.
431func mi_prose_render(src: *u8, n: i64, out: *u8, cap: i64) -> i64 {
432 return nx_html_to_text(src, n, out, cap)
433}
434
435// visible-text bytes of an HTML body, via the estate ONE renderer.
436func mi_prose_bytes(src: *u8, n: i64) -> i64 {
437 let cap: i64=n+MI_RENDER_SLACK
438 let out: *u8=sys_mmap(cap)
439 let m: i64=mi_prose_render(src, n, out, cap)
440 sys_munmap(out, cap)
441 return m
442}
443
444// Classify ONE stored body on the CONTENT axis. Reads the whole body (a JS shell can only be recognised
445// by the prose it does NOT have, and absence is not visible in a head window).
446// prose[0] = visible-text bytes, or -1 when this body is not judgeable on this axis.
447// hb[0] = body bytes.
448func mi_content_class(path: *u8, floor: i64, prose: *i64, hb: *i64) -> i64 {
449 prose[0]=0-1
450 hb[0]=0
451 let lenbox: *i64=sys_mmap(16) as *i64
452 lenbox[0]=0
453 let src: *u8=sys_read_file(path, lenbox)
454 let n: i64=lenbox[0]
455 sys_munmap(lenbox as *u8, 16)
456 if (src as i64)==0 { return MC_NOT_HTML }
457 if n<=0 { sys_free_file(src, n); return MC_NOT_HTML }
458 hb[0]=n
459 if mi_is_html(src, n)==0 { sys_free_file(src, n); return MC_NOT_HTML }
460 // THE FIX (2026-08-25). The rendered text used to be computed here and THROWN AWAY -- only its
461 // LENGTH survived -- and every marker test below then ran against the RAW BODY. That is why a
462 // 2.1 MB JS bundle whose error-handling CODE contains the string "404 not found" was classified
463 // http-error-page-stored-as-body while its url served real documentation: the marker was in the
464 // BUNDLE, never on the PAGE. The renderer had already produced the evidence and the classifier
465 // discarded it. It is retained now, and the two classes whose markers are things a server SAYS
466 // TO A HUMAN are judged on the rendered text instead of the raw bytes.
467 let rcap: i64=n+MI_RENDER_SLACK
468 let rtx: *u8=sys_mmap(rcap)
469 let p: i64=mi_prose_render(src, n, rtx, rcap)
470 prose[0]=p
471 // THE GUARD: real prose acquits, whatever the markers say.
472 if p>=floor { sys_munmap(rtx, rcap); sys_free_file(src, n); return MC_OK }
473 var cls: i64=MC_PROSE_EMPTY
474 var done: i64=0
475 // 1. CHALLENGE -- remedy is a session/PoW lane, NOT a re-render. Tested first on purpose.
476 // DELIBERATELY STILL ON THE RAW BODY (src, n). These markers are transport and script tokens --
477 // cf-mitigated, __cf_chl, cf_chl_opt -- which never appear in rendered prose at all. Moving this
478 // class to rtx the way classes 2 and 3 moved would DISARM it completely.
479 if done==0 { if mi_has(src, n, "just a moment\x00" as *u8)==1 { cls=MC_BOT_CHALLENGE; done=1 } }
480 if done==0 { if mi_has(src, n, "cf-mitigated\x00" as *u8)==1 { cls=MC_BOT_CHALLENGE; done=1 } }
481 if done==0 { if mi_has(src, n, "__cf_chl\x00" as *u8)==1 { cls=MC_BOT_CHALLENGE; done=1 } }
482 if done==0 { if mi_has(src, n, "cf_chl_opt\x00" as *u8)==1 { cls=MC_BOT_CHALLENGE; done=1 } }
483 if done==0 { if mi_has(src, n, "checking your browser\x00" as *u8)==1 { cls=MC_BOT_CHALLENGE; done=1 } }
484 if done==0 { if mi_has(src, n, "attention required\x00" as *u8)==1 { cls=MC_BOT_CHALLENGE; done=1 } }
485 if done==0 { if mi_has(src, n, "anubis\x00" as *u8)==1 { cls=MC_BOT_CHALLENGE; done=1 } }
486 if done==0 { if mi_has(src, n, "verifying you are human\x00" as *u8)==1 { cls=MC_BOT_CHALLENGE; done=1 } }
487 // 2. HTTP ERROR PAGE -- the fetch stored the server complaint, not the document. Remedy is the URL.
488 // JUDGED ON THE RENDERED TEXT (rtx, p), NOT the raw body. "404 not found" is a sentence a server
489 // shows a HUMAN, so its home is prose. In RAW bytes it is also a string literal sitting in every
490 // JS router's error branch, and that is precisely the false positive this class was generating:
491 // measured 2026-08-25, 17 of 17 rows in this class were large bundles serving real documents.
492 if done==0 { if mi_has(rtx, p, "404 not found\x00" as *u8)==1 { cls=MC_HTTP_ERROR; done=1 } }
493 if done==0 { if mi_has(rtx, p, "403 forbidden\x00" as *u8)==1 { cls=MC_HTTP_ERROR; done=1 } }
494 if done==0 { if mi_has(rtx, p, "429 too many requests\x00" as *u8)==1 { cls=MC_HTTP_ERROR; done=1 } }
495 if done==0 { if mi_has(rtx, p, "500 internal server error\x00" as *u8)==1 { cls=MC_HTTP_ERROR; done=1 } }
496 if done==0 { if mi_has(rtx, p, "page not found\x00" as *u8)==1 { cls=MC_HTTP_ERROR; done=1 } }
497 if done==0 { if mi_has(rtx, p, "access denied\x00" as *u8)==1 { cls=MC_HTTP_ERROR; done=1 } }
498 // 3. LOGIN WALL -- remedy is credentials or a different source; a renderer cannot help.
499 // JUDGED ON THE RENDERED TEXT (rtx, p) for the same reason as class 2: a wall is something the
500 // reader is TOLD. "please log in" in raw bytes is ordinary header-nav markup on every storefront
501 // template, so the raw test convicted articles that were served in full.
502 if done==0 { if mi_has(rtx, p, "sign in to continue\x00" as *u8)==1 { cls=MC_LOGIN_WALL; done=1 } }
503 if done==0 { if mi_has(rtx, p, "log in to continue\x00" as *u8)==1 { cls=MC_LOGIN_WALL; done=1 } }
504 if done==0 { if mi_has(rtx, p, "please log in\x00" as *u8)==1 { cls=MC_LOGIN_WALL; done=1 } }
505 if done==0 { if mi_has(rtx, p, "login required\x00" as *u8)==1 { cls=MC_LOGIN_WALL; done=1 } }
506 // 4. JS SHELL -- remedy IS the browser-render lane. Tested last so a challenge never lands here.
507 // DELIBERATELY STILL ON THE RAW BODY (src, n). These markers ARE markup structure -- a root div,
508 // data-reactroot, noscript, __NEXT_DATA__ -- i.e. precisely the bytes a renderer strips. Judging
509 // them on rendered text would disarm this class too.
510 // THE SPLIT IS PRINCIPLED, NOT A PATCH: classes 2 and 3 match on what a server SAYS to a reader
511 // (prose), classes 1 and 4 match on how a body is BUILT (markup and script). Each is tested
512 // against the representation that actually carries its evidence.
513 if done==0 { if mi_has(src, n, "you need to enable javascript\x00" as *u8)==1 { cls=MC_JS_SHELL; done=1 } }
514 if done==0 { if mi_has(src, n, "please enable javascript\x00" as *u8)==1 { cls=MC_JS_SHELL; done=1 } }
515 if done==0 { if mi_has(src, n, "enable javascript to run this app\x00" as *u8)==1 { cls=MC_JS_SHELL; done=1 } }
516 if done==0 { if mi_has(src, n, "__next_data__\x00" as *u8)==1 { cls=MC_JS_SHELL; done=1 } }
517 if done==0 { if mi_has(src, n, "data-reactroot\x00" as *u8)==1 { cls=MC_JS_SHELL; done=1 } }
518 if done==0 { if mi_has(src, n, "<div id=\x22root\x22></div>\x00" as *u8)==1 { cls=MC_JS_SHELL; done=1 } }
519 if done==0 { if mi_has(src, n, "<div id=\x22app\x22></div>\x00" as *u8)==1 { cls=MC_JS_SHELL; done=1 } }
520 if done==0 { if mi_has(src, n, "<noscript\x00" as *u8)==1 { cls=MC_JS_SHELL; done=1 } }
521 sys_munmap(rtx, rcap)
522 sys_free_file(src, n)
523 return cls
524}
525
526func mi_content_reason(c: i64) -> *u8 {
527 if c==MC_OK { return "content-present\x00" as *u8 }
528 if c==MC_JS_SHELL { return "js-shell-rendered-empty\x00" as *u8 }
529 if c==MC_BOT_CHALLENGE { return "bot-challenge-interstitial\x00" as *u8 }
530 if c==MC_LOGIN_WALL { return "login-wall\x00" as *u8 }
531 if c==MC_HTTP_ERROR { return "http-error-page-stored-as-body\x00" as *u8 }
532 if c==MC_PROSE_EMPTY { return "prose-empty-cause-unknown\x00" as *u8 }
533 return "not-html-not-judgeable-on-this-axis\x00" as *u8
534}
535
536// the lane that can actually resolve each class -- carried WITH the row so a worklist is actionable.
537func mi_content_remedy(c: i64) -> *u8 {
538 if c==MC_JS_SHELL { return "cdp-render\x00" as *u8 }
539 if c==MC_BOT_CHALLENGE { return "session-or-alternate-source\x00" as *u8 }
540 if c==MC_LOGIN_WALL { return "credentials-or-alternate-source\x00" as *u8 }
541 if c==MC_HTTP_ERROR { return "fix-url-or-find-successor\x00" as *u8 }
542 if c==MC_PROSE_EMPTY { return "adjudicate-by-hand\x00" as *u8 }
543 return "none\x00" as *u8
544}
545
546// ONE CONTENT WALKER, shared by the census, the re-ingest actuator and the gate, so the three can never
547// disagree about what a failure IS. ccounts[0..6]=content classes by MC_* code, ccounts[7]=files seen,
548// ccounts[8]=html bodies judged. floor<0 selects CALIBRATION mode: every HTML body emits its prose
549// number so the bar can be READ OFF THE DISTRIBUTION instead of invented.
550// AN UNCALIBRATED CLASSIFIER MUST REPORT NUMBERS, NEVER VERDICTS.
551func mi_content_walk(p: *u8, pn: i64, depth: i64, floor: i64, ccounts: *i64, work: *u8, wo: *i64) -> i64 {
552 if depth>MI_MAXDEPTH { return 0 }
553 if pn>MI_PATHCAP-256 { return 0 }
554 p[pn]=0 as u8
555 let fd: i64=sys_openat_rd(p)
556 if fd<0 { return 0 }
557 let dbuf: *u8=sys_mmap(MI_DIRBUF)
558 let prose: *i64=sys_mmap(16) as *i64
559 let hb: *i64=sys_mmap(16) as *i64
560 var go: i64=1
561 while go==1 {
562 let nr: i64=sys_getdents64(fd, dbuf, MI_DIRBUF)
563 if nr<=0 { go=0 } else {
564 var off: i64=0
565 while off<nr {
566 let rec: *u8=(dbuf as i64+off) as *u8
567 let ty: i64=dirent_type(rec)
568 let nm: *u8=dirent_name(rec)
569 if mi_isdotdot(nm)==0 {
570 let cs: i64=mi_join(p, pn, nm)
571 p[cs]=0 as u8
572 if ty==MI_DT_DIR { mi_content_walk(p, cs, depth+1, floor, ccounts, work, wo) }
573 else {
574 ccounts[7]=ccounts[7]+1
575 var ef: i64=floor
576 if ef<0 { ef=0 }
577 let cc: i64=mi_content_class(p, ef, prose, hb)
578 ccounts[cc]=ccounts[cc]+1
579 if cc!=MC_NOT_HTML { ccounts[8]=ccounts[8]+1 }
580 var emit: i64=0
581 if floor<0 { if cc!=MC_NOT_HTML { emit=1 } }
582 else { if cc!=MC_NOT_HTML { if cc!=MC_OK { emit=1 } } }
583 if emit==1 {
584 if wo[0]<MI_WORKFLUSH {
585 var a: i64=wo[0]
586 a=mi_cat(work, a, " CONTENT " as *u8)
587 a=mi_cat(work, a, mi_content_reason(cc))
588 a=mi_cat(work, a, " remedy=" as *u8)
589 a=mi_cat(work, a, mi_content_remedy(cc))
590 a=mi_cat(work, a, " prose=" as *u8)
591 a=mi_catn(work, a, prose[0])
592 a=mi_cat(work, a, " bytes=" as *u8)
593 a=mi_catn(work, a, hb[0])
594 a=mi_cat(work, a, " " as *u8)
595 a=mi_cat(work, a, p)
596 a=mi_cat(work, a, "\n" as *u8)
597 wo[0]=a
598 }
599 }
600 }
601 }
602 off=off+dirent_reclen(rec)
603 }
604 }
605 }
606 sys_close(fd)
607 sys_munmap(dbuf, MI_DIRBUF)
608 return 0
609}
610
611// counts[0..3]=partition, counts[4]=files, counts[5]=cap-exact axis, counts[6]=worst declared shortfall
612func mi_walk(p: *u8, pn: i64, depth: i64, hbuf: *u8, tbuf: *u8, counts: *i64, work: *u8, wo: *i64) -> i64 {
613 if depth>MI_MAXDEPTH { return 0 }
614 if pn>MI_PATHCAP-256 { return 0 }
615 p[pn]=0 as u8
616 let fd: i64=sys_openat_rd(p)
617 if fd<0 { return 0 }
618 let dbuf: *u8=sys_mmap(MI_DIRBUF)
619 let rcode: *i64=sys_mmap(16) as *i64
620 let szp: *i64=sys_mmap(16) as *i64
621 let dcp: *i64=sys_mmap(16) as *i64
622 var go: i64=1
623 while go==1 {
624 let nr: i64=sys_getdents64(fd, dbuf, MI_DIRBUF)
625 if nr<=0 { go=0 } else {
626 var off: i64=0
627 while off<nr {
628 let rec: *u8=(dbuf as i64+off) as *u8
629 let ty: i64=dirent_type(rec)
630 let nm: *u8=dirent_name(rec)
631 if mi_isdotdot(nm)==0 {
632 let cs: i64=mi_join(p, pn, nm)
633 p[cs]=0 as u8
634 if ty==MI_DT_DIR { mi_walk(p, cs, depth+1, hbuf, tbuf, counts, work, wo) }
635 else {
636 let cls: i64=mi_classify(p, hbuf, tbuf, rcode, szp, dcp)
637 counts[cls]=counts[cls]+1
638 counts[4]=counts[4]+1
639 if mi_is_pow2(szp[0])==1 { counts[5]=counts[5]+1 }
640 if cls==MI_PROVEN { if dcp[0]>0 { let sh: i64=dcp[0]-szp[0]; if sh>counts[6] { counts[6]=sh } } }
641 if cls!=MI_COMPLETE { if cls!=MI_UNKNOWN {
642 if wo[0]<MI_WORKFLUSH {
643 var a: i64=wo[0]
644 a=mi_cat(work, a, " " as *u8)
645 if cls==MI_PROVEN { a=mi_cat(work, a, "TRUNCATED-PROVEN " as *u8) } else { a=mi_cat(work, a, "TRUNCATED-SUSPECT" as *u8) }
646 a=mi_cat(work, a, " reason=" as *u8)
647 a=mi_cat(work, a, mi_reason(rcode[0]))
648 a=mi_cat(work, a, " size=" as *u8)
649 a=mi_catn(work, a, szp[0])
650 if dcp[0]>0 { a=mi_cat(work, a, " declared=" as *u8); a=mi_catn(work, a, dcp[0]) }
651 a=mi_cat(work, a, " " as *u8)
652 a=mi_cat(work, a, p)
653 a=mi_cat(work, a, "\n" as *u8)
654 wo[0]=a
655 }
656 } }
657 }
658 }
659 off=off+dirent_reclen(rec)
660 }
661 }
662 }
663 sys_close(fd)
664 return 0
665}
666