nx_compare_sections.nx source
↩ module page · 377 lines · 18886 B
1// nx_compare_sections.nx -- SECTION CENSUS of the published /compare surface: which <h2> sections every
2// domain page carries, the union across the surface, and -- against a DECLARED required shape -- which
3// sections each page is MISSING, named per domain.
4//
5// WHY THIS EXISTS (2026-09-02, operator: "all the /compare on the same emitted output, not random; if a
6// previous compare had more capabilities bring it into the emitter"). Two generators emit the surface
7// (nx_swcompare_matrix, nx_swcompare_sota) and their pages had drifted to two SHAPES: the sota-class pages
8// lacked Evidence profile, Do this next, Critical path and the Capability matrix that every matrix-class
9// page carries. The first attempt to measure that was `nx_shelltool grep "<h2"` over the docroot, which hit
10// its 1 MiB budget on the FIRST page: emitted HTML is minified, so one section is one line of 100 KB and a
11// line-oriented grep measures nothing. A TAG-AWARE WALKER IS THE INSTRUMENT, and it belongs in the estate,
12// not in a shell one-liner (NishiLang for everything; shell is the wire).
13//
14// THE REQUIRED SHAPE IS DATA, NOT A LIST IN THIS FILE: knowledge/compare/sections.required rows
15// section|<title prefix>|<data ext or ->
16// A section is REQUIRED of a domain when its data class exists for that domain (<kdir>/<dom>.<ext>) or
17// when the ext is "-" (required of every page). A page that has the data but not the section is the
18// emitter-shape defect this organ names; a page without the data is not blamed for the section.
19// Title matching is by PREFIX after normalisation (tags stripped, cut at the first " —"), so the
20// generators' long explanatory headings ("Do this next — computed by the ranker...") match their
21// short declared prefix and a reworded tail cannot silently un-match a section.
22//
23// nx_compare_sections <docroot> <kdir> <required.conf>
24// docroot the live compare docroot holding <dom>/index.html (sites/nishifamily/compare from nishihost)
25// kdir the knowledge compare tree the regen READS (buildroot/knowledge/compare)
26// required the declared shape
27// Prints one DOM line per page (sections carried, missing named), one MISSING row per absent required
28// section, the union table (title, domains carrying it), then the partition WITH its sums, then the
29// verdict LAST (gv_last_line anchors by position). Exit: 0 GREEN (no page misses a required section) .
30// 1 RED (at least one MISSING row) . 2 usage . 3 UNOBSERVABLE (docroot or conf unreadable, or zero pages
31// read -- an empty population is never a pass).
32// license_tier: ORIGINAL Read-only on every path it touches. No hw writes (Rule 26).
33import "nx_syscalls.nx"
34
35const CS_DENTBUF: i64 = 262144 // one getdents64 window; the loop drains until 0 so this is a window, not a cap
36const CS_PATH: i64 = 4096
37const CS_SLOT: i64 = 96 // a domain directory name; the longest on the surface is 15 bytes
38const CS_MAX_DOMS: i64 = 512 // regen.list 101 + hand.roster 11 = 112 today; 512 is 4x headroom and ANNOUNCES when hit
39const CS_TITLE: i64 = 128 // a normalised <h2> title; the longest declared prefix is 21 bytes, the longest emitted head before — ~60
40const CS_MAX_SECT: i64 = 512 // distinct titles across the surface: two generators emit ~20 and the hand pages a few dozen more
41const CS_PAGE_SECT: i64 = 96 // distinct titles on ONE page: the richest page today carries 14
42const CS_MAX_REQ: i64 = 64 // rows in sections.required: 10 declared today
43const CS_EXT: i64 = 16
44const CS_DOT: i64 = 46
45const CS_DT_DIR: i64 = 4
46const CS_DT_UNKNOWN: i64 = 0 // an unknown d_type is TAKEN (the page read decides), never skipped
47const CS_LT: i64 = 60 // <
48const CS_GT: i64 = 62 // >
49const CS_AMP: i64 = 38 // &
50const CS_SP: i64 = 32
51const CS_NL: i64 = 10
52const CS_PIPE: i64 = 124
53const CS_HASH: i64 = 35
54const CS_EXIT_GREEN: i64 = 0
55const CS_EXIT_RED: i64 = 1
56const CS_EXIT_USAGE: i64 = 2
57const CS_EXIT_UNOBS: i64 = 3
58
59func cs_slen(s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { i = i + 1 } return i }
60func cs_puts(fd: i64, s: *u8) -> i64 { return sys_write(fd, s, cs_slen(s)) }
61func cs_putn(fd: i64, v: i64) -> i64 {
62 let t: *u8 = sys_mmap(32)
63 var m: i64 = v
64 var k: i64 = 0
65 var neg: i64 = 0
66 if m < 0 { neg = 1; m = 0 - m }
67 if m == 0 { t[0] = 48 as u8; k = 1 }
68 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
69 let o: *u8 = sys_mmap(34)
70 var p: i64 = 0
71 if neg == 1 { o[0] = 45 as u8; p = 1 }
72 var j: i64 = 0
73 while j < k { o[p] = t[k - 1 - j]; p = p + 1; j = j + 1 }
74 sys_write(fd, o, p)
75 sys_munmap(t, 32); sys_munmap(o, 34)
76 return p
77}
78func cs_cat(d: *u8, o: i64, s: *u8) -> i64 { var i: i64 = 0; var p: i64 = o; while s[i] != (0 as u8) { d[p] = s[i]; p = p + 1; i = i + 1 } d[p] = 0 as u8; return p }
79func cs_eq(a: *u8, b: *u8) -> i64 { var i: i64 = 0; while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 } if b[i] != (0 as u8) { return 0 } return 1 }
80func cs_starts(hay: *u8, pre: *u8) -> i64 { var i: i64 = 0; while pre[i] != (0 as u8) { if hay[i] != pre[i] { return 0 } i = i + 1 } return 1 }
81func cs_slot(base: *u8, i: i64, stride: i64) -> *u8 { return ((base as i64) + i * stride) as *u8 }
82func cs_exists(path: *u8) -> i64 { let fd: i64 = sys_openat_rd(path); if fd < 0 { return 0 } sys_close(fd); return 1 }
83
84// bytes at b[i..] equal the literal s (no NUL scan of b: the page is not NUL-terminated per line)
85func cs_at(b: *u8, n: i64, i: i64, s: *u8) -> i64 {
86 var k: i64 = 0
87 while s[k] != (0 as u8) {
88 if i + k >= n { return 0 }
89 if b[i + k] != s[k] { return 0 }
90 k = k + 1
91 }
92 return 1
93}
94
95// Enumerate the docroot's top-level directories into names (CS_SLOT stride). out[0] = capped flag.
96func cs_dirs(root: *u8, names: *u8, cap: i64, out: *i64) -> i64 {
97 let fd: i64 = sys_openat_rd(root)
98 if fd < 0 { return 0 - 1 }
99 let dbuf: *u8 = sys_mmap(CS_DENTBUF)
100 var n: i64 = 0
101 var run: i64 = 1
102 while run == 1 {
103 let got: i64 = sys_getdents64(fd, dbuf, CS_DENTBUF)
104 if got <= 0 { run = 0 } else {
105 var off: i64 = 0
106 var inner: i64 = 1
107 while inner == 1 {
108 if off >= got { inner = 0 } else {
109 let rec: *u8 = ((dbuf as i64) + off) as *u8
110 let reclen: i64 = dirent_reclen(rec)
111 if reclen <= 0 { inner = 0 } else {
112 let name: *u8 = dirent_name(rec)
113 var take: i64 = 1
114 if (name[0] as i64) == CS_DOT { take = 0 }
115 let dt: i64 = dirent_type(rec)
116 if dt != CS_DT_DIR { if dt != CS_DT_UNKNOWN { take = 0 } }
117 if take == 1 {
118 if n >= cap { out[0] = 1 } else {
119 var q: i64 = 0
120 let dst: *u8 = cs_slot(names, n, CS_SLOT)
121 while name[q] != (0 as u8) { if q < CS_SLOT - 1 { dst[q] = name[q] } q = q + 1 }
122 if q > CS_SLOT - 1 { q = CS_SLOT - 1 }
123 dst[q] = 0 as u8
124 n = n + 1
125 }
126 }
127 off = off + reclen
128 }
129 }
130 }
131 }
132 }
133 sys_close(fd)
134 sys_munmap(dbuf, CS_DENTBUF)
135 return n
136}
137
138// Copy the text of one <h2 ...>...</h2> starting at the byte AFTER the opening tag's '>' into title:
139// tags inside are skipped, the copy stops at "</h2>", at " —" (the generators' explanatory tail),
140// or at CS_TITLE-1 bytes; trailing spaces are trimmed. Returns the byte index just past "</h2>" (or n).
141func cs_h2_text(b: *u8, n: i64, start: i64, title: *u8) -> i64 {
142 var i: i64 = start
143 var o: i64 = 0
144 var intag: i64 = 0
145 var cut: i64 = 0
146 while i < n {
147 if cs_at(b, n, i, "</h2>" as *u8) == 1 { i = i + 5; break }
148 let c: i64 = b[i] as i64
149 if intag == 1 {
150 if c == CS_GT { intag = 0 }
151 } else {
152 if c == CS_LT { intag = 1 } else {
153 if cut == 0 {
154 if cs_at(b, n, i, "—" as *u8) == 1 { cut = 1 } else {
155 if o < CS_TITLE - 1 {
156 var cc: i64 = c
157 if cc == CS_NL { cc = CS_SP }
158 title[o] = cc as u8
159 o = o + 1
160 }
161 }
162 }
163 }
164 }
165 i = i + 1
166 }
167 while o > 0 { if (title[o - 1] as i64) == CS_SP { o = o - 1 } else { break } }
168 title[o] = 0 as u8
169 return i
170}
171
172// All distinct <h2> titles of one page into ptitles (CS_TITLE stride, at most cap). Returns the count.
173func cs_page_sections(b: *u8, n: i64, ptitles: *u8, cap: i64, out: *i64) -> i64 {
174 var i: i64 = 0
175 var np: i64 = 0
176 let tmp: *u8 = sys_mmap(CS_TITLE)
177 while i < n {
178 if cs_at(b, n, i, "<h2" as *u8) == 1 {
179 // skip to the end of the opening tag
180 var j: i64 = i + 3
181 while j < n { if (b[j] as i64) == CS_GT { break } j = j + 1 }
182 if j >= n { i = n } else {
183 let nxt: i64 = cs_h2_text(b, n, j + 1, tmp)
184 if tmp[0] != (0 as u8) {
185 var dup: i64 = 0
186 var k: i64 = 0
187 while k < np { if cs_eq(cs_slot(ptitles, k, CS_TITLE), tmp) == 1 { dup = 1; k = np } k = k + 1 }
188 if dup == 0 {
189 if np >= cap { out[0] = 1 } else { cs_cat(cs_slot(ptitles, np, CS_TITLE), 0, tmp); np = np + 1 }
190 }
191 }
192 i = nxt
193 }
194 } else { i = i + 1 }
195 }
196 sys_munmap(tmp, CS_TITLE)
197 return np
198}
199
200// intern a title into the surface-wide table; returns its index, or -1 when the table is full
201func cs_intern(title: *u8, gt: *u8, gn: *i64, cap: i64) -> i64 {
202 var k: i64 = 0
203 while k < gn[0] { if cs_eq(cs_slot(gt, k, CS_TITLE), title) == 1 { return k } k = k + 1 }
204 if gn[0] >= cap { return 0 - 1 }
205 cs_cat(cs_slot(gt, gn[0], CS_TITLE), 0, title)
206 gn[0] = gn[0] + 1
207 return gn[0] - 1
208}
209
210// Parse sections.required: rows `section|<prefix>|<ext or ->`. Returns the row count; prefixes and exts
211// land in their tables (CS_TITLE and CS_EXT strides). Comment and blank lines are skipped.
212func cs_required(path: *u8, pre: *u8, ext: *u8, cap: i64) -> i64 {
213 let lp: *i64 = sys_mmap(16) as *i64
214 lp[0] = 0
215 let b: *u8 = sys_read_file(path, lp)
216 let n: i64 = lp[0]
217 if (b as i64) == 0 { return 0 - 1 }
218 if n <= 0 { return 0 - 1 }
219 var nr: i64 = 0
220 var i: i64 = 0
221 while i < n {
222 var e: i64 = i
223 while e < n { if (b[e] as i64) == CS_NL { break } e = e + 1 }
224 // line is b[i..e)
225 if e > i { if (b[i] as i64) != CS_HASH { if cs_at(b, n, i, "section|" as *u8) == 1 {
226 var p: i64 = i + 8
227 var o: i64 = 0
228 let pd: *u8 = cs_slot(pre, nr, CS_TITLE)
229 while p < e { if (b[p] as i64) == CS_PIPE { break } if o < CS_TITLE - 1 { pd[o] = b[p]; o = o + 1 } p = p + 1 }
230 pd[o] = 0 as u8
231 let xd: *u8 = cs_slot(ext, nr, CS_EXT)
232 var xo: i64 = 0
233 if p < e { p = p + 1; while p < e { if (b[p] as i64) == CS_PIPE { break } if xo < CS_EXT - 1 { xd[xo] = b[p]; xo = xo + 1 } p = p + 1 } }
234 xd[xo] = 0 as u8
235 if o > 0 { if nr < cap { nr = nr + 1 } }
236 } } }
237 i = e + 1
238 }
239 sys_free_file(b, n)
240 return nr
241}
242
243func main(argc: i64, argv: *i64) -> i64 {
244 if argc < 4 {
245 cs_puts(1, "usage: nx_compare_sections <docroot> <knowledge-compare-dir> <sections.required>\nverdict=USAGE\n" as *u8)
246 sys_exit(CS_EXIT_USAGE); return CS_EXIT_USAGE
247 }
248 let root: *u8 = argv[1] as *u8
249 let kdir: *u8 = argv[2] as *u8
250 let reqp: *u8 = argv[3] as *u8
251
252 let rpre: *u8 = sys_mmap(CS_MAX_REQ * CS_TITLE)
253 let rext: *u8 = sys_mmap(CS_MAX_REQ * CS_EXT)
254 let nreq: i64 = cs_required(reqp, rpre, rext, CS_MAX_REQ)
255 if nreq < 0 {
256 cs_puts(1, "NX-COMPARE-SECTIONS required-shape file unreadable: " as *u8); cs_puts(1, reqp); cs_puts(1, "\nverdict=UNOBSERVABLE\n" as *u8)
257 sys_exit(CS_EXIT_UNOBS); return CS_EXIT_UNOBS
258 }
259
260 let names: *u8 = sys_mmap(CS_MAX_DOMS * CS_SLOT)
261 let dout: *i64 = sys_mmap(16) as *i64
262 dout[0] = 0
263 let ndom: i64 = cs_dirs(root, names, CS_MAX_DOMS, dout)
264 if ndom < 0 {
265 cs_puts(1, "NX-COMPARE-SECTIONS docroot unreadable: " as *u8); cs_puts(1, root); cs_puts(1, "\nverdict=UNOBSERVABLE\n" as *u8)
266 sys_exit(CS_EXIT_UNOBS); return CS_EXIT_UNOBS
267 }
268
269 cs_puts(1, "=== NX-COMPARE-SECTIONS docroot=" as *u8); cs_puts(1, root); cs_puts(1, " kdir=" as *u8); cs_puts(1, kdir)
270 cs_puts(1, " required_rows=" as *u8); cs_putn(1, nreq); cs_puts(1, " ===\n" as *u8)
271
272 let gt: *u8 = sys_mmap(CS_MAX_SECT * CS_TITLE) // surface-wide distinct titles
273 let gcount: *i64 = sys_mmap(CS_MAX_SECT * 8) as *i64
274 let gn: *i64 = sys_mmap(16) as *i64
275 gn[0] = 0
276 var gcapped: i64 = 0
277 let ptitles: *u8 = sys_mmap(CS_PAGE_SECT * CS_TITLE)
278 let pout: *i64 = sys_mmap(16) as *i64
279 let path: *u8 = sys_mmap(CS_PATH)
280 let dpath: *u8 = sys_mmap(CS_PATH)
281 let lp: *i64 = sys_mmap(16) as *i64
282
283 var pages_read: i64 = 0
284 var unreadable: i64 = 0
285 var complete: i64 = 0
286 var incomplete: i64 = 0
287 var missing_total: i64 = 0
288 var pcapped: i64 = 0
289 var d: i64 = 0
290 while d < ndom {
291 let dom: *u8 = cs_slot(names, d, CS_SLOT)
292 var o: i64 = cs_cat(path, 0, root); o = cs_cat(path, o, "/" as *u8); o = cs_cat(path, o, dom); o = cs_cat(path, o, "/index.html" as *u8)
293 lp[0] = 0
294 let b: *u8 = sys_read_file(path, lp)
295 let n: i64 = lp[0]
296 if (b as i64) == 0 {
297 unreadable = unreadable + 1
298 cs_puts(1, "UNREADABLE dom=" as *u8); cs_puts(1, dom); cs_puts(1, " (no index.html or not a page directory)\n" as *u8)
299 } else {
300 pages_read = pages_read + 1
301 pout[0] = 0
302 let np: i64 = cs_page_sections(b, n, ptitles, CS_PAGE_SECT, pout)
303 if pout[0] == 1 { pcapped = pcapped + 1 }
304 // union table
305 var k: i64 = 0
306 while k < np {
307 let gi: i64 = cs_intern(cs_slot(ptitles, k, CS_TITLE), gt, gn, CS_MAX_SECT)
308 if gi < 0 { gcapped = 1 } else { gcount[gi] = gcount[gi] + 1 }
309 k = k + 1
310 }
311 // required shape for THIS domain
312 var missing: i64 = 0
313 var r: i64 = 0
314 while r < nreq {
315 let pre: *u8 = cs_slot(rpre, r, CS_TITLE)
316 let ext: *u8 = cs_slot(rext, r, CS_EXT)
317 var required: i64 = 0
318 if cs_eq(ext, "-" as *u8) == 1 { required = 1 } else {
319 var q: i64 = cs_cat(dpath, 0, kdir); q = cs_cat(dpath, q, "/" as *u8); q = cs_cat(dpath, q, dom); q = cs_cat(dpath, q, "." as *u8); q = cs_cat(dpath, q, ext)
320 if cs_exists(dpath) == 1 { required = 1 }
321 }
322 if required == 1 {
323 var have: i64 = 0
324 var t: i64 = 0
325 while t < np { if cs_starts(cs_slot(ptitles, t, CS_TITLE), pre) == 1 { have = 1; t = np } t = t + 1 }
326 if have == 0 {
327 missing = missing + 1
328 cs_puts(1, "MISSING dom=" as *u8); cs_puts(1, dom); cs_puts(1, " section=" as *u8); cs_puts(1, pre)
329 cs_puts(1, " requires=" as *u8); cs_puts(1, ext); cs_puts(1, "\n" as *u8)
330 }
331 }
332 r = r + 1
333 }
334 cs_puts(1, "DOM " as *u8); cs_puts(1, dom); cs_puts(1, " bytes=" as *u8); cs_putn(1, n); cs_puts(1, " sections=" as *u8); cs_putn(1, np)
335 cs_puts(1, " missing=" as *u8); cs_putn(1, missing); cs_puts(1, "\n" as *u8)
336 if missing == 0 { complete = complete + 1 } else { incomplete = incomplete + 1 }
337 missing_total = missing_total + missing
338 sys_free_file(b, n)
339 }
340 d = d + 1
341 }
342
343 // the union table, most-carried first (selection order over at most CS_MAX_SECT rows)
344 cs_puts(1, "--- SECTION UNION (title, pages carrying it, of pages_read=" as *u8); cs_putn(1, pages_read); cs_puts(1, ") ---\n" as *u8)
345 let used: *i64 = sys_mmap(CS_MAX_SECT * 8) as *i64
346 var emitted: i64 = 0
347 while emitted < gn[0] {
348 var best: i64 = 0 - 1
349 var bi: i64 = 0
350 while bi < gn[0] { if used[bi] == 0 { if best < 0 { best = bi } else { if gcount[bi] > gcount[best] { best = bi } } } bi = bi + 1 }
351 used[best] = 1
352 cs_puts(1, " " as *u8); cs_putn(1, gcount[best]); cs_puts(1, " " as *u8); cs_puts(1, cs_slot(gt, best, CS_TITLE)); cs_puts(1, "\n" as *u8)
353 emitted = emitted + 1
354 }
355
356 // partition WITH sums: complete + incomplete = pages_read; pages_read + unreadable = dirs
357 cs_puts(1, "PARTITION dirs=" as *u8); cs_putn(1, ndom); cs_puts(1, " pages_read=" as *u8); cs_putn(1, pages_read); cs_puts(1, " unreadable=" as *u8); cs_putn(1, unreadable)
358 cs_puts(1, " complete=" as *u8); cs_putn(1, complete); cs_puts(1, " incomplete=" as *u8); cs_putn(1, incomplete); cs_puts(1, " missing_total=" as *u8); cs_putn(1, missing_total)
359 cs_puts(1, " distinct_sections=" as *u8); cs_putn(1, gn[0])
360 cs_puts(1, " dirs_capped=" as *u8); cs_putn(1, dout[0]); cs_puts(1, " pages_with_section_cap=" as *u8); cs_putn(1, pcapped); cs_puts(1, " union_capped=" as *u8); cs_putn(1, gcapped)
361 cs_puts(1, " sums: complete+incomplete=" as *u8); cs_putn(1, complete + incomplete); cs_puts(1, " pages_read+unreadable=" as *u8); cs_putn(1, pages_read + unreadable); cs_puts(1, "\n" as *u8)
362
363 if pages_read == 0 {
364 cs_puts(1, "verdict=UNOBSERVABLE (zero pages read -- an empty population is not a pass)\n" as *u8)
365 sys_exit(CS_EXIT_UNOBS); return CS_EXIT_UNOBS
366 }
367 if dout[0] == 1 {
368 cs_puts(1, "verdict=UNOBSERVABLE (the directory table filled: the counts above are a FLOOR, raise CS_MAX_DOMS)\n" as *u8)
369 sys_exit(CS_EXIT_UNOBS); return CS_EXIT_UNOBS
370 }
371 if missing_total > 0 {
372 cs_puts(1, "verdict=RED (" as *u8); cs_putn(1, incomplete); cs_puts(1, " page(s) lack a required section; every MISSING row above names the domain, the section and the data that requires it)\n" as *u8)
373 sys_exit(CS_EXIT_RED); return CS_EXIT_RED
374 }
375 cs_puts(1, "verdict=GREEN (every page carries every section its data requires)\n" as *u8)
376 sys_exit(CS_EXIT_GREEN); return CS_EXIT_GREEN
377}