nx_outbench.nx source

1// nx_outbench.nx -- OUTPUT-CORRECTNESS BENCH: does a generated site's OUTPUT hold together? 2// 3// ★WHY THIS EXISTS (the gap this lane measured and filed, 2026-07-25): nothing scored whether generated 4// output is CORRECT. A generation census scores CAPABILITY ("can it generate"); a craft judge scores 5// design MARKERS on ONE finished page. Neither asks the questions that caught every real defect in this 6// lane -- each of which was LIVE and passed every craft/a11y/contrast checker: 7// * a coffee shop's product page selling the GENERATOR'S OWN software subscriptions (leak) 8// * a site shipping TWO visual identities across its pages (skin) 9// * a tool reporting success having written NOTHING (set) 10// * links pointing at pages that were never emitted (link) 11// So a generator can score well on both existing rulers while shipping broken sites. 12// 13// GENERATOR-AGNOSTIC BY CONSTRUCTION: it reads a DIRECTORY of emitted files and needs no knowledge of who 14// produced them, so it runs against the field's output exactly as it runs against ours. Offline -- no 15// network, no live edge, no vantage problem (deliberately complementary to the live link sentinel). 16// usage: nx_outbench bench <dir> [leakterm ...] | nx_outbench selftest 17// Scores four classes in permille and an overall (worst-class-weighted) figure. 18// expect_exit: 0 license_tier: ORIGINAL 19import "nx_syscalls.nx" 20import "nx_emit_guard.nx" 21const OB_MAGIC_1024: i64 = 1024 22 23const OB_MAXFILES: i64 = 256 24const OB_NAMEW: i64 = 128 25const OB_DIRBUF: i64 = 65536 26const OB_FILEBUF: i64 = 1048576 27const OB_PERMILLE: i64 = 1000 28 29func ob_say(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 } 30func ob_num(v: i64) -> i64 { if v == 0 { sys_write(1, "0" as *u8, 1); return 0 } var m: i64 = v; if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m } let t: *u8 = sys_mmap(28); var k: i64 = 0; while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 } while k > 0 { k = k - 1; sys_write(1, (((t as i64) + k) as *u8), 1) } return 0 } 31func ob_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n } 32func ob_slot(base: *u8, i: i64) -> *u8 { return (((base as i64) + i * OB_NAMEW) as *u8) } 33func ob_eq(a: *u8, b: *u8) -> i64 { 34 var i: i64 = 0 35 while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 } 36 if b[i] != (0 as u8) { return 0 } 37 return 1 38} 39// substring search in a length-bounded buffer 40func ob_find(buf: *u8, len: i64, lit: *u8, from: i64) -> i64 { 41 let ll: i64 = ob_len(lit) 42 if ll == 0 { return 0 - 1 } 43 var i: i64 = from 44 while i + ll <= len { 45 var j: i64 = 0 46 var ok: i64 = 1 47 while j < ll { if buf[i + j] != lit[j] { ok = 0; j = ll } else { j = j + 1 } } 48 if ok == 1 { return i } 49 i = i + 1 50 } 51 return 0 - 1 52} 53func ob_ends(name: *u8, suf: *u8) -> i64 { 54 let nl: i64 = ob_len(name) 55 let sl: i64 = ob_len(suf) 56 if sl > nl { return 0 } 57 var i: i64 = 0 58 while i < sl { if name[nl - sl + i] != suf[i] { return 0 } i = i + 1 } 59 return 1 60} 61func ob_join(dst: *u8, dir: *u8, name: *u8) -> i64 { 62 var o: i64 = 0 63 var i: i64 = 0 64 while dir[i] != (0 as u8) { dst[o] = dir[i]; o = o + 1; i = i + 1 } 65 dst[o] = 47 as u8 66 o = o + 1 67 i = 0 68 while name[i] != (0 as u8) { dst[o] = name[i]; o = o + 1; i = i + 1 } 69 dst[o] = 0 as u8 70 return o 71} 72func ob_has_name(names: *u8, n: i64, want: *u8) -> i64 { 73 var i: i64 = 0 74 while i < n { if ob_eq(ob_slot(names, i), want) == 1 { return 1 } i = i + 1 } 75 return 0 76} 77 78// ---- CLASS 1: LINK RESOLUTION. every plain internal .html href must name a file that exists ---- 79func ob_links(buf: *u8, len: i64, names: *u8, nfiles: i64, tmp: *u8, tot: *i64) -> i64 { 80 var bad: i64 = 0 81 var at: i64 = ob_find(buf, len, "href=\"" as *u8, 0) 82 while at >= 0 { 83 var m: i64 = at + 6 84 var tl: i64 = 0 85 while m < len { 86 if buf[m] == (34 as u8) { m = len } else { 87 if tl < OB_NAMEW - 8 { tmp[tl] = buf[m]; tl = tl + 1 } 88 m = m + 1 89 } 90 } 91 tmp[tl] = 0 as u8 92 var plain: i64 = 1 93 var q: i64 = 0 94 while q < tl { if tmp[q] == (47 as u8) { plain = 0 } if tmp[q] == (58 as u8) { plain = 0 } if tmp[q] == (35 as u8) { plain = 0 } q = q + 1 } 95 if plain == 1 { if ob_ends(tmp, ".html" as *u8) == 1 { 96 tot[0] = tot[0] + 1 97 if ob_has_name(names, nfiles, tmp) == 0 { bad = bad + 1 } 98 } } 99 at = ob_find(buf, len, "href=\"" as *u8, at + 6) 100 } 101 return bad 102} 103 104// ---- CLASS 3: DESIGN SIGNATURE. the first CSS custom-property prefix a page declares ("--kc-", "--nx-") 105// identifies its design language without knowing anything about the generator. Pages that disagree 106// with the plurality signature are shipping a second visual identity. ---- 107func ob_signature(buf: *u8, len: i64, out: *u8) -> i64 { 108 let at: i64 = ob_find(buf, len, "--" as *u8, 0) 109 if at < 0 { out[0] = 0 as u8; return 0 } 110 var o: i64 = 0 111 var i: i64 = at 112 // capture "--word-" up to and including the first dash after the prefix word 113 while i < len { 114 let c: i64 = buf[i] as i64 115 if o >= 24 { i = len } else { 116 if c == 58 { i = len } else { 117 out[o] = buf[i] 118 o = o + 1 119 if o > 3 { if c == 45 { i = len } } 120 i = i + 1 121 } 122 } 123 } 124 out[o] = 0 as u8 125 return o 126} 127 128func main(argc: i64, argv: *i64) -> i64 { 129 if argc < 2 { 130 ob_say("usage: nx_outbench {bench <dir> [leakterm ...] | selftest}\n" as *u8) 131 ob_say(" offline, generator-agnostic correctness bench over a directory of emitted pages\n" as *u8) 132 sys_exit(2) 133 return 2 134 } 135 let verb: *u8 = argv[1] as *u8 136 var dir: *u8 = "web_assets" as *u8 137 var selftest: i64 = 0 138 if ob_eq(verb, "selftest" as *u8) == 1 { selftest = 1; dir = "web_assets/obkat" as *u8 } 139 else { if argc < 3 { ob_say("usage: nx_outbench bench <dir>\n" as *u8); sys_exit(2); return 2 } dir = argv[2] as *u8 } 140 141 // ---- enumerate the directory (generator-agnostic: we are told nothing, we look) ---- 142 let names: *u8 = eg_buf(OB_MAXFILES * OB_NAMEW, "outbench name table" as *u8) 143 let dbuf: *u8 = eg_buf(OB_DIRBUF, "outbench dirent buffer" as *u8) 144 let fbuf: *u8 = eg_buf(OB_FILEBUF, "outbench file buffer" as *u8) 145 let tmp: *u8 = eg_buf(OB_NAMEW, "outbench scratch" as *u8) 146 let sig: *u8 = eg_buf(OB_NAMEW, "outbench signature" as *u8) 147 let sigs: *u8 = eg_buf(OB_MAXFILES * OB_NAMEW, "outbench signature table" as *u8) 148 let path: *u8 = eg_buf(OB_MAGIC_1024, "outbench path" as *u8) 149 var nfiles: i64 = 0 150 let fd: i64 = sys_openat_rd(dir) 151 if fd < 0 { ob_say("OUTBENCH-FAIL cannot open dir " as *u8); ob_say(dir); ob_say("\n" as *u8); sys_exit(3); return 3 } 152 var got: i64 = sys_getdents64(fd, dbuf, OB_DIRBUF) 153 while got > 0 { 154 var off: i64 = 0 155 while off < got { 156 let rec: *u8 = (((dbuf as i64) + off) as *u8) 157 let nm: *u8 = dirent_name(rec) 158 if nm[0] != (46 as u8) { 159 if nfiles < OB_MAXFILES { 160 var k: i64 = 0 161 let slot: *u8 = ob_slot(names, nfiles) 162 while nm[k] != (0 as u8) { if k < OB_NAMEW - 1 { slot[k] = nm[k] } k = k + 1 } 163 slot[k] = 0 as u8 164 nfiles = nfiles + 1 165 } 166 } 167 off = off + dirent_reclen(rec) 168 } 169 got = sys_getdents64(fd, dbuf, OB_DIRBUF) 170 } 171 sys_close(fd) 172 173 // ---- walk every .html page: links + signature + leak ---- 174 let ltot: *i64 = eg_buf(8, "outbench link counter") as *i64 175 ltot[0] = 0 176 var lbad: i64 = 0 177 var pages: i64 = 0 178 var leakhits: i64 = 0 179 var i: i64 = 0 180 while i < nfiles { 181 let nm: *u8 = ob_slot(names, i) 182 if ob_ends(nm, ".html" as *u8) == 1 { 183 ob_join(path, dir, nm) 184 let flen: *i64 = eg_buf(8, "outbench flen") as *i64 185 let raw: *u8 = sys_read_file(path, flen) 186 if (raw as i64) != 0 { 187 let n: i64 = flen[0] 188 lbad = lbad + ob_links(raw, n, names, nfiles, tmp, ltot) 189 ob_signature(raw, n, sig) 190 var k: i64 = 0 191 let ss: *u8 = ob_slot(sigs, pages) 192 while sig[k] != (0 as u8) { ss[k] = sig[k]; k = k + 1 } 193 ss[k] = 0 as u8 194 // leak terms: universal placeholder tells, plus any the caller adds 195 if ob_find(raw, n, "Lorem ipsum" as *u8, 0) >= 0 { leakhits = leakhits + 1 } 196 if ob_find(raw, n, "lorem ipsum" as *u8, 0) >= 0 { leakhits = leakhits + 1 } 197 if ob_find(raw, n, "Your Company" as *u8, 0) >= 0 { leakhits = leakhits + 1 } 198 if ob_find(raw, n, "Placeholder" as *u8, 0) >= 0 { leakhits = leakhits + 1 } 199 if ob_find(raw, n, "TODO" as *u8, 0) >= 0 { leakhits = leakhits + 1 } 200 // NOTE: "example.com" is deliberately NOT a default leak term. RFC 2606 reserves it for 201 // exactly this legitimate use, and a correct contact form carries jane@example.com as a 202 // field PLACEHOLDER. The bench's first real run flagged our own correct forms on it -- 203 // a false positive in this rule, found by pointing the ruler at its author. Callers whose 204 // context warrants it can still pass it as an explicit term. 205 var ai: i64 = 3 206 while ai < argc { 207 if ob_find(raw, n, argv[ai] as *u8, 0) >= 0 { leakhits = leakhits + 1 } 208 ai = ai + 1 209 } 210 pages = pages + 1 211 } 212 } 213 i = i + 1 214 } 215 216 // ---- class scores ---- 217 // ★ NOT-APPLICABLE, never a vacuous pass. Benching the FIELD exposed this: their pages use absolute 218 // and clean URLs, so ZERO plain .html hrefs were found and the class reported a PERFECT 1000 while 219 // checking NOTHING. A ruler that scores full marks on an unexamined property is lying by omission 220 // (the estate law: unproven = ABSENT). -1 means the class could not be assessed on this input. 221 var link_pm: i64 = 0 - 1 222 if ltot[0] > 0 { link_pm = ((ltot[0] - lbad) * OB_PERMILLE) / ltot[0] } 223 // set completeness: pages exist, sitemap present, robots present 224 var setparts: i64 = 0 225 if pages > 0 { setparts = setparts + 1 } 226 if ob_has_name(names, nfiles, "sitemap.xml" as *u8) == 1 { setparts = setparts + 1 } 227 if ob_has_name(names, nfiles, "robots.txt" as *u8) == 1 { setparts = setparts + 1 } 228 let set_pm: i64 = (setparts * OB_PERMILLE) / 3 229 // design signature agreement vs the plurality 230 var best: i64 = 0 231 var bi: i64 = 0 232 while bi < pages { 233 var c: i64 = 0 234 var bj: i64 = 0 235 while bj < pages { if ob_eq(ob_slot(sigs, bi), ob_slot(sigs, bj)) == 1 { c = c + 1 } bj = bj + 1 } 236 if c > best { best = c } 237 bi = bi + 1 238 } 239 var skin_pm: i64 = OB_PERMILLE 240 if pages > 0 { skin_pm = (best * OB_PERMILLE) / pages } 241 var leak_pm: i64 = OB_PERMILLE 242 if pages > 0 { if leakhits > 0 { leak_pm = 0 } } 243 // overall = the WORST class (a site is only as correct as its weakest guarantee) 244 // overall = worst of the APPLICABLE classes only; a class that could not be assessed neither 245 // helps nor hurts, and the count of applicable classes is reported so the figure can be judged. 246 var overall: i64 = OB_PERMILLE 247 var applicable: i64 = 0 248 if link_pm >= 0 { applicable = applicable + 1; if link_pm < overall { overall = link_pm } } 249 if set_pm >= 0 { applicable = applicable + 1; if set_pm < overall { overall = set_pm } } 250 if skin_pm >= 0 { applicable = applicable + 1; if skin_pm < overall { overall = skin_pm } } 251 if leak_pm >= 0 { applicable = applicable + 1; if leak_pm < overall { overall = leak_pm } } 252 if applicable == 0 { overall = 0 - 1 } 253 254 ob_say("{\"organ\":\"nx_outbench\",\"v\":1,\"dir\":\"" as *u8); ob_say(dir) 255 ob_say("\",\"files\":" as *u8); ob_num(nfiles) 256 ob_say(",\"pages\":" as *u8); ob_num(pages) 257 ob_say(",\"link_permille\":" as *u8); ob_num(link_pm) 258 ob_say(",\"links_checked\":" as *u8); ob_num(ltot[0]) 259 ob_say(",\"links_unresolved\":" as *u8); ob_num(lbad) 260 ob_say(",\"set_permille\":" as *u8); ob_num(set_pm) 261 ob_say(",\"skin_permille\":" as *u8); ob_num(skin_pm) 262 ob_say(",\"leak_permille\":" as *u8); ob_num(leak_pm) 263 ob_say(",\"leak_hits\":" as *u8); ob_num(leakhits) 264 ob_say(",\"overall_permille\":" as *u8); ob_num(overall) 265 ob_say(",\"applicable_classes\":" as *u8); ob_num(applicable) 266 ob_say(",\"env\":\"offline; ONE SITE PER DIRECTORY assumed (a folder of unrelated pages makes skin meaningless); link class requires RELATIVE .html hrefs and reports -1 NOT-APPLICABLE for absolute/clean-URL conventions rather than a vacuous pass; set assumes a fully emitted site, not a capture; overall = worst APPLICABLE class\"}\n" as *u8) 267 268 if selftest == 1 { 269 // the fixture is authored to be BROKEN in every class; a bench that scores it well is useless 270 var fails: i64 = 0 271 if lbad < 1 { fails = fails + 1 } 272 if link_pm < 0 { fails = fails + 1 } 273 if leakhits < 1 { fails = fails + 1 } 274 if skin_pm >= OB_PERMILLE { fails = fails + 1 } 275 if set_pm >= OB_PERMILLE { fails = fails + 1 } 276 ob_say("NX-OUTBENCH selftest fails=" as *u8); ob_num(fails) 277 if fails == 0 { ob_say(" verdict=GREEN (a deliberately broken fixture scored badly in ALL FOUR classes)\n" as *u8); sys_exit(0); return 0 } 278 ob_say(" verdict=RED (the bench failed to detect a defect it was shown)\n" as *u8) 279 sys_exit(1) 280 return 1 281 } 282 sys_exit(0) 283 return 0 284}