nx_wpt_reftest.nx source
↩ module page · 394 lines · 19482 B
1// nx_wpt_reftest.nx -- BR1 vNext: THE SOVEREIGN WPT REFTEST RUNNER (2026-08-19).
2//
3// The css half of the BR1 slice. A WPT reftest declares <link rel="match" href="ref.html"> (or
4// rel="mismatch"): the test PASSES when its render is pixel-identical to (match) / different from
5// (mismatch) the reference. This runner paints BOTH files through the browser's own render core
6// (br_layout + br_draw_fb) into FIXED 800x600 framebuffers -- the WPT viewport semantic, so a
7// legitimately different full-page height cannot fail two identical viewports -- and compares the
8// raw pixel buffers byte-for-byte. No PNG, no OCR, no tolerance: the engine is bit-deterministic,
9// so equality is exact and every failure names a real render difference.
10//
11// Telemetry rides the shared spine (nx_wpt_spine): content-addressed (engine sha, corpus sha)
12// skip with --force, one TSV row per counted run (own history file -- reftest columns are NOT the
13// testharness columns; overloading them would be a bucket named for the wrong thing), REGRESSION
14// vs the newest same-corpus row exits 4, and every verdict exit CARRIES its code via sys_exit
15// (the runner's own regression bite caught the bare-return-blesses-failure defect on day one).
16//
17// CONTROLS FIRST, synthesized at runtime: a self-match (same bytes twice) MUST pass and a
18// known-mismatch (two different pages) MUST differ, or the harness refuses to count.
19//
20// DECLARED v1 imprecisions (each a named bucket or counter, never silent):
21// - first rel=match link only (WPT any-of alternates: files with >1 counted in multi_ref);
22// - <meta name=fuzzy> tolerance is UNIMPLEMENTED -- fuzzy tests are compared EXACT and the
23// fuzzy_meta counter travels with the claim so a fail there reads as "stricter than spec";
24// - .xht/.xml tests are out of scope (html only); scripts in reftests do not run (static lane).
25//
26// row: epoch dir engine16 corpus16 files with_ref match_pass match_fail mism_pass
27// mism_fail no_ref ref_missing render_fail fuzzy_meta dur_ms verdict
28// usage: nx_wpt_reftest <dir> [historyfile] [--force]
29// exit: 0 counted | 2 harness unproven | 3 usage | 4 REGRESSION
30// license_tier: ORIGINAL expect_exit: 0
31
32import "nx_browser_render.nx"
33import "nx_wpt_spine.nx"
34
35const RT_VW: i64 = 800 // the WPT screenshot viewport, both dimensions fixed for BOTH sides
36const RT_VH: i64 = 600
37
38func rw(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
39func rwn(v: i64) -> i64 {
40 var m: i64 = v; if m < 0 { rw("-" as *u8); m = 0 - m }
41 let t: *u8 = sys_mmap(24); var k: i64 = 0; if m == 0 { t[0] = 48 as u8; k = 1 }
42 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
43 let o: *u8 = sys_mmap(24); var i: i64 = 0; while i < k { o[i] = t[k-1-i]; i = i + 1 }
44 sys_write(1, o, k); return 0
45}
46func rcat(dst: *u8, off: i64, src: *u8) -> i64 { var i: i64 = 0; while src[i] != (0 as u8) { dst[off+i] = src[i]; i = i + 1 } return off + i }
47func rcatnum(dst: *u8, off: i64, v: i64) -> i64 {
48 var m: i64 = v; var o: i64 = off
49 if m < 0 { dst[o] = 45 as u8; o = o + 1; m = 0 - m }
50 let t: *u8 = sys_mmap(24); var k: i64 = 0; if m == 0 { t[0] = 48 as u8; k = 1 }
51 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
52 var i: i64 = 0; while i < k { dst[o] = t[k-1-i]; o = o + 1; i = i + 1 }
53 return o
54}
55func rlower(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c }
56// case-insensitive substring find in hay[from..n): returns offset or -1
57func rfind_ci(hay: *u8, from: i64, n: i64, needle: *u8) -> i64 {
58 var nl: i64 = 0
59 while needle[nl] != (0 as u8) { nl = nl + 1 }
60 var i: i64 = from
61 while i + nl <= n {
62 var k: i64 = 0
63 var hit: i64 = 1
64 while k < nl { if rlower(hay[i+k] as i64) != rlower(needle[k] as i64) { hit = 0; k = nl } else { k = k + 1 } }
65 if hit == 1 { return i }
66 i = i + 1
67 }
68 return 0 - 1
69}
70// extract the FIRST rel=match / rel=mismatch link href from html. kindout[0]: 1=match 2=mismatch 0=none.
71// nlinks counts ALL match/mismatch links (multi_ref visibility). fuzzy[0]=1 when <meta name=fuzzy present.
72func rt_extract_ref(html: *u8, n: i64, hrefout: *u8, hcap: i64, kindout: *i64, nlinks: *i64, fuzzy: *i64) -> i64 {
73 kindout[0] = 0
74 nlinks[0] = 0
75 fuzzy[0] = 0
76 if rfind_ci(html, 0, n, "name=fuzzy\x00" as *u8) >= 0 { fuzzy[0] = 1 } else {
77 if rfind_ci(html, 0, n, "name=\"fuzzy\"\x00" as *u8) >= 0 { fuzzy[0] = 1 } }
78 var pos: i64 = 0
79 var going: i64 = 1
80 while going == 1 {
81 let lt: i64 = rfind_ci(html, pos, n, "<link\x00" as *u8)
82 if lt < 0 { going = 0 } else {
83 var gt: i64 = lt
84 while gt < n { if html[gt] == (62 as u8) { break } gt = gt + 1 }
85 // rel token inside the tag
86 var kind: i64 = 0
87 if rfind_ci(html, lt, gt, "rel=\"mismatch\"\x00" as *u8) >= 0 { kind = 2 } else {
88 if rfind_ci(html, lt, gt, "rel='mismatch'\x00" as *u8) >= 0 { kind = 2 } else {
89 if rfind_ci(html, lt, gt, "rel=mismatch\x00" as *u8) >= 0 { kind = 2 } else {
90 if rfind_ci(html, lt, gt, "rel=\"match\"\x00" as *u8) >= 0 { kind = 1 } else {
91 if rfind_ci(html, lt, gt, "rel='match'\x00" as *u8) >= 0 { kind = 1 } else {
92 if rfind_ci(html, lt, gt, "rel=match\x00" as *u8) >= 0 { kind = 1 } } } } } }
93 if kind != 0 {
94 nlinks[0] = nlinks[0] + 1
95 if kindout[0] == 0 {
96 let hp: i64 = rfind_ci(html, lt, gt, "href=\x00" as *u8)
97 if hp >= 0 {
98 var q: i64 = hp + 5
99 var qc: i64 = 0
100 if html[q] == (34 as u8) { qc = 34; q = q + 1 }
101 if html[q] == (39 as u8) { qc = 39; q = q + 1 }
102 var o: i64 = 0
103 var cop: i64 = 1
104 while cop == 1 {
105 if q >= gt { cop = 0 } else {
106 let ch: i64 = html[q] as i64
107 var stop: i64 = 0
108 if qc != 0 { if ch == qc { stop = 1 } }
109 if qc == 0 { if ch == 32 { stop = 1 } }
110 if ch == 62 { stop = 1 }
111 if stop == 1 { cop = 0 } else {
112 if o < (hcap - 2) { hrefout[o] = ch as u8; o = o + 1 }
113 q = q + 1
114 }
115 }
116 }
117 hrefout[o] = 0 as u8
118 if o > 0 { kindout[0] = kind }
119 }
120 }
121 }
122 pos = gt + 1
123 }
124 }
125 return 0
126}
127// join dir + href and normalize ./ and ../ segments. 0 ok / 1 escapes the tree.
128func rt_resolve(dir: *u8, href: *u8, out: *u8) -> i64 {
129 let tmp: *u8 = sys_mmap(4096)
130 var o: i64 = rcat(tmp, 0, dir)
131 tmp[o] = 47 as u8; o = o + 1
132 o = rcat(tmp, o, href)
133 tmp[o] = 0 as u8
134 // normalize into out via a segment stack of offsets
135 let segs: *i64 = sys_mmap(8 * 256) as *i64
136 var nseg: i64 = 0
137 var i: i64 = 0
138 var start: i64 = 0
139 var abs: i64 = 0
140 if tmp[0] == (47 as u8) { abs = 1; start = 1; i = 1 }
141 var going: i64 = 1
142 while going == 1 {
143 let c: i64 = tmp[i] as i64
144 if c == 47 { if i > start {
145 // segment [start,i)
146 var dots: i64 = 0
147 if (i - start) == 1 { if tmp[start] == (46 as u8) { dots = 1 } }
148 if (i - start) == 2 { if tmp[start] == (46 as u8) { if tmp[start+1] == (46 as u8) { dots = 2 } } }
149 if dots == 0 { segs[nseg*2] = start; segs[nseg*2+1] = i - start; nseg = nseg + 1 }
150 if dots == 2 { if nseg > 0 { nseg = nseg - 1 } else { return 1 } }
151 }
152 start = i + 1
153 }
154 if c == 0 {
155 if i > start {
156 var dots2: i64 = 0
157 if (i - start) == 1 { if tmp[start] == (46 as u8) { dots2 = 1 } }
158 if (i - start) == 2 { if tmp[start] == (46 as u8) { if tmp[start+1] == (46 as u8) { dots2 = 2 } } }
159 if dots2 == 0 { segs[nseg*2] = start; segs[nseg*2+1] = i - start; nseg = nseg + 1 }
160 if dots2 == 2 { if nseg > 0 { nseg = nseg - 1 } else { return 1 } }
161 }
162 going = 0
163 }
164 i = i + 1
165 }
166 var oo: i64 = 0
167 if abs == 1 { out[oo] = 47 as u8; oo = oo + 1 }
168 var si: i64 = 0
169 while si < nseg {
170 if si > 0 { out[oo] = 47 as u8; oo = oo + 1 }
171 var k: i64 = 0
172 while k < segs[si*2+1] { out[oo] = tmp[segs[si*2] + k]; oo = oo + 1; k = k + 1 }
173 si = si + 1
174 }
175 out[oo] = 0 as u8
176 return 0
177}
178// render html bytes at the fixed viewport into px (RT_VW*RT_VH*4). 0 ok / 1 layout failed.
179func rt_render(html: *u8, hlen: i64, px: *u8) -> i64 {
180 let page: *Page = sys_mmap(NX_PAGE_BYTES) as *Page
181 page.raw = html
182 page.raw_len = hlen
183 br_layout(page, RT_VW)
184 if page.ok != 1 { return 1 }
185 let fb: *Framebuffer = sys_mmap(NX_FRAMEBUFFER_BYTES) as *Framebuffer
186 nx_framebuffer_init(fb, px, RT_VW, RT_VH)
187 br_draw_fb(fb, page, RT_VW, "reftest" as *u8, 7, 1)
188 return 0
189}
190func rt_px_equal(a: *u8, b: *u8) -> i64 {
191 let n: i64 = RT_VW * RT_VH * 4
192 var i: i64 = 0
193 while i < n { if a[i] != b[i] { return 0 } i = i + 1 }
194 return 1
195}
196
197func main(argc: i64, argv: *i64) -> i64 {
198 if argc < 2 {
199 rw("usage: nx_wpt_reftest <dir> [historyfile] [--force] (fixed 800x600 viewport, exact pixel compare;\n" as *u8)
200 rw(" runtime self-match + known-mismatch controls REFUSE counts when the harness cannot see; exit 4 = regression)\n" as *u8)
201 sys_exit(3); return 3
202 }
203 let dir: *u8 = argv[1] as *u8
204 var histpath: *u8 = 0 as *u8
205 var force: i64 = 0
206 var ai: i64 = 2
207 while ai < argc {
208 let a: *u8 = argv[ai] as *u8
209 var isf: i64 = 0
210 if a[0] == (45 as u8) { if a[1] == (45 as u8) { isf = 1 } }
211 if isf == 1 { force = 1 } else { histpath = a }
212 ai = ai + 1
213 }
214 let t0: i64 = sys_now_ms()
215
216 // ---- CONTROLS: self-match MUST pass, known-mismatch MUST differ ----
217 let ca: *u8 = "<body style=\"background-color:#3355aa\"><div style=\"width:200px;height:100px;background-color:#ffcc00\">A</div></body>\x00" as *u8
218 let cb: *u8 = "<body style=\"background-color:#aa5533\"><div style=\"width:300px;height:150px;background-color:#00ccff\">B</div></body>\x00" as *u8
219 var can: i64 = 0
220 while ca[can] != (0 as u8) { can = can + 1 }
221 var cbn: i64 = 0
222 while cb[cbn] != (0 as u8) { cbn = cbn + 1 }
223 let p1: *u8 = sys_mmap(RT_VW * RT_VH * 4 + 64)
224 let p2: *u8 = sys_mmap(RT_VW * RT_VH * 4 + 64)
225 let p3: *u8 = sys_mmap(RT_VW * RT_VH * 4 + 64)
226 if rt_render(ca, can, p1) != 0 { rw("REFTEST verdict=HARNESS-UNPROVEN (control A did not render)\n" as *u8); sys_exit(2); return 2 }
227 if rt_render(ca, can, p2) != 0 { rw("REFTEST verdict=HARNESS-UNPROVEN (control A2 did not render)\n" as *u8); sys_exit(2); return 2 }
228 if rt_render(cb, cbn, p3) != 0 { rw("REFTEST verdict=HARNESS-UNPROVEN (control B did not render)\n" as *u8); sys_exit(2); return 2 }
229 let selfeq: i64 = rt_px_equal(p1, p2)
230 let abneq: i64 = rt_px_equal(p1, p3)
231 rw("REFTEST-CONTROL selfmatch=" as *u8); rwn(selfeq); rw(" ab_equal=" as *u8); rwn(abneq); rw("\n" as *u8)
232 if selfeq != 1 { rw("REFTEST verdict=HARNESS-UNPROVEN (self-match differed: the render is not deterministic HERE, which is itself a finding)\n" as *u8); sys_exit(2); return 2 }
233 if abneq != 0 { rw("REFTEST verdict=HARNESS-UNPROVEN (known-different pages compared equal: the comparator is blind)\n" as *u8); sys_exit(2); return 2 }
234
235 // ---- engine + corpus keys (the spine) ----
236 let esha: *u8 = sys_mmap(24)
237 if wsp_engine_sha(esha) != 0 { rw("REFTEST: cannot hash /proc/self/exe -- refusing an unkeyed run\n" as *u8); sys_exit(3); return 3 }
238 let namebuf: *u8 = sys_mmap(1048576)
239 let offs: *i64 = sys_mmap(8 * 8192) as *i64
240 let nfiles: i64 = wsp_collect_names(dir, ".html" as *u8, namebuf, 1048576, offs, 8192)
241 if nfiles < 0 { rw("REFTEST: cannot open dir\n" as *u8); sys_exit(3); return 3 }
242 let csha: *u8 = sys_mmap(24)
243 wsp_corpus_sha(dir, namebuf, offs, nfiles, csha)
244 let dbase: *u8 = sys_mmap(256)
245 wsp_base_of(dir, dbase)
246
247 var hbuf: *u8 = 0 as *u8
248 var hn: i64 = 0
249 if (histpath as i64) != 0 {
250 let hp: *i64 = sys_mmap(16) as *i64
251 hbuf = sys_read_file(histpath, hp)
252 if (hbuf as i64) != 0 { hn = hp[0] }
253 }
254 let rlen: *i64 = sys_mmap(16) as *i64
255 if force == 0 { if hn > 0 {
256 let hit: i64 = wsp_hist_find(hbuf, hn, dbase, esha, csha, rlen)
257 if hit >= 0 {
258 rw("REFTEST-CACHED dir=" as *u8); rw(dbase)
259 rw(" -- identical (engine, corpus), deterministic engine: the result cannot differ. Prior row:\n" as *u8)
260 sys_write(1, ((hbuf as i64) + hit) as *u8, rlen[0])
261 rw("\nverdict=CACHED (--force to re-run)\n" as *u8)
262 sys_exit(0); return 0
263 }
264 } }
265
266 // ---- the run ----
267 var with_ref: i64 = 0
268 var mpass: i64 = 0
269 var mfail: i64 = 0
270 var xpass: i64 = 0
271 var xfail: i64 = 0
272 var noref: i64 = 0
273 var refmiss: i64 = 0
274 var rfail: i64 = 0
275 var fuzzyn: i64 = 0
276 var multiref: i64 = 0
277 let path: *u8 = sys_mmap(4096)
278 let refpath: *u8 = sys_mmap(4096)
279 let href: *u8 = sys_mmap(2048)
280 let kind: *i64 = sys_mmap(16) as *i64
281 let nlinks: *i64 = sys_mmap(16) as *i64
282 let fuzzy: *i64 = sys_mmap(16) as *i64
283 var fi: i64 = 0
284 while fi < nfiles {
285 let nm: *u8 = ((namebuf as i64) + offs[fi]) as *u8
286 var pp: i64 = rcat(path, 0, dir)
287 pp = rcat(path, pp, "/" as *u8)
288 pp = rcat(path, pp, nm)
289 path[pp] = 0 as u8
290 let lnp: *i64 = sys_mmap(16) as *i64
291 let body: *u8 = sys_read_file(path, lnp)
292 if (body as i64) == 0 { rfail = rfail + 1 } else {
293 rt_extract_ref(body, lnp[0], href, 2048, kind, nlinks, fuzzy)
294 if kind[0] == 0 { noref = noref + 1 } else {
295 with_ref = with_ref + 1
296 if fuzzy[0] == 1 { fuzzyn = fuzzyn + 1 }
297 if nlinks[0] > 1 { multiref = multiref + 1 }
298 if rt_resolve(dir, href, refpath) != 0 { refmiss = refmiss + 1 } else {
299 let rp2: *i64 = sys_mmap(16) as *i64
300 let refb: *u8 = sys_read_file(refpath, rp2)
301 if (refb as i64) == 0 { refmiss = refmiss + 1 } else {
302 let pxa: *u8 = sys_mmap(RT_VW * RT_VH * 4 + 64)
303 let pxb: *u8 = sys_mmap(RT_VW * RT_VH * 4 + 64)
304 let r1: i64 = rt_render(body, lnp[0], pxa)
305 let r2: i64 = rt_render(refb, rp2[0], pxb)
306 if r1 != 0 { rfail = rfail + 1 } else { if r2 != 0 { rfail = rfail + 1 } else {
307 let eq: i64 = rt_px_equal(pxa, pxb)
308 var ok: i64 = 0
309 if kind[0] == 1 { if eq == 1 { ok = 1 } }
310 if kind[0] == 2 { if eq == 0 { ok = 1 } }
311 if kind[0] == 1 { if ok == 1 { mpass = mpass + 1 } else { mfail = mfail + 1 } }
312 if kind[0] == 2 { if ok == 1 { xpass = xpass + 1 } else { xfail = xfail + 1 } }
313 rw("REFTEST-FILE " as *u8); rw(nm)
314 if kind[0] == 1 { rw(" match=" as *u8) } else { rw(" mismatch=" as *u8) }
315 if ok == 1 { rw("PASS" as *u8) } else { rw("FAIL" as *u8) }
316 if fuzzy[0] == 1 { rw(" (fuzzy-meta: compared EXACT, stricter than spec)" as *u8) }
317 rw("\n" as *u8)
318 } }
319 sys_free_file(refb, rp2[0])
320 }
321 }
322 }
323 sys_free_file(body, lnp[0])
324 }
325 fi = fi + 1
326 }
327 let dur: i64 = sys_now_ms() - t0
328
329 // ---- regression vs newest same-corpus row (cols: 6=match_pass 7=match_fail) ----
330 var regr: i64 = 0
331 var regep: i64 = 0 - 1
332 var pvpass: i64 = 0 - 1
333 var pvfail: i64 = 0 - 1
334 if hn > 0 {
335 let base: i64 = wsp_hist_find(hbuf, hn, dbase, 0 as *u8, csha, rlen)
336 if base >= 0 {
337 regep = wsp_hist_col_int(hbuf, base, rlen[0], 0)
338 pvpass = wsp_hist_col_int(hbuf, base, rlen[0], 6)
339 pvfail = wsp_hist_col_int(hbuf, base, rlen[0], 7)
340 if mpass < pvpass { regr = 1 }
341 if mfail > pvfail { regr = 1 }
342 }
343 }
344
345 let row: *u8 = sys_mmap(2048)
346 var ro: i64 = 0
347 ro = rcatnum(row, ro, sys_now_realtime_sec()); row[ro] = WSP_FS as u8; ro = ro + 1
348 ro = rcat(row, ro, dbase); row[ro] = WSP_FS as u8; ro = ro + 1
349 ro = rcat(row, ro, esha); row[ro] = WSP_FS as u8; ro = ro + 1
350 ro = rcat(row, ro, csha); row[ro] = WSP_FS as u8; ro = ro + 1
351 ro = rcatnum(row, ro, nfiles); row[ro] = WSP_FS as u8; ro = ro + 1
352 ro = rcatnum(row, ro, with_ref); row[ro] = WSP_FS as u8; ro = ro + 1
353 ro = rcatnum(row, ro, mpass); row[ro] = WSP_FS as u8; ro = ro + 1
354 ro = rcatnum(row, ro, mfail); row[ro] = WSP_FS as u8; ro = ro + 1
355 ro = rcatnum(row, ro, xpass); row[ro] = WSP_FS as u8; ro = ro + 1
356 ro = rcatnum(row, ro, xfail); row[ro] = WSP_FS as u8; ro = ro + 1
357 ro = rcatnum(row, ro, noref); row[ro] = WSP_FS as u8; ro = ro + 1
358 ro = rcatnum(row, ro, refmiss); row[ro] = WSP_FS as u8; ro = ro + 1
359 ro = rcatnum(row, ro, rfail); row[ro] = WSP_FS as u8; ro = ro + 1
360 ro = rcatnum(row, ro, fuzzyn); row[ro] = WSP_FS as u8; ro = ro + 1
361 ro = rcatnum(row, ro, dur); row[ro] = WSP_FS as u8; ro = ro + 1
362 if regr == 1 { ro = rcat(row, ro, "REGRESSION-vs-" as *u8); ro = rcatnum(row, ro, regep) } else {
363 ro = rcat(row, ro, "COUNTED" as *u8) }
364 row[ro] = 10 as u8; ro = ro + 1
365 rw("REFTEST-ROW " as *u8); sys_write(1, row, ro)
366 if (histpath as i64) != 0 {
367 if wsp_append(histpath, row, ro) != 0 { rw("REFTEST: history append FAILED (the row above is the only copy)\n" as *u8) }
368 }
369 rw("REFTEST dir=" as *u8); rw(dbase)
370 rw(" files=" as *u8); rwn(nfiles)
371 rw(" with_ref=" as *u8); rwn(with_ref)
372 rw(" match_pass=" as *u8); rwn(mpass)
373 rw(" match_fail=" as *u8); rwn(mfail)
374 rw(" mismatch_pass=" as *u8); rwn(xpass)
375 rw(" mismatch_fail=" as *u8); rwn(xfail)
376 rw(" no_ref=" as *u8); rwn(noref)
377 rw(" ref_missing=" as *u8); rwn(refmiss)
378 rw(" render_fail=" as *u8); rwn(rfail)
379 rw(" fuzzy_meta=" as *u8); rwn(fuzzyn)
380 rw(" multi_ref=" as *u8); rwn(multiref)
381 rw(" dur_ms=" as *u8); rwn(dur)
382 rw(" engine=" as *u8); rw(esha)
383 rw(" corpus=" as *u8); rw(csha)
384 rw("\n" as *u8)
385 if regr == 1 {
386 rw("verdict=REGRESSION vs epoch " as *u8); rwn(regep)
387 rw(" (match_pass " as *u8); rwn(pvpass); rw(" -> " as *u8); rwn(mpass)
388 rw(", match_fail " as *u8); rwn(pvfail); rw(" -> " as *u8); rwn(mfail)
389 rw(") -- same corpus, the engine changed for the worse\n" as *u8)
390 sys_exit(4); return 4
391 }
392 rw("verdict=COUNTED (controls proven; exact-pixel at 800x600; the row is the time-spine record)\n" as *u8)
393 sys_exit(0); return 0
394}