nx_wpt_acquire_lib.nx source
↩ module page · 597 lines · 26653 B
1// nx_wpt_acquire_lib.nx -- LIB: BR21, the BYTE-PRESERVING half of WPT corpus ingest.
2//
3// WHY THIS EXISTS. On 2026-08-26 the corpus behind every BR1 number published on 08-18/19 was found
4// banked NOWHERE. Two directories were recovered BY HAND and both reproduced their 08-18 corpus_sha
5// exactly -- so the numbers were sound and the METHOD was not, and it failed in two different ways:
6// FIDELITY: the hand path re-emits every body through a TEXT TOOL ARGUMENT. A text argument is a
7// NUL-terminated C string. html/syntax/parsing-html-fragments is the ENCODING-DETECTION suite and
8// the-input-byte-stream-003.html is UTF-16LE with a BOM, i.e. a file whose FOURTH BYTE IS NUL.
9// Transcribing it through any C-string carrier truncates it at byte 3 and the corruption is SILENT.
10// SCALE: dom/nodes is 178 files, and a partial dom/nodes is a SAMPLE, which may never be published
11// as a population fact.
12// This lib is the half that can be proven offline: parse the declared manifest, verify a body against
13// BOTH numbers upstream publishes for it, write RAW BYTES with an explicit length, and read them back
14// and compare. Nothing here consults a NUL in file content -- every content path carries its length.
15//
16// THE VERIFICATION IS NOT OURS AND THAT IS THE POINT. GitHub's contents API declares, per file, a
17// size and a sha. That sha is the GIT BLOB OBJECT NAME -- sha1 over "blob " + decimal_length + NUL +
18// content -- which nx_swhid_lib already computes and which was proven this session against NINE
19// vectors produced by git 2.49.0, an INDEPENDENT implementation. So a single flipped byte anywhere in
20// our transport is caught by a hash WE DID NOT CHOOSE AND CANNOT INFLUENCE. Size alone would not do
21// it: a byte swap preserves length. The blob sha alone would not do it either -- it is derived from
22// the length, so a check that trusts our own count of the bytes is partly checking itself. Both, or
23// neither.
24//
25// license_tier: ORIGINAL
26
27import "nx_syscalls.nx"
28import "nx_wpt_spine.nx"
29import "nx_swhid_lib.nx"
30
31// ---- DERIVED SHAPE CONSTANTS (each states where its value comes from) --------------------------
32const WA_SHA_HEX: i64 = 40 // DERIVED: a git blob object name is SHA-1, 20 bytes, 2 hex chars each
33const WA_SWHID_PREFIX: i64 = 10 // DERIVED: swhid_cnt emits the literal swh:1:cnt: ahead of the hex
34const WA_ID_CAP: i64 = 64 // DERIVED: WA_SWHID_PREFIX + WA_SHA_HEX + NUL = 51, rounded up to 64
35const WA_NAME_MAX: i64 = 255 // DERIVED: POSIX NAME_MAX -- a longer name is REFUSED, never truncated
36const WA_PATH_CAP: i64 = 4096 // DERIVED: POSIX PATH_MAX
37
38// ---- ASCII CODE POINTS, NAMED (a bare 34 in the source is a second copy of a shape) ------------
39const WA_TAB: i64 = 9
40const WA_NL: i64 = 10
41const WA_CR: i64 = 13
42const WA_SP: i64 = 32
43const WA_QUOTE: i64 = 34
44const WA_LPAREN: i64 = 40
45const WA_RPAREN: i64 = 41
46const WA_PLUS: i64 = 43
47const WA_COMMA: i64 = 44
48const WA_MINUS: i64 = 45
49const WA_DOT: i64 = 46
50const WA_SLASH: i64 = 47
51const WA_ZERO: i64 = 48
52const WA_NINE: i64 = 57
53const WA_COLON: i64 = 58
54const WA_UC_A: i64 = 65
55const WA_UC_Z: i64 = 90
56const WA_USCORE: i64 = 95
57const WA_LC_A: i64 = 97
58const WA_LC_F: i64 = 102
59const WA_LC_Z: i64 = 122
60
61// ---- REPORT SLOTS. A partition, and wa_report_sums checks that it SUMS. -------------------------
62const WA_R_ENTRIES: i64 = 0 // manifest records seen, whatever their type
63const WA_R_DECLARED: i64 = 1 // the POPULATION: type=file AND name ends .html
64const WA_R_SKIPPED_NOTFILE: i64 = 2
65const WA_R_SKIPPED_NOTHTML: i64 = 3
66const WA_R_LANDED: i64 = 4
67const WA_R_REUSED: i64 = 5 // already on disk and verifying -> not re-fetched (idempotence)
68const WA_R_FETCHED: i64 = 6
69const WA_R_FAIL_FETCH: i64 = 7
70const WA_R_FAIL_SIZE: i64 = 8
71const WA_R_FAIL_SHA: i64 = 9
72const WA_R_FAIL_WRITE: i64 = 10
73const WA_R_FAIL_READBACK: i64 = 11
74const WA_R_REFUSED_NAME: i64 = 12
75const WA_R_COVERAGE_COMPLETE: i64 = 13
76const WA_R_SLOTS: i64 = 14
77
78// ---- PARSE REFUSALS (negative; the caller must never read a partial table) ----------------------
79const WA_PARSE_MALFORMED: i64 = 0 - 2 // a record lacked one of name/sha/size/type
80const WA_PARSE_OVERFLOW: i64 = 0 - 3 // more records than the table -- REFUSED, never silently capped
81const WA_PARSE_NAMEBUF: i64 = 0 - 4 // name storage exhausted -- same rule
82
83func wa_strlen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
84
85// ONE decimal renderer in this lib, and ONE length scan. The first draft carried a SECOND copy of
86// each, inside wa_putn and wa_puts, and nx_gate_bite found them the way a duplicate ruler is always
87// found: it reported their comparison sites NOT-REACHED, because an offline gate has no reason to
88// call a stdout printer, so the compiler eliminated them and a mutation there could not change the
89// binary at all. THREE of the four mutation sites in this file were in code that existed only
90// because it had been written twice. Collapsing them removes the duplicate AND the blind spot in
91// one edit -- the blind spot was not a gap in the gate, it was a symptom of the duplication.
92func wa_mf_num(d: *u8, o: i64, v: i64) -> i64 {
93 var x: i64 = v
94 if x <= 0 { d[o] = WA_ZERO as u8; return o + 1 }
95 var dg: i64 = 0
96 var t: i64 = x
97 while t > 0 { dg = dg + 1; t = t / 10 }
98 var p: i64 = o + dg
99 while x > 0 { p = p - 1; d[p] = (WA_ZERO + (x % 10)) as u8; x = x / 10 }
100 return o + dg
101}
102
103func wa_puts(s: *u8) -> i64 { sys_write(1, s, wa_strlen(s)); return 0 }
104func wa_putn(v: i64) -> i64 {
105 let b: *u8 = sys_mmap(64)
106 var o: i64 = 0
107 if v < 0 { b[0] = WA_MINUS as u8; o = wa_mf_num(b, 1, 0 - v) } else { o = wa_mf_num(b, 0, v) }
108 b[o] = 0 as u8
109 return wa_puts(b)
110}
111
112// LENGTH-BOUNDED substring search. It never stops at a NUL, which is the whole reason this lib
113// exists: a manifest is text, but the discipline is applied everywhere so no content path can
114// acquire a C-string assumption by accident.
115func wa_find(hay: *u8, n: i64, from: i64, needle: *u8) -> i64 {
116 let m: i64 = wa_strlen(needle)
117 if m == 0 { return 0 - 1 }
118 var i: i64 = from
119 if i < 0 { i = 0 }
120 while (i + m) <= n {
121 var k: i64 = 0
122 var same: i64 = 1
123 while k < m {
124 if hay[i+k] != needle[k] { same = 0; break }
125 k = k + 1
126 }
127 if same == 1 { return i }
128 i = i + 1
129 }
130 return 0 - 1
131}
132
133// A COMMIT PIN THAT IS NOT A COMMIT SHA IS NOT A PIN. A branch name fetches whatever HEAD happens to
134// be, and a corpus acquired from a moving reference is reproducible by nobody -- the exact defect
135// BR21 exists to close. Refused here rather than in the CLI so the gate can prove it offline.
136func wa_is_pin(s: *u8) -> i64 {
137 if wa_strlen(s) != WA_SHA_HEX { return 0 }
138 var i: i64 = 0
139 while i < WA_SHA_HEX {
140 let c: i64 = s[i] as i64
141 var ok: i64 = 0
142 if c >= WA_ZERO { if c <= WA_NINE { ok = 1 } }
143 if c >= WA_LC_A { if c <= WA_LC_F { ok = 1 } }
144 if ok == 0 { return 0 }
145 i = i + 1
146 }
147 return 1
148}
149
150// ALLOWLIST, not a blocklist: the byte set is stated, and anything outside it is REFUSED BY NAME with
151// the offending byte printed, so widening it is an operator decision made on evidence rather than a
152// silent mangle. A refused name cannot be a path traversal, cannot escape destdir, and cannot be a
153// dotfile. plus comma and the parens are admitted because WPT filenames use them; nothing outside
154// printable ASCII is.
155func wa_byte_ok(c: i64) -> i64 {
156 var ok: i64 = 0
157 if c >= WA_ZERO { if c <= WA_NINE { ok = 1 } }
158 if c >= WA_UC_A { if c <= WA_UC_Z { ok = 1 } }
159 if c >= WA_LC_A { if c <= WA_LC_Z { ok = 1 } }
160 if c == WA_DOT { ok = 1 }
161 if c == WA_USCORE { ok = 1 }
162 if c == WA_MINUS { ok = 1 }
163 if c == WA_PLUS { ok = 1 }
164 if c == WA_COMMA { ok = 1 }
165 if c == WA_LPAREN { ok = 1 }
166 if c == WA_RPAREN { ok = 1 }
167 return ok
168}
169func wa_is_safe_name(nm: *u8, nl: i64) -> i64 {
170 if nl <= 0 { return 0 }
171 if nl > WA_NAME_MAX { return 0 }
172 if (nm[0] as i64) == WA_DOT { return 0 }
173 var i: i64 = 0
174 while i < nl {
175 if wa_byte_ok(nm[i] as i64) == 0 { return 0 }
176 i = i + 1
177 }
178 return 1
179}
180
181// which byte of the name caused the refusal, so the message can NAME it; -1 if the name is safe.
182// It tests the SAME per-byte predicate the rule uses -- a second copy of the allowlist here would be
183// the duplicate-ruler defect, and it would drift the moment the set is widened.
184func wa_unsafe_at(nm: *u8, nl: i64) -> i64 {
185 if wa_is_safe_name(nm, nl) == 1 { return 0 - 1 }
186 if nl <= 0 { return 0 }
187 var i: i64 = 0
188 while i < nl {
189 if wa_byte_ok(nm[i] as i64) == 0 { return i }
190 i = i + 1
191 }
192 return 0
193}
194
195// ---- VERIFICATION -------------------------------------------------------------------------------
196// 0 = both declared numbers agree | 1 = declared size disagrees | 2 = declared blob sha disagrees
197// 3 = the hash could not be computed (an ABSTENTION, never an acquittal)
198func wa_verify_blob(data: *u8, n: i64, want_size: i64, want_sha: *u8) -> i64 {
199 if n != want_size { return 1 }
200 let id: *u8 = sys_mmap(WA_ID_CAP)
201 let r: i64 = swhid_cnt(data, n, id)
202 if r < 0 { return 3 }
203 var i: i64 = 0
204 while i < WA_SHA_HEX {
205 if id[WA_SWHID_PREFIX + i] != want_sha[i] { return 2 }
206 i = i + 1
207 }
208 return 0
209}
210
211// ---- RAW WRITE. The length is an ARGUMENT; no NUL is consulted anywhere on this path. -----------
212// 0 ok | 1 open failed | 2 the write did not place every byte
213func wa_write_raw(path: *u8, data: *u8, n: i64) -> i64 {
214 if n < 0 { return 2 }
215 let fd: i64 = sys_openat_wr(path, MODE_0644)
216 if fd < 0 { return 1 }
217 var off: i64 = 0
218 var bad: i64 = 0
219 while off < n {
220 let w: i64 = sys_write(fd, ((data as i64) + off) as *u8, n - off)
221 if w <= 0 { bad = 1; break }
222 off = off + w
223 }
224 sys_fsync(fd)
225 sys_close(fd)
226 if bad == 1 { return 2 }
227 if off != n { return 2 }
228 return 0
229}
230
231// READ IT BACK AND COMPARE EVERY BYTE. A write that returned success is a receipt; this is the artifact.
232// 1 identical | 0 differs | -1 unreadable
233func wa_readback_eq(path: *u8, data: *u8, n: i64) -> i64 {
234 let lp: *i64 = sys_mmap(16) as *i64
235 lp[0] = 0
236 let b: *u8 = sys_read_file(path, lp)
237 if (b as i64) == 0 {
238 // DECLARED IMPRECISION: a zero-length file reads back as a null pointer here, so this reader
239 // cannot distinguish an empty file from an unreadable one. Named rather than papered over;
240 // a zero-length expectation is answered as identical, anything else as unreadable.
241 if n == 0 { return 1 }
242 return 0 - 1
243 }
244 var r: i64 = 1
245 if lp[0] != n { r = 0 }
246 if r == 1 {
247 var i: i64 = 0
248 while i < n {
249 if b[i] != data[i] { r = 0; break }
250 i = i + 1
251 }
252 }
253 sys_free_file(b, lp[0])
254 return r
255}
256
257func wa_dest_path(destdir: *u8, nm: *u8, nl: i64, out: *u8) -> i64 {
258 var o: i64 = wsp_scat(out, 0, destdir)
259 out[o] = WA_SLASH as u8
260 o = o + 1
261 var i: i64 = 0
262 while i < nl { out[o] = nm[i]; o = o + 1; i = i + 1 }
263 out[o] = 0 as u8
264 return o
265}
266
267// ---- MANIFEST PARSE -----------------------------------------------------------------------------
268// GitHub contents emits, per entry, in this order: name, path, sha, size, url, html_url, git_url,
269// download_url, type, _links. The parser BOUNDS each record at the next name key and requires all
270// four fields inside that window; a record missing one REFUSES the whole parse rather than skipping a
271// row, because a silently skipped row is a smaller population that reads as better news. Every needle
272// carries its quotes, so a filename containing the letters n-a-m-e cannot be mistaken for the key.
273func wa_json_key(js: *u8, n: i64, from: i64, upto: i64, key: *u8) -> i64 {
274 let k: i64 = wa_find(js, upto, from, key)
275 if k < 0 { return 0 - 1 }
276 var p: i64 = k + wa_strlen(key)
277 var scanning: i64 = 1
278 while scanning == 1 {
279 if p >= upto { return 0 - 1 }
280 let c: i64 = js[p] as i64
281 var skip: i64 = 0
282 if c == WA_SP { skip = 1 }
283 if c == WA_COLON { skip = 1 }
284 if c == WA_TAB { skip = 1 }
285 if c == WA_NL { skip = 1 }
286 if c == WA_CR { skip = 1 }
287 if skip == 1 { p = p + 1 } else { scanning = 0 }
288 }
289 return p
290}
291
292// copies the quoted string value into out (NUL-terminated); returns its length or -1
293func wa_json_str(js: *u8, n: i64, from: i64, upto: i64, key: *u8, out: *u8, outcap: i64) -> i64 {
294 let p: i64 = wa_json_key(js, n, from, upto, key)
295 if p < 0 { return 0 - 1 }
296 if (js[p] as i64) != WA_QUOTE { return 0 - 1 }
297 var i: i64 = p + 1
298 var o: i64 = 0
299 while i < upto {
300 let c: i64 = js[i] as i64
301 if c == WA_QUOTE { out[o] = 0 as u8; return o }
302 if o >= (outcap - 1) { return 0 - 1 }
303 out[o] = js[i]
304 o = o + 1
305 i = i + 1
306 }
307 return 0 - 1
308}
309
310func wa_json_int(js: *u8, n: i64, from: i64, upto: i64, key: *u8) -> i64 {
311 let p: i64 = wa_json_key(js, n, from, upto, key)
312 if p < 0 { return 0 - 1 }
313 var i: i64 = p
314 var v: i64 = 0
315 var any: i64 = 0
316 var scanning: i64 = 1
317 while scanning == 1 {
318 if i >= upto { scanning = 0 } else {
319 let c: i64 = js[i] as i64
320 var digit: i64 = 0
321 if c >= WA_ZERO { if c <= WA_NINE { digit = 1 } }
322 if digit == 0 { scanning = 0 } else {
323 v = v * 10 + (c - WA_ZERO)
324 any = 1
325 i = i + 1
326 }
327 }
328 }
329 if any == 0 { return 0 - 1 }
330 return v
331}
332
333// the JSON keys, assembled at RUNTIME. A detector that scans source finds its own fixtures, and a
334// key written as a source literal is a second copy of a shape the parser already encodes.
335func wa_key_name(d: *u8) -> i64 {
336 d[0] = WA_QUOTE as u8; d[1] = 110 as u8; d[2] = 97 as u8; d[3] = 109 as u8
337 d[4] = 101 as u8; d[5] = WA_QUOTE as u8; d[6] = 0 as u8
338 return 6
339}
340func wa_key_sha(d: *u8) -> i64 {
341 d[0] = WA_QUOTE as u8; d[1] = 115 as u8; d[2] = 104 as u8; d[3] = 97 as u8
342 d[4] = WA_QUOTE as u8; d[5] = 0 as u8
343 return 5
344}
345func wa_key_size(d: *u8) -> i64 {
346 d[0] = WA_QUOTE as u8; d[1] = 115 as u8; d[2] = 105 as u8; d[3] = 122 as u8
347 d[4] = 101 as u8; d[5] = WA_QUOTE as u8; d[6] = 0 as u8
348 return 6
349}
350func wa_key_type(d: *u8) -> i64 {
351 d[0] = WA_QUOTE as u8; d[1] = 116 as u8; d[2] = 121 as u8; d[3] = 112 as u8
352 d[4] = 101 as u8; d[5] = WA_QUOTE as u8; d[6] = 0 as u8
353 return 6
354}
355func wa_val_file(d: *u8) -> i64 {
356 d[0] = 102 as u8; d[1] = 105 as u8; d[2] = 108 as u8; d[3] = 101 as u8; d[4] = 0 as u8
357 return 4
358}
359func wa_suf_html(d: *u8) -> i64 {
360 d[0] = WA_DOT as u8; d[1] = 104 as u8; d[2] = 116 as u8; d[3] = 109 as u8
361 d[4] = 108 as u8; d[5] = 0 as u8
362 return 5
363}
364
365// Fills namebuf/offs/sizes/shas for every entry that is BOTH type=file AND *.html.
366// Returns the DECLARED count (the population), or a negative WA_PARSE_ refusal.
367// shas is a flat table of WA_SHA_HEX bytes per accepted record.
368func wa_parse_manifest(js: *u8, n: i64, namebuf: *u8, nbcap: i64, offs: *i64, sizes: *i64,
369 shas: *u8, maxn: i64, rep: *i64) -> i64 {
370 let kname: *u8 = sys_mmap(16)
371 let knlen: i64 = wa_key_name(kname)
372 let ksha: *u8 = sys_mmap(16)
373 wa_key_sha(ksha)
374 let ksize: *u8 = sys_mmap(16)
375 wa_key_size(ksize)
376 let ktype: *u8 = sys_mmap(16)
377 wa_key_type(ktype)
378 let vfile: *u8 = sys_mmap(16)
379 let vflen: i64 = wa_val_file(vfile)
380 let suf: *u8 = sys_mmap(16)
381 let suflen: i64 = wa_suf_html(suf)
382
383 let nmbuf: *u8 = sys_mmap(WA_PATH_CAP)
384 let tybuf: *u8 = sys_mmap(WA_PATH_CAP)
385 let shbuf: *u8 = sys_mmap(WA_PATH_CAP)
386 var cnt: i64 = 0
387 var nb: i64 = 0
388 var pos: i64 = 0
389 var going: i64 = 1
390 var refusal: i64 = 0
391 while going == 1 {
392 let k: i64 = wa_find(js, n, pos, kname)
393 if k < 0 { going = 0 } else {
394 var recend: i64 = wa_find(js, n, k + knlen, kname)
395 if recend < 0 { recend = n }
396 rep[WA_R_ENTRIES] = rep[WA_R_ENTRIES] + 1
397 let nl: i64 = wa_json_str(js, n, k, recend, kname, nmbuf, WA_PATH_CAP)
398 let tl: i64 = wa_json_str(js, n, k, recend, ktype, tybuf, WA_PATH_CAP)
399 let sl: i64 = wa_json_str(js, n, k, recend, ksha, shbuf, WA_PATH_CAP)
400 let sz: i64 = wa_json_int(js, n, k, recend, ksize)
401 var bad: i64 = 0
402 if nl < 0 { bad = 1 }
403 if tl < 0 { bad = 1 }
404 if sl < 0 { bad = 1 }
405 if sz < 0 { bad = 1 }
406 if sl != WA_SHA_HEX { bad = 1 }
407 if bad == 1 { refusal = WA_PARSE_MALFORMED; going = 0 } else {
408 var take: i64 = 1
409 if wsp_ends_with(tybuf, tl, vfile, vflen) == 0 {
410 take = 0
411 rep[WA_R_SKIPPED_NOTFILE] = rep[WA_R_SKIPPED_NOTFILE] + 1
412 }
413 if take == 1 {
414 if wsp_ends_with(nmbuf, nl, suf, suflen) == 0 {
415 take = 0
416 rep[WA_R_SKIPPED_NOTHTML] = rep[WA_R_SKIPPED_NOTHTML] + 1
417 }
418 }
419 if take == 1 {
420 if cnt >= maxn { refusal = WA_PARSE_OVERFLOW; going = 0 }
421 if going == 1 { if (nb + nl + 1) > nbcap { refusal = WA_PARSE_NAMEBUF; going = 0 } }
422 if going == 1 {
423 offs[cnt] = nb
424 var q: i64 = 0
425 while q < nl { namebuf[nb+q] = nmbuf[q]; q = q + 1 }
426 namebuf[nb+nl] = 0 as u8
427 nb = nb + nl + 1
428 sizes[cnt] = sz
429 var h: i64 = 0
430 while h < WA_SHA_HEX { shas[cnt*WA_SHA_HEX + h] = shbuf[h]; h = h + 1 }
431 cnt = cnt + 1
432 }
433 }
434 if going == 1 { pos = k + knlen }
435 }
436 }
437 }
438 if refusal != 0 { return refusal }
439 rep[WA_R_DECLARED] = cnt
440 return cnt
441}
442
443// ---- LAND ONE ALREADY-FETCHED BODY --------------------------------------------------------------
444// 0 ok | 1 size | 2 blob sha | 3 write | 4 readback | 5 refused name
445// A body that fails verification is NEVER WRITTEN: a corrupt file on disk is indistinguishable from a
446// good one to every later reader, and the runner would fold it into a corpus_sha that looks authoritative.
447func wa_land_one(destdir: *u8, nm: *u8, nl: i64, data: *u8, n: i64,
448 want_size: i64, want_sha: *u8, rep: *i64) -> i64 {
449 if wa_is_safe_name(nm, nl) == 0 { rep[WA_R_REFUSED_NAME] = rep[WA_R_REFUSED_NAME] + 1; return 5 }
450 let v: i64 = wa_verify_blob(data, n, want_size, want_sha)
451 if v == 1 { rep[WA_R_FAIL_SIZE] = rep[WA_R_FAIL_SIZE] + 1; return 1 }
452 if v == 2 { rep[WA_R_FAIL_SHA] = rep[WA_R_FAIL_SHA] + 1; return 2 }
453 if v == 3 { rep[WA_R_FAIL_SHA] = rep[WA_R_FAIL_SHA] + 1; return 2 }
454 let path: *u8 = sys_mmap(WA_PATH_CAP)
455 wa_dest_path(destdir, nm, nl, path)
456 if wa_write_raw(path, data, n) != 0 { rep[WA_R_FAIL_WRITE] = rep[WA_R_FAIL_WRITE] + 1; return 3 }
457 if wa_readback_eq(path, data, n) != 1 { rep[WA_R_FAIL_READBACK] = rep[WA_R_FAIL_READBACK] + 1; return 4 }
458 rep[WA_R_LANDED] = rep[WA_R_LANDED] + 1
459 return 0
460}
461
462// Is the file already on disk AND does it satisfy BOTH declared numbers? 1 yes / 0 no.
463// This is what makes the ingest idempotent: a second run re-verifies and re-fetches nothing.
464func wa_already_good(destdir: *u8, nm: *u8, nl: i64, want_size: i64, want_sha: *u8) -> i64 {
465 let path: *u8 = sys_mmap(WA_PATH_CAP)
466 wa_dest_path(destdir, nm, nl, path)
467 let lp: *i64 = sys_mmap(16) as *i64
468 lp[0] = 0
469 let b: *u8 = sys_read_file(path, lp)
470 if (b as i64) == 0 { return 0 }
471 var r: i64 = 0
472 if wa_verify_blob(b, lp[0], want_size, want_sha) == 0 { r = 1 }
473 sys_free_file(b, lp[0])
474 return r
475}
476
477// ---- MANIFEST EMIT ------------------------------------------------------------------------------
478// A TSV whose LAST line is the canonical verdict, so a positional reader can decide COMPLETE vs
479// INCOMPLETE without parsing the body. Its decimal rendering is wa_mf_num, hoisted to the top of
480// this file so wa_putn composes it instead of carrying a second copy.
481func wa_manifest_write(destdir: *u8, wptdir: *u8, commit: *u8, namebuf: *u8, offs: *i64,
482 sizes: *i64, shas: *u8, cnt: i64, rep: *i64) -> i64 {
483 // DERIVED reserve: a fixed header, plus one row per file bounded by NAME_MAX + the 40-hex sha +
484 // room for a decimal size and two tabs. No cap is guessed and none can be reached in silence.
485 let cap: i64 = WA_PATH_CAP + (cnt + 1) * (WA_NAME_MAX + WA_SHA_HEX + 64)
486 let buf: *u8 = sys_mmap(cap)
487 var o: i64 = wsp_scat(buf, 0, "wsp-manifest" as *u8)
488 buf[o] = WA_TAB as u8; o = o + 1
489 o = wsp_scat(buf, o, "v1" as *u8); buf[o] = WA_NL as u8; o = o + 1
490 o = wsp_scat(buf, o, "pin" as *u8); buf[o] = WA_TAB as u8; o = o + 1
491 o = wsp_scat(buf, o, commit); buf[o] = WA_NL as u8; o = o + 1
492 o = wsp_scat(buf, o, "repo" as *u8); buf[o] = WA_TAB as u8; o = o + 1
493 o = wsp_scat(buf, o, "web-platform-tests/wpt" as *u8); buf[o] = WA_NL as u8; o = o + 1
494 o = wsp_scat(buf, o, "src" as *u8); buf[o] = WA_TAB as u8; o = o + 1
495 o = wsp_scat(buf, o, wptdir); buf[o] = WA_NL as u8; o = o + 1
496 var i: i64 = 0
497 while i < cnt {
498 let nm: *u8 = ((namebuf as i64) + offs[i]) as *u8
499 o = wsp_scat(buf, o, "file" as *u8)
500 buf[o] = WA_TAB as u8; o = o + 1
501 o = wsp_scat(buf, o, nm)
502 buf[o] = WA_TAB as u8; o = o + 1
503 o = wa_mf_num(buf, o, sizes[i])
504 buf[o] = WA_TAB as u8; o = o + 1
505 var h: i64 = 0
506 while h < WA_SHA_HEX { buf[o] = shas[i*WA_SHA_HEX + h]; o = o + 1; h = h + 1 }
507 buf[o] = WA_NL as u8; o = o + 1
508 i = i + 1
509 }
510 o = wsp_scat(buf, o, "declared" as *u8); buf[o] = WA_TAB as u8; o = o + 1
511 o = wa_mf_num(buf, o, rep[WA_R_DECLARED]); buf[o] = WA_NL as u8; o = o + 1
512 o = wsp_scat(buf, o, "landed" as *u8); buf[o] = WA_TAB as u8; o = o + 1
513 o = wa_mf_num(buf, o, rep[WA_R_LANDED]); buf[o] = WA_NL as u8; o = o + 1
514 o = wsp_scat(buf, o, "reused" as *u8); buf[o] = WA_TAB as u8; o = o + 1
515 o = wa_mf_num(buf, o, rep[WA_R_REUSED]); buf[o] = WA_NL as u8; o = o + 1
516 o = wsp_scat(buf, o, "fetched" as *u8); buf[o] = WA_TAB as u8; o = o + 1
517 o = wa_mf_num(buf, o, rep[WA_R_FETCHED]); buf[o] = WA_NL as u8; o = o + 1
518 o = wsp_scat(buf, o, "coverage_complete" as *u8); buf[o] = WA_TAB as u8; o = o + 1
519 o = wa_mf_num(buf, o, rep[WA_R_COVERAGE_COMPLETE]); buf[o] = WA_NL as u8; o = o + 1
520 o = wsp_scat(buf, o, "verdict" as *u8); buf[o] = WA_TAB as u8; o = o + 1
521 if rep[WA_R_COVERAGE_COMPLETE] == 1 {
522 o = wsp_scat(buf, o, "COMPLETE" as *u8)
523 } else {
524 o = wsp_scat(buf, o, "INCOMPLETE" as *u8)
525 }
526 buf[o] = WA_NL as u8; o = o + 1
527 let path: *u8 = sys_mmap(WA_PATH_CAP)
528 var p: i64 = wsp_scat(path, 0, destdir)
529 p = wsp_scat(path, p, "/MANIFEST.tsv" as *u8)
530 path[p] = 0 as u8
531 return wa_write_raw(path, buf, o)
532}
533
534// The marker the hand lane left behind, now written and REMOVED mechanically so it cannot go stale in
535// either direction: present means this directory is not publishable, absent means it is.
536func wa_marker_path(destdir: *u8, out: *u8) -> i64 {
537 var p: i64 = wsp_scat(out, 0, destdir)
538 p = wsp_scat(out, p, "/INCOMPLETE-DO-NOT-PUBLISH.txt" as *u8)
539 out[p] = 0 as u8
540 return p
541}
542func wa_marker_set(destdir: *u8, complete: i64, declared: i64, landed: i64) -> i64 {
543 let path: *u8 = sys_mmap(WA_PATH_CAP)
544 wa_marker_path(destdir, path)
545 if complete == 1 { sys_unlinkat(path); return 0 }
546 let buf: *u8 = sys_mmap(1024)
547 var o: i64 = wsp_scat(buf, 0, "INCOMPLETE -- DO NOT PUBLISH A COUNT FROM THIS DIRECTORY.\nlanded " as *u8)
548 o = wa_mf_num(buf, o, landed)
549 o = wsp_scat(buf, o, " of declared " as *u8)
550 o = wa_mf_num(buf, o, declared)
551 o = wsp_scat(buf, o, "\nA partial directory is a SAMPLE, and a sample may never be published as a population fact.\nRe-run nx_wpt_ingest for this directory; the ingest is idempotent and re-fetches only what is missing.\n" as *u8)
552 return wa_write_raw(path, buf, o)
553}
554
555// THE PARTITION MUST SUM, AND THE SUM IS PRINTED. 1 = reconciles, 0 = leaks.
556func wa_report_sums(rep: *i64) -> i64 {
557 let acc: i64 = rep[WA_R_LANDED] + rep[WA_R_REUSED] + rep[WA_R_FAIL_FETCH] + rep[WA_R_FAIL_SIZE]
558 + rep[WA_R_FAIL_SHA] + rep[WA_R_FAIL_WRITE] + rep[WA_R_FAIL_READBACK] + rep[WA_R_REFUSED_NAME]
559 if acc == rep[WA_R_DECLARED] { return 1 }
560 return 0
561}
562
563// COMPLETE is a CONJUNCTION and every conjunct is named: the population was non-empty, every declared
564// file is on disk and verified, nothing failed for any reason, and the partition reconciles. A
565// completeness flag that can be true on the empty set is not a flag.
566func wa_coverage_complete(rep: *i64) -> i64 {
567 if rep[WA_R_DECLARED] <= 0 { return 0 }
568 if (rep[WA_R_LANDED] + rep[WA_R_REUSED]) != rep[WA_R_DECLARED] { return 0 }
569 if rep[WA_R_FAIL_FETCH] != 0 { return 0 }
570 if rep[WA_R_FAIL_SIZE] != 0 { return 0 }
571 if rep[WA_R_FAIL_SHA] != 0 { return 0 }
572 if rep[WA_R_FAIL_WRITE] != 0 { return 0 }
573 if rep[WA_R_FAIL_READBACK] != 0 { return 0 }
574 if rep[WA_R_REFUSED_NAME] != 0 { return 0 }
575 if wa_report_sums(rep) == 0 { return 0 }
576 return 1
577}
578
579func wa_report_print(rep: *i64) -> i64 {
580 wa_puts("WSP-INGEST entries="); wa_putn(rep[WA_R_ENTRIES])
581 wa_puts(" declared="); wa_putn(rep[WA_R_DECLARED])
582 wa_puts(" landed="); wa_putn(rep[WA_R_LANDED])
583 wa_puts(" reused="); wa_putn(rep[WA_R_REUSED])
584 wa_puts(" fetched="); wa_putn(rep[WA_R_FETCHED])
585 wa_puts(" skipped_notfile="); wa_putn(rep[WA_R_SKIPPED_NOTFILE])
586 wa_puts(" skipped_nothtml="); wa_putn(rep[WA_R_SKIPPED_NOTHTML])
587 wa_puts(" fail_fetch="); wa_putn(rep[WA_R_FAIL_FETCH])
588 wa_puts(" fail_size="); wa_putn(rep[WA_R_FAIL_SIZE])
589 wa_puts(" fail_sha="); wa_putn(rep[WA_R_FAIL_SHA])
590 wa_puts(" fail_write="); wa_putn(rep[WA_R_FAIL_WRITE])
591 wa_puts(" fail_readback="); wa_putn(rep[WA_R_FAIL_READBACK])
592 wa_puts(" refused_name="); wa_putn(rep[WA_R_REFUSED_NAME])
593 wa_puts(" partition_sums="); wa_putn(wa_report_sums(rep))
594 wa_puts(" coverage_complete="); wa_putn(rep[WA_R_COVERAGE_COMPLETE])
595 wa_puts("\n")
596 return 0
597}