nx_wpt_acquire_lib.nx
buildroot/runtime/nx_wpt_acquire_lib.nx
about
nx_wpt_acquire_lib.nx -- LIB: BR21, the BYTE-PRESERVING half of WPT corpus ingest.
WHY THIS EXISTS. On 2026-08-26 the corpus behind every BR1 number published on 08-18/19 was found
banked NOWHERE. Two directories were recovered BY HAND and both reproduced their 08-18 corpus_sha
exactly -- so the numbers were sound and the METHOD was not, and it failed in two different ways:
FIDELITY: the hand path re-emits every body through a TEXT TOOL ARGUMENT. A text argument is a
NUL-terminated C string. html/syntax/parsing-html-fragments is the ENCODING-DETECTION suite and
the-input-byte-stream-003.html is UTF-16LE with a BOM, i.e. a file whose FOURTH BYTE IS NUL.
Transcribing it through any C-string carrier truncates it at byte 3 and the corruption is SILENT.
SCALE: dom/nodes is 178 files, and a partial dom/nodes is a SAMPLE, which may never be published
as a population fact.
This lib is the half that can be proven offline: parse the declared manifest, verify a body against
BOTH numbers upstream publishes for it, write RAW BYTES with an explicit length, and read them back
and compare. Nothing here consults a NUL in file content -- every content path carries its length.
THE VERIFICATION IS NOT OURS AND THAT IS THE POINT. GitHub's contents API declares, per file, a
size and a sha. That sha is the GIT BLOB OBJECT NAME -- sha1 over "blob " + decimal_length + NUL +
content -- which nx_swhid_lib already computes and which was proven this session against NINE
vectors produced by git 2.49.0, an INDEPENDENT implementation. So a single flipped byte anywhere in
our transport is caught by a hash WE DID NOT CHOOSE AND CANNOT INFLUENCE. Size alone would not do
it: a byte swap preserves length. The blob sha alone would not do it either -- it is derived from
the length, so a check that trusts our own count of the bytes is partly checking itself. Both, or
neither.
license_tier: ORIGINAL
dependencies 3 imports · 2 importers
imports: nx_syscalls.nxnx_wpt_spine.nxnx_swhid_lib.nx
imported by: nx_wpt_ingest.nxnx_wpt_ingest_gate.nx
structs
| none |
consts
| 32 | const WA_SHA_HEX: i64 = 40 // DERIVED: a git blob object name is SHA-1, 20 bytes, 2 hex chars each |
| 33 | const WA_SWHID_PREFIX: i64 = 10 // DERIVED: swhid_cnt emits the literal swh:1:cnt: ahead of the hex |
| 34 | const WA_ID_CAP: i64 = 64 // DERIVED: WA_SWHID_PREFIX + WA_SHA_HEX + NUL = 51, rounded up to 64 |
| 35 | const WA_NAME_MAX: i64 = 255 // DERIVED: POSIX NAME_MAX -- a longer name is REFUSED, never truncated |
| 36 | const WA_PATH_CAP: i64 = 4096 // DERIVED: POSIX PATH_MAX |
| 39 | const WA_TAB: i64 = 9 |
| 40 | const WA_NL: i64 = 10 |
| 41 | const WA_CR: i64 = 13 |
| 42 | const WA_SP: i64 = 32 |
| 43 | const WA_QUOTE: i64 = 34 |
| 44 | const WA_LPAREN: i64 = 40 |
| 45 | const WA_RPAREN: i64 = 41 |
| 46 | const WA_PLUS: i64 = 43 |
| 47 | const WA_COMMA: i64 = 44 |
| 48 | const WA_MINUS: i64 = 45 |
| 49 | const WA_DOT: i64 = 46 |
| 50 | const WA_SLASH: i64 = 47 |
| 51 | const WA_ZERO: i64 = 48 |
| 52 | const WA_NINE: i64 = 57 |
| 53 | const WA_COLON: i64 = 58 |
| 54 | const WA_UC_A: i64 = 65 |
| 55 | const WA_UC_Z: i64 = 90 |
| 56 | const WA_USCORE: i64 = 95 |
| 57 | const WA_LC_A: i64 = 97 |
| 58 | const WA_LC_F: i64 = 102 |
| 59 | const WA_LC_Z: i64 = 122 |
| 62 | const WA_R_ENTRIES: i64 = 0 // manifest records seen, whatever their type |
| 63 | const WA_R_DECLARED: i64 = 1 // the POPULATION: type=file AND name ends .html |
| 64 | const WA_R_SKIPPED_NOTFILE: i64 = 2 |
| 65 | const WA_R_SKIPPED_NOTHTML: i64 = 3 |
| 66 | const WA_R_LANDED: i64 = 4 |
| 67 | const WA_R_REUSED: i64 = 5 // already on disk and verifying -> not re-fetched (idempotence) |
| 68 | const WA_R_FETCHED: i64 = 6 |
| 69 | const WA_R_FAIL_FETCH: i64 = 7 |
| 70 | const WA_R_FAIL_SIZE: i64 = 8 |
| 71 | const WA_R_FAIL_SHA: i64 = 9 |
| 72 | const WA_R_FAIL_WRITE: i64 = 10 |
| 73 | const WA_R_FAIL_READBACK: i64 = 11 |
| 74 | const WA_R_REFUSED_NAME: i64 = 12 |
| 75 | const WA_R_COVERAGE_COMPLETE: i64 = 13 |
| 76 | const WA_R_SLOTS: i64 = 14 |
| 79 | const WA_PARSE_MALFORMED: i64 = 0 - 2 // a record lacked one of name/sha/size/type |
| 80 | const WA_PARSE_OVERFLOW: i64 = 0 - 3 // more records than the table -- REFUSED, never silently capped |
| 81 | const WA_PARSE_NAMEBUF: i64 = 0 - 4 // name storage exhausted -- same rule |
functions
| 83 | func wa_strlen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n } |
| 92 | func wa_mf_num(d: *u8, o: i64, v: i64) -> i64 |
| 103 | func wa_puts(s: *u8) -> i64 { sys_write(1, s, wa_strlen(s)); return 0 } |
| 104 | func wa_putn(v: i64) -> i64 |
| 115 | func wa_find(hay: *u8, n: i64, from: i64, needle: *u8) -> i64 |
| 136 | func wa_is_pin(s: *u8) -> i64 |
| 155 | func wa_byte_ok(c: i64) -> i64 |
| 169 | func wa_is_safe_name(nm: *u8, nl: i64) -> i64 |
| 184 | func wa_unsafe_at(nm: *u8, nl: i64) -> i64 |
| 198 | func wa_verify_blob(data: *u8, n: i64, want_size: i64, want_sha: *u8) -> i64 |
| 213 | func wa_write_raw(path: *u8, data: *u8, n: i64) -> i64 |
| 233 | func wa_readback_eq(path: *u8, data: *u8, n: i64) -> i64 |
| 257 | func wa_dest_path(destdir: *u8, nm: *u8, nl: i64, out: *u8) -> i64 |
| 273 | func wa_json_key(js: *u8, n: i64, from: i64, upto: i64, key: *u8) -> i64 |
| 293 | func wa_json_str(js: *u8, n: i64, from: i64, upto: i64, key: *u8, out: *u8, outcap: i64) -> i64 |
| 310 | func wa_json_int(js: *u8, n: i64, from: i64, upto: i64, key: *u8) -> i64 |
| 335 | func wa_key_name(d: *u8) -> i64 |
| 340 | func wa_key_sha(d: *u8) -> i64 |
| 345 | func wa_key_size(d: *u8) -> i64 |
| 350 | func wa_key_type(d: *u8) -> i64 |
| 355 | func wa_val_file(d: *u8) -> i64 |
| 359 | func wa_suf_html(d: *u8) -> i64 |
| 368 | func wa_parse_manifest(js: *u8, n: i64, namebuf: *u8, nbcap: i64, offs: *i64, sizes: *i64, |
| 447 | func wa_land_one(destdir: *u8, nm: *u8, nl: i64, data: *u8, n: i64, |
| 464 | func wa_already_good(destdir: *u8, nm: *u8, nl: i64, want_size: i64, want_sha: *u8) -> i64 |
| 481 | func wa_manifest_write(destdir: *u8, wptdir: *u8, commit: *u8, namebuf: *u8, offs: *i64, |
| 536 | func wa_marker_path(destdir: *u8, out: *u8) -> i64 |
| 542 | func wa_marker_set(destdir: *u8, complete: i64, declared: i64, landed: i64) -> i64 |
| 556 | func wa_report_sums(rep: *i64) -> i64 |
| 566 | func wa_coverage_complete(rep: *i64) -> i64 |
| 579 | func wa_report_print(rep: *i64) -> i64 |