code wiki / (root) / nx_wpt_acquire_lib.nx

nx_wpt_acquire_lib.nx

buildroot/runtime/nx_wpt_acquire_lib.nx

26653 B597 linesdepth 6pulls 8 transitivereach 2 importersview sourcekind librarytopic wpt
docsdependenciesstructsconstsfunctions

about

nx_wpt_acquire_lib.nx -- LIB: BR21, the BYTE-PRESERVING half of WPT corpus ingest. WHY THIS EXISTS. On 2026-08-26 the corpus behind every BR1 number published on 08-18/19 was found banked NOWHERE. Two directories were recovered BY HAND and both reproduced their 08-18 corpus_sha exactly -- so the numbers were sound and the METHOD was not, and it failed in two different ways: FIDELITY: the hand path re-emits every body through a TEXT TOOL ARGUMENT. A text argument is a NUL-terminated C string. html/syntax/parsing-html-fragments is the ENCODING-DETECTION suite and the-input-byte-stream-003.html is UTF-16LE with a BOM, i.e. a file whose FOURTH BYTE IS NUL. Transcribing it through any C-string carrier truncates it at byte 3 and the corruption is SILENT. SCALE: dom/nodes is 178 files, and a partial dom/nodes is a SAMPLE, which may never be published as a population fact. This lib is the half that can be proven offline: parse the declared manifest, verify a body against BOTH numbers upstream publishes for it, write RAW BYTES with an explicit length, and read them back and compare. Nothing here consults a NUL in file content -- every content path carries its length. THE VERIFICATION IS NOT OURS AND THAT IS THE POINT. GitHub's contents API declares, per file, a size and a sha. That sha is the GIT BLOB OBJECT NAME -- sha1 over "blob " + decimal_length + NUL + content -- which nx_swhid_lib already computes and which was proven this session against NINE vectors produced by git 2.49.0, an INDEPENDENT implementation. So a single flipped byte anywhere in our transport is caught by a hash WE DID NOT CHOOSE AND CANNOT INFLUENCE. Size alone would not do it: a byte swap preserves length. The blob sha alone would not do it either -- it is derived from the length, so a check that trusts our own count of the bytes is partly checking itself. Both, or neither. license_tier: ORIGINAL

dependencies 3 imports · 2 importers

nx_syscalls.nx nx_wpt_spine.nx nx_swhid_lib.nx nx_wpt_acquire_lib.nx nx_wpt_ingest.nx nx_wpt_ingest_gate.nx

imports: nx_syscalls.nxnx_wpt_spine.nxnx_swhid_lib.nx

imported by: nx_wpt_ingest.nxnx_wpt_ingest_gate.nx

structs

none

consts

32const WA_SHA_HEX: i64 = 40 // DERIVED: a git blob object name is SHA-1, 20 bytes, 2 hex chars each
33const WA_SWHID_PREFIX: i64 = 10 // DERIVED: swhid_cnt emits the literal swh:1:cnt: ahead of the hex
34const WA_ID_CAP: i64 = 64 // DERIVED: WA_SWHID_PREFIX + WA_SHA_HEX + NUL = 51, rounded up to 64
35const WA_NAME_MAX: i64 = 255 // DERIVED: POSIX NAME_MAX -- a longer name is REFUSED, never truncated
36const WA_PATH_CAP: i64 = 4096 // DERIVED: POSIX PATH_MAX
39const WA_TAB: i64 = 9
40const WA_NL: i64 = 10
41const WA_CR: i64 = 13
42const WA_SP: i64 = 32
43const WA_QUOTE: i64 = 34
44const WA_LPAREN: i64 = 40
45const WA_RPAREN: i64 = 41
46const WA_PLUS: i64 = 43
47const WA_COMMA: i64 = 44
48const WA_MINUS: i64 = 45
49const WA_DOT: i64 = 46
50const WA_SLASH: i64 = 47
51const WA_ZERO: i64 = 48
52const WA_NINE: i64 = 57
53const WA_COLON: i64 = 58
54const WA_UC_A: i64 = 65
55const WA_UC_Z: i64 = 90
56const WA_USCORE: i64 = 95
57const WA_LC_A: i64 = 97
58const WA_LC_F: i64 = 102
59const WA_LC_Z: i64 = 122
62const WA_R_ENTRIES: i64 = 0 // manifest records seen, whatever their type
63const WA_R_DECLARED: i64 = 1 // the POPULATION: type=file AND name ends .html
64const WA_R_SKIPPED_NOTFILE: i64 = 2
65const WA_R_SKIPPED_NOTHTML: i64 = 3
66const WA_R_LANDED: i64 = 4
67const WA_R_REUSED: i64 = 5 // already on disk and verifying -> not re-fetched (idempotence)
68const WA_R_FETCHED: i64 = 6
69const WA_R_FAIL_FETCH: i64 = 7
70const WA_R_FAIL_SIZE: i64 = 8
71const WA_R_FAIL_SHA: i64 = 9
72const WA_R_FAIL_WRITE: i64 = 10
73const WA_R_FAIL_READBACK: i64 = 11
74const WA_R_REFUSED_NAME: i64 = 12
75const WA_R_COVERAGE_COMPLETE: i64 = 13
76const WA_R_SLOTS: i64 = 14
79const WA_PARSE_MALFORMED: i64 = 0 - 2 // a record lacked one of name/sha/size/type
80const WA_PARSE_OVERFLOW: i64 = 0 - 3 // more records than the table -- REFUSED, never silently capped
81const WA_PARSE_NAMEBUF: i64 = 0 - 4 // name storage exhausted -- same rule

functions

83func wa_strlen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
92func wa_mf_num(d: *u8, o: i64, v: i64) -> i64
103func wa_puts(s: *u8) -> i64 { sys_write(1, s, wa_strlen(s)); return 0 }
104func wa_putn(v: i64) -> i64
115func wa_find(hay: *u8, n: i64, from: i64, needle: *u8) -> i64
136func wa_is_pin(s: *u8) -> i64
155func wa_byte_ok(c: i64) -> i64
169func wa_is_safe_name(nm: *u8, nl: i64) -> i64
184func wa_unsafe_at(nm: *u8, nl: i64) -> i64
198func wa_verify_blob(data: *u8, n: i64, want_size: i64, want_sha: *u8) -> i64
213func wa_write_raw(path: *u8, data: *u8, n: i64) -> i64
233func wa_readback_eq(path: *u8, data: *u8, n: i64) -> i64
257func wa_dest_path(destdir: *u8, nm: *u8, nl: i64, out: *u8) -> i64
273func wa_json_key(js: *u8, n: i64, from: i64, upto: i64, key: *u8) -> i64
293func wa_json_str(js: *u8, n: i64, from: i64, upto: i64, key: *u8, out: *u8, outcap: i64) -> i64
310func wa_json_int(js: *u8, n: i64, from: i64, upto: i64, key: *u8) -> i64
335func wa_key_name(d: *u8) -> i64
340func wa_key_sha(d: *u8) -> i64
345func wa_key_size(d: *u8) -> i64
350func wa_key_type(d: *u8) -> i64
355func wa_val_file(d: *u8) -> i64
359func wa_suf_html(d: *u8) -> i64
368func wa_parse_manifest(js: *u8, n: i64, namebuf: *u8, nbcap: i64, offs: *i64, sizes: *i64,
447func wa_land_one(destdir: *u8, nm: *u8, nl: i64, data: *u8, n: i64,
464func wa_already_good(destdir: *u8, nm: *u8, nl: i64, want_size: i64, want_sha: *u8) -> i64
481func wa_manifest_write(destdir: *u8, wptdir: *u8, commit: *u8, namebuf: *u8, offs: *i64,
536func wa_marker_path(destdir: *u8, out: *u8) -> i64
542func wa_marker_set(destdir: *u8, complete: i64, declared: i64, landed: i64) -> i64
556func wa_report_sums(rep: *i64) -> i64
566func wa_coverage_complete(rep: *i64) -> i64
579func wa_report_print(rep: *i64) -> i64