nx_wpt_ingest.nx source
↩ module page · 249 lines · 12669 B
1// nx_wpt_ingest.nx -- BR21: byte-preserving WPT corpus ingest. Carries wsp_ingest_dir.
2//
3// ONE CALL: nx_wpt_ingest <wpt-dir-path> <40-hex-commit> <dest-dir>
4// nx_wpt_ingest dom/nodes a7b5671e50ee3610ec3ad2e1278a33b2cb11339c bench/wpt/dom_nodes
5//
6// WHAT IT DOES, AND WHY EACH STEP IS THERE:
7// 1. REFUSES a commit that is not 40 lowercase hex. A branch name resolves to whatever HEAD happens
8// to be, and a corpus acquired from a moving reference is reproducible by nobody -- which is the
9// whole defect BR21 exists to close. A pin that is not a pin is worse than no pin, because the
10// manifest would still record it and read as authoritative.
11// 2. Fetches the directory manifest ONCE from GitHub's contents API, at the pin. That is one
12// request against the API's 60/hour unauthenticated quota per DIRECTORY, not per file; every
13// body then comes from raw.githubusercontent.com, which is not on that quota.
14// 3. For each declared .html file: if it is already on disk AND satisfies both declared numbers,
15// it is REUSED and not re-fetched (the ingest is idempotent). Otherwise it is fetched into a
16// reserve DERIVED FROM ITS OWN DECLARED SIZE, verified against BOTH the declared size and the
17// declared git blob sha, written as RAW BYTES with an explicit length, and READ BACK AND
18// COMPARED BYTE FOR BYTE.
19// 4. Writes MANIFEST.tsv carrying the pin, and writes or REMOVES the INCOMPLETE-DO-NOT-PUBLISH
20// marker so the directory's publishability is a fact on disk rather than a memory.
21//
22// THE TEXT LAYER IS THE DEFECT AND THERE IS NONE HERE. Every content path in wa_ carries an explicit
23// length. The transport hands back a body and a byte count; nothing between the socket and the file
24// looks for a NUL. That is what makes html/syntax/parsing-html-fragments reachable at all -- its
25// UTF-16LE-with-BOM fixture has a NUL as its fourth byte, and any C-string carrier truncates it there
26// and reports success.
27//
28// license_tier: ORIGINAL
29
30import "nx_syscalls.nx"
31import "nx_x509_trust_store.nx"
32import "nx_trust_store_load_from_certdata.nx"
33import "nx_https_fetch_follow.nx"
34import "nx_wpt_acquire_lib.nx"
35
36// DERIVED: Mozilla certdata.txt measures ~1 MB; this is its reserve alone, matching nx_research_fetch,
37// which is the organ this acquisition path composes rather than re-implements beside.
38const WI_CERTDATA_CAP: i64 = 4194304
39const WI_CERTDATA_ROOTS_MAX: i64 = 512
40
41// DERIVED FROM A MEASUREMENT, not chosen: the dom/lists manifest measured 6 entries in 5,904 bytes,
42// i.e. 984 bytes per entry. GitHub's contents API returns at most 1,000 entries for one directory.
43// 1000 * 2048 is a 2x margin on the measured per-entry size at the API's own maximum page, and the
44// brim ANNOUNCES rather than truncating -- a manifest that fills its reserve is refused, never parsed.
45const WI_MANIFEST_PER_ENTRY: i64 = 2048
46const WI_MANIFEST_MAX_ENTRIES: i64 = 1000
47const WI_MANIFEST_CAP: i64 = 2048000
48
49// DERIVED: the fetcher's buffer must hold the response headers as well as the body (measured ~1 KB
50// from raw.githubusercontent.com), plus room PAST the declared length so an over-long body is
51// OBSERVED rather than truncated into a false size match. Charged per file, not per directory.
52const WI_FETCH_SLACK: i64 = 131072
53
54// DERIVED: the table is bounded by the API's own per-directory maximum, so it can never be the
55// binding constraint; wa_parse_manifest REFUSES at the brim rather than capping in silence.
56const WI_MAX_FILES: i64 = 1000
57const WI_NAMEBUF_CAP: i64 = 262144 // DERIVED: WI_MAX_FILES * (WA_NAME_MAX + 1), rounded up
58
59const WI_REDIRECT_HOPS: i64 = 6 // matches nx_research_fetch; raw.githubusercontent 302s once
60
61const WI_API_BASE: *u8 = "https://api.github.com/repos/web-platform-tests/wpt/contents/"
62const WI_RAW_BASE: *u8 = "https://raw.githubusercontent.com/web-platform-tests/wpt/"
63
64func wi_url_manifest(out: *u8, wptdir: *u8, commit: *u8) -> i64 {
65 var o: i64 = wsp_scat(out, 0, WI_API_BASE)
66 o = wsp_scat(out, o, wptdir)
67 o = wsp_scat(out, o, "?ref=" as *u8)
68 o = wsp_scat(out, o, commit)
69 out[o] = 0 as u8
70 return o
71}
72
73func wi_url_raw(out: *u8, wptdir: *u8, commit: *u8, nm: *u8) -> i64 {
74 var o: i64 = wsp_scat(out, 0, WI_RAW_BASE)
75 o = wsp_scat(out, o, commit)
76 out[o] = WA_SLASH as u8
77 o = o + 1
78 o = wsp_scat(out, o, wptdir)
79 out[o] = WA_SLASH as u8
80 o = o + 1
81 o = wsp_scat(out, o, nm)
82 out[o] = 0 as u8
83 return o
84}
85
86// THE CONTRACT SYMBOL. One call: manifest -> per-file acquisition -> manifest emit -> marker.
87// 0 = the directory is COMPLETE and publishable. Non-zero = it is not, and rep says why.
88// 1 pin refused | 2 destdir unusable | 3 manifest fetch failed | 4 manifest malformed or over cap
89// 5 one or more files did not land
90func wsp_ingest_dir(wptdir: *u8, commit: *u8, destdir: *u8, store: *TrustStore, rep: *i64) -> i64 {
91 var s: i64 = 0
92 while s < WA_R_SLOTS { rep[s] = 0; s = s + 1 }
93
94 if wa_is_pin(commit) == 0 {
95 wa_puts("WSP-REFUSED rule=pin-is-not-a-commit-sha value=")
96 wa_puts(commit)
97 wa_puts(" -- a pin must be 40 lowercase hex. A branch or tag resolves to whatever HEAD happens to be, and a corpus acquired from a moving reference is reproducible by nobody.\n")
98 return 1
99 }
100
101 sys_mkdir(destdir, MODE_0755)
102 let dfd: i64 = sys_openat_rd(destdir)
103 if dfd < 0 {
104 wa_puts("WSP-REFUSED rule=destdir-unusable path=")
105 wa_puts(destdir)
106 wa_puts(" -- mkdir did not leave a readable directory here; its PARENT probably does not exist. Nothing was fetched.\n")
107 return 2
108 }
109 sys_close(dfd)
110
111 // ---- 1. the manifest, once, at the pin -----------------------------------------------------
112 let url: *u8 = sys_mmap(WA_PATH_CAP)
113 wi_url_manifest(url, wptdir, commit)
114 let js: *u8 = sys_mmap(WI_MANIFEST_CAP)
115 let status: *i64 = sys_mmap(8) as *i64
116 status[0] = 0
117 wa_puts("WSP-MANIFEST url="); wa_puts(url); wa_puts("\n")
118 let jn: i64 = nx_https_fetch_follow_best(url, store, js, WI_MANIFEST_CAP, WI_REDIRECT_HOPS, status)
119 wa_puts("WSP-MANIFEST status="); wa_putn(status[0])
120 wa_puts(" bytes="); wa_putn(jn)
121 wa_puts(" cap="); wa_putn(WI_MANIFEST_CAP)
122 wa_puts(" cap_provenance=DERIVED-measured-984B-per-entry-x2-at-the-API-1000-entry-maximum\n")
123 if jn <= 0 { wa_puts("WSP-REFUSED rule=manifest-fetch-empty\n"); return 3 }
124 if status[0] != 200 { wa_puts("WSP-REFUSED rule=manifest-status-not-200\n"); return 3 }
125 if jn >= WI_MANIFEST_CAP {
126 wa_puts("WSP-REFUSED rule=manifest-filled-its-reserve -- a body that fills the brim is indistinguishable from a cut one, so it is REFUSED rather than parsed as complete\n")
127 return 3
128 }
129
130 // ---- 2. parse ------------------------------------------------------------------------------
131 let namebuf: *u8 = sys_mmap(WI_NAMEBUF_CAP)
132 let offs: *i64 = sys_mmap(WI_MAX_FILES * 8) as *i64
133 let sizes: *i64 = sys_mmap(WI_MAX_FILES * 8) as *i64
134 let shas: *u8 = sys_mmap(WI_MAX_FILES * WA_SHA_HEX)
135 let cnt: i64 = wa_parse_manifest(js, jn, namebuf, WI_NAMEBUF_CAP, offs, sizes, shas, WI_MAX_FILES, rep)
136 if cnt < 0 {
137 wa_puts("WSP-REFUSED rule=manifest-parse code="); wa_putn(cnt)
138 wa_puts(" (-2 a record lacked name/sha/size/type, -3 more entries than the table, -4 name storage exhausted). NOTHING was written: a partially parsed manifest is a smaller population that reads as better news.\n")
139 return 4
140 }
141 wa_puts("WSP-DECLARED files="); wa_putn(cnt)
142 wa_puts(" of entries="); wa_putn(rep[WA_R_ENTRIES])
143 wa_puts(" (skipped_notfile="); wa_putn(rep[WA_R_SKIPPED_NOTFILE])
144 wa_puts(" skipped_nothtml="); wa_putn(rep[WA_R_SKIPPED_NOTHTML])
145 wa_puts(")\n")
146
147 // ---- 3. per file ---------------------------------------------------------------------------
148 let rurl: *u8 = sys_mmap(WA_PATH_CAP)
149 var i: i64 = 0
150 while i < cnt {
151 let nm: *u8 = ((namebuf as i64) + offs[i]) as *u8
152 let nl: i64 = wa_strlen(nm)
153 let want: i64 = sizes[i]
154 let wsha: *u8 = ((shas as i64) + i * WA_SHA_HEX) as *u8
155
156 if wa_is_safe_name(nm, nl) == 0 {
157 let at: i64 = wa_unsafe_at(nm, nl)
158 wa_puts("WSP-REFUSED-NAME name="); wa_puts(nm)
159 wa_puts(" at="); wa_putn(at)
160 wa_puts(" byte="); wa_putn(nm[at] as i64)
161 wa_puts(" -- outside the declared allowlist. Widening it is an operator decision made on this evidence; nothing was written for this file.\n")
162 rep[WA_R_REFUSED_NAME] = rep[WA_R_REFUSED_NAME] + 1
163 i = i + 1
164 } else {
165 if wa_already_good(destdir, nm, nl, want, wsha) == 1 {
166 rep[WA_R_REUSED] = rep[WA_R_REUSED] + 1
167 i = i + 1
168 } else {
169 wi_url_raw(rurl, wptdir, commit, nm)
170 let cap: i64 = want + WI_FETCH_SLACK
171 let body: *u8 = sys_mmap(cap)
172 status[0] = 0
173 let bn: i64 = nx_https_fetch_follow_best(rurl, store, body, cap, WI_REDIRECT_HOPS, status)
174 if bn <= 0 {
175 wa_puts("WSP-FAIL-FETCH name="); wa_puts(nm)
176 wa_puts(" status="); wa_putn(status[0])
177 wa_puts(" rc="); wa_putn(bn); wa_puts("\n")
178 rep[WA_R_FAIL_FETCH] = rep[WA_R_FAIL_FETCH] + 1
179 } else {
180 if status[0] != 200 {
181 wa_puts("WSP-FAIL-FETCH name="); wa_puts(nm)
182 wa_puts(" status="); wa_putn(status[0]); wa_puts("\n")
183 rep[WA_R_FAIL_FETCH] = rep[WA_R_FAIL_FETCH] + 1
184 } else {
185 rep[WA_R_FETCHED] = rep[WA_R_FETCHED] + 1
186 let r: i64 = wa_land_one(destdir, nm, nl, body, bn, want, wsha, rep)
187 if r != 0 {
188 wa_puts("WSP-FAIL name="); wa_puts(nm)
189 wa_puts(" stage="); wa_putn(r)
190 wa_puts(" (1 declared size, 2 declared git blob sha, 3 write, 4 read-back compare, 5 name)")
191 wa_puts(" got_bytes="); wa_putn(bn)
192 wa_puts(" declared_bytes="); wa_putn(want)
193 wa_puts("\n")
194 }
195 }
196 }
197 // DELIBERATELY NOT UNMAPPED. sys_mmap serves small requests from a shared arena, so a
198 // munmap of an arena sub-block would unmap pages other allocations still hold. The
199 // reserve is per-file and lazily faulted: a 1,000-file directory costs address space,
200 // not resident memory, because only the bytes actually received are ever touched.
201 i = i + 1
202 }
203 }
204 }
205
206 // ---- 4. verdict, manifest, marker ----------------------------------------------------------
207 rep[WA_R_COVERAGE_COMPLETE] = wa_coverage_complete(rep)
208 wa_manifest_write(destdir, wptdir, commit, namebuf, offs, sizes, shas, cnt, rep)
209 wa_marker_set(destdir, rep[WA_R_COVERAGE_COMPLETE], rep[WA_R_DECLARED], rep[WA_R_LANDED] + rep[WA_R_REUSED])
210 wa_report_print(rep)
211 if rep[WA_R_COVERAGE_COMPLETE] == 1 { return 0 }
212 return 5
213}
214
215func main(argc: i64, argv: *i64) -> i64 {
216 if argc < 4 {
217 wa_puts("usage: nx_wpt_ingest <wpt-dir-path> <40-hex-commit> <dest-dir>\n")
218 wa_puts(" e.g. nx_wpt_ingest dom/nodes a7b5671e50ee3610ec3ad2e1278a33b2cb11339c bench/wpt/dom_nodes\n")
219 return 3
220 }
221 let wptdir: *u8 = argv[1] as *u8
222 let commit: *u8 = argv[2] as *u8
223 let destdir: *u8 = argv[3] as *u8
224
225 let cpath: *u8 = "data/mozilla_certdata.txt\x00"
226 let tr: i64 = nx_trust_store_load_from_certdata(cpath, WI_CERTDATA_ROOTS_MAX, WI_CERTDATA_CAP)
227 if tr <= 0 {
228 wa_puts("WSP-REFUSED rule=certdata-load-failed path=data/mozilla_certdata.txt -- run from the nishihost root\n")
229 return 1
230 }
231 let store: *TrustStore = tr as *TrustStore
232 wa_puts("WSP-CA roots="); wa_putn(trust_store_count(store)); wa_puts("\n")
233
234 let rep: *i64 = sys_mmap(WA_R_SLOTS * 8) as *i64
235 let rc: i64 = wsp_ingest_dir(wptdir, commit, destdir, store, rep)
236 wa_puts("WSP-INGEST-VERDICT dir=")
237 wa_puts(wptdir)
238 wa_puts(" pin=")
239 wa_puts(commit)
240 wa_puts(" declared=")
241 wa_putn(rep[WA_R_DECLARED])
242 wa_puts(" present=")
243 wa_putn(rep[WA_R_LANDED] + rep[WA_R_REUSED])
244 wa_puts(" rc=")
245 wa_putn(rc)
246 wa_puts(" verdict=")
247 if rc == 0 { wa_puts("COMPLETE\n") } else { wa_puts("INCOMPLETE\n") }
248 return rc
249}