nx_https_fetch_lib.nx source
↩ module page · 462 lines · 19855 B
1// nx_https_fetch_lib.nx -- the sovereign HTTPS GET composition, as a LIBRARY.
2//
3// WHY THIS EXISTS (2026-07-31): the whole working fetch path -- trust-store load,
4// CSPRNG, url parse, connect (with optional override), TLS 1.3 chrome-JA3
5// handshake, cert cache, path+query assembly, HTTP GET -- lived inside
6// nx_https_get_cli.nx's main(). Any second consumer (the album downloader) had
7// exactly two bad options: duplicate ~90 lines of crypto setup, or shell out.
8// Rule 15: a pattern needed by more than one consumer belongs in a lib, and
9// duplicated crypto setup is the kind that diverges SILENTLY -- one copy gets the
10// cert-cache fix or the recv_hs reassembly fix and the other quietly does not.
11//
12// The logic here is LIFTED VERBATIM from the proven CLI, only parameterised. The
13// CLI is then re-pointed at this lib so there is ONE implementation, and its
14// behaviour is re-verified live after the move (a refactor of a crown-jewel path
15// is not done until the old entry point is proven still working).
16//
17// ★ THE STORE IS LOADED SEPARATELY ON PURPOSE. nx_trust_store_load_from_certdata
18// parses a ~4 MiB Mozilla bundle. Doing that per file would make a 300-file album
19// pay it 300 times. hf_store_load() once, then hf_fetch() per url.
20// license_tier: ORIGINAL
21
22import "nx_syscalls.nx"
23import "nx_connect.nx" // bounded connect: a raw sys_connect hangs ~127s on a black-holed host
24import "nx_csprng.nx"
25import "nx_x509_trust_store.nx"
26import "nx_trust_store_load_from_certdata.nx"
27import "nx_tls13_client_validate_certificate.nx"
28import "nx_tls13_client_session_run.nx"
29import "nx_tls13_chrome_session.nx"
30import "nx_https_url_for_fetch.nx"
31import "nx_https_url_connect.nx"
32import "nx_https_get_complete.nx"
33import "nx_https_get_stream.nx" // streaming tail: media to an fd, with Range resume
34import "nx_tls_cert_cache.nx"
35import "nx_tls12_client_session.nx" // R10: the AUTHENTICATED TLS-1.2 session (chain + SKE-signature verified)
36const HF_MAGIC_2047: i64 = 2047
37const HF_MAGIC_2048: i64 = 2048
38const HF_T12_PT_CAP: i64 = 20000
39const HF_T12_MAX_RECORDS: i64 = 4096
40
41const HF_CERTDATA: *u8 = "data/mozilla_certdata.txt\x00"
42const HF_STORE_CAP: i64 = 4194304
43
44// Distinct negative codes so a caller can tell WHICH stage failed. A single -1
45// would make "the site is down" and "our trust store is missing" the same answer.
46const HF_ERR_STORE: i64 = 0 - 2
47const HF_ERR_URL: i64 = 0 - 3
48const HF_ERR_CONNECT: i64 = 0 - 4
49const HF_ERR_TLS: i64 = 0 - 5
50const HF_ERR_HTTP: i64 = 0 - 6
51
52// Load the Mozilla trust store ONCE. Returns the *TrustStore as i64, or 0.
53func hf_store_load() -> i64 {
54 let r: i64 = nx_trust_store_load_from_certdata(HF_CERTDATA, 512, HF_STORE_CAP)
55 if r <= 0 { return 0 }
56 return r
57}
58
59// GET <url> over sovereign TLS 1.3 into out; returns bytes of the RAW response
60// (status line + headers + body) or one of the HF_ERR_* codes.
61//
62// cip/cport: optional connect override (curl --connect-to). cip==0 means resolve
63// the url host normally. The override opens TCP+TLS to that endpoint while SNI,
64// Host and the cert name all stay the URL's host -- which is how our own vhosts
65// get fetched deterministically from the sovereign edge instead of coin-flipping
66// against the DSM nginx that co-squats :443.
67// Connect + handshake + assemble the request path, leaving a live session ready
68// for EITHER tail (buffered read or stream-to-file). Extracted so the two tails
69// cannot drift apart -- duplicating this setup inside my own lib would be the
70// same DRY sin that made extracting it from the CLI necessary in the first place.
71//
72// box[0]=session box[1]=fd box[2]=path_ptr box[3]=path_len
73// box[4]=host_ptr box[5]=host_len. Returns 1 on success, else an HF_ERR_*.
74func hf_open(store_i: i64, url: *u8, cip: i64, cport: i64, box: *i64) -> i64 {
75 if store_i <= 0 { return HF_ERR_STORE }
76 let store: *TrustStore = store_i as *TrustStore
77 let now: i64 = sys_now_realtime_sec()
78
79 let cr: *u8 = sys_mmap(32)
80 nx_csprng_fill(cr, 32)
81 let priv: *u8 = sys_mmap(32)
82 nx_csprng_fill(priv, 32)
83
84 let url_p: *NxUrl = nx_url_new()
85 let target_raw: *u8 = sys_mmap(32)
86 let target: *NxHttpsTarget = target_raw as *NxHttpsTarget
87 target.url = url_p
88 target.port = 0
89 if nx_https_url_for_fetch(url, target) != NX_HTTPS_URL_OK { return HF_ERR_URL }
90
91 let fd_p: *i64 = sys_mmap(16) as *i64
92 if cip != 0 {
93 let sa: *u8 = sys_mmap(16)
94 nx_https_build_sockaddr(sa, cip, cport)
95 let cfd: i64 = sys_socket(NX_HTTPS_AF_INET, NX_HTTPS_SOCK_STREAM, 0)
96 if cfd < 0 { return HF_ERR_CONNECT }
97 sys_set_socket_timeout(cfd, 15)
98 if nx_connect_bounded(cfd, sa, 16, NX_CONN_DEFAULT_MS) < 0 { sys_close(cfd); return HF_ERR_CONNECT }
99 fd_p[0] = cfd
100 } else {
101 if nx_https_url_connect(target, url, now, fd_p) != NX_HTTPS_CONNECT_OK { return HF_ERR_CONNECT }
102 }
103 let fd: i64 = fd_p[0]
104
105 let val_raw: *u8 = sys_mmap(128)
106 let val_ctx: *TlsValidationContext = val_raw as *TlsValidationContext
107 val_ctx.store = store
108 val_ctx.sni_host = url + target.url.host_off
109 val_ctx.sni_host_len = target.url.host_len
110 val_ctx.now_epoch = now
111 tcc_load(url + target.url.host_off, target.url.host_len, now, val_ctx)
112 tcc_arm(val_ctx)
113 let sr: i64 = nx_tls13_client_session_run_chrome(fd, url + target.url.host_off, target.url.host_len, cr, priv, val_ctx)
114 if sr < 0 { sys_close(fd); return HF_ERR_TLS }
115 tcc_save(url + target.url.host_off, target.url.host_len, now, val_ctx)
116
117 var path_ptr: *u8 = url + target.url.path_off
118 var path_len: i64 = target.url.path_len
119 if path_len == 0 {
120 let dp: *u8 = sys_mmap(2)
121 dp[0] = 47 as u8
122 path_ptr = dp
123 path_len = 1
124 }
125 if target.url.query_len > 0 {
126 let full: *u8 = sys_mmap(path_len + target.url.query_len + 4)
127 var fo: i64 = 0
128 var pci: i64 = 0
129 while pci < path_len { full[fo] = path_ptr[pci]; fo = fo + 1; pci = pci + 1 }
130 full[fo] = 63 as u8; fo = fo + 1
131 let qp: *u8 = url + target.url.query_off
132 var qci: i64 = 0
133 while qci < target.url.query_len { full[fo] = qp[qci]; fo = fo + 1; qci = qci + 1 }
134 path_ptr = full
135 path_len = fo
136 }
137
138 box[0] = sr
139 box[1] = fd
140 box[2] = path_ptr as i64
141 box[3] = path_len
142 box[4] = (url + target.url.host_off) as i64
143 box[5] = target.url.host_len
144 return 1
145}
146
147// STREAM a url straight to an open fd -- never through a full-size buffer, so a
148// 5 GB video costs the socket buffer, not 5 GB of RAM. range_start>0 resumes a
149// partial file (HTTP Range), which is what makes an interrupted album re-runnable
150// instead of restart-from-zero.
151// Returns bytes written, or an HF_ERR_*; out_status carries the HTTP status.
152func hf_fetch_to_file_once(store_i: i64, url: *u8, cip: i64, cport: i64, loc: *u8,
153 dest_fd: i64, range_start: i64, out_status: *i64) -> i64 {
154 let box: *i64 = sys_mmap(64) as *i64
155 let o: i64 = hf_open(store_i, url, cip, cport, box)
156 if o != 1 { return o }
157 let session: *Tls13ClientSession = box[0] as *Tls13ClientSession
158 let fd: i64 = box[1]
159 loc[0] = 0 as u8
160 let n: i64 = nx_https_get_stream(session, fd, box[2] as *u8, box[3], box[4] as *u8, box[5],
161 range_start, dest_fd, out_status, loc, HF_MAGIC_2048)
162 sys_close(fd)
163 if n < 0 { return HF_ERR_HTTP }
164 return n
165}
166
167// R10: TLS-1.2 FALLBACK FETCH -- the whole GET over an AUTHENTICATED TLS 1.2 session.
168//
169// WHY (measured 2026-08-05): graphis.ne.jp is Apache 2.2.31 / OpenSSL 1.0.0 and negotiates
170// TLS 1.2 ONLY. Our fetch path was 1.3-only, so it failed at ServerHello -- and the crawler
171// then counted those failures toward WC_HD_RETIRE and retired the host PERMANENTLY. A whole
172// class of the web (older Apache/nginx estates) was therefore not "uncrawlable", it was
173// unreachable BY US, and the scheduler laundered that into permanent coverage loss.
174// ★A TRANSPORT GAP BECOMES PERMANENT COVERAGE LOSS WHEN THE SCHEDULER RETIRES WHAT IT
175// CANNOT FETCH -- so the transport gap is the thing to close.
176//
177// This calls nx_tls12_client_session_run (NOT t12_request in nx_tls12_req.nx, which states in
178// its own header that chain + SKE-signature validation are unwired). A fallback that quietly
179// drops peer authentication would trade a coverage gap for a MITM surface; this one keeps the
180// same TlsValidationContext the 1.3 path uses, so a 1.2 fetch is authenticated or it fails.
181//
182// ⚠DUPLICATION, DELIBERATE AND RECORDED: the drain loop below is the same shape as
183// _acme_http12_roundtrip in nx_acme_http.nx, which already solved this for Porkbun. Merging
184// them means editing the live ACME cert-issuance path, which is a separate risk from adding a
185// function here -- debt filed for the merge rather than papered over.
186func hf_fetch12_once(store_i: i64, url: *u8, cip: i64, cport: i64, out: *u8, cap: i64) -> i64 {
187 if store_i <= 0 { return HF_ERR_STORE }
188 let store: *TrustStore = store_i as *TrustStore
189 let now: i64 = sys_now_realtime_sec()
190
191 let url_p: *NxUrl = nx_url_new()
192 let target_raw: *u8 = sys_mmap(32)
193 let target: *NxHttpsTarget = target_raw as *NxHttpsTarget
194 target.url = url_p
195 target.port = 0
196 if nx_https_url_for_fetch(url, target) != NX_HTTPS_URL_OK { return HF_ERR_URL }
197
198 let fd_p: *i64 = sys_mmap(16) as *i64
199 if cip != 0 {
200 let sa: *u8 = sys_mmap(16)
201 nx_https_build_sockaddr(sa, cip, cport)
202 let cfd: i64 = sys_socket(NX_HTTPS_AF_INET, NX_HTTPS_SOCK_STREAM, 0)
203 if cfd < 0 { return HF_ERR_CONNECT }
204 sys_set_socket_timeout(cfd, 15)
205 if nx_connect_bounded(cfd, sa, 16, NX_CONN_DEFAULT_MS) < 0 { sys_close(cfd); return HF_ERR_CONNECT }
206 fd_p[0] = cfd
207 } else {
208 if nx_https_url_connect(target, url, now, fd_p) != NX_HTTPS_CONNECT_OK { return HF_ERR_CONNECT }
209 }
210 let fd: i64 = fd_p[0]
211
212 let host: *u8 = (url as i64 + target.url.host_off) as *u8
213 let host_len: i64 = target.url.host_len
214
215 let vc_raw: *u8 = sys_mmap(128)
216 let vc: *TlsValidationContext = vc_raw as *TlsValidationContext
217 vc.store = store
218 vc.sni_host = host
219 vc.sni_host_len = host_len
220 vc.now_epoch = now
221
222 // Fresh ephemerals per connection -- forward secrecy is per-handshake, never reused.
223 let cr: *u8 = sys_mmap(32)
224 nx_csprng_fill(cr, 32)
225 let seed: *u8 = sys_mmap(32)
226 nx_csprng_fill(seed, 32)
227
228 let sr: i64 = nx_tls12_client_session_run(fd, host, host_len, cr, seed, vc)
229 if sr < 0 { sys_close(fd); return HF_ERR_TLS }
230 let s: *Tls12ClientSession = sr as *Tls12ClientSession
231
232 var path_ptr: *u8 = (url as i64 + target.url.path_off) as *u8
233 var path_len: i64 = target.url.path_len
234 if path_len == 0 {
235 let dp: *u8 = sys_mmap(2)
236 dp[0] = 47 as u8
237 path_ptr = dp
238 path_len = 1
239 }
240 if target.url.query_len > 0 {
241 let full: *u8 = sys_mmap(path_len + target.url.query_len + 4)
242 var fo: i64 = 0
243 var pci: i64 = 0
244 while pci < path_len { full[fo] = path_ptr[pci]; fo = fo + 1; pci = pci + 1 }
245 full[fo] = 63 as u8; fo = fo + 1
246 let qp: *u8 = (url as i64 + target.url.query_off) as *u8
247 var qci: i64 = 0
248 while qci < target.url.query_len { full[fo] = qp[qci]; fo = fo + 1; qci = qci + 1 }
249 path_ptr = full
250 path_len = fo
251 }
252
253 let req: *u8 = sys_mmap(HF_MAGIC_2048 + path_len + host_len)
254 let req_len: i64 = nx_http_client_build_request(path_ptr, path_len, host, host_len, req)
255 if req_len <= 0 { sys_close(fd); return HF_ERR_HTTP }
256 if nx_tls12_session_send(s, fd, req, req_len) != 0 { sys_close(fd); return HF_ERR_HTTP }
257
258 let pt: *u8 = sys_mmap(HF_T12_PT_CAP)
259 let ctp: *i64 = sys_mmap(16) as *i64
260 var acc: i64 = 0
261 var rounds: i64 = 0
262 var draining: i64 = 1
263 while draining == 1 {
264 if rounds >= HF_T12_MAX_RECORDS { draining = 0 }
265 else {
266 let pl: i64 = nx_tls12_session_recv(s, fd, pt, HF_T12_PT_CAP, ctp)
267 rounds = rounds + 1
268 if pl < 0 { draining = 0 }
269 else {
270 if ctp[0] == 23 {
271 var j: i64 = 0
272 while j < pl {
273 if acc < cap { out[acc] = pt[j]; acc = acc + 1 }
274 j = j + 1
275 }
276 }
277 if ctp[0] == 21 { draining = 0 }
278 }
279 }
280 }
281 sys_close(fd)
282 if acc <= 0 { return HF_ERR_HTTP }
283 return acc
284}
285
286// Buffered read into `out` -- for PAGES (album/item HTML), where the caller wants
287// the whole document to parse. Media goes through hf_fetch_to_file instead.
288//
289// R10: on a TLS failure the 1.2 fallback runs. The retry is scoped to HF_ERR_TLS ONLY --
290// a connect failure or an HTTP failure is a real answer and is NOT masked by a second attempt.
291func hf_fetch_once(store_i: i64, url: *u8, cip: i64, cport: i64, out: *u8, cap: i64) -> i64 {
292 let box: *i64 = sys_mmap(64) as *i64
293 let o: i64 = hf_open(store_i, url, cip, cport, box)
294 if o == HF_ERR_TLS { return hf_fetch12_once(store_i, url, cip, cport, out, cap) }
295 if o != 1 { return o }
296 let session: *Tls13ClientSession = box[0] as *Tls13ClientSession
297 let fd: i64 = box[1]
298 let n: i64 = nx_https_get_complete(session, fd, box[2] as *u8, box[3], box[4] as *u8, box[5], out, cap)
299 sys_close(fd)
300 if n < 0 { return HF_ERR_HTTP }
301 return n
302}
303
304// Extract the Location header value from a raw response. Case-insensitive on
305// the field name because header casing is not guaranteed. Returns length, 0 if
306// absent.
307func hf_location(resp: *u8, n: i64, out: *u8, cap: i64) -> i64 {
308 out[0] = 0 as u8
309 var i: i64 = 0
310 while i + 10 < n {
311 var atline: i64 = 0
312 if i == 0 { atline = 1 } else { if resp[i-1] == (10 as u8) { atline = 1 } }
313 if atline == 1 {
314 let key: *u8 = "location:" as *u8
315 var m: i64 = 1
316 var j: i64 = 0
317 while j < 9 {
318 var c: i64 = resp[i+j] as i64
319 if c >= 65 { if c <= 90 { c = c + 32 } }
320 if c != (key[j] as i64) { m = 0; j = 9 } else { j = j + 1 }
321 }
322 if m == 1 {
323 var v: i64 = i + 9
324 var gs: i64 = 1
325 while gs == 1 { if v >= n { gs = 0 } else { if resp[v] == (32 as u8) { v = v + 1 } else { gs = 0 } } }
326 var o: i64 = 0
327 var ge: i64 = 1
328 while ge == 1 {
329 if v >= n { ge = 0 } else {
330 if resp[v] == (13 as u8) { ge = 0 } else {
331 if resp[v] == (10 as u8) { ge = 0 } else {
332 if o + 1 < cap { out[o] = resp[v]; o = o + 1 }
333 v = v + 1
334 }
335 }
336 }
337 }
338 out[o] = 0 as u8
339 return o
340 }
341 }
342 i = i + 1
343 }
344 return 0
345}
346
347// Buffered fetch that FOLLOWS REDIRECTS, bounded.
348// The streaming tail already followed them; this one did not, and the two tails
349// disagreeing was its own defect: the ALBUM PAGE goes through here, so a site
350// whose entry URL 3xx-redirects (en.wikipedia.org does) yielded a 301 body with
351// no links and the ingest refused with -- no item links matched -- naming the
352// adapter rule as the culprit when the real cause was an unfollowed redirect.
353func hf_fetch(store_i: i64, url: *u8, cip: i64, cport: i64, out: *u8, cap: i64) -> i64 {
354 let cur: *u8 = sys_mmap(HF_MAGIC_2048)
355 var ci: i64 = 0
356 while url[ci] != (0 as u8) { if ci < HF_MAGIC_2047 { cur[ci] = url[ci] } ci = ci + 1 }
357 cur[ci] = 0 as u8
358 let loc: *u8 = sys_mmap(HF_MAGIC_2048)
359 var ip: i64 = cip
360 var pt: i64 = cport
361 var hops: i64 = 0
362 var res: i64 = HF_ERR_HTTP
363 var go: i64 = 1
364 while go == 1 {
365 let n: i64 = hf_fetch_once(store_i, cur, ip, pt, out, cap)
366 if n < 0 { return n }
367 let st: i64 = hf_status(out, n)
368 var redir: i64 = 0
369 if st >= 300 { if st < 400 { if hf_location(out, n, loc, HF_MAGIC_2048) > 0 { redir = 1 } } }
370 if redir == 1 {
371 if hops < 3 {
372 var k: i64 = 0
373 while loc[k] != (0 as u8) { if k < HF_MAGIC_2047 { cur[k] = loc[k] } k = k + 1 }
374 cur[k] = 0 as u8
375 ip = 0
376 pt = 0
377 hops = hops + 1
378 } else { go = 0; res = n }
379 } else { go = 0; res = n }
380 }
381 return res
382}
383
384// Offset of the body inside a raw response (past the CRLFCRLF), or -1 if the
385// header terminator never appears -- REFUSING rather than returning 0, because
386// treating a malformed response as "body starts at 0" would hand the caller the
387// HTTP headers as if they were file bytes.
388// Follow redirects, BOUNDED. nx_https_get_stream returns 0 WITHOUT writing any
389// body on a 3xx and fills loc_buf with the Location (verified at
390// nx_https_get_stream.nx:138), so the destination fd is untouched between hops
391// and there is nothing to truncate -- which matters because no ftruncate
392// primitive exists here. The wrapper only has to ACT on what the primitive
393// already hands back, which it previously ignored.
394//
395// Why this is not a nicety: album hosts redirect item URLs to signed CDN
396// endpoints as the NORM, so without following, EVERY real download lands as
397// zero bytes while the album and site collections still get declared -- empty
398// albums that look structurally correct. Measured live 2026-07-31.
399//
400// The override is dropped after hop 1: it pins a connect endpoint for a
401// specific host, and a redirect by definition changes the host.
402func hf_fetch_to_file(store_i: i64, url: *u8, cip: i64, cport: i64,
403 dest_fd: i64, range_start: i64, out_status: *i64) -> i64 {
404 let cur: *u8 = sys_mmap(HF_MAGIC_2048)
405 var ci: i64 = 0
406 while url[ci] != (0 as u8) { if ci < HF_MAGIC_2047 { cur[ci] = url[ci] } ci = ci + 1 }
407 cur[ci] = 0 as u8
408 let loc: *u8 = sys_mmap(HF_MAGIC_2048)
409 var ip: i64 = cip
410 var pt: i64 = cport
411 var hops: i64 = 0
412 var res: i64 = HF_ERR_HTTP
413 var go: i64 = 1
414 while go == 1 {
415 loc[0] = 0 as u8
416 let n: i64 = hf_fetch_to_file_once(store_i, cur, ip, pt, loc, dest_fd, range_start, out_status)
417 if n < 0 { return n }
418 let st: i64 = out_status[0]
419 var redir: i64 = 0
420 if st >= 300 { if st < 400 { if loc[0] != (0 as u8) { redir = 1 } } }
421 if redir == 1 {
422 if hops < 3 {
423 var k: i64 = 0
424 while loc[k] != (0 as u8) { if k < HF_MAGIC_2047 { cur[k] = loc[k] } k = k + 1 }
425 cur[k] = 0 as u8
426 ip = 0
427 pt = 0
428 hops = hops + 1
429 } else { go = 0; res = n }
430 } else { go = 0; res = n }
431 }
432 return res
433}
434
435func hf_body_off(resp: *u8, n: i64) -> i64 {
436 var i: i64 = 0
437 while i + 3 < n {
438 if resp[i]==(13 as u8) { if resp[i+1]==(10 as u8) { if resp[i+2]==(13 as u8) { if resp[i+3]==(10 as u8) { return i + 4 } } } }
439 i = i + 1
440 }
441 return 0 - 1
442}
443
444// HTTP status code from the status line, or -1.
445func hf_status(resp: *u8, n: i64) -> i64 {
446 var i: i64 = 0
447 while i < n { if resp[i]==(32 as u8) { i = n + 1 } else { i = i + 1 } }
448 if i != n + 1 { return 0 - 1 }
449 var p: i64 = 0
450 while p < n { if resp[p]==(32 as u8) { p = p + 1; i = p; p = n } else { p = p + 1 } }
451 var v: i64 = 0
452 var d: i64 = 0
453 while d < 3 {
454 if i + d >= n { return 0 - 1 }
455 let c: i64 = resp[i + d] as i64
456 if c < 48 { return 0 - 1 }
457 if c > 57 { return 0 - 1 }
458 v = v * 10 + (c - 48)
459 d = d + 1
460 }
461 return v
462}