nx_https_fetch_lib.nx source
↩ module page · 783 lines · 34757 B
1// nx_https_fetch_lib.nx -- the sovereign HTTPS GET composition, as a LIBRARY.
2//
3// WHY THIS EXISTS (2026-07-31): the whole working fetch path -- trust-store load,
4// CSPRNG, url parse, connect (with optional override), TLS 1.3 chrome-JA3
5// handshake, cert cache, path+query assembly, HTTP GET -- lived inside
6// nx_https_get_cli.nx's main(). Any second consumer (the album downloader) had
7// exactly two bad options: duplicate ~90 lines of crypto setup, or shell out.
8// Rule 15: a pattern needed by more than one consumer belongs in a lib, and
9// duplicated crypto setup is the kind that diverges SILENTLY -- one copy gets the
10// cert-cache fix or the recv_hs reassembly fix and the other quietly does not.
11//
12// The logic here is LIFTED VERBATIM from the proven CLI, only parameterised. The
13// CLI is then re-pointed at this lib so there is ONE implementation, and its
14// behaviour is re-verified live after the move (a refactor of a crown-jewel path
15// is not done until the old entry point is proven still working).
16//
17// ★ THE STORE IS LOADED SEPARATELY ON PURPOSE. nx_trust_store_load_from_certdata
18// parses a ~4 MiB Mozilla bundle. Doing that per file would make a 300-file album
19// pay it 300 times. hf_store_load() once, then hf_fetch() per url.
20// license_tier: ORIGINAL
21
22import "nx_syscalls.nx"
23import "nx_connect.nx" // bounded connect: a raw sys_connect hangs ~127s on a black-holed host
24import "nx_csprng.nx"
25import "nx_x509_trust_store.nx"
26import "nx_trust_store_load_from_certdata.nx"
27import "nx_tls13_client_validate_certificate.nx"
28import "nx_tls13_client_session_run.nx"
29import "nx_tls13_chrome_session.nx"
30import "nx_https_url_for_fetch.nx"
31import "nx_http_resolve_redirect.nx"
32import "nx_redirect_resolve.nx"
33import "nx_https_url_connect.nx"
34import "nx_https_get_complete.nx"
35import "nx_https_get_stream.nx" // streaming tail: media to an fd, with Range resume
36import "nx_tls_cert_cache.nx"
37import "nx_tls12_client_session.nx" // R10: the AUTHENTICATED TLS-1.2 session (chain + SKE-signature verified)
38import "nx_http_response_parse.nx"
39import "nx_gzip_wrap.nx" // Content-Encoding: gzip -- the canonical in-tree inflate (KAT: nx_gzip_inflate_kat_test)
40import "nx_zlib_wrap.nx" // Content-Encoding: deflate -- HTTP `deflate` means RFC 1950 zlib
41const HF_MAGIC_2047: i64 = 2047
42const HF_MAGIC_2048: i64 = 2048
43const HF_T12_PT_CAP: i64 = 20000
44const HF_T12_MAX_RECORDS: i64 = 4096
45
46const HF_CERTDATA: *u8 = "data/mozilla_certdata.txt\x00"
47const HF_STORE_CAP: i64 = 4194304
48
49// Distinct negative codes so a caller can tell WHICH stage failed. A single -1
50// would make "the site is down" and "our trust store is missing" the same answer.
51const HF_ERR_STORE: i64 = 0 - 2
52const HF_ERR_URL: i64 = 0 - 3
53const HF_ERR_CONNECT: i64 = 0 - 4
54const HF_ERR_TLS: i64 = 0 - 5
55const HF_ERR_HTTP: i64 = 0 - 6
56const HF_ERR_HEADERS: i64 = 0 - 7
57const HF_ERR_HEADER_REDIRECT: i64 = 0 - 8
58
59// Transport-decode outcomes (see hf_decode_transport at the foot of this file).
60// Declared HERE, above hf_fetch, because a module const read before its declaration would
61// silently evaluate to 0 -- and 0 is a VALID decoded length, so the bug would be invisible.
62const HF_DEC_IDENTITY: i64 = 0 - 30 // nothing to strip: caller keeps the original bytes verbatim
63const HF_DEC_ECODING: i64 = 0 - 31 // br/zstd/stacked: we cannot decode it, so we REFUSE rather than emit wrong bytes
64const HF_DEC_CHUNK: i64 = 0 - 32 // malformed chunked framing -- refuse a partial document
65const HF_DEC_INFLATE: i64 = 0 - 33 // invalid or unsupported compressed stream
66const HF_DEC_CAPACITY: i64 = 0 - 34 // caller-owned decoded output budget exhausted
67
68// Load the Mozilla trust store ONCE. Returns the *TrustStore as i64, or 0.
69func hf_store_load() -> i64 {
70 let r: i64 = nx_trust_store_load_from_certdata(HF_CERTDATA, 512, HF_STORE_CAP)
71 if r <= 0 { return 0 }
72 return r
73}
74
75// GET <url> over sovereign TLS 1.3 into out; returns bytes of the RAW response
76// (status line + headers + body) or one of the HF_ERR_* codes.
77//
78// cip/cport: optional connect override (curl --connect-to). cip==0 means resolve
79// the url host normally. The override opens TCP+TLS to that endpoint while SNI,
80// Host and the cert name all stay the URL's host -- which is how our own vhosts
81// get fetched deterministically from the sovereign edge instead of coin-flipping
82// against the DSM nginx that co-squats :443.
83// Connect + handshake + assemble the request path, leaving a live session ready
84// for EITHER tail (buffered read or stream-to-file). Extracted so the two tails
85// cannot drift apart -- duplicating this setup inside my own lib would be the
86// same DRY sin that made extracting it from the CLI necessary in the first place.
87//
88// box[0]=session box[1]=fd box[2]=path_ptr box[3]=path_len
89// box[4]=host_ptr box[5]=host_len. Returns 1 on success, else an HF_ERR_*.
90func hf_open(store_i: i64, url: *u8, cip: i64, cport: i64, box: *i64) -> i64 {
91 if store_i <= 0 { return HF_ERR_STORE }
92 let store: *TrustStore = store_i as *TrustStore
93 let now: i64 = sys_now_realtime_sec()
94
95 let cr: *u8 = sys_mmap(32)
96 nx_csprng_fill(cr, 32)
97 let priv: *u8 = sys_mmap(32)
98 nx_csprng_fill(priv, 32)
99
100 let url_p: *NxUrl = nx_url_new()
101 let target_raw: *u8 = sys_mmap(32)
102 let target: *NxHttpsTarget = target_raw as *NxHttpsTarget
103 target.url = url_p
104 target.port = 0
105 if nx_https_url_for_fetch(url, target) != NX_HTTPS_URL_OK { return HF_ERR_URL }
106
107 // The socket is opened INSIDE the hello-retry loop below, not here: a rejected handshake
108 // kills the connection, so a single pre-loop connect could not be retried on attempt 1.
109 let fd_p: *i64 = sys_mmap(16) as *i64
110
111 let val_raw: *u8 = sys_mmap(128)
112 let val_ctx: *TlsValidationContext = val_raw as *TlsValidationContext
113
114 // TWO HELLOS, NOT ONE (2026-08-06). This path was chrome-JA3 ONLY, and a chrome-mimic hello is
115 // REJECTED by a whole class of federal hosts (www.sec.gov, efts.sec.gov, api.nhtsa.gov,
116 // www.justice.gov) that accept our plain minimal hello. nx_https_get_cli2 had already learned
117 // this on 2026-07-25 and carried the fallback -- but only in ITS OWN main(), so the LIB and
118 // every consumer of it stayed blind. MEASURED THIS SESSION: nx_feed_gate against the lib-backed
119 // fetcher = 7/11 (4x exit=4 TLS), against cli2 = 11/11. Same feeds, same minute; the ONLY
120 // difference was the second hello.
121 // ★THE SAME LESSON LEARNED TWICE IN TWO BINARIES IS A LESSON THE CODEBASE NEVER LEARNED.
122 // Neither fetcher was complete: this one had the shared lib, redirects and the TLS-1.2
123 // fallback; that one had the hello fallback. Merging the missing half here makes ONE path
124 // strictly superior, which is what lets the duplicates be retired instead of maintained.
125 // No single hello serves both sets, so: attempt 0 = chrome-JA3 (byte-identical behaviour for
126 // every host that already worked), attempt 1 = plain hello. A rejected handshake kills the
127 // socket, so each attempt reconnects and draws FRESH ephemerals -- a client-random is never
128 // reused across handshakes.
129 var attempt: i64 = 0
130 var fd: i64 = 0 - 1
131 var sr: i64 = 0 - 1
132 while attempt < 2 {
133 if cip != 0 {
134 let sa2: *u8 = sys_mmap(16)
135 nx_https_build_sockaddr(sa2, cip, cport)
136 let cfd2: i64 = sys_socket(NX_HTTPS_AF_INET, NX_HTTPS_SOCK_STREAM, 0)
137 if cfd2 < 0 { return HF_ERR_CONNECT }
138 sys_set_socket_timeout(cfd2, 15)
139 if nx_connect_bounded(cfd2, sa2, 16, NX_CONN_DEFAULT_MS) < 0 { sys_close(cfd2); return HF_ERR_CONNECT }
140 fd_p[0] = cfd2
141 } else {
142 if nx_https_url_connect(target, url, now, fd_p) != NX_HTTPS_CONNECT_OK { return HF_ERR_CONNECT }
143 }
144 fd = fd_p[0]
145
146 nx_csprng_fill(cr, 32)
147 nx_csprng_fill(priv, 32)
148
149 val_ctx.store = store
150 val_ctx.sni_host = url + target.url.host_off
151 val_ctx.sni_host_len = target.url.host_len
152 val_ctx.now_epoch = now
153 tcc_load(url + target.url.host_off, target.url.host_len, now, val_ctx)
154 tcc_arm(val_ctx)
155
156 if attempt == 0 { sr = nx_tls13_client_session_run_chrome(fd, url + target.url.host_off, target.url.host_len, cr, priv, val_ctx) }
157 else { sr = nx_tls13_client_session_run(fd, url + target.url.host_off, target.url.host_len, cr, priv, val_ctx) }
158
159 if sr >= 0 { attempt = 99 } else { sys_close(fd); attempt = attempt + 1 }
160 }
161 // Still failing after both hellos: hand HF_ERR_TLS back so hf_fetch_once's TLS-1.2 fallback
162 // still gets its turn. The 1.3 hello retry is ADDITIVE to R10, never a replacement for it.
163 if sr < 0 { return HF_ERR_TLS }
164 tcc_save(url + target.url.host_off, target.url.host_len, now, val_ctx)
165
166 var path_ptr: *u8 = url + target.url.path_off
167 var path_len: i64 = target.url.path_len
168 if path_len == 0 {
169 let dp: *u8 = sys_mmap(2)
170 dp[0] = 47 as u8
171 path_ptr = dp
172 path_len = 1
173 }
174 if target.url.query_len > 0 {
175 let full: *u8 = sys_mmap(path_len + target.url.query_len + 4)
176 var fo: i64 = 0
177 var pci: i64 = 0
178 while pci < path_len { full[fo] = path_ptr[pci]; fo = fo + 1; pci = pci + 1 }
179 full[fo] = 63 as u8; fo = fo + 1
180 let qp: *u8 = url + target.url.query_off
181 var qci: i64 = 0
182 while qci < target.url.query_len { full[fo] = qp[qci]; fo = fo + 1; qci = qci + 1 }
183 path_ptr = full
184 path_len = fo
185 }
186
187 box[0] = sr
188 box[1] = fd
189 box[2] = path_ptr as i64
190 box[3] = path_len
191 box[4] = (url + target.url.host_off) as i64
192 box[5] = target.url.host_len
193 return 1
194}
195
196// STREAM a url straight to an open fd -- never through a full-size buffer, so a
197// 5 GB video costs the socket buffer, not 5 GB of RAM. range_start>0 resumes a
198// partial file (HTTP Range), which is what makes an interrupted album re-runnable
199// instead of restart-from-zero.
200// Returns bytes written, or an HF_ERR_*; out_status carries the HTTP status.
201func hf_fetch_to_file_once(store_i: i64, url: *u8, cip: i64, cport: i64, loc: *u8,
202 dest_fd: i64, range_start: i64, out_status: *i64) -> i64 {
203 let box: *i64 = sys_mmap(64) as *i64
204 let o: i64 = hf_open(store_i, url, cip, cport, box)
205 if o != 1 { return o }
206 let session: *Tls13ClientSession = box[0] as *Tls13ClientSession
207 let fd: i64 = box[1]
208 loc[0] = 0 as u8
209 let n: i64 = nx_https_get_stream(session, fd, box[2] as *u8, box[3], box[4] as *u8, box[5],
210 range_start, dest_fd, out_status, loc, HF_MAGIC_2048)
211 sys_close(fd)
212 if n < 0 { return HF_ERR_HTTP }
213 return n
214}
215
216// R10: TLS-1.2 FALLBACK FETCH -- the whole GET over an AUTHENTICATED TLS 1.2 session.
217//
218// WHY (measured 2026-08-05): graphis.ne.jp is Apache 2.2.31 / OpenSSL 1.0.0 and negotiates
219// TLS 1.2 ONLY. Our fetch path was 1.3-only, so it failed at ServerHello -- and the crawler
220// then counted those failures toward WC_HD_RETIRE and retired the host PERMANENTLY. A whole
221// class of the web (older Apache/nginx estates) was therefore not "uncrawlable", it was
222// unreachable BY US, and the scheduler laundered that into permanent coverage loss.
223// ★A TRANSPORT GAP BECOMES PERMANENT COVERAGE LOSS WHEN THE SCHEDULER RETIRES WHAT IT
224// CANNOT FETCH -- so the transport gap is the thing to close.
225//
226// This calls nx_tls12_client_session_run (NOT t12_request in nx_tls12_req.nx, which states in
227// its own header that chain + SKE-signature validation are unwired). A fallback that quietly
228// drops peer authentication would trade a coverage gap for a MITM surface; this one keeps the
229// same TlsValidationContext the 1.3 path uses, so a 1.2 fetch is authenticated or it fails.
230//
231// ⚠DUPLICATION, DELIBERATE AND RECORDED: the drain loop below is the same shape as
232// _acme_http12_roundtrip in nx_acme_http.nx, which already solved this for Porkbun. Merging
233// them means editing the live ACME cert-issuance path, which is a separate risk from adding a
234// function here -- debt filed for the merge rather than papered over.
235func hf_fetch12_headers_once(store_i: i64, url: *u8, cip: i64, cport: i64, out: *u8, cap: i64, xhdr: *u8, xhdr_len: i64) -> i64 {
236 if store_i <= 0 { return HF_ERR_STORE }
237 let store: *TrustStore = store_i as *TrustStore
238 let now: i64 = sys_now_realtime_sec()
239
240 let url_p: *NxUrl = nx_url_new()
241 let target_raw: *u8 = sys_mmap(32)
242 let target: *NxHttpsTarget = target_raw as *NxHttpsTarget
243 target.url = url_p
244 target.port = 0
245 if nx_https_url_for_fetch(url, target) != NX_HTTPS_URL_OK { return HF_ERR_URL }
246
247 let fd_p: *i64 = sys_mmap(16) as *i64
248 if cip != 0 {
249 let sa: *u8 = sys_mmap(16)
250 nx_https_build_sockaddr(sa, cip, cport)
251 let cfd: i64 = sys_socket(NX_HTTPS_AF_INET, NX_HTTPS_SOCK_STREAM, 0)
252 if cfd < 0 { return HF_ERR_CONNECT }
253 sys_set_socket_timeout(cfd, 15)
254 if nx_connect_bounded(cfd, sa, 16, NX_CONN_DEFAULT_MS) < 0 { sys_close(cfd); return HF_ERR_CONNECT }
255 fd_p[0] = cfd
256 } else {
257 if nx_https_url_connect(target, url, now, fd_p) != NX_HTTPS_CONNECT_OK { return HF_ERR_CONNECT }
258 }
259 let fd: i64 = fd_p[0]
260
261 let host: *u8 = (url as i64 + target.url.host_off) as *u8
262 let host_len: i64 = target.url.host_len
263
264 let vc_raw: *u8 = sys_mmap(128)
265 let vc: *TlsValidationContext = vc_raw as *TlsValidationContext
266 vc.store = store
267 vc.sni_host = host
268 vc.sni_host_len = host_len
269 vc.now_epoch = now
270
271 // Fresh ephemerals per connection -- forward secrecy is per-handshake, never reused.
272 let cr: *u8 = sys_mmap(32)
273 nx_csprng_fill(cr, 32)
274 let seed: *u8 = sys_mmap(32)
275 nx_csprng_fill(seed, 32)
276
277 let sr: i64 = nx_tls12_client_session_run(fd, host, host_len, cr, seed, vc)
278 if sr < 0 { sys_close(fd); return HF_ERR_TLS }
279 let s: *Tls12ClientSession = sr as *Tls12ClientSession
280
281 var path_ptr: *u8 = (url as i64 + target.url.path_off) as *u8
282 var path_len: i64 = target.url.path_len
283 if path_len == 0 {
284 let dp: *u8 = sys_mmap(2)
285 dp[0] = 47 as u8
286 path_ptr = dp
287 path_len = 1
288 }
289 if target.url.query_len > 0 {
290 let full: *u8 = sys_mmap(path_len + target.url.query_len + 4)
291 var fo: i64 = 0
292 var pci: i64 = 0
293 while pci < path_len { full[fo] = path_ptr[pci]; fo = fo + 1; pci = pci + 1 }
294 full[fo] = 63 as u8; fo = fo + 1
295 let qp: *u8 = (url as i64 + target.url.query_off) as *u8
296 var qci: i64 = 0
297 while qci < target.url.query_len { full[fo] = qp[qci]; fo = fo + 1; qci = qci + 1 }
298 path_ptr = full
299 path_len = fo
300 }
301
302 let req: *u8 = sys_mmap(nx_http_client_request_cap(path_len,host_len,0,xhdr_len))
303 let req_len: i64 = nx_http_client_build_request_cookie_xhdr(path_ptr,path_len,host,host_len,0 as *u8,0,xhdr,xhdr_len,req)
304 if req_len <= 0 { sys_close(fd); return HF_ERR_HTTP }
305 let sent: i64=nx_tls12_session_send(s,fd,req,req_len)
306 sys_munmap(req,nx_http_client_request_cap(path_len,host_len,0,xhdr_len))
307 if sent!=0 { sys_close(fd); return HF_ERR_HTTP }
308
309 let pt: *u8 = sys_mmap(HF_T12_PT_CAP)
310 let ctp: *i64 = sys_mmap(16) as *i64
311 var acc: i64 = 0
312 var rounds: i64 = 0
313 var draining: i64 = 1
314 while draining == 1 {
315 if rounds >= HF_T12_MAX_RECORDS { draining = 0 }
316 else {
317 let pl: i64 = nx_tls12_session_recv(s, fd, pt, HF_T12_PT_CAP, ctp)
318 rounds = rounds + 1
319 if pl < 0 { draining = 0 }
320 else {
321 if ctp[0] == 23 {
322 var j: i64 = 0
323 while j < pl {
324 if acc < cap { out[acc] = pt[j]; acc = acc + 1 }
325 j = j + 1
326 }
327 }
328 if ctp[0] == 21 { draining = 0 }
329 }
330 }
331 }
332 sys_close(fd)
333 if acc <= 0 { return HF_ERR_HTTP }
334 return acc
335}
336
337func hf_fetch12_once(store_i: i64,url: *u8,cip: i64,cport: i64,out: *u8,cap: i64) -> i64 {
338 return hf_fetch12_headers_once(store_i,url,cip,cport,out,cap,0 as *u8,0)
339}
340
341// Buffered read into `out` -- for PAGES (album/item HTML), where the caller wants
342// the whole document to parse. Media goes through hf_fetch_to_file instead.
343//
344// R10: on a TLS failure the 1.2 fallback runs. The retry is scoped to HF_ERR_TLS ONLY --
345// a connect failure or an HTTP failure is a real answer and is NOT masked by a second attempt.
346func hf_fetch_headers_once(store_i: i64, url: *u8, cip: i64, cport: i64, out: *u8, cap: i64, xhdr: *u8, xhdr_len: i64) -> i64 {
347 let box: *i64 = sys_mmap(64) as *i64
348 let o: i64 = hf_open(store_i, url, cip, cport, box)
349 if o == HF_ERR_TLS { return hf_fetch12_headers_once(store_i,url,cip,cport,out,cap,xhdr,xhdr_len) }
350 if o != 1 { return o }
351 let session: *Tls13ClientSession = box[0] as *Tls13ClientSession
352 let fd: i64 = box[1]
353 let n: i64 = nx_https_get_complete_cookie_xhdr(session,fd,box[2] as *u8,box[3],box[4] as *u8,box[5],0 as *u8,0,xhdr,xhdr_len,out,cap)
354 sys_close(fd)
355 if n < 0 { return HF_ERR_HTTP }
356 return n
357}
358
359func hf_fetch_once(store_i: i64,url: *u8,cip: i64,cport: i64,out: *u8,cap: i64) -> i64 {
360 return hf_fetch_headers_once(store_i,url,cip,cport,out,cap,0 as *u8,0)
361}
362
363// Extract the Location header value from a raw response. Case-insensitive on
364// the field name because header casing is not guaranteed. Returns length, 0 if
365// absent.
366func hf_location(resp: *u8, n: i64, out: *u8, cap: i64) -> i64 {
367 out[0] = 0 as u8
368 var i: i64 = 0
369 while i + 10 < n {
370 var atline: i64 = 0
371 if i == 0 { atline = 1 } else { if resp[i-1] == (10 as u8) { atline = 1 } }
372 if atline == 1 {
373 let key: *u8 = "location:" as *u8
374 var m: i64 = 1
375 var j: i64 = 0
376 while j < 9 {
377 var c: i64 = resp[i+j] as i64
378 if c >= 65 { if c <= 90 { c = c + 32 } }
379 if c != (key[j] as i64) { m = 0; j = 9 } else { j = j + 1 }
380 }
381 if m == 1 {
382 var v: i64 = i + 9
383 var gs: i64 = 1
384 while gs == 1 { if v >= n { gs = 0 } else { if resp[v] == (32 as u8) { v = v + 1 } else { gs = 0 } } }
385 var o: i64 = 0
386 var ge: i64 = 1
387 while ge == 1 {
388 if v >= n { ge = 0 } else {
389 if resp[v] == (13 as u8) { ge = 0 } else {
390 if resp[v] == (10 as u8) { ge = 0 } else {
391 if o + 1 < cap { out[o] = resp[v]; o = o + 1 }
392 v = v + 1
393 }
394 }
395 }
396 }
397 out[o] = 0 as u8
398 return o
399 }
400 }
401 i = i + 1
402 }
403 return 0
404}
405
406// Buffered fetch that FOLLOWS REDIRECTS, bounded.
407// The streaming tail already followed them; this one did not, and the two tails
408// disagreeing was its own defect: the ALBUM PAGE goes through here, so a site
409// whose entry URL 3xx-redirects (en.wikipedia.org does) yielded a 301 body with
410// no links and the ingest refused with -- no item links matched -- naming the
411// adapter rule as the culprit when the real cause was an unfollowed redirect.
412// DECODE-MODE variant. decode=1 strips the transport coding -- what every caller that wants a
413// DOCUMENT needs, and now the default. decode=0 returns the true wire bytes, which a debugging
414// client legitimately needs and which we must not take away (rule 25: never strip a feature to
415// fix a default).
416func hf_fetch_mode_headers(store_i: i64, url: *u8, cip: i64, cport: i64, out: *u8, cap: i64, decode: i64, xhdr: *u8, xhdr_len: i64) -> i64 {
417 if hf_header_block_valid(xhdr,xhdr_len)==0 { return HF_ERR_HEADERS }
418 var cur_n: i64=_gc_slen(url)
419 var cur_cap: i64=cur_n+1
420 var cur: *u8=sys_mmap(cur_cap)
421 if (cur as i64)<=0 { return HF_ERR_URL }
422 var ci: i64=0
423 while ci<cur_n { cur[ci]=url[ci];ci=ci+1 }
424 cur[cur_n]=0 as u8
425 var ip: i64=cip
426 var pt: i64=cport
427 var hops: i64=0
428 var res: i64=HF_ERR_HTTP
429 let resolved_n: *i64=sys_mmap(__size_of(i64)) as *i64
430 if (resolved_n as i64)<=0 { sys_munmap(cur,cur_cap);return HF_ERR_URL }
431 while 1==1 {
432 let n: i64=hf_fetch_headers_once(store_i,cur,ip,pt,out,cap,xhdr,xhdr_len)
433 if n<0 { res=n;break }
434 res=n
435 let st: i64=hf_status(out,n)
436 if nx_redirect_status_is_followable(st)==0 || hops>=3 { break }
437 let head_n: i64=hf_body_off(out,n)
438 if head_n<0 { res=HF_ERR_HTTP;break }
439 // A resolved reference cannot exceed base + response headers + terminator.
440 // Retaining the preceding URL until resolution succeeds avoids aliasing.
441 let next_cap: i64=cur_n+head_n+1
442 let next: *u8=sys_mmap(next_cap)
443 if (next as i64)<=0 { res=HF_ERR_URL;break }
444 let rr: i64=nx_http_resolve_redirect(out,head_n,st,cur,cur_n,next,next_cap-1,resolved_n)
445 if rr==NX_REDIRECT_NO_LOCATION { sys_munmap(next,next_cap);break }
446 if rr!=NX_REDIRECT_RESOLVED { sys_munmap(next,next_cap);res=HF_ERR_URL;break }
447 next[resolved_n[0]]=0 as u8
448 if _rr_is_https_prefix(next,resolved_n[0])==0 { sys_munmap(next,next_cap);res=HF_ERR_URL;break }
449 if xhdr_len>0 && hf_header_redirect_same_origin(cur,next)==0 { sys_munmap(next,next_cap);res=HF_ERR_HEADER_REDIRECT;break }
450 sys_munmap(cur,cur_cap)
451 cur=next;cur_n=resolved_n[0];cur_cap=next_cap
452 ip=0;pt=0;hops=hops+1
453 }
454 sys_munmap(cur,cur_cap);sys_munmap(resolved_n,__size_of(i64))
455 if res<0 || decode==0 { return res }
456 let dec: *u8=sys_mmap(cap)
457 if (dec as i64)<=0 { return HF_ERR_HTTP }
458 let dn: i64=hf_decode_transport(out,res,dec,cap)
459 if dn==HF_DEC_IDENTITY { sys_munmap(dec,cap);return res }
460 if dn<0 { sys_munmap(dec,cap);return dn }
461 var wi: i64=0
462 while wi<dn { out[wi]=dec[wi];wi=wi+1 }
463 sys_munmap(dec,cap)
464 return dn
465}
466
467func hf_fetch_mode(store_i: i64,url: *u8,cip: i64,cport: i64,out: *u8,cap: i64,decode: i64) -> i64 {
468 return hf_fetch_mode_headers(store_i,url,cip,cport,out,cap,decode,0 as *u8,0)
469}
470
471// The name every existing consumer already calls. Decoding is the DEFAULT because a caller that
472// asked for a page and received gzip has been handed the wrong bytes, not a feature.
473func hf_fetch(store_i: i64, url: *u8, cip: i64, cport: i64, out: *u8, cap: i64) -> i64 {
474 return hf_fetch_mode(store_i, url, cip, cport, out, cap, 1)
475}
476
477// Offset of the body inside a raw response (past the CRLFCRLF), or -1 if the
478// header terminator never appears -- REFUSING rather than returning 0, because
479// treating a malformed response as "body starts at 0" would hand the caller the
480// HTTP headers as if they were file bytes.
481// Follow redirects, BOUNDED. nx_https_get_stream returns 0 WITHOUT writing any
482// body on a 3xx and fills loc_buf with the Location (verified at
483// nx_https_get_stream.nx:138), so the destination fd is untouched between hops
484// and there is nothing to truncate -- which matters because no ftruncate
485// primitive exists here. The wrapper only has to ACT on what the primitive
486// already hands back, which it previously ignored.
487//
488// Why this is not a nicety: album hosts redirect item URLs to signed CDN
489// endpoints as the NORM, so without following, EVERY real download lands as
490// zero bytes while the album and site collections still get declared -- empty
491// albums that look structurally correct. Measured live 2026-07-31.
492//
493// The override is dropped after hop 1: it pins a connect endpoint for a
494// specific host, and a redirect by definition changes the host.
495func hf_fetch_to_file(store_i: i64, url: *u8, cip: i64, cport: i64,
496 dest_fd: i64, range_start: i64, out_status: *i64) -> i64 {
497 let cur: *u8 = sys_mmap(HF_MAGIC_2048)
498 var ci: i64 = 0
499 while url[ci] != (0 as u8) { if ci < HF_MAGIC_2047 { cur[ci] = url[ci] } ci = ci + 1 }
500 cur[ci] = 0 as u8
501 let loc: *u8 = sys_mmap(HF_MAGIC_2048)
502 var ip: i64 = cip
503 var pt: i64 = cport
504 var hops: i64 = 0
505 var res: i64 = HF_ERR_HTTP
506 var go: i64 = 1
507 while go == 1 {
508 loc[0] = 0 as u8
509 let n: i64 = hf_fetch_to_file_once(store_i, cur, ip, pt, loc, dest_fd, range_start, out_status)
510 if n < 0 { return n }
511 let st: i64 = out_status[0]
512 var redir: i64 = 0
513 if st >= 300 { if st < 400 { if loc[0] != (0 as u8) { redir = 1 } } }
514 if redir == 1 {
515 if hops < 3 {
516 var k: i64 = 0
517 while loc[k] != (0 as u8) { if k < HF_MAGIC_2047 { cur[k] = loc[k] } k = k + 1 }
518 cur[k] = 0 as u8
519 ip = 0
520 pt = 0
521 hops = hops + 1
522 } else { go = 0; res = n }
523 } else { go = 0; res = n }
524 }
525 return res
526}
527
528func hf_body_off(resp: *u8, n: i64) -> i64 {
529 var i: i64 = 0
530 while i + 3 < n {
531 if resp[i]==(13 as u8) { if resp[i+1]==(10 as u8) { if resp[i+2]==(13 as u8) { if resp[i+3]==(10 as u8) { return i + 4 } } } }
532 i = i + 1
533 }
534 return 0 - 1
535}
536
537// HTTP status code from the status line, or -1.
538func hf_status(resp: *u8, n: i64) -> i64 {
539 var i: i64 = 0
540 while i < n { if resp[i]==(32 as u8) { i = n + 1 } else { i = i + 1 } }
541 if i != n + 1 { return 0 - 1 }
542 var p: i64 = 0
543 while p < n { if resp[p]==(32 as u8) { p = p + 1; i = p; p = n } else { p = p + 1 } }
544 var v: i64 = 0
545 var d: i64 = 0
546 while d < 3 {
547 if i + d >= n { return 0 - 1 }
548 let c: i64 = resp[i + d] as i64
549 if c < 48 { return 0 - 1 }
550 if c > 57 { return 0 - 1 }
551 v = v * 10 + (c - 48)
552 d = d + 1
553 }
554 return v
555}
556
557// ---- TRANSPORT DECODING (2026-08-06) ---------------------------------------------------------
558// WHY THIS LIVES IN THE LIB AND NOT IN A CLI. nx_https_get_cli2 already implemented chunked+gzip
559// decoding -- but ONLY behind a `--body` flag, and the MCP tool rows that front these fetchers
560// pass no flags. So every agent calling nx_https_get / nx_https_get3 over MCP got COMPRESSED
561// BYTES and had to know a magic flag to receive a document. The decoder existed for over a
562// month; the FLEET never had it. Measured 2026-08-06 on https://www.bio-sourced.com/kultevat/ :
563// default = 13171 bytes of gzip garbage, `--body` = 59674 characters of real text, same URL,
564// same second.
565// ★A DECODER ONE CONSUMER IMPORTS IS NOT A CAPABILITY THE FLEET HAS -- so it belongs at the
566// SHARED chokepoint (hf_fetch), where the CLI, the album page fetch, and every future consumer
567// inherit it without opting in.
568//
569// Transport coding is FRAMING, NOT CONTENT. Stripping it is not an API-contract change (rule 19);
570// it is the difference between the wire and the document. The status line and headers are
571// PRESERVED so status-asserting callers (nx_feed_gate asserts an `HTTP/1.1 200` line) are
572// untouched. Identity, unchunked responses are returned BYTE-IDENTICAL -- the common case pays
573// nothing and cannot regress.
574//
575// ★HEADERS THAT DESCRIBED THE WIRE MUST NOT SURVIVE THE DECODE. After inflating we would
576// otherwise emit a body alongside `Content-Encoding: gzip` and a stale `Content-Length` -- headers
577// that are now LIES about the bytes beneath them, and a caller trusting Content-Length would
578// truncate a correct document. The three affected names are renamed IN PLACE to equal-length
579// `X-Was-*` forms, so nothing is hidden and nothing misleads.
580
581func hfd_streq(a: *u8, b: *u8) -> i64 {
582 var i: i64 = 0
583 while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 }
584 if b[i] != (0 as u8) { return 0 }
585 return 1
586}
587
588func hfd_hexval(c: i64) -> i64 {
589 if c >= 48 { if c <= 57 { return c - 48 } }
590 if c >= 97 { if c <= 102 { return c - 87 } }
591 if c >= 65 { if c <= 70 { return c - 55 } }
592 return 0 - 1
593}
594
595// Header-region scans ONLY. A body that merely CONTAINS the word `chunked` must never drive a
596// framing decision -- same discipline the CLI already established.
597func hfd_hdr_chunked(b: *u8, he: i64) -> i64 {
598 let pat: *u8 = "chunked" as *u8
599 var i: i64 = 0
600 while i + 7 <= he {
601 var m: i64 = 1
602 var j: i64 = 0
603 while j < 7 {
604 var c: i64 = b[i+j] as i64
605 if c >= 65 { if c <= 90 { c = c + 32 } }
606 if c != (pat[j] as i64) { m = 0 }
607 j = j + 1
608 }
609 if m == 1 { return 1 }
610 i = i + 1
611 }
612 return 0
613}
614
615// 0 = identity/none, 1 = gzip, 2 = deflate, 3 = present but undecodable here (br, zstd, stacked).
616// Header-driven ON PURPOSE, never magic-sniff: a .gz FILE served with identity encoding IS the
617// document, and sniff-inflating it would corrupt the download.
618func hfd_hdr_enc(b: *u8, he: i64) -> i64 {
619 let pat: *u8 = "content-encoding:" as *u8
620 var at: i64 = 0 - 1
621 var i: i64 = 0
622 while i + 17 <= he {
623 var m: i64 = 1
624 var j: i64 = 0
625 while j < 17 {
626 var c: i64 = b[i+j] as i64
627 if c >= 65 { if c <= 90 { c = c + 32 } }
628 if c != (pat[j] as i64) { m = 0 }
629 j = j + 1
630 }
631 if m == 1 { at = i + 17; i = he + 1 } else { i = i + 1 }
632 }
633 if at < 0 { return 0 }
634 var sk: i64 = 0
635 while sk == 0 {
636 if at >= he { sk = 1 } else {
637 let cs: i64 = b[at] as i64
638 if cs == 32 { at = at + 1 } else { if cs == 9 { at = at + 1 } else { sk = 1 } }
639 }
640 }
641 let tok: *u8 = sys_mmap(16)
642 var tl: i64 = 0
643 var sc: i64 = 0
644 while sc == 0 {
645 if at >= he { sc = 1 } else {
646 var cv: i64 = b[at] as i64
647 if cv == 13 { sc = 1 } else {
648 if cv == 10 { sc = 1 } else {
649 if cv == 44 { return 3 } else {
650 if cv == 32 { sc = 1 } else {
651 if cv == 59 { sc = 1 } else {
652 if tl >= 15 { return 3 }
653 if cv >= 65 { if cv <= 90 { cv = cv + 32 } }
654 tok[tl] = cv as u8
655 tl = tl + 1
656 at = at + 1
657 } } } } }
658 }
659 }
660 tok[tl] = 0 as u8
661 if tl == 0 { return 0 }
662 if hfd_streq(tok, "gzip" as *u8) == 1 { return 1 }
663 if hfd_streq(tok, "x-gzip" as *u8) == 1 { return 1 }
664 if hfd_streq(tok, "deflate" as *u8) == 1 { return 2 }
665 if hfd_streq(tok, "identity" as *u8) == 1 { return 0 }
666 return 3
667}
668
669// Decode a chunked body, bounded by cap. Tolerates chunk extensions, stops at the 0-size chunk.
670// Returns the decoded length, or -1 when the first size line carries no hex digits at all.
671func hfd_dechunk(src: *u8, n: i64, out: *u8, cap: i64) -> i64 {
672 return nx_http_dechunk(src,n,out,cap)
673}
674
675// Rename a header name IN PLACE, equal length, case-insensitive, line-anchored. `pat` lowercase.
676func hfd_rename_hdr(b: *u8, he: i64, pat: *u8, rep: *u8, plen: i64) -> i64 {
677 var i: i64 = 0
678 var hits: i64 = 0
679 while i + plen <= he {
680 var atline: i64 = 0
681 if i == 0 { atline = 1 } else { if b[i-1] == (10 as u8) { atline = 1 } }
682 if atline == 1 {
683 var m: i64 = 1
684 var j: i64 = 0
685 while j < plen {
686 var c: i64 = b[i+j] as i64
687 if c >= 65 { if c <= 90 { c = c + 32 } }
688 if c != (pat[j] as i64) { m = 0 }
689 j = j + 1
690 }
691 if m == 1 {
692 var k: i64 = 0
693 while k < plen { b[i+k] = rep[k]; k = k + 1 }
694 hits = hits + 1
695 }
696 }
697 i = i + 1
698 }
699 return hits
700}
701
702// Rewrite a raw response into (status line + headers + DECODED body) in `out`.
703// Returns the new total length, HF_DEC_IDENTITY when there was nothing to strip, or a negative
704// HF_DEC_* when the response is framed in a way we refuse to guess about.
705func hf_decode_transport(resp: *u8, n: i64, out: *u8, cap: i64) -> i64 {
706 let he: i64 = hf_body_off(resp, n)
707 if he < 0 { return HF_DEC_IDENTITY }
708 let chunked: i64 = hfd_hdr_chunked(resp, he)
709 let enc: i64 = hfd_hdr_enc(resp, he)
710 if chunked == 0 { if enc == 0 { return HF_DEC_IDENTITY } }
711 if enc == 3 { return HF_DEC_ECODING }
712
713 var ent: *u8 = resp + he
714 var en: i64 = n - he
715
716 if chunked == 1 {
717 let db: *u8 = sys_mmap(cap)
718 let dn: i64 = hfd_dechunk(ent, en, db, cap)
719 if dn < 0 { return HF_DEC_CHUNK }
720 ent = db
721 en = dn
722 }
723 if enc == 1 {
724 let gr: *NxGzipResult = nx_gzip_inflate(ent, en, cap)
725 let gerr: i64 = gr.error_code
726 if gerr == NX_GZ_ERR_OUTPUT_CAPACITY { return HF_DEC_CAPACITY }
727 if gerr != 0 { return HF_DEC_INFLATE }
728 ent = gr.output_data
729 en = gr.output_size
730 }
731 if enc == 2 {
732 let zr: *NxZlibResult = nx_zlib_inflate(ent, en, cap)
733 let zerr: i64 = zr.error_code
734 if zerr == NX_ZLIB_ERR_OUTPUT_CAPACITY { return HF_DEC_CAPACITY }
735 if zerr != 0 { return HF_DEC_INFLATE }
736 ent = zr.output_data
737 en = zr.output_size
738 }
739 // A decoded body must not silently become a raw success when headers do not fit.
740 if he > cap { return HF_DEC_CAPACITY }
741 if en > cap - he { return HF_DEC_CAPACITY }
742
743 var o: i64 = 0
744 while o < he { out[o] = resp[o]; o = o + 1 }
745 var k: i64 = 0
746 while k < en { out[he + k] = ent[k]; k = k + 1 }
747
748 hfd_rename_hdr(out, he, "content-encoding" as *u8, "X-Was-C-Encoding" as *u8, 16)
749 hfd_rename_hdr(out, he, "transfer-encoding" as *u8, "X-Was-Tr-Encoding" as *u8, 17)
750 hfd_rename_hdr(out, he, "content-length" as *u8, "X-Was-C-Length" as *u8, 14)
751 return he + en
752}
753
754func hf_header_block_valid(p: *u8,n: i64) -> i64 {
755 if n<0 { return 0 }
756 if n==0 { return 1 }
757 if (p as i64)==0 { return 0 }
758 var at: i64=0
759 while at<n {
760 var end: i64=at
761 while end<n && p[end]!=(13 as u8) { end=end+1 }
762 if end>=n-1 || p[end+1]!=(10 as u8) { return 0 }
763 if hc_header_line_size(p+at,end-at)<0 { return 0 }
764 at=end+2
765 }
766 return 1
767}
768// Header-bearing redirects require the same explicit HTTPS authority. Returning a
769// distinct result lets the caller authorize a new origin without leaking credentials.
770func hf_header_redirect_same_origin(a: *u8,b: *u8) -> i64 {
771 let scheme: *u8="https://"
772 var i: i64=0
773 while scheme[i]!=(0 as u8) {
774 if a[i]!=scheme[i] || b[i]!=scheme[i] { return 0 }
775 i=i+1
776 }
777 while a[i]!=(0 as u8) && a[i]!=(47 as u8) && a[i]!=(63 as u8) && a[i]!=(35 as u8) {
778 if a[i]!=b[i] { return 0 }
779 i=i+1
780 }
781 let c: i64=b[i] as i64
782 return (c==0 || c==47 || c==63 || c==35) as i64
783}