nx_https_fetch_follow.nx source
↩ module page · 761 lines · 45204 B
1// nx_https_fetch_follow.nx -- Fetcher upgrade F-2: LIVE redirect-following fetch.
2// Composes the proven sovereign TLS-1.3 fetch sequence (url_for_fetch -> connect
3// -> client_session_run -> get_complete -> response_parse) with F-1's redirect
4// logic (nx_https_redirect) in a bounded loop: on a 3xx with Location, re-fetch
5// the new URL up to max_hops. 100% sovereign (nx_cc->nxasm, own TLS); the only
6// non-Nishi inputs are the CA-root DATA + the fetched pages.
7//
8// API: nx_https_fetch_follow(url, store, out, out_cap, max_hops, out_status)
9// -> POSITIVE body_bytes (final non-redirect response, dechunked into out)
10// | NEGATIVE -code on failure. out_status[0] = final HTTP status.
11//
12// nx_safety_envelope:
13// intended_use: sovereign web fetcher (live network)
14// sil_target: SIL1
15// verdict: NOT_YET_EVALUATED
16// genealogy_id: ietf/rfc_9110+9112 lineage_id: nx_https_fetch_follow_v1
17
18import "nx_syscalls.nx"
19import "nx_x509_trust_store.nx"
20import "nx_tls13_client_validate_certificate.nx"
21import "nx_tls13_client_session_run.nx"
22import "nx_https_url_for_fetch.nx"
23import "nx_https_url_connect.nx"
24import "nx_https_get_complete.nx"
25import "nx_tls13_chrome_session.nx" // run_chrome = the Chrome-JA3 hello path (beats CDN fingerprinting + RSA-PSS CV)
26import "nx_http_response_parse.nx"
27import "nx_https_redirect.nx"
28import "nx_http_range_req.nx" // Range: bytes=S-E builder -> the Common Crawl targeted-record fetch
29import "nx_tls12_req.nx" // t12_request = the TLS-1.2 client that ACTUALLY reaches 1.2-only hosts
30import "nx_tls12_client_session.nx" // TLS 1.2 session (run/send/recv) -> reach TLS-1.2-only hosts (ukdevilz-class)
31import "nx_csprng.nx" // nx_csprng_fill -> fresh handshake entropy for the 1.2 client
32import "nx_gzip_wrap.nx" // nx_gzip_inflate -- we ADVERTISE gzip, so the chokepoint must decode it
33// Content-Encoding: gzip -- inflate AT THE CHOKEPOINT so every caller of the shared fetch gets TEXT.
34// nx_codec_caps DERIVES Accept-Encoding from the decoders that exist and advertises gzip, so servers
35// legitimately compress -- and returning those bytes to a caller that asked for a page hands it garbage
36// that LOOKS like a successful fetch (positive length, status 200), which is the worst possible failure
37// shape for a crawler: it INDEXES the garbage instead of erroring.
38// MEASURED 2026-08-06 on https://www.v2ph.com/actor/JULIA?page=3&hl=en -- SAME url, same second:
39// 1.3 leg -> 200, 6,079 bytes beginning 1f 8b (gzip, undecoded)
40// 1.2 leg -> 200, 30,385 bytes of <!DOCTYPE html> (it sends Accept-Encoding: identity)
41// nx_https_get_cli2 imported this decoder long ago; the SHARED path never did, so ~200 call sites and
42// the web crawler have been storing compressed bytes as page text.
43// ★A DECODER ONE CONSUMER IMPORTS IS NOT A CAPABILITY THE FLEET HAS.
44const NX_FF_GZ_MAX_OUT: i64 = 33554432 // declared: 32MB inflate ceiling, so a zip-bomb cannot take the host
45func ff_gunzip(out: *u8, n: i64, out_cap: i64) -> i64 {
46 if n < 2 { return n }
47 if (out[0] as i64) != 0x1f { return n }
48 if (out[1] as i64) != 0x8b { return n }
49 let gr: *NxGzipResult = nx_gzip_inflate(out, n, NX_FF_GZ_MAX_OUT)
50 // FAIL-SOFT, NEVER FAIL-SILENT-ZERO: an undecodable body is handed back exactly as received. Returning
51 // 0 here would turn "we could not inflate" into "the page was empty", which is the lie that cost this
52 // lane two sessions on the 1.2 leg (debt 1785971025).
53 if gr.error_code != 0 { return n }
54 let gsz: i64 = gr.output_size
55 if gsz <= 0 { return n }
56 var m: i64 = gsz
57 if m > out_cap { m = out_cap }
58 let src: *u8 = gr.output_data
59 var i: i64 = 0
60 while i < m { out[i] = src[i]; i = i + 1 }
61 return m
62}
63const NX_MAGIC_4096: i64 = 4096
64const NX_MAGIC_2048: i64 = 2048
65const NX_MAGIC_65536: i64 = 65536
66const NX_MAGIC_20000: i64 = 20000
67const NX_MAGIC_8192: i64 = 8192
68
69const NX_FF_BAD_URL: i64 = 1
70const NX_FF_CONNECT: i64 = 2
71const NX_FF_HANDSHAKE: i64 = 3
72const NX_FF_GET: i64 = 4
73const NX_FF_PARSE: i64 = 5
74const NX_FF_NO_LOCATION: i64 = 6
75const NX_FF_TOO_MANY: i64 = 7
76// A CUT BODY IS ITS OWN OUTCOME, NOT A HANDSHAKE FAILURE AND NOT A SUCCESS. Callers branch on this to
77// refuse to SAVE rather than retrying a deterministic truncation.
78const NX_FF_TRUNCATED: i64 = 9
79// Per-attempt recv+send timeout (seconds) on the fetch socket. Without it, an anti-bot CDN (Cloudflare) that ACCEPTS
80// the TCP connection but STALLS a non-browser TLS hello hangs the blocking read forever -> the whole fetch never
81// returns, and the escalating best-effort path (minimal -> Chrome-JA3 -> TLS 1.2) never reaches the hello that works.
82// 8s is generous for a healthy handshake+TTFB (<2s typical) yet fails a true stall fast enough for the next hello to
83// run inside the edge's proxy window. Read timeout surfaces as sys_read<0 -> READ_ERR -> the read loop returns (no
84// busy-loop; verified in nx_tls13_read_record_from_fd._read_n).
85const NX_FF_FETCH_TIMEOUT_SECS: i64 = 8
86
87// Extract the request path from the URL string into path_out (NUL-terminated).
88// Path starts after host[:port]; defaults to "/" when absent. Returns length.
89func ff_path(urlbuf: *u8, target: *NxHttpsTarget, path_out: *u8) -> i64 {
90 var p: i64 = target.url.host_off + target.url.host_len
91 if urlbuf[p] == 0x3A as u8 { // ':' port present -> skip :digits
92 p = p + 1
93 while urlbuf[p] >= 0x30 as u8 { if urlbuf[p] <= 0x39 as u8 { p = p + 1 } else { break } }
94 }
95 if urlbuf[p] != 0x2F as u8 { // no '/' -> path is "/"
96 path_out[0] = 0x2F as u8
97 path_out[1] = 0 as u8
98 return 1
99 }
100 var k: i64 = 0
101 while urlbuf[p] != 0 as u8 { path_out[k] = urlbuf[p]; p = p + 1; k = k + 1 }
102 path_out[k] = 0 as u8
103 return k
104}
105
106// ---- RFC-3986 relative-reference resolution for redirect Location ----
107// The Location header MAY be a relative reference (RFC 9110 §10.2.2): absolute
108// (https://h/p), protocol-relative (//h/p), path-absolute (/p), or bare (p).
109// The original follow loop copied Location verbatim + re-parsed as absolute, so
110// any non-absolute Location failed nx_url_parse (NO_SCHEME) -> -NX_FF_BAD_URL.
111// These helpers resolve it against the current request URI (scheme is always
112// https on this fetcher; host/port carried from the current hop).
113
114func ff_apps(dst: *u8, k0: i64, src: *u8) -> i64 { // append NUL-term src
115 var k: i64 = k0; var j: i64 = 0
116 while src[j] != 0 as u8 { dst[k] = src[j]; k = k + 1; j = j + 1 }
117 return k
118}
119func ff_appn(dst: *u8, k0: i64, src: *u8, n: i64) -> i64 { // append n bytes
120 var k: i64 = k0; var j: i64 = 0
121 while j < n { dst[k] = src[j]; k = k + 1; j = j + 1 }
122 return k
123}
124func ff_appi(dst: *u8, k0: i64, v: i64) -> i64 { // append decimal int
125 var k: i64 = k0
126 if v == 0 { dst[k] = 0x30 as u8; return k + 1 }
127 let tmp: *u8 = sys_mmap(24); var m: i64 = v; var t: i64 = 0
128 while m > 0 { tmp[t] = (0x30 + (m % 10)) as u8; m = m / 10; t = t + 1 }
129 var i: i64 = t - 1
130 while i >= 0 { dst[k] = tmp[i]; k = k + 1; i = i - 1 }
131 return k
132}
133func ff_starts(s: *u8, p: *u8) -> i64 { // s starts with prefix p?
134 var j: i64 = 0
135 while p[j] != 0 as u8 { if s[j] != p[j] { return 0 } j = j + 1 }
136 return 1
137}
138// Resolve `loc` against current URI (host/port from `cur` over `cur_urlbuf`) into
139// outp (NUL-terminated absolute https URL). http:// is UPGRADED to https:// -- we
140// never follow a redirect into plaintext (never-relax-security guardrail).
141func ff_resolve_location(loc: *u8, cur_urlbuf: *u8, cur: *NxHttpsTarget, outp: *u8) -> i64 {
142 var k: i64 = 0
143 if ff_starts(loc, "https://" as *u8) == 1 { // already absolute https
144 k = ff_apps(outp, k, loc); outp[k] = 0 as u8; return k
145 }
146 if ff_starts(loc, "http://" as *u8) == 1 { // secure upgrade http->https
147 k = ff_apps(outp, k, "https://" as *u8)
148 k = ff_apps(outp, k, loc + 7) // skip "http://"
149 outp[k] = 0 as u8; return k
150 }
151 if ff_starts(loc, "//" as *u8) == 1 { // protocol-relative
152 k = ff_apps(outp, k, "https:" as *u8)
153 k = ff_apps(outp, k, loc); outp[k] = 0 as u8; return k
154 }
155 // path-absolute (/p) or bare (p): keep the current authority.
156 k = ff_apps(outp, k, "https://" as *u8)
157 k = ff_appn(outp, k, cur_urlbuf + cur.url.host_off, cur.url.host_len)
158 if cur.url.port != 0 { outp[k] = 0x3A as u8; k = k + 1; k = ff_appi(outp, k, cur.url.port) }
159 if loc[0] == 0x2F as u8 { k = ff_apps(outp, k, loc) }
160 else { outp[k] = 0x2F as u8; k = k + 1; k = ff_apps(outp, k, loc) }
161 outp[k] = 0 as u8; return k
162}
163
164// ---- Cookie jar for the redirect loop (browser-faithful session carry) ----
165// Real browsers keep a cookie jar across a redirect chain; our fetcher dropped cookies between
166// hops, so cookie-gated sites (chaturbate's /?next= age-gate hands back csrftoken/sbr/AG_Key)
167// could never be reached. These helpers collect Set-Cookie into a flat "n=v; n=v" jar (dedup by
168// name = last-writer-wins, RFC-6265 jar semantics) and detect the "gate-and-return" ?next= idiom.
169const NX_FF_JAR_BYTES: i64 = 3072
170
171// Case-insensitive match of a lowercase pattern at buf[pos..]. Returns 1 on match, 0 otherwise.
172func ff_hdr_match(buf: *u8, pos: i64, n: i64, pat: *u8, patlen: i64) -> i64 {
173 if pos + patlen > n { return 0 }
174 var k: i64 = 0
175 while k < patlen {
176 var c: i64 = buf[pos+k] as i64
177 if c >= 65 { if c <= 90 { c = c + 32 } } // ASCII upper -> lower
178 if c != (pat[k] as i64) { return 0 }
179 k = k + 1
180 }
181 return 1
182}
183
184// Set cookie `name`=`val` in the jar (jl = current length), replacing any prior entry for the
185// same name (delete-then-append) so the freshest value wins. Fail-safe: skips if it won't fit.
186func ff_jar_set(jar: *u8, jl: *i64, cap: i64, name: *u8, name_len: i64, val: *u8, val_len: i64) -> i64 {
187 var i: i64 = 0
188 let cur: i64 = jl[0]
189 while i < cur {
190 var m: i64 = 1
191 var k: i64 = 0
192 while k < name_len { if jar[i+k] != name[k] { m = 0; k = name_len } else { k = k + 1 } }
193 if m == 1 {
194 if jar[i+name_len] == (61 as u8) { // matched "name="
195 var e: i64 = i + name_len
196 while e < cur { if jar[e] == (59 as u8) { break } e = e + 1 } // to ';'
197 if e < cur { if jar[e] == (59 as u8) { e = e + 1; if e < cur { if jar[e] == (32 as u8) { e = e + 1 } } } }
198 var s: i64 = 0
199 while e + s < cur { jar[i+s] = jar[e+s]; s = s + 1 }
200 jl[0] = cur - (e - i)
201 i = cur
202 } else { i = i + 1 }
203 } else {
204 while i < cur { if jar[i] == (59 as u8) { break } i = i + 1 }
205 if i < cur { i = i + 1; if i < cur { if jar[i] == (32 as u8) { i = i + 1 } } }
206 }
207 }
208 var o: i64 = jl[0]
209 if o + name_len + val_len + 4 > cap { return 0 } // no room -> skip (fail-safe)
210 if o > 0 { jar[o]=59;o=o+1; jar[o]=32;o=o+1 } // "; "
211 var a: i64 = 0
212 while a < name_len { jar[o]=name[a]; o=o+1; a=a+1 }
213 jar[o]=61;o=o+1 // '='
214 var b: i64 = 0
215 while b < val_len { jar[o]=val[b]; o=o+1; b=b+1 }
216 jl[0] = o
217 return 1
218}
219
220// Scan a raw HTTP response's headers for Set-Cookie lines; jar-set each name=value (value up to
221// the first ';'). Cookies live only in headers, so scanning the whole buffer is safe+simple.
222func ff_collect_cookies(buf: *u8, n: i64, jar: *u8, jl: *i64, cap: i64) -> i64 {
223 let pat: *u8 = "set-cookie:" as *u8
224 var i: i64 = 0
225 while i < n {
226 if ff_hdr_match(buf, i, n, pat, 11) == 1 {
227 var p: i64 = i + 11
228 while p < n { if buf[p] != (32 as u8) { break } p = p + 1 } // skip SP
229 let nstart: i64 = p
230 while p < n { if buf[p] == (61 as u8) { break } if buf[p] == (59 as u8) { break } if buf[p] == (13 as u8) { break } p = p + 1 }
231 if p < n { if buf[p] == (61 as u8) {
232 let name_len: i64 = p - nstart
233 p = p + 1
234 let vstart: i64 = p
235 while p < n { if buf[p] == (59 as u8) { break } if buf[p] == (13 as u8) { break } if buf[p] == (10 as u8) { break } p = p + 1 }
236 let val_len: i64 = p - vstart
237 if name_len > 0 { if name_len < 128 { ff_jar_set(jar, jl, cap, buf + nstart, name_len, buf + vstart, val_len) } }
238 } }
239 }
240 while i < n { if buf[i] == (10 as u8) { break } i = i + 1 } // to end of line
241 i = i + 1
242 }
243 return 0
244}
245
246// True if the URL carries a "?next="/"&next=" gate-and-return param (chaturbate-class age gate):
247// we visit it to collect the session, then re-request the original target with the jar.
248func ff_is_gate_url(url: *u8) -> i64 {
249 var i: i64 = 0
250 while url[i] != 0 as u8 {
251 if url[i] == (63 as u8) { if ff_hdr_match(url, i+1, i+7, "next=" as *u8, 5) == 1 { return 1 } }
252 if url[i] == (38 as u8) { if ff_hdr_match(url, i+1, i+7, "next=" as *u8, 5) == 1 { return 1 } }
253 i = i + 1
254 }
255 return 0
256}
257
258// ff_core: the redirect-following TLS-1.3 GET. use_chrome=0 -> the minimal fleet hello; use_chrome=1 -> the
259// Chrome-JA3 hello (run_chrome) that passes anti-bot CDN fingerprinting + verifies RSA-PSS CertificateVerify.
260// Carries a cookie jar across hops (ff_collect_cookies/ff_jar_set) + one gate-bounce back to the original
261// URL after a ?next= age-gate, so cookie-gated rooms resolve. Public sigs unchanged (Cardinal 19).
262// ---- REDIRECT DEPTH AS AN OBSERVABLE (2026-08-16) --------------------------------------------
263// max_hops goes IN; the hop count never came OUT. The chain was walked and its length discarded at
264// the boundary, so nothing downstream could distinguish a page that DELIVERED from one that BOUNCED --
265// and a doorway that redirects onward without ever serving the artifact is the single most
266// goal-defeating result a search engine can return. It looks like an answer and hands over nothing.
267// Exposed as a STATIC + accessor rather than a new parameter ON PURPOSE: ff_core_x is reached through
268// four wrappers and ~200 call sites, so a signature change would be a large blast radius for a
269// read-only diagnostic. Nothing existing moves; a caller that wants the depth asks for it.
270// SCOPE, DECLARED: single-threaded fork-per-job organs, last-call-wins. Read it immediately after the
271// fetch you care about. It is a diagnostic, never an authorisation input.
272static ff_last_hops_g: i64
273func nx_https_last_hops() -> i64 { return ff_last_hops_g }
274func ff_core_x(url0: *u8, store: *TrustStore, out: *u8, out_cap: i64,
275 max_hops: i64, out_status: *i64, use_chrome: i64, xhdr: *u8, xhdr_len: i64) -> i64 {
276 out_status[0] = 0
277 // -1 = UNOBSERVED: the loop was never entered (bad url, etc). MEASURED 2026-08-16: initialising
278 // this to 0 made a failed fetch read IDENTICALLY to a clean direct fetch -- an http:// url returned
279 // body_bytes=-1 with redirect_hops=0, the same 0 a successful no-redirect fetch reports. A counter
280 // that cannot say "I never measured" reports absence of evidence as evidence of absence.
281 ff_last_hops_g = 0 - 1
282 let urlbuf: *u8 = sys_mmap(NX_MAGIC_4096)
283 var ui: i64 = 0
284 while url0[ui] != 0 as u8 { urlbuf[ui] = url0[ui]; ui = ui + 1 }
285 urlbuf[ui] = 0 as u8
286
287 // cookie jar (carried across hops) + original-target snapshot for the one gate-bounce
288 let jar: *u8 = sys_mmap(NX_FF_JAR_BYTES)
289 let jl: *i64 = (sys_mmap(8)) as *i64
290 jl[0] = 0
291 let orig: *u8 = sys_mmap(NX_MAGIC_4096)
292 var oi: i64 = 0
293 while url0[oi] != 0 as u8 { orig[oi] = url0[oi]; oi = oi + 1 }
294 orig[oi] = 0 as u8
295 var bounced: i64 = 0
296
297 let cr: *u8 = sys_mmap(32)
298 let priv: *u8 = sys_mmap(32)
299 var hop: i64 = 0
300 while hop <= max_hops {
301 // record depth per iteration, so ANY return path (success, refusal, transport failure) leaves
302 // the static holding the hop actually reached. A value set only on success would read 0 for
303 // exactly the bounced chains this exists to expose.
304 ff_last_hops_g = hop
305 let target_raw: *u8 = sys_mmap(64)
306 let target: *NxHttpsTarget = target_raw as *NxHttpsTarget
307 target.url = nx_url_new()
308 target.port = 0
309 if nx_https_url_for_fetch(urlbuf, target) != NX_HTTPS_URL_OK { return 0 - NX_FF_BAD_URL }
310
311 let fd_p: *i64 = (sys_mmap(16)) as *i64
312 if nx_https_url_connect(target, urlbuf, sys_now_realtime_sec(), fd_p) != NX_HTTPS_CONNECT_OK { return 0 - NX_FF_CONNECT }
313 let fd: i64 = fd_p[0]
314 sys_set_socket_timeout(fd, NX_FF_FETCH_TIMEOUT_SECS) // bound handshake+read: a stalling host fails fast so best-effort can escalate (never hangs forever)
315
316 // SEV-9 FIX 2026-08-05 (debt 1785970852, CWE-330): this loop used to hardcode the X25519
317 // scalar to 0xA0..0xBF and client_random to 0xC0..0xDF, so EVERY session the fleet opened
318 // shared ONE "ephemeral" private key derivable from any binary -- forward secrecy absent,
319 // not merely weak, and retroactively so for anything ever recorded. nx_csprng_fill was
320 // already imported in this very file for the TLS 1.2 leg: the primitive was never missing.
321 // ★A TEST CONSTANT THAT ESCAPES INTO PRODUCTION IS INDISTINGUISHABLE FROM A BACKDOOR.
322 nx_csprng_fill(cr, 32)
323 nx_csprng_fill(priv, 32)
324 let vc_raw: *u8 = sys_mmap(64)
325 let vc: *TlsValidationContext = vc_raw as *TlsValidationContext
326 vc.store = store
327 vc.sni_host = urlbuf + target.url.host_off
328 vc.sni_host_len = target.url.host_len
329 vc.now_epoch = sys_now_realtime_sec()
330 var sr: i64 = 0
331 if use_chrome == 1 { sr = nx_tls13_client_session_run_chrome(fd, urlbuf + target.url.host_off, target.url.host_len, cr, priv, vc) }
332 else { sr = nx_tls13_client_session_run(fd, urlbuf + target.url.host_off, target.url.host_len, cr, priv, vc) }
333 if sr <= 0 { sys_close(fd); return 0 - NX_FF_HANDSHAKE }
334 let session: *Tls13ClientSession = sr as *Tls13ClientSession
335
336 let path: *u8 = sys_mmap(NX_MAGIC_2048)
337 let plen: i64 = ff_path(urlbuf, target, path)
338 let icap: i64 = out_cap + out_cap/8 + NX_MAGIC_65536 // internal raw-response buffer sized to the caller's out_cap (grow for complete repos)
339 let buf: *u8 = sys_mmap(icap)
340 let gc: i64 = nx_https_get_complete_cookie_xhdr(session, fd, path, plen, urlbuf + target.url.host_off, target.url.host_len, jar, jl[0], xhdr, xhdr_len, buf, icap)
341 sys_close(fd)
342 if gc < 0 { sys_munmap(buf as *u8, icap); return 0 - NX_FF_GET }
343
344 let r: *i64 = (sys_mmap(128)) as *i64
345 if nx_http_response_parse(buf, gc, r) != 0 { sys_munmap(buf as *u8, icap); return 0 - NX_FF_PARSE }
346 let status: i64 = r[1]
347 out_status[0] = status
348 ff_collect_cookies(buf, gc, jar, jl, NX_FF_JAR_BYTES) // carry Set-Cookie to the next hop
349
350 if nx_redir_is_redirect(status) == 1 {
351 let loc: *u8 = sys_mmap(NX_MAGIC_4096)
352 let ln: i64 = nx_redir_location(buf, gc, loc, NX_MAGIC_4096)
353 if ln <= 0 { sys_munmap(buf as *u8, icap); return 0 - NX_FF_NO_LOCATION }
354 // Resolve a possibly-relative Location against the current URI into a
355 // temp buffer (reading host from urlbuf), THEN copy back into urlbuf.
356 let resolved: *u8 = sys_mmap(NX_MAGIC_4096)
357 ff_resolve_location(loc, urlbuf, target, resolved)
358 var k: i64 = 0
359 while resolved[k] != 0 as u8 { urlbuf[k] = resolved[k]; k = k + 1 }
360 urlbuf[k] = 0 as u8
361 sys_munmap(buf as *u8, icap); sys_munmap(loc as *u8, NX_MAGIC_4096); sys_munmap(resolved as *u8, NX_MAGIC_4096)
362 hop = hop + 1
363 } else {
364 // Gate-bounce: if this hop landed on a ?next= age-gate (not the target) and we now hold a
365 // session, re-request the ORIGINAL url ONCE with the jar to reach the real content. do_bounce
366 // is per-iteration so the re-request's 200 still returns normally (no loop, bounded by max_hops).
367 var do_bounce: i64 = 0
368 if bounced == 0 { if jl[0] > 0 { if ff_is_gate_url(urlbuf) == 1 { do_bounce = 1 } } }
369 if do_bounce == 1 {
370 bounced = 1
371 var z: i64 = 0
372 while orig[z] != 0 as u8 { urlbuf[z] = orig[z]; z = z + 1 }
373 urlbuf[z] = 0 as u8
374 sys_munmap(buf as *u8, icap)
375 hop = hop + 1
376 } else {
377 let body_off: i64 = r[6]
378 if r[10] == 1 { // chunked -> dechunk
379 let dn: i64 = nx_http_dechunk(buf + body_off, gc - body_off, out, out_cap)
380 sys_munmap(buf as *u8, icap) // FREE the 8MB response buffer (was the dominant per-fetch leak)
381 if dn < 0 { return dn }
382 return ff_gunzip(out, dn, out_cap)
383 }
384 var blen: i64 = gc - body_off // identity -> copy received body
385 if blen < 0 { blen = 0 }
386 if blen > out_cap { blen = out_cap }
387 var c: i64 = 0
388 while c < blen { out[c] = buf[body_off + c]; c = c + 1 }
389 sys_munmap(buf as *u8, icap) // FREE the 8MB response buffer (was the dominant per-fetch leak)
390 return ff_gunzip(out, blen, out_cap)
391 }
392 }
393 }
394 return 0 - NX_FF_TOO_MANY
395}
396
397// ff_core: STABLE 7-arg shim delegating to ff_core_x with NO caller header (xhdr_len=0), so every existing
398// caller (nx_https_fetch_follow_13only/_chrome/_best) is byte-identical and untouched -- their call sites are
399// not edited. The header-capable entry point passes a real xhdr to ff_core_x directly.
400// (fold 2026-08-12: caller-supplied headers on the redirect-following best-effort ladder.)
401func ff_core(url0: *u8, store: *TrustStore, out: *u8, out_cap: i64,
402 max_hops: i64, out_status: *i64, use_chrome: i64) -> i64 {
403 return ff_core_x(url0, store, out, out_cap, max_hops, out_status, use_chrome, "" as *u8, 0)
404}
405
406// Stable API (Cardinal 19): the minimal fleet hello, byte-for-byte behavior.
407// RENAMED 2026-07-31 (debt 1785516905) -- kept verbatim under an EXPLICIT name for any caller that
408// genuinely needs narrow 1.3-only behavior. The nx_https_fetch_follow NAME now delegates to _best;
409// see the tail of this file for why that is additive rather than a contract break.
410func nx_https_fetch_follow_13only(url0: *u8, store: *TrustStore, out: *u8, out_cap: i64,
411 max_hops: i64, out_status: *i64) -> i64 {
412 return ff_core(url0, store, out, out_cap, max_hops, out_status, 0)
413}
414
415// Chrome-JA3 fetch: for anti-bot-CDN targets (Cloudflare image hosts etc.). Same signature.
416func nx_https_fetch_follow_chrome(url0: *u8, store: *TrustStore, out: *u8, out_cap: i64,
417 max_hops: i64, out_status: *i64) -> i64 {
418 return ff_core(url0, store, out, out_cap, max_hops, out_status, 1)
419}
420// nx_https_fetch_range: fetch ONE byte-range of a URL (RFC 9110 Range) -> the Common Crawl targeted-record
421// path (CDX gives warc-file/offset/length; this pulls exactly that gzip member, no 344TiB download). NO
422// redirect loop (the CC data host serves ranges directly); reuses the proven connect+handshake +
423// nx_https_req_complete send/recv core. use_chrome=1 for anti-bot CDNs. out_status = 206 (partial) or 200.
424// Returns body_bytes (the raw range bytes -- a gzip member for CC) or -code.
425func nx_https_fetch_range(url0: *u8, store: *TrustStore, out: *u8, out_cap: i64,
426 rstart: i64, rend: i64, out_status: *i64, use_chrome: i64) -> i64 {
427 return _ff_range_core(url0, store, out, out_cap, 0, rstart, rend, out_status, use_chrome)
428}
429// nx_https_fetch_head: the SAME range GET, read as a BOUNDED PREFIX (2026-09-05). A server that honours
430// Range answers 206 with the window; a server that ignores it (the static front door, measured on
431// /world/ref9d.nxa, 10,758,232 B) answers 200 with the whole body -- and the completing reader then refuses
432// on overflow, so the caller learns NOTHING about a body it only wanted eight bytes of. This entry reads
433// through nx_https_req_prefix instead: the response is taken up to the internal cap, sealed TRUNCATED
434// (announced on stderr), and the first min(body, out_cap) bytes are returned with their status, so a
435// verifier can prove a rig's magic or a media head without reading the rig. Same signature as the
436// completing sibling; the choice between them is the caller's declared intent, never a size guess.
437func nx_https_fetch_head(url0: *u8, store: *TrustStore, out: *u8, out_cap: i64,
438 rstart: i64, rend: i64, out_status: *i64, use_chrome: i64) -> i64 {
439 return _ff_range_core(url0, store, out, out_cap, 1, rstart, rend, out_status, use_chrome)
440}
441func _ff_range_core(url0: *u8, store: *TrustStore, out: *u8, out_cap: i64, prefix_ok: i64,
442 rstart: i64, rend: i64, out_status: *i64, use_chrome: i64) -> i64 {
443 out_status[0] = 0
444 let urlbuf: *u8 = sys_mmap(NX_MAGIC_4096)
445 var ui: i64 = 0
446 while url0[ui] != 0 as u8 { urlbuf[ui] = url0[ui]; ui = ui + 1 }
447 urlbuf[ui] = 0 as u8
448
449 let target_raw: *u8 = sys_mmap(64)
450 let target: *NxHttpsTarget = target_raw as *NxHttpsTarget
451 target.url = nx_url_new()
452 target.port = 0
453 if nx_https_url_for_fetch(urlbuf, target) != NX_HTTPS_URL_OK { return 0 - NX_FF_BAD_URL }
454
455 let fd_p: *i64 = (sys_mmap(16)) as *i64
456 if nx_https_url_connect(target, urlbuf, sys_now_realtime_sec(), fd_p) != NX_HTTPS_CONNECT_OK { return 0 - NX_FF_CONNECT }
457 let fd: i64 = fd_p[0]
458
459 let cr: *u8 = sys_mmap(32)
460 let priv: *u8 = sys_mmap(32)
461 // SEV-9 FIX 2026-08-05 (debt 1785970852, CWE-330) -- see the note at the sibling site above.
462 // Fresh per-connection entropy is what makes the key ephemeral; a constant makes it a shared secret.
463 nx_csprng_fill(cr, 32)
464 nx_csprng_fill(priv, 32)
465 let vc_raw: *u8 = sys_mmap(64)
466 let vc: *TlsValidationContext = vc_raw as *TlsValidationContext
467 vc.store = store
468 vc.sni_host = urlbuf + target.url.host_off
469 vc.sni_host_len = target.url.host_len
470 vc.now_epoch = sys_now_realtime_sec()
471 var sr: i64 = 0
472 if use_chrome == 1 { sr = nx_tls13_client_session_run_chrome(fd, urlbuf + target.url.host_off, target.url.host_len, cr, priv, vc) }
473 else { sr = nx_tls13_client_session_run(fd, urlbuf + target.url.host_off, target.url.host_len, cr, priv, vc) }
474 if sr <= 0 { sys_close(fd); return 0 - NX_FF_HANDSHAKE }
475 let session: *Tls13ClientSession = sr as *Tls13ClientSession
476
477 let path: *u8 = sys_mmap(NX_MAGIC_2048)
478 let plen: i64 = ff_path(urlbuf, target, path)
479 let req: *u8 = sys_mmap(NX_MAGIC_4096)
480 let req_len: i64 = nx_http_build_range_get(path, plen, urlbuf + target.url.host_off, target.url.host_len, rstart, rend, req)
481 let icap: i64 = out_cap + out_cap/8 + NX_MAGIC_65536
482 let buf: *u8 = sys_mmap(icap)
483 // the caller's declared intent picks the reader: a completing read refuses overflow, a prefix read returns it
484 var gc: i64 = 0
485 if prefix_ok == 1 { gc = nx_https_req_prefix(session, fd, req, req_len, buf, icap) } else { gc = nx_https_req_complete(session, fd, req, req_len, buf, icap) }
486 sys_close(fd)
487 if gc < 0 { sys_munmap(buf as *u8, icap); return 0 - NX_FF_GET }
488
489 let r: *i64 = (sys_mmap(128)) as *i64
490 if nx_http_response_parse(buf, gc, r) != 0 { sys_munmap(buf as *u8, icap); return 0 - NX_FF_PARSE }
491 out_status[0] = r[1]
492 let body_off: i64 = r[6]
493 if r[10] == 1 {
494 let dn: i64 = nx_http_dechunk(buf + body_off, gc - body_off, out, out_cap)
495 sys_munmap(buf as *u8, icap)
496 return dn
497 }
498 var blen: i64 = gc - body_off
499 if blen < 0 { blen = 0 }
500 if blen > out_cap { blen = out_cap }
501 var c: i64 = 0
502 while c < blen { out[c] = buf[body_off + c]; c = c + 1 }
503 sys_munmap(buf as *u8, icap)
504 return blen
505}
506
507// TLS 1.2 GET: reach TLS-1.2-ONLY hosts that reject our TLS-1.3 ClientHello (ukdevilz-class -- the server
508// sends NO ServerHello to a 1.3 hello). Completes ECDHE-RSA AND ECDHE-ECDSA (0xC02F/0xC02B) over P-256 (the
509// hello forces secp256r1 groups). Follows redirects (bounded) so a 1.2-only host that 301s reaches content.
510// Reuses the shipped, gated nx_tls12 client + the same connect/URL/response-parse/redirect pieces as ff_core.
511// SURFACE THE 1.2 SUB-VERDICT (debt 1787075987, 2026-08-18). t12_request returns a NAMED failure code
512// (-1 url, -2 connect, -3 incomplete SH/Cert/SKE flight, -4 ecdh, -5 write, -6 no-cert, -7 trust-store,
513// -(200+cv) cert-pipeline verdict, -(400+sv) SKE-signature verdict) and this leg collapsed ALL of them
514// into NX_FF_HANDSHAKE(-3), so the failing STEP was invisible to every caller -- 11 refs-lane mirrors
515// were declared absent against an undiagnosable -3. This prints the sub-verdict line-anchored on the
516// failure path only; the RETURN CONTRACT IS UNCHANGED (Cardinal 19: callers still see -NX_FF_HANDSHAKE).
517func ff_t12_diag(rc: i64) -> i64 {
518 let w0: *u8 = "nishi-t12 rc=" as *u8
519 var n0: i64 = 0
520 while w0[n0] != (0 as u8) { n0 = n0 + 1 }
521 sys_write(1, w0, n0)
522 var m: i64 = 0 - rc
523 if m == 0 { sys_write(1, "0" as *u8, 1) } else {
524 sys_write(1, "-" as *u8, 1)
525 let d: *u8 = sys_mmap(24)
526 var k: i64 = 0
527 while m > 0 { d[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
528 let o: *u8 = sys_mmap(24)
529 var wi: i64 = 0
530 while wi < k { o[wi] = d[k - 1 - wi]; wi = wi + 1 }
531 sys_write(1, o, k)
532 }
533 var name: *u8 = " t12-unknown\n" as *u8
534 let a: i64 = 0 - rc
535 if a == 1 { name = " t12-bad-url\n" as *u8 }
536 if a == 2 { name = " t12-connect\n" as *u8 }
537 if a == 3 { name = " t12-incomplete-flight (no SH/SKE inside the read budget)\n" as *u8 }
538 if a == 4 { name = " t12-ecdh\n" as *u8 }
539 if a == 5 { name = " t12-write\n" as *u8 }
540 if a == 6 { name = " t12-no-certificate\n" as *u8 }
541 if a == 7 { name = " t12-trust-store-load\n" as *u8 }
542 if a == TR_TRUNCATED { name = " t12-TRUNCATED: the peer declared a Content-Length this read did not reach -- the T12-TRUNCATED line carries both numbers\n" as *u8 }
543 if a == TR_RECORD_OVERSIZE { name = " t12-record-oversize: one TLS record exceeds the receive window\n" as *u8 }
544 if a == TR_ALERTED { name = " t12-alert: the peer answered the 1.2 ClientHello with a TLS alert (T12-ALERT desc= on stderr) -- a 1.3-only host refusing 1.2, not a slow flight\n" as *u8 }
545 if a >= 200 { if a < 400 { name = " t12-cert-pipeline (verdict = -(rc+200))\n" as *u8 } }
546 if a >= 400 { name = " t12-ske-signature (verdict = -(rc+400))\n" as *u8 }
547 sys_write(1, name, tr_slen(name))
548 return 0
549}
550func nx_https_fetch_follow_12(url0: *u8, store: *TrustStore, out: *u8, out_cap: i64, out_status: *i64) -> i64 {
551 out_status[0] = 0
552 let urlbuf: *u8 = sys_mmap(NX_MAGIC_4096)
553 var ui: i64 = 0
554 while url0[ui] != 0 as u8 { urlbuf[ui] = url0[ui]; ui = ui + 1 }
555 urlbuf[ui] = 0 as u8
556 var hop: i64 = 0
557 while hop <= 6 {
558 let target_raw: *u8 = sys_mmap(64)
559 let target: *NxHttpsTarget = target_raw as *NxHttpsTarget
560 target.url = nx_url_new()
561 target.port = 0
562 if nx_https_url_for_fetch(urlbuf, target) != NX_HTTPS_URL_OK { return 0 - NX_FF_BAD_URL }
563 // REACH FIX 2026-07-31 (debt 1785519121): this leg used nx_tls12_client_session_run, which
564 // does NOT complete a handshake with hosts that nx_tls12_req.t12_request reaches. PROVEN by
565 // nx_reach12_test, running BOTH impls in ONE process on FRESH connections with no 1.3 attempt
566 // before either: usgs.gov -> t12_request bytes=1684, client_session bytes=-3 HANDSHAKE. The
567 // ladder was wired to the broken one of two TLS-1.2 clients, so every -3 row stayed
568 // unreachable even after the chokepoint fix.
569 // NOT a security downgrade: t12_request is FAIL-CLOSED -- chain-to-Mozilla-roots + hostname +
570 // validity via nx_https_cert_pipeline_verify_with_store, plus an SKE-signature check binding
571 // the ephemeral key to the validated leaf. It loads its own trust store, so `store` is now
572 // unused here; the parameter is KEPT so the signature stays stable (Cardinal 19).
573 let icap: i64 = out_cap + out_cap/8 + NX_MAGIC_65536
574 let buf: *u8 = sys_mmap(icap)
575 let acc: i64 = t12_request(urlbuf, "GET" as *u8, 3, "" as *u8, 0, "" as *u8, 0, buf, icap)
576 // A BUCKET NAMED FOR HOW THE READER FAILED MERGES A REAL FAILURE WITH A DIFFERENT ONE: every negative
577 // t12 code used to collapse into NX_FF_HANDSHAKE, so a TRUNCATED body and a dead handshake arrived at
578 // the caller as the same word and the caller retried a deterministic cut four times.
579 if acc <= 0 {
580 ff_t12_diag(acc)
581 sys_munmap(buf as *u8, icap)
582 if acc == 0 - TR_TRUNCATED { return 0 - NX_FF_TRUNCATED }
583 return 0 - NX_FF_HANDSHAKE
584 }
585 let r: *i64 = (sys_mmap(128)) as *i64
586 if nx_http_response_parse(buf, acc, r) != 0 { sys_munmap(buf as *u8, icap); return 0 - NX_FF_PARSE }
587 let status: i64 = r[1]
588 out_status[0] = status
589 if nx_redir_is_redirect(status) == 1 {
590 let loc: *u8 = sys_mmap(NX_MAGIC_4096)
591 let ln: i64 = nx_redir_location(buf, acc, loc, NX_MAGIC_4096)
592 if ln <= 0 { sys_munmap(buf as *u8, icap); return 0 - NX_FF_NO_LOCATION }
593 let resolved: *u8 = sys_mmap(NX_MAGIC_4096)
594 ff_resolve_location(loc, urlbuf, target, resolved)
595 var k: i64 = 0
596 while resolved[k] != 0 as u8 { urlbuf[k] = resolved[k]; k = k + 1 }
597 urlbuf[k] = 0 as u8
598 sys_munmap(buf as *u8, icap)
599 hop = hop + 1
600 } else {
601 let body_off: i64 = r[6]
602 if r[10] == 1 {
603 let dn: i64 = nx_http_dechunk(buf + body_off, acc - body_off, out, out_cap)
604 sys_munmap(buf as *u8, icap)
605 if dn < 0 { return dn }
606 return ff_gunzip(out, dn, out_cap)
607 }
608 var blen: i64 = acc - body_off
609 if blen < 0 { blen = 0 }
610 if blen > out_cap { blen = out_cap }
611 var c: i64 = 0
612 while c < blen { out[c] = buf[body_off + c]; c = c + 1 }
613 sys_munmap(buf as *u8, icap)
614 return ff_gunzip(out, blen, out_cap)
615 }
616 }
617 return 0 - NX_FF_TOO_MANY
618}
619
620// BEST-EFFORT fetch: minimal fleet hello (fast) -> Chrome-JA3 hello (anti-bot CDNs) -> TLS 1.2 (1.2-only
621// hosts that reject a 1.3 hello, ukdevilz-class). Widest reach; extra handshakes only when earlier tries fail.
622// LEG VERDICT -- THE ONE PLACE THE LADDER DECIDES WHAT A TLS-1.3 LEG'S ANSWER MEANS (2026-09-17). Composed by
623// BOTH _best twins below, because the rule used to be hand-written in each and both carried the same defect:
624// only a 2xx counted as an answer, so a COMPLETE 404 fell through to the TLS-1.2 leg, a 1.3-only host refused
625// that leg, the leg zeroed out_status on entry, and the 404 body was handed back under status=0 -- which the
626// mirror lane reads as "no HTTP answer, transient" and retries (about.marginalia-search.com, twice, 2026-09-17).
627// The plain hello (leg 0) is NOT the last word on a non-2xx: a CDN fingerprint wall answers it with a complete
628// 403 and answers the Chrome-JA3 hello (leg 1) with 200, so only a complete 2xx settles leg 0. The Chrome hello
629// is the last 1.3 arm: a complete answer of ANY status class ends the ladder there, because a different
630// handshake cannot change what the origin said. Retry policy for 5xx belongs to the caller, never to a
631// TLS-version ladder.
632const FF_LEG_PLAIN: i64 = 0 // ff_core mode 0, the plain 1.3 hello
633const FF_LEG_CHROME: i64 = 1 // ff_core mode 1, the Chrome-JA3 hello -- the last 1.3 arm
634const FF_LEG_NEXT: i64 = 0 // no HTTP answer parsed (n <= 0 or status <= 0), or a non-2xx from the plain hello: next leg
635const FF_LEG_DONE: i64 = 1 // a complete HTTP answer this leg is allowed to settle: the ladder is finished
636const FF_LEG_KEEP: i64 = 2 // an answer whose body fell short of its declared length: bank it, try the next leg
637func ff_leg_verdict(leg: i64, n: i64, status: i64, body_state: i64) -> i64 {
638 if n <= 0 { return FF_LEG_NEXT }
639 if status <= 0 { return FF_LEG_NEXT }
640 if leg == FF_LEG_PLAIN { if status < 200 { return FF_LEG_NEXT } if status >= 300 { return FF_LEG_NEXT } }
641 if body_state == NX_HTTPS_BODY_TRUNCATED { return FF_LEG_KEEP }
642 return FF_LEG_DONE
643}
644func nx_https_fetch_follow_best(url0: *u8, store: *TrustStore, out: *u8, out_cap: i64,
645 max_hops: i64, out_status: *i64) -> i64 {
646 // SAME LAW AS _hdr_best BELOW, AND IT HAD TO BE WRITTEN IN BOTH: this is the entry point
647 // nx_research_fetch calls, so fixing only the header-capable twin would have shipped a fix the
648 // mirror lane never reaches. A FIX THAT LIVES IN ONE SIBLING AND NOT THE OTHER IS HALF A FIX, AND
649 // THE MISSING HALF IS INVISIBLE UNTIL SOMETHING RUNS IT.
650 var keep: *u8 = 0 as *u8
651 var keep_n: i64 = 0
652 var keep_st: i64 = 0
653 // ONE DECISION, TWO CALLERS: ff_leg_verdict (above) says what each leg's answer means.
654 let n: i64 = ff_core(url0, store, out, out_cap, max_hops, out_status, 0)
655 let st1: i64 = out_status[0]
656 var v: i64 = ff_leg_verdict(FF_LEG_PLAIN, n, st1, nx_https_last_body_state())
657 if v == FF_LEG_DONE { return n }
658 if v == FF_LEG_KEEP { keep = ff_keep(out, n); keep_n = n; keep_st = st1 }
659 let n2: i64 = ff_core(url0, store, out, out_cap, max_hops, out_status, 1)
660 let st2: i64 = out_status[0]
661 v = ff_leg_verdict(FF_LEG_CHROME, n2, st2, nx_https_last_body_state())
662 if v == FF_LEG_DONE { return n2 }
663 if v == FF_LEG_KEEP { if n2 > keep_n { keep = ff_keep(out, n2); keep_n = n2; keep_st = st2 } }
664 // no 1.3 leg produced a complete HTTP answer (or every answer fell short of its declared length) -> the 1.2 leg
665 let n3: i64 = nx_https_fetch_follow_12(url0, store, out, out_cap, out_status)
666 if n3 > 0 { _gc_body_seal(0, n3); return n3 }
667 // nothing did better -> hand back the short body unchanged rather than lose it, with its own status
668 if keep_n > 0 { out_status[0] = keep_st; return ff_restore(keep, out, keep_n) }
669 // nothing worked -> the status returned must belong to the BODY returned, never to the leg that died last:
670 // the 1.2 leg zeroes out_status on entry, and a 1.3 body handed back under that zero read as "no HTTP
671 // answer" to every caller.
672 if n2 > 0 { out_status[0] = st2; return n2 }
673 out_status[0] = st1
674 return n
675}
676
677// REACH FIX 2026-07-31 (debt 1785516905, ws=library-datasets). MEASURED: 198 call sites used the
678// narrow 1.3-minimal-hello entry point vs 43 on _best, so 82pct of the fleet could not reach
679// TLS-1.2-only hosts -- which is exactly where US federal open data lives (usgs/usda/nsf/noaa/
680// eia/bls). PROOF: nx_https_get3 on waterservices.usgs.gov died verdict=5 at recvSH while
681// nx_tls12_probe on the SAME URL returned chain=VALID + HTTP 200 + 1307B. NEG-CONTROL: wikipedia
682// via the 1.3 path returned 163307B, so the 1.3 client was never broken -- a whole HOST CLASS was
683// simply unreachable, and that is why the research corpus was 44 Wikipedia pages.
684//
685// Delegating the STABLE NAME to _best is ADDITIVE, not a Cardinal-19 break: _best runs the exact
686// same minimal hello FIRST and returns its result untouched whenever that already yields 2xx, so
687// every currently-working fetch is byte-identical. The chrome hello and the TLS-1.2 ladder run
688// ONLY after the minimal hello has already failed -- i.e. only in cases that returned nothing
689// before. Callers needing byte-exact narrow behavior call nx_https_fetch_follow_13only above.
690// NOTE: this lib is compiled INTO each organ, so a caller inherits the reach only when REBUILT.
691func nx_https_fetch_follow(url0: *u8, store: *TrustStore, out: *u8, out_cap: i64,
692 max_hops: i64, out_status: *i64) -> i64 {
693 return nx_https_fetch_follow_best(url0, store, out, out_cap, max_hops, out_status)
694}
695
696// HEADER-CAPABLE best-effort fetch (the fold): the FULL ladder -- minimal TLS-1.3 hello -> Chrome-JA3 ->
697// TLS-1.2 -- redirect-following + cookie jar, PLUS caller-supplied request header(s) xhdr (CRLF-terminated
698// line(s), xhdr_len bytes; 0 = none) injected into every 1.3 request. This is what lets an authed/Cloudflare
699// API be reached in ONE call: Authorization: Bearer <token> (redgifs/reddit-OAuth), a pre-seeded
700// Cookie: over18=1, a Referer, etc. Same return contract as _best (body_bytes>0 | -code; out_status[0]=status).
701// With xhdr_len=0 this is byte-identical to nx_https_fetch_follow_best.
702// BOUNDED LIMITATION (fail-safe, NAMED): the final TLS-1.2 fallback leg carries NO caller header -- t12_request
703// uses a different header convention and is a shared API for unrelated callers (nx_gpu_ctl/Porkbun); every
704// header use case is a modern TLS-1.3/Cloudflare host reached by the 1.3 legs, and a 1.2-only host that needed
705// a header simply does not get it (no security downgrade, no crash). Widen t12_request if that ever changes.
706// A TRUNCATED 200 IS NOT A SUCCESS, AND IT IS ALSO NOT A FAILURE -- IT IS A REASON TO TRY THE NEXT
707// RUNG. The ladder below used to stop at the FIRST 2xx, so a TLS-1.3 leg that returned a body the peer
708// had cut short SHORT-CIRCUITED PAST the TLS-1.2 leg, which has checked declared-vs-received length
709// since 2026-08-18 (TR_TRUNCATED). The short body was then saved as a clean mirror.
710//
711// THIS INTRODUCES NO REFUSAL, WHICH IS THE WHOLE POINT. If no later rung does better, the short body
712// is RESTORED BYTE-FOR-BYTE and returned exactly as before, with its own status -- so every call that
713// works today still works and still returns the same bytes. What is removed is the short-circuit, not
714// the result. The keep buffer is sized to the bytes ACTUALLY RECEIVED, never to out_cap, so a 128 MB
715// reserve does not become a 128 MB copy.
716func ff_keep(out: *u8, n: i64) -> *u8 {
717 if n <= 0 { return 0 as *u8 }
718 let k: *u8 = sys_mmap(n + 16)
719 var i: i64 = 0
720 while i < n { k[i] = out[i]; i = i + 1 }
721 return k
722}
723func ff_restore(keep: *u8, out: *u8, n: i64) -> i64 {
724 var i: i64 = 0
725 while i < n { out[i] = keep[i]; i = i + 1 }
726 return n
727}
728func nx_https_fetch_follow_hdr_best(url0: *u8, store: *TrustStore, out: *u8, out_cap: i64,
729 max_hops: i64, out_status: *i64, xhdr: *u8, xhdr_len: i64) -> i64 {
730 var keep: *u8 = 0 as *u8
731 var keep_n: i64 = 0
732 var keep_st: i64 = 0
733 // ONE DECISION, TWO CALLERS: ff_leg_verdict (above) says what each leg's answer means -- the same law the
734 // plain twin composes, so the two can no longer drift apart.
735 let n: i64 = ff_core_x(url0, store, out, out_cap, max_hops, out_status, 0, xhdr, xhdr_len)
736 let st1: i64 = out_status[0]
737 var v: i64 = ff_leg_verdict(FF_LEG_PLAIN, n, st1, nx_https_last_body_state())
738 if v == FF_LEG_DONE { return n }
739 if v == FF_LEG_KEEP { keep = ff_keep(out, n); keep_n = n; keep_st = st1 }
740 let n2: i64 = ff_core_x(url0, store, out, out_cap, max_hops, out_status, 1, xhdr, xhdr_len)
741 let st2: i64 = out_status[0]
742 v = ff_leg_verdict(FF_LEG_CHROME, n2, st2, nx_https_last_body_state())
743 if v == FF_LEG_DONE { return n2 }
744 if v == FF_LEG_KEEP { if n2 > keep_n { keep = ff_keep(out, n2); keep_n = n2; keep_st = st2 } }
745 // no 1.3 leg produced a complete HTTP answer (or every answer fell short of its declared length) -> the 1.2 leg
746 // (caller header NOT carried on this leg -- see the note above)
747 let n3: i64 = nx_https_fetch_follow_12(url0, store, out, out_cap, out_status)
748 // THE 1.2 LEG DELIVERED, SO THE 1.3 AXIS MUST STOP TALKING ABOUT THIS BODY. Leaving the static
749 // holding rung-1's TRUNCATED verdict would report a 1.2 body as cut -- a stale reading from an
750 // instrument whose subject changed underneath it. body_target=0 seals it UNJUDGEABLE-BY-THIS-AXIS,
751 // which is exactly true: the 1.2 leg carries its own TR_TRUNCATED check.
752 if n3 > 0 { _gc_body_seal(0, n3); return n3 }
753 // NOTHING DID BETTER: hand back the short body unchanged rather than lose it. Ordered ahead of the
754 // bare n2/n fallbacks because those two return a LENGTH whose BYTES a later rung has since
755 // overwritten -- a pre-existing hazard this path deliberately does not inherit.
756 if keep_n > 0 { out_status[0] = keep_st; return ff_restore(keep, out, keep_n) }
757 // the status returned must belong to the BODY returned, never to the leg that died last
758 if n2 > 0 { out_status[0] = st2; return n2 }
759 out_status[0] = st1
760 return n
761}