nx_https_fetch_follow.nx source
↩ module page · 555 lines · 30365 B
1// nx_https_fetch_follow.nx -- Fetcher upgrade F-2: LIVE redirect-following fetch.
2// Composes the proven sovereign TLS-1.3 fetch sequence (url_for_fetch -> connect
3// -> client_session_run -> get_complete -> response_parse) with F-1's redirect
4// logic (nx_https_redirect) in a bounded loop: on a 3xx with Location, re-fetch
5// the new URL up to max_hops. 100% sovereign (nx_cc->nxasm, own TLS); the only
6// non-Nishi inputs are the CA-root DATA + the fetched pages.
7//
8// API: nx_https_fetch_follow(url, store, out, out_cap, max_hops, out_status)
9// -> POSITIVE body_bytes (final non-redirect response, dechunked into out)
10// | NEGATIVE -code on failure. out_status[0] = final HTTP status.
11//
12// nx_safety_envelope:
13// intended_use: sovereign web fetcher (live network)
14// sil_target: SIL1
15// verdict: NOT_YET_EVALUATED
16// genealogy_id: ietf/rfc_9110+9112 lineage_id: nx_https_fetch_follow_v1
17
18import "nx_syscalls.nx"
19import "nx_x509_trust_store.nx"
20import "nx_tls13_client_validate_certificate.nx"
21import "nx_tls13_client_session_run.nx"
22import "nx_https_url_for_fetch.nx"
23import "nx_https_url_connect.nx"
24import "nx_https_get_complete.nx"
25import "nx_tls13_chrome_session.nx" // run_chrome = the Chrome-JA3 hello path (beats CDN fingerprinting + RSA-PSS CV)
26import "nx_http_response_parse.nx"
27import "nx_https_redirect.nx"
28import "nx_http_range_req.nx" // Range: bytes=S-E builder -> the Common Crawl targeted-record fetch
29import "nx_tls12_req.nx" // t12_request = the TLS-1.2 client that ACTUALLY reaches 1.2-only hosts
30import "nx_tls12_client_session.nx" // TLS 1.2 session (run/send/recv) -> reach TLS-1.2-only hosts (ukdevilz-class)
31import "nx_csprng.nx" // nx_csprng_fill -> fresh handshake entropy for the 1.2 client
32import "nx_gzip_wrap.nx" // nx_gzip_inflate -- we ADVERTISE gzip, so the chokepoint must decode it
33// Content-Encoding: gzip -- inflate AT THE CHOKEPOINT so every caller of the shared fetch gets TEXT.
34// nx_codec_caps DERIVES Accept-Encoding from the decoders that exist and advertises gzip, so servers
35// legitimately compress -- and returning those bytes to a caller that asked for a page hands it garbage
36// that LOOKS like a successful fetch (positive length, status 200), which is the worst possible failure
37// shape for a crawler: it INDEXES the garbage instead of erroring.
38// MEASURED 2026-08-06 on https://www.v2ph.com/actor/JULIA?page=3&hl=en -- SAME url, same second:
39// 1.3 leg -> 200, 6,079 bytes beginning 1f 8b (gzip, undecoded)
40// 1.2 leg -> 200, 30,385 bytes of <!DOCTYPE html> (it sends Accept-Encoding: identity)
41// nx_https_get_cli2 imported this decoder long ago; the SHARED path never did, so ~200 call sites and
42// the web crawler have been storing compressed bytes as page text.
43// ★A DECODER ONE CONSUMER IMPORTS IS NOT A CAPABILITY THE FLEET HAS.
44const NX_FF_GZ_MAX_OUT: i64 = 33554432 // declared: 32MB inflate ceiling, so a zip-bomb cannot take the host
45func ff_gunzip(out: *u8, n: i64, out_cap: i64) -> i64 {
46 if n < 2 { return n }
47 if (out[0] as i64) != 0x1f { return n }
48 if (out[1] as i64) != 0x8b { return n }
49 let gr: *NxGzipResult = nx_gzip_inflate(out, n, NX_FF_GZ_MAX_OUT)
50 // FAIL-SOFT, NEVER FAIL-SILENT-ZERO: an undecodable body is handed back exactly as received. Returning
51 // 0 here would turn "we could not inflate" into "the page was empty", which is the lie that cost this
52 // lane two sessions on the 1.2 leg (debt 1785971025).
53 if gr.error_code != 0 { return n }
54 let gsz: i64 = gr.output_size
55 if gsz <= 0 { return n }
56 var m: i64 = gsz
57 if m > out_cap { m = out_cap }
58 let src: *u8 = gr.output_data
59 var i: i64 = 0
60 while i < m { out[i] = src[i]; i = i + 1 }
61 return m
62}
63const NX_MAGIC_4096: i64 = 4096
64const NX_MAGIC_2048: i64 = 2048
65const NX_MAGIC_65536: i64 = 65536
66const NX_MAGIC_20000: i64 = 20000
67const NX_MAGIC_8192: i64 = 8192
68
69const NX_FF_BAD_URL: i64 = 1
70const NX_FF_CONNECT: i64 = 2
71const NX_FF_HANDSHAKE: i64 = 3
72const NX_FF_GET: i64 = 4
73const NX_FF_PARSE: i64 = 5
74const NX_FF_NO_LOCATION: i64 = 6
75const NX_FF_TOO_MANY: i64 = 7
76// Per-attempt recv+send timeout (seconds) on the fetch socket. Without it, an anti-bot CDN (Cloudflare) that ACCEPTS
77// the TCP connection but STALLS a non-browser TLS hello hangs the blocking read forever -> the whole fetch never
78// returns, and the escalating best-effort path (minimal -> Chrome-JA3 -> TLS 1.2) never reaches the hello that works.
79// 8s is generous for a healthy handshake+TTFB (<2s typical) yet fails a true stall fast enough for the next hello to
80// run inside the edge's proxy window. Read timeout surfaces as sys_read<0 -> READ_ERR -> the read loop returns (no
81// busy-loop; verified in nx_tls13_read_record_from_fd._read_n).
82const NX_FF_FETCH_TIMEOUT_SECS: i64 = 8
83
84// Extract the request path from the URL string into path_out (NUL-terminated).
85// Path starts after host[:port]; defaults to "/" when absent. Returns length.
86func ff_path(urlbuf: *u8, target: *NxHttpsTarget, path_out: *u8) -> i64 {
87 var p: i64 = target.url.host_off + target.url.host_len
88 if urlbuf[p] == 0x3A as u8 { // ':' port present -> skip :digits
89 p = p + 1
90 while urlbuf[p] >= 0x30 as u8 { if urlbuf[p] <= 0x39 as u8 { p = p + 1 } else { break } }
91 }
92 if urlbuf[p] != 0x2F as u8 { // no '/' -> path is "/"
93 path_out[0] = 0x2F as u8
94 path_out[1] = 0 as u8
95 return 1
96 }
97 var k: i64 = 0
98 while urlbuf[p] != 0 as u8 { path_out[k] = urlbuf[p]; p = p + 1; k = k + 1 }
99 path_out[k] = 0 as u8
100 return k
101}
102
103// ---- RFC-3986 relative-reference resolution for redirect Location ----
104// The Location header MAY be a relative reference (RFC 9110 §10.2.2): absolute
105// (https://h/p), protocol-relative (//h/p), path-absolute (/p), or bare (p).
106// The original follow loop copied Location verbatim + re-parsed as absolute, so
107// any non-absolute Location failed nx_url_parse (NO_SCHEME) -> -NX_FF_BAD_URL.
108// These helpers resolve it against the current request URI (scheme is always
109// https on this fetcher; host/port carried from the current hop).
110
111func ff_apps(dst: *u8, k0: i64, src: *u8) -> i64 { // append NUL-term src
112 var k: i64 = k0; var j: i64 = 0
113 while src[j] != 0 as u8 { dst[k] = src[j]; k = k + 1; j = j + 1 }
114 return k
115}
116func ff_appn(dst: *u8, k0: i64, src: *u8, n: i64) -> i64 { // append n bytes
117 var k: i64 = k0; var j: i64 = 0
118 while j < n { dst[k] = src[j]; k = k + 1; j = j + 1 }
119 return k
120}
121func ff_appi(dst: *u8, k0: i64, v: i64) -> i64 { // append decimal int
122 var k: i64 = k0
123 if v == 0 { dst[k] = 0x30 as u8; return k + 1 }
124 let tmp: *u8 = sys_mmap(24); var m: i64 = v; var t: i64 = 0
125 while m > 0 { tmp[t] = (0x30 + (m % 10)) as u8; m = m / 10; t = t + 1 }
126 var i: i64 = t - 1
127 while i >= 0 { dst[k] = tmp[i]; k = k + 1; i = i - 1 }
128 return k
129}
130func ff_starts(s: *u8, p: *u8) -> i64 { // s starts with prefix p?
131 var j: i64 = 0
132 while p[j] != 0 as u8 { if s[j] != p[j] { return 0 } j = j + 1 }
133 return 1
134}
135// Resolve `loc` against current URI (host/port from `cur` over `cur_urlbuf`) into
136// outp (NUL-terminated absolute https URL). http:// is UPGRADED to https:// -- we
137// never follow a redirect into plaintext (never-relax-security guardrail).
138func ff_resolve_location(loc: *u8, cur_urlbuf: *u8, cur: *NxHttpsTarget, outp: *u8) -> i64 {
139 var k: i64 = 0
140 if ff_starts(loc, "https://" as *u8) == 1 { // already absolute https
141 k = ff_apps(outp, k, loc); outp[k] = 0 as u8; return k
142 }
143 if ff_starts(loc, "http://" as *u8) == 1 { // secure upgrade http->https
144 k = ff_apps(outp, k, "https://" as *u8)
145 k = ff_apps(outp, k, loc + 7) // skip "http://"
146 outp[k] = 0 as u8; return k
147 }
148 if ff_starts(loc, "//" as *u8) == 1 { // protocol-relative
149 k = ff_apps(outp, k, "https:" as *u8)
150 k = ff_apps(outp, k, loc); outp[k] = 0 as u8; return k
151 }
152 // path-absolute (/p) or bare (p): keep the current authority.
153 k = ff_apps(outp, k, "https://" as *u8)
154 k = ff_appn(outp, k, cur_urlbuf + cur.url.host_off, cur.url.host_len)
155 if cur.url.port != 0 { outp[k] = 0x3A as u8; k = k + 1; k = ff_appi(outp, k, cur.url.port) }
156 if loc[0] == 0x2F as u8 { k = ff_apps(outp, k, loc) }
157 else { outp[k] = 0x2F as u8; k = k + 1; k = ff_apps(outp, k, loc) }
158 outp[k] = 0 as u8; return k
159}
160
161// ---- Cookie jar for the redirect loop (browser-faithful session carry) ----
162// Real browsers keep a cookie jar across a redirect chain; our fetcher dropped cookies between
163// hops, so cookie-gated sites (chaturbate's /?next= age-gate hands back csrftoken/sbr/AG_Key)
164// could never be reached. These helpers collect Set-Cookie into a flat "n=v; n=v" jar (dedup by
165// name = last-writer-wins, RFC-6265 jar semantics) and detect the "gate-and-return" ?next= idiom.
166const NX_FF_JAR_BYTES: i64 = 3072
167
168// Case-insensitive match of a lowercase pattern at buf[pos..]. Returns 1 on match, 0 otherwise.
169func ff_hdr_match(buf: *u8, pos: i64, n: i64, pat: *u8, patlen: i64) -> i64 {
170 if pos + patlen > n { return 0 }
171 var k: i64 = 0
172 while k < patlen {
173 var c: i64 = buf[pos+k] as i64
174 if c >= 65 { if c <= 90 { c = c + 32 } } // ASCII upper -> lower
175 if c != (pat[k] as i64) { return 0 }
176 k = k + 1
177 }
178 return 1
179}
180
181// Set cookie `name`=`val` in the jar (jl = current length), replacing any prior entry for the
182// same name (delete-then-append) so the freshest value wins. Fail-safe: skips if it won't fit.
183func ff_jar_set(jar: *u8, jl: *i64, cap: i64, name: *u8, name_len: i64, val: *u8, val_len: i64) -> i64 {
184 var i: i64 = 0
185 let cur: i64 = jl[0]
186 while i < cur {
187 var m: i64 = 1
188 var k: i64 = 0
189 while k < name_len { if jar[i+k] != name[k] { m = 0; k = name_len } else { k = k + 1 } }
190 if m == 1 {
191 if jar[i+name_len] == (61 as u8) { // matched "name="
192 var e: i64 = i + name_len
193 while e < cur { if jar[e] == (59 as u8) { break } e = e + 1 } // to ';'
194 if e < cur { if jar[e] == (59 as u8) { e = e + 1; if e < cur { if jar[e] == (32 as u8) { e = e + 1 } } } }
195 var s: i64 = 0
196 while e + s < cur { jar[i+s] = jar[e+s]; s = s + 1 }
197 jl[0] = cur - (e - i)
198 i = cur
199 } else { i = i + 1 }
200 } else {
201 while i < cur { if jar[i] == (59 as u8) { break } i = i + 1 }
202 if i < cur { i = i + 1; if i < cur { if jar[i] == (32 as u8) { i = i + 1 } } }
203 }
204 }
205 var o: i64 = jl[0]
206 if o + name_len + val_len + 4 > cap { return 0 } // no room -> skip (fail-safe)
207 if o > 0 { jar[o]=59;o=o+1; jar[o]=32;o=o+1 } // "; "
208 var a: i64 = 0
209 while a < name_len { jar[o]=name[a]; o=o+1; a=a+1 }
210 jar[o]=61;o=o+1 // '='
211 var b: i64 = 0
212 while b < val_len { jar[o]=val[b]; o=o+1; b=b+1 }
213 jl[0] = o
214 return 1
215}
216
217// Scan a raw HTTP response's headers for Set-Cookie lines; jar-set each name=value (value up to
218// the first ';'). Cookies live only in headers, so scanning the whole buffer is safe+simple.
219func ff_collect_cookies(buf: *u8, n: i64, jar: *u8, jl: *i64, cap: i64) -> i64 {
220 let pat: *u8 = "set-cookie:" as *u8
221 var i: i64 = 0
222 while i < n {
223 if ff_hdr_match(buf, i, n, pat, 11) == 1 {
224 var p: i64 = i + 11
225 while p < n { if buf[p] != (32 as u8) { break } p = p + 1 } // skip SP
226 let nstart: i64 = p
227 while p < n { if buf[p] == (61 as u8) { break } if buf[p] == (59 as u8) { break } if buf[p] == (13 as u8) { break } p = p + 1 }
228 if p < n { if buf[p] == (61 as u8) {
229 let name_len: i64 = p - nstart
230 p = p + 1
231 let vstart: i64 = p
232 while p < n { if buf[p] == (59 as u8) { break } if buf[p] == (13 as u8) { break } if buf[p] == (10 as u8) { break } p = p + 1 }
233 let val_len: i64 = p - vstart
234 if name_len > 0 { if name_len < 128 { ff_jar_set(jar, jl, cap, buf + nstart, name_len, buf + vstart, val_len) } }
235 } }
236 }
237 while i < n { if buf[i] == (10 as u8) { break } i = i + 1 } // to end of line
238 i = i + 1
239 }
240 return 0
241}
242
243// True if the URL carries a "?next="/"&next=" gate-and-return param (chaturbate-class age gate):
244// we visit it to collect the session, then re-request the original target with the jar.
245func ff_is_gate_url(url: *u8) -> i64 {
246 var i: i64 = 0
247 while url[i] != 0 as u8 {
248 if url[i] == (63 as u8) { if ff_hdr_match(url, i+1, i+7, "next=" as *u8, 5) == 1 { return 1 } }
249 if url[i] == (38 as u8) { if ff_hdr_match(url, i+1, i+7, "next=" as *u8, 5) == 1 { return 1 } }
250 i = i + 1
251 }
252 return 0
253}
254
255// ff_core: the redirect-following TLS-1.3 GET. use_chrome=0 -> the minimal fleet hello; use_chrome=1 -> the
256// Chrome-JA3 hello (run_chrome) that passes anti-bot CDN fingerprinting + verifies RSA-PSS CertificateVerify.
257// Carries a cookie jar across hops (ff_collect_cookies/ff_jar_set) + one gate-bounce back to the original
258// URL after a ?next= age-gate, so cookie-gated rooms resolve. Public sigs unchanged (Cardinal 19).
259func ff_core(url0: *u8, store: *TrustStore, out: *u8, out_cap: i64,
260 max_hops: i64, out_status: *i64, use_chrome: i64) -> i64 {
261 out_status[0] = 0
262 let urlbuf: *u8 = sys_mmap(NX_MAGIC_4096)
263 var ui: i64 = 0
264 while url0[ui] != 0 as u8 { urlbuf[ui] = url0[ui]; ui = ui + 1 }
265 urlbuf[ui] = 0 as u8
266
267 // cookie jar (carried across hops) + original-target snapshot for the one gate-bounce
268 let jar: *u8 = sys_mmap(NX_FF_JAR_BYTES)
269 let jl: *i64 = (sys_mmap(8)) as *i64
270 jl[0] = 0
271 let orig: *u8 = sys_mmap(NX_MAGIC_4096)
272 var oi: i64 = 0
273 while url0[oi] != 0 as u8 { orig[oi] = url0[oi]; oi = oi + 1 }
274 orig[oi] = 0 as u8
275 var bounced: i64 = 0
276
277 let cr: *u8 = sys_mmap(32)
278 let priv: *u8 = sys_mmap(32)
279 var hop: i64 = 0
280 while hop <= max_hops {
281 let target_raw: *u8 = sys_mmap(64)
282 let target: *NxHttpsTarget = target_raw as *NxHttpsTarget
283 target.url = nx_url_new()
284 target.port = 0
285 if nx_https_url_for_fetch(urlbuf, target) != NX_HTTPS_URL_OK { return 0 - NX_FF_BAD_URL }
286
287 let fd_p: *i64 = (sys_mmap(16)) as *i64
288 if nx_https_url_connect(target, urlbuf, sys_now_realtime_sec(), fd_p) != NX_HTTPS_CONNECT_OK { return 0 - NX_FF_CONNECT }
289 let fd: i64 = fd_p[0]
290 sys_set_socket_timeout(fd, NX_FF_FETCH_TIMEOUT_SECS) // bound handshake+read: a stalling host fails fast so best-effort can escalate (never hangs forever)
291
292 // SEV-9 FIX 2026-08-05 (debt 1785970852, CWE-330): this loop used to hardcode the X25519
293 // scalar to 0xA0..0xBF and client_random to 0xC0..0xDF, so EVERY session the fleet opened
294 // shared ONE "ephemeral" private key derivable from any binary -- forward secrecy absent,
295 // not merely weak, and retroactively so for anything ever recorded. nx_csprng_fill was
296 // already imported in this very file for the TLS 1.2 leg: the primitive was never missing.
297 // ★A TEST CONSTANT THAT ESCAPES INTO PRODUCTION IS INDISTINGUISHABLE FROM A BACKDOOR.
298 nx_csprng_fill(cr, 32)
299 nx_csprng_fill(priv, 32)
300 let vc_raw: *u8 = sys_mmap(64)
301 let vc: *TlsValidationContext = vc_raw as *TlsValidationContext
302 vc.store = store
303 vc.sni_host = urlbuf + target.url.host_off
304 vc.sni_host_len = target.url.host_len
305 vc.now_epoch = sys_now_realtime_sec()
306 var sr: i64 = 0
307 if use_chrome == 1 { sr = nx_tls13_client_session_run_chrome(fd, urlbuf + target.url.host_off, target.url.host_len, cr, priv, vc) }
308 else { sr = nx_tls13_client_session_run(fd, urlbuf + target.url.host_off, target.url.host_len, cr, priv, vc) }
309 if sr <= 0 { sys_close(fd); return 0 - NX_FF_HANDSHAKE }
310 let session: *Tls13ClientSession = sr as *Tls13ClientSession
311
312 let path: *u8 = sys_mmap(NX_MAGIC_2048)
313 let plen: i64 = ff_path(urlbuf, target, path)
314 let icap: i64 = out_cap + out_cap/8 + NX_MAGIC_65536 // internal raw-response buffer sized to the caller's out_cap (grow for complete repos)
315 let buf: *u8 = sys_mmap(icap)
316 let gc: i64 = nx_https_get_complete_cookie(session, fd, path, plen, urlbuf + target.url.host_off, target.url.host_len, jar, jl[0], buf, icap)
317 sys_close(fd)
318 if gc < 0 { sys_munmap(buf as *u8, icap); return 0 - NX_FF_GET }
319
320 let r: *i64 = (sys_mmap(128)) as *i64
321 if nx_http_response_parse(buf, gc, r) != 0 { sys_munmap(buf as *u8, icap); return 0 - NX_FF_PARSE }
322 let status: i64 = r[1]
323 out_status[0] = status
324 ff_collect_cookies(buf, gc, jar, jl, NX_FF_JAR_BYTES) // carry Set-Cookie to the next hop
325
326 if nx_redir_is_redirect(status) == 1 {
327 let loc: *u8 = sys_mmap(NX_MAGIC_4096)
328 let ln: i64 = nx_redir_location(buf, gc, loc, NX_MAGIC_4096)
329 if ln <= 0 { sys_munmap(buf as *u8, icap); return 0 - NX_FF_NO_LOCATION }
330 // Resolve a possibly-relative Location against the current URI into a
331 // temp buffer (reading host from urlbuf), THEN copy back into urlbuf.
332 let resolved: *u8 = sys_mmap(NX_MAGIC_4096)
333 ff_resolve_location(loc, urlbuf, target, resolved)
334 var k: i64 = 0
335 while resolved[k] != 0 as u8 { urlbuf[k] = resolved[k]; k = k + 1 }
336 urlbuf[k] = 0 as u8
337 sys_munmap(buf as *u8, icap); sys_munmap(loc as *u8, NX_MAGIC_4096); sys_munmap(resolved as *u8, NX_MAGIC_4096)
338 hop = hop + 1
339 } else {
340 // Gate-bounce: if this hop landed on a ?next= age-gate (not the target) and we now hold a
341 // session, re-request the ORIGINAL url ONCE with the jar to reach the real content. do_bounce
342 // is per-iteration so the re-request's 200 still returns normally (no loop, bounded by max_hops).
343 var do_bounce: i64 = 0
344 if bounced == 0 { if jl[0] > 0 { if ff_is_gate_url(urlbuf) == 1 { do_bounce = 1 } } }
345 if do_bounce == 1 {
346 bounced = 1
347 var z: i64 = 0
348 while orig[z] != 0 as u8 { urlbuf[z] = orig[z]; z = z + 1 }
349 urlbuf[z] = 0 as u8
350 sys_munmap(buf as *u8, icap)
351 hop = hop + 1
352 } else {
353 let body_off: i64 = r[6]
354 if r[10] == 1 { // chunked -> dechunk
355 let dn: i64 = nx_http_dechunk(buf + body_off, gc - body_off, out, out_cap)
356 sys_munmap(buf as *u8, icap) // FREE the 8MB response buffer (was the dominant per-fetch leak)
357 if dn < 0 { return dn }
358 return ff_gunzip(out, dn, out_cap)
359 }
360 var blen: i64 = gc - body_off // identity -> copy received body
361 if blen < 0 { blen = 0 }
362 if blen > out_cap { blen = out_cap }
363 var c: i64 = 0
364 while c < blen { out[c] = buf[body_off + c]; c = c + 1 }
365 sys_munmap(buf as *u8, icap) // FREE the 8MB response buffer (was the dominant per-fetch leak)
366 return ff_gunzip(out, blen, out_cap)
367 }
368 }
369 }
370 return 0 - NX_FF_TOO_MANY
371}
372
373// Stable API (Cardinal 19): the minimal fleet hello, byte-for-byte behavior.
374// RENAMED 2026-07-31 (debt 1785516905) -- kept verbatim under an EXPLICIT name for any caller that
375// genuinely needs narrow 1.3-only behavior. The nx_https_fetch_follow NAME now delegates to _best;
376// see the tail of this file for why that is additive rather than a contract break.
377func nx_https_fetch_follow_13only(url0: *u8, store: *TrustStore, out: *u8, out_cap: i64,
378 max_hops: i64, out_status: *i64) -> i64 {
379 return ff_core(url0, store, out, out_cap, max_hops, out_status, 0)
380}
381
382// Chrome-JA3 fetch: for anti-bot-CDN targets (Cloudflare image hosts etc.). Same signature.
383func nx_https_fetch_follow_chrome(url0: *u8, store: *TrustStore, out: *u8, out_cap: i64,
384 max_hops: i64, out_status: *i64) -> i64 {
385 return ff_core(url0, store, out, out_cap, max_hops, out_status, 1)
386}
387// nx_https_fetch_range: fetch ONE byte-range of a URL (RFC 9110 Range) -> the Common Crawl targeted-record
388// path (CDX gives warc-file/offset/length; this pulls exactly that gzip member, no 344TiB download). NO
389// redirect loop (the CC data host serves ranges directly); reuses the proven connect+handshake +
390// nx_https_req_complete send/recv core. use_chrome=1 for anti-bot CDNs. out_status = 206 (partial) or 200.
391// Returns body_bytes (the raw range bytes -- a gzip member for CC) or -code.
392func nx_https_fetch_range(url0: *u8, store: *TrustStore, out: *u8, out_cap: i64,
393 rstart: i64, rend: i64, out_status: *i64, use_chrome: i64) -> i64 {
394 out_status[0] = 0
395 let urlbuf: *u8 = sys_mmap(NX_MAGIC_4096)
396 var ui: i64 = 0
397 while url0[ui] != 0 as u8 { urlbuf[ui] = url0[ui]; ui = ui + 1 }
398 urlbuf[ui] = 0 as u8
399
400 let target_raw: *u8 = sys_mmap(64)
401 let target: *NxHttpsTarget = target_raw as *NxHttpsTarget
402 target.url = nx_url_new()
403 target.port = 0
404 if nx_https_url_for_fetch(urlbuf, target) != NX_HTTPS_URL_OK { return 0 - NX_FF_BAD_URL }
405
406 let fd_p: *i64 = (sys_mmap(16)) as *i64
407 if nx_https_url_connect(target, urlbuf, sys_now_realtime_sec(), fd_p) != NX_HTTPS_CONNECT_OK { return 0 - NX_FF_CONNECT }
408 let fd: i64 = fd_p[0]
409
410 let cr: *u8 = sys_mmap(32)
411 let priv: *u8 = sys_mmap(32)
412 // SEV-9 FIX 2026-08-05 (debt 1785970852, CWE-330) -- see the note at the sibling site above.
413 // Fresh per-connection entropy is what makes the key ephemeral; a constant makes it a shared secret.
414 nx_csprng_fill(cr, 32)
415 nx_csprng_fill(priv, 32)
416 let vc_raw: *u8 = sys_mmap(64)
417 let vc: *TlsValidationContext = vc_raw as *TlsValidationContext
418 vc.store = store
419 vc.sni_host = urlbuf + target.url.host_off
420 vc.sni_host_len = target.url.host_len
421 vc.now_epoch = sys_now_realtime_sec()
422 var sr: i64 = 0
423 if use_chrome == 1 { sr = nx_tls13_client_session_run_chrome(fd, urlbuf + target.url.host_off, target.url.host_len, cr, priv, vc) }
424 else { sr = nx_tls13_client_session_run(fd, urlbuf + target.url.host_off, target.url.host_len, cr, priv, vc) }
425 if sr <= 0 { sys_close(fd); return 0 - NX_FF_HANDSHAKE }
426 let session: *Tls13ClientSession = sr as *Tls13ClientSession
427
428 let path: *u8 = sys_mmap(NX_MAGIC_2048)
429 let plen: i64 = ff_path(urlbuf, target, path)
430 let req: *u8 = sys_mmap(NX_MAGIC_4096)
431 let req_len: i64 = nx_http_build_range_get(path, plen, urlbuf + target.url.host_off, target.url.host_len, rstart, rend, req)
432 let icap: i64 = out_cap + out_cap/8 + NX_MAGIC_65536
433 let buf: *u8 = sys_mmap(icap)
434 let gc: i64 = nx_https_req_complete(session, fd, req, req_len, buf, icap)
435 sys_close(fd)
436 if gc < 0 { sys_munmap(buf as *u8, icap); return 0 - NX_FF_GET }
437
438 let r: *i64 = (sys_mmap(128)) as *i64
439 if nx_http_response_parse(buf, gc, r) != 0 { sys_munmap(buf as *u8, icap); return 0 - NX_FF_PARSE }
440 out_status[0] = r[1]
441 let body_off: i64 = r[6]
442 if r[10] == 1 {
443 let dn: i64 = nx_http_dechunk(buf + body_off, gc - body_off, out, out_cap)
444 sys_munmap(buf as *u8, icap)
445 return dn
446 }
447 var blen: i64 = gc - body_off
448 if blen < 0 { blen = 0 }
449 if blen > out_cap { blen = out_cap }
450 var c: i64 = 0
451 while c < blen { out[c] = buf[body_off + c]; c = c + 1 }
452 sys_munmap(buf as *u8, icap)
453 return blen
454}
455
456// TLS 1.2 GET: reach TLS-1.2-ONLY hosts that reject our TLS-1.3 ClientHello (ukdevilz-class -- the server
457// sends NO ServerHello to a 1.3 hello). Completes ECDHE-RSA AND ECDHE-ECDSA (0xC02F/0xC02B) over P-256 (the
458// hello forces secp256r1 groups). Follows redirects (bounded) so a 1.2-only host that 301s reaches content.
459// Reuses the shipped, gated nx_tls12 client + the same connect/URL/response-parse/redirect pieces as ff_core.
460func nx_https_fetch_follow_12(url0: *u8, store: *TrustStore, out: *u8, out_cap: i64, out_status: *i64) -> i64 {
461 out_status[0] = 0
462 let urlbuf: *u8 = sys_mmap(NX_MAGIC_4096)
463 var ui: i64 = 0
464 while url0[ui] != 0 as u8 { urlbuf[ui] = url0[ui]; ui = ui + 1 }
465 urlbuf[ui] = 0 as u8
466 var hop: i64 = 0
467 while hop <= 6 {
468 let target_raw: *u8 = sys_mmap(64)
469 let target: *NxHttpsTarget = target_raw as *NxHttpsTarget
470 target.url = nx_url_new()
471 target.port = 0
472 if nx_https_url_for_fetch(urlbuf, target) != NX_HTTPS_URL_OK { return 0 - NX_FF_BAD_URL }
473 // REACH FIX 2026-07-31 (debt 1785519121): this leg used nx_tls12_client_session_run, which
474 // does NOT complete a handshake with hosts that nx_tls12_req.t12_request reaches. PROVEN by
475 // nx_reach12_test, running BOTH impls in ONE process on FRESH connections with no 1.3 attempt
476 // before either: usgs.gov -> t12_request bytes=1684, client_session bytes=-3 HANDSHAKE. The
477 // ladder was wired to the broken one of two TLS-1.2 clients, so every -3 row stayed
478 // unreachable even after the chokepoint fix.
479 // NOT a security downgrade: t12_request is FAIL-CLOSED -- chain-to-Mozilla-roots + hostname +
480 // validity via nx_https_cert_pipeline_verify_with_store, plus an SKE-signature check binding
481 // the ephemeral key to the validated leaf. It loads its own trust store, so `store` is now
482 // unused here; the parameter is KEPT so the signature stays stable (Cardinal 19).
483 let icap: i64 = out_cap + out_cap/8 + NX_MAGIC_65536
484 let buf: *u8 = sys_mmap(icap)
485 let acc: i64 = t12_request(urlbuf, "GET" as *u8, 3, "" as *u8, 0, "" as *u8, 0, buf, icap)
486 if acc <= 0 { sys_munmap(buf as *u8, icap); return 0 - NX_FF_HANDSHAKE }
487 let r: *i64 = (sys_mmap(128)) as *i64
488 if nx_http_response_parse(buf, acc, r) != 0 { sys_munmap(buf as *u8, icap); return 0 - NX_FF_PARSE }
489 let status: i64 = r[1]
490 out_status[0] = status
491 if nx_redir_is_redirect(status) == 1 {
492 let loc: *u8 = sys_mmap(NX_MAGIC_4096)
493 let ln: i64 = nx_redir_location(buf, acc, loc, NX_MAGIC_4096)
494 if ln <= 0 { sys_munmap(buf as *u8, icap); return 0 - NX_FF_NO_LOCATION }
495 let resolved: *u8 = sys_mmap(NX_MAGIC_4096)
496 ff_resolve_location(loc, urlbuf, target, resolved)
497 var k: i64 = 0
498 while resolved[k] != 0 as u8 { urlbuf[k] = resolved[k]; k = k + 1 }
499 urlbuf[k] = 0 as u8
500 sys_munmap(buf as *u8, icap)
501 hop = hop + 1
502 } else {
503 let body_off: i64 = r[6]
504 if r[10] == 1 {
505 let dn: i64 = nx_http_dechunk(buf + body_off, acc - body_off, out, out_cap)
506 sys_munmap(buf as *u8, icap)
507 if dn < 0 { return dn }
508 return ff_gunzip(out, dn, out_cap)
509 }
510 var blen: i64 = acc - body_off
511 if blen < 0 { blen = 0 }
512 if blen > out_cap { blen = out_cap }
513 var c: i64 = 0
514 while c < blen { out[c] = buf[body_off + c]; c = c + 1 }
515 sys_munmap(buf as *u8, icap)
516 return ff_gunzip(out, blen, out_cap)
517 }
518 }
519 return 0 - NX_FF_TOO_MANY
520}
521
522// BEST-EFFORT fetch: minimal fleet hello (fast) -> Chrome-JA3 hello (anti-bot CDNs) -> TLS 1.2 (1.2-only
523// hosts that reject a 1.3 hello, ukdevilz-class). Widest reach; extra handshakes only when earlier tries fail.
524func nx_https_fetch_follow_best(url0: *u8, store: *TrustStore, out: *u8, out_cap: i64,
525 max_hops: i64, out_status: *i64) -> i64 {
526 let n: i64 = ff_core(url0, store, out, out_cap, max_hops, out_status, 0)
527 if n > 0 { if out_status[0] >= 200 { if out_status[0] < 300 { return n } } }
528 let n2: i64 = ff_core(url0, store, out, out_cap, max_hops, out_status, 1)
529 if n2 > 0 { if out_status[0] >= 200 { if out_status[0] < 300 { return n2 } } }
530 // both 1.3 hellos failed -> the host may be TLS-1.2-only
531 let n3: i64 = nx_https_fetch_follow_12(url0, store, out, out_cap, out_status)
532 if n3 > 0 { return n3 }
533 // nothing worked -> report the 1.2 status (or the last 1.3 result if 1.2 also died)
534 if n2 > 0 { return n2 }
535 return n
536}
537
538// REACH FIX 2026-07-31 (debt 1785516905, ws=library-datasets). MEASURED: 198 call sites used the
539// narrow 1.3-minimal-hello entry point vs 43 on _best, so 82pct of the fleet could not reach
540// TLS-1.2-only hosts -- which is exactly where US federal open data lives (usgs/usda/nsf/noaa/
541// eia/bls). PROOF: nx_https_get3 on waterservices.usgs.gov died verdict=5 at recvSH while
542// nx_tls12_probe on the SAME URL returned chain=VALID + HTTP 200 + 1307B. NEG-CONTROL: wikipedia
543// via the 1.3 path returned 163307B, so the 1.3 client was never broken -- a whole HOST CLASS was
544// simply unreachable, and that is why the research corpus was 44 Wikipedia pages.
545//
546// Delegating the STABLE NAME to _best is ADDITIVE, not a Cardinal-19 break: _best runs the exact
547// same minimal hello FIRST and returns its result untouched whenever that already yields 2xx, so
548// every currently-working fetch is byte-identical. The chrome hello and the TLS-1.2 ladder run
549// ONLY after the minimal hello has already failed -- i.e. only in cases that returned nothing
550// before. Callers needing byte-exact narrow behavior call nx_https_fetch_follow_13only above.
551// NOTE: this lib is compiled INTO each organ, so a caller inherits the reach only when REBUILT.
552func nx_https_fetch_follow(url0: *u8, store: *TrustStore, out: *u8, out_cap: i64,
553 max_hops: i64, out_status: *i64) -> i64 {
554 return nx_https_fetch_follow_best(url0, store, out, out_cap, max_hops, out_status)
555}