nx_http_client.nx source
↩ module page · 759 lines · 37352 B
1// nx_http_client.nx -- sovereign HTTP/1.1 GET client.
2//
3// Phase 0 of the Nishi browser per docs/NISHI_BROWSER_ROADMAP.md.
4// Smallest possible NX-side HTTP client: TCP connect + send GET +
5// receive response. No libc. No curl. No fetch shim.
6//
7// What it does today:
8// - opens a TCP socket
9// - connects to a caller-provided IPv4 address + port
10// - sends "GET <path> HTTP/1.1\r\nHost: <host>\r\nConnection: close\r\n\r\n"
11// - reads the response until the server closes
12// - hands the caller the raw bytes (header + body)
13//
14// What it doesn't do yet:
15// - DNS lookup (caller passes a packed IPv4 + port). L20b queued.
16// - HTTP/1.1 keep-alive (we Connection: close to make the read-to-EOF
17// pattern correct).
18// - HTTP/2, HTTP/3, ALPN. Eventually.
19// - TLS. Phase 8.
20// - Redirects (302). Caller handles.
21//
22// genealogy_id: rfc_7230_http_1_1 + linux_socket_api + nishi_pages_v1_pages_nx
23// lineage_id: nishi_browser_http_client_q10
24
25// nx_safety_envelope:
26// intended_use: AUTO_APPLIED -- primitive-specific tuning queued
27// sil_target: SIL1
28// evidence: [bulk_applied_2026-05-16, see-file-comment-for-detail]
29// verdict: NOT_YET_EVALUATED
30
31import "nx_syscalls.nx"
32import "nx_connect.nx" // bounded connect: a raw sys_connect here hangs ~127s on a black-holed host
33import "nx_dec_emit.nx"
34import "nx_codec_caps.nx" // Accept/Accept-Encoding are DERIVED, never hand-written
35const NX_MAGIC_1024: i64 = 1024
36
37// ---- REQUEST IDENTITY: ONE OWNER (2026-08-25) ------------------------------------------------------
38// THE DEFECT THIS REPLACES. The same User-Agent line was written out FOUR times in this file, and all
39// four claimed to be Chrome. The comments above the call sites record three DIFFERENT policies stacked
40// in sequence -- "present as a real, polite browser", then "HONEST non-browser identity", then "The
41// Nishi Browser IS a browser" -- and the code ended up applying the last one to EVERYTHING. Two of those
42// comments assert that "the bulk crawler keeps its honest NishiBot identity on its SEPARATE path".
43// MEASURED 2026-08-25: it does not. nx_web_crawl_step -> nx_https_fetch_follow_best -> ff_core ->
44// nx_https_get_complete_cookie_xhdr -> nx_http_client_build_request_cookie_xhdr, which is one of these
45// four sites. WE CRAWL THE OPEN WEB AS FORGED CHROME.
46//
47// Two consequences that are worse than the forgery itself:
48// * we MATCH the robots.txt group `nishibot` while SENDING Chrome -- we obey rules written for a token
49// we never present, so a site operator has no way to address us, and no way to see that we complied.
50// * the later identity mechanism appended its UA via the xhdr slot, AFTER this one. RFC 9110 5.3
51// forbids repeating a non-list field and receivers take the FIRST, so the declared identity never
52// reached the wire at all.
53//
54// ★A COMMENT DESCRIBING A SEPARATE HONEST PATH IS NOT A SEPARATE HONEST PATH.
55//
56// THE FIX IS A SELECTOR, NOT A REPLACEMENT. The browser lane's Chrome UA is LOAD-BEARING -- it matches
57// our Chrome JA3 TLS fingerprint, and sites that 403 non-browser clients serve real content only to the
58// pair. So the default is BYTE-IDENTICAL to the old behaviour and every existing caller is unchanged;
59// a caller that is a CRAWLER says so explicitly. Additive per Cardinal 19.
60// ASCII case-fold bounds, named for PURPOSE. The magic-number ratchet refused the build over the bare
61// 65/90/32 I first wrote into hc_xhdr_has_ua -- correctly: this file is full of grandfathered raw byte
62// emission, so a NEW bare literal is exactly what the ratchet exists to stop, and I had already named
63// this same set properly in nx_textcut and then not here.
64const HC_CH_UPPER_A: i64 = 65
65const HC_CH_UPPER_Z: i64 = 90
66const HC_CH_CASE_DELTA: i64 = 32
67const HC_UA_BROWSER: i64 = 0 // default: Chrome UA matching our JA3. The browsing/HTTPS-GET/torrent paths.
68const HC_UA_BOT: i64 = 1 // honest crawler identity. The bulk crawler MUST select this.
69static hc_ua_mode_g: i64
70func hc_set_identity(mode: i64) -> i64 { hc_ua_mode_g = mode; return mode }
71func hc_identity() -> i64 { return hc_ua_mode_g }
72
73// Emit the identity block (UA + Accept-Language). ONE definition; the four builders below call it.
74// The bot form carries a CONTACT URL because an identity a site operator cannot reach is not an identity
75// -- and a contact url that 404s is worse than none, so /bot is served (see sites/nishifamily/bot).
76func hc_put_ua(out: *u8, o0: i64) -> i64 {
77 var o: i64 = o0
78 var ua: *u8 = "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36\r\nAccept-Language: en-US,en;q=0.9\r\n" as *u8
79 if hc_ua_mode_g == HC_UA_BOT { ua = "User-Agent: NishiBot/1.0 (+https://nishifamily.com/bot)\r\nAccept-Language: en-US,en;q=0.9\r\n" as *u8 }
80 var ui: i64 = 0
81 while ua[ui] != (0 as u8) { out[o] = ua[ui]; o = o + 1; ui = ui + 1 }
82 // ONE OWNER: every builder that emits an identity inherits the matching client hints in the SAME act,
83 // so the UA and its sec-ch-ua version can never drift apart into the mismatch anti-bot scoring hunts.
84 // hc_put_ua_x returns BEFORE reaching hc_put_ua when the caller declared their own UA, so a declared
85 // identity is never contradicted by Chrome hints.
86 o = hc_put_hints(out, o)
87 return o
88}
89
90// Append the Accept + Accept-Encoding pair, derived from the codecs we
91// can actually decode. Returns the new offset.
92//
93// Every one of the three request builders below used to carry its own
94// copy of a hand-written header literal. Two facts in that literal were
95// false: it advertised `image/avif` with no AVIF codec in the tree, and
96// it sent `Accept-Encoding: identity` with the comment "because we don't
97// inflate gzip/deflate yet" -- while nx_gzip_inflate and nx_zlib_inflate
98// were both complete. A server that believed either answered with bytes
99// we could not read, and nothing errored: a blank image, or a compressed
100// body handed to an HTML parser.
101//
102// Routing all three through one call means the next codec we land makes
103// every builder honest in the same act. nx_codec_caps_gate proves the
104// emitted header equals the capability in both directions.
105// Does the caller's extra-header block ALREADY carry a User-Agent? Case-insensitive, because header
106// field names are case-insensitive (RFC 9110 5.1) and a case-sensitive check here would miss
107// "user-agent:" and re-create the exact duplicate it exists to prevent.
108func hc_xhdr_has_field(xhdr: *u8, xhdr_len: i64, pat: *u8) -> i64 {
109 var pl: i64 = 0
110 while pat[pl] != (0 as u8) { pl = pl + 1 }
111 var i: i64 = 0
112 while i + pl <= xhdr_len {
113 var m: i64 = 1
114 var j: i64 = 0
115 while j < pl {
116 var c: i64 = xhdr[i + j] as i64
117 if c >= HC_CH_UPPER_A { if c <= HC_CH_UPPER_Z { c = c + HC_CH_CASE_DELTA } }
118 if c != (pat[j] as i64) { m = 0; j = pl } else { j = j + 1 }
119 }
120 if m == 1 { return 1 }
121 i = i + 1
122 }
123 return 0
124}
125
126func hc_xhdr_has_ua(xhdr: *u8, xhdr_len: i64) -> i64 { return hc_xhdr_has_field(xhdr, xhdr_len, "user-agent:" as *u8) }
127// Does the caller's block already carry an Accept-Encoding? Then the CALLER decides what it can decode and the
128// derived codec caps are NOT appended: a client that cannot decode must not advertise. Measured 2026-09-06: the
129// POST completer returns the reply verbatim (no gzip stage) while hc_put_caps advertised gzip, so the public
130// edge gzipped every JSON receipt and nx_content_put_client read no transfer id. ONE scanner for both fields.
131func hc_xhdr_has_ae(xhdr: *u8, xhdr_len: i64) -> i64 { return hc_xhdr_has_field(xhdr, xhdr_len, "accept-encoding:" as *u8) }
132
133// THE DUPLICATE-HEADER FIX (2026-08-25). The two xhdr builders emitted OUR identity and then appended
134// the CALLER's, producing two User-Agent field lines. RFC 9110 5.3 forbids repeating a non-list field,
135// and receivers resolve it by taking the FIRST -- ours. So every caller that had gone to the trouble of
136// declaring an honest identity (edgar_ua_header, from knowledge/fetch_identity.conf, used by
137// nx_fin_edgar_fetch, nx_edgar_submissions and nx_hops_probe) had it silently discarded on the wire.
138// THE CALLER WINS. Fixing it HERE repairs all three callers at once and needs no edit in any of them --
139// the alternative was a custom-UA API plus three call-site changes, which is three chances to miss one.
140func hc_put_ua_x(out: *u8, o: i64, xhdr: *u8, xhdr_len: i64) -> i64 {
141 if hc_xhdr_has_ua(xhdr, xhdr_len) == 1 { return o }
142 return hc_put_ua(out, o)
143}
144
145// '"' emitted BY NAME. sec-ch-ua's grammar (RFC 8941 structured-field strings) REQUIRES real double
146// quotes, and this file's own lexer notes already record that string literals bite on '#' and '!' --
147// a quote is not a character to discover the hard way inside a literal. One named byte, no escape.
148const HC_CH_DQUOTE: i64 = 34
149
150func hc_puts(out: *u8, o0: i64, s: *u8) -> i64 {
151 var o: i64 = o0
152 var i: i64 = 0
153 while s[i] != (0 as u8) { out[o] = s[i]; o = o + 1; i = i + 1 }
154 return o
155}
156
157func hc_putb(out: *u8, o: i64, b: i64) -> i64 { out[o] = b as u8; return o + 1 }
158
159// THE CLIENT-HINT / FETCH-METADATA BLOCK (2026-09-03) -- the OTHER HALF OF THE CHROME IMPERSONATION.
160// We already pay for a Chrome-JA3 ClientHello (nx_tls13_chrome_session, imported by nx_https_get
161// precisely so "anti-bot CDNs complete the handshake instead of RST-ing our minimal hello"). But the
162// REQUEST we then sent carried only UA + Accept-Language + the derived Accept pair -- a header set no
163// Chrome has ever sent. A Chrome TLS fingerprint under a non-Chrome header set is not a weaker signal
164// than either alone; it is the MISMATCH that anti-bot scoring is built to find.
165//
166// MEASURED 2026-09-03, control-separated: fab.com (Cloudflare) returns 403 to the UA-alone profile and
167// 200 (885,467 B, cf-mitigated absent) to the full profile OVER THE SAME TLS STACK -- so on that host
168// the blocker is provably the HEADER PROFILE, not the JA3. The TLS half shipped and the header half did
169// not: a law applied in one layer and not its sibling.
170//
171// IDENTITY-GATED ON PURPOSE: HC_UA_BOT returns untouched. An honest crawler identity that ships Chrome
172// client hints is no longer honest, and nx_http_client_gate's neg-control tooth asserts exactly that.
173// Callers that declare their OWN User-Agent inherit the same protection for free, because hc_put_ua_x
174// already returns before reaching here (the 2026-08-25 duplicate-header lesson: THE CALLER WINS).
175//
176// IMPRECISION DECLARED: the Sec-Fetch-* values describe a top-level document navigation, which is the
177// dominant case for this client and correct for a form POST, but a cross-origin API POST would really
178// send Sec-Fetch-Mode: cors / Sec-Fetch-Dest: empty. Emitting one owner's worth of headers for all six
179// builders is this file's own established doctrine ("fixing it HERE repairs all callers at once"); a
180// per-method Sec-Fetch table is a NAMED next rung, not an oversight.
181func hc_put_hints(out: *u8, o0: i64) -> i64 {
182 if hc_ua_mode_g == HC_UA_BOT { return o0 }
183 var o: i64 = o0
184 o = hc_puts(out, o, "sec-ch-ua: " as *u8)
185 o = hc_putb(out, o, HC_CH_DQUOTE); o = hc_puts(out, o, "Chromium" as *u8); o = hc_putb(out, o, HC_CH_DQUOTE)
186 o = hc_puts(out, o, ";v=" as *u8)
187 o = hc_putb(out, o, HC_CH_DQUOTE); o = hc_puts(out, o, "121" as *u8); o = hc_putb(out, o, HC_CH_DQUOTE)
188 o = hc_puts(out, o, ", " as *u8)
189 o = hc_putb(out, o, HC_CH_DQUOTE); o = hc_puts(out, o, "Not A(Brand" as *u8); o = hc_putb(out, o, HC_CH_DQUOTE)
190 o = hc_puts(out, o, ";v=" as *u8)
191 o = hc_putb(out, o, HC_CH_DQUOTE); o = hc_puts(out, o, "99" as *u8); o = hc_putb(out, o, HC_CH_DQUOTE)
192 o = hc_puts(out, o, ", " as *u8)
193 o = hc_putb(out, o, HC_CH_DQUOTE); o = hc_puts(out, o, "Google Chrome" as *u8); o = hc_putb(out, o, HC_CH_DQUOTE)
194 o = hc_puts(out, o, ";v=" as *u8)
195 o = hc_putb(out, o, HC_CH_DQUOTE); o = hc_puts(out, o, "121" as *u8); o = hc_putb(out, o, HC_CH_DQUOTE)
196 o = hc_puts(out, o, "\r\nsec-ch-ua-mobile: ?0\r\nsec-ch-ua-platform: " as *u8)
197 o = hc_putb(out, o, HC_CH_DQUOTE); o = hc_puts(out, o, "Windows" as *u8); o = hc_putb(out, o, HC_CH_DQUOTE)
198 o = hc_puts(out, o, "\r\nSec-Fetch-Dest: document\r\nSec-Fetch-Mode: navigate\r\nSec-Fetch-Site: none\r\nSec-Fetch-User: ?1\r\nUpgrade-Insecure-Requests: 1\r\n" as *u8)
199 return o
200}
201
202func hc_put_caps(out: *u8, o: i64) -> i64 {
203 let cch: *u8 = sys_mmap(NX_MAGIC_1024)
204 var i: i64 = 0
205 var p: i64 = o
206 nx_codec_caps_headers(cch)
207 while cch[i] != (0 as u8) {
208 out[p] = cch[i]
209 p = p + 1
210 i = i + 1
211 }
212 return p
213}
214const NX_MAGIC_4096: i64 = 4096
215
216// Sealed verdict for an HTTP GET attempt.
217const NX_HTTP_C_VERDICT_UNKNOWN: i64 = 0
218const NX_HTTP_C_VERDICT_OK: i64 = 1
219const NX_HTTP_C_VERDICT_SOCKET_FAIL: i64 = 2
220const NX_HTTP_C_VERDICT_CONNECT_FAIL: i64 = 3
221const NX_HTTP_C_VERDICT_SEND_FAIL: i64 = 4
222const NX_HTTP_C_VERDICT_RECV_FAIL: i64 = 5
223const NX_HTTP_C_VERDICT_N: i64 = 6
224
225// Build the request line + headers into `out`. Returns the byte
226// length written. Caller must give a buffer of >= 4 KB.
227//
228// Format:
229// GET <path>\r\n (path bytes)
230// plus "HTTP/1.1\r\n"
231// Host: <host>\r\n (host bytes)
232// Connection: close\r\n\r\n
233// (terminating)
234//
235// Pre: path_bytes / host_bytes are caller-supplied; no allocation.
236// Connect budget for one HTTP request. Replaces the kernel's ~127s SYN-retry ceiling, which is
237// not a timeout anyone chose -- it is just what happens when nobody sets one.
238const NX_HTTP_C_CONNECT_MS: i64 = 6000
239
240// ---- THE REQUEST BUFFER IS DERIVED HERE, NEVER GUESSED BY A CALLER (2026-08-27) -----------------
241// MEASURED: nx_browser_page_demo_test handed nx_http_client_build_request a hand-picked 256-byte
242// arena for `GET / ... Host: example.com`. The day the identity block grew (the Chrome UA +
243// Accept-Language, then the DERIVED codec-caps Accept/Accept-Encoding pair) the request outran it,
244// the crash guard printed ARENA-OVERRUN prev_alloc_size=256 with the Accept header spilling into
245// the next cell, and the "rendered page" the test judged was arena garbage -- read as exit 80,
246// "the browser cannot render example.com". A CAP THAT HAS TO BE GUESSED IS A DEFECT GENERATOR IN
247// BOTH DIRECTIONS, and the guess was made in a file that does not own the thing being measured.
248// This is the ONE owner of the bound: the fixed part (request line, Host, identity, caps,
249// Connection: close) is bounded by the contract every builder above documents ("a buffer of
250// >= 4 KB" -- NX_MAGIC_4096 is what this file's own nx_http_client_get allocates), and the
251// caller's VARIABLE parts are added rather than assumed to fit inside it. Compose it:
252// let req: *u8 = sys_mmap(nx_http_client_request_cap(path_len, host_len, cookie_len, xhdr_len))
253// A new header this file learns to emit changes this function, and every caller inherits the
254// change in the same act -- which is precisely what a per-caller literal can never do.
255func nx_http_client_request_cap(path_len: i64, host_len: i64, cookie_len: i64, xhdr_len: i64) -> i64 {
256 return NX_MAGIC_4096 + path_len + host_len + cookie_len + xhdr_len
257}
258
259func nx_http_client_build_request(
260 path: *u8, path_len: i64,
261 host: *u8, host_len: i64,
262 out: *u8
263) -> i64 {
264 var o: i64 = 0
265 // "GET "
266 out[o]=71;o=o+1; out[o]=69;o=o+1; out[o]=84;o=o+1; out[o]=32;o=o+1
267 // path
268 var pi: i64 = 0
269 while pi < path_len { out[o+pi] = path[pi]; pi = pi + 1 }
270 o = o + path_len
271 // " HTTP/1.1\r\n"
272 out[o]=32;o=o+1
273 out[o]=72;o=o+1; out[o]=84;o=o+1; out[o]=84;o=o+1; out[o]=80;o=o+1
274 out[o]=47;o=o+1; out[o]=49;o=o+1; out[o]=46;o=o+1; out[o]=49;o=o+1
275 out[o]=13;o=o+1; out[o]=10;o=o+1
276 // "Host: "
277 out[o]=72;o=o+1; out[o]=111;o=o+1; out[o]=115;o=o+1; out[o]=116;o=o+1
278 out[o]=58;o=o+1; out[o]=32;o=o+1
279 var hi: i64 = 0
280 while hi < host_len { out[o+hi] = host[hi]; hi = hi + 1 }
281 o = o + host_len
282 out[o]=13;o=o+1; out[o]=10;o=o+1
283 // Present as a real, polite browser ("like a person") so sites that 403 bots/anonymous clients serve
284 // their real content (operator 2026-06-23: access everywhere, human in presentation). Accept-Encoding:
285 // identity because we don't inflate gzip/deflate yet. The bulk crawler keeps its honest NishiBot identity
286 // via the SEPARATE nx_http_get_ua path; this builder serves the browser/HTTPS-GET+POST/torrent paths.
287 // HONEST non-browser identity. mangadex's API 400s ANY "Mozilla" (browser) UA on purpose -- it forces API
288 // clients to identify as an app, not a browser (curl's plain UA gets 200). Cloudflare-fronted image CDNs
289 // (nhentai) passed our TLS JA3 even with a mismatched UA, so the TLS fingerprint (not the UA) is what beats
290 // them. So: an honest app UA satisfies mangadex AND still rides the Chrome-JA3 for Cloudflare. (op 2026-07-03)
291 // The Nishi Browser IS a browser: present a real Chrome UA (matching our Chrome-JA3 TLS fingerprint) + a browser
292 // Accept set, so sites that 403/406 non-browser clients (jable/javxxx/cam sites) serve their real content. This
293 // is the browsing/HTTPS-GET path; the bulk crawler keeps its honest NishiBot identity on its SEPARATE path.
294 // (mangadex's API prefers an app UA -- if that host is re-added, give it a per-host override; browser is the default.)
295 o = hc_put_ua(out, o)
296 o = hc_put_caps(out, o)
297 // "Connection: close\r\n\r\n"
298 out[o]=67;o=o+1; out[o]=111;o=o+1; out[o]=110;o=o+1; out[o]=110;o=o+1
299 out[o]=101;o=o+1; out[o]=99;o=o+1; out[o]=116;o=o+1; out[o]=105;o=o+1
300 out[o]=111;o=o+1; out[o]=110;o=o+1
301 out[o]=58;o=o+1; out[o]=32;o=o+1
302 out[o]=99;o=o+1; out[o]=108;o=o+1; out[o]=111;o=o+1; out[o]=115;o=o+1
303 out[o]=101;o=o+1
304 out[o]=13;o=o+1; out[o]=10;o=o+1
305 out[o]=13;o=o+1; out[o]=10;o=o+1
306 return o
307}
308
309// Generic extra-header variant (ADDITIVE, base kept byte-identical per Cardinal 19): injects caller-supplied
310// header line(s) `xhdr` (each CRLF-terminated, NO trailing blank line) right after the UA/Accept block and
311// before Connection: close. Powers the sovereign authed GitHub-oracle fetch (Authorization: token <PAT>) with
312// NO curl and NO graphql. cookie/POST variants queued to fold into this (DRY consolidation rung).
313func nx_http_client_build_request_xhdr(
314 path: *u8, path_len: i64,
315 host: *u8, host_len: i64,
316 xhdr: *u8, xhdr_len: i64,
317 out: *u8
318) -> i64 {
319 var o: i64 = 0
320 out[o]=71;o=o+1; out[o]=69;o=o+1; out[o]=84;o=o+1; out[o]=32;o=o+1
321 var pi: i64 = 0
322 while pi < path_len { out[o+pi] = path[pi]; pi = pi + 1 }
323 o = o + path_len
324 out[o]=32;o=o+1
325 out[o]=72;o=o+1; out[o]=84;o=o+1; out[o]=84;o=o+1; out[o]=80;o=o+1
326 out[o]=47;o=o+1; out[o]=49;o=o+1; out[o]=46;o=o+1; out[o]=49;o=o+1
327 out[o]=13;o=o+1; out[o]=10;o=o+1
328 out[o]=72;o=o+1; out[o]=111;o=o+1; out[o]=115;o=o+1; out[o]=116;o=o+1
329 out[o]=58;o=o+1; out[o]=32;o=o+1
330 var hi: i64 = 0
331 while hi < host_len { out[o+hi] = host[hi]; hi = hi + 1 }
332 o = o + host_len
333 out[o]=13;o=o+1; out[o]=10;o=o+1
334 o = hc_put_ua_x(out, o, xhdr, xhdr_len)
335 if hc_xhdr_has_ae(xhdr, xhdr_len) == 0 { o = hc_put_caps(out, o) }
336 var xi: i64 = 0
337 while xi < xhdr_len { out[o] = xhdr[xi]; o = o + 1; xi = xi + 1 }
338 out[o]=67;o=o+1; out[o]=111;o=o+1; out[o]=110;o=o+1; out[o]=110;o=o+1
339 out[o]=101;o=o+1; out[o]=99;o=o+1; out[o]=116;o=o+1; out[o]=105;o=o+1
340 out[o]=111;o=o+1; out[o]=110;o=o+1
341 out[o]=58;o=o+1; out[o]=32;o=o+1
342 out[o]=99;o=o+1; out[o]=108;o=o+1; out[o]=111;o=o+1; out[o]=115;o=o+1
343 out[o]=101;o=o+1
344 out[o]=13;o=o+1; out[o]=10;o=o+1
345 out[o]=13;o=o+1; out[o]=10;o=o+1
346 return o
347}
348
349// Cookie-aware variant: identical to nx_http_client_build_request but injects a
350// "Cookie: <cookie>\r\n" header when cookie_len>0, so a redirect-following fetch can carry a
351// session across hops (browser-faithful). E.g. chaturbate's /?next= age-gate hands back
352// csrftoken/sbr/AG_Key that must be replayed to reach the room -- without a jar we land on the
353// gate page forever. Base builder kept byte-identical (Cardinal 19); this is purely additive.
354// `cookie` = a jar buffer "name=value; name2=value2" (NO CRLF, caller-terminated by cookie_len).
355func nx_http_client_build_request_cookie(
356 path: *u8, path_len: i64,
357 host: *u8, host_len: i64,
358 cookie: *u8, cookie_len: i64,
359 out: *u8
360) -> i64 {
361 var o: i64 = 0
362 out[o]=71;o=o+1; out[o]=69;o=o+1; out[o]=84;o=o+1; out[o]=32;o=o+1
363 var pi: i64 = 0
364 while pi < path_len { out[o+pi] = path[pi]; pi = pi + 1 }
365 o = o + path_len
366 out[o]=32;o=o+1
367 out[o]=72;o=o+1; out[o]=84;o=o+1; out[o]=84;o=o+1; out[o]=80;o=o+1
368 out[o]=47;o=o+1; out[o]=49;o=o+1; out[o]=46;o=o+1; out[o]=49;o=o+1
369 out[o]=13;o=o+1; out[o]=10;o=o+1
370 out[o]=72;o=o+1; out[o]=111;o=o+1; out[o]=115;o=o+1; out[o]=116;o=o+1
371 out[o]=58;o=o+1; out[o]=32;o=o+1
372 var hi: i64 = 0
373 while hi < host_len { out[o+hi] = host[hi]; hi = hi + 1 }
374 o = o + host_len
375 out[o]=13;o=o+1; out[o]=10;o=o+1
376 o = hc_put_ua(out, o)
377 o = hc_put_caps(out, o)
378 if cookie_len > 0 {
379 let ckh: *u8 = "Cookie: " as *u8
380 var ci: i64 = 0
381 while ckh[ci] != (0 as u8) { out[o] = ckh[ci]; o = o + 1; ci = ci + 1 }
382 var cj: i64 = 0
383 while cj < cookie_len { out[o] = cookie[cj]; o = o + 1; cj = cj + 1 }
384 out[o]=13;o=o+1; out[o]=10;o=o+1
385 }
386 out[o]=67;o=o+1; out[o]=111;o=o+1; out[o]=110;o=o+1; out[o]=110;o=o+1
387 out[o]=101;o=o+1; out[o]=99;o=o+1; out[o]=116;o=o+1; out[o]=105;o=o+1
388 out[o]=111;o=o+1; out[o]=110;o=o+1
389 out[o]=58;o=o+1; out[o]=32;o=o+1
390 out[o]=99;o=o+1; out[o]=108;o=o+1; out[o]=111;o=o+1; out[o]=115;o=o+1
391 out[o]=101;o=o+1
392 out[o]=13;o=o+1; out[o]=10;o=o+1
393 out[o]=13;o=o+1; out[o]=10;o=o+1
394 return o
395}
396
397// Cookie + generic extra-header variant (ADDITIVE; base builders kept byte-identical per Cardinal 19). Emits
398// the Cookie block (when cookie_len>0) THEN the caller's xhdr line(s) (when xhdr_len>0; each CRLF-terminated,
399// NO trailing blank line) after the UA/Accept block and before Connection: close. This is the fold the _xhdr
400// builder's own comment queued: it lets a redirect-following fetch carry BOTH a harvested cookie jar AND a
401// caller-supplied header (Authorization: Bearer <token>, a pre-seeded Cookie: over18=1, Referer, ...). With
402// cookie_len=0 AND xhdr_len=0 the emitted bytes are IDENTICAL to nx_http_client_build_request_cookie with
403// cookie_len=0, i.e. to the plain nx_http_client_build_request (the byte-preservation the fold gate proves).
404// THE CRAWLER'S AND EDGAR'S PATH. This is the builder nx_https_get_complete_cookie_xhdr reaches, so it
405// is the one that was emitting a hardcoded Chrome UA and THEN appending the caller's -- two User-Agent
406// field lines, which RFC 9110 5.3 forbids and receivers resolve by taking the first. hc_put_ua_x below
407// yields to a caller-supplied UA instead of duplicating it.
408func nx_http_client_build_request_cookie_xhdr(
409 path: *u8, path_len: i64,
410 host: *u8, host_len: i64,
411 cookie: *u8, cookie_len: i64,
412 xhdr: *u8, xhdr_len: i64,
413 out: *u8
414) -> i64 {
415 var o: i64 = 0
416 out[o]=71;o=o+1; out[o]=69;o=o+1; out[o]=84;o=o+1; out[o]=32;o=o+1
417 var pi: i64 = 0
418 while pi < path_len { out[o+pi] = path[pi]; pi = pi + 1 }
419 o = o + path_len
420 out[o]=32;o=o+1
421 out[o]=72;o=o+1; out[o]=84;o=o+1; out[o]=84;o=o+1; out[o]=80;o=o+1
422 out[o]=47;o=o+1; out[o]=49;o=o+1; out[o]=46;o=o+1; out[o]=49;o=o+1
423 out[o]=13;o=o+1; out[o]=10;o=o+1
424 out[o]=72;o=o+1; out[o]=111;o=o+1; out[o]=115;o=o+1; out[o]=116;o=o+1
425 out[o]=58;o=o+1; out[o]=32;o=o+1
426 var hi: i64 = 0
427 while hi < host_len { out[o+hi] = host[hi]; hi = hi + 1 }
428 o = o + host_len
429 out[o]=13;o=o+1; out[o]=10;o=o+1
430 o = hc_put_ua_x(out, o, xhdr, xhdr_len)
431 if hc_xhdr_has_ae(xhdr, xhdr_len) == 0 { o = hc_put_caps(out, o) }
432 if cookie_len > 0 {
433 let ckh: *u8 = "Cookie: " as *u8
434 var ci: i64 = 0
435 while ckh[ci] != (0 as u8) { out[o] = ckh[ci]; o = o + 1; ci = ci + 1 }
436 var cj: i64 = 0
437 while cj < cookie_len { out[o] = cookie[cj]; o = o + 1; cj = cj + 1 }
438 out[o]=13;o=o+1; out[o]=10;o=o+1
439 }
440 if xhdr_len > 0 {
441 var xi: i64 = 0
442 while xi < xhdr_len { out[o] = xhdr[xi]; o = o + 1; xi = xi + 1 }
443 }
444 out[o]=67;o=o+1; out[o]=111;o=o+1; out[o]=110;o=o+1; out[o]=110;o=o+1
445 out[o]=101;o=o+1; out[o]=99;o=o+1; out[o]=116;o=o+1; out[o]=105;o=o+1
446 out[o]=111;o=o+1; out[o]=110;o=o+1
447 out[o]=58;o=o+1; out[o]=32;o=o+1
448 out[o]=99;o=o+1; out[o]=108;o=o+1; out[o]=111;o=o+1; out[o]=115;o=o+1
449 out[o]=101;o=o+1
450 out[o]=13;o=o+1; out[o]=10;o=o+1
451 out[o]=13;o=o+1; out[o]=10;o=o+1
452 return o
453}
454
455// Build an HTTP/1.1 POST request line + headers + body into `out`.
456// Returns the byte length written. Caller must give a buffer of
457// >= 4 KB + body_len.
458//
459// Format:
460// POST <path> HTTP/1.1\r\n
461// Host: <host>\r\n
462// Content-Type: <content_type>\r\n
463// Content-Length: <body_len>\r\n
464// Connection: close\r\n
465// \r\n
466// <body bytes>
467//
468// Per RFC 7230 §3.3.2 Content-Length is the canonical body delimiter
469// for client-side requests with known length. Composes
470// nx_dec_emit_u63 for the length emission (cardinal: no inline
471// decimal emitters).
472func nx_http_client_build_request_post(
473 path: *u8, path_len: i64,
474 host: *u8, host_len: i64,
475 content_type: *u8, content_type_len: i64,
476 body: *u8, body_len: i64,
477 out: *u8
478) -> i64 {
479 var o: i64 = 0
480 // "POST "
481 out[o]=80;o=o+1; out[o]=79;o=o+1; out[o]=83;o=o+1; out[o]=84;o=o+1
482 out[o]=32;o=o+1
483 // path
484 var pi: i64 = 0
485 while pi < path_len { out[o+pi] = path[pi]; pi = pi + 1 }
486 o = o + path_len
487 // " HTTP/1.1\r\n"
488 out[o]=32;o=o+1
489 out[o]=72;o=o+1; out[o]=84;o=o+1; out[o]=84;o=o+1; out[o]=80;o=o+1
490 out[o]=47;o=o+1; out[o]=49;o=o+1; out[o]=46;o=o+1; out[o]=49;o=o+1
491 out[o]=13;o=o+1; out[o]=10;o=o+1
492 // "Host: "
493 out[o]=72;o=o+1; out[o]=111;o=o+1; out[o]=115;o=o+1; out[o]=116;o=o+1
494 out[o]=58;o=o+1; out[o]=32;o=o+1
495 var hi: i64 = 0
496 while hi < host_len { out[o+hi] = host[hi]; hi = hi + 1 }
497 o = o + host_len
498 out[o]=13;o=o+1; out[o]=10;o=o+1
499 // "Content-Type: "
500 out[o]=67;o=o+1; out[o]=111;o=o+1; out[o]=110;o=o+1; out[o]=116;o=o+1
501 out[o]=101;o=o+1; out[o]=110;o=o+1; out[o]=116;o=o+1; out[o]=45;o=o+1
502 out[o]=84;o=o+1; out[o]=121;o=o+1; out[o]=112;o=o+1; out[o]=101;o=o+1
503 out[o]=58;o=o+1; out[o]=32;o=o+1
504 var ci: i64 = 0
505 while ci < content_type_len { out[o+ci] = content_type[ci]; ci = ci + 1 }
506 o = o + content_type_len
507 out[o]=13;o=o+1; out[o]=10;o=o+1
508 // "Content-Length: "
509 out[o]=67;o=o+1; out[o]=111;o=o+1; out[o]=110;o=o+1; out[o]=116;o=o+1
510 out[o]=101;o=o+1; out[o]=110;o=o+1; out[o]=116;o=o+1; out[o]=45;o=o+1
511 out[o]=76;o=o+1; out[o]=101;o=o+1; out[o]=110;o=o+1; out[o]=103;o=o+1
512 out[o]=116;o=o+1; out[o]=104;o=o+1; out[o]=58;o=o+1; out[o]=32;o=o+1
513 let dlen: i64 = nx_dec_emit_u63(out, o, body_len)
514 o = o + dlen
515 out[o]=13;o=o+1; out[o]=10;o=o+1
516 // "Connection: close\r\n"
517 out[o]=67;o=o+1; out[o]=111;o=o+1; out[o]=110;o=o+1; out[o]=110;o=o+1
518 out[o]=101;o=o+1; out[o]=99;o=o+1; out[o]=116;o=o+1; out[o]=105;o=o+1
519 out[o]=111;o=o+1; out[o]=110;o=o+1
520 out[o]=58;o=o+1; out[o]=32;o=o+1
521 out[o]=99;o=o+1; out[o]=108;o=o+1; out[o]=111;o=o+1; out[o]=115;o=o+1
522 out[o]=101;o=o+1
523 out[o]=13;o=o+1; out[o]=10;o=o+1
524 // header terminator "\r\n"
525 out[o]=13;o=o+1; out[o]=10;o=o+1
526 // body bytes
527 var bi: i64 = 0
528 while bi < body_len { out[o+bi] = body[bi]; bi = bi + 1 }
529 o = o + body_len
530 return o
531}
532
533// Build a packed sockaddr_in for IPv4 dotted-quad `a.b.c.d` at port.
534// `out` must point at >= 16 bytes (struct sockaddr_in size).
535//
536// Linux sockaddr_in layout:
537// [0..1] sin_family = AF_INET = 2 (little-endian short)
538// [2..3] sin_port = port in BIG-endian
539// [4..7] sin_addr = IPv4 bytes a.b.c.d in network order
540// [8..15] padding zero
541// POST with a UA, a cookie jar and caller headers -- the variant the GET family has had since
542// Cardinal 19 and the POST path never got. MEASURED 2026-09-01: nx_http_client_build_request_post
543// emits ONLY Host, Content-Type, Content-Length and Connection: close. No User-Agent, no way to add
544// one. A POST from that builder is therefore structurally unable to reach any edge that fingerprints
545// its clients -- which is every anti-bot CDN we care about, and specifically the chaturbate
546// /get_edge_hls_url_ajax/ endpoint that the whole live-capture lane depends on (it wants
547// X-Requested-With: XMLHttpRequest, and the room path wants the /?next= age-gate cookies this
548// file's own build_request_cookie comment already documents).
549//
550// STRICTLY ADDITIVE: nx_http_client_build_request_post is untouched, so nx_acme_http, nx_funcheck,
551// nx_edge_probe and nx_porkbun_ping_probe keep byte-identical requests. UA and Accept/Accept-Encoding
552// come from hc_put_ua_x / hc_put_caps, the SAME helpers the GET builders use -- this adds a caller
553// surface, not a second identity ruler, so hc_set_identity still governs bot-vs-browser in one place.
554// hc_put_ua_x suppresses our UA when the caller already supplied one in xhdr, so a caller can override
555// without emitting two User-Agent headers.
556//
557// `cookie` = jar bytes "name=value; name2=value2", NO CRLF (cookie_len==0 to omit the header).
558// `xhdr` = zero or more COMPLETE "Name: value\r\n" lines (xhdr_len==0 to omit).
559// Size the out buffer with nx_http_client_request_cap(path_len, host_len, cookie_len, xhdr_len)
560// PLUS content_type_len + body_len + the Content-Type/Content-Length header text.
561func nx_http_client_build_request_post_xhdr(
562 path: *u8, path_len: i64,
563 host: *u8, host_len: i64,
564 content_type: *u8, content_type_len: i64,
565 body: *u8, body_len: i64,
566 cookie: *u8, cookie_len: i64,
567 xhdr: *u8, xhdr_len: i64,
568 out: *u8
569) -> i64 {
570 var o: i64 = 0
571 // "POST "
572 out[o]=80;o=o+1; out[o]=79;o=o+1; out[o]=83;o=o+1; out[o]=84;o=o+1; out[o]=32;o=o+1
573 var pi: i64 = 0
574 while pi < path_len { out[o+pi] = path[pi]; pi = pi + 1 }
575 o = o + path_len
576 // " HTTP/1.1\r\n"
577 out[o]=32;o=o+1
578 out[o]=72;o=o+1; out[o]=84;o=o+1; out[o]=84;o=o+1; out[o]=80;o=o+1
579 out[o]=47;o=o+1; out[o]=49;o=o+1; out[o]=46;o=o+1; out[o]=49;o=o+1
580 out[o]=13;o=o+1; out[o]=10;o=o+1
581 // "Host: "
582 out[o]=72;o=o+1; out[o]=111;o=o+1; out[o]=115;o=o+1; out[o]=116;o=o+1
583 out[o]=58;o=o+1; out[o]=32;o=o+1
584 var hi: i64 = 0
585 while hi < host_len { out[o+hi] = host[hi]; hi = hi + 1 }
586 o = o + host_len
587 out[o]=13;o=o+1; out[o]=10;o=o+1
588 // User-Agent (suppressed when xhdr already carries one) + derived Accept/Accept-Encoding
589 o = hc_put_ua_x(out, o, xhdr, xhdr_len)
590 if hc_xhdr_has_ae(xhdr, xhdr_len) == 0 { o = hc_put_caps(out, o) }
591 // "Cookie: " <jar> CRLF
592 if cookie_len > 0 {
593 out[o]=67;o=o+1; out[o]=111;o=o+1; out[o]=111;o=o+1; out[o]=107;o=o+1
594 out[o]=105;o=o+1; out[o]=101;o=o+1; out[o]=58;o=o+1; out[o]=32;o=o+1
595 var ki: i64 = 0
596 while ki < cookie_len { out[o+ki] = cookie[ki]; ki = ki + 1 }
597 o = o + cookie_len
598 out[o]=13;o=o+1; out[o]=10;o=o+1
599 }
600 // caller headers verbatim (each already CRLF-terminated)
601 var xi: i64 = 0
602 while xi < xhdr_len { out[o] = xhdr[xi]; o = o + 1; xi = xi + 1 }
603 // "Content-Type: "
604 out[o]=67;o=o+1; out[o]=111;o=o+1; out[o]=110;o=o+1; out[o]=116;o=o+1
605 out[o]=101;o=o+1; out[o]=110;o=o+1; out[o]=116;o=o+1; out[o]=45;o=o+1
606 out[o]=84;o=o+1; out[o]=121;o=o+1; out[o]=112;o=o+1; out[o]=101;o=o+1
607 out[o]=58;o=o+1; out[o]=32;o=o+1
608 var ci: i64 = 0
609 while ci < content_type_len { out[o+ci] = content_type[ci]; ci = ci + 1 }
610 o = o + content_type_len
611 out[o]=13;o=o+1; out[o]=10;o=o+1
612 // "Content-Length: " <n>
613 out[o]=67;o=o+1; out[o]=111;o=o+1; out[o]=110;o=o+1; out[o]=116;o=o+1
614 out[o]=101;o=o+1; out[o]=110;o=o+1; out[o]=116;o=o+1; out[o]=45;o=o+1
615 out[o]=76;o=o+1; out[o]=101;o=o+1; out[o]=110;o=o+1; out[o]=103;o=o+1
616 out[o]=116;o=o+1; out[o]=104;o=o+1; out[o]=58;o=o+1; out[o]=32;o=o+1
617 let dlen: i64 = nx_dec_emit_u63(out, o, body_len)
618 o = o + dlen
619 out[o]=13;o=o+1; out[o]=10;o=o+1
620 // "Connection: close\r\n"
621 out[o]=67;o=o+1; out[o]=111;o=o+1; out[o]=110;o=o+1; out[o]=110;o=o+1
622 out[o]=101;o=o+1; out[o]=99;o=o+1; out[o]=116;o=o+1; out[o]=105;o=o+1
623 out[o]=111;o=o+1; out[o]=110;o=o+1
624 out[o]=58;o=o+1; out[o]=32;o=o+1
625 out[o]=99;o=o+1; out[o]=108;o=o+1; out[o]=111;o=o+1; out[o]=115;o=o+1
626 out[o]=101;o=o+1
627 out[o]=13;o=o+1; out[o]=10;o=o+1
628 // header terminator
629 out[o]=13;o=o+1; out[o]=10;o=o+1
630 // body
631 var bi: i64 = 0
632 while bi < body_len { out[o+bi] = body[bi]; bi = bi + 1 }
633 o = o + body_len
634 return o
635}
636
637func nx_http_client_sockaddr_ipv4(
638 out: *u8,
639 a: i64, b: i64, c: i64, d: i64,
640 port: i64
641) -> i64 {
642 out[0] = 2; out[1] = 0
643 out[2] = (port >> 8) & 0xff
644 out[3] = port & 0xff
645 out[4] = a
646 out[5] = b
647 out[6] = c
648 out[7] = d
649 out[8] = 0; out[9] = 0; out[10] = 0; out[11] = 0
650 out[12] = 0; out[13] = 0; out[14] = 0; out[15] = 0
651 return 16
652}
653
654// Read all available bytes from `fd` into `buf` (cap bytes max) until
655// the peer closes the connection (sys_read returns 0). Returns bytes
656// read.
657func _drain(fd: i64, buf: *u8, cap: i64) -> i64 {
658 var off: i64 = 0
659 var keep: i64 = 1
660 while keep == 1 {
661 if off >= cap { keep = 0 }
662 else {
663 let r: i64 = sys_read(fd, (buf as i64 + off) as *u8, cap - off)
664 if r <= 0 { keep = 0 }
665 else { off = off + r }
666 }
667 }
668 return off
669}
670
671// Top-level: connect, send a GET, read the full response. Returns
672// bytes read (-1 on socket error) and writes the verdict through
673// out_verdict. Response (headers + body) is in out_buf[0..ret).
674func nx_http_client_get(
675 addr_packed: *u8,
676 path: *u8, path_len: i64,
677 host: *u8, host_len: i64,
678 out_buf: *u8, out_cap: i64,
679 out_verdict: *i64
680) -> i64 {
681 *out_verdict = NX_HTTP_C_VERDICT_UNKNOWN
682
683 let fd: i64 = sys_socket(AF_INET, SOCK_STREAM, 0)
684 if fd < 0 {
685 *out_verdict = NX_HTTP_C_VERDICT_SOCKET_FAIL
686 return -1
687 }
688 // BOUNDED. This exact line is why nx_torrent_get had to double-fork its tracker announce:
689 // its own comment read "nx_http_client has no connect timeout". It has one now, so the
690 // workaround is no longer load-bearing.
691 let cr: i64 = nx_connect_bounded(fd, addr_packed, 16, NX_HTTP_C_CONNECT_MS)
692 if cr < 0 {
693 sys_close(fd)
694 *out_verdict = NX_HTTP_C_VERDICT_CONNECT_FAIL
695 return -1
696 }
697 let req: *u8 = sys_mmap(NX_MAGIC_4096)
698 let req_len: i64 = nx_http_client_build_request(
699 path, path_len, host, host_len, req)
700 let wr: i64 = sys_write(fd, req, req_len)
701 if wr != req_len {
702 sys_close(fd)
703 *out_verdict = NX_HTTP_C_VERDICT_SEND_FAIL
704 return -1
705 }
706 let got: i64 = _drain(fd, out_buf, out_cap)
707 sys_close(fd)
708 *out_verdict = NX_HTTP_C_VERDICT_OK
709 return got
710}
711
712// Sealed-enum validity gate.
713func nx_http_c_verdict_is_valid(v: i64) -> i64 {
714 if v < 0 { return 0 }
715 if v >= NX_HTTP_C_VERDICT_N { return 0 }
716 return 1
717}
718
719const HC_HEADER_BAD_NAME: i64 = 0 - 40
720const HC_HEADER_BAD_VALUE: i64 = 0 - 41
721const HC_HEADER_OWNED_FIELD: i64 = 0 - 42
722
723// HTTP field-name grammar (RFC 9110 token); these byte values describe syntax, not resource budgets.
724func hc_header_token(c: i64) -> i64 {
725 if c>=48 && c<=57 { return 1 }
726 if c>=65 && c<=90 { return 1 }
727 if c>=97 && c<=122 { return 1 }
728 if c==33 || c==35 || c==36 || c==37 || c==38 || c==39 || c==42 || c==43 || c==45 || c==46 || c==94 || c==95 || c==96 || c==124 || c==126 { return 1 }
729 return 0
730}
731func hc_header_name_equal(p: *u8,n: i64,name: *u8) -> i64 {
732 var i: i64=0
733 while i<n {
734 if name[i]==(0 as u8) { return 0 }
735 var c: i64=p[i] as i64
736 if c>=HC_CH_UPPER_A && c<=HC_CH_UPPER_Z { c=c+HC_CH_CASE_DELTA }
737 if c!=(name[i] as i64) { return 0 }
738 i=i+1
739 }
740 return (name[n]==(0 as u8)) as i64
741}
742// Validate before allocating or emitting. Framing and authority belong to the request builder.
743func hc_header_line_size(p: *u8,n: i64) -> i64 {
744 if n<=0 || (p as i64)==0 { return HC_HEADER_BAD_NAME }
745 var colon: i64=0
746 while colon<n && p[colon]!=(58 as u8) {
747 if hc_header_token(p[colon] as i64)==0 { return HC_HEADER_BAD_NAME }
748 colon=colon+1
749 }
750 if colon==0 || colon==n { return HC_HEADER_BAD_NAME }
751 if hc_header_name_equal(p,colon,"host")==1 || hc_header_name_equal(p,colon,"content-length")==1 || hc_header_name_equal(p,colon,"transfer-encoding")==1 || hc_header_name_equal(p,colon,"connection")==1 || hc_header_name_equal(p,colon,"trailer")==1 || hc_header_name_equal(p,colon,"upgrade")==1 { return HC_HEADER_OWNED_FIELD }
752 var i: i64=colon+1
753 while i<n {
754 let c: i64=p[i] as i64
755 if c==127 || (c<32 && c!=9) { return HC_HEADER_BAD_VALUE }
756 i=i+1
757 }
758 return n+2
759}