nx_http_client.nx source
↩ module page · 400 lines · 16741 B
1// nx_http_client.nx -- sovereign HTTP/1.1 GET client.
2//
3// Phase 0 of the Nishi browser per docs/NISHI_BROWSER_ROADMAP.md.
4// Smallest possible NX-side HTTP client: TCP connect + send GET +
5// receive response. No libc. No curl. No fetch shim.
6//
7// What it does today:
8// - opens a TCP socket
9// - connects to a caller-provided IPv4 address + port
10// - sends "GET <path> HTTP/1.1\r\nHost: <host>\r\nConnection: close\r\n\r\n"
11// - reads the response until the server closes
12// - hands the caller the raw bytes (header + body)
13//
14// What it doesn't do yet:
15// - DNS lookup (caller passes a packed IPv4 + port). L20b queued.
16// - HTTP/1.1 keep-alive (we Connection: close to make the read-to-EOF
17// pattern correct).
18// - HTTP/2, HTTP/3, ALPN. Eventually.
19// - TLS. Phase 8.
20// - Redirects (302). Caller handles.
21//
22// genealogy_id: rfc_7230_http_1_1 + linux_socket_api + nishi_pages_v1_pages_nx
23// lineage_id: nishi_browser_http_client_q10
24
25// nx_safety_envelope:
26// intended_use: AUTO_APPLIED -- primitive-specific tuning queued
27// sil_target: SIL1
28// evidence: [bulk_applied_2026-05-16, see-file-comment-for-detail]
29// verdict: NOT_YET_EVALUATED
30
31import "nx_syscalls.nx"
32import "nx_connect.nx" // bounded connect: a raw sys_connect here hangs ~127s on a black-holed host
33import "nx_dec_emit.nx"
34import "nx_codec_caps.nx" // Accept/Accept-Encoding are DERIVED, never hand-written
35const NX_MAGIC_1024: i64 = 1024
36
37// Append the Accept + Accept-Encoding pair, derived from the codecs we
38// can actually decode. Returns the new offset.
39//
40// Every one of the three request builders below used to carry its own
41// copy of a hand-written header literal. Two facts in that literal were
42// false: it advertised `image/avif` with no AVIF codec in the tree, and
43// it sent `Accept-Encoding: identity` with the comment "because we don't
44// inflate gzip/deflate yet" -- while nx_gzip_inflate and nx_zlib_inflate
45// were both complete. A server that believed either answered with bytes
46// we could not read, and nothing errored: a blank image, or a compressed
47// body handed to an HTML parser.
48//
49// Routing all three through one call means the next codec we land makes
50// every builder honest in the same act. nx_codec_caps_gate proves the
51// emitted header equals the capability in both directions.
52func hc_put_caps(out: *u8, o: i64) -> i64 {
53 let cch: *u8 = sys_mmap(NX_MAGIC_1024)
54 var i: i64 = 0
55 var p: i64 = o
56 nx_codec_caps_headers(cch)
57 while cch[i] != (0 as u8) {
58 out[p] = cch[i]
59 p = p + 1
60 i = i + 1
61 }
62 return p
63}
64const NX_MAGIC_4096: i64 = 4096
65
66// Sealed verdict for an HTTP GET attempt.
67const NX_HTTP_C_VERDICT_UNKNOWN: i64 = 0
68const NX_HTTP_C_VERDICT_OK: i64 = 1
69const NX_HTTP_C_VERDICT_SOCKET_FAIL: i64 = 2
70const NX_HTTP_C_VERDICT_CONNECT_FAIL: i64 = 3
71const NX_HTTP_C_VERDICT_SEND_FAIL: i64 = 4
72const NX_HTTP_C_VERDICT_RECV_FAIL: i64 = 5
73const NX_HTTP_C_VERDICT_N: i64 = 6
74
75// Build the request line + headers into `out`. Returns the byte
76// length written. Caller must give a buffer of >= 4 KB.
77//
78// Format:
79// GET <path>\r\n (path bytes)
80// plus "HTTP/1.1\r\n"
81// Host: <host>\r\n (host bytes)
82// Connection: close\r\n\r\n
83// (terminating)
84//
85// Pre: path_bytes / host_bytes are caller-supplied; no allocation.
86// Connect budget for one HTTP request. Replaces the kernel's ~127s SYN-retry ceiling, which is
87// not a timeout anyone chose -- it is just what happens when nobody sets one.
88const NX_HTTP_C_CONNECT_MS: i64 = 6000
89
90func nx_http_client_build_request(
91 path: *u8, path_len: i64,
92 host: *u8, host_len: i64,
93 out: *u8
94) -> i64 {
95 var o: i64 = 0
96 // "GET "
97 out[o]=71;o=o+1; out[o]=69;o=o+1; out[o]=84;o=o+1; out[o]=32;o=o+1
98 // path
99 var pi: i64 = 0
100 while pi < path_len { out[o+pi] = path[pi]; pi = pi + 1 }
101 o = o + path_len
102 // " HTTP/1.1\r\n"
103 out[o]=32;o=o+1
104 out[o]=72;o=o+1; out[o]=84;o=o+1; out[o]=84;o=o+1; out[o]=80;o=o+1
105 out[o]=47;o=o+1; out[o]=49;o=o+1; out[o]=46;o=o+1; out[o]=49;o=o+1
106 out[o]=13;o=o+1; out[o]=10;o=o+1
107 // "Host: "
108 out[o]=72;o=o+1; out[o]=111;o=o+1; out[o]=115;o=o+1; out[o]=116;o=o+1
109 out[o]=58;o=o+1; out[o]=32;o=o+1
110 var hi: i64 = 0
111 while hi < host_len { out[o+hi] = host[hi]; hi = hi + 1 }
112 o = o + host_len
113 out[o]=13;o=o+1; out[o]=10;o=o+1
114 // Present as a real, polite browser ("like a person") so sites that 403 bots/anonymous clients serve
115 // their real content (operator 2026-06-23: access everywhere, human in presentation). Accept-Encoding:
116 // identity because we don't inflate gzip/deflate yet. The bulk crawler keeps its honest NishiBot identity
117 // via the SEPARATE nx_http_get_ua path; this builder serves the browser/HTTPS-GET+POST/torrent paths.
118 // HONEST non-browser identity. mangadex's API 400s ANY "Mozilla" (browser) UA on purpose -- it forces API
119 // clients to identify as an app, not a browser (curl's plain UA gets 200). Cloudflare-fronted image CDNs
120 // (nhentai) passed our TLS JA3 even with a mismatched UA, so the TLS fingerprint (not the UA) is what beats
121 // them. So: an honest app UA satisfies mangadex AND still rides the Chrome-JA3 for Cloudflare. (op 2026-07-03)
122 // The Nishi Browser IS a browser: present a real Chrome UA (matching our Chrome-JA3 TLS fingerprint) + a browser
123 // Accept set, so sites that 403/406 non-browser clients (jable/javxxx/cam sites) serve their real content. This
124 // is the browsing/HTTPS-GET path; the bulk crawler keeps its honest NishiBot identity on its SEPARATE path.
125 // (mangadex's API prefers an app UA -- if that host is re-added, give it a per-host override; browser is the default.)
126 let ua: *u8 = "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36\r\nAccept-Language: en-US,en;q=0.9\r\n" as *u8
127 var ui: i64 = 0
128 while ua[ui] != (0 as u8) { out[o] = ua[ui]; o = o + 1; ui = ui + 1 }
129 o = hc_put_caps(out, o)
130 // "Connection: close\r\n\r\n"
131 out[o]=67;o=o+1; out[o]=111;o=o+1; out[o]=110;o=o+1; out[o]=110;o=o+1
132 out[o]=101;o=o+1; out[o]=99;o=o+1; out[o]=116;o=o+1; out[o]=105;o=o+1
133 out[o]=111;o=o+1; out[o]=110;o=o+1
134 out[o]=58;o=o+1; out[o]=32;o=o+1
135 out[o]=99;o=o+1; out[o]=108;o=o+1; out[o]=111;o=o+1; out[o]=115;o=o+1
136 out[o]=101;o=o+1
137 out[o]=13;o=o+1; out[o]=10;o=o+1
138 out[o]=13;o=o+1; out[o]=10;o=o+1
139 return o
140}
141
142// Generic extra-header variant (ADDITIVE, base kept byte-identical per Cardinal 19): injects caller-supplied
143// header line(s) `xhdr` (each CRLF-terminated, NO trailing blank line) right after the UA/Accept block and
144// before Connection: close. Powers the sovereign authed GitHub-oracle fetch (Authorization: token <PAT>) with
145// NO curl and NO graphql. cookie/POST variants queued to fold into this (DRY consolidation rung).
146func nx_http_client_build_request_xhdr(
147 path: *u8, path_len: i64,
148 host: *u8, host_len: i64,
149 xhdr: *u8, xhdr_len: i64,
150 out: *u8
151) -> i64 {
152 var o: i64 = 0
153 out[o]=71;o=o+1; out[o]=69;o=o+1; out[o]=84;o=o+1; out[o]=32;o=o+1
154 var pi: i64 = 0
155 while pi < path_len { out[o+pi] = path[pi]; pi = pi + 1 }
156 o = o + path_len
157 out[o]=32;o=o+1
158 out[o]=72;o=o+1; out[o]=84;o=o+1; out[o]=84;o=o+1; out[o]=80;o=o+1
159 out[o]=47;o=o+1; out[o]=49;o=o+1; out[o]=46;o=o+1; out[o]=49;o=o+1
160 out[o]=13;o=o+1; out[o]=10;o=o+1
161 out[o]=72;o=o+1; out[o]=111;o=o+1; out[o]=115;o=o+1; out[o]=116;o=o+1
162 out[o]=58;o=o+1; out[o]=32;o=o+1
163 var hi: i64 = 0
164 while hi < host_len { out[o+hi] = host[hi]; hi = hi + 1 }
165 o = o + host_len
166 out[o]=13;o=o+1; out[o]=10;o=o+1
167 let ua: *u8 = "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36\r\nAccept-Language: en-US,en;q=0.9\r\n" as *u8
168 var ui: i64 = 0
169 while ua[ui] != (0 as u8) { out[o] = ua[ui]; o = o + 1; ui = ui + 1 }
170 o = hc_put_caps(out, o)
171 var xi: i64 = 0
172 while xi < xhdr_len { out[o] = xhdr[xi]; o = o + 1; xi = xi + 1 }
173 out[o]=67;o=o+1; out[o]=111;o=o+1; out[o]=110;o=o+1; out[o]=110;o=o+1
174 out[o]=101;o=o+1; out[o]=99;o=o+1; out[o]=116;o=o+1; out[o]=105;o=o+1
175 out[o]=111;o=o+1; out[o]=110;o=o+1
176 out[o]=58;o=o+1; out[o]=32;o=o+1
177 out[o]=99;o=o+1; out[o]=108;o=o+1; out[o]=111;o=o+1; out[o]=115;o=o+1
178 out[o]=101;o=o+1
179 out[o]=13;o=o+1; out[o]=10;o=o+1
180 out[o]=13;o=o+1; out[o]=10;o=o+1
181 return o
182}
183
184// Cookie-aware variant: identical to nx_http_client_build_request but injects a
185// "Cookie: <cookie>\r\n" header when cookie_len>0, so a redirect-following fetch can carry a
186// session across hops (browser-faithful). E.g. chaturbate's /?next= age-gate hands back
187// csrftoken/sbr/AG_Key that must be replayed to reach the room -- without a jar we land on the
188// gate page forever. Base builder kept byte-identical (Cardinal 19); this is purely additive.
189// `cookie` = a jar buffer "name=value; name2=value2" (NO CRLF, caller-terminated by cookie_len).
190func nx_http_client_build_request_cookie(
191 path: *u8, path_len: i64,
192 host: *u8, host_len: i64,
193 cookie: *u8, cookie_len: i64,
194 out: *u8
195) -> i64 {
196 var o: i64 = 0
197 out[o]=71;o=o+1; out[o]=69;o=o+1; out[o]=84;o=o+1; out[o]=32;o=o+1
198 var pi: i64 = 0
199 while pi < path_len { out[o+pi] = path[pi]; pi = pi + 1 }
200 o = o + path_len
201 out[o]=32;o=o+1
202 out[o]=72;o=o+1; out[o]=84;o=o+1; out[o]=84;o=o+1; out[o]=80;o=o+1
203 out[o]=47;o=o+1; out[o]=49;o=o+1; out[o]=46;o=o+1; out[o]=49;o=o+1
204 out[o]=13;o=o+1; out[o]=10;o=o+1
205 out[o]=72;o=o+1; out[o]=111;o=o+1; out[o]=115;o=o+1; out[o]=116;o=o+1
206 out[o]=58;o=o+1; out[o]=32;o=o+1
207 var hi: i64 = 0
208 while hi < host_len { out[o+hi] = host[hi]; hi = hi + 1 }
209 o = o + host_len
210 out[o]=13;o=o+1; out[o]=10;o=o+1
211 let ua: *u8 = "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36\r\nAccept-Language: en-US,en;q=0.9\r\n" as *u8
212 var ui: i64 = 0
213 while ua[ui] != (0 as u8) { out[o] = ua[ui]; o = o + 1; ui = ui + 1 }
214 o = hc_put_caps(out, o)
215 if cookie_len > 0 {
216 let ckh: *u8 = "Cookie: " as *u8
217 var ci: i64 = 0
218 while ckh[ci] != (0 as u8) { out[o] = ckh[ci]; o = o + 1; ci = ci + 1 }
219 var cj: i64 = 0
220 while cj < cookie_len { out[o] = cookie[cj]; o = o + 1; cj = cj + 1 }
221 out[o]=13;o=o+1; out[o]=10;o=o+1
222 }
223 out[o]=67;o=o+1; out[o]=111;o=o+1; out[o]=110;o=o+1; out[o]=110;o=o+1
224 out[o]=101;o=o+1; out[o]=99;o=o+1; out[o]=116;o=o+1; out[o]=105;o=o+1
225 out[o]=111;o=o+1; out[o]=110;o=o+1
226 out[o]=58;o=o+1; out[o]=32;o=o+1
227 out[o]=99;o=o+1; out[o]=108;o=o+1; out[o]=111;o=o+1; out[o]=115;o=o+1
228 out[o]=101;o=o+1
229 out[o]=13;o=o+1; out[o]=10;o=o+1
230 out[o]=13;o=o+1; out[o]=10;o=o+1
231 return o
232}
233
234// Build an HTTP/1.1 POST request line + headers + body into `out`.
235// Returns the byte length written. Caller must give a buffer of
236// >= 4 KB + body_len.
237//
238// Format:
239// POST <path> HTTP/1.1\r\n
240// Host: <host>\r\n
241// Content-Type: <content_type>\r\n
242// Content-Length: <body_len>\r\n
243// Connection: close\r\n
244// \r\n
245// <body bytes>
246//
247// Per RFC 7230 ยง3.3.2 Content-Length is the canonical body delimiter
248// for client-side requests with known length. Composes
249// nx_dec_emit_u63 for the length emission (cardinal: no inline
250// decimal emitters).
251func nx_http_client_build_request_post(
252 path: *u8, path_len: i64,
253 host: *u8, host_len: i64,
254 content_type: *u8, content_type_len: i64,
255 body: *u8, body_len: i64,
256 out: *u8
257) -> i64 {
258 var o: i64 = 0
259 // "POST "
260 out[o]=80;o=o+1; out[o]=79;o=o+1; out[o]=83;o=o+1; out[o]=84;o=o+1
261 out[o]=32;o=o+1
262 // path
263 var pi: i64 = 0
264 while pi < path_len { out[o+pi] = path[pi]; pi = pi + 1 }
265 o = o + path_len
266 // " HTTP/1.1\r\n"
267 out[o]=32;o=o+1
268 out[o]=72;o=o+1; out[o]=84;o=o+1; out[o]=84;o=o+1; out[o]=80;o=o+1
269 out[o]=47;o=o+1; out[o]=49;o=o+1; out[o]=46;o=o+1; out[o]=49;o=o+1
270 out[o]=13;o=o+1; out[o]=10;o=o+1
271 // "Host: "
272 out[o]=72;o=o+1; out[o]=111;o=o+1; out[o]=115;o=o+1; out[o]=116;o=o+1
273 out[o]=58;o=o+1; out[o]=32;o=o+1
274 var hi: i64 = 0
275 while hi < host_len { out[o+hi] = host[hi]; hi = hi + 1 }
276 o = o + host_len
277 out[o]=13;o=o+1; out[o]=10;o=o+1
278 // "Content-Type: "
279 out[o]=67;o=o+1; out[o]=111;o=o+1; out[o]=110;o=o+1; out[o]=116;o=o+1
280 out[o]=101;o=o+1; out[o]=110;o=o+1; out[o]=116;o=o+1; out[o]=45;o=o+1
281 out[o]=84;o=o+1; out[o]=121;o=o+1; out[o]=112;o=o+1; out[o]=101;o=o+1
282 out[o]=58;o=o+1; out[o]=32;o=o+1
283 var ci: i64 = 0
284 while ci < content_type_len { out[o+ci] = content_type[ci]; ci = ci + 1 }
285 o = o + content_type_len
286 out[o]=13;o=o+1; out[o]=10;o=o+1
287 // "Content-Length: "
288 out[o]=67;o=o+1; out[o]=111;o=o+1; out[o]=110;o=o+1; out[o]=116;o=o+1
289 out[o]=101;o=o+1; out[o]=110;o=o+1; out[o]=116;o=o+1; out[o]=45;o=o+1
290 out[o]=76;o=o+1; out[o]=101;o=o+1; out[o]=110;o=o+1; out[o]=103;o=o+1
291 out[o]=116;o=o+1; out[o]=104;o=o+1; out[o]=58;o=o+1; out[o]=32;o=o+1
292 let dlen: i64 = nx_dec_emit_u63(out, o, body_len)
293 o = o + dlen
294 out[o]=13;o=o+1; out[o]=10;o=o+1
295 // "Connection: close\r\n"
296 out[o]=67;o=o+1; out[o]=111;o=o+1; out[o]=110;o=o+1; out[o]=110;o=o+1
297 out[o]=101;o=o+1; out[o]=99;o=o+1; out[o]=116;o=o+1; out[o]=105;o=o+1
298 out[o]=111;o=o+1; out[o]=110;o=o+1
299 out[o]=58;o=o+1; out[o]=32;o=o+1
300 out[o]=99;o=o+1; out[o]=108;o=o+1; out[o]=111;o=o+1; out[o]=115;o=o+1
301 out[o]=101;o=o+1
302 out[o]=13;o=o+1; out[o]=10;o=o+1
303 // header terminator "\r\n"
304 out[o]=13;o=o+1; out[o]=10;o=o+1
305 // body bytes
306 var bi: i64 = 0
307 while bi < body_len { out[o+bi] = body[bi]; bi = bi + 1 }
308 o = o + body_len
309 return o
310}
311
312// Build a packed sockaddr_in for IPv4 dotted-quad `a.b.c.d` at port.
313// `out` must point at >= 16 bytes (struct sockaddr_in size).
314//
315// Linux sockaddr_in layout:
316// [0..1] sin_family = AF_INET = 2 (little-endian short)
317// [2..3] sin_port = port in BIG-endian
318// [4..7] sin_addr = IPv4 bytes a.b.c.d in network order
319// [8..15] padding zero
320func nx_http_client_sockaddr_ipv4(
321 out: *u8,
322 a: i64, b: i64, c: i64, d: i64,
323 port: i64
324) -> i64 {
325 out[0] = 2; out[1] = 0
326 out[2] = (port >> 8) & 0xff
327 out[3] = port & 0xff
328 out[4] = a
329 out[5] = b
330 out[6] = c
331 out[7] = d
332 out[8] = 0; out[9] = 0; out[10] = 0; out[11] = 0
333 out[12] = 0; out[13] = 0; out[14] = 0; out[15] = 0
334 return 16
335}
336
337// Read all available bytes from `fd` into `buf` (cap bytes max) until
338// the peer closes the connection (sys_read returns 0). Returns bytes
339// read.
340func _drain(fd: i64, buf: *u8, cap: i64) -> i64 {
341 var off: i64 = 0
342 var keep: i64 = 1
343 while keep == 1 {
344 if off >= cap { keep = 0 }
345 else {
346 let r: i64 = sys_read(fd, (buf as i64 + off) as *u8, cap - off)
347 if r <= 0 { keep = 0 }
348 else { off = off + r }
349 }
350 }
351 return off
352}
353
354// Top-level: connect, send a GET, read the full response. Returns
355// bytes read (-1 on socket error) and writes the verdict through
356// out_verdict. Response (headers + body) is in out_buf[0..ret).
357func nx_http_client_get(
358 addr_packed: *u8,
359 path: *u8, path_len: i64,
360 host: *u8, host_len: i64,
361 out_buf: *u8, out_cap: i64,
362 out_verdict: *i64
363) -> i64 {
364 *out_verdict = NX_HTTP_C_VERDICT_UNKNOWN
365
366 let fd: i64 = sys_socket(AF_INET, SOCK_STREAM, 0)
367 if fd < 0 {
368 *out_verdict = NX_HTTP_C_VERDICT_SOCKET_FAIL
369 return -1
370 }
371 // BOUNDED. This exact line is why nx_torrent_get had to double-fork its tracker announce:
372 // its own comment read "nx_http_client has no connect timeout". It has one now, so the
373 // workaround is no longer load-bearing.
374 let cr: i64 = nx_connect_bounded(fd, addr_packed, 16, NX_HTTP_C_CONNECT_MS)
375 if cr < 0 {
376 sys_close(fd)
377 *out_verdict = NX_HTTP_C_VERDICT_CONNECT_FAIL
378 return -1
379 }
380 let req: *u8 = sys_mmap(NX_MAGIC_4096)
381 let req_len: i64 = nx_http_client_build_request(
382 path, path_len, host, host_len, req)
383 let wr: i64 = sys_write(fd, req, req_len)
384 if wr != req_len {
385 sys_close(fd)
386 *out_verdict = NX_HTTP_C_VERDICT_SEND_FAIL
387 return -1
388 }
389 let got: i64 = _drain(fd, out_buf, out_cap)
390 sys_close(fd)
391 *out_verdict = NX_HTTP_C_VERDICT_OK
392 return got
393}
394
395// Sealed-enum validity gate.
396func nx_http_c_verdict_is_valid(v: i64) -> i64 {
397 if v < 0 { return 0 }
398 if v >= NX_HTTP_C_VERDICT_N { return 0 }
399 return 1
400}