nx_https_get.nx source
↩ module page · 228 lines · 9605 B
1// nx_https_get.nx -- THE TOP-LEVEL USER-FACING HTTPS FETCH PRIMITIVE.
2//
3// This is the single call that wraps the 4 wiring-arc steps into
4// one user-facing primitive. Takes a URL string + entropy +
5// trust store + clock + output buffer, returns the response bytes
6// (HTTP headers + body) accumulated until peer-close.
7//
8// Step 5 of the nx_https_client wiring arc -- the convenience
9// wrapper that turns 4 discrete shipped primitives into a
10// one-liner the rest of the substrate (Browser arc / NX-INGEST /
11// etc.) calls.
12//
13// Composes 4 shipped substrate primitives:
14// 1. nx_https_url_for_fetch -- URL parse + scheme + port
15// 2. nx_https_url_connect -- URL+DNS+TCP → fd
16// 3. nx_tls13_client_session_run -- full TLS 1.3 handshake
17// 4. nx_https_get_complete -- HTTP GET round-trip
18//
19// Plus internal: builds TlsValidationContext from (store, host
20// from parsed URL, now_epoch); calls sys_close on fd to release
21// the socket after response received.
22//
23// Public API:
24// nx_https_get(
25// url_str: *u8, NUL-terminated URL
26// client_random: *u8, 32 bytes of entropy
27// x25519_priv: *u8, 32 bytes ephemeral key
28// store: *TrustStore, boot-loaded CA bundle
29// now_epoch_secs: i64, for cert validity check
30// out_buf: *u8, out_cap: i64
31// ) -> POSITIVE bytes_received | NEGATIVE -NX_HTTPS_GET_* code
32// nx_https_get_verdict_is_valid(v) -> 0|1
33//
34// Sealed verdict (one per failure mode, mapped from sub-step
35// verdicts):
36// NX_HTTPS_GET_OK
37// NX_HTTPS_GET_BAD_URL url_for_fetch returned non-OK
38// NX_HTTPS_GET_CONNECT_FAIL url_connect returned non-OK
39// NX_HTTPS_GET_HANDSHAKE_FAIL session_run returned negative
40// NX_HTTPS_GET_FETCH_FAIL get_complete returned negative
41//
42// Caller responsibility:
43// - Supply 32 bytes of REAL entropy for client_random (read
44// from /dev/urandom via sys_openat + sys_read, OR from a
45// hardware RNG; do NOT use a deterministic PRNG for production)
46// - Supply 32 bytes of REAL entropy for x25519_priv (same)
47// - Pre-populate the TrustStore with trusted CAs (boot-time via
48// nx_x509_trust_store_load + nx_nss_certdata_parse on a real
49// Mozilla certdata.txt drop)
50// - now_epoch_secs from a real wall-clock source
51// - out_cap sized for the expected response (small static HTML =
52// ~10KB; larger pages need bigger buffer)
53//
54// Per Cardinals 9 (single-responsibility -- THE convenience
55// wrapper, no new logic), 12 (defensive at boundaries -- fd is
56// closed on every code path), 19 (composes shipped primitives
57// unchanged), 22 (composition -- 4 substrate primitives compose
58// into one user-facing call), 23 (preamble names every caller
59// responsibility item).
60//
61// license_tier: INDEPENDENT_REDERIVE
62// genealogy_id: international-research-sources/ietf/rfc_8446 + rfc_9112 + rfc_3986
63// lineage_id: nishi_https_get_q10
64
65// nx_safety_envelope:
66// intended_use: AUTO_APPLIED -- primitive-specific tuning queued
67// sil_target: SIL1
68// evidence: [bulk_applied_2026-05-19, https-get-user-facing-step-5]
69// verdict: NOT_YET_EVALUATED
70
71import "nx_syscalls.nx"
72import "nx_url.nx"
73import "nx_x509_trust_store.nx"
74import "nx_tls13_client_validate_certificate.nx"
75import "nx_tls13_client_session.nx"
76import "nx_tls13_client_session_run.nx"
77import "nx_tls13_chrome_session.nx" // Chrome-JA3 ClientHello: anti-bot CDNs (Cloudflare-class) complete the handshake instead of RST-ing our minimal hello; inherits the recv_hs cross-record reassembly fix
78import "nx_https_url_for_fetch.nx"
79import "nx_https_url_connect.nx"
80import "nx_https_get_complete.nx"
81
82const NX_HTTPS_GET_OK: i64 = 1
83const NX_HTTPS_GET_BAD_URL: i64 = 2
84const NX_HTTPS_GET_CONNECT_FAIL: i64 = 3
85const NX_HTTPS_GET_HANDSHAKE_FAIL: i64 = 4
86const NX_HTTPS_GET_FETCH_FAIL: i64 = 5
87const NX_HTTPS_GET_VERDICT_N: i64 = 6
88
89func nx_https_get_verdict_is_valid(v: i64) -> i64 {
90 if v < NX_HTTPS_GET_OK { return 0 }
91 if v >= NX_HTTPS_GET_VERDICT_N { return 0 }
92 return 1
93}
94
95// THE FETCH.
96// native sovereign perf helper: write a base-10 integer to stderr (fd 2).
97func _hg_pn(v: i64) -> i64 {
98 let b: *u8 = sys_mmap(24)
99 var n: i64 = v
100 if n < 0 { n = 0 - n }
101 var i: i64 = 22
102 if n == 0 { b[i] = 0x30 as u8; i = i - 1 }
103 else { while n > 0 { b[i] = (0x30 + (n - (n/10)*10)) as u8; n = n / 10; i = i - 1 } }
104 sys_write(2, ((b as i64) + i + 1) as *u8, 22 - i)
105 return 0
106}
107
108func nx_https_get(
109 url_str: *u8,
110 client_random: *u8,
111 x25519_priv: *u8,
112 store: *TrustStore,
113 now_epoch_secs: i64,
114 out_buf: *u8, out_cap: i64
115) -> i64 {
116 let t_a: i64 = sys_now_ms()
117 // ---- Step 1: parse URL + validate scheme=https ----
118 let url_p: *NxUrl = nx_url_new()
119 let target_raw: *u8 = sys_mmap(32)
120 let target: *NxHttpsTarget = target_raw as *NxHttpsTarget
121 target.url = url_p
122 target.port = 0
123 let url_v: i64 = nx_https_url_for_fetch(url_str, target)
124 if url_v != NX_HTTPS_URL_OK { return 0 - NX_HTTPS_GET_BAD_URL }
125
126 // ---- Step 2: DNS resolve + TCP connect ----
127 let fd_p: *i64 = sys_mmap(16) as *i64
128 let conn_v: i64 = nx_https_url_connect(target, url_str, now_epoch_secs, fd_p)
129 if conn_v != NX_HTTPS_CONNECT_OK { return 0 - NX_HTTPS_GET_CONNECT_FAIL }
130 var fd: i64 = *fd_p // var: the handshake fallback below RECONNECTS on a fresh fd
131 let t_b: i64 = sys_now_ms()
132
133 // ---- Build TlsValidationContext (SNI = host from URL) ----
134 let val_ctx_raw: *u8 = sys_mmap(64)
135 let val_ctx: *TlsValidationContext = val_ctx_raw as *TlsValidationContext
136 val_ctx.store = store
137 val_ctx.sni_host = url_str + target.url.host_off
138 val_ctx.sni_host_len = target.url.host_len
139 val_ctx.now_epoch = now_epoch_secs
140
141 // ---- Step 3: TLS 1.3 handshake, CHROME-JA3 FIRST then STANDARD FALLBACK -------------
142 // The Chrome-fingerprint hello exists because anti-bot CDNs RST our minimal hello. But
143 // swapping to it UNCONDITIONALLY traded one failure set for another: MEASURED 2026-07-27,
144 // bing / duckduckgo / news.ycombinator.com all failed the fetch at verdict=5 while the
145 // ServerHello probe showed their handshakes parse PERFECTLY under the standard hello
146 // (HR=0, inner recv_sh RV=1, same as the example.com/wikipedia controls). One hello does
147 // not fit every server, so try both instead of picking a side.
148 // A failed handshake leaves the socket unusable, so the retry RECONNECTS -- a fresh fd is
149 // part of the retry, not an optimisation. Bounded: exactly two attempts, no loop.
150 var session_r: i64 = nx_tls13_client_session_run_chrome(
151 fd,
152 url_str + target.url.host_off, target.url.host_len,
153 client_random, x25519_priv,
154 val_ctx
155 )
156 if session_r < 0 {
157 sys_close(fd)
158 let fd2_p: *i64 = sys_mmap(16) as *i64
159 let conn2: i64 = nx_https_url_connect(target, url_str, now_epoch_secs, fd2_p)
160 if conn2 != NX_HTTPS_CONNECT_OK { return 0 - NX_HTTPS_GET_CONNECT_FAIL }
161 fd = *fd2_p
162 session_r = nx_tls13_client_session_run(
163 fd,
164 url_str + target.url.host_off, target.url.host_len,
165 client_random, x25519_priv,
166 val_ctx
167 )
168 if session_r < 0 {
169 sys_close(fd)
170 return 0 - NX_HTTPS_GET_HANDSHAKE_FAIL
171 }
172 }
173 let session: *Tls13ClientSession = session_r as *Tls13ClientSession
174 let t_c: i64 = sys_now_ms()
175
176 // ---- Step 4: HTTP GET round-trip ----
177 // If URL had no path, default to "/"
178 var path_off: i64 = target.url.path_off
179 var path_len: i64 = target.url.path_len
180 let default_path: *u8 = sys_mmap(2)
181 default_path[0] = 0x2F // '/'
182 var path_ptr: *u8 = url_str + path_off
183 if path_len == 0 {
184 path_ptr = default_path
185 path_len = 1
186 }
187
188 // The URL parser (nx_url.nx) splits the query string off the path, but the HTTP request
189 // line needs "path?query" VERBATIM. Without this, GET /sparql?query=... was sent as
190 // GET /sparql (query SILENTLY DROPPED), breaking every query-string API -- e.g. Wikidata
191 // SPARQL. Guarded on query_len>0, so plain fetches (no query) are byte-identical = no regression.
192 if target.url.query_len > 0 {
193 let full: *u8 = sys_mmap(path_len + target.url.query_len + 4)
194 var fo: i64 = 0
195 var pci: i64 = 0
196 while pci < path_len { full[fo] = path_ptr[pci]; fo = fo + 1; pci = pci + 1 }
197 full[fo] = 0x3F as u8
198 fo = fo + 1
199 let qp: *u8 = url_str + target.url.query_off
200 var qci: i64 = 0
201 while qci < target.url.query_len { full[fo] = qp[qci]; fo = fo + 1; qci = qci + 1 }
202 path_ptr = full
203 path_len = fo
204 }
205
206 let n: i64 = nx_https_get_complete(
207 session, fd,
208 path_ptr, path_len,
209 url_str + target.url.host_off, target.url.host_len,
210 out_buf, out_cap
211 )
212 sys_close(fd)
213 if n < 0 { return 0 - NX_HTTPS_GET_FETCH_FAIL }
214 let t_d: i64 = sys_now_ms()
215
216 // native sovereign per-fetch network breakdown (stderr): where the connect time goes.
217 sys_write(2, "nishi-net dns+tcp=" as *u8, 18); _hg_pn(t_b - t_a)
218 sys_write(2, "ms tls=" as *u8, 7); _hg_pn(t_c - t_b)
219 sys_write(2, "ms http=" as *u8, 8); _hg_pn(t_d - t_c)
220 sys_write(2, "ms\n" as *u8, 3)
221
222 return n
223}
224
225// Compile-only smoke. Real KAT in nx_https_get_test.nx.
226func main() -> i64 {
227 return 0
228}