nx_https_client.nx source
↩ module page · 349 lines · 16813 B
1// nx_https_client.nx -- HTTPS convenience wrapper (TLS+DNS glue).
2//
3// module: nishi-core.net.https_client
4// depends: nishi-core.net.http_client, nishi-core.crypto.tls13_client,
5// nishi-core.net.dns, nishi-core.io.url, nishi-core.io.syscalls
6// disk_kb: 5
7// capability: CORE_NET
8//
9// license_tier: PUBLIC_NISHI_SUBSTRATE
10// genealogy_id: rfc_7230_http_1_1 + rfc_8446_tls_1_3 + rfc_1035_dns +
11// rfc_3986_uri_generic +
12// nishi_substrate_phase_8_tls_http_dns_wiring_2026
13//
14// The "Phase 8" substrate wiring per nx_http_client.nx's own header
15// note: glue layer composing existing shipped substrate (nx_http_client
16// HTTP/1.1 + nx_tls13_client TLS 1.3 + nx_dns DNS resolution + nx_url
17// URL parsing) into a single HTTPS GET entry-point.
18//
19// This is the SUBSTRATE PHASE 8 UNBLOCK that every NX-INGEST adapter
20// has been honest-stubbing against. Once this primitive ships, the
21// honest-stub fetch lines in all 10 agronomic adapters can swap to
22// real nx_https_get() calls with NO caller-side API change per
23// Cardinal 19 (API contract stability).
24//
25// ===== Architecture ===============================================
26//
27// Caller invokes nx_https_get(url_ptr, url_len, out_buf, out_cap):
28//
29// 1. Parse URL into (scheme, host, port, path, query) via nx_url
30// 2. Resolve host → IPv4 via nx_dns_resolve
31// 3. TCP connect to (IPv4, port) via nx_socket
32// 4. TLS 1.3 handshake via nx_tls13_client_handshake
33// 5. Inside the TLS session, send HTTP/1.1 GET via nx_http_client
34// 6. Read TLS-wrapped HTTP response
35// 7. Close TLS session + TCP socket cleanly
36// 8. Return response bytes + verdict to caller
37//
38// All steps are bits-up NishiLang substrate composition. Zero
39// third-party deps (no libcurl, no OpenSSL, no Go net/http).
40
41// nx_safety_envelope:
42// intended_use: AUTO_APPLIED -- primitive-specific tuning queued
43// sil_target: SIL1
44// evidence: [bulk_applied_2026-05-16, see-file-comment-for-detail]
45// verdict: NOT_YET_EVALUATED
46
47import "nx_syscalls.nx"
48import "nx_http_client.nx"
49import "nx_url.nx"
50
51// ===== Verdict ====================================================
52
53const NX_HTTPS_OK: i64 = 1
54const NX_HTTPS_URL_PARSE_FAIL: i64 = 2
55const NX_HTTPS_DNS_FAIL: i64 = 3
56const NX_HTTPS_CONNECT_FAIL: i64 = 4
57const NX_HTTPS_TLS_HANDSHAKE_FAIL: i64 = 5
58const NX_HTTPS_TLS_CERT_VERIFY_FAIL: i64 = 6
59const NX_HTTPS_SEND_FAIL: i64 = 7
60const NX_HTTPS_RECV_FAIL: i64 = 8
61const NX_HTTPS_RESPONSE_TOO_LARGE: i64 = 9
62const NX_HTTPS_TIMEOUT: i64 = 10
63const NX_HTTPS_REDIRECT_LIMIT: i64 = 11 // too many redirects (caller handles)
64// NO REQUEST WAS PERFORMED. Added 2026-08-06 so this file's scaffolding can REFUSE instead of
65// reporting success -- see the note in nx_https_get. Distinct from every failure above it: those
66// mean "we tried and it failed", this means "we never tried".
67const NX_HTTPS_NOT_IMPLEMENTED: i64 = 12
68
69func nx_https_verdict_name(v: i64) -> *u8 {
70 if v == NX_HTTPS_OK { return "OK" }
71 if v == NX_HTTPS_URL_PARSE_FAIL { return "URL_PARSE_FAIL" }
72 if v == NX_HTTPS_DNS_FAIL { return "DNS_FAIL" }
73 if v == NX_HTTPS_CONNECT_FAIL { return "CONNECT_FAIL" }
74 if v == NX_HTTPS_TLS_HANDSHAKE_FAIL { return "TLS_HANDSHAKE_FAIL" }
75 if v == NX_HTTPS_TLS_CERT_VERIFY_FAIL { return "TLS_CERT_VERIFY_FAIL" }
76 if v == NX_HTTPS_SEND_FAIL { return "SEND_FAIL" }
77 if v == NX_HTTPS_RECV_FAIL { return "RECV_FAIL" }
78 if v == NX_HTTPS_RESPONSE_TOO_LARGE { return "RESPONSE_TOO_LARGE" }
79 if v == NX_HTTPS_TIMEOUT { return "TIMEOUT" }
80 if v == NX_HTTPS_REDIRECT_LIMIT { return "REDIRECT_LIMIT" }
81 return "UNKNOWN"
82}
83
84// ===== HttpsResponse struct =======================================
85
86struct HttpsResponse {
87 response_hk: i64,
88 status_code: i64, // HTTP status (200, 404, 429, 500, etc.)
89 headers_ptr: *u8, // raw header block (caller parses)
90 headers_len: i64,
91 body_ptr: *u8,
92 body_len: i64,
93 bytes_total: i64, // header + body
94 tls_session_resumed: i64, // 1 if TLS resumed (PSK)
95 elapsed_ms: i64,
96 verdict: i64, // NX_HTTPS_*
97 requested_url_ptr: *u8,
98 final_url_ptr: *u8, // post-redirect URL (if any)
99 n_redirects: i64,
100}
101
102const NX_HTTPS_RESPONSE_BYTES: i64 = 104 // 13 fields * 8 bytes
103
104// ===== Defaults ===================================================
105
106const NX_HTTPS_DEFAULT_MAX_RESPONSE_BYTES: i64 = 67108864 // 64 MB cap
107const NX_HTTPS_DEFAULT_TIMEOUT_SECONDS: i64 = 60
108const NX_HTTPS_MAX_REDIRECTS: i64 = 5
109
110// ===== Main entry-point ===========================================
111//
112// nx_https_get(url, out_buf, out_cap, now_unix) → *HttpsResponse
113//
114// Composes the substrate's existing primitives in sequence. Each
115// step has its own verdict; this primitive maps them onto the
116// HttpsVerdict + populates HttpsResponse.
117
118func nx_https_get(
119 url_ptr: *u8,
120 url_len: i64,
121 out_buf: *u8,
122 out_cap: i64,
123 timeout_seconds: i64,
124 now_unix: i64
125) -> *HttpsResponse {
126 let raw: *u8 = sys_mmap(NX_HTTPS_RESPONSE_BYTES)
127 let r: *HttpsResponse = raw as *HttpsResponse
128 r.response_hk = 0
129 r.status_code = 0
130 r.headers_ptr = 0 as *u8
131 r.headers_len = 0
132 r.body_ptr = 0 as *u8
133 r.body_len = 0
134 r.bytes_total = 0
135 r.tls_session_resumed = 0
136 r.elapsed_ms = 0
137 // REFUSE -- DO NOT REPORT SUCCESS. Every step below this line is still commented-out scaffolding
138 // (STEP 1 onward), so this function performs NO REQUEST AT ALL -- and it used to return
139 // NX_HTTPS_OK with status_code=0, body_len=0. A caller checking `verdict == NX_HTTPS_OK`, which is
140 // the entire purpose of a verdict, would conclude the fetch SUCCEEDED and the server returned an
141 // empty document. The two convenience wrappers below inherit this and are worse still:
142 // nx_https_post_json accepts json_body/json_len and DISCARDS them (it calls the GET path), and
143 // nx_https_get_with_api_key accepts a key and DISCARDS it -- both then reporting OK. A caller
144 // would POST data, be told OK, and have sent nothing.
145 // MEASURED 2026-08-06: this file has ZERO live callers -- the nx_acme/nx_acme_es256 references to
146 // it are COMMENTS naming it as a future dependency, not imports -- so this is a LATENT TRAP, not
147 // live damage. The first person to wire it would have inherited a silent success.
148 // ★A STUB THAT RETURNS THE SUCCESS VERDICT IS NOT A STUB, IT IS A FALSE WITNESS. The file did say
149 // "honest stub" in a comment -- but honesty that lives in a comment is invisible to every caller.
150 // It has to live in the RETURN VALUE, which is the only thing code acts on.
151 // Whoever lands the real implementation: delete this line. The working primitives already exist
152 // and are proven in production -- nx_https_fetch_lib (hf_fetch) and nx_https_post_complete, the
153 // latter already used by nx_acme_http and nx_funcheck.
154 r.verdict = NX_HTTPS_NOT_IMPLEMENTED
155 r.requested_url_ptr = url_ptr
156 r.final_url_ptr = url_ptr
157 r.n_redirects = 0
158
159 // STEP 1: Parse URL
160 // let url: *NxUrl = nx_url_parse(url_ptr, url_len)
161 // if url.scheme != "https" → mark verdict + return
162 // Extract: host_ptr, host_len, port (default 443), path_ptr,
163 // path_len, query_ptr, query_len
164 //
165 // STEP 2: DNS resolve
166 // let ipv4: i64 = nx_dns_resolve_a_record(host_ptr, host_len)
167 // if ipv4 == 0 → NX_HTTPS_DNS_FAIL
168 //
169 // STEP 3: TCP connect
170 // let socket_fd: i64 = nx_socket_tcp_connect(ipv4, port)
171 // if socket_fd < 0 → NX_HTTPS_CONNECT_FAIL
172 //
173 // STEP 4: TLS 1.3 handshake (sovereign substrate; no OpenSSL)
174 // let tls_session: *NxTls13Session = nx_tls13_client_handshake(
175 // socket_fd, host_ptr, host_len)
176 // if tls_session.verdict != HANDSHAKE_OK → NX_HTTPS_TLS_HANDSHAKE_FAIL
177 // if tls_session.cert_verify_verdict != OK → NX_HTTPS_TLS_CERT_VERIFY_FAIL
178 //
179 // STEP 5: Build HTTP/1.1 GET request
180 // let req_buf: *u8 = sys_mmap(4096)
181 // let req_len: i64 = nx_http_client_build_request(
182 // path_ptr, path_len, host_ptr, host_len, req_buf)
183 // (caller may append query string + extra headers like
184 // Authorization / User-Agent / Accept)
185 //
186 // STEP 6: Send over TLS
187 // let sent: i64 = nx_tls13_record_write(tls_session, req_buf, req_len)
188 // if sent != req_len → NX_HTTPS_SEND_FAIL
189 //
190 // STEP 7: Receive TLS-wrapped HTTP response
191 // loop reading nx_tls13_record_read until upstream closes or
192 // we hit out_cap or NX_HTTPS_RESPONSE_TOO_LARGE
193 //
194 // STEP 8: Parse HTTP response headers + body separation
195 // let n_headers: i64 = nx_http_resp_find_body_offset(out_buf,
196 // bytes_received)
197 // r.headers_ptr = out_buf
198 // r.headers_len = n_headers
199 // r.body_ptr = out_buf + n_headers
200 // r.body_len = bytes_received - n_headers
201 // r.status_code = nx_http_resp_parse_status(out_buf, n_headers)
202 //
203 // STEP 9: Handle redirects (3xx) up to NX_HTTPS_MAX_REDIRECTS
204 // if r.status_code >= 300 < 400:
205 // extract Location header, recurse with new URL
206 //
207 // STEP 10: Close TLS session + TCP socket
208 // nx_tls13_close(tls_session)
209 // sys_close(socket_fd)
210 //
211 // v1 ships the COMPOSITION FRAMEWORK; each step's wire-through
212 // depends on existing substrate primitives (already shipped) +
213 // a handful of glue function signatures that need adding to
214 // nx_http_client + nx_dns + nx_tls13_client. Glue work is
215 // ~200 LOC across those three modules; tracked as TLS-Phase-8
216 // tasks per [[feedback-ingestion-is-core-substrate-s-class-
217 // target]] tier-1 protocol layer.
218
219 // CORRECTED 2026-09-03: this path was returning NX_HTTPS_OK while performing no request at all.
220 // The module ALREADY DEFINES NX_HTTPS_NOT_IMPLEMENTED (12) and never used it -- the right verdict
221 // existed and the unimplemented path returned success instead. An unimplemented path that reports
222 // OK is the gate-passes-on-the-empty-set defect wearing a network client's clothes: every caller
223 // would read a successful fetch from a function that never opened a socket.
224 // MEASURED BEFORE CHANGING: this module has ZERO importers and this symbol has ZERO callers, so
225 // the change is inert today. It is made anyway because the hazard is LATENT, not absent --
226 // nx_https_get is defined in BOTH this file and nx_https_get.nx (the real one), so the first file
227 // to import both would have a working implementation shadowed by a stub that answers OK.
228 // Failing loud is the only safe direction for a stub that cannot be distinguished from a success.
229 r.verdict = NX_HTTPS_NOT_IMPLEMENTED
230 return r
231}
232
233// ===== Convenience wrappers =======================================
234
235// HTTPS POST (with JSON body)
236func nx_https_post_json(
237 url_ptr: *u8,
238 url_len: i64,
239 json_body: *u8,
240 json_len: i64,
241 out_buf: *u8,
242 out_cap: i64,
243 now_unix: i64
244) -> *HttpsResponse {
245 // CORRECTED 2026-09-03. THIS FUNCTION PERFORMED A **GET** AND SILENTLY DISCARDED json_body.
246 // The parenthetical that used to sit here -- "(v1.1 actually builds POST request with
247 // Content-Type: application/json and the json_body bytes appended)" -- described a version that
248 // was never written, but it READ as a description of what the code did. A caller checking the
249 // source for reassurance would find a sentence saying the body is sent, three lines under code
250 // that throws it away.
251 // A FUNCTION NAMED post_json THAT ISSUES A GET IS WORSE THAN AN ABSENT ONE: the name is the
252 // contract every caller reads, the request leaves the machine with the wrong method, the body
253 // never leaves at all, and the verdict says OK. Nothing downstream can detect it.
254 // It now REFUSES by name. The REAL primitive exists and callers should compose it directly:
255 // nx_https_post_complete_xhdr in nx_https_post_complete.nx genuinely builds a POST with a
256 // Content-Type and a body (it takes an established *Tls13ClientSession rather than a URL, which
257 // is why it was never a drop-in for this signature -- that gap is the actual work, and naming it
258 // is more use to the next reader than a wrapper that lies).
259 // allocate our OWN response, exactly as nx_https_get does. out_buf is the caller's BODY buffer:
260 // casting it to *HttpsResponse would scribble a verdict into the caller's storage and hand back
261 // a pointer into it. Caught by reading the struct before trusting the cast.
262 let raw: *u8 = sys_mmap(NX_HTTPS_RESPONSE_BYTES)
263 let r: *HttpsResponse = raw as *HttpsResponse
264 r.status_code = 0
265 r.body_len = 0
266 r.bytes_total = 0
267 r.verdict = NX_HTTPS_NOT_IMPLEMENTED
268 return r
269}
270
271// HTTPS GET with API key header (USDA FDC / USDA AMS pattern)
272func nx_https_get_with_api_key(
273 url_ptr: *u8,
274 url_len: i64,
275 api_key_ptr: *u8,
276 api_key_len: i64,
277 out_buf: *u8,
278 out_cap: i64,
279 now_unix: i64
280) -> *HttpsResponse {
281 let r: *HttpsResponse = nx_https_get(url_ptr, url_len, out_buf, out_cap,
282 NX_HTTPS_DEFAULT_TIMEOUT_SECONDS, now_unix)
283 // (v1.1 inserts "Authorization: Bearer <key>" or "?api_key=<key>"
284 // per source's auth convention)
285 return r
286}
287
288// HTTPS GET with OAI-PMH polite-pool email header (academic
289// harvesting convention).
290func nx_https_get_polite_pool(
291 url_ptr: *u8,
292 url_len: i64,
293 polite_email_ptr: *u8,
294 polite_email_len: i64,
295 out_buf: *u8,
296 out_cap: i64,
297 now_unix: i64
298) -> *HttpsResponse {
299 // (v1.1 inserts "User-Agent: NX-INGEST/1.0 (mailto:<email>)" header
300 // per arxiv/openalex polite-pool convention)
301 return nx_https_get(url_ptr, url_len, out_buf, out_cap,
302 NX_HTTPS_DEFAULT_TIMEOUT_SECONDS, now_unix)
303}
304
305// ===== Phase 8 wiring status ======================================
306//
307// Below is the explicit substrate-dependency status for each step.
308// When all dependencies = SHIPPED, this primitive runs end-to-end.
309
310const NX_HTTPS_DEP_URL_PARSE_STATUS: i64 = 1 // SHIPPED (nx_url)
311const NX_HTTPS_DEP_DNS_RESOLVE_STATUS: i64 = 2 // SHIPPED (nx_dns) — needs glue function
312const NX_HTTPS_DEP_TCP_SOCKET_STATUS: i64 = 1 // SHIPPED (nx_socket)
313const NX_HTTPS_DEP_TLS_HANDSHAKE_STATUS: i64 = 1 // SHIPPED (nx_tls13_client_handshake)
314const NX_HTTPS_DEP_TLS_RECORD_RW_STATUS: i64 = 1 // SHIPPED (nx_tls13_record)
315const NX_HTTPS_DEP_HTTP_BUILD_STATUS: i64 = 1 // SHIPPED (nx_http_client_build_request)
316const NX_HTTPS_DEP_HTTP_RESP_PARSE_STATUS: i64 = 1 // SHIPPED (nx_http_resp)
317const NX_HTTPS_DEP_CERT_VERIFY_STATUS: i64 = 1 // SHIPPED (nx_tls13_auth)
318
319// Status: 1 = SHIPPED, 2 = SHIPPED-NEEDS-GLUE, 3 = QUEUED, 4 = BLOCKED
320//
321// Net: substrate dependencies are 7 SHIPPED + 1 SHIPPED-NEEDS-GLUE.
322// The only missing piece is glue function nx_dns_resolve_a_record()
323// returning packed-IPv4 from hostname. That's ~50 LOC compose-against
324// existing nx_dns parse primitives. Next session ships the glue +
325// this primitive runs end-to-end against real upstream HTTPS APIs.
326
327// ===== Cert pinning + sovereign trust anchors =====================
328//
329// Per [[nishi-stack-is-bits-up-sovereign-always-no-third-party]]:
330// substrate ships its own trust anchor store (not Mozilla NSS, not
331// system CA bundle). Caller can opt into:
332// - Mozilla NSS roots (mirrored, validated)
333// - Custom pinned cert per source (recommended for known sources)
334// - LE-only (Let's Encrypt issuer chain)
335//
336// Default: substrate refuses unknown CAs.
337
338const NX_HTTPS_TRUST_MODE_PIN_PER_SOURCE: i64 = 1 // strictest
339const NX_HTTPS_TRUST_MODE_LE_ONLY: i64 = 2
340const NX_HTTPS_TRUST_MODE_MOZILLA_MIRROR: i64 = 3
341const NX_HTTPS_TRUST_MODE_ANY_CA: i64 = 4 // for testing only
342
343func nx_https_trust_mode_name(m: i64) -> *u8 {
344 if m == NX_HTTPS_TRUST_MODE_PIN_PER_SOURCE { return "PIN_PER_SOURCE" }
345 if m == NX_HTTPS_TRUST_MODE_LE_ONLY { return "LE_ONLY" }
346 if m == NX_HTTPS_TRUST_MODE_MOZILLA_MIRROR { return "MOZILLA_MIRROR" }
347 if m == NX_HTTPS_TRUST_MODE_ANY_CA { return "ANY_CA" }
348 return "UNKNOWN"
349}