code wiki / (root) / nx_https_get_cli2.nx
`nx_https_get_cli2.nx` source

↩ module page · 367 lines · 18035 B
1// nx_https_get_cli2.nx -- CANARY of the sovereign HTTPS fetch with HELLO AUTO-FALLBACK (seq759 design, finally real).
2//   nx_https_get <url> [connect-host:port]
3// WHY THIS EXISTS (measured 2026-07-25): the live chrome-ONLY build regressed every host that rejects the
4// Chrome-mimic hello but accepts our plain minimal hello (api.nhtsa.gov proven 200 on 07-23, verdict=5 now;
5// same class: www.sec.gov, efts.sec.gov, www.justice.gov). Cloudflare-walled hosts (api.worldbank.org) still
6// NEED the chrome hello. No single hello serves both sets, so: attempt 1 = chrome-JA3 (identical to live
7// behavior for every currently-working feed), on handshake failure reconnect and attempt 2 = plain hello.
8// Fallback is LEGIBLE (stderr nishi-hello line, never a silently-different code path). license_tier: ORIGINAL
9import "nx_syscalls.nx"
10import "nx_connect.nx"   // bounded connect: a raw sys_connect hangs ~127s on a black-holed host
11import "nx_csprng.nx"
12import "nx_x509_trust_store.nx"
13import "nx_trust_store_load_from_certdata.nx"
14import "nx_tls13_client_validate_certificate.nx"
15import "nx_tls13_client_session_run.nx"
16import "nx_tls13_chrome_session.nx"   // Chrome-JA3 ClientHello runner (beats anti-bot CDN TLS walls; inherits the recv_hs reassembly fix)
17import "nx_https_url_for_fetch.nx"
18import "nx_https_url_connect.nx"
19import "nx_https_get_complete.nx"
20import "nx_tls_cert_cache.nx"
21const HGC_MAGIC_4194304: i64 = 4194304
22const HGC_MAGIC_65535: i64 = 65535
23
24const HGC_CERTDATA: *u8 = "data/mozilla_certdata.txt" as *u8
25const HGC_OUTCAP:   i64 = 4194304    // 4 MiB response cap
26
27func hgc_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
28// One header line plus its CRLF. Named for the one thing it bounds; a value longer than this is a
29// caller error, not a case to grow silently into.
30const HGC_XHDR_CAP: i64 = 8192
31func hgc_put(s: *u8) -> i64 { sys_write(1, s, hgc_slen(s)); return 0 }
32func hgc_putn(v: i64) -> i64 {
  let b: *u8 = sys_mmap(24)
  var m: i64 = v
  if m == 0 { b[0] = 48 as u8; sys_write(1, b, 1); return 0 }
  if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
  var nd: i64 = 0
  var t: i64 = m
  while t > 0 { nd = nd + 1; t = t / 10 }
  var i: i64 = nd - 1
  while i >= 0 { b[i] = (48 + (m % 10)) as u8; m = m / 10; i = i - 1 }
  sys_write(1, b, nd)
  return 0
44}
45// stderr twins (the fallback diagnostic must not pollute the stdout HTTP payload)
46func hgc_put2(s: *u8) -> i64 { sys_write(2, s, hgc_slen(s)); return 0 }
47func hgc_putn2(v: i64) -> i64 {
  let b: *u8 = sys_mmap(24)
  var m: i64 = v
  if m == 0 { b[0] = 48 as u8; sys_write(2, b, 1); return 0 }
  if m < 0 { sys_write(2, "-" as *u8, 1); m = 0 - m }
  var nd: i64 = 0
  var t: i64 = m
  while t > 0 { nd = nd + 1; t = t / 10 }
  var i: i64 = nd - 1
  while i >= 0 { b[i] = (48 + (m % 10)) as u8; m = m / 10; i = i - 1 }
  sys_write(2, b, nd)
  return 0
59}
60
61// ---- BODY MODE (additive; the DEFAULT output is byte-for-byte unchanged, rule 19) -------------
62// This client has always emitted the RAW response: status line, headers, and -- when the origin
63// uses Transfer-Encoding: chunked -- the hex chunk-size markers interleaved with the payload.
64// That is right for a debugging client and CORRUPTING for any caller that content-addresses what
65// it fetched: a sha256 over that stream hashes a framing artifact, not the document.
66// â˜…â˜…â˜…â˜…â˜…IT DOES NOT FAIL -- IT SUCCEEDS WITH THE WRONG BYTES, which is why it went unnoticed. A
67// fetch that errors gets fixed; a fetch that returns plausible-but-reframed bytes gets TRUSTED.
68// Measured 2026-07-31 against https://www.rfc-editor.org/rfc/rfc7748.txt: the first chunk header
69// read `9982` = 39298 decimal, exactly the true body length, sitting inline in the output.
70// `--body` emits the de-chunked entity body only. Callers of the default path are untouched.
71// NOTE ON DRY (rule 15): _hdl_build/nx_http_chunked.nx already implements this exact decode, but
72// NO file in this corpus imports across directories, so it is unreachable from runtime/. Fixing
73// the import model is the right long-term move; blocking a pinned-evidence chain on it is not.
74// --hdr SUPPORT. It composes the SHIPPED primitive nx_https_get_complete_cookie_xhdr rather than adding
75// a second request builder: the extra-header capability already existed in the library and only the CLI
76// lacked a way to reach it, so this is an exposure, not a new ruler.
77//
78// THE GUARD IS THE LOAD-BEARING PART. A CR or LF inside a caller-supplied header value is HTTP REQUEST
79// SPLITTING: it closes the header the caller was given and starts one the caller never authorised, on a
80// client that speaks to arbitrary hosts. So a value carrying either byte is REFUSED BY NAME rather than
81// sanitised -- ★SANITISING AN INJECTION ATTEMPT HIDES IT; REFUSING IT REPORTS IT. A missing ':' is also
82// refused, because a header with no field-name separator is not a header and the server's parse of it is
83// undefined. Returns the built length, or a negative code naming which rule fired.
84const HGC_HDR_ERR_NOCOLON: i64 = 0 - 1
85const HGC_HDR_ERR_CTLBYTE: i64 = 0 - 2
86func hgc_build_xhdr(v: *u8, out: *u8) -> i64 {
  var i: i64 = 0
  var colon: i64 = 0
  while v[i] != (0 as u8) {
      let c: i64 = v[i] as i64
      if c == 13 { return HGC_HDR_ERR_CTLBYTE }
      if c == 10 { return HGC_HDR_ERR_CTLBYTE }
      if c == 58 { colon = 1 }
      out[i] = v[i]
      i = i + 1
  }
  if colon == 0 { return HGC_HDR_ERR_NOCOLON }
  if i == 0 { return HGC_HDR_ERR_NOCOLON }
  out[i] = 13 as u8
  out[i+1] = 10 as u8
  return i + 2
102}
103func hgc_streq(a: *u8, b: *u8) -> i64 {
  var i: i64 = 0
  while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 }
  if b[i] != (0 as u8) { return 0 }
  return 1
108}
109
110func hgc_hexval(c: i64) -> i64 {
  if c >= 48 { if c <= 57 { return c - 48 } }
  if c >= 97 { if c <= 102 { return c - 87 } }
  if c >= 65 { if c <= 70 { return c - 55 } }
  return 0 - 1
115}
116
117func hgc_hdr_end(b: *u8, n: i64) -> i64 {
  var i: i64 = 0
  while i + 3 < n {
      if (b[i] as i64) == 13 { if (b[i+1] as i64) == 10 { if (b[i+2] as i64) == 13 { if (b[i+3] as i64) == 10 { return i + 4 } } } }
      i = i + 1
  }
  return 0 - 1
124}
125
126// Case-insensitive scan of the HEADER REGION ONLY. Scanning the whole response would false-positive
127// on any document that merely contains the word, and RFC 7748 is not it -- but the next fetched doc
128// might be, and a framing decision must never depend on payload text.
129func hgc_hdr_chunked(b: *u8, n: i64, he: i64) -> i64 {
  let pat: *u8 = "chunked" as *u8
  var i: i64 = 0
  while i + 7 <= he {
      var m: i64 = 1
      var j: i64 = 0
      while j < 7 {
          var c: i64 = b[i+j] as i64
          if c >= 65 { if c <= 90 { c = c + 32 } }
          if c != (pat[j] as i64) { m = 0 }
          j = j + 1
      }
      if m == 1 { return 1 }
      i = i + 1
  }
  return 0
145}
146
147// Decode a chunked body. Tolerates chunk extensions after the size, stops at the 0-size chunk.
148// Returns the decoded length, or -1 when the first size line has no hex digits at all.
149func hgc_dechunk(src: *u8, n: i64, out: *u8) -> i64 {
  var i: i64 = 0
  var o: i64 = 0
  var done: i64 = 0
  while done == 0 {
      var sz: i64 = 0
      var got: i64 = 0
      var st: i64 = 0
      while st == 0 {
          if i >= n { st = 1 } else {
              let hv: i64 = hgc_hexval(src[i] as i64)
              if hv >= 0 { sz = (sz * 16) + hv; got = got + 1; i = i + 1 } else { st = 1 }
          }
      }
      if got == 0 { return 0 - 1 }
      var st2: i64 = 0
      while st2 == 0 { if i >= n { st2 = 1 } else { if (src[i] as i64) == 13 { st2 = 1 } else { i = i + 1 } } }
      if i + 1 < n { if (src[i] as i64) == 13 { if (src[i+1] as i64) == 10 { i = i + 2 } } }
      if sz == 0 { done = 1 } else {
          var k: i64 = 0
          while k < sz { if i < n { out[o] = src[i]; o = o + 1; i = i + 1 } k = k + 1 }
          if i + 1 < n { if (src[i] as i64) == 13 { if (src[i+1] as i64) == 10 { i = i + 2 } } }
      }
  }
  return o
174}
175
176func main(argc: i64, argv: *i64) -> i64 {
  if argc < 2 { hgc_put("ERROR: usage: nx_https_get <url> [connect-host:port] [--body] [--hdr 'K: V']\n" as *u8); return 2 }
  let url: *u8 = argv[1] as *u8
  let now: i64 = sys_now_realtime_sec()
180
  // ---- trust store + real TLS entropy buffers (refilled per attempt) ----
  let r: i64 = nx_trust_store_load_from_certdata(HGC_CERTDATA, 512, HGC_MAGIC_4194304)
  if r <= 0 { hgc_put("ERROR: trust-store load failed (data/mozilla_certdata.txt on daemon CWD?)\n" as *u8); return 3 }
  let store: *TrustStore = r as *TrustStore
  let cr: *u8 = sys_mmap(32)
  let priv: *u8 = sys_mmap(32)
187
  // ---- parse the REAL url (drives SNI / Host / path / cert-name) ----
  let url_p: *NxUrl = nx_url_new()
  let target_raw: *u8 = sys_mmap(32)
  let target: *NxHttpsTarget = target_raw as *NxHttpsTarget
  target.url = url_p
  target.port = 0
  if nx_https_url_for_fetch(url, target) != NX_HTTPS_URL_OK { hgc_put("ERROR: bad url\n" as *u8); return 2 }
195
  // ---- connect override parsed ONCE (an attempt reconnects with the same endpoint) ----
  var have_ov: i64 = 0
  let ipbox: *i64 = sys_mmap(8) as *i64
  let portbox: *i64 = sys_mmap(8) as *i64
  // Flags and the connect-override are position-independent so `--body` may appear before or
  // after the override. Parsing argv[2] positionally would have made `nx_https_get <url> --body`
  // fail as a malformed ip:port -- an error message pointing at the wrong argument.
  var body_only: i64 = 0
  // One --hdr, deliberately: a repeatable flag needs an accumulating buffer and a cap on the total,
  // and one extra header is what the roster needs today (asserting a header-CONDITIONAL behaviour on a
  // live surface). A second one can be added when something needs it -- not before.
  var xhdr: *u8 = 0 as *u8
  var xhdr_len: i64 = 0
  var ai: i64 = 2
  while ai < argc {
      let a: *u8 = argv[ai] as *u8
      if hgc_streq(a, "--body" as *u8) == 1 { body_only = 1 } else {
      if hgc_streq(a, "--hdr" as *u8) == 1 {
          // It takes a VALUE, so it must be matched BEFORE the connect-override arm: otherwise the
          // header text falls through to hgc_parse_ipport and is reported as a malformed ip:port --
          // an error naming the wrong argument, which is the defect the comment above already warns of.
          if ai + 1 >= argc { hgc_put("ERROR: --hdr needs a value, e.g. --hdr 'X-Name: value'\n" as *u8); return 2 }
          let hv: *u8 = argv[ai+1] as *u8
          xhdr = sys_mmap(HGC_XHDR_CAP)
          xhdr_len = hgc_build_xhdr(hv, xhdr)
          if xhdr_len == HGC_HDR_ERR_CTLBYTE { hgc_put("ERROR: --hdr value contains CR or LF -- REFUSED. That is HTTP request splitting: it would close the header you asked for and open one you did not authorise. Value NOT sanitised, NOT sent.\n" as *u8); return 2 }
          if xhdr_len == HGC_HDR_ERR_NOCOLON { hgc_put("ERROR: --hdr value has no ':' field-name separator, so it is not a header. Use --hdr 'X-Name: value'\n" as *u8); return 2 }
          ai = ai + 1
      } else {
          if hgc_parse_ipport(a, ipbox, portbox) != 1 { hgc_put("ERROR: bad connect-override (want a.b.c.d:port)\n" as *u8); return 2 }
          have_ov = 1
      } }
      ai = ai + 1
  }
230
  let fd_p: *i64 = sys_mmap(16) as *i64
  let val_raw: *u8 = sys_mmap(128)
  let val_ctx: *TlsValidationContext = val_raw as *TlsValidationContext
234
  // ---- connect + handshake, TWO attempts: 0 = chrome-JA3 hello (live-identical), 1 = plain hello fallback ----
  var attempt: i64 = 0
  var session_i: i64 = 0
  var fd: i64 = 0 - 1
  while attempt < 2 {
      // fresh connect per attempt (a rejected handshake kills the socket)
      if have_ov == 1 {
          let sa: *u8 = sys_mmap(16)
          nx_https_build_sockaddr(sa, ipbox[0], portbox[0])
          let cfd: i64 = sys_socket(NX_HTTPS_AF_INET, NX_HTTPS_SOCK_STREAM, 0)
          if cfd < 0 { hgc_put("ERROR: socket failed\n" as *u8); return 3 }
          sys_set_socket_timeout(cfd, 15)
          if nx_connect_bounded(cfd, sa, 16, NX_CONN_DEFAULT_MS) < 0 { sys_close(cfd); hgc_put("ERROR: connect override failed (edge unreachable at that ip:port)\n" as *u8); return 3 }
          fd_p[0] = cfd
      } else {
          if nx_https_url_connect(target, url, now, fd_p) != NX_HTTPS_CONNECT_OK { hgc_put("ERROR: connect failed\n" as *u8); return 3 }
      }
      fd = fd_p[0]
253
      // fresh entropy per attempt (never reuse a client-random across handshakes)
      nx_csprng_fill(cr, 32)
      nx_csprng_fill(priv, 32)
257
      // validation ctx + F799 certloop cache, re-armed per attempt
      val_ctx.store = store
      val_ctx.sni_host = url + target.url.host_off
      val_ctx.sni_host_len = target.url.host_len
      val_ctx.now_epoch = now
      let tcc_hit: i64 = tcc_load(url + target.url.host_off, target.url.host_len, now, val_ctx)
      tcc_arm(val_ctx)
      if tcc_hit == 1 { sys_write(2, "nishi-tcc candidate loaded\n" as *u8, 27) }
266
      var sr: i64 = 0
      if attempt == 0 { sr = nx_tls13_client_session_run_chrome(fd, url + target.url.host_off, target.url.host_len, cr, priv, val_ctx) }
      else { sr = nx_tls13_client_session_run(fd, url + target.url.host_off, target.url.host_len, cr, priv, val_ctx) }
270
      if sr >= 0 { session_i = sr; attempt = 99 }
      else {
          sys_close(fd)
          if attempt == 0 {
              hgc_put2("nishi-hello chrome-hello REJECTED verdict=" as *u8)
              hgc_putn2(0 - sr)
              hgc_put2(" -- retrying with plain hello\n" as *u8)
              attempt = 1
          } else {
              hgc_put("ERROR: TLS handshake failed verdict=" as *u8)
              hgc_putn(0 - sr)
              hgc_put(" (2=emitCH 3=wrCH 4=readSH 5=recvSH 6=readHS 7=recvHS/CERT 8=emitCF 10=deriveApp 11=budget)\n" as *u8)
              return 4
          }
      }
  }
  if session_i == 0 { hgc_put("ERROR: TLS handshake failed (no session)\n" as *u8); return 4 }
  let session: *Tls13ClientSession = session_i as *Tls13ClientSession
  tcc_save(url + target.url.host_off, target.url.host_len, now, val_ctx)
290
  // ---- path (default "/", append ?query verbatim -- else query-string APIs silently drop) ----
  var path_ptr: *u8 = url + target.url.path_off
  var path_len: i64 = target.url.path_len
  if path_len == 0 {
      let dp: *u8 = sys_mmap(2)
      dp[0] = 47 as u8
      path_ptr = dp
      path_len = 1
  }
  if target.url.query_len > 0 {
      let full: *u8 = sys_mmap(path_len + target.url.query_len + 4)
      var fo: i64 = 0
      var pci: i64 = 0
      while pci < path_len { full[fo] = path_ptr[pci]; fo = fo + 1; pci = pci + 1 }
      full[fo] = 63 as u8; fo = fo + 1   // '?'
      let qp: *u8 = url + target.url.query_off
      var qci: i64 = 0
      while qci < target.url.query_len { full[fo] = qp[qci]; fo = fo + 1; qci = qci + 1 }
      path_ptr = full
      path_len = fo
  }
312
  // ---- HTTP GET; Host header = the REAL url host ----
  let out: *u8 = sys_mmap(HGC_OUTCAP)
  // WITHOUT --hdr THIS IS THE EXACT CALL IT ALWAYS WAS. The xhdr variant is reached only when a header
  // was actually supplied, so every existing invocation is byte-identical and the 61 library callers of
  // the base primitive are untouched. The cookie argument is empty because this flag is about headers;
  // a cookie CLI is a separate question and is not being answered here by accident.
  var n: i64 = 0
  if xhdr_len > 0 {
      n = nx_https_get_complete_cookie_xhdr(session, fd, path_ptr, path_len, url + target.url.host_off, target.url.host_len, "" as *u8, 0, xhdr, xhdr_len, out, HGC_OUTCAP)
  }
  if xhdr_len <= 0 {
      n = nx_https_get_complete(session, fd, path_ptr, path_len, url + target.url.host_off, target.url.host_len, out, HGC_OUTCAP)
  }
  sys_close(fd)
  if n < 0 { hgc_put("ERROR: HTTP fetch failed code=" as *u8); hgc_putn(0 - n); hgc_put("\n" as *u8); return 5 }
  if body_only == 0 { sys_write(1, out, n); return 0 }
  let he: i64 = hgc_hdr_end(out, n)
  if he < 0 { hgc_put("ERROR: --body: no CRLFCRLF header terminator; refusing to guess where the body starts\n" as *u8); return 6 }
  if hgc_hdr_chunked(out, n, he) == 0 { sys_write(1, out + he, n - he); return 0 }
  let db: *u8 = sys_mmap(HGC_OUTCAP)
  let dn: i64 = hgc_dechunk(out + he, n - he, db)
  if dn < 0 { hgc_put("ERROR: --body: malformed chunked framing; refusing to emit a partial document\n" as *u8); return 7 }
  sys_write(1, db, dn)
  return 0
337}
338
339// parse "a.b.c.d:port" -> ip_out (big-endian packed u32, e.g. 127.0.0.1 -> 0x7F000001) + port_out. 1 ok / 0 bad.
340func hgc_parse_ipport(s: *u8, ip_out: *i64, port_out: *i64) -> i64 {
  var packed: i64 = 0
  var val: i64 = 0
  var nocts: i64 = 0
  var port: i64 = 0
  var indots: i64 = 1
  var i: i64 = 0
  while s[i] != (0 as u8) {
      let c: i64 = s[i] as i64
      if indots == 1 {
          if c == 46 { packed = (packed << 8) | (val & 0xff); nocts = nocts + 1; val = 0 }
          else { if c == 58 { packed = (packed << 8) | (val & 0xff); nocts = nocts + 1; val = 0; indots = 0 }
          else { if c < 48 { return 0 } if c > 57 { return 0 } val = val * 10 + (c - 48) } }
      } else {
          if c < 48 { return 0 }
          if c > 57 { return 0 }
          port = port * 10 + (c - 48)
      }
      i = i + 1
  }
  if indots == 1 { return 0 }   // no ':' -> no port given
  if nocts != 4 { return 0 }    // need exactly 4 octets
  if port <= 0 { return 0 }
  if port > HGC_MAGIC_65535 { return 0 }
  ip_out[0] = packed
  port_out[0] = port
  return 1
367}
nx_https_get_cli2.nx source

`nx_https_get_cli2.nx` source
