code wiki / (root) / nx_https_get_cli2.nx
`nx_https_get_cli2.nx` source

↩ module page · 400 lines · 18612 B
1// nx_https_get_cli2.nx -- CANARY of the sovereign HTTPS fetch with HELLO AUTO-FALLBACK (seq759 design, finally real).
2//   nx_https_get <url> [connect-host:port]
3// WHY THIS EXISTS (measured 2026-07-25): the live chrome-ONLY build regressed every host that rejects the
4// Chrome-mimic hello but accepts our plain minimal hello (api.nhtsa.gov proven 200 on 07-23, verdict=5 now;
5// same class: www.sec.gov, efts.sec.gov, www.justice.gov). Cloudflare-walled hosts (api.worldbank.org) still
6// NEED the chrome hello. No single hello serves both sets, so: attempt 1 = chrome-JA3 (identical to live
7// behavior for every currently-working feed), on handshake failure reconnect and attempt 2 = plain hello.
8// Fallback is LEGIBLE (stderr nishi-hello line, never a silently-different code path). license_tier: ORIGINAL
9import "nx_syscalls.nx"
10import "nx_connect.nx"   // bounded connect: a raw sys_connect hangs ~127s on a black-holed host
11import "nx_csprng.nx"
12import "nx_x509_trust_store.nx"
13import "nx_trust_store_load_from_certdata.nx"
14import "nx_tls13_client_validate_certificate.nx"
15import "nx_tls13_client_session_run.nx"
16import "nx_tls13_chrome_session.nx"   // Chrome-JA3 ClientHello runner (beats anti-bot CDN TLS walls; inherits the recv_hs reassembly fix)
17import "nx_https_url_for_fetch.nx"
18import "nx_https_url_connect.nx"
19import "nx_https_get_complete.nx"
20import "nx_tls_cert_cache.nx"
21import "nx_gzip_wrap.nx"   // Content-Encoding: gzip -- the canonical in-tree inflate (KAT: nx_gzip_inflate_kat_test)
22import "nx_zlib_wrap.nx"   // Content-Encoding: deflate -- HTTP `deflate` means RFC 1950 zlib
23const HGC_MAGIC_1950: i64 = 1950
24const HGC_MAGIC_4194304: i64 = 4194304
25const HGC_MAGIC_65535: i64 = 65535
26
27const HGC_CERTDATA: *u8 = "data/mozilla_certdata.txt" as *u8
28const HGC_OUTCAP:   i64 = 33554432   // 32 MiB response cap (4 MiB refused a 5MB Koikatsu card 2026-08-04 -- code=8 overflow; real assets are routinely >4MiB)
29
30func hgc_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
31func hgc_put(s: *u8) -> i64 { sys_write(1, s, hgc_slen(s)); return 0 }
32func hgc_putn(v: i64) -> i64 {
  let b: *u8 = sys_mmap(24)
  var m: i64 = v
  if m == 0 { b[0] = 48 as u8; sys_write(1, b, 1); return 0 }
  if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
  var nd: i64 = 0
  var t: i64 = m
  while t > 0 { nd = nd + 1; t = t / 10 }
  var i: i64 = nd - 1
  while i >= 0 { b[i] = (48 + (m % 10)) as u8; m = m / 10; i = i - 1 }
  sys_write(1, b, nd)
  return 0
44}
45// stderr twins (the fallback diagnostic must not pollute the stdout HTTP payload)
46func hgc_put2(s: *u8) -> i64 { sys_write(2, s, hgc_slen(s)); return 0 }
47func hgc_putn2(v: i64) -> i64 {
  let b: *u8 = sys_mmap(24)
  var m: i64 = v
  if m == 0 { b[0] = 48 as u8; sys_write(2, b, 1); return 0 }
  if m < 0 { sys_write(2, "-" as *u8, 1); m = 0 - m }
  var nd: i64 = 0
  var t: i64 = m
  while t > 0 { nd = nd + 1; t = t / 10 }
  var i: i64 = nd - 1
  while i >= 0 { b[i] = (48 + (m % 10)) as u8; m = m / 10; i = i - 1 }
  sys_write(2, b, nd)
  return 0
59}
60
61// ---- BODY MODE (additive; the DEFAULT output is byte-for-byte unchanged, rule 19) -------------
62// This client has always emitted the RAW response: status line, headers, and -- when the origin
63// uses Transfer-Encoding: chunked -- the hex chunk-size markers interleaved with the payload.
64// That is right for a debugging client and CORRUPTING for any caller that content-addresses what
65// it fetched: a sha256 over that stream hashes a framing artifact, not the document.
66// â˜…â˜…â˜…â˜…â˜…IT DOES NOT FAIL -- IT SUCCEEDS WITH THE WRONG BYTES, which is why it went unnoticed. A
67// fetch that errors gets fixed; a fetch that returns plausible-but-reframed bytes gets TRUSTED.
68// Measured 2026-07-31 against https://www.rfc-editor.org/rfc/rfc7748.txt: the first chunk header
69// read `9982` = 39298 decimal, exactly the true body length, sitting inline in the output.
70// `--body` emits the de-chunked entity body only. Callers of the default path are untouched.
71// 2026-08-04: --body also DECODES the transport coding (Content-Encoding gzip/deflate via
72// nx_gzip_wrap / nx_zlib_wrap, the canonical in-tree inflates). Live-hit that forced this:
73// db.bepis.moe answered gzip and the emitted "body" was compressed bytes -- success with the
74// wrong bytes again. Decode failure or a coding we cannot decode (br, zstd, stacked) is a LOUD
75// nonzero exit (8/9), never wrong bytes; the raw default path still serves the actual wire bytes.
76// NOTE ON DRY (rule 15): _hdl_build/nx_http_chunked.nx already implements this exact decode, but
77// NO file in this corpus imports across directories, so it is unreachable from runtime/. Fixing
78// the import model is the right long-term move; blocking a pinned-evidence chain on it is not.
79func hgc_streq(a: *u8, b: *u8) -> i64 {
  var i: i64 = 0
  while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 }
  if b[i] != (0 as u8) { return 0 }
  return 1
84}
85
86func hgc_hexval(c: i64) -> i64 {
  if c >= 48 { if c <= 57 { return c - 48 } }
  if c >= 97 { if c <= 102 { return c - 87 } }
  if c >= 65 { if c <= 70 { return c - 55 } }
  return 0 - 1
91}
92
93func hgc_hdr_end(b: *u8, n: i64) -> i64 {
  var i: i64 = 0
  while i + 3 < n {
      if (b[i] as i64) == 13 { if (b[i+1] as i64) == 10 { if (b[i+2] as i64) == 13 { if (b[i+3] as i64) == 10 { return i + 4 } } } }
      i = i + 1
  }
  return 0 - 1
100}
101
102// Case-insensitive scan of the HEADER REGION ONLY. Scanning the whole response would false-positive
103// on any document that merely contains the word, and RFC 7748 is not it -- but the next fetched doc
104// might be, and a framing decision must never depend on payload text.
105func hgc_hdr_chunked(b: *u8, n: i64, he: i64) -> i64 {
  let pat: *u8 = "chunked" as *u8
  var i: i64 = 0
  while i + 7 <= he {
      var m: i64 = 1
      var j: i64 = 0
      while j < 7 {
          var c: i64 = b[i+j] as i64
          if c >= 65 { if c <= 90 { c = c + 32 } }
          if c != (pat[j] as i64) { m = 0 }
          j = j + 1
      }
      if m == 1 { return 1 }
      i = i + 1
  }
  return 0
121}
122
123// Decode a chunked body. Tolerates chunk extensions after the size, stops at the 0-size chunk.
124// Returns the decoded length, or -1 when the first size line has no hex digits at all.
125func hgc_dechunk(src: *u8, n: i64, out: *u8) -> i64 {
  var i: i64 = 0
  var o: i64 = 0
  var done: i64 = 0
  while done == 0 {
      var sz: i64 = 0
      var got: i64 = 0
      var st: i64 = 0
      while st == 0 {
          if i >= n { st = 1 } else {
              let hv: i64 = hgc_hexval(src[i] as i64)
              if hv >= 0 { sz = (sz * 16) + hv; got = got + 1; i = i + 1 } else { st = 1 }
          }
      }
      if got == 0 { return 0 - 1 }
      var st2: i64 = 0
      while st2 == 0 { if i >= n { st2 = 1 } else { if (src[i] as i64) == 13 { st2 = 1 } else { i = i + 1 } } }
      if i + 1 < n { if (src[i] as i64) == 13 { if (src[i+1] as i64) == 10 { i = i + 2 } } }
      if sz == 0 { done = 1 } else {
          var k: i64 = 0
          while k < sz { if i < n { out[o] = src[i]; o = o + 1; i = i + 1 } k = k + 1 }
          if i + 1 < n { if (src[i] as i64) == 13 { if (src[i+1] as i64) == 10 { i = i + 2 } } }
      }
  }
  return o
150}
151
152// Which Content-Encoding did the server apply? HEADER REGION ONLY -- a body that merely
153// contains the words must never drive a framing decision (same discipline as hgc_hdr_chunked).
154// Header-driven ON PURPOSE, not magic-sniff: a .gz FILE fetched over identity encoding IS the
155// document, and sniff-inflating it would corrupt the download. Transport coding is the only
156// thing --body may strip. Returns 0 = identity/none, 1 = gzip, 2 = deflate, 3 = undecodable
157// here (br, zstd, stacked codings) -- the caller must REFUSE, because emitting still-compressed
158// bytes from a mode whose contract is "the decoded document" is success-with-wrong-bytes, the
159// exact shape that poisoned the search index (debt 1785794199).
160func hgc_hdr_enc(b: *u8, n: i64, he: i64) -> i64 {
  let pat: *u8 = "content-encoding:" as *u8
  var at: i64 = 0 - 1
  var i: i64 = 0
  while i + 17 <= he {
      var m: i64 = 1
      var j: i64 = 0
      while j < 17 {
          var c: i64 = b[i+j] as i64
          if c >= 65 { if c <= 90 { c = c + 32 } }
          if c != (pat[j] as i64) { m = 0 }
          j = j + 1
      }
      if m == 1 { at = i + 17; i = he + 1 } else { i = i + 1 }
  }
  if at < 0 { return 0 }
  var sk: i64 = 0
  while sk == 0 {
      if at >= he { sk = 1 } else {
          let cs: i64 = b[at] as i64
          if cs == 32 { at = at + 1 } else { if cs == 9 { at = at + 1 } else { sk = 1 } }
      }
  }
  let tok: *u8 = sys_mmap(16)
  var tl: i64 = 0
  var sc: i64 = 0
  while sc == 0 {
      if at >= he { sc = 1 } else {
          var cv: i64 = b[at] as i64
          if cv == 13 { sc = 1 } else {
          if cv == 10 { sc = 1 } else {
          if cv == 44 { return 3 } else {
          if cv == 32 { sc = 1 } else {
          if cv == 59 { sc = 1 } else {
              if tl >= 15 { return 3 }
              if cv >= 65 { if cv <= 90 { cv = cv + 32 } }
              tok[tl] = cv as u8
              tl = tl + 1
              at = at + 1
          } } } } }
      }
  }
  tok[tl] = 0 as u8
  if tl == 0 { return 0 }
  if hgc_streq(tok, "gzip" as *u8) == 1 { return 1 }
  if hgc_streq(tok, "x-gzip" as *u8) == 1 { return 1 }
  if hgc_streq(tok, "deflate" as *u8) == 1 { return 2 }
  if hgc_streq(tok, "identity" as *u8) == 1 { return 0 }
  return 3
209}
210
211func main(argc: i64, argv: *i64) -> i64 {
  if argc < 2 { hgc_put("ERROR: usage: nx_https_get <url> [connect-host:port] [--body]\n" as *u8); return 2 }
  let url: *u8 = argv[1] as *u8
  let now: i64 = sys_now_realtime_sec()
215
  // ---- trust store + real TLS entropy buffers (refilled per attempt) ----
  let r: i64 = nx_trust_store_load_from_certdata(HGC_CERTDATA, 512, HGC_MAGIC_4194304)
  if r <= 0 { hgc_put("ERROR: trust-store load failed (data/mozilla_certdata.txt on daemon CWD?)\n" as *u8); return 3 }
  let store: *TrustStore = r as *TrustStore
  let cr: *u8 = sys_mmap(32)
  let priv: *u8 = sys_mmap(32)
222
  // ---- parse the REAL url (drives SNI / Host / path / cert-name) ----
  let url_p: *NxUrl = nx_url_new()
  let target_raw: *u8 = sys_mmap(32)
  let target: *NxHttpsTarget = target_raw as *NxHttpsTarget
  target.url = url_p
  target.port = 0
  if nx_https_url_for_fetch(url, target) != NX_HTTPS_URL_OK { hgc_put("ERROR: bad url\n" as *u8); return 2 }
230
  // ---- connect override parsed ONCE (an attempt reconnects with the same endpoint) ----
  var have_ov: i64 = 0
  let ipbox: *i64 = sys_mmap(8) as *i64
  let portbox: *i64 = sys_mmap(8) as *i64
  // Flags and the connect-override are position-independent so `--body` may appear before or
  // after the override. Parsing argv[2] positionally would have made `nx_https_get <url> --body`
  // fail as a malformed ip:port -- an error message pointing at the wrong argument.
  var body_only: i64 = 0
  var ai: i64 = 2
  while ai < argc {
      let a: *u8 = argv[ai] as *u8
      if hgc_streq(a, "--body" as *u8) == 1 { body_only = 1 } else {
          if hgc_parse_ipport(a, ipbox, portbox) != 1 { hgc_put("ERROR: bad connect-override (want a.b.c.d:port)\n" as *u8); return 2 }
          have_ov = 1
      }
      ai = ai + 1
  }
248
  let fd_p: *i64 = sys_mmap(16) as *i64
  let val_raw: *u8 = sys_mmap(128)
  let val_ctx: *TlsValidationContext = val_raw as *TlsValidationContext
252
  // ---- connect + handshake, TWO attempts: 0 = chrome-JA3 hello (live-identical), 1 = plain hello fallback ----
  var attempt: i64 = 0
  var session_i: i64 = 0
  var fd: i64 = 0 - 1
  while attempt < 2 {
      // fresh connect per attempt (a rejected handshake kills the socket)
      if have_ov == 1 {
          let sa: *u8 = sys_mmap(16)
          nx_https_build_sockaddr(sa, ipbox[0], portbox[0])
          let cfd: i64 = sys_socket(NX_HTTPS_AF_INET, NX_HTTPS_SOCK_STREAM, 0)
          if cfd < 0 { hgc_put("ERROR: socket failed\n" as *u8); return 3 }
          sys_set_socket_timeout(cfd, 15)
          if nx_connect_bounded(cfd, sa, 16, NX_CONN_DEFAULT_MS) < 0 { sys_close(cfd); hgc_put("ERROR: connect override failed (edge unreachable at that ip:port)\n" as *u8); return 3 }
          fd_p[0] = cfd
      } else {
          if nx_https_url_connect(target, url, now, fd_p) != NX_HTTPS_CONNECT_OK { hgc_put("ERROR: connect failed\n" as *u8); return 3 }
      }
      fd = fd_p[0]
271
      // fresh entropy per attempt (never reuse a client-random across handshakes)
      nx_csprng_fill(cr, 32)
      nx_csprng_fill(priv, 32)
275
      // validation ctx + F799 certloop cache, re-armed per attempt
      val_ctx.store = store
      val_ctx.sni_host = url + target.url.host_off
      val_ctx.sni_host_len = target.url.host_len
      val_ctx.now_epoch = now
      let tcc_hit: i64 = tcc_load(url + target.url.host_off, target.url.host_len, now, val_ctx)
      tcc_arm(val_ctx)
      if tcc_hit == 1 { sys_write(2, "nishi-tcc candidate loaded\n" as *u8, 27) }
284
      var sr: i64 = 0
      if attempt == 0 { sr = nx_tls13_client_session_run_chrome(fd, url + target.url.host_off, target.url.host_len, cr, priv, val_ctx) }
      else { sr = nx_tls13_client_session_run(fd, url + target.url.host_off, target.url.host_len, cr, priv, val_ctx) }
288
      if sr >= 0 { session_i = sr; attempt = 99 }
      else {
          sys_close(fd)
          if attempt == 0 {
              hgc_put2("nishi-hello chrome-hello REJECTED verdict=" as *u8)
              hgc_putn2(0 - sr)
              hgc_put2(" -- retrying with plain hello\n" as *u8)
              attempt = 1
          } else {
              hgc_put("ERROR: TLS handshake failed verdict=" as *u8)
              hgc_putn(0 - sr)
              hgc_put(" (2=emitCH 3=wrCH 4=readSH 5=recvSH 6=readHS 7=recvHS/CERT 8=emitCF 10=deriveApp 11=budget)\n" as *u8)
              return 4
          }
      }
  }
  if session_i == 0 { hgc_put("ERROR: TLS handshake failed (no session)\n" as *u8); return 4 }
  let session: *Tls13ClientSession = session_i as *Tls13ClientSession
  tcc_save(url + target.url.host_off, target.url.host_len, now, val_ctx)
308
  // ---- path (default "/", append ?query verbatim -- else query-string APIs silently drop) ----
  var path_ptr: *u8 = url + target.url.path_off
  var path_len: i64 = target.url.path_len
  if path_len == 0 {
      let dp: *u8 = sys_mmap(2)
      dp[0] = 47 as u8
      path_ptr = dp
      path_len = 1
  }
  if target.url.query_len > 0 {
      let full: *u8 = sys_mmap(path_len + target.url.query_len + 4)
      var fo: i64 = 0
      var pci: i64 = 0
      while pci < path_len { full[fo] = path_ptr[pci]; fo = fo + 1; pci = pci + 1 }
      full[fo] = 63 as u8; fo = fo + 1   // '?'
      let qp: *u8 = url + target.url.query_off
      var qci: i64 = 0
      while qci < target.url.query_len { full[fo] = qp[qci]; fo = fo + 1; qci = qci + 1 }
      path_ptr = full
      path_len = fo
  }
330
  // ---- HTTP GET; Host header = the REAL url host ----
  let out: *u8 = sys_mmap(HGC_OUTCAP)
  let n: i64 = nx_https_get_complete(session, fd, path_ptr, path_len, url + target.url.host_off, target.url.host_len, out, HGC_OUTCAP)
  sys_close(fd)
  if n < 0 { hgc_put("ERROR: HTTP fetch failed code=" as *u8); hgc_putn(0 - n); hgc_put("\n" as *u8); return 5 }
  if body_only == 0 { sys_write(1, out, n); return 0 }
  let he: i64 = hgc_hdr_end(out, n)
  if he < 0 { hgc_put("ERROR: --body: no CRLFCRLF header terminator; refusing to guess where the body starts\n" as *u8); return 6 }
  var ent: *u8 = out + he
  var en: i64 = n - he
  if hgc_hdr_chunked(out, n, he) == 1 {
      let db: *u8 = sys_mmap(HGC_OUTCAP)
      let dn: i64 = hgc_dechunk(ent, en, db)
      if dn < 0 { hgc_put("ERROR: --body: malformed chunked framing; refusing to emit a partial document\n" as *u8); return 7 }
      ent = db
      en = dn
  }
  let enc: i64 = hgc_hdr_enc(out, n, he)
  if enc == 3 { hgc_put("ERROR: --body: unsupported Content-Encoding (not identity/gzip/deflate); rerun without --body for the raw response\n" as *u8); return 9 }
  if enc == 1 {
      let gr: *NxGzipResult = nx_gzip_inflate(ent, en, HGC_OUTCAP)
      let gerr: i64 = gr.error_code
      if gerr != 0 { hgc_put("ERROR: --body: Content-Encoding gzip but inflate failed code=" as *u8); hgc_putn(gerr); hgc_put(" -- refusing to emit compressed bytes as the document\n" as *u8); return 8 }
      let gout: *u8 = gr.output_data
      let gsz: i64 = gr.output_size
      ent = gout
      en = gsz
  }
  if enc == 2 {
      let zr: *NxZlibResult = nx_zlib_inflate(ent, en, HGC_OUTCAP)
      let zerr: i64 = zr.error_code
      if zerr != 0 { hgc_put("ERROR: --body: Content-Encoding deflate but zlib inflate failed code=" as *u8); hgc_putn(zerr); hgc_put(" -- refusing to emit compressed bytes as the document\n" as *u8); return 8 }
      let zout: *u8 = zr.output_data
      let zsz: i64 = zr.output_size
      ent = zout
      en = zsz
  }
  sys_write(1, ent, en)
  return 0
370}
371
372// parse "a.b.c.d:port" -> ip_out (big-endian packed u32, e.g. 127.0.0.1 -> 0x7F000001) + port_out. 1 ok / 0 bad.
373func hgc_parse_ipport(s: *u8, ip_out: *i64, port_out: *i64) -> i64 {
  var packed: i64 = 0
  var val: i64 = 0
  var nocts: i64 = 0
  var port: i64 = 0
  var indots: i64 = 1
  var i: i64 = 0
  while s[i] != (0 as u8) {
      let c: i64 = s[i] as i64
      if indots == 1 {
          if c == 46 { packed = (packed << 8) | (val & 0xff); nocts = nocts + 1; val = 0 }
          else { if c == 58 { packed = (packed << 8) | (val & 0xff); nocts = nocts + 1; val = 0; indots = 0 }
          else { if c < 48 { return 0 } if c > 57 { return 0 } val = val * 10 + (c - 48) } }
      } else {
          if c < 48 { return 0 }
          if c > 57 { return 0 }
          port = port * 10 + (c - 48)
      }
      i = i + 1
  }
  if indots == 1 { return 0 }   // no ':' -> no port given
  if nocts != 4 { return 0 }    // need exactly 4 octets
  if port <= 0 { return 0 }
  if port > HGC_MAGIC_65535 { return 0 }
  ip_out[0] = packed
  port_out[0] = port
  return 1
400}
nx_https_get_cli2.nx source

`nx_https_get_cli2.nx` source
