code wiki / (root) / nx_https_get_cli2_candidate.nx
`nx_https_get_cli2_candidate.nx` source

↩ module page · 355 lines · 18045 B
1// nx_https_get_cli2.nx -- CANARY of the sovereign HTTPS fetch with HELLO AUTO-FALLBACK (seq759 design, finally real).
2//   nx_https_get <url> [connect-host:port]
3// WHY THIS EXISTS (measured 2026-07-25): the live chrome-ONLY build regressed every host that rejects the
4// Chrome-mimic hello but accepts our plain minimal hello (api.nhtsa.gov proven 200 on 07-23, verdict=5 now;
5// same class: www.sec.gov, efts.sec.gov, www.justice.gov). Cloudflare-walled hosts (api.worldbank.org) still
6// NEED the chrome hello. No single hello serves both sets, so: attempt 1 = chrome-JA3 (identical to live
7// behavior for every currently-working feed), on handshake failure reconnect and attempt 2 = plain hello.
8// Fallback is LEGIBLE (stderr nishi-hello line, never a silently-different code path). license_tier: ORIGINAL
9import "nx_syscalls.nx"
10import "nx_http_response_parse_candidate.nx"
11import "nx_connect.nx"   // bounded connect: a raw sys_connect hangs ~127s on a black-holed host
12import "nx_csprng.nx"
13import "nx_x509_trust_store.nx"
14import "nx_trust_store_load_from_certdata.nx"
15import "nx_tls13_client_validate_certificate.nx"
16import "nx_tls13_client_session_run.nx"
17import "nx_tls13_chrome_session.nx"   // Chrome-JA3 ClientHello runner (beats anti-bot CDN TLS walls; inherits the recv_hs reassembly fix)
18import "nx_https_url_for_fetch.nx"
19import "nx_https_url_connect.nx"
20import "nx_https_get_complete.nx"
21import "nx_tls_cert_cache.nx"
22const HGC_MAGIC_4194304: i64 = 4194304
23const HGC_MAGIC_65535: i64 = 65535
24
25const HGC_CERTDATA: *u8 = "data/mozilla_certdata.txt" as *u8
26const HGC_OUTCAP:   i64 = 4194304    // 4 MiB response cap
27
28func hgc_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
29// One header line plus its CRLF. Named for the one thing it bounds; a value longer than this is a
30// caller error, not a case to grow silently into.
31const HGC_XHDR_CAP: i64 = 8192
32func hgc_put(s: *u8) -> i64 { sys_write(1, s, hgc_slen(s)); return 0 }
33func hgc_putn(v: i64) -> i64 {
  let b: *u8 = sys_mmap(24)
  var m: i64 = v
  if m == 0 { b[0] = 48 as u8; sys_write(1, b, 1); return 0 }
  if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
  var nd: i64 = 0
  var t: i64 = m
  while t > 0 { nd = nd + 1; t = t / 10 }
  var i: i64 = nd - 1
  while i >= 0 { b[i] = (48 + (m % 10)) as u8; m = m / 10; i = i - 1 }
  sys_write(1, b, nd)
  return 0
45}
46// stderr twins (the fallback diagnostic must not pollute the stdout HTTP payload)
47func hgc_put2(s: *u8) -> i64 { sys_write(2, s, hgc_slen(s)); return 0 }
48func hgc_putn2(v: i64) -> i64 {
  let b: *u8 = sys_mmap(24)
  var m: i64 = v
  if m == 0 { b[0] = 48 as u8; sys_write(2, b, 1); return 0 }
  if m < 0 { sys_write(2, "-" as *u8, 1); m = 0 - m }
  var nd: i64 = 0
  var t: i64 = m
  while t > 0 { nd = nd + 1; t = t / 10 }
  var i: i64 = nd - 1
  while i >= 0 { b[i] = (48 + (m % 10)) as u8; m = m / 10; i = i - 1 }
  sys_write(2, b, nd)
  return 0
60}
61
62// ---- BODY MODE (additive; the DEFAULT output is byte-for-byte unchanged, rule 19) -------------
63// This client has always emitted the RAW response: status line, headers, and -- when the origin
64// uses Transfer-Encoding: chunked -- the hex chunk-size markers interleaved with the payload.
65// That is right for a debugging client and CORRUPTING for any caller that content-addresses what
66// it fetched: a sha256 over that stream hashes a framing artifact, not the document.
67// â˜…â˜…â˜…â˜…â˜…IT DOES NOT FAIL -- IT SUCCEEDS WITH THE WRONG BYTES, which is why it went unnoticed. A
68// fetch that errors gets fixed; a fetch that returns plausible-but-reframed bytes gets TRUSTED.
69// Measured 2026-07-31 against https://www.rfc-editor.org/rfc/rfc7748.txt: the first chunk header
70// read `9982` = 39298 decimal, exactly the true body length, sitting inline in the output.
71// `--body` emits the de-chunked entity body only. Callers of the default path are untouched.
72// NOTE ON DRY (rule 15): _hdl_build/nx_http_chunked.nx already implements this exact decode, but
73// NO file in this corpus imports across directories, so it is unreachable from runtime/. Fixing
74// the import model is the right long-term move; blocking a pinned-evidence chain on it is not.
75// --hdr SUPPORT. It composes the SHIPPED primitive nx_https_get_complete_cookie_xhdr rather than adding
76// a second request builder: the extra-header capability already existed in the library and only the CLI
77// lacked a way to reach it, so this is an exposure, not a new ruler.
78//
79// THE GUARD IS THE LOAD-BEARING PART. A CR or LF inside a caller-supplied header value is HTTP REQUEST
80// SPLITTING: it closes the header the caller was given and starts one the caller never authorised, on a
81// client that speaks to arbitrary hosts. So a value carrying either byte is REFUSED BY NAME rather than
82// sanitised -- ★SANITISING AN INJECTION ATTEMPT HIDES IT; REFUSING IT REPORTS IT. A missing ':' is also
83// refused, because a header with no field-name separator is not a header and the server's parse of it is
84// undefined. Returns the built length, or a negative code naming which rule fired.
85const HGC_HDR_ERR_NOCOLON: i64 = 0 - 1
86const HGC_HDR_ERR_CTLBYTE: i64 = 0 - 2
87const HGC_HDR_ERR_CAPACITY: i64 = 0 - 3
88func hgc_build_xhdr(v: *u8, out: *u8) -> i64 {
  var i: i64 = 0
  var colon: i64 = 0
  while v[i] != (0 as u8) {
      if i >= HGC_XHDR_CAP-2 { return HGC_HDR_ERR_CAPACITY }
      let c: i64 = v[i] as i64
      if c == 13 { return HGC_HDR_ERR_CTLBYTE }
      if c == 10 { return HGC_HDR_ERR_CTLBYTE }
      if c == 58 { colon = 1 }
      out[i] = v[i]
      i = i + 1
  }
  if colon == 0 { return HGC_HDR_ERR_NOCOLON }
  if i == 0 { return HGC_HDR_ERR_NOCOLON }
  out[i] = 13 as u8
  out[i+1] = 10 as u8
  return i + 2
105}
106func hgc_streq(a: *u8, b: *u8) -> i64 {
  var i: i64 = 0
  while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 }
  if b[i] != (0 as u8) { return 0 }
  return 1
111}
112
113func hgc_hexval(c: i64) -> i64 {
  if c >= 48 { if c <= 57 { return c - 48 } }
  if c >= 97 { if c <= 102 { return c - 87 } }
  if c >= 65 { if c <= 70 { return c - 55 } }
  return 0 - 1
118}
119
120func hgc_hdr_end(b: *u8, n: i64) -> i64 {
  var i: i64 = 0
  while i + 3 < n {
      if (b[i] as i64) == 13 { if (b[i+1] as i64) == 10 { if (b[i+2] as i64) == 13 { if (b[i+3] as i64) == 10 { return i + 4 } } } }
      i = i + 1
  }
  return 0 - 1
127}
128
129// Only an exact Transfer-Encoding field controls decoding. Return -1 for unsupported or duplicate
130// coding declarations so --body cannot silently treat encoded bytes as an entity body.
131func hgc_hdr_chunked(b: *u8, n: i64, he: i64) -> i64 {
  if he < 0 || he > n { return -1 }
  var line: i64 = 0
  var found: i64 = 0
  while line < he {
      var end: i64 = line
      while end+1 < he { if b[end] == 13 as u8 && b[end+1] == 10 as u8 { break }; end=end+1 }
      if end+1 >= he { return -1 }
      if _gc_ci_match(b,line,end,"transfer-encoding:" as *u8,18) == 1 {
          if found != 0 { return -1 }
          var start: i64 = line+18
          while start < end { if b[start] != 32 as u8 && b[start] != 9 as u8 { break }; start=start+1 }
          var stop: i64 = end
          while stop > start { if b[stop-1] != 32 as u8 && b[stop-1] != 9 as u8 { break }; stop=stop-1 }
          // This client only decodes a single chunked transfer coding. Other chains need their own decoder.
          if stop-start != 7 { return -1 }
          if _gc_ci_match(b,start,stop,"chunked" as *u8,7) != 1 { return -1 }
          found=1
      }
      line=end+2
  }
  return found
153}
154
155// Decode a chunked body. Tolerates chunk extensions after the size, stops at the 0-size chunk.
156// Returns the decoded length, or -1 when the first size line has no hex digits at all.
157func hgc_dechunk(src: *u8, n: i64, out: *u8) -> i64 {
  return nx_http_dechunk(src,n,out,HGC_OUTCAP)
159}
160
161func main(argc: i64, argv: *i64) -> i64 {
  if argc < 2 { hgc_put("ERROR: usage: nx_https_get <url> [connect-host:port] [--body] [--hdr 'K: V']\n" as *u8); return 2 }
  let url: *u8 = argv[1] as *u8
  let now: i64 = sys_now_realtime_sec()
165
  // ---- trust store + real TLS entropy buffers (refilled per attempt) ----
  let r: i64 = nx_trust_store_load_from_certdata(HGC_CERTDATA, 512, HGC_MAGIC_4194304)
  if r <= 0 { hgc_put("ERROR: trust-store load failed (data/mozilla_certdata.txt on daemon CWD?)\n" as *u8); return 3 }
  let store: *TrustStore = r as *TrustStore
  let cr: *u8 = sys_mmap(32)
  let priv: *u8 = sys_mmap(32)
172
  // ---- parse the REAL url (drives SNI / Host / path / cert-name) ----
  let url_p: *NxUrl = nx_url_new()
  let target_raw: *u8 = sys_mmap(32)
  let target: *NxHttpsTarget = target_raw as *NxHttpsTarget
  target.url = url_p
  target.port = 0
  if nx_https_url_for_fetch(url, target) != NX_HTTPS_URL_OK { hgc_put("ERROR: bad url\n" as *u8); return 2 }
180
  // ---- connect override parsed ONCE (an attempt reconnects with the same endpoint) ----
  var have_ov: i64 = 0
  let ipbox: *i64 = sys_mmap(8) as *i64
  let portbox: *i64 = sys_mmap(8) as *i64
  // Flags and the connect-override are position-independent so `--body` may appear before or
  // after the override. Parsing argv[2] positionally would have made `nx_https_get <url> --body`
  // fail as a malformed ip:port -- an error message pointing at the wrong argument.
  var body_only: i64 = 0
  // One --hdr, deliberately: a repeatable flag needs an accumulating buffer and a cap on the total,
  // and one extra header is what the roster needs today (asserting a header-CONDITIONAL behaviour on a
  // live surface). A second one can be added when something needs it -- not before.
  var xhdr: *u8 = 0 as *u8
  var xhdr_len: i64 = 0
  var ai: i64 = 2
  while ai < argc {
      let a: *u8 = argv[ai] as *u8
      if hgc_streq(a, "--body" as *u8) == 1 { body_only = 1 } else {
      if hgc_streq(a, "--hdr" as *u8) == 1 {
          // It takes a VALUE, so it must be matched BEFORE the connect-override arm: otherwise the
          // header text falls through to hgc_parse_ipport and is reported as a malformed ip:port --
          // an error naming the wrong argument, which is the defect the comment above already warns of.
          if ai + 1 >= argc { hgc_put("ERROR: --hdr needs a value, e.g. --hdr 'X-Name: value'\n" as *u8); return 2 }
          let hv: *u8 = argv[ai+1] as *u8
          xhdr = sys_mmap(HGC_XHDR_CAP)
          xhdr_len = hgc_build_xhdr(hv, xhdr)
          if xhdr_len == HGC_HDR_ERR_CAPACITY { hgc_put("ERROR: --hdr exceeds header buffer capacity\n" as *u8); return 2 }
          if xhdr_len == HGC_HDR_ERR_CTLBYTE { hgc_put("ERROR: --hdr value contains CR or LF -- REFUSED. That is HTTP request splitting: it would close the header you asked for and open one you did not authorise. Value NOT sanitised, NOT sent.\n" as *u8); return 2 }
          if xhdr_len == HGC_HDR_ERR_NOCOLON { hgc_put("ERROR: --hdr value has no ':' field-name separator, so it is not a header. Use --hdr 'X-Name: value'\n" as *u8); return 2 }
          ai = ai + 1
      } else {
          if hgc_parse_ipport(a, ipbox, portbox) != 1 { hgc_put("ERROR: bad connect-override (want a.b.c.d:port)\n" as *u8); return 2 }
          have_ov = 1
      } }
      ai = ai + 1
  }
216
  let fd_p: *i64 = sys_mmap(16) as *i64
  let val_raw: *u8 = sys_mmap(128)
  let val_ctx: *TlsValidationContext = val_raw as *TlsValidationContext
220
  // ---- connect + handshake, TWO attempts: 0 = chrome-JA3 hello (live-identical), 1 = plain hello fallback ----
  var attempt: i64 = 0
  var session_i: i64 = 0
  var fd: i64 = 0 - 1
  while attempt < 2 {
      // fresh connect per attempt (a rejected handshake kills the socket)
      if have_ov == 1 {
          let sa: *u8 = sys_mmap(16)
          nx_https_build_sockaddr(sa, ipbox[0], portbox[0])
          let cfd: i64 = sys_socket(NX_HTTPS_AF_INET, NX_HTTPS_SOCK_STREAM, 0)
          if cfd < 0 { hgc_put("ERROR: socket failed\n" as *u8); return 3 }
          sys_set_socket_timeout(cfd, 15)
          if nx_connect_bounded(cfd, sa, 16, NX_CONN_DEFAULT_MS) < 0 { sys_close(cfd); hgc_put("ERROR: connect override failed (edge unreachable at that ip:port)\n" as *u8); return 3 }
          fd_p[0] = cfd
      } else {
          if nx_https_url_connect(target, url, now, fd_p) != NX_HTTPS_CONNECT_OK { hgc_put("ERROR: connect failed\n" as *u8); return 3 }
      }
      fd = fd_p[0]
239
      // fresh entropy per attempt (never reuse a client-random across handshakes)
      nx_csprng_fill(cr, 32)
      nx_csprng_fill(priv, 32)
243
      // validation ctx + F799 certloop cache, re-armed per attempt
      val_ctx.store = store
      val_ctx.sni_host = url + target.url.host_off
      val_ctx.sni_host_len = target.url.host_len
      val_ctx.now_epoch = now
      let tcc_hit: i64 = tcc_load(url + target.url.host_off, target.url.host_len, now, val_ctx)
      tcc_arm(val_ctx)
      if tcc_hit == 1 { sys_write(2, "nishi-tcc candidate loaded\n" as *u8, 27) }
252
      var sr: i64 = 0
      if attempt == 0 { sr = nx_tls13_client_session_run_chrome(fd, url + target.url.host_off, target.url.host_len, cr, priv, val_ctx) }
      else { sr = nx_tls13_client_session_run(fd, url + target.url.host_off, target.url.host_len, cr, priv, val_ctx) }
256
      if sr >= 0 { session_i = sr; attempt = 99 }
      else {
          sys_close(fd)
          if attempt == 0 {
              hgc_put2("nishi-hello chrome-hello REJECTED verdict=" as *u8)
              hgc_putn2(0 - sr)
              hgc_put2(" -- retrying with plain hello\n" as *u8)
              attempt = 1
          } else {
              hgc_put("ERROR: TLS handshake failed verdict=" as *u8)
              hgc_putn(0 - sr)
              hgc_put(" (2=emitCH 3=wrCH 4=readSH 5=recvSH 6=readHS 7=recvHS/CERT 8=emitCF 10=deriveApp 11=budget)\n" as *u8)
              return 4
          }
      }
  }
  if session_i == 0 { hgc_put("ERROR: TLS handshake failed (no session)\n" as *u8); return 4 }
  let session: *Tls13ClientSession = session_i as *Tls13ClientSession
  tcc_save(url + target.url.host_off, target.url.host_len, now, val_ctx)
276
  // ---- path (default "/", append ?query verbatim -- else query-string APIs silently drop) ----
  var path_ptr: *u8 = url + target.url.path_off
  var path_len: i64 = target.url.path_len
  if path_len == 0 {
      let dp: *u8 = sys_mmap(2)
      dp[0] = 47 as u8
      path_ptr = dp
      path_len = 1
  }
  if target.url.query_len > 0 {
      let full: *u8 = sys_mmap(path_len + target.url.query_len + 4)
      var fo: i64 = 0
      var pci: i64 = 0
      while pci < path_len { full[fo] = path_ptr[pci]; fo = fo + 1; pci = pci + 1 }
      full[fo] = 63 as u8; fo = fo + 1   // '?'
      let qp: *u8 = url + target.url.query_off
      var qci: i64 = 0
      while qci < target.url.query_len { full[fo] = qp[qci]; fo = fo + 1; qci = qci + 1 }
      path_ptr = full
      path_len = fo
  }
298
  // ---- HTTP GET; Host header = the REAL url host ----
  let out: *u8 = sys_mmap(HGC_OUTCAP)
  // WITHOUT --hdr THIS IS THE EXACT CALL IT ALWAYS WAS. The xhdr variant is reached only when a header
  // was actually supplied, so every existing invocation is byte-identical and the 61 library callers of
  // the base primitive are untouched. The cookie argument is empty because this flag is about headers;
  // a cookie CLI is a separate question and is not being answered here by accident.
  var n: i64 = 0
  if xhdr_len > 0 {
      n = nx_https_get_complete_cookie_xhdr(session, fd, path_ptr, path_len, url + target.url.host_off, target.url.host_len, "" as *u8, 0, xhdr, xhdr_len, out, HGC_OUTCAP)
  }
  if xhdr_len <= 0 {
      n = nx_https_get_complete(session, fd, path_ptr, path_len, url + target.url.host_off, target.url.host_len, out, HGC_OUTCAP)
  }
  sys_close(fd)
  if n < 0 { hgc_put("ERROR: HTTP fetch failed code=" as *u8); hgc_putn(0 - n); hgc_put("\n" as *u8); return 5 }
  if body_only == 0 { sys_write(1, out, n); return 0 }
  let he: i64 = hgc_hdr_end(out, n)
  if he < 0 { hgc_put("ERROR: --body: no CRLFCRLF header terminator; refusing to guess where the body starts\n" as *u8); return 6 }
  let coding: i64 = hgc_hdr_chunked(out,n,he)
  if coding < 0 { hgc_put("ERROR: --body: unsupported or ambiguous transfer coding\n" as *u8); return 7 }
  if coding == 0 { sys_write(1, out + he, n - he); return 0 }
  let db: *u8 = sys_mmap(HGC_OUTCAP)
  let dn: i64 = hgc_dechunk(out + he, n - he, db)
  if dn < 0 { hgc_put("ERROR: --body: malformed chunked framing; refusing to emit a partial document\n" as *u8); return 7 }
  sys_write(1, db, dn)
  return 0
325}
326
327// parse "a.b.c.d:port" -> ip_out (big-endian packed u32, e.g. 127.0.0.1 -> 0x7F000001) + port_out. 1 ok / 0 bad.
328func hgc_parse_ipport(s: *u8, ip_out: *i64, port_out: *i64) -> i64 {
  var packed: i64 = 0
  var val: i64 = 0
  var nocts: i64 = 0
  var port: i64 = 0
  var indots: i64 = 1
  var i: i64 = 0
  while s[i] != (0 as u8) {
      let c: i64 = s[i] as i64
      if indots == 1 {
          if c == 46 { packed = (packed << 8) | (val & 0xff); nocts = nocts + 1; val = 0 }
          else { if c == 58 { packed = (packed << 8) | (val & 0xff); nocts = nocts + 1; val = 0; indots = 0 }
          else { if c < 48 { return 0 } if c > 57 { return 0 } val = val * 10 + (c - 48) } }
      } else {
          if c < 48 { return 0 }
          if c > 57 { return 0 }
          port = port * 10 + (c - 48)
      }
      i = i + 1
  }
  if indots == 1 { return 0 }   // no ':' -> no port given
  if nocts != 4 { return 0 }    // need exactly 4 octets
  if port <= 0 { return 0 }
  if port > HGC_MAGIC_65535 { return 0 }
  ip_out[0] = packed
  port_out[0] = port
  return 1
355}
nx_https_get_cli2_candidate.nx source

`nx_https_get_cli2_candidate.nx` source
