nx_vecfetch.nx source

1// nx_vecfetch.nx -- ACQUIRE EXTERNAL ANSWERS WITH THE AGENT OUT OF THE BYTE PATH. 2// 3// THE HOLE THIS CLOSES (measured 2026-07-31): every clause in the evidence apparatus verifies the 4// COMPARISON and none verifies the PROVENANCE OF THE ANSWERS. An agent can write a gate against 5// "RFC 8032 test vectors" by typing them from its own recall -- it compiles, runs GREEN, and yields a row 6// with a real ref=, a real refdig= and a real execution. refdig= commits to the bytes we compared against; 7// it CANNOT prove those bytes came from the IETF rather than from the model that wrote the file. 8// 9// ============================ v2, 2026-07-31: TWO DEFECTS FOUND AND FIXED ============================ 10// 11// ⚠DEFECT 1 -- I HASHED THE ENVELOPE, NOT THE DOCUMENT. v1 hashed whatever nx_https_get returned. By 12// contract that is the ENTIRE HTTP RESPONSE: status line, headers, and chunked-transfer framing. So all 14 13// pinned "authority documents" were HTTP WIRE TRANSCRIPTS -- rfc2202.txt carried 707 bytes of Cloudflare 14// headers and a `2ea6` chunk-size line before the RFC even began. nx_extvec_authentic_gate now reports 15// 14/14 CONTAMINATED against the v1 corpus. Verified against a stack sharing no code (.NET WebClient): 16// chunk size 0x2ea6 = 11942 = the true document length exactly. 17// 18// ★★★★★ THE LAW v1 GOT WRONG, CORRECTED: 19// v1 said "hash the bytes before anything touches them". That is too crude -- it hashes the ENVELOPE. 20// THE RULE IS: HASH THE OBJECT YOU ARE MAKING A CLAIM ABOUT, AT THE EARLIEST POINT IT EXISTS. 21// The document does not exist until the transport framing is removed. Hashing before that point is not 22// "hashing earlier and therefore safer" -- it is hashing A DIFFERENT OBJECT. 23// The anti-tamper property v1 was protecting is PRESERVED: the decode below is a pure, deterministic, 24// in-process transform. No agent, no second tool, and no disk sits between the socket and the digest. 25// 26// ⚠DEFECT 2 -- THE DECODER ALREADY EXISTED AND I DID NOT ADOPT IT. nx_fetch_unit.nx has had fu_body_off / 27// fu_status / fu_find_ci / fu_dechunk, and a fully staged nx_fetch_staged, the whole time. v1 called the 28// RAW primitive underneath them. ★THE ADOPTION GAP, NOT A MISSING PRIMITIVE -- the same class already in 29// memory. The fix is to ADOPT the proven decoder, not to write a ninth private copy of one. (`nx_body_off` 30// is currently duplicated across at least 9 files -- that Rule-15 extraction is filed separately; this 31// organ deliberately adds no new copy.) 32// 33// ============================ CORROBORATION: THE JULY-2026 BAR ============================ 34// A digest pin proves INTEGRITY-SINCE-FETCH. It cannot prove AUTHENTICITY-AT-SOURCE, because a corrupted 35// or re-framed fetch pins exactly as cleanly as a good one -- the pin is self-consistent by construction. 36// The state of the art does not solve this with a better hash; it solves it with ANOTHER PARTY: 37// - CA/Browser Forum MPIC (mandatory for TLS/S-MIME issuance): validate from MULTIPLE INDEPENDENT 38// NETWORK PERSPECTIVES, and if the perspectives DISAGREE, THE PROCESS HALTS. It does not pick one. 39// - Sigstore/Rekor: a transparency log exists so verification need not trust a single party. 40// So this organ takes an OPTIONAL third argument: a digest obtained by an INDEPENDENT STACK. If supplied 41// and it disagrees, the fetch is REFUSED -- halt, never reconcile, never prefer our own. If not supplied, 42// the output is explicitly stamped UNCORROBORATED so a single-perspective acquisition can never be 43// mistaken for a corroborated one downstream. 44// ★TWO FETCHES THROUGH THE SAME CODE ARE NOT TWO PERSPECTIVES. The corroborating digest MUST come from a 45// stack that shares no parser with this one, or it only re-measures our own bug. 46// 47// ORDERING, AND IT IS NOT REARRANGEABLE: 48// 1 fetch into a buffer 2 verify status, strip headers, de-chunk -> THE DOCUMENT 49// 3 sha256 THE DOCUMENT, in this process, before it touches disk 50// 4 compare against the independent perspective (halt on disagreement) 51// 5 only then write 6 print refsrc= / refsrcdig= / bytes= / corroboration state 52// 53// usage: nx_vecfetch <url> <destpath> [independent-sha256-hex] 54// license_tier: ORIGINAL expect_exit: 0 55import "nx_syscalls.nx" 56import "nx_csprng.nx" 57import "nx_x509_trust_store.nx" 58import "nx_trust_store_load_from_certdata.nx" 59import "nx_https_get.nx" 60import "nx_fetch_unit.nx" 61import "nx_sha256_wasm.nx" 62import "nx_inflate.nx" 63 64const INF_RC_NOT_GZIP: i64 = 0 - 7 // inf_gunzip's rc for a body that is not a gzip container (identity) 65 66func vf_put(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 } 67func vf_wb(b: *u8, n: i64) -> i64 { sys_write(1, b, n); return 0 } 68 69func vf_num(v: i64) -> i64 { 70 var m: i64 = v 71 if m < 0 { vf_put("-" as *u8); m = 0 - m } 72 let t: *u8 = sys_mmap(32) 73 var k: i64 = 0 74 if m == 0 { t[0] = 48 as u8; k = 1 } 75 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 } 76 let b: *u8 = sys_mmap(32) 77 var j: i64 = 0 78 while j < k { b[j] = t[k - 1 - j]; j = j + 1 } 79 sys_write(1, b, k) 80 return 0 81} 82 83func vf_hexnib(v: i64) -> i64 { if v < 10 { return 48 + v } return 87 + v } 84func vf_lc(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c } 85 86func main(argc: i64, argv: *i64) -> i64 { 87 if argc < 3 { vf_put("usage: nx_vecfetch <url> <destpath> [independent-sha256-hex]\n" as *u8); return 2 } 88 let url: *u8 = argv[1] as *u8 89 let dest: *u8 = argv[2] as *u8 90 var corro: *u8 = 0 as *u8 91 if argc >= 4 { corro = argv[3] as *u8 } 92 let now: i64 = sys_now_realtime_sec() 93 94 let r: i64 = nx_trust_store_load_from_certdata("data/mozilla_certdata.txt\x00" as *u8, 512, 4194304) 95 if r <= 0 { vf_put("ERROR: trust-store load failed (need data/mozilla_certdata.txt on CWD)\n" as *u8); return 3 } 96 let store: *TrustStore = r as *TrustStore 97 98 let cr: *u8 = sys_mmap(32) 99 nx_csprng_fill(cr, 32) 100 let priv: *u8 = sys_mmap(32) 101 nx_csprng_fill(priv, 32) 102 103 // ---- STEP 1: FETCH (returns the WHOLE response: status line + headers + framed body) ---- 104 let cap: i64 = 8388608 105 let buf: *u8 = sys_mmap(cap) 106 let v: i64 = nx_https_get(url, cr, priv, store, now, buf, cap) 107 vf_put("fetch_rc=" as *u8); vf_num(v); vf_put("\n" as *u8) 108 // ★MY FIRST GUARD ASKED THE WRONG QUESTION. nx_https_get_verdict_is_valid(v) is a RANGE CHECK ON THE 109 // VERDICT ENUM, not "did the fetch succeed": a success returns the RESPONSE LENGTH, far above the enum 110 // bound. Narrowed to exactly the success shape -- widening to `v >= 0` would let every failure verdict 111 // through as a "body" and manufacture provenance for bytes never received. 112 if v < NX_HTTPS_GET_VERDICT_N { 113 vf_put("ERROR: fetch returned a FAILURE VERDICT -- refusing to write or hash anything.\n" as *u8) 114 return 3 115 } 116 if v > cap { vf_put("ERROR: reported length exceeds the buffer -- truncation suspected, refusing.\n" as *u8); return 3 } 117 118 // ---- STEP 2: DECODE THE TRANSPORT ENVELOPE. THE DOCUMENT DOES NOT EXIST UNTIL THIS COMPLETES. ---- 119 // 2a. Status. A 404 body or a redirect stub pins just as cleanly as the real document; only 200 may be 120 // treated as the authority's answer. v1 never looked at the status line at all. 121 let st: i64 = fu_status(buf, v) 122 vf_put("http_status=" as *u8); vf_num(st); vf_put("\n" as *u8) 123 if st != 200 { 124 vf_put("ERROR: HTTP status is not 200 -- refusing to pin a non-answer as an external answer.\n" as *u8) 125 return 3 126 } 127 128 // 2b. Header/body split. 129 let bo: i64 = fu_body_off(buf, v) 130 if bo <= 0 { 131 vf_put("ERROR: no CRLFCRLF header/body break found -- cannot identify the document.\n" as *u8) 132 return 3 133 } 134 135 // 2c. Chunked? Header names are case-insensitive (RFC 9110), and the probe is bounded to the header 136 // block so a body that DISCUSSES chunked encoding cannot flip the decoder. 137 var chunked: i64 = 0 138 let te: i64 = fu_find_ci(buf, bo, "\ntransfer-encoding:" as *u8) 139 if te >= 0 { 140 if fu_find_ci(((buf as i64) + te) as *u8, 96, "chunked" as *u8) >= 0 { chunked = 1 } 141 } 142 143 let doc: *u8 = sys_mmap(cap) 144 var L: i64 = 0 145 if chunked == 1 { 146 L = fu_dechunk(buf, bo, v, doc, cap) 147 vf_put("framing=chunked decoded_bytes=" as *u8); vf_num(L); vf_put("\n" as *u8) 148 } else { 149 var i: i64 = 0 150 while bo + i < v { doc[i] = buf[bo + i]; i = i + 1 } 151 L = i 152 vf_put("framing=identity body_bytes=" as *u8); vf_num(L); vf_put("\n" as *u8) 153 } 154 if L <= 0 { vf_put("ERROR: empty document after decode\n" as *u8); return 3 } 155 156 // 2d. SELF-CHECK: the decoded document must be strictly smaller than the transcript, and must not still 157 // begin with an HTTP status line. A decoder that silently no-ops is the exact failure that produced 158 // the v1 corpus, and it must be LOUD rather than merely absent. 159 if L >= v { vf_put("ERROR: decode produced no reduction -- envelope not removed, refusing.\n" as *u8); return 3 } 160 var looks: i64 = 1 161 let sig: *u8 = "HTTP/" as *u8 162 var s: i64 = 0 163 while s < 5 { if doc[s] != sig[s] { looks = 0 } s = s + 1 } 164 if looks == 1 { vf_put("ERROR: decoded document still begins 'HTTP/' -- refusing.\n" as *u8); return 3 } 165 166 // ---- STEP 2e: CONTENT-ENCODING. The framing decode above strips CHUNKING, not Content-Encoding. 167 // rfc-editor.org (and most origins) gzip the body regardless of what we asked, so `doc` may still 168 // be a gzip stream -- pinning it would commit the digest to COMPRESSED bytes and every downstream 169 // KAT that parses the document as ASCII fails on the pin or the parse. We do NOT re-derive what a 170 // gzip container looks like (that ruler lives in inf_gunzip): attempt to inflate, and if the body 171 // is not a gzip container inf_gunzip returns INF_RC_NOT_GZIP and we keep the identity body as-is. 172 // Any OTHER negative rc is a CORRUPT gzip stream and fails LOUD -- inf_gunzip verifies CRC32 and 173 // ISIZE, so we never pin bytes we could not fully and correctly decompress. Reuses the existing 174 // `cap` buffer and copies back into `doc` so the hash and write steps below are unchanged. 175 let plain: *u8 = sys_mmap(cap) 176 let PL: i64 = inf_gunzip(doc, L, plain, cap) 177 if PL >= 0 { 178 var ci: i64 = 0 179 while ci < PL { doc[ci] = plain[ci]; ci = ci + 1 } 180 L = PL 181 vf_put("content_encoding=gzip inflated_bytes=" as *u8); vf_num(PL); vf_put("\n" as *u8) 182 } else { 183 if PL != INF_RC_NOT_GZIP { vf_put("ERROR: Content-Encoding gzip inflate failed rc=" as *u8); vf_num(PL); vf_put(" -- refusing to pin a body we cannot decompress.\n" as *u8); return 3 } 184 vf_put("content_encoding=identity\n" as *u8) 185 } 186 187 // ---- STEP 3: HASH THE DOCUMENT, HERE, BEFORE IT TOUCHES DISK OR ANY OTHER PROCESS ---- 188 let ctx: *u8 = sys_mmap(1024) 189 let dig: *u8 = sys_mmap(64) 190 nx_sha256_one_shot(doc, L, ctx, dig) 191 let hx: *u8 = sys_mmap(80) 192 var i2: i64 = 0 193 while i2 < 32 { 194 hx[i2 * 2] = vf_hexnib(((dig[i2] as i64) / 16) & 15) as u8 195 hx[i2 * 2 + 1] = vf_hexnib((dig[i2] as i64) & 15) as u8 196 i2 = i2 + 1 197 } 198 199 // ---- STEP 4: CORROBORATION. HALT ON DISAGREEMENT -- NEVER RECONCILE, NEVER PREFER OUR OWN. ---- 200 var corro_state: i64 = 0 // 0 = none supplied, 1 = agreed 201 if (corro as i64) != 0 { 202 var cl: i64 = 0 203 while corro[cl] != (0 as u8) { cl = cl + 1 } 204 if cl != 64 { 205 vf_put("ERROR: independent digest must be 64 hex chars, got " as *u8); vf_num(cl); vf_put("\n" as *u8) 206 return 3 207 } 208 var agree: i64 = 1 209 var k: i64 = 0 210 while k < 64 { if vf_lc(corro[k] as i64) != (hx[k] as i64) { agree = 0 } k = k + 1 } 211 if agree == 0 { 212 vf_put("\nHALT: PERSPECTIVES DISAGREE -- REFUSING TO WRITE.\n" as *u8) 213 vf_put(" ours(sovereign) = " as *u8); vf_wb(hx, 64); vf_put("\n" as *u8) 214 vf_put(" independent = " as *u8); vf_put(corro); vf_put("\n" as *u8) 215 vf_put(" Per CA/Browser Forum MPIC: on disagreement the process HALTS. Picking either side\n" as *u8) 216 vf_put(" would be choosing which party to trust, which is the thing corroboration exists to\n" as *u8) 217 vf_put(" avoid. Resolve the disagreement, do not average it.\n" as *u8) 218 return 4 219 } 220 corro_state = 1 221 } 222 223 // ---- STEP 5: only now, write ---- 224 let fd: i64 = sys_openat_wr(dest, 0x1a4) 225 if fd < 0 { vf_put("ERROR: cannot open destpath for write\n" as *u8); return 3 } 226 var off: i64 = 0 227 while off < L { 228 let wn: i64 = sys_write(fd, ((doc as i64) + off) as *u8, L - off) 229 if wn <= 0 { sys_close(fd); vf_put("ERROR: short write -- destination NOT trustworthy\n" as *u8); return 3 } 230 off = off + wn 231 } 232 sys_close(fd) 233 234 // ---- STEP 6: report ---- 235 vf_put("\n refsrc=" as *u8); vf_put(url); vf_put("\n" as *u8) 236 vf_put(" refsrcdig=" as *u8); vf_wb(hx, 64); vf_put("\n" as *u8) 237 vf_put(" bytes=" as *u8); vf_num(L); vf_put("\n" as *u8) 238 vf_put(" transcript_bytes=" as *u8); vf_num(v); vf_put(" document_bytes=" as *u8); vf_num(L); vf_put("\n" as *u8) 239 if corro_state == 1 { 240 vf_put(" corroboration=AGREED (independent stack confirms this digest)\n" as *u8) 241 } else { 242 vf_put(" corroboration=UNCORROBORATED -- ONE PERSPECTIVE ONLY.\n" as *u8) 243 vf_put(" This digest proves integrity-since-fetch, NOT authenticity-at-source. Supply a digest\n" as *u8) 244 vf_put(" from a stack sharing no code with this one to raise it.\n" as *u8) 245 } 246 return 0 247}