code wiki / (root) / nx_content_put_client.nx

nx_content_put_client.nx source

↩ module page · 929 lines · 50154 B

1// nx_content_put_client.nx -- THE SOVEREIGN laptop->NAS UPLOAD CLIENT. Retires nx_content_put_client.py. 2// 3// WHY THIS EXISTS (operator standing order 2026-09-03: "i dont want py or js or sh or anything else in the 4// build lanes i want us nishilang soverign", and again "dont build python wtf build nishi"). 5// nx_content_put has been the sovereign RECEIVER since 2026-08-23; its only driver was an 8,525-byte 6// Python script shelling out to a Python MCP wire. This is the NishiLang driver. 7// 8// IT WAS BLOCKED ON ONE MISSING PRIMITIVE, NOW BUILT. Driving the receiver needs POST-a-body-to-a-URL. 9// The estate had no such thing: nx_https_post_json took a URL and a body and issued a GET (corrected to 10// refuse, 2026-09-03), and nx_https_post_complete_xhdr needs an already-established session. hp_post_json 11// (nx_https_post_lib.nx) is the glue, and this file is its first consumer. 12// 13// nx_content_put_client [dryrun] <local_path> <dest> <capfile> <url> [expect=<sha256hex>] 14// 15// IT REFUSES TO ESCAPE JSON RATHER THAN ESCAPING IT BADLY. 16// Every value this client puts on the wire comes from a KNOWN-SAFE alphabet: a destination path, decimal 17// digits, lowercase hex, base64 (A-Za-z0-9+/=) and a capability token. None of them can contain a quote, 18// a backslash or a control byte. So instead of hand-rolling a JSON escaper -- a defect generator that is 19// wrong in exactly the cases nobody tests -- cc_json_safe() REFUSES any argument carrying a character 20// that would need escaping. A hand-rolled escaper fails SILENTLY and produces a malformed request the 21// server rejects for the wrong reason; a refusal names the offending byte and stops. 22// 23// THE CHUNK SIZE IS THE SERVER'S, READ FROM ITS OWN RECEIPT -- NEVER COMPUTED HERE. 24// nx_content_put derives chunk_raw from the wire body cap, the request skeleton measured at runtime, the 25// largest observed capability token and the i64 digit bound. Recomputing that here would install a second 26// ruler that drifts the moment the server's reserve changes, and the drift would show up as truncated 27// uploads. The client reads chunk_raw= out of the begin receipt and uses it verbatim. 28// 29// A CHUNK RE-SEND IS SAFE; A COMMIT RETRY IS NOT DECIDABLE FROM HERE. 30// The receiver treats an identical re-send as a receipted no-op and REFUSES a different body for an index 31// that already landed (CAS), so a chunk retry can never double-apply. A lost COMMIT response is genuinely 32// UNKNOWN -- the transfer id may be gone because it committed. This client says so and tells the caller 33// to verify by hash rather than guessing; it does not retry commit. 34// 35// STATUS 2026-09-04: THE ROUND TRIP IS NOW PROVEN AND THIS ORGAN IS LIVE. It had never completed a 36// transfer before today and had never been promoted at all -- no nx_content_put_client.elf existed in 37// the serving root -- so the Python it claims to retire was in fact the only working shipper. Two 38// defects stood between the two facts: this client did not follow the edge's JOB-STARTED promotion (it 39// read the JOB id as the transfer id, both being spelled `id=`), and the shared TLS transport wrote 40// every request as ONE record, capping a body at 8 KB against the 48,402-byte chunks the server offers. 41// Both are fixed; measured since: 24 B, 12,083 B, and 410,443 B (9 chunks) all E2E OK, promoted live. 42// The 2026-09-03 line this replaces said the envelope was proven and the round trip was not. It was an 43// honest gap then and it is closed now -- but note it stood for a DAY while the header above it claimed 44// the organ retired the Python, which is the overclaiming-header class this estate keeps paying for. 45// 46// exit: 0 ok | 2 usage | 3 local-io | 4 refused-unsafe-arg | 5 begin-failed | 6 chunk-failed | 7 commit-unknown 47// license_tier: ORIGINAL No hw writes (Rule 26). expect_exit: 0 48import "nx_syscalls.nx" 49import "nx_base64.nx" 50import "nx_sha256.nx" 51import "nx_cdc_lib.nx" // DI9: the local plan is cut by the SAME rule as the door's have-list (parameters read from the begin receipt) 52import "nx_https_post_lib.nx" 53import "nx_jobfollow_lib.nx" 54 55const CC_DIGEST: i64 = 32 56const CC_SHAHEX: i64 = 64 57const CC_RESP: i64 = 262144 58const CC_REQ: i64 = 262144 59const CC_NUMBUF: i64 = 32 60// Poll budget for following a JOB-STARTED promotion: 40 x 3000ms = 120s. Derived from the put lane 61// observed behaviour, not picked: a begin is fast, but the artifact appears only after the job is 62// scheduled, and the estate array is degraded. Exhausting this budget reports PENDING, never failure. 63const CC_POLLS: i64 = 40 64const CC_POLL_MS: i64 = 3000 65// Per-chunk re-send budget. 3 tries with a 4s gap: enough to ride out the transient that cost a 66// 410443 byte ship on 2026-09-04, small enough that a genuinely dead endpoint is reported in seconds. 67const CC_CHUNK_TRIES: i64 = 3 68const CC_CHUNK_RETRY_MS: i64 = 4000 69const CC_PATHCAP: i64 = 4096 70const CC_MODE_RW: i64 = 420 71// Ceiling on the missing-index table a resume can hold. NOT a guess: the server derives chunk_raw from 72// its own body cap (48,402 raw today), so 65,536 slots covers a 3.1 GB artifact -- three orders of 73// magnitude above anything this estate ships. It is a TABLE size, not a transfer limit: exceeding it 74// only disables RESUME for that transfer, which falls back to the full send that worked before. 75const CC_MAXCHUNKS: i64 = 65536 76// the JSON-RPC envelope this client sends. Its length is MEASURED at runtime wherever it matters -- 77// a hand-counted length beside a literal is two copies of the same shape that drift apart. 78// It opens exactly THREE objects and ONE array, so the tail closes "]" then three braces. That pairing 79// is asserted arithmetically by nx_content_put_client_gate rather than trusted: the first draft of this 80// constant carried a DOUBLED "params" key with a four-brace tail that balanced against itself, which 81// would have produced well-formed JSON of the WRONG SHAPE -- rejected by the server for a reason naming 82// neither the client nor the real fault. 83const CC_PRE: *u8 = "{\x22jsonrpc\x22:\x222.0\x22,\x22id\x22:1,\x22method\x22:\x22tools/call\x22,\x22params\x22:{\x22name\x22:\x22nx_content_put\x22,\x22arguments\x22:{\x22argv\x22:[" 84const CC_TAIL: *u8 = "}}}" 85const CC_QUOTE: i64 = 34 86const CC_BACKSLASH: i64 = 92 87const CC_SPACE: i64 = 32 88const CC_DEL: i64 = 127 89 90func cc_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n } 91func cc_out(s: *u8) -> i64 { sys_write(1, s, cc_slen(s)); return 0 } 92func cc_eo(s: *u8) -> i64 { sys_write(2, s, cc_slen(s)); return 0 } 93func cc_num(v: i64) -> i64 { 94 let b: *u8 = sys_mmap(CC_NUMBUF) 95 var m: i64 = v 96 var i: i64 = CC_NUMBUF - 1 97 if m == 0 { i = i - 1; b[i] = 48 as u8 } 98 while m > 0 { i = i - 1; b[i] = (48 + (m % 10)) as u8; m = m / 10 } 99 sys_write(1, (b as i64 + i) as *u8, (CC_NUMBUF - 1) - i) 100 return 0 101} 102// write decimal into `out`, return length (no allocation, so it can be used inside the request builder) 103func cc_itoa(v: i64, out: *u8) -> i64 { 104 if v == 0 { out[0] = 48 as u8; return 1 } 105 let tmp: *u8 = sys_mmap(CC_NUMBUF) 106 var m: i64 = v 107 var i: i64 = 0 108 while m > 0 { tmp[i] = (48 + (m % 10)) as u8; m = m / 10; i = i + 1 } 109 var j: i64 = 0 110 while j < i { out[j] = tmp[i - 1 - j]; j = j + 1 } 111 return i 112} 113func cc_hex_into(d: *u8, out: *u8) -> i64 { 114 var i: i64 = 0 115 while i < CC_DIGEST { 116 let v: i64 = d[i] as i64 117 let hi: i64 = (v >> 4) & 15 118 let lo: i64 = v & 15 119 if hi < 10 { out[i * 2] = (48 + hi) as u8 } else { out[i * 2] = (87 + hi) as u8 } 120 if lo < 10 { out[i * 2 + 1] = (48 + lo) as u8 } else { out[i * 2 + 1] = (87 + lo) as u8 } 121 i = i + 1 122 } 123 out[CC_SHAHEX] = 0 as u8 124 return 0 125} 126 127// REFUSE rather than escape. Returns 1 when every byte is safe to place inside a JSON string as-is. 128func cc_json_safe(s: *u8, n: i64) -> i64 { 129 var i: i64 = 0 130 while i < n { 131 let c: i64 = s[i] as i64 132 if c == CC_QUOTE { return 0 } 133 if c == CC_BACKSLASH { return 0 } 134 if c < CC_SPACE { return 0 } 135 if c == CC_DEL { return 0 } 136 i = i + 1 137 } 138 return 1 139} 140 141// find needle in buf[0..n); index just past it, or -1. Exits on a FLAG, never by clobbering the cursor. 142func cc_find(buf: *u8, n: i64, needle: *u8) -> i64 { 143 let m: i64 = cc_slen(needle) 144 if m == 0 { return 0 - 1 } 145 var i: i64 = 0 146 var hit: i64 = 0 - 1 147 while i + m <= n { 148 var j: i64 = 0 149 var same: i64 = 1 150 while j < m { 151 if buf[i + j] != needle[j] { same = 0; j = m } else { j = j + 1 } 152 } 153 if same == 1 { if hit < 0 { hit = i + m } } 154 i = i + 1 155 } 156 return hit 157} 158func cc_int_after(buf: *u8, n: i64, key: *u8) -> i64 { 159 let p: i64 = cc_find(buf, n, key) 160 if p < 0 { return 0 - 1 } 161 var i: i64 = p 162 var v: i64 = 0 163 var got: i64 = 0 164 var go: i64 = 1 165 while go == 1 { 166 if i >= n { go = 0 } else { 167 let c: i64 = buf[i] as i64 168 if c < 48 { go = 0 } else { 169 if c > 57 { go = 0 } else { v = v * 10 + (c - 48); got = 1; i = i + 1 } 170 } 171 } 172 } 173 if got == 0 { return 0 - 1 } 174 return v 175} 176 177// append a quoted JSON string element; returns new offset, or -1 if the value is not safe as-is. 178func cc_put_arg(req: *u8, off: i64, val: *u8, vlen: i64, first: i64) -> i64 { 179 if cc_json_safe(val, vlen) == 0 { return 0 - 1 } 180 var o: i64 = off 181 if first == 0 { req[o] = 44 as u8; o = o + 1 } 182 req[o] = CC_QUOTE as u8; o = o + 1 183 var i: i64 = 0 184 while i < vlen { req[o + i] = val[i]; i = i + 1 } 185 o = o + vlen 186 req[o] = CC_QUOTE as u8; o = o + 1 187 return o 188} 189func cc_streq(a: *u8, b: *u8) -> i64 { 190 var i: i64 = 0 191 while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 } 192 if b[i] != (0 as u8) { return 0 } 193 return 1 194} 195func cc_put_lit(req: *u8, off: i64, lit: *u8) -> i64 { 196 var o: i64 = off 197 let n: i64 = cc_slen(lit) 198 var i: i64 = 0 199 while i < n { req[o + i] = lit[i]; i = i + 1 } 200 return o + n 201} 202 203// ---- IDEMPOTENCY KEY ON EVERY CALL (DI4, 2026-09-05) -------------------------------------------------- 204// The tools/call transport now reads params._idem (nx_toolcall_idem_lib): the first applied outcome is recorded 205// under the key and a retry carrying the same key is REPLAYED, never re-executed. So every request this client 206// sends names itself: cpc-<stage>-<sha16 of the file>-<a>[-<b>]-<t0>, where t0 is THIS RUN's start epoch. Within a 207// run a re-send under an Outcome-Unknown replays; a fresh run (or a --resume) reserves fresh keys instead of 208// replaying a begin whose transfer the server may already have reaped. The alphabet is dash, digits and hex, so 209// the key can never need JSON escaping and cc_json_safe's refusal rule is untouched. 210const CC_IDEM_SHA16: i64 = 16 211func cc_put_idem(req: *u8, off: i64, stage: *u8, shahex: *u8, a: i64, b: i64, t0: i64) -> i64 { 212 var o: i64 = cc_put_lit(req, off, ",\x22_idem\x22:\x22cpc-" as *u8) 213 o = cc_put_lit(req, o, stage) 214 req[o] = 45 as u8; o = o + 1 215 var i: i64 = 0 216 while i < CC_IDEM_SHA16 { req[o] = shahex[i]; o = o + 1; i = i + 1 } 217 req[o] = 45 as u8; o = o + 1 218 let nb: *u8 = sys_mmap(CC_NUMBUF) 219 var l: i64 = cc_itoa(a, nb) 220 i = 0 221 while i < l { req[o] = nb[i]; o = o + 1; i = i + 1 } 222 if b >= 0 { 223 req[o] = 45 as u8; o = o + 1 224 l = cc_itoa(b, nb) 225 i = 0 226 while i < l { req[o] = nb[i]; o = o + 1; i = i + 1 } 227 } 228 req[o] = 45 as u8; o = o + 1 229 l = cc_itoa(t0, nb) 230 i = 0 231 while i < l { req[o] = nb[i]; o = o + 1; i = i + 1 } 232 req[o] = CC_QUOTE as u8; o = o + 1 233 return o 234} 235 236// ---- CROSS-PROCESS RESUME (DL10) ------------------------------------------------------------------- 237// The in-flight retry above survives a TRANSIENT. It cannot survive the process dying, and on a degraded 238// array that is the case that costs a 400 KB ship. The server already exposes everything needed -- 239// `status <id>` returns dest, total, chunk_raw, nchunks and the MISSING index list, and its header says 240// chunk_raw travels in status ON PURPOSE because a resuming client cannot derive it. So the only thing 241// missing was a client that remembers the id across runs. 242// 243// THE SIDECAR IS THE MEMORY AND THE DIGEST IS THE GUARD. It records the transfer id beside the LOCAL 244// file's sha256. Resuming a transfer whose local file has CHANGED would splice two generations into one 245// artifact -- byte-valid and semantically nonsense, exactly the franken-file the get client's header 246// warns about -- so a sha mismatch abandons the sidecar and begins fresh rather than trying to be clever. 247// A size check would not do: a same-size rewrite passes it. 248func cc_state_path(lpath: *u8, out: *u8) -> i64 { 249 var o: i64 = 0 250 while lpath[o] != (0 as u8) { out[o] = lpath[o]; o = o + 1 } 251 out[o] = 46 as u8; out[o+1] = 99 as u8; out[o+2] = 112 as u8 252 out[o+3] = 115 as u8; out[o+4] = 116 as u8; out[o+5] = 97 as u8 253 out[o+6] = 116 as u8; out[o+7] = 101 as u8; out[o+8] = 0 as u8 254 return o + 8 255} 256// Mark every index the server reports MISSING. The list is whitespace-separated decimals after 257// "missing:" and runs to end of line; anything already received is left 0 and will be SKIPPED. 258// Returns how many were marked, so a caller can assert the parse found something rather than 259// silently sending nothing -- a resume that skips every chunk and commits is the worst possible bug. 260func cc_mark_missing(resp: *u8, n: i64, miss: *i64, nch: i64) -> i64 { 261 var i: i64 = 0 262 while i < nch { miss[i] = 0; i = i + 1 } 263 let p: i64 = cc_find(resp, n, "missing:" as *u8) 264 if p < 0 { return 0 - 1 } 265 var k: i64 = p 266 var marked: i64 = 0 267 var cur: i64 = 0 268 var indig: i64 = 0 269 var go: i64 = 1 270 while go == 1 { 271 if k >= n { go = 0 } else { 272 let c: i64 = resp[k] as i64 273 if c == 10 { go = 0 } else { 274 if c >= 48 { if c <= 57 { cur = cur * 10 + (c - 48); indig = 1 } else { go = 0 } } 275 else { 276 if indig == 1 { if cur < nch { miss[cur] = 1; marked = marked + 1 } cur = 0; indig = 0 } 277 } 278 k = k + 1 279 } 280 } 281 } 282 if indig == 1 { if cur < nch { miss[cur] = 1; marked = marked + 1 } } 283 return marked 284} 285 286// Does this argument carry a readcap= prefix? Exactly 8 bytes, compared explicitly rather than by a 287// shared prefix helper, because a loose match would swallow the expect= argument beside it. 288func cc_is_readcap(a: *u8) -> i64 { 289 let want: *u8 = "readcap=" as *u8 290 var i: i64 = 0 291 while i < 8 { 292 if a[i] == (0 as u8) { return 0 } 293 if a[i] != want[i] { return 0 } 294 i = i + 1 295 } 296 return 1 297} 298// THE CONTRACT THIS SATISFIES IS PUBLISHED ON /compare/deploy AS DL10 _ABSENT_:cc_resume_from_status, 299// AND IT IS EXTRACTED HERE SO THE CODE MEETS THE CONTRACT RATHER THAN THE CONTRACT BEING BENT TO FIT 300// THE CODE. The logic first shipped inline in main, which left the board cell reading WATCHING while 301// the capability was live -- exactly the mechanically-green-editorially-false mismatch the prose axis 302// of nx_board_contract_gate exists to catch, so leaving it would have been shipping a defect my own 303// gate hunts. 304// 305// Returns 1 when a resumable transfer was adopted, 0 otherwise. On 1, out[] carries 306// [0]=transfer id [1]=chunk_raw [2]=nchunks [3]=chunks still missing, and `miss` is marked. 307// EVERY ONE OF THOSE NUMBERS COMES FROM THE SERVER'S `status` RECEIPT, never from anything this client 308// remembered about what it thinks it sent -- the client's only memory is the transfer id. 309func cc_resume_from_status(store_i: i64, url: *u8, spath: *u8, shahex: *u8, 310 capraw: *u8, caplen: i64, 311 resp: *u8, respcap: i64, 312 rcraw: *u8, rclen: i64, 313 miss: *i64, out: *i64, t0: i64) -> i64 { 314 out[0] = 0 - 1 315 out[1] = 0 - 1 316 out[2] = 0 - 1 317 out[3] = 0 318 var res: i64 = 0 319 let sbox: *i64 = sys_mmap(16) as *i64 320 let sbuf: *u8 = sys_read_file(spath, sbox) 321 if (sbuf as i64) != 0 { 322 let sid: i64 = cc_int_after(sbuf, sbox[0], "id=" as *u8) 323 // THE DIGEST IS THE GUARD, and a size check would not do: a same-size rewrite passes it while 324 // splicing two generations into one artifact. 325 if sid > 0 { if cc_find(sbuf, sbox[0], shahex) >= 0 { 326 let sreq: *u8 = sys_mmap(CC_REQ) 327 var so: i64 = cc_put_lit(sreq, 0, CC_PRE) 328 so = cc_put_arg(sreq, so, "status" as *u8, 6, 1) 329 let idb: *u8 = sys_mmap(CC_NUMBUF) 330 let idl: i64 = cc_itoa(sid, idb) 331 so = cc_put_arg(sreq, so, idb, idl, 0) 332 so = cc_put_lit(sreq, so, "],\x22_cap\x22:" as *u8) 333 so = cc_put_arg(sreq, so, capraw, caplen, 1) 334 so = cc_put_idem(sreq, so, "status" as *u8, shahex, sid, 0 - 1, t0) 335 so = cc_put_lit(sreq, so, CC_TAIL) 336 let sn: i64 = jf_post_follow(store_i, url, sreq, so, resp, CC_RESP, rcraw, rclen, CC_POLLS, CC_POLL_MS) 337 if sn > 0 { if cc_find(resp, sn, "CP-STATUS id=" as *u8) >= 0 { 338 let r2: i64 = cc_int_after(resp, sn, "id=" as *u8) 339 let c2: i64 = cc_int_after(resp, sn, "chunk_raw=" as *u8) 340 let n2: i64 = cc_int_after(resp, sn, "nchunks=" as *u8) 341 if r2 > 0 { if c2 > 0 { if n2 > 0 { if n2 <= CC_MAXCHUNKS { 342 let marked: i64 = cc_mark_missing(resp, sn, miss, n2) 343 // A RESUME THAT MARKS NOTHING MUST NOT PROCEED TO COMMIT. Either every chunk is 344 // already there (in which case the previous run should have committed and the 345 // honest move is to let it commit now) or the parse failed -- and a client that 346 // skipped every chunk and committed would publish whatever the server happened to 347 // hold. Only a POSITIVE mark count authorises the skip path. 348 if marked > 0 { 349 out[0] = r2; out[1] = c2; out[2] = n2; out[3] = marked; res = 1 350 cc_out("CP-RESUME id=" as *u8); cc_num(r2) 351 cc_out(" chunk_raw=" as *u8); cc_num(c2) 352 cc_out(" nchunks=" as *u8); cc_num(n2) 353 cc_out(" still_missing=" as *u8); cc_num(marked) 354 cc_out(" already_received=" as *u8); cc_num(n2 - marked) 355 cc_out(" (a prior run left state for this exact file and its digest still matches)\n" as *u8) 356 } 357 } } } } 358 } } 359 } } 360 } 361 return res 362} 363 364// ---- ONE CHUNK POST (extracted 2026-09-06 so the fixed-size path and the DI9 plan-addressed path share it) ---------- 365// Posts bytes [lo,hi) of `data` as chunk `idx` with its declared digest, retrying a transient at the same index 366// (idempotent by construction), naming size and code on a transport failure, and accepting both success spellings. 367// Returns the request body length on success (the wire cost), or -1 after printing the reason. 368func cc_post_chunk(store_i: i64, url: *u8, req: *u8, resp: *u8, rcraw: *u8, rclen: i64, capraw: *u8, caplen: i64, 369 rid: i64, idx: i64, lo: i64, hi: i64, data: *u8, enc: *u8, nb: *u8, cdg: *u8, chex: *u8, dbuf: *u8, 370 shahex: *u8, t0: i64, craw: i64) -> i64 { 371 let elen: i64 = b64_encode((data as i64 + lo) as *u8, hi - lo, enc) 372 var co: i64 = cc_put_lit(req, 0, CC_PRE) 373 co = cc_put_arg(req, co, "chunk" as *u8, 5, 1) 374 let il: i64 = cc_itoa(rid, nb) 375 co = cc_put_arg(req, co, nb, il, 0) 376 let xl: i64 = cc_itoa(idx, nb) 377 co = cc_put_arg(req, co, nb, xl, 0) 378 co = cc_put_arg(req, co, enc, elen, 0) 379 if co < 0 { cc_eo("CC-REFUSED: base64 payload was not JSON-safe (this cannot happen; the alphabet is A-Za-z0-9+/=)\n" as *u8); return 0 - 1 } 380 // DI10: declare THIS chunk's digest with the chunk, so the door refuses a corrupted or truncated chunk ON 381 // ARRIVAL (CP-REFUSED-CHUNK-DIGEST index=N) instead of at commit; a legacy door ignores the extra argument 382 sha256_digest((data as i64 + lo) as *u8, hi - lo, cdg) 383 cc_hex_into(cdg, chex) 384 var dcl: i64 = cc_put_lit(dbuf, 0, "sha256=" as *u8) 385 dcl = cc_put_lit(dbuf, dcl, chex) 386 co = cc_put_arg(req, co, dbuf, dcl, 0) 387 co = cc_put_lit(req, co, "],\x22_cap\x22:" as *u8) 388 co = cc_put_arg(req, co, capraw, caplen, 1) 389 co = cc_put_idem(req, co, "chunk" as *u8, shahex, rid, idx, t0) 390 co = cc_put_lit(req, co, CC_TAIL) 391 // RE-SENDING ONE CHUNK IS IDEMPOTENT BY CONSTRUCTION, WHICH IS WHY THIS IS NOT THE BANNED RETRY. 392 // A chunk is addressed by INDEX -- `chunk <id> <index> <b64>` overwrites that one staging slot -- 393 // so a second send of the same index cannot double-apply the way an append can. The estate ban 394 // on blind retries is about calls whose outcome is UNKNOWN and whose effect is CUMULATIVE; this 395 // is neither. 396 // AND IT REDUCES LOAD ON A DEGRADED ARRAY RATHER THAN ADDING IT. MEASURED 2026-09-04: a 410443 397 // byte ship lost chunk 7 of 9 to a transient and discarded the ENTIRE transfer, so the operator 398 // paid NINE chunk posts again to recover one. Retrying the single failed index costs one request 399 // instead of nine -- this is an amplifier in reverse. 400 // SCOPE, STATED: this survives a TRANSIENT, not a process death. Cross-process resume -- query 401 // `status <id>` and send only the missing indices -- is rung DL10 on the deploy board and stays 402 // OPEN; do not read this as closing it. 403 var cn: i64 = 0 404 var att: i64 = 0 405 var cdone: i64 = 0 406 while cdone == 0 { 407 cn = jf_post_follow(store_i, url, req, co, resp, CC_RESP, rcraw, rclen, CC_POLLS, CC_POLL_MS) 408 if cn > 0 { cdone = 1 } else { 409 att = att + 1 410 // Exits on a FLAG, never by clobbering the attempt counter -- the loop-exit sentinel 411 // defect this estate has recorded four times in one day. 412 if att >= CC_CHUNK_TRIES { cdone = 1 } else { 413 cc_out("CC-CHUNK retry index " as *u8); cc_num(idx) 414 cc_out(" attempt " as *u8); cc_num(att) 415 cc_out(" of " as *u8); cc_num(CC_CHUNK_TRIES) 416 cc_out(" -- same index, idempotent by construction\n" as *u8) 417 sys_sleep_ms(CC_CHUNK_RETRY_MS) 418 } 419 } 420 } 421 // A FAILURE THAT NAMES NEITHER ITS SIZE NOR ITS CODE SENDS EVERY READER INTO THE WRONG ORGAN. 422 // This one cost eleven days: the transport refuses a body over its own ceiling with a NAMED 423 // code, hp_post_once flattens every negative into HF_ERR_HTTP, and the caller printed only an 424 // index -- so a ceiling mismatch read as a dead server. Print the two numbers that decide it. 425 if cn <= 0 { 426 // ONE STREAM. cc_num writes fd 1 and cc_eo writes fd 2, so a diagnostic mixing them is 427 // torn into two interleaved fragments and the numbers are lost exactly when they matter. 428 cc_out("CC-CHUNK: post failed at index " as *u8); cc_num(idx) 429 cc_out(" transport_rc=" as *u8); cc_num(0 - cn) 430 cc_out(" request_body_bytes=" as *u8); cc_num(co) 431 cc_out(" server_offered_chunk_raw=" as *u8); cc_num(craw) 432 cc_out("\n" as *u8) 433 return 0 - 1 434 } 435 // SUCCESS HAS TWO SPELLINGS AND ACCEPTING ONLY ONE TURNS A SUCCESS INTO A REFUSAL. 436 // The receiver answers `CP-CHUNK OK` for a fresh chunk and `CP-CHUNK DUPLICATE-IDENTICAL` for a 437 // byte-identical re-send (nx_content_put.nx:510 and :532). Testing only for OK made the retry 438 // added above WORSE THAN NO RETRY: the first post landed, its response was lost, the re-send was 439 // correctly receipted as a duplicate, and this client called that a refusal and aborted a live 440 // transfer. MEASURED here on the first live run of the retry path, 2026-09-04. 441 // Both spellings are READ FROM THE SERVER SOURCE, not guessed, so this list is grounded -- the 442 // same defect class as a marker list keyed on the vocabulary of only one of the two producers. 443 var chunk_ok: i64 = 0 444 if cc_find(resp, cn, "CP-CHUNK OK" as *u8) >= 0 { chunk_ok = 1 } 445 if cc_find(resp, cn, "CP-CHUNK DUPLICATE-IDENTICAL" as *u8) >= 0 { chunk_ok = 1 } 446 if chunk_ok == 0 { 447 cc_out("CC-CHUNK refused at index " as *u8); cc_num(idx); cc_out("\n" as *u8) 448 // a CAS refusal means a DIFFERENT body already landed for this index: stop, do not retry. 449 if cc_find(resp, cn, "CHUNK-CAS" as *u8) >= 0 { 450 cc_eo("CHUNK-CAS: a different body already landed for this index -- abort and re-begin, or repair the source\n" as *u8) 451 } 452 return 0 - 1 453 } 454 return co 455} 456 457// ---- DI9 REUSE BATCH (2026-09-06): rows `k off len sha` staged from the incumbent by the door; returns the rows the 458// door accepted (staged + noop), or -1 when the batch was refused or lost (the caller then sends those chunks' bytes). 459// THE FOLLOW BUDGET SCALES WITH THE ROWS (2026-09-06): every reuse row costs the door one staged chunk write -- the 460// same work a chunk post costs -- so a batch of N rows is followed for N chunk budgets. MEASURED before this fix: a 461// 122-row batch on a storming array outlived the single-post budget (HP-PENDING: accepted, still running), the 462// client called the batch FAILED and re-sent every chunk as bytes while the door was still staging the same rows. 463func cc_flush_reuse(store_i: i64, url: *u8, req: *u8, resp: *u8, rcraw: *u8, rclen: i64, capraw: *u8, caplen: i64, 464 rid: i64, rbuf: *u8, rl: i64, rows: i64, enc: *u8, nb: *u8, shahex: *u8, t0: i64, batchno: i64, wire: *i64) -> i64 { 465 let elen: i64 = b64_encode(rbuf, rl, enc) 466 var o: i64 = cc_put_lit(req, 0, CC_PRE) 467 o = cc_put_arg(req, o, "reuse" as *u8, 5, 1) 468 let il: i64 = cc_itoa(rid, nb) 469 o = cc_put_arg(req, o, nb, il, 0) 470 o = cc_put_arg(req, o, enc, elen, 0) 471 if o < 0 { return 0 - 1 } 472 o = cc_put_lit(req, o, "],\x22_cap\x22:" as *u8) 473 o = cc_put_arg(req, o, capraw, caplen, 1) 474 o = cc_put_idem(req, o, "reuse" as *u8, shahex, rid, batchno, t0) 475 o = cc_put_lit(req, o, CC_TAIL) 476 wire[0] = wire[0] + o 477 var polls: i64 = CC_POLLS * rows 478 if polls < CC_POLLS { polls = CC_POLLS } 479 let n: i64 = jf_post_follow(store_i, url, req, o, resp, CC_RESP, rcraw, rclen, polls, CC_POLL_MS) 480 if n <= 0 { cc_out("CC-REUSE: batch post failed batch=" as *u8); cc_num(batchno); cc_out(" rows=" as *u8); cc_num(rows); cc_out(" polls=" as *u8); cc_num(polls); cc_out("\n" as *u8); return 0 - 1 } 481 if cc_find(resp, n, "CP-REUSE-BATCH" as *u8) < 0 { 482 cc_out("CC-REUSE: batch refused batch=" as *u8); cc_num(batchno); cc_out(" -- the door answered: " as *u8); sys_write(1, resp, n); cc_out("\n" as *u8) 483 return 0 - 1 484 } 485 let refused: i64 = cc_int_after(resp, n, "refused=" as *u8) 486 if refused != 0 { cc_out("CC-REUSE: batch had refused rows batch=" as *u8); cc_num(batchno); cc_out(" refused=" as *u8); cc_num(refused); cc_out(" -- those chunks are sent as bytes\n" as *u8); return 0 - 1 } 487 return cc_int_after(resp, n, "staged=" as *u8) + cc_int_after(resp, n, "noop=" as *u8) 488} 489 490// ---- DI9 THE DEDUPE SENDER (2026-09-06) ------------------------------------------------------------------------------ 491// Plan the local file by the DOOR's parameters, declare the plan, read the incumbent's digests page by page, and send 492// only the chunks the receiver lacks -- every other chunk is staged by a `reuse` row, batched to the wire chunk. Prints 493// CC-DEDUPE with bytes on the wire against the total. Returns the count of chunks whose BYTES were sent, or -1. 494func cc_send_cdc(store_i: i64, url: *u8, req: *u8, resp: *u8, rcraw: *u8, rclen: i64, capraw: *u8, caplen: i64, 495 rid: i64, craw: i64, cmin: i64, cavg: i64, cmax: i64, data: *u8, total: i64, shahex: *u8, t0: i64, 496 nb: *u8, enc: *u8, cdg: *u8, chex: *u8, dbuf: *u8) -> i64 { 497 if cmin <= 0 { cc_eo("CC-CDC: the begin receipt carried no cdc_min= -- this door predates plan-addressed transfers\n" as *u8); return 0 - 1 } 498 let tbl: *u8 = sys_mmap(CDC_GEAR_BYTES) 499 cdc_gear_table(tbl) 500 let pcap: i64 = total / cmin + 2 501 let poffs: *i64 = sys_mmap(pcap * 8) as *i64 502 let pcount: i64 = cdc_plan(tbl, data, total, cmin, cavg, cmax, poffs, pcap) 503 if pcount <= 0 { cc_eo("CC-CDC: the local plan was refused (not a chunking)\n" as *u8); return 0 - 1 } 504 let wire: *i64 = sys_mmap(16) as *i64 505 wire[0] = 0 506 // declare the plan 507 var o: i64 = cc_put_lit(req, 0, CC_PRE) 508 o = cc_put_arg(req, o, "plan" as *u8, 4, 1) 509 let il: i64 = cc_itoa(rid, nb) 510 o = cc_put_arg(req, o, nb, il, 0) 511 let pl: i64 = cc_itoa(pcount, nb) 512 o = cc_put_arg(req, o, nb, pl, 0) 513 o = cc_put_lit(req, o, "],\x22_cap\x22:" as *u8) 514 o = cc_put_arg(req, o, capraw, caplen, 1) 515 o = cc_put_idem(req, o, "plan" as *u8, shahex, rid, pcount, t0) 516 o = cc_put_lit(req, o, CC_TAIL) 517 wire[0] = wire[0] + o 518 var n: i64 = jf_post_follow(store_i, url, req, o, resp, CC_RESP, rcraw, rclen, CC_POLLS, CC_POLL_MS) 519 if n <= 0 { cc_eo("CC-CDC: plan post failed\n" as *u8); return 0 - 1 } 520 if cc_find(resp, n, "CP-PLAN OK" as *u8) < 0 { cc_eo("CC-CDC: plan refused -- the door answered: " as *u8); sys_write(2, resp, n); cc_eo("\n" as *u8); return 0 - 1 } 521 // the incumbent's digests, page by page (next=-1 ends the list) 522 var hn: i64 = 0 523 var hdecl: i64 = 0 524 var incb: i64 = 0 525 var hoff: *i64 = 0 as *i64 526 var hlen: *i64 = 0 as *i64 527 var hsha: *u8 = 0 as *u8 528 var from: i64 = 0 529 var pages: i64 = 0 530 var go: i64 = 1 531 while go == 1 { 532 o = cc_put_lit(req, 0, CC_PRE) 533 o = cc_put_arg(req, o, "have" as *u8, 4, 1) 534 let il2: i64 = cc_itoa(rid, nb) 535 o = cc_put_arg(req, o, nb, il2, 0) 536 var fb: i64 = cc_put_lit(dbuf, 0, "from=" as *u8) 537 let fl: i64 = cc_itoa(from, nb) 538 var fi: i64 = 0 539 while fi < fl { dbuf[fb] = nb[fi]; fb = fb + 1; fi = fi + 1 } 540 o = cc_put_arg(req, o, dbuf, fb, 0) 541 o = cc_put_lit(req, o, "],\x22_cap\x22:" as *u8) 542 o = cc_put_arg(req, o, capraw, caplen, 1) 543 o = cc_put_idem(req, o, "have" as *u8, shahex, rid, from, t0) 544 o = cc_put_lit(req, o, CC_TAIL) 545 wire[0] = wire[0] + o 546 n = jf_post_follow(store_i, url, req, o, resp, CC_RESP, rcraw, rclen, CC_POLLS, CC_POLL_MS) 547 if n <= 0 { cc_eo("CC-CDC: have post failed\n" as *u8); return 0 - 1 } 548 if cc_find(resp, n, "CP-HAVE id=" as *u8) < 0 { cc_eo("CC-CDC: have refused -- the door answered: " as *u8); sys_write(2, resp, n); cc_eo("\n" as *u8); return 0 - 1 } 549 pages = pages + 1 550 if pages == 1 { 551 incb = cc_int_after(resp, n, "incumbent_bytes=" as *u8) 552 hdecl = cc_int_after(resp, n, " chunks=" as *u8) 553 if incb < 0 { incb = 0 } 554 if hdecl < 0 { hdecl = 0 } 555 hoff = sys_mmap((hdecl + 1) * 8) as *i64 556 hlen = sys_mmap((hdecl + 1) * 8) as *i64 557 hsha = sys_mmap((hdecl + 1) * CC_SHAHEX + 8) 558 } 559 // rows: h<k> off=<a> len=<b> sha256=<hex>, one per line; parsed line by line, bounded by the declared count. 560 // A line ends at byte 10 OR at the two-byte escape `\n` -- a fast `have` answers synchronously INSIDE the 561 // JSON-RPC envelope, where the door's newlines arrive escaped, while a job-lane answer arrives raw from the 562 // artifact. MEASURED 2026-09-06: the first live dedupe pass read `chunks=122 parsed=0` because every row sat 563 // on one escaped "line" whose first byte was the header's C, and the whole incumbent would have been re-sent. 564 var i: i64 = 0 565 while i < n { 566 var e: i64 = i 567 var step: i64 = 1 568 while e < n { 569 if resp[e] == (10 as u8) { break } 570 if resp[e] == (92 as u8) { if e + 1 < n { if resp[e + 1] == (110 as u8) { step = 2; break } } } 571 e = e + 1 572 } 573 let ll: i64 = e - i 574 let line: *u8 = ((resp as i64) + i) as *u8 575 if ll > 2 { if line[0] == (104 as u8) { if line[1] >= (48 as u8) { if line[1] <= (57 as u8) { if hn < hdecl { 576 let a: i64 = cc_int_after(line, ll, " off=" as *u8) 577 let b: i64 = cc_int_after(line, ll, " len=" as *u8) 578 let sp: i64 = cc_find(line, ll, " sha256=" as *u8) 579 if a >= 0 { if b > 0 { if sp >= 0 { if sp + CC_SHAHEX <= ll { 580 hoff[hn] = a 581 hlen[hn] = b 582 var q: i64 = 0 583 while q < CC_SHAHEX { hsha[hn * CC_SHAHEX + q] = line[sp + q]; q = q + 1 } 584 hn = hn + 1 585 } } } } 586 } } } } } 587 i = e + step 588 } 589 let nxt: i64 = cc_int_after(resp, n, " next=" as *u8) 590 if nxt <= 0 { go = 0 } else { from = nxt } 591 } 592 cc_out("CC-HAVE incumbent_bytes=" as *u8); cc_num(incb) 593 cc_out(" chunks=" as *u8); cc_num(hdecl) 594 cc_out(" parsed=" as *u8); cc_num(hn) 595 cc_out(" pages=" as *u8); cc_num(pages) 596 cc_out("\n" as *u8) 597 // send only what the receiver lacks; reuse rows are batched to the wire chunk and flushed by `reuse` 598 let rbuf: *u8 = sys_mmap(craw + CC_NUMBUF * 3 + CC_SHAHEX + 8) 599 let rowb: *u8 = sys_mmap(CC_NUMBUF * 3 + CC_SHAHEX + 8) 600 let bks: *i64 = sys_mmap(pcap * 8) as *i64 601 var rl: i64 = 0 602 var bn: i64 = 0 603 var reused: i64 = 0 604 var sent: i64 = 0 605 var batches: i64 = 0 606 var failed: i64 = 0 607 var k: i64 = 0 608 while k < pcount { 609 if failed == 1 { k = pcount } else { 610 let lo: i64 = poffs[k] 611 let hi: i64 = poffs[k + 1] 612 sha256_digest((data as i64 + lo) as *u8, hi - lo, cdg) 613 cc_hex_into(cdg, chex) 614 var j: i64 = 0 615 var found: i64 = 0 - 1 616 while j < hn { 617 var same: i64 = 1 618 var q2: i64 = 0 619 while q2 < CC_SHAHEX { if hsha[j * CC_SHAHEX + q2] != chex[q2] { same = 0; q2 = CC_SHAHEX } else { q2 = q2 + 1 } } 620 if same == 1 { found = j; j = hn } else { j = j + 1 } 621 } 622 if found >= 0 { 623 // one reuse row: k off len sha 624 var rw: i64 = cc_itoa(k, nb) 625 var ro: i64 = 0 626 var z: i64 = 0 627 while z < rw { rowb[ro] = nb[z]; ro = ro + 1; z = z + 1 } 628 rowb[ro] = 32 as u8 629 ro = ro + 1 630 rw = cc_itoa(hoff[found], nb) 631 z = 0 632 while z < rw { rowb[ro] = nb[z]; ro = ro + 1; z = z + 1 } 633 rowb[ro] = 32 as u8 634 ro = ro + 1 635 rw = cc_itoa(hlen[found], nb) 636 z = 0 637 while z < rw { rowb[ro] = nb[z]; ro = ro + 1; z = z + 1 } 638 rowb[ro] = 32 as u8 639 ro = ro + 1 640 z = 0 641 while z < CC_SHAHEX { rowb[ro] = chex[z]; ro = ro + 1; z = z + 1 } 642 rowb[ro] = 10 as u8 643 ro = ro + 1 644 // flush when the batch would outgrow the wire chunk 645 if rl + ro > craw { 646 batches = batches + 1 647 let acc: i64 = cc_flush_reuse(store_i, url, req, resp, rcraw, rclen, capraw, caplen, rid, rbuf, rl, bn, enc, nb, shahex, t0, batches, wire) 648 if acc < 0 { 649 var f: i64 = 0 650 while f < bn { let bk: i64 = bks[f]; let pc: i64 = cc_post_chunk(store_i, url, req, resp, rcraw, rclen, capraw, caplen, rid, bk, poffs[bk], poffs[bk + 1], data, enc, nb, cdg, chex, dbuf, shahex, t0, craw); if pc < 0 { failed = 1; f = bn } else { sent = sent + 1; wire[0] = wire[0] + pc; f = f + 1 } } 651 } else { reused = reused + bn } 652 rl = 0 653 bn = 0 654 } 655 z = 0 656 while z < ro { rbuf[rl + z] = rowb[z]; z = z + 1 } 657 rl = rl + ro 658 bks[bn] = k 659 bn = bn + 1 660 } else { 661 let pc2: i64 = cc_post_chunk(store_i, url, req, resp, rcraw, rclen, capraw, caplen, rid, k, lo, hi, data, enc, nb, cdg, chex, dbuf, shahex, t0, craw) 662 if pc2 < 0 { failed = 1 } else { sent = sent + 1; wire[0] = wire[0] + pc2 } 663 } 664 k = k + 1 665 } 666 } 667 if failed == 0 { if bn > 0 { 668 batches = batches + 1 669 let acc2: i64 = cc_flush_reuse(store_i, url, req, resp, rcraw, rclen, capraw, caplen, rid, rbuf, rl, bn, enc, nb, shahex, t0, batches, wire) 670 if acc2 < 0 { 671 var f2: i64 = 0 672 while f2 < bn { let bk2: i64 = bks[f2]; let pc3: i64 = cc_post_chunk(store_i, url, req, resp, rcraw, rclen, capraw, caplen, rid, bk2, poffs[bk2], poffs[bk2 + 1], data, enc, nb, cdg, chex, dbuf, shahex, t0, craw); if pc3 < 0 { failed = 1; f2 = bn } else { sent = sent + 1; wire[0] = wire[0] + pc3; f2 = f2 + 1 } } 673 } else { reused = reused + bn } 674 } } 675 cc_out("CC-DEDUPE mode=cdc plan_chunks=" as *u8); cc_num(pcount) 676 cc_out(" incumbent_chunks=" as *u8); cc_num(hn) 677 cc_out(" reused=" as *u8); cc_num(reused) 678 cc_out(" sent=" as *u8); cc_num(sent) 679 cc_out(" reuse_batches=" as *u8); cc_num(batches) 680 cc_out(" wire_bytes=" as *u8); cc_num(wire[0]) 681 cc_out(" total=" as *u8); cc_num(total) 682 cc_out(" (reused + sent = plan_chunks when nothing failed)\n" as *u8) 683 if failed == 1 { return 0 - 1 } 684 return sent 685} 686 687func main(argc: i64, argv: *i64) -> i64 { 688 // `dryrun` emits the BEGIN request this client WOULD send and exits without opening a socket. 689 // It exists because a malformed envelope is rejected by the SERVER, for a reason that names neither 690 // the client nor the real fault -- so the request must be inspectable without a network. It is also 691 // what nx_content_put_client_gate forks: the envelope is then a decidable, deterministic artifact. 692 var base: i64 = 0 693 if argc > 1 { if cc_streq(argv[1] as *u8, "dryrun" as *u8) == 1 { base = 1 } } 694 if argc < 5 + base { 695 cc_eo("usage: nx_content_put_client [dryrun] <local_path> <dest> <capfile> <url> [expect=<sha256hex>]\n" as *u8) 696 return 2 697 } 698 let lpath: *u8 = argv[1 + base] as *u8 699 let dest: *u8 = argv[2 + base] as *u8 700 let capfile: *u8 = argv[3 + base] as *u8 701 let url: *u8 = argv[4 + base] as *u8 702 703 // OPTIONAL read capability, used ONLY to follow the edge JOB-STARTED promotion to its artifact. 704 // A SCAN, not a positional slot, so it composes with the existing optional expect= argument in 705 // either order and breaks no existing caller -- the same shape nx_gate_bite uses for its subject 706 // keyword, adopted here for the same reason. 707 var rcraw: *u8 = 0 as *u8 708 var rclen: i64 = 0 709 // DI9: the literal token `cdc` anywhere after the four positional arguments selects the dedupe path 710 var cdcmode: i64 = 0 711 var ai: i64 = 1 712 while ai < argc { 713 let av: *u8 = argv[ai] as *u8 714 if cc_streq(av, "cdc" as *u8) == 1 { cdcmode = 1 } 715 if cc_is_readcap(av) == 1 { 716 let rcbox: *i64 = sys_mmap(16) as *i64 717 rcraw = sys_read_file(((av as i64) + 8) as *u8, rcbox) 718 if rcraw as i64 == 0 { cc_eo("CC-IO: cannot read readcap file\n" as *u8); return 3 } 719 rclen = rcbox[0] 720 while rclen > 0 { 721 let c2: i64 = rcraw[rclen - 1] as i64 722 if c2 <= CC_SPACE { rclen = rclen - 1 } else { rclen = 0 - rclen } 723 } 724 if rclen < 0 { rclen = 0 - rclen } 725 } 726 ai = ai + 1 727 } 728 729 // ---- local file ---- 730 let lenbox: *i64 = sys_mmap(16) as *i64 731 let data: *u8 = sys_read_file(lpath, lenbox) 732 if data as i64 == 0 { cc_eo("CC-IO: cannot read local file\n" as *u8); return 3 } 733 let total: i64 = lenbox[0] 734 let d: *u8 = sys_mmap(CC_DIGEST + 8) 735 sha256_digest(data, total, d) 736 let shahex: *u8 = sys_mmap(CC_SHAHEX + 8) 737 cc_hex_into(d, shahex) 738 cc_out("local file=" as *u8); cc_out(lpath) 739 cc_out(" bytes=" as *u8); cc_num(total) 740 cc_out(" sha256=" as *u8); cc_out(shahex); cc_out("\n" as *u8) 741 742 // ---- cap token ---- 743 let capbox: *i64 = sys_mmap(16) as *i64 744 let capraw: *u8 = sys_read_file(capfile, capbox) 745 if capraw as i64 == 0 { cc_eo("CC-IO: cannot read cap file\n" as *u8); return 3 } 746 var caplen: i64 = capbox[0] 747 // trim trailing newline/whitespace -- a cap with a stray newline is the silent-401 class 748 while caplen > 0 { 749 let c: i64 = capraw[caplen - 1] as i64 750 if c <= CC_SPACE { caplen = caplen - 1 } else { caplen = 0 - caplen } 751 } 752 if caplen < 0 { caplen = 0 - caplen } 753 if cc_json_safe(capraw, caplen) == 0 { 754 cc_eo("CC-REFUSED: the capability token carries a byte that would need JSON escaping -- refusing rather than escaping badly\n" as *u8) 755 return 4 756 } 757 758 let store_i: i64 = hf_store_load() 759 let resp: *u8 = sys_mmap(CC_RESP + 16) 760 let req: *u8 = sys_mmap(CC_REQ + 16) 761 let nb: *u8 = sys_mmap(CC_NUMBUF) 762 // the run's start epoch names every idempotency key this run sends (set BEFORE the begin envelope is built, 763 // so the dryrun envelope carries it too and the gate can see the key's shape) 764 let t0: i64 = sys_now_realtime_sec() 765 766 // ---- BEGIN ---- 767 var o: i64 = cc_put_lit(req, 0, CC_PRE) 768 o = cc_put_arg(req, o, "begin" as *u8, 5, 1) 769 if o < 0 { cc_eo("CC-REFUSED: unsafe argument\n" as *u8); return 4 } 770 o = cc_put_arg(req, o, dest, cc_slen(dest), 0) 771 if o < 0 { cc_eo("CC-REFUSED: destination carries a byte needing JSON escaping\n" as *u8); return 4 } 772 let tl: i64 = cc_itoa(total, nb) 773 o = cc_put_arg(req, o, nb, tl, 0) 774 o = cc_put_arg(req, o, shahex, CC_SHAHEX, 0) 775 if argc > 5 + base { 776 let ex: *u8 = argv[5 + base] as *u8 777 // A readcap= argument in this position is NOT an expect token. Without this guard the CAS slot 778 // silently receives a FILE PATH, and the server refuses the transfer naming neither the real 779 // cause nor the offending argument -- the wrong-subject error class this estate keeps paying for. 780 if cc_is_readcap(ex) == 0 { if cc_streq(ex, "cdc" as *u8) == 0 { 781 o = cc_put_arg(req, o, ex, cc_slen(ex), 0) 782 if o < 0 { cc_eo("CC-REFUSED: expect argument carries a byte needing JSON escaping\n" as *u8); return 4 } 783 } } 784 } 785 o = cc_put_lit(req, o, "],\x22_cap\x22:" as *u8) 786 o = cc_put_arg(req, o, capraw, caplen, 1) 787 o = cc_put_idem(req, o, "begin" as *u8, shahex, total, 0 - 1, t0) 788 o = cc_put_lit(req, o, CC_TAIL) 789 790 if base == 1 { 791 sys_write(1, req, o) 792 sys_write(1, "\n" as *u8, 1) 793 return 0 794 } 795 796 // ---- RESUME FIRST, IF A PRIOR RUN LEFT STATE FOR THIS EXACT FILE ---- 797 // Everything here is additive: any failure to resume falls through to the ordinary begin below, so 798 // the path that has been shipping all day is unchanged whenever there is no sidecar. 799 let spath: *u8 = sys_mmap(CC_PATHCAP) 800 cc_state_path(lpath, spath) 801 let miss: *i64 = sys_mmap(CC_MAXCHUNKS * 8) as *i64 802 var rid: i64 = 0 - 1 803 var craw: i64 = 0 - 1 804 var nch: i64 = 0 - 1 805 // DI9: the door's content-defined parameters, read from its begin receipt (the server owns these numbers) 806 var cmin: i64 = 0 - 1 807 var cavg: i64 = 0 - 1 808 var cmax: i64 = 0 - 1 809 var resumed: i64 = 0 810 let rout: *i64 = sys_mmap(64) as *i64 811 // DI9: a dedupe run always begins fresh (its re-run is cheap by construction); the sidecar resume is the fixed path's 812 if cdcmode == 0 { resumed = cc_resume_from_status(store_i, url, spath, shahex, capraw, caplen, resp, CC_RESP, rcraw, rclen, miss, rout, t0) } 813 if resumed == 1 { rid = rout[0]; craw = rout[1]; nch = rout[2] } 814 815 var n: i64 = 0 816 if resumed == 0 { 817 n = jf_post_follow(store_i, url, req, o, resp, CC_RESP, rcraw, rclen, CC_POLLS, CC_POLL_MS) 818 if n <= 0 { cc_eo("CC-BEGIN: post failed\n" as *u8); return 5 } 819 rid = cc_int_after(resp, n, "id=" as *u8) 820 craw = cc_int_after(resp, n, "chunk_raw=" as *u8) 821 nch = cc_int_after(resp, n, "nchunks=" as *u8) 822 cmin = cc_int_after(resp, n, "cdc_min=" as *u8) 823 cavg = cc_int_after(resp, n, "cdc_avg=" as *u8) 824 cmax = cc_int_after(resp, n, "cdc_max=" as *u8) 825 cc_out("CP-BEGIN id=" as *u8); cc_num(rid) 826 cc_out(" chunk_raw=" as *u8); cc_num(craw) 827 cc_out(" nchunks=" as *u8); cc_num(nch); cc_out("\n" as *u8) 828 if rid < 0 { 829 // ECHO THE DOOR'S ANSWER. A client that reports only "no id" has destroyed the refusal that names the 830 // remedy (measured 2026-09-05: three failed begins, the cause -- a token spelling -- visible in none). 831 cc_eo("CC-BEGIN: no transfer id in receipt -- the door answered: " as *u8) 832 sys_write(2, resp, n) 833 cc_eo("\n" as *u8) 834 return 5 835 } 836 if craw <= 0 { cc_eo("CC-BEGIN: no chunk_raw in receipt (the server owns this number; it is never computed here)\n" as *u8); return 5 } 837 // A FRESH BEGIN MEANS EVERY CHUNK IS MISSING. Stated explicitly rather than inferred from 838 // `resumed == 0` at the send site, so the chunk loop has exactly ONE rule to follow and cannot 839 // acquire a second meaning later. 840 var mi: i64 = 0 841 while mi < nch { if mi < CC_MAXCHUNKS { miss[mi] = 1 } mi = mi + 1 } 842 // WRITE THE SIDECAR BEFORE SENDING ANYTHING, because a crash between begin and the first chunk is 843 // exactly the case resume exists for -- recording the id only on success would leave the one 844 // failure mode this feature targets unrecorded. 845 let sfd: i64 = sys_openat_wr(spath, CC_MODE_RW) 846 if sfd >= 0 { 847 let sb2: *u8 = sys_mmap(CC_PATHCAP) 848 var sp: i64 = cc_put_lit(sb2, 0, "id=" as *u8) 849 let nb2: *u8 = sys_mmap(CC_NUMBUF) 850 let nl2: i64 = cc_itoa(rid, nb2) 851 var z: i64 = 0 852 while z < nl2 { sb2[sp] = nb2[z]; sp = sp + 1; z = z + 1 } 853 sp = cc_put_lit(sb2, sp, " sha=" as *u8) 854 z = 0 855 while z < CC_SHAHEX { sb2[sp] = shahex[z]; sp = sp + 1; z = z + 1 } 856 sb2[sp] = 10 as u8; sp = sp + 1 857 sys_write(sfd, sb2, sp) 858 sys_close(sfd) 859 } 860 } 861 862 // ---- CHUNKS ---- 863 let enc: *u8 = sys_mmap(((craw + 2) / 3) * 4 + 64) 864 var idx: i64 = 0 865 var sent: i64 = 0 866 var skipped: i64 = 0 867 // DI10: buffers for the per-chunk digest declared with each chunk, allocated ONCE outside the loop 868 let cdg: *u8 = sys_mmap(CC_DIGEST + 8) 869 let chex: *u8 = sys_mmap(CC_SHAHEX + 8) 870 let dbuf: *u8 = sys_mmap(CC_SHAHEX + 16) 871 // DI9: the dedupe path plans, declares, reads the incumbent's digests and sends only what the door lacks; the fixed 872 // loop below is then skipped (idx = nch), and the commit that follows is the same commit 873 if cdcmode == 1 { 874 let cs: i64 = cc_send_cdc(store_i, url, req, resp, rcraw, rclen, capraw, caplen, rid, craw, cmin, cavg, cmax, data, total, shahex, t0, nb, enc, cdg, chex, dbuf) 875 if cs < 0 { return 6 } 876 sent = cs 877 idx = nch 878 } 879 while idx < nch { 880 // THE ONE RULE: send a chunk iff the server says it is missing. On a fresh begin every index is 881 // marked, so this is a no-op and the proven path is byte-for-byte what it was. On a resume it is 882 // the whole point -- and it reads from a table the SERVER filled, never from anything this 883 // client remembered about what it thinks it sent. 884 if miss[idx] == 0 { skipped = skipped + 1 } else { 885 let lo: i64 = idx * craw 886 var hi: i64 = lo + craw 887 if hi > total { hi = total } 888 // ONE chunk post for both paths (extracted 2026-09-06 for DI9): retry, diagnostics and both success spellings live once 889 let pc: i64 = cc_post_chunk(store_i, url, req, resp, rcraw, rclen, capraw, caplen, rid, idx, lo, hi, data, enc, nb, cdg, chex, dbuf, shahex, t0, craw) 890 if pc < 0 { return 6 } 891 sent = sent + 1 892 } 893 idx = idx + 1 894 } 895 896 // ---- COMMIT ---- 897 var mo: i64 = cc_put_lit(req, 0, CC_PRE) 898 mo = cc_put_arg(req, mo, "commit" as *u8, 6, 1) 899 let ml: i64 = cc_itoa(rid, nb) 900 mo = cc_put_arg(req, mo, nb, ml, 0) 901 mo = cc_put_lit(req, mo, "],\x22_cap\x22:" as *u8) 902 mo = cc_put_arg(req, mo, capraw, caplen, 1) 903 mo = cc_put_idem(req, mo, "commit" as *u8, shahex, rid, 0 - 1, t0) 904 mo = cc_put_lit(req, mo, CC_TAIL) 905 let mn: i64 = jf_post_follow(store_i, url, req, mo, resp, CC_RESP, rcraw, rclen, CC_POLLS, CC_POLL_MS) 906 if mn <= 0 { 907 // A LOST COMMIT RESPONSE IS *UNKNOWN*, NOT A FAILURE, AND THIS CLIENT DOES NOT GUESS. 908 cc_eo("CC-COMMIT-UNKNOWN: the commit response was lost. The transfer may well have committed.\n" as *u8) 909 cc_eo(" VERIFY BY HASH, DO NOT RETRY: nx_filehash <dest> and compare against the sha256 printed above.\n" as *u8) 910 return 7 911 } 912 var ok: i64 = 0 913 if cc_find(resp, mn, shahex) >= 0 { if cc_find(resp, mn, "CP-COMMIT" as *u8) >= 0 { ok = 1 } } 914 cc_out("E2E " as *u8) 915 if ok == 1 { cc_out("OK" as *u8) } else { cc_out("FAILED" as *u8) } 916 cc_out(": bytes=" as *u8); cc_num(total) 917 cc_out(" chunks_sent=" as *u8); cc_num(sent) 918 cc_out(" of=" as *u8); cc_num(nch) 919 if skipped > 0 { cc_out(" skipped_already_received=" as *u8); cc_num(skipped) } 920 cc_out("\n" as *u8) 921 // THE SIDECAR IS REMOVED ONLY ON A CONFIRMED COMMIT. Removing it on ANY exit would throw away the 922 // resume state in precisely the failure this feature exists for; leaving it after a commit would 923 // point the next run at a transfer id the server has already retired. Neither is silent: a stale 924 // sidecar whose transfer is gone falls through to a fresh begin, and this delete only fires on the 925 // path where the whole-file digest was echoed back by the server. 926 if ok == 1 { sys_unlinkat(spath) } 927 if ok == 1 { return 0 } 928 return 7 929}