nx_store_ingest.nx source

1// AUTHORED BY THE NISHI BUILDER (pattern: PNG_INGEST compose) -- 5-stage ingest, no Claude logic 2import "_pe_pngchunk.nx" 3import "_pe_pngtext.nx" 4import "nx_canon_cid.nx" 5import "nx_seg_store.nx" 6import "nx_syscalls.nx" 7const K_MAGIC_4096: i64 = 4096 8const K_MAGIC_4095: i64 = 4095 9const K_MAGIC_1950701684: i64 = 1950701684 10const K_MAGIC_2048: i64 = 2048 11const K_MAGIC_2047: i64 = 2047 12const K_MAGIC_8192: i64 = 8192 13func nx_store_ingest_keq(buf: *u8, off: i64, kl: i64, key: *u8) -> i64 { 14 var i: i64 = 0 15 while i < kl { if (buf[off + i] & 0xff) != (key[i] & 0xff) { return 0 } i = i + 1 } 16 if (key[kl] & 0xff) != 0 { return 0 } 17 return 1 18} 19// ★ONE CANON, ONE PLACE (2026-08-04, debt 1785882708). This 210-line block -- the length guard, 20// the seven canonical key declarations and the tEXt chunk walk that harvests them -- was 21// DUPLICATED VERBATIM in nx_store_ingest_ingest and nx_store_ingest_compute_cid. Adding `size` and 22// then `args` to the canon meant editing BOTH by hand; the day someone edits one, the two disagree 23// about what a CID *is*, and the symptom is silent gallery data loss (a render overwriting another). 24// Extracted verbatim: both callers now harvest through here, so they CANNOT drift. 25// Returns the number of canonical fields found (gn), or -1 when the input is not a GENREC PNG. 26func nx_store_ingest_harvest(buf: *u8, flen: i64, gkeys: *i64, gvals: *i64) -> i64 { 27 if flen < 8 { return 0 - 1 } 28 let cb: *u8 = (buf as i64 + 8) as *u8 29 let cn: i64 = flen - 8 30 // gkeys/gvals are the CALLER'S arrays -- re-declaring them here would shadow the parameters and 31 // silently leave the caller with an empty canon (and therefore a wrong, colliding CID). 32 var gn: i64 = 0 33 let key0: *u8 = sys_mmap(5) 34 key0[0] = 115 as u8 35 key0[1] = 101 as u8 36 key0[2] = 101 as u8 37 key0[3] = 100 as u8 38 key0[4] = 0 as u8 39 let key1: *u8 = sys_mmap(6) 40 key1[0] = 109 as u8 41 key1[1] = 111 as u8 42 key1[2] = 100 as u8 43 key1[3] = 101 as u8 44 key1[4] = 108 as u8 45 key1[5] = 0 as u8 46 let key2: *u8 = sys_mmap(8) 47 key2[0] = 115 as u8 48 key2[1] = 97 as u8 49 key2[2] = 109 as u8 50 key2[3] = 112 as u8 51 key2[4] = 108 as u8 52 key2[5] = 101 as u8 53 key2[6] = 114 as u8 54 key2[7] = 0 as u8 55 let key3: *u8 = sys_mmap(6) 56 key3[0] = 115 as u8 57 key3[1] = 116 as u8 58 key3[2] = 101 as u8 59 key3[3] = 112 as u8 60 key3[4] = 115 as u8 61 key3[5] = 0 as u8 62 let key4: *u8 = sys_mmap(15) 63 key4[0] = 103 as u8 64 key4[1] = 101 as u8 65 key4[2] = 110 as u8 66 key4[3] = 101 as u8 67 key4[4] = 114 as u8 68 key4[5] = 97 as u8 69 key4[6] = 116 as u8 70 key4[7] = 111 as u8 71 key4[8] = 114 as u8 72 key4[9] = 95 as u8 73 key4[10] = 104 as u8 74 key4[11] = 111 as u8 75 key4[12] = 115 as u8 76 key4[13] = 116 as u8 77 key4[14] = 0 as u8 78 let key5: *u8 = sys_mmap(7) 79 key5[0] = 112 as u8 80 key5[1] = 114 as u8 81 key5[2] = 111 as u8 82 key5[3] = 109 as u8 83 key5[4] = 112 as u8 84 key5[5] = 116 as u8 85 key5[6] = 0 as u8 86 // key6 = "size" (WxH), added 2026-08-04 for the CID-COLLISION DATA-LOSS fix (debt 1785881333). 87 // The CID is a hash of THIS canonical key set, NOT of the pixels, so before this key two renders 88 // differing only in resolution produced an IDENTICAL cid and the second silently OVERWROTE the 89 // first blob. ★ADDITIVE AND PROVABLY NON-BREAKING FOR EVERY EXISTING CONSUMER (GALX included): 90 // a PNG carrying no `size` tEXt never matches this key, so gn is unchanged, the canonical form 91 // is unchanged, and its CID is byte-identical to before. Nothing already in a store moves. 92 // ⚠This canon builder is DUPLICATED in nx_store_ingest_ingest and nx_store_ingest_compute_cid; 93 // the key is added to BOTH so the two can never disagree about what a CID is. 94 let key6: *u8 = sys_mmap(5) 95 key6[0] = 115 as u8 96 key6[1] = 105 as u8 97 key6[2] = 122 as u8 98 key6[3] = 101 as u8 99 key6[4] = 0 as u8 100 // key7 = "args": the EXACT generator-facing argument string. `size` alone did not close the 101 // collision class -- cfg 1.0 vs 5.0 still produced two different images under ONE cid. Keying on 102 // what was actually sent to the engine closes it generically, for every present and future knob. 103 let key7: *u8 = sys_mmap(5) 104 key7[0] = 97 as u8 105 key7[1] = 114 as u8 106 key7[2] = 103 as u8 107 key7[3] = 115 as u8 108 key7[4] = 0 as u8 109 let out: *i64 = sys_mmap(64) as *i64 110 var off: i64 = 0 111 while off < cn { 112 if off + 8 > cn { return 0 - 1 } 113 let clen: i64 = _pe_pngchunk_rdbe(cb, off + 0, 4) 114 let cty: i64 = _pe_pngchunk_rdbe(cb, off + 4, 4) 115 if off + 12 + clen > cn { return 0 - 1 } 116 if cty == K_MAGIC_1950701684 { 117 let body: *u8 = (cb as i64 + off + 8) as *u8 118 if _pe_pngtext_locate(body, clen, out) == 0 { 119 let kwoff: i64 = out[2] 120 let kwlen: i64 = out[3] 121 let voff: i64 = out[0] 122 let vlen: i64 = out[1] 123 if nx_store_ingest_keq(body, kwoff, kwlen, key0) == 1 { 124 let kk: *u8 = sys_mmap(80) 125 var ci: i64 = 0 126 while ci < kwlen { kk[ci] = body[kwoff + ci]; ci = ci + 1 } 127 kk[kwlen] = 0 as u8 128 let vv: *u8 = sys_mmap(512) 129 ci = 0 130 while ci < vlen { vv[ci] = body[voff + ci]; ci = ci + 1 } 131 vv[vlen] = 0 as u8 132 gkeys[gn] = kk as i64 133 gvals[gn] = vv as i64 134 gn = gn + 1 135 } 136 if nx_store_ingest_keq(body, kwoff, kwlen, key1) == 1 { 137 let kk: *u8 = sys_mmap(80) 138 var ci: i64 = 0 139 while ci < kwlen { kk[ci] = body[kwoff + ci]; ci = ci + 1 } 140 kk[kwlen] = 0 as u8 141 let vv: *u8 = sys_mmap(512) 142 ci = 0 143 while ci < vlen { vv[ci] = body[voff + ci]; ci = ci + 1 } 144 vv[vlen] = 0 as u8 145 gkeys[gn] = kk as i64 146 gvals[gn] = vv as i64 147 gn = gn + 1 148 } 149 if nx_store_ingest_keq(body, kwoff, kwlen, key2) == 1 { 150 let kk: *u8 = sys_mmap(80) 151 var ci: i64 = 0 152 while ci < kwlen { kk[ci] = body[kwoff + ci]; ci = ci + 1 } 153 kk[kwlen] = 0 as u8 154 let vv: *u8 = sys_mmap(512) 155 ci = 0 156 while ci < vlen { vv[ci] = body[voff + ci]; ci = ci + 1 } 157 vv[vlen] = 0 as u8 158 gkeys[gn] = kk as i64 159 gvals[gn] = vv as i64 160 gn = gn + 1 161 } 162 if nx_store_ingest_keq(body, kwoff, kwlen, key3) == 1 { 163 let kk: *u8 = sys_mmap(80) 164 var ci: i64 = 0 165 while ci < kwlen { kk[ci] = body[kwoff + ci]; ci = ci + 1 } 166 kk[kwlen] = 0 as u8 167 let vv: *u8 = sys_mmap(512) 168 ci = 0 169 while ci < vlen { vv[ci] = body[voff + ci]; ci = ci + 1 } 170 vv[vlen] = 0 as u8 171 gkeys[gn] = kk as i64 172 gvals[gn] = vv as i64 173 gn = gn + 1 174 } 175 if nx_store_ingest_keq(body, kwoff, kwlen, key4) == 1 { 176 let kk: *u8 = sys_mmap(80) 177 var ci: i64 = 0 178 while ci < kwlen { kk[ci] = body[kwoff + ci]; ci = ci + 1 } 179 kk[kwlen] = 0 as u8 180 let vv: *u8 = sys_mmap(512) 181 ci = 0 182 while ci < vlen { vv[ci] = body[voff + ci]; ci = ci + 1 } 183 vv[vlen] = 0 as u8 184 gkeys[gn] = kk as i64 185 gvals[gn] = vv as i64 186 gn = gn + 1 187 } 188 if nx_store_ingest_keq(body, kwoff, kwlen, key7) == 1 { 189 let kk7: *u8 = sys_mmap(80) 190 var c7: i64 = 0 191 while c7 < kwlen { kk7[c7] = body[kwoff + c7]; c7 = c7 + 1 } 192 kk7[kwlen] = 0 as u8 193 let vv7: *u8 = sys_mmap(K_MAGIC_4096) 194 var v7: i64 = vlen 195 if v7 > K_MAGIC_4095 { v7 = K_MAGIC_4095 } 196 c7 = 0 197 while c7 < v7 { vv7[c7] = body[voff + c7]; c7 = c7 + 1 } 198 vv7[v7] = 0 as u8 199 gkeys[gn] = kk7 as i64 200 gvals[gn] = vv7 as i64 201 gn = gn + 1 202 } 203 if nx_store_ingest_keq(body, kwoff, kwlen, key6) == 1 { 204 let kk6: *u8 = sys_mmap(80) 205 var c6: i64 = 0 206 while c6 < kwlen { kk6[c6] = body[kwoff + c6]; c6 = c6 + 1 } 207 kk6[kwlen] = 0 as u8 208 let vv6: *u8 = sys_mmap(512) 209 var v6: i64 = vlen 210 if v6 > 511 { v6 = 511 } 211 c6 = 0 212 while c6 < v6 { vv6[c6] = body[voff + c6]; c6 = c6 + 1 } 213 vv6[v6] = 0 as u8 214 gkeys[gn] = kk6 as i64 215 gvals[gn] = vv6 as i64 216 gn = gn + 1 217 } 218 if nx_store_ingest_keq(body, kwoff, kwlen, key5) == 1 { 219 let kk: *u8 = sys_mmap(80) 220 var ci: i64 = 0 221 while ci < kwlen { kk[ci] = body[kwoff + ci]; ci = ci + 1 } 222 kk[kwlen] = 0 as u8 223 let vv: *u8 = sys_mmap(K_MAGIC_2048) 224 var vc: i64 = vlen 225 if vc > K_MAGIC_2047 { vc = K_MAGIC_2047 } 226 ci = 0 227 while ci < vc { vv[ci] = body[voff + ci]; ci = ci + 1 } 228 vv[vc] = 0 as u8 229 gkeys[gn] = kk as i64 230 gvals[gn] = vv as i64 231 gn = gn + 1 232 } 233 } 234 } 235 off = off + 12 + clen 236 } 237 return gn 238} 239func nx_store_ingest_ingest(buf: *u8, flen: i64, prefix: *u8) -> i64 { 240 let gkeys: *i64 = sys_mmap(8 * 64) as *i64 241 let gvals: *i64 = sys_mmap(8 * 64) as *i64 242 let gn: i64 = nx_store_ingest_harvest(buf, flen, gkeys, gvals) 243 if gn < 1 { return 0 - 1 } 244 let canon: *u8 = sys_mmap(K_MAGIC_8192) 245 let clen2: i64 = canon_encode(gkeys, gvals, gn, canon) 246 let cid: *u8 = sys_mmap(80) 247 cid_of(canon, clen2, cid) 248 let rk: *u8 = sys_mmap(96) 249 rk[0] = 105 as u8 250 rk[1] = 109 as u8 251 rk[2] = 103 as u8 252 rk[3] = 58 as u8 253 var ri: i64 = 0 254 while ri < 69 { rk[4 + ri] = cid[ri]; ri = ri + 1 } 255 rk[73] = 0 as u8 256 let pp: *i64 = sys_mmap(16) as *i64 257 let ll: *i64 = sys_mmap(16) as *i64 258 if ss_get(prefix, rk, pp, ll) == 1 { return 0 } 259 let segs: *i64 = sys_mmap(8 * 260) as *i64 260 let nseg: i64 = ss_manifest(prefix, segs) 261 var segid: i64 = 1 262 if nseg >= 0 { segid = 1 + nseg } 263 let wr: *i64 = ss_begin() 264 ss_add(wr, 1, rk, canon, clen2) 265 let rc: i64 = ss_commit(prefix, wr, segid) 266 if rc != 0 { return 0 - 1 } 267 return 1 268} 269// ADDITIVE sibling (GALX-PROD): same 5-stage ingest, but ALSO writes the 69-char CID to cidout 270// so a caller can build a CID->path sidecar without re-parsing. Returns 1=new, 0=already-present 271// (cidout still filled), -1=not a GENREC PNG. The original nx_store_ingest_ingest is UNTOUCHED. 272// GALX-PROD-FULL: dedup + segid use the cap-aware store siblings (8192) so re-ingest stays 273// idempotent past image #256 (the original ss_get/ss_manifest cap of 256 would have created 274// duplicate segments once the store grows beyond 256 segments). 275// SHARED compute (refactor): parse PNG -> harvest GENREC -> canon_encode -> CID -> record key. 276// NO store side-effect. Returns canon length (clen2>0) on success, -1 if <8 bytes or no GENREC found. 277// Writes the 69B CID (+NUL) to cidout, the canon bytes to canonout (caller mmaps >=8192), and the 278// record key "img:"+CID (73B +NUL) to rkout (>=96). nx_store_ingest_ingest_cid AND the bulk-ingest 279// driver BOTH call this, so their CIDs are byte-identical (no replication drift). 280func nx_store_ingest_compute_cid(buf: *u8, flen: i64, cidout: *u8, canonout: *u8, rkout: *u8) -> i64 { 281 let gkeys: *i64 = sys_mmap(8 * 64) as *i64 282 let gvals: *i64 = sys_mmap(8 * 64) as *i64 283 let gn: i64 = nx_store_ingest_harvest(buf, flen, gkeys, gvals) 284 if gn < 1 { return 0 - 1 } 285 let clen2: i64 = canon_encode(gkeys, gvals, gn, canonout) 286 let cid: *u8 = sys_mmap(80) 287 cid_of(canonout, clen2, cid) 288 var co: i64 = 0 289 while co < 69 { cidout[co] = cid[co]; co = co + 1 } 290 cidout[69] = 0 as u8 291 rkout[0] = 105 as u8 292 rkout[1] = 109 as u8 293 rkout[2] = 103 as u8 294 rkout[3] = 58 as u8 295 var ri: i64 = 0 296 while ri < 69 { rkout[4 + ri] = cid[ri]; ri = ri + 1 } 297 rkout[73] = 0 as u8 298 return clen2 299} 300 301// store wrapper -- compute then dedup (cap-aware) then commit ONE segment. Behavior is byte-identical 302// to the pre-refactor ingest_cid (GALX-001 gate verifies). The bulk path skips this and batches. 303func nx_store_ingest_ingest_cid(buf: *u8, flen: i64, prefix: *u8, cidout: *u8) -> i64 { 304 let canon: *u8 = sys_mmap(K_MAGIC_8192) 305 let rk: *u8 = sys_mmap(96) 306 let clen2: i64 = nx_store_ingest_compute_cid(buf, flen, cidout, canon, rk) 307 if clen2 < 0 { return 0 - 1 } 308 let pp: *i64 = sys_mmap(16) as *i64 309 let ll: *i64 = sys_mmap(16) as *i64 310 // GALX-PROD-FULL: dedup + segid must see the WHOLE store (cap-aware siblings, 8192). 311 if ss_get_cap(prefix, rk, pp, ll, K_MAGIC_8192) == 1 { return 0 } 312 let segs: *i64 = sys_mmap(8 * K_MAGIC_8192) as *i64 313 let nseg: i64 = ss_manifest_cap(prefix, segs, K_MAGIC_8192) 314 var segid: i64 = 1 315 if nseg >= 0 { segid = 1 + nseg } 316 let wr: *i64 = ss_begin() 317 ss_add(wr, 1, rk, canon, clen2) 318 let rc: i64 = ss_commit(prefix, wr, segid) 319 if rc != 0 { return 0 - 1 } 320 return 1 321}