nx_plane_append.nx source

1// nx_plane_append.nx -- SAFE APPEND OF ONE ROW TO A SOVEREIGN SEG-STORE PLANE. 2// 3// WHY THIS EXISTS (proved by experiment 2026-07-26, debt seq1067). `sts_seed` REPLACES a plane; it does 4// not append. Seeding the same 2-row fixture twice into a throwaway prefix left 2 rows, not 4. So the 5// obvious way to "add one row" to a shared catalog -- call the seeder with your row -- DESTROYS every 6// row already there. Adding one row to knowledge/store/commontask- that way would have turned its 42 7// rows into 1. Two unrelated pieces of work were blocked on this single missing primitive: registering 8// an organ in the capability catalog, and adding the additive KIND column the atlas measure-gap axis 9// needs (seq1046). It was never a catalog problem. It was a WRITE-API problem. 10// 11// FAIL-SAFE BY CONSTRUCTION, not by promise (Rule 13 / Rule 26 spirit): 12// * the ORIGINAL bytes are held in memory for the whole operation, so a restore is always possible; 13// * an absent or empty plane is REFUSED, never silently created -- a typo in a prefix must not 14// conjure a new plane that later reads as authoritative; 15// * the new buffer must be strictly LONGER than the old one AND must still start with the old bytes, 16// checked before the commit -- an append that does not preserve its input is not an append; 17// * after the commit the plane is RE-LOADED and the row count verified; on any mismatch the original 18// buffer is written back and the tool reports RED rather than leaving a mangled registry. 19// 20// HONEST LIMIT, stated because it matters: the lock serialises appends made THROUGH THIS ORGAN. Any 21// other writer calling sts_seed directly bypasses it. This is the one correct path, not a guarantee 22// about every path, and it cannot become one until the other writers route through here. 23// 24// nx_plane_append <prefix> <row> 25// nx_plane_append selftest 26// expect_exit: 0 license_tier: ORIGINAL No hw writes (Rule 26). 27import "nx_syscalls.nx" 28import "nx_itoa_lib.nx" // shared MSB-first emitter (zero-alloc) 29import "nx_store_seed_lib.nx" 30import "nx_seg_store.nx" 31const PA_MAGIC_65536: i64 = 65536 32const PA_MAGIC_4096: i64 = 4096 33 34const PA_CAP: i64 = 4194304 35const PA_LOCK_MODE: i64 = 420 36const PA_NL: i64 = 10 37const PA_OUTFD: i64 = 1 38const PA_EXIT_OK: i64 = 0 39const PA_EXIT_USAGE: i64 = 2 40const PA_EXIT_REFUSED: i64 = 3 41const PA_EXIT_IO: i64 = 4 42const PA_EXIT_CORRUPT: i64 = 5 43 44func pa_w(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(PA_OUTFD, s, n); return 0 } 45// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer 46// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the 47// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls). 48// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign. 49func pa_n(v: i64) -> i64 { nxi_out(v); return 0 } 50func pa_streq(a: *u8, b: *u8) -> i64 { 51 var i: i64 = 0 52 while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 } 53 if b[i] != (0 as u8) { return 0 } 54 return 1 55} 56func pa_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n } 57 58// Rows are newline-separated. A trailing newline does not open a new row. 59func pa_rows(buf: *u8, n: i64) -> i64 { 60 var c: i64 = 0 61 var i: i64 = 0 62 while i < n { if buf[i] == (PA_NL as u8) { c = c + 1 } i = i + 1 } 63 if n > 0 { if buf[n-1] != (PA_NL as u8) { c = c + 1 } } 64 return c 65} 66 67// An append must PRESERVE its input. Verified against the in-memory original before any commit, so a 68// codec that silently rewrote the body could never reach the plane. 69func pa_prefix_ok(oldb: *u8, oldn: i64, newb: *u8, newn: i64) -> i64 { 70 if newn <= oldn { return 0 } 71 var i: i64 = 0 72 while i < oldn { if oldb[i] != newb[i] { return 0 } i = i + 1 } 73 return 1 74} 75 76// LOCK DOMAIN UNIFIED 2026-07-30 (ws=sev-eater). WAS: a SINGLE GLOBAL lock file 77// /tmp/nx_plane_append.lock, which made the honest limit above weaker than it read. That file 78// serialised this organ against ITSELF and against nothing else -- nx_debt / nx_ecomat_put / 79// nx_frontier_put lock <prefix>plock, so an append here and a debt write there took DIFFERENT locks 80// and did not exclude at all. A lock that the other writers do not share is not mutual exclusion, 81// it is a comment with a syscall in it. 82// Two further hazards it carried: (1) /tmp is banned for sovereign state and is CLEARED -- if the 83// file is removed between two writers they flock DIFFERENT INODES and both proceed, so the guard 84// silently evaporates exactly when the box is busy enough to matter; (2) one global lock serialises 85// appends to UNRELATED planes, which is a bottleneck bought for no safety. 86// NOW: delegate to the shared lib's per-plane lock, the SAME <prefix>plock every other writer uses. 87// One domain, no /tmp, and unrelated planes append concurrently. 88func pa_lock(prefix: *u8) -> i64 { 89 return sts_lock(prefix) 90} 91 92func pa_unlock(fd: i64) -> i64 { 93 return sts_unlock(fd) 94} 95 96// Returns 0 on success. Never leaves the plane shorter than it found it. 97func pa_append(prefix: *u8, row: *u8, quiet: i64) -> i64 { 98 let lk: i64 = pa_lock(prefix) 99 if lk < 0 { if quiet == 0 { pa_w("{\"verdict\":\"RED\",\"why\":\"cannot take the append lock\"}\n" as *u8) } return PA_EXIT_IO } 100 101 let oldb: *u8 = sts_mm(PA_CAP) 102 let oldn: i64 = sts_load(prefix, oldb, PA_CAP) 103 if oldn <= 0 { 104 // FAIL-CLOSED. A mistyped prefix must not create a plane that later reads as authoritative. 105 if quiet == 0 { pa_w("{\"verdict\":\"REFUSED\",\"why\":\"plane is absent or empty -- this verb appends to an EXISTING plane and will not create one\"}\n" as *u8) } 106 pa_unlock(lk) 107 return PA_EXIT_REFUSED 108 } 109 let oldrows: i64 = pa_rows(oldb, oldn) 110 111 let rl: i64 = pa_slen(row) 112 let newb: *u8 = sts_mm(PA_CAP) 113 var o: i64 = 0 114 while o < oldn { newb[o] = oldb[o]; o = o + 1 } 115 if newb[o-1] != (PA_NL as u8) { newb[o] = (PA_NL as u8); o = o + 1 } 116 var j: i64 = 0 117 while j < rl { newb[o] = row[j]; o = o + 1; j = j + 1 } 118 newb[o] = (PA_NL as u8) 119 o = o + 1 120 121 if pa_prefix_ok(oldb, oldn, newb, o) != 1 { 122 if quiet == 0 { pa_w("{\"verdict\":\"RED\",\"why\":\"assembled buffer does not preserve the original bytes -- refusing to commit\"}\n" as *u8) } 123 pa_unlock(lk) 124 return PA_EXIT_CORRUPT 125 } 126 127 let wrote: i64 = sts_seed(prefix, newb, o) 128 if wrote < 0 { 129 // The seeder failed. Put back exactly what we found. 130 sts_seed(prefix, oldb, oldn) 131 if quiet == 0 { pa_w("{\"verdict\":\"RED\",\"why\":\"commit failed; original restored\"}\n" as *u8) } 132 pa_unlock(lk) 133 return PA_EXIT_IO 134 } 135 136 // VERIFY BY RE-READING. A commit that returns success is not evidence the plane is correct. 137 let chk: *u8 = sts_mm(PA_CAP) 138 let chkn: i64 = sts_load(prefix, chk, PA_CAP) 139 let chkrows: i64 = pa_rows(chk, chkn) 140 if chkrows != oldrows + 1 { 141 sts_seed(prefix, oldb, oldn) 142 if quiet == 0 { 143 pa_w("{\"verdict\":\"RED\",\"why\":\"post-commit row count wrong; ORIGINAL RESTORED\",\"expected\":" as *u8) 144 pa_n(oldrows + 1) 145 pa_w(",\"got\":" as *u8) 146 pa_n(chkrows) 147 pa_w("}\n" as *u8) 148 } 149 pa_unlock(lk) 150 return PA_EXIT_CORRUPT 151 } 152 if pa_prefix_ok(oldb, oldn, chk, chkn) != 1 { 153 sts_seed(prefix, oldb, oldn) 154 if quiet == 0 { pa_w("{\"verdict\":\"RED\",\"why\":\"post-commit body no longer starts with the original rows; ORIGINAL RESTORED\"}\n" as *u8) } 155 pa_unlock(lk) 156 return PA_EXIT_CORRUPT 157 } 158 159 if quiet == 0 { 160 pa_w("{\"verdict\":\"GREEN\",\"appended\":1,\"rows_before\":" as *u8) 161 pa_n(oldrows) 162 pa_w(",\"rows_after\":" as *u8) 163 pa_n(chkrows) 164 pa_w(",\"bytes_after\":" as *u8) 165 pa_n(chkn) 166 pa_w("}\n" as *u8) 167 } 168 pa_unlock(lk) 169 return PA_EXIT_OK 170} 171 172func pa_selftest() -> i64 { 173 var pass: i64 = 0 174 var total: i64 = 0 175 let P: *u8 = "knowledge/store/_pa_selftest-" as *u8 176 177 // Ground truth: plant a known 2-row plane using the RAW seeder. 178 let seed: *u8 = sts_mm(PA_MAGIC_4096) 179 var so: i64 = 0 180 so = ss_cat(seed, so, "alpha\tone\n" as *u8) 181 so = ss_cat(seed, so, "beta\ttwo\n" as *u8) 182 sts_seed(P, seed, so) 183 184 // T1 NEGATIVE CONTROL FIRST -- prove the danger this organ exists for is REAL. A raw sts_seed of 185 // just the new row must leave the plane with ONE row, destroying both originals. If this ever 186 // passes with 3, the seeder gained append semantics and this whole organ is obsolete. 187 let raw: *u8 = sts_mm(PA_MAGIC_4096) 188 let ro: i64 = ss_cat(raw, 0, "gamma\tthree\n" as *u8) 189 sts_seed(P, raw, ro) 190 let after_raw: *u8 = sts_mm(PA_MAGIC_4096) 191 let arn: i64 = sts_load(P, after_raw, PA_MAGIC_4096) 192 let arrows: i64 = pa_rows(after_raw, arn) 193 total = total + 1 194 if arrows == 1 { pass = pass + 1; pa_w(" [PASS] " as *u8) } else { pa_w(" [FAIL] " as *u8) } 195 pa_w("T1 NEGATIVE CONTROL: a raw sts_seed DESTROYS the plane (2 rows -> " as *u8); pa_n(arrows); pa_w(", want 1)\n" as *u8) 196 197 // Re-plant the 2-row ground truth for the real test. 198 sts_seed(P, seed, so) 199 200 // T2 the append preserves BOTH originals and adds exactly one. 201 total = total + 1 202 let rc: i64 = pa_append(P, "gamma\tthree" as *u8, 1) 203 let got: *u8 = sts_mm(PA_MAGIC_4096) 204 let gn: i64 = sts_load(P, got, PA_MAGIC_4096) 205 let grows: i64 = pa_rows(got, gn) 206 if rc == 0 { if grows == 3 { pass = pass + 1; pa_w(" [PASS] " as *u8) } else { pa_w(" [FAIL] " as *u8) } } else { pa_w(" [FAIL] " as *u8) } 207 pa_w("T2 APPEND ADDS ONE: 2 rows + append -> " as *u8); pa_n(grows); pa_w(" rows (want 3, rc=" as *u8); pa_n(rc); pa_w(")\n" as *u8) 208 209 // T3 the original bytes survive byte-for-byte at the head. Row-count alone would pass even if the 210 // codec rewrote every original row, so the identity of the ORIGINALS is checked separately. 211 total = total + 1 212 if pa_prefix_ok(seed, so, got, gn) == 1 { pass = pass + 1; pa_w(" [PASS] " as *u8) } else { pa_w(" [FAIL] " as *u8) } 213 pa_w("T3 ORIGINALS INTACT: the post-append body still starts with the exact original bytes\n" as *u8) 214 215 // T4 fail-closed on an absent plane. A typo must never conjure a plane. 216 total = total + 1 217 let rc2: i64 = pa_append("knowledge/store/_pa_does_not_exist-" as *u8, "x\ty" as *u8, 1) 218 if rc2 == PA_EXIT_REFUSED { pass = pass + 1; pa_w(" [PASS] " as *u8) } else { pa_w(" [FAIL] " as *u8) } 219 pa_w("T4 FAIL-CLOSED: appending to an absent plane is REFUSED, not created (rc=" as *u8); pa_n(rc2); pa_w(")\n" as *u8) 220 221 // T5 the preservation check must REJECT a shorter buffer, or every guard above is decorative. 222 total = total + 1 223 let shortb: *u8 = sts_mm(64) 224 let shn: i64 = ss_cat(shortb, 0, "alpha\n" as *u8) 225 if pa_prefix_ok(seed, so, shortb, shn) == 0 { pass = pass + 1; pa_w(" [PASS] " as *u8) } else { pa_w(" [FAIL] " as *u8) } 226 pa_w("T5 GUARD IS NOT DECORATIVE: a shorter/divergent buffer is rejected by the preservation check\n" as *u8) 227 228 // T6 two appends in sequence accumulate. Proves the organ is not itself replace-flavoured. 229 total = total + 1 230 pa_append(P, "delta\tfour" as *u8, 1) 231 let g2: *u8 = sts_mm(PA_MAGIC_4096) 232 let g2n: i64 = sts_load(P, g2, PA_MAGIC_4096) 233 let g2rows: i64 = pa_rows(g2, g2n) 234 if g2rows == 4 { pass = pass + 1; pa_w(" [PASS] " as *u8) } else { pa_w(" [FAIL] " as *u8) } 235 pa_w("T6 APPENDS ACCUMULATE: a second append -> " as *u8); pa_n(g2rows); pa_w(" rows (want 4)\n" as *u8) 236 237 pa_w("\n=== nx_plane_append " as *u8) 238 pa_n(pass) 239 pa_w("/" as *u8) 240 pa_n(total) 241 if pass == total { 242 pa_w(" verdict=GREEN -- proves the raw seeder destroys a plane, then proves this path adds exactly one row while the originals survive byte-for-byte.\n" as *u8) 243 return 0 244 } 245 pa_w(" verdict=RED\n" as *u8) 246 return 1 247} 248 249// --------------------------------------------------------------------------------------------------- 250// FAST APPEND -- O(1) instead of O(plane). THE seq724 AMPLIFICATION FIX. 251// 252// pa_append above is SAFE but not CHEAP: it holds the whole plane in memory and re-seeds it, so appending 253// one row to dp-web-pub- (1.5GB / 4,291,836 entries) rewrites 1.5GB. sts_seed has the same shape, which is 254// why EVERY plane write in the ecosystem is O(plane). 255// 256// The seg-store already supports the cheap path and nothing else needs to change: segments are append-only 257// and reads are NEWEST-WINS per key, so one small segment carrying just the new row plus the updated count 258// IS a complete, correct append. Readers already walk q:0..q:n-1 ACROSS segments -- no reader changes. 259// ss_begin_cap(small) -> ss_add(q:<n>, row) -> ss_add(q:n, n+1) -> ss_commit 260// q:<n> is a key that has never been written; q:n is overwritten and the newest wins. 261// 262// SEGMENT COUNT GROWS BY ONE PER APPEND, AND THAT IS THE INTENDED DESIGN, not a leak: nx_store_fold_beat 263// merges bloated planes on the standing sweep and nx_store_janitor_beat reclaims what folding supersedes. 264// Cheap appends + background compaction is the LSM shape this store was built for, and which re-seeding 265// defeats. Additive: pa_append is untouched, so every existing caller keeps identical semantics. 266// 267// FAIL-CLOSED, same spirit as pa_append: an absent or countless plane is REFUSED, never conjured -- a typo 268// in a prefix must not create a plane that later reads as authoritative. 269func pa_append_fast(prefix: *u8, row: *u8, quiet: i64) -> i64 { 270 let rl: i64 = pa_slen(row) 271 if rl == 0 { 272 pa_w("PLANE-APPEND-FAST REFUSED empty row\n" as *u8) 273 return PA_EXIT_USAGE 274 } 275 let lk: i64 = pa_lock(prefix) 276 let pq: *i64 = sys_mmap(32) as *i64 277 let lq: *i64 = sys_mmap(32) as *i64 278 if ss_get(prefix, "q:n" as *u8, pq, lq) != 1 { 279 pa_unlock(lk) 280 pa_w("PLANE-APPEND-FAST REFUSED plane absent or has no q:n -- never conjuring a new plane\n" as *u8) 281 return PA_EXIT_USAGE 282 } 283 let n: i64 = sts_atoi(pq[0] as *u8, lq[0]) 284 let key: *u8 = sys_mmap(64) 285 sts_rowkey(n, key) 286 let cnt: *u8 = sys_mmap(32) 287 let cl: i64 = ss_catn(cnt, 0, n + 1) 288 let w: *i64 = ss_begin_cap(rl + PA_MAGIC_65536) 289 if ss_add(w, 1, key, row, rl) < 0 { 290 pa_unlock(lk) 291 pa_w("PLANE-APPEND-FAST REFUSED row add failed\n" as *u8) 292 return PA_EXIT_USAGE 293 } 294 if ss_add(w, 1, "q:n" as *u8, cnt, cl) < 0 { 295 pa_unlock(lk) 296 pa_w("PLANE-APPEND-FAST REFUSED count add failed\n" as *u8) 297 return PA_EXIT_USAGE 298 } 299 if ss_commit(prefix, w, ss_next_segid(prefix)) != 0 { 300 pa_unlock(lk) 301 pa_w("PLANE-APPEND-FAST REFUSED commit failed\n" as *u8) 302 return PA_EXIT_USAGE 303 } 304 pa_unlock(lk) 305 if quiet == 0 { 306 pa_w("PLANE-APPEND-FAST ok rows " as *u8) 307 pa_n(n) 308 pa_w(" -> " as *u8) 309 pa_n(n + 1) 310 pa_w(" (one small segment, plane NOT rewritten)\n" as *u8) 311 } 312 return 0 313} 314 315func main(argc: i64, argv: *i64) -> i64 { 316 if argc < 2 { 317 pa_w("usage: nx_plane_append <prefix> <row> | selftest\n" as *u8) 318 sys_exit(PA_EXIT_USAGE) 319 return PA_EXIT_USAGE 320 } 321 if pa_streq(argv[1] as *u8, "selftest" as *u8) == 1 { 322 let f: i64 = pa_selftest() 323 sys_exit(f) 324 return f 325 } 326 if argc < 3 { 327 pa_w("usage: nx_plane_append <prefix> <row> | selftest\n" as *u8) 328 sys_exit(PA_EXIT_USAGE) 329 return PA_EXIT_USAGE 330 } 331 if pa_streq(argv[1] as *u8, "fast" as *u8) == 1 { 332 if argc < 4 { 333 pa_w("usage: nx_plane_append fast <prefix> <row>\n" as *u8) 334 sys_exit(PA_EXIT_USAGE) 335 return PA_EXIT_USAGE 336 } 337 let rcf: i64 = pa_append_fast(argv[2] as *u8, argv[3] as *u8, 0) 338 sys_exit(rcf) 339 return rcf 340 } 341 let rc: i64 = pa_append(argv[1] as *u8, argv[2] as *u8, 0) 342 sys_exit(rc) 343 return rc 344}