nx_store_seed_lib.nx source

1// nx_store_seed_lib.nx -- THE canonical GENERIC store-seed codec (operator 2026-07-17: unify the 2// ~30 bespoke per-plane seeders). Migrates ANY flat staging buffer (# and blank lines skipped) into 3// ANY sovereign seg-store prefix as records key "q:<seq>"=row + "q:n"=count, and reconstructs a 4// newline-joined buffer byte-identical to the flat body. Prefix is a PARAMETER (gates isolate). 5// This is nx_frontier_store's frs_* promoted to the shared home; frontier + commontask delegate here. 6// license_tier: ORIGINAL No hw writes (Rule 26). 7import "nx_seg_store.nx" 8import "nx_syscalls.nx" 9import "nx_record_tsv.nx" // rtv_to_tsv + nxr_count -- the compatibility view (seq1326 rung 3) 10 11const STS_WCAP: i64 = 8388608 // RETIRED as the writer cap (see sts_seed); kept only for callers that still name it 12const STS_ROWOVH: i64 = 32 // per-entry seg-store header: kind + u32 klen + u32 vlen, with margin 13const STS_WSLACK: i64 = 65536 // count row + commit slack 14// ---- PLANE LOCK (seq1544 follow-on) ------------------------------------------------------------- 15// MEASURED 2026-07-30: 55 files call sts_seed; only THREE take any lock. sts_seed is LOAD-ALL / 16// WRITE-ALL -- it REPLACES the whole plane from the caller's buffer -- so two unlocked writers that 17// overlap LOSE DATA, silently and by construction: 18// A loads (n rows) | B loads (n rows) | A commits (n+1) | B commits (n+1, WITHOUT A's row) 19// A's row is simply gone, and the next reader sees a declared/found mismatch it will blame on a 20// "prior writer dropped rows" -- which is exactly what it is, but the cause is the MISSING LOCK. 21// 22// WHY THIS IS A CALLER-HELD LOCK AND NOT AN sts_seed INTERNAL: the critical section is the whole 23// READ-MODIFY-WRITE (load -> mutate -> seed), not the commit alone. A lock taken inside sts_seed would 24// cover only the write half and leave the exact race above wide open. Worse, flock is per OPEN FILE 25// DESCRIPTION, so locking inside sts_seed would DEADLOCK the three callers that already hold 26// <prefix>plock on their own fd -- same process, different fd, blocks forever. 27// 28// ADDITIVE AND OPT-IN: nothing here changes existing behaviour. A caller wraps its RMW: 29// let lk: i64 = sts_lock(prefix) // <=0 means could not lock: refuse, do not proceed unlocked 30// ... sts_load_honest ... mutate ... sts_seed ... 31// sts_unlock(lk) 32// The lock file is <prefix>plock -- the SAME name nx_debt/nx_ecomat_put/nx_frontier_put already use 33// by hand, so adopting this composes with them instead of introducing a second, incompatible lock. 34// ─── DUPLICATE LOCK API REMOVED 2026-07-31 (ws=debt-intelligence) ───────────────────────────────── 35// This file defined sts_lock/sts_unlock TWICE -- here (the original) and again at the STS_LOCKMODE 36// block below. Harmless-looking while the compiler silently picked one; a HARD BUILD FAILURE for 37// EVERY consumer the moment the duplicate-definition guard went fail-closed on 2026-07-31, which is 38// how it was found (nx_debtcluster could not compile, and neither could any other store-seed 39// consumer -- nx_debtlive/nx_debt included). 40// THE SURVIVOR IS THE LOWER ONE, ON MERIT, NOT ON POSITION: it names the failing PATH on an open 41// error instead of letting "cannot lock" blame flock, and its sts_unlock issues an explicit 42// LOCK_UN before close rather than relying on close to drop the flock implicitly. 43// ★LAW: A DUPLICATE DEFINITION IS NOT A STYLE PROBLEM -- IT IS AN UNDECLARED COIN FLIP OVER WHICH 44// IMPLEMENTATION RUNS, AND IT SURVIVES EXACTLY UNTIL SOMETHING STARTS CHECKING. 45 46const STS_HASH: i64 = 35 // '#' 47const STS_NL: i64 = 10 // '\n' 48const STS_Q: i64 = 113 // 'q' 49const STS_COLON: i64 = 58 // ':' 50const STS_ZERO: i64 = 48 // '0' 51const STS_NINE: i64 = 57 // '9' 52const STS_KIND_LIVE: i64 = 1 // seg-store live-record kind (2 = tombstone) 53const STS_KEYCAP: i64 = 64 54const STS_NUMCAP: i64 = 28 55const STS_OUTCAP: i64 = 16 56const STS_BASE10: i64 = 10 57const STS_KPFX: i64 = 2 // "q:" prefix length 58const STS_EXIT_MMAP: i64 = 9 59const STS_MMAP_ERRMAX: i64 = 4096 60const STS_PROBE_WINDOW: i64 = 4096 // detect-only probe span past the declared q:n 61const STS_PROBE_MISS_RUN: i64 = 64 // consecutive misses that end the probe (sparse-tail guard) 62const STS_ERRFD: i64 = 2 63 64// CHECKED allocator (seq350, INCIDENT-0720): under host memory exhaustion sys_mmap returns an 65// error code in the top page; every unchecked caller then writes to it and SEGFAULTS (observed at 66// 0xfffffffffffffff4 = -ENOMEM, with multi-GB cores). This is the SHARED home of the seg-store 67// reader family (nx_debt/nx_debt_hygiene/nx_ws_cycle/nx_sweep_daemon all load through here), so the 68// check lives here ONCE and every member inherits it on its next rebuild. Fail LOUD with a 69// structured constant message (allocation-free by construction), never a segfault. 70func sts_werr(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(STS_ERRFD, s, n); return 0 } 71func sts_mm(n: i64) -> *u8 { 72 let p: *u8 = sys_mmap(n) 73 let v: i64 = p as i64 74 if v < 0 { if v > (0 - STS_MMAP_ERRMAX) { 75 sts_werr("STORE-RED {\"error\":\"mmap-fail\",\"lib\":\"nx_store_seed_lib\",\"hint\":\"host memory exhausted; see debt seq350 / INCIDENT-0720\"}\n" as *u8) 76 sys_exit(STS_EXIT_MMAP) 77 } } 78 return p 79} 80 81const STS_NXR_HDR: i64 = 6 82const STS_NXR_N: i64 = 78 83const STS_NXR_X: i64 = 88 84const STS_NXR_R: i64 = 82 85const STS_NXR_1: i64 = 49 86 87// FORMAT AUTO-DETECT (seq1326 rung 3) -- MUST STAY IDENTICAL TO _hdl_build/nx_store_seed_lib.nx. 88// ⚠⚠⚠DO NOT DROP THIS BLOCK WHEN REWRITING THIS FILE. It was clobbered twice on 2026-07-30 (14:12:33 and 89// 14:22:33) by full-file writes from a concurrent session, each time returning the file to its pre-edit 90// 10199B. knowledge/store/riskreg- is MIGRATED TO NXR1 on disk, so WITHOUT this block nx_info_registry 91// reads that live plane as rows=[] total=54 malformed=54 and nx_store_put load returns shredded garbage. 92// It is purely ADDITIVE: the render path is unreachable for any value not starting with the NXR1 magic, 93// so it is provably inert on every legacy plane and costs other lanes nothing. 94// ⚠Run nx_libcheck before building anything that imports this file -- the runtime/ and _hdl_build/ copies 95// are EACH OTHER'S CONTROL, and divergence means half the tree will silently build old behaviour. 96func sts_emit_row(src: *u8, vl: i64, out: *u8, off: i64, cap: i64) -> i64 { 97 var o: i64 = off 98 if vl >= STS_NXR_HDR { 99 if src[0] == (STS_NXR_N as u8) { if src[1] == (STS_NXR_X as u8) { 100 if src[2] == (STS_NXR_R as u8) { if src[3] == (STS_NXR_1 as u8) { 101 let cols: i64 = nxr_count(src) 102 let n2: i64 = rtv_to_tsv(src, vl, cols, (out as i64 + o) as *u8, cap - o) 103 if n2 >= 0 { return o + n2 } 104 return o 105 } } } } 106 } 107 var t2: i64 = 0 108 while t2 < vl { if o < cap { out[o] = src[t2]; o = o + 1 } t2 = t2 + 1 } 109 return o 110} 111 112func sts_atoi(b: *u8, n: i64) -> i64 { 113 var v: i64 = 0 114 var i: i64 = 0 115 var go: i64 = 1 116 while i < n { 117 if go == 1 { 118 let c: i64 = b[i] as i64 119 if c >= STS_ZERO { if c <= STS_NINE { v = v * STS_BASE10 + (c - STS_ZERO) } else { go = 0 } } else { go = 0 } 120 } 121 i = i + 1 122 } 123 return v 124} 125// ---- THE CAS / GENERATION GUARD (2026-08-06) ---------------------------------------------------- 126// MEASURED by nx_sts_cas_gate on the pre-guard code: six unlocked writers, SIX told they succeeded, 127// ONE row present. Five callers were handed a row count for a write that does not exist. 128// 129// The lock cannot be moved inside sts_seed -- that argument is made twice above and it is correct: 130// the critical section is the whole load -> mutate -> seed, and only the caller can express it. 131// BUT SILENCE IS A SEPARATE DEFECT FROM LOSS, and this half CAN be fixed here. sts_seed cannot make 132// the conflicting writes both survive; it can refuse to LIE about the one it is destroying. 133// 134// THE RULE: remember, per process, the generation of the plane this process last LOADED. If sts_seed 135// is handed that same prefix and the plane has moved since, another writer committed inside our 136// read-modify-write, so seeding would overwrite them from a stale snapshot -> REFUSE. 137// 138// WHY THIS DOES NOT REPEAT THE 2026-08-02 REVERT. A shrink-refusal guard was built here and pulled 139// before promote because it broke LEGITIMATE whole-plane rewriters. This one cannot, because it is 140// keyed on THIS PROCESS HAVING LOADED THIS PREFIX, not on row counts: 141// * nx_plane_append ROLLBACK -- its first seed UPDATES the remembered generation, so the rollback 142// seed that follows matches and is ALLOWED. The failure mode that guard created is gone. 143// * nx_plane_repair / nx_store_put close/setcol -- load then seed with no other writer -> allowed. 144// * fresh seeders that never loaded, and nx_plane_append's T1 negative control that deliberately 145// raw-seeds to prove a raw seed destroys a plane -> no recorded generation for that prefix, so 146// NO CHECK RUNS and the honest gate stays honest. 147// It fires ONLY when this process loaded the plane and somebody else committed in between, which is 148// exactly and only the lost-update case. 149// 150// Two static SCALARS, deliberately not an array: a BSS static array crashes the module on startup in 151// this compiler, and scalars are the proven-safe shape. `static` (no initialiser) is zero-init and 152// persists for the process; a fork gives each child its own copy-on-write copy, which is what makes 153// this correct across the fork-per-worker shape every one of these writers uses. 154const STS_HASHMUL: i64 = 131 155const STS_ERR_STALE: i64 = 0 - 5 156static STS_GEN_KEY: i64 157static STS_GEN_VAL: i64 158 159// Never returns 0 -- 0 is the "nothing recorded" sentinel. A collision would only ever cause a 160// SPURIOUS REFUSAL of an unrelated prefix, never a missed one: it fails closed. 161func sts_pfxhash(prefix: *u8) -> i64 { 162 var h: i64 = 0 163 var i: i64 = 0 164 while prefix[i] != (0 as u8) { 165 h = h * STS_HASHMUL + (prefix[i] as i64) 166 i = i + 1 167 } 168 if h == 0 { h = 1 } 169 return h 170} 171 172func sts_gen_note(prefix: *u8) -> i64 { 173 STS_GEN_KEY = sts_pfxhash(prefix) 174 STS_GEN_VAL = ss_max_segid(prefix) 175 return 0 176} 177 178func sts_rowkey(seq: i64, out: *u8) -> i64 { 179 out[0] = STS_Q as u8 180 out[1] = STS_COLON as u8 181 var o: i64 = ss_catn(out, STS_KPFX, seq) 182 out[o] = 0 as u8 183 return o 184} 185// migrate a flat buffer into the store at `prefix`; ONE commit. returns row count or -1. 186func sts_seed(prefix: *u8, buf: *u8, n: i64) -> i64 { 187 // CAS GUARD -- see the block above sts_rowkey. Runs ONLY if THIS process loaded THIS prefix. 188 // Refusing costs the caller its own write; NOT refusing costs another writer theirs, silently, 189 // and hands this caller a row count for data that will never exist. 190 let gh: i64 = sts_pfxhash(prefix) 191 var expect: i64 = SS_CAS_ANY 192 if STS_GEN_KEY == gh { 193 expect = STS_GEN_VAL 194 } 195 // WRITER CAPACITY IS DATA-DRIVEN, NOT A CEILING (2026-07-30). This was ss_begin_cap(STS_WCAP) 196 // with STS_WCAP a fixed 1MiB. sts_seed rewrites the WHOLE plane on every write, so the moment a 197 // plane outgrew 1MiB the trailing ss_add calls failed -- and their return value was DISCARDED -- 198 // while the q:n count row still recorded the INTENDED total. The plane then declared more rows 199 // than it contained: silent, unbounded data loss that only surfaced later as DEBT-REFUSED 200 // lossy-load, with the debt board (the ecosystem ledger) frozen read-only. 201 // Size from the actual payload the way reg_put and ss_compact_cap already do: every byte of 202 // input, plus per-row key+header overhead, plus the count row and slack. A plane can now grow 203 // to whatever it needs; there is no number here for a future plane to outgrow. 204 var rowcount: i64 = 0 205 var scan: i64 = 0 206 while scan < n { if buf[scan] == (STS_NL as u8) { rowcount = rowcount + 1 } scan = scan + 1 } 207 // ⚠A SHRINK-REFUSAL GUARD WAS ATTEMPTED HERE 2026-08-02 AND DELIBERATELY REVERTED BEFORE PROMOTE. 208 // It would have refused any sts_seed whose input holds fewer rows than the plane already has -- 209 // correct for the clobber class (dedupq- seq756, commontask- 08-02) but WRONG HERE, because a 210 // blast-radius read of the 118 call sites found LEGITIMATE shrinking writers that the guard would 211 // have broken, some catastrophically: 212 // * nx_plane_append ROLLBACK -- sts_seed(prefix, oldb, oldn) restores the PRE-append buffer after a 213 // failed write; refusing that strands the plane in the failed state (the guard would cause the 214 // very loss it exists to prevent). 215 // * nx_plane_repair -- repairing a corrupted plane legitimately produces FEWER rows. 216 // * nx_plane_append's own T1 NEGATIVE CONTROL deliberately raw-seeds to PROVE a raw seed destroys a 217 // plane (2 rows -> 1); refusing it turns a healthy liar-killed gate RED. 218 // * nx_store_put close/setcol paths rewrite the whole plane by design. 219 // THE GUARD BELONGS AT THE PLANE-WRITER LAYER, NOT IN THE SHARED PRIMITIVE: see debt 1785710884 -- 220 // either a checked twin (sts_seed_checked) that intentional shrinkers opt out of, or the guard inside 221 // nx_store_put's put path only. The primitive stays raw and DOCUMENTED-DANGEROUS; nx_plane_append is 222 // already the one correct append path. 223 let needed: i64 = n + (rowcount + 2) * (STS_KEYCAP + STS_ROWOVH) + STS_WSLACK 224 let w: *i64 = ss_begin_cap(needed) 225 let key: *u8 = sts_mm(STS_KEYCAP) 226 let base: i64 = buf as i64 227 var seq: i64 = 0 228 var i: i64 = 0 229 while i < n { 230 var le: i64 = i 231 var s: i64 = 1 232 while s == 1 { if le >= n { s = 0 } else { if buf[le] == (STS_NL as u8) { s = 0 } else { le = le + 1 } } } 233 if le > i { if buf[i] != (STS_HASH as u8) { 234 sts_rowkey(seq, key) 235 // CHECKED: a dropped row used to be invisible here, which is what let q:n over-declare. 236 if ss_add(w, STS_KIND_LIVE, key, (base + i) as *u8, le - i) < 0 { return 0 - 2 } 237 seq = seq + 1 238 } } 239 i = le + 1 240 } 241 let cb: *u8 = sts_mm(STS_NUMCAP) 242 let cl: i64 = ss_catn(cb, 0, seq) 243 if ss_add(w, STS_KIND_LIVE, "q:n" as *u8, cb, cl) < 0 { return 0 - 2 } 244 // CAS: the plane must still be at the generation this process loaded, checked UNDER THE PLANE 245 // LOCK inside ss_commit_cas. Checking it here instead is what the first attempt did, and it 246 // measured EXACTLY ZERO improvement -- see the note above ss_commit_cas. 247 let rc: i64 = ss_commit_cas(prefix, w, ss_next_segid(prefix), expect) 248 if rc == SS_ERR_STALE { return STS_ERR_STALE } 249 // Re-baseline ON SUCCESS ONLY. This is what keeps a legitimate second write from this same 250 // process -- nx_plane_append's rollback seed above all -- from being refused for the generation 251 // bump its OWN first seed just caused. 252 if rc == 0 { sts_gen_note(prefix) } 253 if rc != 0 { return 0 - 1 } 254 return seq 255} 256// --------------------------------------------------------------------------------------------------- 257// sts_append_fast -- O(1) APPEND OF ONE ROW. The seq724 amplification fix, living beside sts_seed so every 258// writer that already imports this lib can reach it without a new dependency. 259// 260// sts_seed REWRITES THE WHOLE PLANE. Measured 2026-07-30 on a 2000-row / 210,906-byte plane: appending one 261// row via re-seed wrote a 210,969-byte segment; this path wrote 47 BYTES. 4489x less. On dp-web-pub- 262// (1.5GB, 4,291,836 entries) that is 1.5GB rewritten versus about 50 bytes. 263// 264// IT IS NOT ONLY A SPEED FIX -- IT REMOVES TWO WHOLE DEFECT CLASSES BY CONSTRUCTION: 265// * PARTIAL-LOAD DROPS ROWS: a re-seed rebuilds the plane from whatever the loader could reach, so a 266// partial read silently deletes every row past the declared count (the lossy-plane class). An append 267// rewrites NOTHING, so an incomplete read cannot destroy a row it never saw. 268// * ORDINAL SHIFT: a re-seed RE-SEGMENTS rows (any row containing a newline splits), shifting every 269// ordinal after it -- which is how a positional seq issued before a re-seed later addresses a 270// DIFFERENT row. An append adds q:<n> and touches no existing key, so ordinals are stable. 271// 272// Correctness rests on properties the store already guarantees: segments are append-only and reads are 273// NEWEST-WINS per key; q:<n> is a key never written before; q:n is simply overwritten; and every reader 274// already walks q:0..q:n-1 ACROSS segments. NO READER CHANGES ARE REQUIRED. 275// Segment count grows by one per append BY DESIGN -- nx_store_fold_beat merges on the standing sweep and 276// nx_store_janitor_beat reclaims what folding supersedes. Cheap appends + background compaction is the LSM 277// shape this store was built for and that re-seeding defeats. 278// FAIL-CLOSED: a plane with no q:n is REFUSED (-1), never conjured. Returns the NEW row count, or -1/-2. 279func sts_append_fast(prefix: *u8, row: *u8, rowlen: i64) -> i64 { 280 if rowlen <= 0 { return 0 - 1 } 281 let pq: *i64 = sts_mm(STS_OUTCAP) as *i64 282 let lq: *i64 = sts_mm(STS_OUTCAP) as *i64 283 // CAS BASELINE, captured BEFORE the q:n read on purpose. Taking it after would leave a window in 284 // which a commit lands between the read and the baseline and is never noticed; taking it first can 285 // only ever over-refuse, which is the safe direction. Unlike sts_seed this needs no process-local 286 // state at all -- the read and the commit are both inside THIS function, so the generation is just 287 // a local. MEASURED before the fix by nx_sts_cas_gate T3: 6 writers told yes, 1 row present. 288 let gen0: i64 = ss_max_segid(prefix) 289 if ss_get(prefix, "q:n" as *u8, pq, lq) != 1 { return 0 - 1 } 290 let n: i64 = sts_atoi(pq[0] as *u8, lq[0]) 291 let key: *u8 = sts_mm(STS_KEYCAP) 292 sts_rowkey(n, key) 293 let cb: *u8 = sts_mm(STS_NUMCAP) 294 let cl: i64 = ss_catn(cb, 0, n + 1) 295 let w: *i64 = ss_begin_cap(rowlen + STS_WSLACK) 296 if ss_add(w, STS_KIND_LIVE, key, row, rowlen) < 0 { return 0 - 2 } 297 if ss_add(w, STS_KIND_LIVE, "q:n" as *u8, cb, cl) < 0 { return 0 - 2 } 298 // If ANY writer committed since gen0, this appender's q:<n> would collide with theirs and the 299 // NEWEST-WINS read would silently discard one of the two rows. Refuse instead: the caller gets a 300 // negative return and can retry against the new n, which is the whole difference between 301 // backpressure and corruption. 302 let crc: i64 = ss_commit_cas(prefix, w, ss_next_segid(prefix), gen0) 303 if crc == SS_ERR_STALE { return STS_ERR_STALE } 304 if crc != 0 { return 0 - 2 } 305 return n + 1 306} 307 308// UPDATED 2026-08-06 -- THIS PARAGRAPH USED TO OPEN "sts_append_fast IS STILL A READ-MODIFY-WRITE 309// AND IT STILL LOSES ROWS UNDER CONCURRENCY". THAT IS NO LONGER TRUE OF THE CODE ABOVE, and leaving 310// it would have been worse than never writing it: a comment that contradicts its own function is read 311// as the specification by the next person, who then re-derives a defect that is already fixed. 312// IT IS STILL A READ-MODIFY-WRITE. It no longer loses rows SILENTLY. 313// The diagnosis below is preserved verbatim because it is exactly right about the MECHANISM -- it is 314// the WHY for the CAS baseline in the body above. Only its conclusion changed. 315// MEASURED both ways by nx_sts_cas_gate T3/T4: before, 6 concurrent appenders were ALL told they 316// succeeded while 1 row existed (5 silent losses); after, every writer told YES has written, and the 317// losers get STS_ERR_STALE and can retry against the new n. 318// Measured/reasoned 2026-07-30 (ws=sev-eater): it ss_get's q:n to learn n, then writes q:<n> and 319// q:n=n+1. Two concurrent appenders both read n, both write THE SAME q:<n> key -- and reads are 320// NEWEST-WINS per key, so one row does not merely fail to appear, it is silently OVERWRITTEN -- and 321// both then set q:n=n+1, so the count agrees with the loss and no guard ever notices. It is 4489x 322// cheaper than a re-seed and exactly as lossy; SPEED IS NOT SAFETY. 323// 324// THE LOCK IS DELIBERATELY *NOT* INSIDE sts_append_fast. Audited its callers first, and nx_debt 325// (nx_debt.nx:320) ALREADY HOLDS <prefix>plock via db_lock and then calls it on THAT SAME prefix -- 326// flock is per open-file-description, so an internal lock would make nx_debt block against ITSELF 327// and hang the debt board for every seat. Same trap that rules out locking sts_seed. 328// ★LAW, twice proven in one file now: BEFORE LOCKING A SHARED PRIMITIVE, AUDIT WHO ALREADY HOLDS 329// THAT LOCK -- the callers doing it right are what the internal lock breaks first. 330// 331// So the safe variant is a WRAPPER. Callers that do NOT already hold the plane lock use this one; 332// callers that do (nx_debt, nx_frontier_put) keep calling the raw primitive and stay correct. 333// (the wrapper itself is defined at the END of this file, below sts_lock -- NishiLang resolves 334// identifiers in TEXTUAL order and a forward reference to sts_lock from here would not compile.) 335 336// read the store at `prefix` -> newline-joined row buffer. returns bytes. 337// seq356 (INCIDENT-0720 residual): the original body did one ss_get PER ROW, and every ss_get 338// re-opens + re-maps the whole seglist => O(rows x segments) file IO -- the amplification that 339// helped the reader family (nx_debt/nx_ws_cycle/...) OOM the host. NOW: ss_open ONCE -> 340// ss_hget per row (zero file IO per call; the proven WMS audit-scale pattern). Semantics 341// identical BY ORACLE: sts_load_slow below is the verbatim old body and the bulkload gate 342// proves byte-identical output on single-seg, multi-seg-supersede, and shrink planes. 343func sts_load(prefix: *u8, out: *u8, cap: i64) -> i64 { 344 // Record the generation we are ABOUT to read at. Deliberately before the read, not after: if a 345 // writer commits DURING our load our snapshot is already torn, and stamping the earlier 346 // generation means sts_seed will catch it. Conservative on purpose -- it can only over-refuse. 347 sts_gen_note(prefix) 348 // ss_open_cached, NOT ss_open (seq905/962 CLASS FIX, 2026-07-30). ss_open reads every live segment 349 // fully into anonymous RAM and nothing frees it, so a consumer that loads repeatedly leaks the WHOLE 350 // store per load -- the fingerprint behind ~23.6 GiB of anonymous address space across nine organs and 351 // the 92.3%-swap thrash. MEASURED 2026-07-30: 133 files reach ss_open but only 1 called ss_close, so 352 // the fix existed and was dark. This lib is a CHOKEPOINT -- 63 importers, and it is the exact reader 353 // family (nx_debt / nx_ws_cycle / ...) named above as having OOM'd the host -- so migrating the two 354 // opens here reaches far more consumers than migrating leaf call sites one at a time (rule 15). 355 // Cache is prefix-keyed with manifest (st_size, st_mtime) invalidation, so LIVE EDITS ARE STILL PICKED 356 // UP on the next call with no restart; you simply cannot leak what you never allocate twice. 357 // ⚠Safe here because these loaders are single-threaded and run to completion before the next call, so 358 // the handle's shared query-scratch arena is never re-entered. Do NOT blanket-alias ss_open itself. 359 let h: *i64 = ss_open_cached(prefix) 360 if (h as i64) == 0 { return 0 } 361 let pq: *i64 = sts_mm(STS_OUTCAP) as *i64 362 let lq: *i64 = sts_mm(STS_OUTCAP) as *i64 363 if ss_hget(h, "q:n" as *u8, pq, lq) != 1 { return 0 } 364 let cnt: i64 = sts_atoi(pq[0] as *u8, lq[0]) 365 let key: *u8 = sts_mm(STS_KEYCAP) 366 let pr: *i64 = sts_mm(STS_OUTCAP) as *i64 367 let lr: *i64 = sts_mm(STS_OUTCAP) as *i64 368 var o: i64 = 0 369 var seq: i64 = 0 370 var lost: i64 = 0 371 while seq < cnt { 372 sts_rowkey(seq, key) 373 if ss_hget(h, key, pr, lr) == 1 { 374 let src: *u8 = pr[0] as *u8 375 let vl: i64 = lr[0] 376 o = sts_emit_row(src, vl, out, o, cap) 377 if o >= cap { lost = lost + 1 } 378 if o < cap { out[o] = STS_NL as u8; o = o + 1 } 379 if o >= cap { lost = lost + 1 } 380 } 381 seq = seq + 1 382 } 383 // ⚠TRUNCATION IS LOUD (ported from the authoring tree 2026-08-06 -- it was authored there on 384 // 2026-07-30 and NEVER REACHED THE TREE THAT COMPILES AND DEPLOYS, so every organ built here has 385 // been truncating silently ever since). The loop above drops bytes whenever o >= cap and returns a 386 // byte count that looks perfectly healthy, so a caller whose cap is smaller than the plane reads a 387 // PARTIAL board and reports on it as if complete. 388 // MEASURED SCOPE (authoring-tree note, preserved): 126 call sites use this plain loader against 15 389 // on sts_load_honest -- ~89% of all plane reads were silent; once the debt plane passed 1MB, 390 // nx_sheriff / nx_pm_board / nx_pm_cockpit / nx_law_warden / nx_dora / nx_debt_hygiene / 391 // nx_backlog_feeder were ALL reading a TRUNCATED board without knowing. 392 // ★WHY STDERR AND NOT A RETURN CODE: the return is a BYTE COUNT 126 callers already consume; 393 // changing that contract would be a silent semantic break across the estate -- the very failure 394 // being fixed. A diagnostic makes the condition VISIBLE without changing any contract. 395 // ★LAW: a reader that cannot fit its source must SAY SO. Under-reporting is indistinguishable from 396 // good news, which is what makes it dangerous. 397 // ⚠DELIBERATE SHAPE DIFFERENCE FROM THE AUTHORING COPY: that one counts DROPPED BYTES inside an 398 // inline byte loop; this tree emits rows through sts_emit_row (the NXR1 auto-detect the authoring 399 // copy does not have), so `lost` counts ROWS-at-the-boundary instead. The warning predicate is 400 // `lost > 0` in both, so the DIAGNOSTIC IS IDENTICAL; only the unit behind it differs. 401 if lost > 0 { 402 sts_werr("STS-LOAD TRUNCATED plane=" as *u8) 403 sts_werr(prefix) 404 sts_werr(" -- read cap too small; bytes DROPPED, the returned buffer is a PARTIAL board. Raise the caller's cap or use sts_load_honest.\n" as *u8) 405 } 406 return o 407} 408 409// ---- sts_load_fit -- A LOADER THAT CANNOT BE UNDER-CAPPED (2026-08-06) -------------------------- 410// nx_debt's DB_CAP comment states the requirement exactly: "THIS IS THE THIRD RAISE, NOT A FIX: the 411// structural answer is to compare the cap against the actual plane size and REFUSE LOUDLY instead of 412// silently returning a prefix -- A CAP THAT CAN BE CROSSED IN SILENCE WILL BE CROSSED AGAIN." 413// Raising a constant buys time; it does not remove the class. This removes the class: the caller 414// stops choosing a number at all, and the loader sizes itself from the plane. 415// 416// ★THE TEST IS EXACT AND NEEDS NO STAT: sts_load truncates ONLY when its buffer fills, so a return 417// STRICTLY LESS THAN cap PROVES the read was complete. n == cap is the only ambiguous case -- it may 418// be an exact fit or a truncation -- so that is the single case we grow on. No sizing heuristic, no 419// estimate that can be wrong in the dangerous direction. 420// 421// ★STARTS ABOVE EVERY MEASURED PLANE ON PURPOSE. nx_planefit swept all 1,235 planes on 2026-08-06: 422// the largest was knowledge/store/debt- at 4,416,561B. An 8 MiB first attempt therefore completes in 423// ONE pass for every plane that exists today, so the common path costs one mmap and prints no 424// spurious truncation warning; growth is the rare fallback, not the norm. 425// 426// ★FAILURE IS LOUD AND BOUNDED. Doubling is capped, and exhausting it RETURNS NULL with a named 427// diagnostic rather than handing back a partial board -- the whole defect being closed here is a 428// short read that looks healthy, so this must never be able to produce one. 429// ⚠Each failed attempt is MUNMAPPED before growing: sys_mmap is page-granular with no allocator 430// behind it, and an unpaired scratch mmap leaks a full page per call (the 27.7GB class). 431// out_len[0] = bytes on success, -1 on refusal. Returns the buffer, or null on refusal. 432const STS_FIT_START: i64 = 8388608 // 8 MiB: ~1.9x the largest plane measured 2026-08-06 433const STS_FIT_MAXTRIES: i64 = 8 // 8 MiB .. 1 GiB, then REFUSE 434func sts_load_fit(prefix: *u8, out_len: *i64) -> *u8 { 435 var cap: i64 = STS_FIT_START 436 var tries: i64 = 0 437 while tries < STS_FIT_MAXTRIES { 438 let buf: *u8 = sts_mm(cap) 439 let n: i64 = sts_load(prefix, buf, cap) 440 // STRICTLY less than cap => the buffer never filled => nothing was dropped. Complete. 441 if n < cap { 442 out_len[0] = n 443 return buf 444 } 445 sys_munmap(buf, cap) 446 cap = cap * 2 447 tries = tries + 1 448 } 449 sts_werr("STS-LOAD-FIT REFUSED plane=" as *u8) 450 sts_werr(prefix) 451 sts_werr(" -- still filling the buffer at 1 GiB. REFUSING to return a partial board; shard the plane or raise STS_FIT_MAXTRIES deliberately.\n" as *u8) 452 out_len[0] = 0 - 1 453 return 0 as *u8 454} 455// HONEST LOADER (2026-07-25, ws=debt-instrument). sts_load trusts the q:n count key and stops 456// at it with NO error and NO flag, so a low/stale q:n SILENTLY truncates every reader that uses 457// it. MEASURED on the debt- plane: q:n resolved 125 while 629 segments held 734+ rows, and 458// nx_debt list reported total=125 AS FACT. Loads byte-identically to sts_load (which is left 459// UNTOUCHED for API-contract stability, rule 19) but additionally REPORTS what it could not 460// reach, so partial coverage can never again be presented as complete (honesty law L011). 461// flags[0]=declared q:n flags[1]=rows actually loaded flags[2]=rows reachable BEYOND q:n 462// DETECT-ONLY BY CONSTRUCTION: never loads a row past the declared count. sts_seed REASSIGNS 463// q:<i> on every commit, so rows past q:n belong to an OLDER seeding generation -- loading them 464// would duplicate live rows and revive eaten ones. Reporting the gap is safe; reconciling it is 465// a separate operation that must diff the generations. 466// SAFE ASYMMETRY it relies on: a too-LOW q:n silently truncates, a too-HIGH q:n is harmless on 467// read because ss_hget misses are skipped -- so probing past the count cannot corrupt. 468func sts_load_honest(prefix: *u8, out: *u8, cap: i64, flags: *i64) -> i64 { 469 sts_gen_note(prefix) 470 flags[0] = 0 471 flags[1] = 0 472 flags[2] = 0 473 // ss_open_cached -- same seq905/962 class fix and same single-threaded-loader reasoning as sts_load. 474 let h: *i64 = ss_open_cached(prefix) 475 if (h as i64) == 0 { return 0 } 476 let pq: *i64 = sts_mm(STS_OUTCAP) as *i64 477 let lq: *i64 = sts_mm(STS_OUTCAP) as *i64 478 if ss_hget(h, "q:n" as *u8, pq, lq) != 1 { return 0 } 479 let cnt: i64 = sts_atoi(pq[0] as *u8, lq[0]) 480 flags[0] = cnt 481 let key: *u8 = sts_mm(STS_KEYCAP) 482 let pr: *i64 = sts_mm(STS_OUTCAP) as *i64 483 let lr: *i64 = sts_mm(STS_OUTCAP) as *i64 484 var o: i64 = 0 485 var seq: i64 = 0 486 while seq < cnt { 487 sts_rowkey(seq, key) 488 if ss_hget(h, key, pr, lr) == 1 { 489 let src: *u8 = pr[0] as *u8 490 let vl: i64 = lr[0] 491 o = sts_emit_row(src, vl, out, o, cap) 492 if o < cap { out[o] = STS_NL as u8; o = o + 1 } 493 flags[1] = flags[1] + 1 494 } 495 seq = seq + 1 496 } 497 var beyond: i64 = 0 498 var miss: i64 = 0 499 var p: i64 = cnt 500 let lim: i64 = cnt + STS_PROBE_WINDOW 501 while p < lim { 502 if miss < STS_PROBE_MISS_RUN { 503 sts_rowkey(p, key) 504 if ss_hget(h, key, pr, lr) == 1 { beyond = beyond + 1; miss = 0 } else { miss = miss + 1 } 505 } 506 p = p + 1 507 } 508 flags[2] = beyond 509 return o 510} 511// ---- ATOMIC PLANE APPEND (root fix for seq1559, ws=sev-eater 2026-07-30) ---------------------- 512// 55 files call sts_seed(); only THREE take a lock. sts_seed REPLACES the whole plane from the 513// caller's buffer, and the universal call shape is READ-MODIFY-WRITE: 514// n = sts_load(prefix, buf, cap); ...append a row...; sts_seed(prefix, buf, n) 515// Two overlapping unlocked writers interleave as A-loads / B-loads / A-commits n+1 / 516// B-commits n+1-WITHOUT-A. A's row is gone, silently, and the next reader's declared-vs-found 517// mismatch blames a prior writer for dropping rows -- true, but the wrong cause. 518// 519// WHY THE OBVIOUS FIX IS WRONG AND IS NOT DONE HERE: putting the lock INSIDE sts_seed would 520// DEADLOCK the three callers that already do it right. flock(2) locks are per OPEN FILE 521// DESCRIPTION, not per process, so nx_debt -- which already holds <prefix>plock via db_lock and 522// then calls sts_seed -- would block against ITSELF on a second fd of the same file. That trades 523// silent data loss for a hard hang of the debt board for every seat, which is strictly worse. 524// ★LAW: A CHOKEPOINT FIX MUST NOT ASSUME ITS CALLERS ARE ALL WRONG -- the ones already doing it 525// right are exactly what a naive fix breaks first. 526// 527// So sts_seed is left UNLOCKED and this is added ALONGSIDE it: the whole critical section as ONE 528// call. ★LAW: THE RIGHT PRIMITIVE IS THE CRITICAL SECTION, NOT THE LOCK -- a lock 52 callers must 529// remember to take is the same class of defect as the missing lock. Callers migrate to 530// sts_append_row and become correct by construction; nothing has to be remembered. 531// 532// The lock file is <prefix>plock -- deliberately THE SAME PATH nx_debt/nx_ecomat_put/ 533// nx_frontier_put already use, so migrated and unmigrated writers share ONE lock domain. A new 534// lock file would have been easier and would have protected nothing. 535const STS_LOCK_EX: i64 = 2 536const STS_LOCK_UN: i64 = 8 537const STS_LOCKMODE: i64 = 420 538const STS_LOCKPATHCAP: i64 = 256 539const STS_ERR_LOCK: i64 = 0 - 3 540const STS_ERR_CAP: i64 = 0 - 4 541 542// Take the plane's exclusive lock. Returns the held fd, or -1 (LOUD: names the path, because 543// "cannot lock" alone has historically blamed flock when the cause was an open failure). 544func sts_lock(prefix: *u8) -> i64 { 545 let p: *u8 = sts_mm(STS_LOCKPATHCAP) 546 var o: i64 = ss_cat(p, 0, prefix) 547 o = ss_cat(p, o, "plock" as *u8) 548 p[o] = 0 as u8 549 let fd: i64 = sys_openat_append(p, STS_LOCKMODE) 550 if fd < 0 { 551 sts_werr("STS-LOCK open-failed (not flock) path=" as *u8) 552 sts_werr(p) 553 sts_werr("\n" as *u8) 554 return 0 - 1 555 } 556 sys_flock(fd, STS_LOCK_EX) 557 return fd 558} 559 560func sts_unlock(fd: i64) -> i64 { 561 if fd < 0 { return 0 } 562 sys_flock(fd, STS_LOCK_UN) 563 sys_close(fd) 564 return 0 565} 566 567// ATOMIC ROW APPEND: lock -> load -> append -> commit -> unlock, with no window in which another 568// writer can load a snapshot this one is about to invalidate. Returns sts_seed's row count, or 569// STS_ERR_LOCK / STS_ERR_CAP. Fail-closed: if the buffer cannot hold the appended row the plane is 570// left EXACTLY as it was rather than committed short (a truncated commit is the very loss this 571// exists to prevent). 572func sts_append_row(prefix: *u8, row: *u8, rowlen: i64, cap: i64) -> i64 { 573 let fd: i64 = sts_lock(prefix) 574 if fd < 0 { return STS_ERR_LOCK } 575 let buf: *u8 = sts_mm(cap) 576 var n: i64 = sts_load(prefix, buf, cap) 577 if n < 0 { n = 0 } 578 if n + rowlen + 1 > cap { sts_unlock(fd); return STS_ERR_CAP } 579 var i: i64 = 0 580 while i < rowlen { buf[n] = row[i]; n = n + 1; i = i + 1 } 581 buf[n] = STS_NL as u8 582 n = n + 1 583 let rc: i64 = sts_seed(prefix, buf, n) 584 sts_unlock(fd) 585 return rc 586} 587 588// Locked form of sts_append_fast, defined HERE because it needs sts_lock above it (textual order). 589// Use this from any caller that does NOT already hold the plane lock; callers that DO (nx_debt, 590// nx_frontier_put) must keep calling the raw sts_append_fast or they will deadlock against 591// themselves -- see the audit note beside sts_append_fast. 592func sts_append_fast_locked(prefix: *u8, row: *u8, rowlen: i64) -> i64 { 593 let fd: i64 = sts_lock(prefix) 594 if fd < 0 { return STS_ERR_LOCK } 595 let rc: i64 = sts_append_fast(prefix, row, rowlen) 596 sts_unlock(fd) 597 return rc 598} 599 600// the pre-seq356 body VERBATIM (one ss_get per row) -- kept as the equivalence ORACLE for 601// nx_sts_bulkload_gate; not for production use (the amplification lives here). 602func sts_load_slow(prefix: *u8, out: *u8, cap: i64) -> i64 { 603 let pq: *i64 = sts_mm(STS_OUTCAP) as *i64 604 let lq: *i64 = sts_mm(STS_OUTCAP) as *i64 605 if ss_get(prefix, "q:n" as *u8, pq, lq) != 1 { return 0 } 606 let cnt: i64 = sts_atoi(pq[0] as *u8, lq[0]) 607 let key: *u8 = sts_mm(STS_KEYCAP) 608 let pr: *i64 = sts_mm(STS_OUTCAP) as *i64 609 let lr: *i64 = sts_mm(STS_OUTCAP) as *i64 610 var o: i64 = 0 611 var seq: i64 = 0 612 while seq < cnt { 613 sts_rowkey(seq, key) 614 if ss_get(prefix, key, pr, lr) == 1 { 615 let src: *u8 = pr[0] as *u8 616 let vl: i64 = lr[0] 617 o = sts_emit_row(src, vl, out, o, cap) 618 if o < cap { out[o] = STS_NL as u8; o = o + 1 } 619 } 620 seq = seq + 1 621 } 622 return o 623} 624 625// ---- ROW-LEVEL WRITES WITHOUT A RESEED (2026-09-02, loadgov LV17 sts_put_append) ----------------------------- 626// THE STORM, MEASURED: nx_store_put put/close/setcol reload the whole plane and hand it to sts_seed, so ONE row costs 627// ONE fsync'd rewrite of the WHOLE plane (the debt plane is 4.9 MB). Five such writers in flight held the RAID array in 628// D-state for minutes and every /api/build on the estate was refused for hours (nx_dstate roster, 2026-09-02). The store 629// already had the O(1) shape for NEW rows (sts_append_fast: one segment holding q:<n> + q:n). What it lacked was the 630// same shape for a MODIFIED row. These two functions supply it, and nx_store_put composes them. 631// sts_find_seq: the seq (0..q:n-1) whose row's FIRST column equals `id`, or -1. Walks the same q:0..q:n-1 order every 632// reader uses, on the cached handle (zero file IO per key), decoding NXR1 rows exactly as sts_load does so the id 633// comparison is on the tsv form every caller sees. -1 is also returned for an unseeded plane (no q:n). 634const STS_ROWTAB: i64 = 9 635const STS_FINDCAP: i64 = 65536 // one decoded row; the largest plane row measured is far below this, and a longer row 636 // simply fails to match (never a wrong match): sts_emit_row stops at the cap 637func sts_find_seq(prefix: *u8, id: *u8) -> i64 { 638 let h: *i64 = ss_open_cached(prefix) 639 if (h as i64) == 0 { return 0 - 1 } 640 let pq: *i64 = sts_mm(STS_OUTCAP) as *i64 641 let lq: *i64 = sts_mm(STS_OUTCAP) as *i64 642 if ss_hget(h, "q:n" as *u8, pq, lq) != 1 { return 0 - 1 } 643 let cnt: i64 = sts_atoi(pq[0] as *u8, lq[0]) 644 let key: *u8 = sts_mm(STS_KEYCAP) 645 let pr: *i64 = sts_mm(STS_OUTCAP) as *i64 646 let lr: *i64 = sts_mm(STS_OUTCAP) as *i64 647 let tmp: *u8 = sts_mm(STS_FINDCAP) 648 var idl: i64 = 0 649 while id[idl] != (0 as u8) { idl = idl + 1 } 650 var seq: i64 = 0 651 var hit: i64 = 0 - 1 652 while seq < cnt { 653 if hit < 0 { 654 sts_rowkey(seq, key) 655 if ss_hget(h, key, pr, lr) == 1 { 656 let n2: i64 = sts_emit_row(pr[0] as *u8, lr[0], tmp, 0, STS_FINDCAP) 657 var c: i64 = 0 658 var go: i64 = 1 659 while go == 1 { if c >= n2 { go = 0 } else { if tmp[c] == (STS_ROWTAB as u8) { go = 0 } else { c = c + 1 } } } 660 if c == idl { 661 var same: i64 = 1 662 var k: i64 = 0 663 while k < idl { if tmp[k] != id[k] { same = 0; k = idl } else { k = k + 1 } } 664 if same == 1 { hit = seq } 665 } 666 } 667 } 668 seq = seq + 1 669 } 670 sys_munmap(tmp, STS_FINDCAP) 671 return hit 672} 673// sts_replace_fast: overwrite the row at `seq` in ONE new segment. NEWEST-WINS per key means every reader sees the new 674// bytes at the same position: row count and order are unchanged, q:n is untouched, NO READER CHANGES ARE REQUIRED. 675// FAIL-CLOSED: no q:n, or seq outside 0..q:n-1, is refused (-1) with the plane untouched. Same CAS discipline as 676// sts_append_fast (baseline segid taken BEFORE the q:n read; a commit that lands in between returns STS_ERR_STALE). 677// LOCKING: like sts_append_fast this is the RAW primitive -- the caller holds <prefix>plock (nx_store_put does, at main). 678// Returns q:n (the unchanged row count) on success, -1 refused, -2 write error, STS_ERR_STALE on a lost race. 679func sts_replace_fast(prefix: *u8, seq: i64, row: *u8, rowlen: i64) -> i64 { 680 if rowlen <= 0 { return 0 - 1 } 681 if seq < 0 { return 0 - 1 } 682 let pq: *i64 = sts_mm(STS_OUTCAP) as *i64 683 let lq: *i64 = sts_mm(STS_OUTCAP) as *i64 684 let gen0: i64 = ss_max_segid(prefix) 685 if ss_get(prefix, "q:n" as *u8, pq, lq) != 1 { return 0 - 1 } 686 let n: i64 = sts_atoi(pq[0] as *u8, lq[0]) 687 if seq >= n { return 0 - 1 } 688 let key: *u8 = sts_mm(STS_KEYCAP) 689 sts_rowkey(seq, key) 690 let w: *i64 = ss_begin_cap(rowlen + STS_WSLACK) 691 if ss_add(w, STS_KIND_LIVE, key, row, rowlen) < 0 { return 0 - 2 } 692 let crc: i64 = ss_commit_cas(prefix, w, ss_next_segid(prefix), gen0) 693 if crc == SS_ERR_STALE { return STS_ERR_STALE } 694 if crc != 0 { return 0 - 2 } 695 return n 696}