nx_store_seed_lib.nx source

1// nx_store_seed_lib.nx -- THE canonical GENERIC store-seed codec (operator 2026-07-17: unify the 2// ~30 bespoke per-plane seeders). Migrates ANY flat staging buffer (# and blank lines skipped) into 3// ANY sovereign seg-store prefix as records key "q:<seq>"=row + "q:n"=count, and reconstructs a 4// newline-joined buffer byte-identical to the flat body. Prefix is a PARAMETER (gates isolate). 5// This is nx_frontier_store's frs_* promoted to the shared home; frontier + commontask delegate here. 6// license_tier: ORIGINAL No hw writes (Rule 26). 7import "nx_seg_store.nx" 8import "nx_syscalls.nx" 9import "nx_record_tsv.nx" // rtv_to_tsv + nxr_count -- the compatibility view (seq1326 rung 3) 10 11const STS_WCAP: i64 = 8388608 // RETIRED as the writer cap (see sts_seed); kept only for callers that still name it 12const STS_ROWOVH: i64 = 32 // per-entry seg-store header: kind + u32 klen + u32 vlen, with margin 13const STS_WSLACK: i64 = 65536 // count row + commit slack 14// ---- PLANE LOCK (seq1544 follow-on) ------------------------------------------------------------- 15// MEASURED 2026-07-30: 55 files call sts_seed; only THREE take any lock. sts_seed is LOAD-ALL / 16// WRITE-ALL -- it REPLACES the whole plane from the caller's buffer -- so two unlocked writers that 17// overlap LOSE DATA, silently and by construction: 18// A loads (n rows) | B loads (n rows) | A commits (n+1) | B commits (n+1, WITHOUT A's row) 19// A's row is simply gone, and the next reader sees a declared/found mismatch it will blame on a 20// "prior writer dropped rows" -- which is exactly what it is, but the cause is the MISSING LOCK. 21// 22// WHY THIS IS A CALLER-HELD LOCK AND NOT AN sts_seed INTERNAL: the critical section is the whole 23// READ-MODIFY-WRITE (load -> mutate -> seed), not the commit alone. A lock taken inside sts_seed would 24// cover only the write half and leave the exact race above wide open. Worse, flock is per OPEN FILE 25// DESCRIPTION, so locking inside sts_seed would DEADLOCK the three callers that already hold 26// <prefix>plock on their own fd -- same process, different fd, blocks forever. 27// 28// ADDITIVE AND OPT-IN: nothing here changes existing behaviour. A caller wraps its RMW: 29// let lk: i64 = sts_lock(prefix) // <=0 means could not lock: refuse, do not proceed unlocked 30// ... sts_load_honest ... mutate ... sts_seed ... 31// sts_unlock(lk) 32// The lock file is <prefix>plock -- the SAME name nx_debt/nx_ecomat_put/nx_frontier_put already use 33// by hand, so adopting this composes with them instead of introducing a second, incompatible lock. 34// ─── DUPLICATE LOCK API REMOVED 2026-07-31 (ws=debt-intelligence) ───────────────────────────────── 35// This file defined sts_lock/sts_unlock TWICE -- here (the original) and again at the STS_LOCKMODE 36// block below. Harmless-looking while the compiler silently picked one; a HARD BUILD FAILURE for 37// EVERY consumer the moment the duplicate-definition guard went fail-closed on 2026-07-31, which is 38// how it was found (nx_debtcluster could not compile, and neither could any other store-seed 39// consumer -- nx_debtlive/nx_debt included). 40// THE SURVIVOR IS THE LOWER ONE, ON MERIT, NOT ON POSITION: it names the failing PATH on an open 41// error instead of letting "cannot lock" blame flock, and its sts_unlock issues an explicit 42// LOCK_UN before close rather than relying on close to drop the flock implicitly. 43// ★LAW: A DUPLICATE DEFINITION IS NOT A STYLE PROBLEM -- IT IS AN UNDECLARED COIN FLIP OVER WHICH 44// IMPLEMENTATION RUNS, AND IT SURVIVES EXACTLY UNTIL SOMETHING STARTS CHECKING. 45 46const STS_HASH: i64 = 35 // '#' 47const STS_NL: i64 = 10 // '\n' 48const STS_Q: i64 = 113 // 'q' 49const STS_COLON: i64 = 58 // ':' 50const STS_ZERO: i64 = 48 // '0' 51const STS_NINE: i64 = 57 // '9' 52const STS_KIND_LIVE: i64 = 1 // seg-store live-record kind (2 = tombstone) 53const STS_KEYCAP: i64 = 64 54const STS_NUMCAP: i64 = 28 55const STS_OUTCAP: i64 = 16 56const STS_BASE10: i64 = 10 57const STS_KPFX: i64 = 2 // "q:" prefix length 58const STS_EXIT_MMAP: i64 = 9 59const STS_MMAP_ERRMAX: i64 = 4096 60const STS_PROBE_WINDOW: i64 = 4096 // detect-only probe span past the declared q:n 61const STS_PROBE_MISS_RUN: i64 = 64 // consecutive misses that end the probe (sparse-tail guard) 62const STS_ERRFD: i64 = 2 63 64// CHECKED allocator (seq350, INCIDENT-0720): under host memory exhaustion sys_mmap returns an 65// error code in the top page; every unchecked caller then writes to it and SEGFAULTS (observed at 66// 0xfffffffffffffff4 = -ENOMEM, with multi-GB cores). This is the SHARED home of the seg-store 67// reader family (nx_debt/nx_debt_hygiene/nx_ws_cycle/nx_sweep_daemon all load through here), so the 68// check lives here ONCE and every member inherits it on its next rebuild. Fail LOUD with a 69// structured constant message (allocation-free by construction), never a segfault. 70func sts_werr(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(STS_ERRFD, s, n); return 0 } 71func sts_mm(n: i64) -> *u8 { 72 let p: *u8 = sys_mmap(n) 73 let v: i64 = p as i64 74 if v < 0 { if v > (0 - STS_MMAP_ERRMAX) { 75 sts_werr("STORE-RED {\"error\":\"mmap-fail\",\"lib\":\"nx_store_seed_lib\",\"hint\":\"host memory exhausted; see debt seq350 / INCIDENT-0720\"}\n" as *u8) 76 sys_exit(STS_EXIT_MMAP) 77 } } 78 return p 79} 80 81const STS_NXR_HDR: i64 = 6 82const STS_NXR_N: i64 = 78 83const STS_NXR_X: i64 = 88 84const STS_NXR_R: i64 = 82 85const STS_NXR_1: i64 = 49 86 87// FORMAT AUTO-DETECT (seq1326 rung 3) -- MUST STAY IDENTICAL TO _hdl_build/nx_store_seed_lib.nx. 88// ⚠⚠⚠DO NOT DROP THIS BLOCK WHEN REWRITING THIS FILE. It was clobbered twice on 2026-07-30 (14:12:33 and 89// 14:22:33) by full-file writes from a concurrent session, each time returning the file to its pre-edit 90// 10199B. knowledge/store/riskreg- is MIGRATED TO NXR1 on disk, so WITHOUT this block nx_info_registry 91// reads that live plane as rows=[] total=54 malformed=54 and nx_store_put load returns shredded garbage. 92// It is purely ADDITIVE: the render path is unreachable for any value not starting with the NXR1 magic, 93// so it is provably inert on every legacy plane and costs other lanes nothing. 94// ⚠Run nx_libcheck before building anything that imports this file -- the runtime/ and _hdl_build/ copies 95// are EACH OTHER'S CONTROL, and divergence means half the tree will silently build old behaviour. 96func sts_emit_row(src: *u8, vl: i64, out: *u8, off: i64, cap: i64) -> i64 { 97 var o: i64 = off 98 if vl >= STS_NXR_HDR { 99 if src[0] == (STS_NXR_N as u8) { if src[1] == (STS_NXR_X as u8) { 100 if src[2] == (STS_NXR_R as u8) { if src[3] == (STS_NXR_1 as u8) { 101 let cols: i64 = nxr_count(src) 102 let n2: i64 = rtv_to_tsv(src, vl, cols, (out as i64 + o) as *u8, cap - o) 103 if n2 >= 0 { return o + n2 } 104 return o 105 } } } } 106 } 107 var t2: i64 = 0 108 while t2 < vl { if o < cap { out[o] = src[t2]; o = o + 1 } t2 = t2 + 1 } 109 return o 110} 111 112func sts_atoi(b: *u8, n: i64) -> i64 { 113 var v: i64 = 0 114 var i: i64 = 0 115 var go: i64 = 1 116 while i < n { 117 if go == 1 { 118 let c: i64 = b[i] as i64 119 if c >= STS_ZERO { if c <= STS_NINE { v = v * STS_BASE10 + (c - STS_ZERO) } else { go = 0 } } else { go = 0 } 120 } 121 i = i + 1 122 } 123 return v 124} 125func sts_rowkey(seq: i64, out: *u8) -> i64 { 126 out[0] = STS_Q as u8 127 out[1] = STS_COLON as u8 128 var o: i64 = ss_catn(out, STS_KPFX, seq) 129 out[o] = 0 as u8 130 return o 131} 132// migrate a flat buffer into the store at `prefix`; ONE commit. returns row count or -1. 133func sts_seed(prefix: *u8, buf: *u8, n: i64) -> i64 { 134 // WRITER CAPACITY IS DATA-DRIVEN, NOT A CEILING (2026-07-30). This was ss_begin_cap(STS_WCAP) 135 // with STS_WCAP a fixed 1MiB. sts_seed rewrites the WHOLE plane on every write, so the moment a 136 // plane outgrew 1MiB the trailing ss_add calls failed -- and their return value was DISCARDED -- 137 // while the q:n count row still recorded the INTENDED total. The plane then declared more rows 138 // than it contained: silent, unbounded data loss that only surfaced later as DEBT-REFUSED 139 // lossy-load, with the debt board (the ecosystem ledger) frozen read-only. 140 // Size from the actual payload the way reg_put and ss_compact_cap already do: every byte of 141 // input, plus per-row key+header overhead, plus the count row and slack. A plane can now grow 142 // to whatever it needs; there is no number here for a future plane to outgrow. 143 var rowcount: i64 = 0 144 var scan: i64 = 0 145 while scan < n { if buf[scan] == (STS_NL as u8) { rowcount = rowcount + 1 } scan = scan + 1 } 146 // ⚠A SHRINK-REFUSAL GUARD WAS ATTEMPTED HERE 2026-08-02 AND DELIBERATELY REVERTED BEFORE PROMOTE. 147 // It would have refused any sts_seed whose input holds fewer rows than the plane already has -- 148 // correct for the clobber class (dedupq- seq756, commontask- 08-02) but WRONG HERE, because a 149 // blast-radius read of the 118 call sites found LEGITIMATE shrinking writers that the guard would 150 // have broken, some catastrophically: 151 // * nx_plane_append ROLLBACK -- sts_seed(prefix, oldb, oldn) restores the PRE-append buffer after a 152 // failed write; refusing that strands the plane in the failed state (the guard would cause the 153 // very loss it exists to prevent). 154 // * nx_plane_repair -- repairing a corrupted plane legitimately produces FEWER rows. 155 // * nx_plane_append's own T1 NEGATIVE CONTROL deliberately raw-seeds to PROVE a raw seed destroys a 156 // plane (2 rows -> 1); refusing it turns a healthy liar-killed gate RED. 157 // * nx_store_put close/setcol paths rewrite the whole plane by design. 158 // THE GUARD BELONGS AT THE PLANE-WRITER LAYER, NOT IN THE SHARED PRIMITIVE: see debt 1785710884 -- 159 // either a checked twin (sts_seed_checked) that intentional shrinkers opt out of, or the guard inside 160 // nx_store_put's put path only. The primitive stays raw and DOCUMENTED-DANGEROUS; nx_plane_append is 161 // already the one correct append path. 162 let needed: i64 = n + (rowcount + 2) * (STS_KEYCAP + STS_ROWOVH) + STS_WSLACK 163 let w: *i64 = ss_begin_cap(needed) 164 let key: *u8 = sts_mm(STS_KEYCAP) 165 let base: i64 = buf as i64 166 var seq: i64 = 0 167 var i: i64 = 0 168 while i < n { 169 var le: i64 = i 170 var s: i64 = 1 171 while s == 1 { if le >= n { s = 0 } else { if buf[le] == (STS_NL as u8) { s = 0 } else { le = le + 1 } } } 172 if le > i { if buf[i] != (STS_HASH as u8) { 173 sts_rowkey(seq, key) 174 // CHECKED: a dropped row used to be invisible here, which is what let q:n over-declare. 175 if ss_add(w, STS_KIND_LIVE, key, (base + i) as *u8, le - i) < 0 { return 0 - 2 } 176 seq = seq + 1 177 } } 178 i = le + 1 179 } 180 let cb: *u8 = sts_mm(STS_NUMCAP) 181 let cl: i64 = ss_catn(cb, 0, seq) 182 if ss_add(w, STS_KIND_LIVE, "q:n" as *u8, cb, cl) < 0 { return 0 - 2 } 183 let rc: i64 = ss_commit(prefix, w, ss_next_segid(prefix)) 184 if rc != 0 { return 0 - 1 } 185 return seq 186} 187// --------------------------------------------------------------------------------------------------- 188// sts_append_fast -- O(1) APPEND OF ONE ROW. The seq724 amplification fix, living beside sts_seed so every 189// writer that already imports this lib can reach it without a new dependency. 190// 191// sts_seed REWRITES THE WHOLE PLANE. Measured 2026-07-30 on a 2000-row / 210,906-byte plane: appending one 192// row via re-seed wrote a 210,969-byte segment; this path wrote 47 BYTES. 4489x less. On dp-web-pub- 193// (1.5GB, 4,291,836 entries) that is 1.5GB rewritten versus about 50 bytes. 194// 195// IT IS NOT ONLY A SPEED FIX -- IT REMOVES TWO WHOLE DEFECT CLASSES BY CONSTRUCTION: 196// * PARTIAL-LOAD DROPS ROWS: a re-seed rebuilds the plane from whatever the loader could reach, so a 197// partial read silently deletes every row past the declared count (the lossy-plane class). An append 198// rewrites NOTHING, so an incomplete read cannot destroy a row it never saw. 199// * ORDINAL SHIFT: a re-seed RE-SEGMENTS rows (any row containing a newline splits), shifting every 200// ordinal after it -- which is how a positional seq issued before a re-seed later addresses a 201// DIFFERENT row. An append adds q:<n> and touches no existing key, so ordinals are stable. 202// 203// Correctness rests on properties the store already guarantees: segments are append-only and reads are 204// NEWEST-WINS per key; q:<n> is a key never written before; q:n is simply overwritten; and every reader 205// already walks q:0..q:n-1 ACROSS segments. NO READER CHANGES ARE REQUIRED. 206// Segment count grows by one per append BY DESIGN -- nx_store_fold_beat merges on the standing sweep and 207// nx_store_janitor_beat reclaims what folding supersedes. Cheap appends + background compaction is the LSM 208// shape this store was built for and that re-seeding defeats. 209// FAIL-CLOSED: a plane with no q:n is REFUSED (-1), never conjured. Returns the NEW row count, or -1/-2. 210func sts_append_fast(prefix: *u8, row: *u8, rowlen: i64) -> i64 { 211 if rowlen <= 0 { return 0 - 1 } 212 let pq: *i64 = sts_mm(STS_OUTCAP) as *i64 213 let lq: *i64 = sts_mm(STS_OUTCAP) as *i64 214 if ss_get(prefix, "q:n" as *u8, pq, lq) != 1 { return 0 - 1 } 215 let n: i64 = sts_atoi(pq[0] as *u8, lq[0]) 216 let key: *u8 = sts_mm(STS_KEYCAP) 217 sts_rowkey(n, key) 218 let cb: *u8 = sts_mm(STS_NUMCAP) 219 let cl: i64 = ss_catn(cb, 0, n + 1) 220 let w: *i64 = ss_begin_cap(rowlen + STS_WSLACK) 221 if ss_add(w, STS_KIND_LIVE, key, row, rowlen) < 0 { return 0 - 2 } 222 if ss_add(w, STS_KIND_LIVE, "q:n" as *u8, cb, cl) < 0 { return 0 - 2 } 223 if ss_commit(prefix, w, ss_next_segid(prefix)) != 0 { return 0 - 2 } 224 return n + 1 225} 226 227// sts_append_fast IS STILL A READ-MODIFY-WRITE AND IT STILL LOSES ROWS UNDER CONCURRENCY. 228// Measured/reasoned 2026-07-30 (ws=sev-eater): it ss_get's q:n to learn n, then writes q:<n> and 229// q:n=n+1. Two concurrent appenders both read n, both write THE SAME q:<n> key -- and reads are 230// NEWEST-WINS per key, so one row does not merely fail to appear, it is silently OVERWRITTEN -- and 231// both then set q:n=n+1, so the count agrees with the loss and no guard ever notices. It is 4489x 232// cheaper than a re-seed and exactly as lossy; SPEED IS NOT SAFETY. 233// 234// THE LOCK IS DELIBERATELY *NOT* INSIDE sts_append_fast. Audited its callers first, and nx_debt 235// (nx_debt.nx:320) ALREADY HOLDS <prefix>plock via db_lock and then calls it on THAT SAME prefix -- 236// flock is per open-file-description, so an internal lock would make nx_debt block against ITSELF 237// and hang the debt board for every seat. Same trap that rules out locking sts_seed. 238// ★LAW, twice proven in one file now: BEFORE LOCKING A SHARED PRIMITIVE, AUDIT WHO ALREADY HOLDS 239// THAT LOCK -- the callers doing it right are what the internal lock breaks first. 240// 241// So the safe variant is a WRAPPER. Callers that do NOT already hold the plane lock use this one; 242// callers that do (nx_debt, nx_frontier_put) keep calling the raw primitive and stay correct. 243// (the wrapper itself is defined at the END of this file, below sts_lock -- NishiLang resolves 244// identifiers in TEXTUAL order and a forward reference to sts_lock from here would not compile.) 245 246// read the store at `prefix` -> newline-joined row buffer. returns bytes. 247// seq356 (INCIDENT-0720 residual): the original body did one ss_get PER ROW, and every ss_get 248// re-opens + re-maps the whole seglist => O(rows x segments) file IO -- the amplification that 249// helped the reader family (nx_debt/nx_ws_cycle/...) OOM the host. NOW: ss_open ONCE -> 250// ss_hget per row (zero file IO per call; the proven WMS audit-scale pattern). Semantics 251// identical BY ORACLE: sts_load_slow below is the verbatim old body and the bulkload gate 252// proves byte-identical output on single-seg, multi-seg-supersede, and shrink planes. 253func sts_load(prefix: *u8, out: *u8, cap: i64) -> i64 { 254 // ss_open_cached, NOT ss_open (seq905/962 CLASS FIX, 2026-07-30). ss_open reads every live segment 255 // fully into anonymous RAM and nothing frees it, so a consumer that loads repeatedly leaks the WHOLE 256 // store per load -- the fingerprint behind ~23.6 GiB of anonymous address space across nine organs and 257 // the 92.3%-swap thrash. MEASURED 2026-07-30: 133 files reach ss_open but only 1 called ss_close, so 258 // the fix existed and was dark. This lib is a CHOKEPOINT -- 63 importers, and it is the exact reader 259 // family (nx_debt / nx_ws_cycle / ...) named above as having OOM'd the host -- so migrating the two 260 // opens here reaches far more consumers than migrating leaf call sites one at a time (rule 15). 261 // Cache is prefix-keyed with manifest (st_size, st_mtime) invalidation, so LIVE EDITS ARE STILL PICKED 262 // UP on the next call with no restart; you simply cannot leak what you never allocate twice. 263 // ⚠Safe here because these loaders are single-threaded and run to completion before the next call, so 264 // the handle's shared query-scratch arena is never re-entered. Do NOT blanket-alias ss_open itself. 265 let h: *i64 = ss_open_cached(prefix) 266 if (h as i64) == 0 { return 0 } 267 let pq: *i64 = sts_mm(STS_OUTCAP) as *i64 268 let lq: *i64 = sts_mm(STS_OUTCAP) as *i64 269 if ss_hget(h, "q:n" as *u8, pq, lq) != 1 { return 0 } 270 let cnt: i64 = sts_atoi(pq[0] as *u8, lq[0]) 271 let key: *u8 = sts_mm(STS_KEYCAP) 272 let pr: *i64 = sts_mm(STS_OUTCAP) as *i64 273 let lr: *i64 = sts_mm(STS_OUTCAP) as *i64 274 var o: i64 = 0 275 var seq: i64 = 0 276 while seq < cnt { 277 sts_rowkey(seq, key) 278 if ss_hget(h, key, pr, lr) == 1 { 279 let src: *u8 = pr[0] as *u8 280 let vl: i64 = lr[0] 281 o = sts_emit_row(src, vl, out, o, cap) 282 if o < cap { out[o] = STS_NL as u8; o = o + 1 } 283 } 284 seq = seq + 1 285 } 286 return o 287} 288// HONEST LOADER (2026-07-25, ws=debt-instrument). sts_load trusts the q:n count key and stops 289// at it with NO error and NO flag, so a low/stale q:n SILENTLY truncates every reader that uses 290// it. MEASURED on the debt- plane: q:n resolved 125 while 629 segments held 734+ rows, and 291// nx_debt list reported total=125 AS FACT. Loads byte-identically to sts_load (which is left 292// UNTOUCHED for API-contract stability, rule 19) but additionally REPORTS what it could not 293// reach, so partial coverage can never again be presented as complete (honesty law L011). 294// flags[0]=declared q:n flags[1]=rows actually loaded flags[2]=rows reachable BEYOND q:n 295// DETECT-ONLY BY CONSTRUCTION: never loads a row past the declared count. sts_seed REASSIGNS 296// q:<i> on every commit, so rows past q:n belong to an OLDER seeding generation -- loading them 297// would duplicate live rows and revive eaten ones. Reporting the gap is safe; reconciling it is 298// a separate operation that must diff the generations. 299// SAFE ASYMMETRY it relies on: a too-LOW q:n silently truncates, a too-HIGH q:n is harmless on 300// read because ss_hget misses are skipped -- so probing past the count cannot corrupt. 301func sts_load_honest(prefix: *u8, out: *u8, cap: i64, flags: *i64) -> i64 { 302 flags[0] = 0 303 flags[1] = 0 304 flags[2] = 0 305 // ss_open_cached -- same seq905/962 class fix and same single-threaded-loader reasoning as sts_load. 306 let h: *i64 = ss_open_cached(prefix) 307 if (h as i64) == 0 { return 0 } 308 let pq: *i64 = sts_mm(STS_OUTCAP) as *i64 309 let lq: *i64 = sts_mm(STS_OUTCAP) as *i64 310 if ss_hget(h, "q:n" as *u8, pq, lq) != 1 { return 0 } 311 let cnt: i64 = sts_atoi(pq[0] as *u8, lq[0]) 312 flags[0] = cnt 313 let key: *u8 = sts_mm(STS_KEYCAP) 314 let pr: *i64 = sts_mm(STS_OUTCAP) as *i64 315 let lr: *i64 = sts_mm(STS_OUTCAP) as *i64 316 var o: i64 = 0 317 var seq: i64 = 0 318 while seq < cnt { 319 sts_rowkey(seq, key) 320 if ss_hget(h, key, pr, lr) == 1 { 321 let src: *u8 = pr[0] as *u8 322 let vl: i64 = lr[0] 323 o = sts_emit_row(src, vl, out, o, cap) 324 if o < cap { out[o] = STS_NL as u8; o = o + 1 } 325 flags[1] = flags[1] + 1 326 } 327 seq = seq + 1 328 } 329 var beyond: i64 = 0 330 var miss: i64 = 0 331 var p: i64 = cnt 332 let lim: i64 = cnt + STS_PROBE_WINDOW 333 while p < lim { 334 if miss < STS_PROBE_MISS_RUN { 335 sts_rowkey(p, key) 336 if ss_hget(h, key, pr, lr) == 1 { beyond = beyond + 1; miss = 0 } else { miss = miss + 1 } 337 } 338 p = p + 1 339 } 340 flags[2] = beyond 341 return o 342} 343// ---- ATOMIC PLANE APPEND (root fix for seq1559, ws=sev-eater 2026-07-30) ---------------------- 344// 55 files call sts_seed(); only THREE take a lock. sts_seed REPLACES the whole plane from the 345// caller's buffer, and the universal call shape is READ-MODIFY-WRITE: 346// n = sts_load(prefix, buf, cap); ...append a row...; sts_seed(prefix, buf, n) 347// Two overlapping unlocked writers interleave as A-loads / B-loads / A-commits n+1 / 348// B-commits n+1-WITHOUT-A. A's row is gone, silently, and the next reader's declared-vs-found 349// mismatch blames a prior writer for dropping rows -- true, but the wrong cause. 350// 351// WHY THE OBVIOUS FIX IS WRONG AND IS NOT DONE HERE: putting the lock INSIDE sts_seed would 352// DEADLOCK the three callers that already do it right. flock(2) locks are per OPEN FILE 353// DESCRIPTION, not per process, so nx_debt -- which already holds <prefix>plock via db_lock and 354// then calls sts_seed -- would block against ITSELF on a second fd of the same file. That trades 355// silent data loss for a hard hang of the debt board for every seat, which is strictly worse. 356// ★LAW: A CHOKEPOINT FIX MUST NOT ASSUME ITS CALLERS ARE ALL WRONG -- the ones already doing it 357// right are exactly what a naive fix breaks first. 358// 359// So sts_seed is left UNLOCKED and this is added ALONGSIDE it: the whole critical section as ONE 360// call. ★LAW: THE RIGHT PRIMITIVE IS THE CRITICAL SECTION, NOT THE LOCK -- a lock 52 callers must 361// remember to take is the same class of defect as the missing lock. Callers migrate to 362// sts_append_row and become correct by construction; nothing has to be remembered. 363// 364// The lock file is <prefix>plock -- deliberately THE SAME PATH nx_debt/nx_ecomat_put/ 365// nx_frontier_put already use, so migrated and unmigrated writers share ONE lock domain. A new 366// lock file would have been easier and would have protected nothing. 367const STS_LOCK_EX: i64 = 2 368const STS_LOCK_UN: i64 = 8 369const STS_LOCKMODE: i64 = 420 370const STS_LOCKPATHCAP: i64 = 256 371const STS_ERR_LOCK: i64 = 0 - 3 372const STS_ERR_CAP: i64 = 0 - 4 373 374// Take the plane's exclusive lock. Returns the held fd, or -1 (LOUD: names the path, because 375// "cannot lock" alone has historically blamed flock when the cause was an open failure). 376func sts_lock(prefix: *u8) -> i64 { 377 let p: *u8 = sts_mm(STS_LOCKPATHCAP) 378 var o: i64 = ss_cat(p, 0, prefix) 379 o = ss_cat(p, o, "plock" as *u8) 380 p[o] = 0 as u8 381 let fd: i64 = sys_openat_append(p, STS_LOCKMODE) 382 if fd < 0 { 383 sts_werr("STS-LOCK open-failed (not flock) path=" as *u8) 384 sts_werr(p) 385 sts_werr("\n" as *u8) 386 return 0 - 1 387 } 388 sys_flock(fd, STS_LOCK_EX) 389 return fd 390} 391 392func sts_unlock(fd: i64) -> i64 { 393 if fd < 0 { return 0 } 394 sys_flock(fd, STS_LOCK_UN) 395 sys_close(fd) 396 return 0 397} 398 399// ATOMIC ROW APPEND: lock -> load -> append -> commit -> unlock, with no window in which another 400// writer can load a snapshot this one is about to invalidate. Returns sts_seed's row count, or 401// STS_ERR_LOCK / STS_ERR_CAP. Fail-closed: if the buffer cannot hold the appended row the plane is 402// left EXACTLY as it was rather than committed short (a truncated commit is the very loss this 403// exists to prevent). 404func sts_append_row(prefix: *u8, row: *u8, rowlen: i64, cap: i64) -> i64 { 405 let fd: i64 = sts_lock(prefix) 406 if fd < 0 { return STS_ERR_LOCK } 407 let buf: *u8 = sts_mm(cap) 408 var n: i64 = sts_load(prefix, buf, cap) 409 if n < 0 { n = 0 } 410 if n + rowlen + 1 > cap { sts_unlock(fd); return STS_ERR_CAP } 411 var i: i64 = 0 412 while i < rowlen { buf[n] = row[i]; n = n + 1; i = i + 1 } 413 buf[n] = STS_NL as u8 414 n = n + 1 415 let rc: i64 = sts_seed(prefix, buf, n) 416 sts_unlock(fd) 417 return rc 418} 419 420// Locked form of sts_append_fast, defined HERE because it needs sts_lock above it (textual order). 421// Use this from any caller that does NOT already hold the plane lock; callers that DO (nx_debt, 422// nx_frontier_put) must keep calling the raw sts_append_fast or they will deadlock against 423// themselves -- see the audit note beside sts_append_fast. 424func sts_append_fast_locked(prefix: *u8, row: *u8, rowlen: i64) -> i64 { 425 let fd: i64 = sts_lock(prefix) 426 if fd < 0 { return STS_ERR_LOCK } 427 let rc: i64 = sts_append_fast(prefix, row, rowlen) 428 sts_unlock(fd) 429 return rc 430} 431 432// the pre-seq356 body VERBATIM (one ss_get per row) -- kept as the equivalence ORACLE for 433// nx_sts_bulkload_gate; not for production use (the amplification lives here). 434func sts_load_slow(prefix: *u8, out: *u8, cap: i64) -> i64 { 435 let pq: *i64 = sts_mm(STS_OUTCAP) as *i64 436 let lq: *i64 = sts_mm(STS_OUTCAP) as *i64 437 if ss_get(prefix, "q:n" as *u8, pq, lq) != 1 { return 0 } 438 let cnt: i64 = sts_atoi(pq[0] as *u8, lq[0]) 439 let key: *u8 = sts_mm(STS_KEYCAP) 440 let pr: *i64 = sts_mm(STS_OUTCAP) as *i64 441 let lr: *i64 = sts_mm(STS_OUTCAP) as *i64 442 var o: i64 = 0 443 var seq: i64 = 0 444 while seq < cnt { 445 sts_rowkey(seq, key) 446 if ss_get(prefix, key, pr, lr) == 1 { 447 let src: *u8 = pr[0] as *u8 448 let vl: i64 = lr[0] 449 o = sts_emit_row(src, vl, out, o, cap) 450 if o < cap { out[o] = STS_NL as u8; o = o + 1 } 451 } 452 seq = seq + 1 453 } 454 return o 455}