nx_capreg_librarian.nx source

1// nx_capreg_librarian.nx -- the LIBRARIAN's registry survival + collision repair (operator-flagged debt: 2// "CAPREG index collision needs the Librarian dedup / atomic allocation fix" -- and the command-prompt 3// crash that wiped /tmp PROVED the durability gap: the live registry died, only the NTFS journal at 4// knowledge/status/cap_registry_durable.log survived). Three capabilities, one module: 5// 1. REGEN -- cl_regen(journal, live): rebuild the clean live registry FROM the append-only journal. 6// Byte-identical re-snapshot copies dedup away; TRUE collisions (same idx, different capability -- 7// the two-parallel-sessions bug at idx 212-227) are RENUMBERED past the global max with an explicit 8// `remapped_from=` marker (additive: history preserved, nothing deleted, the journal untouched). 9// 2. ALLOC -- cl_next_idx(live, journal): max idx across BOTH files + 1. Register modules call this 10// instead of hardcoding an index = the atomic-allocation fix (collisions can't recur in one lane; 11// cross-lane races are repaired by the next regen, which is the backstop by construction). 12// 3. DUAL-WRITE -- cl_register_dual(...): every registration appends to the live registry AND the 13// NTFS journal in one call. A /tmp wipe now loses NOTHING (regen restores live from journal). 14// RACI: the LIBRARIAN owns this (GATHER/REGISTER verb -- the registry is the gathered source the 15// self-model regenerates from). LAWS: struct-free, integer-only, flat ifs. license_tier: ORIGINAL 16import "nx_syscalls.nx" 17import "nx_itoa_lib.nx" // shared MSB-first emitter (zero-alloc) 18const K_MAGIC_1000000007: i64 = 1000000007 19const K_MAGIC_32768: i64 = 32768 20 21// atomic rename via DIRECT x86 renameat=264 (4 args, no flags). sys_renameat issues rv64 renameat2=276 22// whose translate row postdates the frozen known-good cc -> 276 hits x86 tee = EINVAL (probe-proven 23// rc=-22 by _cl_probe2; same NXASM-SYSCALL-XLATE-GAP class as the filed mkdirat/unlinkat cases). 24func cl_rename(oldpath: *u8, newpath: *u8) -> i64 { 25 return __syscall(264, 0 - 100, oldpath, 0 - 100, newpath, 0, 0) 26} 27 28func cl_w(fd: i64, s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(fd,s,n); return 0 } 29// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer 30// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the 31// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls). 32// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign. 33func cl_wn(fd: i64, v: i64) -> i64 { nxi_fd(fd, v); return 0 } 34// append nul-terminated s into dst at j; returns the new end (dst stays nul-terminated) 35func cl_cats(dst: *u8, j: i64, s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){ dst[j+n]=s[n]; n=n+1 } dst[j+n]=0 as u8; return j+n } 36 37// does the line starting at i match "CAPREG idx="? (the journal's entry marker) 38func cl_is_entry(b: *u8, i: i64, n: i64) -> i64 { 39 let m: *u8 = "CAPREG idx=" as *u8 40 var k: i64 = 0 41 while m[k] != (0 as u8) { 42 if i + k >= n { return 0 } 43 if b[i+k] != m[k] { return 0 } 44 k = k + 1 45 } 46 return 1 47} 48// index of the line's '\n' (or n if the file ends without one) 49func cl_line_end(b: *u8, i: i64, n: i64) -> i64 { 50 var e: i64 = i 51 while e < n { if b[e] == (10 as u8) { return e } e = e + 1 } 52 return n 53} 54// parse the decimal run at i; -1 if no digit there 55func cl_parse_int(b: *u8, i: i64, n: i64) -> i64 { 56 var v: i64 = 0 - 1 57 var j: i64 = i 58 while j < n { 59 let c: i64 = b[j] as i64 60 if c < 48 { return v } 61 if c > 57 { return v } 62 if v < 0 { v = 0 } 63 v = v * 10 + (c - 48) 64 j = j + 1 65 } 66 return v 67} 68// first index past the decimal run at i 69func cl_digits_end(b: *u8, i: i64, n: i64) -> i64 { 70 var j: i64 = i 71 while j < n { 72 let c: i64 = b[j] as i64 73 if c < 48 { return j } 74 if c > 57 { return j } 75 j = j + 1 76 } 77 return j 78} 79// content hash of bytes j..e (the entry text AFTER the idx digits): same capability re-snapshotted 80// hashes equal -> dedup; different capability at the same idx hashes different -> collision 81func cl_hash_range(b: *u8, j: i64, e: i64) -> i64 { 82 var h: i64 = 7 83 var k: i64 = j 84 while k < e { h = (h * 131 + (b[k] as i64)) % K_MAGIC_1000000007; k = k + 1 } 85 return h 86} 87 88// max CAPREG idx in a registry file (0 if missing/empty) -- the allocator's scan 89func cl_max_idx(path: *u8) -> i64 { 90 let lenp: *i64 = sys_mmap(16) as *i64 91 let b: *u8 = sys_read_file(path, lenp) 92 let n: i64 = lenp[0] 93 if n <= 0 { return 0 } 94 var mx: i64 = 0 95 var i: i64 = 0 96 while i < n { 97 if cl_is_entry(b, i, n) == 1 { 98 let v: i64 = cl_parse_int(b, i + 11, n) 99 if v > mx { mx = v } 100 } 101 i = cl_line_end(b, i, n) + 1 102 } 103 return mx 104} 105 106// THE ATOMIC ALLOCATOR: next free idx = max(live, journal) + 1. Register modules call this instead of 107// hardcoding -- the root-cause fix for the cross-session collision. 108func cl_next_idx(livepath: *u8, journalpath: *u8) -> i64 { 109 var mx: i64 = cl_max_idx(livepath) 110 let mj: i64 = cl_max_idx(journalpath) 111 if mj > mx { mx = mj } 112 return mx + 1 113} 114 115// DUAL-WRITE registration: one validated CAPREG line appended to BOTH the live registry and the NTFS 116// journal. The crash-survival fix: live is now a cache, the journal is the truth. 117func cl_register_dual(livepath: *u8, journalpath: *u8, idx: i64, layer: i64, status: i64, name: *u8) -> i64 { 118 var pass: i64 = 0 119 var f: i64 = 0 120 while f < 2 { 121 var fd: i64 = 0 - 1 122 if f == 0 { fd = sys_openat_append(livepath, 0x1a4) } 123 if f == 1 { fd = sys_openat_append(journalpath, 0x1a4) } 124 if fd >= 0 { 125 cl_w(fd, "CAPREG idx=" as *u8); cl_wn(fd, idx) 126 cl_w(fd, " layer=" as *u8); cl_wn(fd, layer) 127 cl_w(fd, " status=" as *u8); cl_wn(fd, status) 128 cl_w(fd, " name=" as *u8); cl_w(fd, name); cl_w(fd, "\n" as *u8) 129 sys_close(fd) 130 pass = pass + 1 131 } 132 f = f + 1 133 } 134 if pass == 2 { return 1 } 135 return 0 136} 137 138// one journal entry -> the clean output (helper kept flat: the lane has a filed 4-deep nested-if 139// miscompile, so the per-entry decision lives here, not inlined under the scan loop; and kept to 6 140// args -- the toolchain family has a filed >6-stack-args miscompile, tables ride inside st). 141// st[0]=nt st[1]=maxidx st[2]=tidx-table st[3]=thsh-table. Emits unless the (idx, content-hash) pair 142// was already emitted; a reused idx with NEW content is renumbered past max with remapped_from. 143func cl_emit_entry(out: i64, b: *u8, i: i64, e: i64, n: i64, st: *i64) -> i64 { 144 let tidx: *i64 = st[2] as *i64 145 let thsh: *i64 = st[3] as *i64 146 let idx: i64 = cl_parse_int(b, i + 11, n) 147 let j: i64 = cl_digits_end(b, i + 11, n) 148 let h: i64 = cl_hash_range(b, j, e) 149 var used: i64 = 0 150 var exact: i64 = 0 151 var k: i64 = 0 152 while k < st[0] { 153 if tidx[k] == idx { used = 1; if thsh[k] == h { exact = 1 } } 154 k = k + 1 155 } 156 if exact == 1 { return 0 } 157 var assigned: i64 = idx 158 if used == 1 { st[1] = st[1] + 1; assigned = st[1] } 159 tidx[st[0]] = idx; thsh[st[0]] = h; st[0] = st[0] + 1 160 cl_w(out, "CAPREG idx=" as *u8); cl_wn(out, assigned) 161 sys_write(out, (b as i64 + j) as *u8, e - j) 162 if assigned != idx { cl_w(out, " remapped_from=" as *u8); cl_wn(out, idx) } 163 cl_w(out, "\n" as *u8) 164 return 1 165} 166 167// THE REGEN: journal -> clean live registry. Dedups byte-identical re-snapshots, renumbers true 168// collisions past the global max (with `remapped_from=` so history stays traceable), writes the new 169// live file atomically (.new + rename -> a concurrent reader never sees a torn registry). 170// Returns the number of unique entries written (-1 on read/write failure). 171func cl_regen(journalpath: *u8, livepath: *u8) -> i64 { 172 let lenp: *i64 = sys_mmap(16) as *i64 173 let b: *u8 = sys_read_file(journalpath, lenp) 174 let n: i64 = lenp[0] 175 if n <= 0 { return 0 - 1 } 176 // pass 1: the global max original idx (renumbers start past EVERYTHING) 177 var maxidx: i64 = 0 178 var i: i64 = 0 179 while i < n { 180 if cl_is_entry(b, i, n) == 1 { 181 let v: i64 = cl_parse_int(b, i + 11, n) 182 if v > maxidx { maxidx = v } 183 } 184 i = cl_line_end(b, i, n) + 1 185 } 186 // seen-table: original idx + content hash per unique entry (linear scan -- registry is small) 187 let tidx: *i64 = sys_mmap(K_MAGIC_32768) as *i64 188 let thsh: *i64 = sys_mmap(K_MAGIC_32768) as *i64 189 let st: *i64 = sys_mmap(64) as *i64 190 st[0] = 0; st[1] = maxidx; st[2] = tidx as i64; st[3] = thsh as i64 191 // atomic publish: write .new then rename over live 192 let tmp: *u8 = sys_mmap(256) 193 var tl: i64 = cl_cats(tmp, 0, livepath) 194 tl = cl_cats(tmp, tl, ".new" as *u8) 195 let out: i64 = sys_openat_wr(tmp, 0x1a4) 196 if out < 0 { return 0 - 1 } 197 // pass 2: emit unique entries, renumber collisions 198 i = 0 199 while i < n { 200 let e: i64 = cl_line_end(b, i, n) 201 if cl_is_entry(b, i, n) == 1 { cl_emit_entry(out, b, i, e, n, st) } 202 i = e + 1 203 } 204 sys_close(out) 205 if cl_rename(tmp, livepath) < 0 { return 0 - 1 } 206 return st[0] 207} 208 209// INGEST-LIVE: append live-registry entries the journal lacks INTO the journal. The legacy register 210// modules append to /tmp live ONLY (pre-dual-write) -- without this, a canonicalize would regenerate 211// live FROM the journal and silently DROP their registrations. Preloads the seen-table from the 212// journal, then routes live's lines through cl_emit_entry with the journal as the output: absentees 213// append (idx collisions renumber, exactly like regen), known lines skip. Returns entries ingested. 214func cl_ingest_live(livepath: *u8, journalpath: *u8) -> i64 { 215 let llen: *i64 = sys_mmap(16) as *i64 216 let lb: *u8 = sys_read_file(livepath, llen) 217 let ln: i64 = llen[0] 218 if ln <= 0 { return 0 } 219 let jlen: *i64 = sys_mmap(16) as *i64 220 let jb: *u8 = sys_read_file(journalpath, jlen) 221 let jn: i64 = jlen[0] 222 // preload the seen-table + global max from the journal 223 let tidx: *i64 = sys_mmap(K_MAGIC_32768) as *i64 224 let thsh: *i64 = sys_mmap(K_MAGIC_32768) as *i64 225 let st: *i64 = sys_mmap(64) as *i64 226 st[0] = 0; st[1] = 0; st[2] = tidx as i64; st[3] = thsh as i64 227 var i: i64 = 0 228 while i < jn { 229 let e: i64 = cl_line_end(jb, i, jn) 230 if cl_is_entry(jb, i, jn) == 1 { 231 let idx: i64 = cl_parse_int(jb, i + 11, jn) 232 let j: i64 = cl_digits_end(jb, i + 11, jn) 233 tidx[st[0]] = idx; thsh[st[0]] = cl_hash_range(jb, j, e); st[0] = st[0] + 1 234 if idx > st[1] { st[1] = idx } 235 } 236 i = e + 1 237 } 238 // renumber base must clear BOTH files' originals (cl_regen gets this from its pass-1 over all 239 // input; here live is a second input, so fold its max in too) 240 let lmx: i64 = cl_max_idx(livepath) 241 if lmx > st[1] { st[1] = lmx } 242 let pre: i64 = st[0] 243 let out: i64 = sys_openat_append(journalpath, 0x1a4) 244 if out < 0 { return 0 - 1 } 245 i = 0 246 while i < ln { 247 let e2: i64 = cl_line_end(lb, i, ln) 248 if cl_is_entry(lb, i, ln) == 1 { cl_emit_entry(out, lb, i, e2, ln, st) } 249 i = e2 + 1 250 } 251 sys_close(out) 252 return st[0] - pre 253} 254 255// CANONICALIZE: the one-time legacy repair made durable. Regen renumbers collisions relative to the 256// journal's CURRENT max -- so as long as collided lines sit in the journal, their assigned idx would 257// DRIFT on every regen (proven by the KAT: GAMMA 12 -> 14 after one more registration). The fix: after 258// a regen, the CLEAN set replaces the journal (old journal archived additively at <journal>.raw -- 259// history sacred, nothing deleted). Post-canonicalize the journal has one idx per entry by 260// construction -> every future regen is a stable pure-dedup, zero renumbers, zero drift. 261func cl_canonicalize(journalpath: *u8, livepath: *u8) -> i64 { 262 let nu: i64 = cl_regen(journalpath, livepath) 263 if nu < 0 { return 0 - 1 } 264 // archive the raw journal (first canonicalize only: keep the oldest raw, don't clobber it) 265 let raw: *u8 = sys_mmap(256) 266 var rl: i64 = cl_cats(raw, 0, journalpath) 267 rl = cl_cats(raw, rl, ".raw" as *u8) 268 let probe: i64 = sys_openat_rd(raw) 269 if probe >= 0 { sys_close(probe) } 270 if probe < 0 { cl_rename(journalpath, raw) } 271 // the clean live set becomes the new journal (atomic: .new + rename) 272 let lenp: *i64 = sys_mmap(16) as *i64 273 let lb: *u8 = sys_read_file(livepath, lenp) 274 if lenp[0] <= 0 { return 0 - 1 } 275 let tmp: *u8 = sys_mmap(256) 276 var tl: i64 = cl_cats(tmp, 0, journalpath) 277 tl = cl_cats(tmp, tl, ".new" as *u8) 278 let out: i64 = sys_openat_wr(tmp, 0x1a4) 279 if out < 0 { return 0 - 1 } 280 sys_write(out, lb, lenp[0]) 281 sys_close(out) 282 if cl_rename(tmp, journalpath) < 0 { return 0 - 1 } 283 return nu 284}