nx_galx_cidput.nx source

1// nx_galx_cidput.nx -- NATIVE incremental insert into the sovereign cid->path store. No TSV. 2// 3// Operator 2026-08-07: *"we shouldnt be using tsvs thats probably why it should be nishi native"*. 4// Correct, and the estate had already said so: nx_galx_cid_index's own header reads *"The .idx + 5// .blob ARE the runtime store; the TSV is only a one-time build input (and can then be retired)."* 6// It was never retired -- because the store shipped READ-ONLY. gs_cidindex_build() can only build 7// the whole thing FROM a TSV, and there was no way to add one entry. So every addition fell back 8// to appending 70 bytes of text to a 34MB file and recompiling all 235,445 rows. 9// ★★★★★★ A STORE YOU CAN ONLY BUILD, NEVER APPEND TO, KEEPS ITS OWN LEGACY INPUT FORMAT ALIVE 10// FOREVER -- the "retired" format is load-bearing precisely because the replacement has no writer. 11// ★ THE MISSING PRIMITIVE WAS NOT THE INDEX, IT WAS INSERT. 12// 13// The format already supports O(1) insert; nobody had written it. Open-addressed, fixed buckets: 14// idx: [8 n_buckets][8 n_entries] then n_buckets * 24: [8 cid_hash][8 path_off][8 path_len] (big-endian) 15// blob: concatenated path bytes 16// So an insert is: append the path to blob, linear-probe to an empty bucket, write 24 bytes, bump 17// n_entries. Three small writes at known offsets -- sys_lseek + sys_openat_rdwr, no file rewrite. 18// 19// ⚠LOAD FACTOR IS A HARD LIMIT, NOT A SUGGESTION. n_buckets is fixed at build time; an open- 20// addressed table that passes ~0.5 degrades toward O(n) and at 1.0 never terminates its probe. 21// This REFUSES past the builder's own nent*2 <= nb rule and names the remedy (rebuild, which 22// re-sizes), because ★ A GUARD THAT CANNOT BE SATISFIED PRODUCES A BYPASS. 23// license_tier: ORIGINAL 24 25import "nx_syscalls.nx" 26import "nx_galx_cid_index.nx" 27 28const CP_SEEK_SET: i64 = 0 29const CP_SEEK_END: i64 = 2 30 31// verdicts 32const CP_INSERTED: i64 = 1 33const CP_PRESENT: i64 = 0 34const CP_EIO: i64 = 0 - 1 35const CP_EFULL: i64 = 0 - 2 // load factor exhausted -> rebuild to resize 36const CP_EBADIDX: i64 = 0 - 3 37 38func cp_w64(b: *u8, o: i64, v: i64) -> i64 { 39 b[o] = (v >> 56) as u8 40 b[o+1] = (v >> 48) as u8 41 b[o+2] = (v >> 40) as u8 42 b[o+3] = (v >> 32) as u8 43 b[o+4] = (v >> 24) as u8 44 b[o+5] = (v >> 16) as u8 45 b[o+6] = (v >> 8) as u8 46 b[o+7] = v as u8 47 return 8 48} 49func cp_len(s: *u8) -> i64 { 50 var n: i64 = 0 51 while s[n] != (0 as u8) { n = n + 1 } 52 return n 53} 54 55// Insert by HASH. The table is keyed by hash, never by the cid text, so this is the real primitive 56// -- and it is what lets a rebuild re-adopt entries whose cid string is no longer available (the 57// index stores hashes and paths, not cids). 58func nx_galx_hashput(idx_path: *u8, blob_path: *u8, h_in: i64, path: *u8, plen_in: i64) -> i64 { 59 let fi: i64 = sys_openat_rdwr(idx_path, 0x1a4) 60 if fi < 0 { return CP_EIO } 61 let hdr: *u8 = sys_mmap(64) 62 if sys_read(fi, hdr, 16) != 16 { sys_close(fi); return CP_EBADIDX } 63 let nb: i64 = cidx_rd64(hdr, 0) 64 let nent: i64 = cidx_rd64(hdr, 8) 65 if nb <= 0 { sys_close(fi); return CP_EBADIDX } 66 // the builder sizes nb as the smallest power of two >= nent*2; hold that same invariant here 67 // rather than inventing a looser one, or the two writers would disagree about when it is full. 68 if (nent + 1) * 2 > nb { sys_close(fi); return CP_EFULL } 69 70 var h: i64 = h_in 71 if h == 0 { h = 1 } 72 let ent: *u8 = sys_mmap(64) 73 74 // ---- probe for an empty bucket (or find it already present) ---- 75 var bk: i64 = h & (nb - 1) 76 var bo: i64 = 0 - 1 77 var scanning: i64 = 1 78 var probes: i64 = 0 79 while scanning == 1 { 80 let at: i64 = 16 + bk * 24 81 if sys_lseek(fi, at, CP_SEEK_SET) < 0 { sys_close(fi); return CP_EIO } 82 if sys_read(fi, ent, 24) != 24 { sys_close(fi); return CP_EIO } 83 let cur: i64 = cidx_rd64(ent, 0) 84 if cur == 0 { bo = at; scanning = 0 } 85 else { 86 if cur == h { sys_close(fi); return CP_PRESENT } 87 bk = (bk + 1) & (nb - 1) 88 probes = probes + 1 89 // Belt and braces: the load-factor check above already makes a full wrap impossible, 90 // but a corrupt header (nent understated) would otherwise spin forever. 91 if probes > nb { sys_close(fi); return CP_EFULL } 92 } 93 } 94 95 // ---- append the path to the blob; its END offset is the entry's path_off ---- 96 let fb: i64 = sys_openat_rdwr(blob_path, 0x1a4) 97 if fb < 0 { sys_close(fi); return CP_EIO } 98 let poff: i64 = sys_lseek(fb, 0, CP_SEEK_END) 99 if poff < 0 { sys_close(fb); sys_close(fi); return CP_EIO } 100 var plen: i64 = plen_in 101 if plen < 0 { plen = cp_len(path) } 102 if sys_write(fb, path, plen) != plen { sys_close(fb); sys_close(fi); return CP_EIO } 103 sys_close(fb) 104 105 // ---- commit the bucket, THEN the counter ---- 106 // Order matters: the bucket is what lookups read. Bumping n_entries first would advertise an 107 // entry that is not yet resolvable if we died in between; this way a crash leaves an entry 108 // that works but is uncounted, which is recoverable by a rebuild rather than a dangling read. 109 cp_w64(ent, 0, h) 110 cp_w64(ent, 8, poff) 111 cp_w64(ent, 16, plen) 112 if sys_lseek(fi, bo, CP_SEEK_SET) < 0 { sys_close(fi); return CP_EIO } 113 if sys_write(fi, ent, 24) != 24 { sys_close(fi); return CP_EIO } 114 cp_w64(hdr, 0, nent + 1) 115 if sys_lseek(fi, 8, CP_SEEK_SET) < 0 { sys_close(fi); return CP_EIO } 116 if sys_write(fi, hdr, 8) != 8 { sys_close(fi); return CP_EIO } 117 sys_close(fi) 118 return CP_INSERTED 119} 120 121// Insert cid -> path. Idempotent: a cid whose hash is already in the table is left alone. This 122// matches the builder's own identity rule (it compares hashes, not full cids -- argued safe there 123// because the cid is itself a 64-hex content hash, so a 64-bit FNV collision across 235k entries 124// is ~3e-9). plen -1 = NUL-terminated. 125func nx_galx_cidput(idx_path: *u8, blob_path: *u8, cid: *u8, path: *u8) -> i64 { 126 return nx_galx_hashput(idx_path, blob_path, cidx_fnv1a(cid, CIDX_CIDLEN), path, 0 - 1) 127} 128 129// Rebuild from the TSV, then RE-ADOPT every entry the old index had that the TSV does not mention. 130// 131// ⚠WITHOUT THIS, THE CONDITIONAL REBUILD IS A LANDMINE. Bite-proven: touching the TSV made it 132// newer, the server rebuilt, and every natively-inserted cid went from 200 to 404 -- correct by the 133// rule, and catastrophic by intent, because the destruction is triggered by merely touching an 134// unrelated legacy file and nothing reports it. 135// ★★★★★★ "AUTHORITATIVE SOURCE WINS" IS A DATA-LOSS POLICY WHENEVER THE OTHER WRITER IS ALSO 136// LEGITIMATE. A rebuild must be a MERGE, or the second writer is a bug the first writer erases. 137// The old index is walked by BUCKET, not by cid, because the store keeps hashes and paths only -- 138// which is exactly why insert-by-hash is the primitive and insert-by-cid the convenience wrapper. 139func gs_cidindex_build_preserving(tsv_path: *u8, idx_path: *u8, blob_path: *u8) -> i64 { 140 // Snapshot BEFORE the build overwrites both files. 141 let ip: *i64 = sys_mmap(16) as *i64 142 let oidx: *u8 = sys_read_file(idx_path, ip) 143 let bp: *i64 = sys_mmap(16) as *i64 144 let oblob: *u8 = sys_read_file(blob_path, bp) 145 146 let rc: i64 = gs_cidindex_build(tsv_path, idx_path, blob_path) 147 if rc < 0 { return rc } 148 if (oidx as i64) == 0 { return rc } 149 if (oblob as i64) == 0 { return rc } 150 151 let onb: i64 = cidx_rd64(oidx, 0) 152 if onb <= 0 { return rc } 153 var readopted: i64 = 0 154 var b: i64 = 0 155 while b < onb { 156 let bo: i64 = 16 + b * 24 157 let h: i64 = cidx_rd64(oidx, bo) 158 if h != 0 { 159 let poff: i64 = cidx_rd64(oidx, bo + 8) 160 let plen: i64 = cidx_rd64(oidx, bo + 16) 161 if plen > 0 { 162 if poff + plen <= bp[0] { 163 if nx_galx_hashput(idx_path, blob_path, h, 164 ((oblob as i64) + poff) as *u8, plen) == CP_INSERTED { 165 readopted = readopted + 1 166 } 167 } 168 } 169 } 170 b = b + 1 171 } 172 return rc + readopted 173}