code wiki / _hdl_build / nx_dr_dedup.nx
nx_dr_dedup.nx source
↩ module page · 86 lines · 5996 B
1// nx_dr_dedup.nx -- CAP-DR-DEDUP: content-addressed dedup (store once by hash), the restic/borg space primitive.
2// For each manifest source: sha256 -> hex; the CAS blob path is <store>/<hex>. If that blob already exists the file
3// is a DUPLICATE (same content under another name, or an unchanged re-archive) -> stored ZERO extra bytes; else the
4// blob is written once. An index maps name->hash so restore can reassemble. Composes nx_sha256. Additive (CAS is
5// write-once-by-hash; a hash collision would need SHA-256 to break). Reports unique blobs + bytes saved.
6// nx_dr_dedup <manifest> <store-dir>
7// license_tier: ORIGINAL
8import "nx_syscalls.nx"
9import "nx_itoa_lib.nx" // shared MSB-first emitter (zero-alloc)
10import "nx_sha256.nx"
11import "nx_site_lock_lib.nx"
12import "nx_seg_store.nx"
13const DD_MAGIC_262144: i64 = 262144
14const DD_MAGIC_1024: i64 = 1024
15const DD_MAGIC_1536: i64 = 1536
16
17const DD_CAP: i64 = 67108864
18
19func dd_w(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
20// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
21// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
22// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
23// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
24func dd_wn(v: i64) -> i64 { nxi_out(v); return 0 }
25func dd_read(path: *u8, out: *u8, cap: i64) -> i64 {
26 let fd: i64 = sys_openat_rd(path); if fd < 0 { return 0 - 1 }
27 var total: i64 = 0; var go: i64 = 1
28 while go == 1 { let nr: i64 = sys_read(fd, (out as i64 + total) as *u8, cap - total); if nr <= 0 { go = 0 } if nr > 0 { total = total + nr } if total >= cap { go = 0 } }
29 sys_close(fd); return total
30}
31func dd_write(path: *u8, buf: *u8, n: i64) -> i64 { let fd: i64 = sys_openat_wr(path, 0x1a4); if fd < 0 { return 0 - 1 } sys_write(fd, buf, n); sys_close(fd); return n }
32func dd_have(path: *u8) -> i64 { let fd: i64 = sys_openat_rd(path); if fd < 0 { return 0 } sys_close(fd); return 1 }
33func dd_join(out: *u8, dir: *u8, name: *u8) -> i64 { var o: i64=0; var i: i64=0; while dir[i]!=(0 as u8){out[o]=dir[i];o=o+1;i=i+1} out[o]=47 as u8; o=o+1; i=0; while name[i]!=(0 as u8){out[o]=name[i];o=o+1;i=i+1} out[o]=0 as u8; return o }
34func dd_field(reg: *u8, ls: i64, le: i64, f: i64, out: *u8, cap: i64) -> i64 {
35 let fs: *i64=sys_mmap(8); let fe: *i64=sys_mmap(8)
36 if slk_field(reg, ls, le, f, fs, fe) == 1 { var o: i64=0; let l: i64=fe[0]-fs[0]; while o<l { if o<cap-1 { out[o]=reg[fs[0]+o] } o=o+1 } out[o]=0 as u8; return l }
37 out[0]=0 as u8; return 0
38}
39
40func main(argc: i64, argv: *i64) -> i64 {
41 if argc < 3 { dd_w("usage: nx_dr_dedup <manifest> <store-dir>\n" as *u8); sys_exit(2); return 2 }
42 let manifest: *u8 = argv[1] as *u8; let store: *u8 = argv[2] as *u8
43 dd_w("=== nx_dr_dedup: content-addressed store-once-by-hash ===\n" as *u8)
44 sys_mkdir(store, 0x1ed)
45 let reg: *u8 = sys_mmap(DD_MAGIC_262144); let rn: i64 = dd_read(manifest, reg, DD_MAGIC_262144)
46 if rn <= 0 { dd_w(" FATAL: cannot read manifest\n" as *u8); sys_exit(3); return 3 }
47
48 let src: *u8=sys_mmap(DD_MAGIC_1024); let name: *u8=sys_mmap(512); let blobp: *u8=sys_mmap(DD_MAGIC_1536)
49 let fbuf: *u8=sys_mmap(DD_CAP); let hexb: *u8=sys_mmap(80); let dig: *u8=sys_mmap(64)
50 // SOVEREIGN index (name -> blob hash) via nx_seg_store -> <store>/index.docs + index.keys -- NO TSV
51 let iprefix: *u8=sys_mmap(DD_MAGIC_1536); dd_join(iprefix, store, "index" as *u8)
52 // ROOT FIX (count-as-segid, id 1785519700): ss_manifest RETURNS THE ROW COUNT, not an id. Past
53 // SS_MANIFEST_LEGACY_CAP (256) the count PINS at 256, so every later write reuses id 257 and OVERWRITES
54 // the previous segment -- data loss by CLOBBER (distinct from the seq1730 pointer poison, which loses
55 // data by SHADOWING). ss_next_segid parses the real ids and is what the lib header prescribes.
56 // The ss_segs[] buffer was filled and never read, so it goes with the call.
57 let ss_segid: i64 = ss_next_segid(iprefix)
58 let w: *i64 = ss_begin()
59 let hx: *u8="0123456789abcdef" as *u8
60 var ls: i64=0
61 var total: i64=0; var unique: i64=0; var dup: i64=0; var total_bytes: i64=0; var saved: i64=0
62
63 while ls < rn {
64 let le: i64 = slk_line_end(reg, rn, ls)
65 if le > ls { if reg[ls] != (35 as u8) {
66 if dd_field(reg, ls, le, 0, src, DD_MAGIC_1024) > 0 {
67 dd_field(reg, ls, le, 1, name, 512)
68 let n: i64 = dd_read(src, fbuf, DD_CAP)
69 if n >= 0 {
70 total=total+1; total_bytes=total_bytes+n
71 sha256_digest(fbuf, n, dig)
72 var h: i64=0; while h<32 { let by: i64=(dig[h] as i64)&0xff; hexb[h*2]=hx[(by>>4)&15]; hexb[h*2+1]=hx[by&15]; h=h+1 } hexb[64]=0 as u8
73 dd_join(blobp, store, hexb)
74 if dd_have(blobp) == 1 { dup=dup+1; saved=saved+n; dd_w(" DEDUP " as *u8); dd_w(name); dd_w(" -> " as *u8); sys_write(1, hexb, 12); dd_w(".. (blob exists, 0 new bytes)\n" as *u8) }
75 else { dd_write(blobp, fbuf, n); unique=unique+1; dd_w(" STORE " as *u8); dd_w(name); dd_w(" -> " as *u8); sys_write(1, hexb, 12); dd_w(".. bytes=" as *u8); dd_wn(n); dd_w("\n" as *u8) }
76 ss_add(w, 1, name, hexb, 64) // sovereign index: name -> blob hash (64-hex)
77 }
78 }
79 } }
80 ls = le + 1
81 }
82 ss_commit(iprefix, w, ss_segid) // sovereign index -> <store>/index.docs + index.keys (NO TSV)
83 dd_w(" ---- files=" as *u8); dd_wn(total); dd_w(" unique-blobs=" as *u8); dd_wn(unique); dd_w(" dedup'd=" as *u8); dd_wn(dup); dd_w(" bytes=" as *u8); dd_wn(total_bytes); dd_w(" saved=" as *u8); dd_wn(saved); dd_w("\n" as *u8)
84 dd_w("=== DEDUP OK (identical content stored once; sovereign index -> <store>/index.docs+.keys, NO TSV) ===\n" as *u8)
85 sys_exit(0); return 0
86}