nx_mvault_reindex.nx source
↩ module page · 112 lines · 5501 B
1// nx_mvault_reindex.nx -- the per-item MIGRATE pipeline core (dry-run/commit).
2//
3// The live driver walks the real galx paths (getdents) and calls this per file;
4// the outer walk also folds a numbered-image-sequence DIR into ONE manga work
5// (ctx=IMGSEQ). This core is the gate-provable heart: for one item it does
6// contextual-classify -> content CID (dedup key) -> DSM dest path
7// -> dedup check vs the store -> reclaim tally -> (commit) seg-store record.
8//
9// NO-COPY MIGRATE (operator "no space for a copy, too many TB"): the LIVE
10// driver relocates each blob with sys_renameat WITHIN /volume1 (zero-copy,
11// additive: rename preserves the bytes) to the dest this core computes. commit=0
12// is DRY-RUN (classify + plan + dedup tally, writes NOTHING) so the whole run
13// can be previewed before a single byte moves.
14//
15// Composes ONLY proven organs. license_tier: ORIGINAL
16
17import "nx_syscalls.nx"
18import "nx_canon_cid.nx"
19import "nx_media_pool.nx"
20import "nx_media_sniff.nx"
21import "nx_mvault_ingest.nx"
22import "nx_mvault_context.nx"
23import "nx_mvault_record.nx"
24import "nx_mvault_layout.nx"
25import "nx_mvault_reclaim.nx"
26const K_MAGIC_4096: i64 = 4096
27
28// Classify + plan + tally one item.
29// rc reclaim accumulator (mv_reclaim_new)
30// store_prefix / dest_root seg-store + DSM root
31// content,clen the item bytes (full content -> content CID for dedup)
32// class_code MV_CLASS_GEN | MV_CLASS_REAL (from the source root)
33// ctx MV_CTX_* (dir-derived: numbered-seq / archive / ebook / none)
34// source,ext visible source tag + file extension
35// large_thresh bytes above which a KEPT item is surfaced for value review
36// commit 1 = write the seg-store record (new items only); 0 = dry-run
37// out_path receives the DSM dest path; out_cid receives the content CID
38// returns is_dup (1 = a copy of already-stored content -> reclaimable).
39// PURE half: contextual type from HEAD bytes + DSM dest path for a
40// PRE-COMPUTED CID. No store I/O -- bulk drivers dedup in memory and batch
41// their own commits. Frees its sniff scratch (bulk loops call this per file).
42const MV_EXTPATH: i64 = 256 // scratch for the synthetic "f.<ext>" sniff path
43
44func mv_classify_plan(head: *u8, headlen: i64, class_code: i64, ctx: i64,
45 dest_root: *u8, source: *u8, ext: *u8,
46 out_path: *u8, cid: *u8) -> i64 {
47 let empty: *u8 = sys_mmap(1)
48 // ★2026-07-30: build a synthetic "f.<ext>" path so the EXTENSION TIER
49 // actually participates. This call used to pass an EMPTY path, so the bulk
50 // migration was magic-only -- every format with no magic signature (.obj,
51 // .gcode, .dae, .stl-ascii-variants, .cbz, .m4b, .3mf) stayed invisible to
52 // the walk even after the sniffer learned them. Real magic still wins the
53 // tier order; this only supplies the fallback the walk was withholding.
54 let pbuf: *u8 = sys_mmap(MV_EXTPATH)
55 var po: i64 = 0
56 pbuf[po] = 102 as u8
57 po = po + 1
58 pbuf[po] = 46 as u8
59 po = po + 1
60 var ei: i64 = 0
61 while ext[ei] != (0 as u8) {
62 if po < MV_EXTPATH - 8 { pbuf[po] = ext[ei]; po = po + 1 }
63 ei = ei + 1
64 }
65 pbuf[po] = 0 as u8
66 let stype: i64 = nx_sniff(head, headlen, empty, 0, pbuf, po)
67 let vt: i64 = mv_context_type(stype, ctx)
68 mv_disk_path(out_path, dest_root, class_code, vt, cid, source, ext)
69 sys_munmap(pbuf, MV_EXTPATH)
70 sys_munmap(empty, 1)
71 return vt
72}
73
74// Post-CID per-item pipeline: the CID is ALREADY computed (streaming path for
75// multi-GB files -- only HEAD bytes are in RAM; clen = FULL size from stat so
76// reclaim accounting stays byte-honest).
77func mv_reindex_cid(rc: *i64, store_prefix: *u8, dest_root: *u8,
78 head: *u8, headlen: i64, clen: i64, class_code: i64, ctx: i64,
79 source: *u8, ext: *u8, large_thresh: i64, commit: i64,
80 out_path: *u8, out_cid: *u8) -> i64 {
81 let vtype: i64 = mv_classify_plan(head, headlen, class_code, ctx, dest_root, source, ext, out_path, out_cid)
82 // dedup check against the store
83 let gp: *i64 = sys_mmap(16) as *i64
84 let gl: *i64 = sys_mmap(16) as *i64
85 let found: i64 = mv_store_get(store_prefix, out_cid, gp, gl)
86 var is_dup: i64 = 0
87 if found == 1 { is_dup = 1 }
88 // reclamation accounting (dup bytes are recoverable)
89 mv_reclaim_add(rc, clen, is_dup, large_thresh)
90 // record a NEW item (commit only); the classified NXR1 record, ref=dest
91 if is_dup == 0 {
92 if commit == 1 {
93 let rec: *u8 = sys_mmap(K_MAGIC_4096)
94 let rl: i64 = mv_record_build(rec, class_code, vtype, MV_LVL_TS, source, out_path)
95 mv_store_put(store_prefix, out_cid, rec, rl)
96 }
97 }
98 return is_dup
99}
100
101// Classify + plan + tally one item from FULL in-RAM content (the original
102// entry point; small files). CID computed here, then the shared post-CID half.
103func mv_reindex_one(rc: *i64, store_prefix: *u8, dest_root: *u8,
104 content: *u8, clen: i64, class_code: i64, ctx: i64,
105 source: *u8, ext: *u8, large_thresh: i64, commit: i64,
106 out_path: *u8, out_cid: *u8) -> i64 {
107 // content CID = the dedup key (identical bytes -> identical CID)
108 cid_of(content, clen, out_cid)
109 return mv_reindex_cid(rc, store_prefix, dest_root, content, clen, clen,
110 class_code, ctx, source, ext, large_thresh, commit,
111 out_path, out_cid)
112}