nx_store_janitor.nx source
↩ module page · 261 lines · 12138 B
1// nx_store_janitor.nx -- reclaim SUPERSEDED seg-store segment files. The janitor pass that seq724/RK013
2// named as reclaimable and nobody had built.
3//
4// MEASURED 2026-07-30 BEFORE WRITING THIS: knowledge/store holds 63,669 MB across 14,585 segment .docs files,
5// while all 1,088 manifests TOGETHER reference only 3,031 live segments. debt-manifest.txt lists exactly TWO
6// segments while 2,055 debt-seg-*.docs exist on disk, ~2.4 GB for one plane. So roughly 79 percent of the
7// information plane's bytes are segments no reader can ever reach: the seg-store is append-only and every
8// commit writes a NEW segment, so superseded generations simply accumulate forever.
9//
10// WHY THIS IS SAFE BY CONSTRUCTION: the manifest IS the definition of live. ss_manifest/ss_scan_seglist read
11// ONLY segments listed there, so a file whose segid is absent from its own plane's manifest is unreachable BY
12// DEFINITION, not by inference. That is the whole safety argument and it is why this tool refuses to run when
13// it cannot read the manifest -- an unknown live set means an unknown safe set.
14//
15// RULE 13 -- MOVE ASIDE, NEVER DELETE. Superseded files are RENAMED into knowledge/store/retired/ (the
16// convention already present on the box). Nothing is unlinked, so a mistake is reversible by renaming back.
17// DRY RUN IS THE DEFAULT: it reports what it WOULD move and touches nothing. Pass 'apply' to actually move.
18//
19// EXIT: 0 clean/reported - 2 usage - 3 REFUSED (manifest unreadable or empty live set) - 4 nothing superseded
20// license_tier: ORIGINAL No hw writes (Rule 26).
21import "nx_syscalls.nx"
22
23const SJ_MANI: i64 = 262144
24const SJ_DBUF: i64 = 262144
25const SJ_LIVE: i64 = 65536
26const SJ_MAXLIVE: i64 = 8192
27const SJ_PATH: i64 = 1024
28const SJ_MSG: i64 = 8192
29const SJ_STAT: i64 = 256
30const SJ_STAT_SIZE_OFF: i64 = 48
31const SJ_NL: i64 = 10
32const SJ_DOT: i64 = 46
33const SJ_SLASH: i64 = 47
34const SJ_STDOUT: i64 = 1
35const SJ_EXIT_USAGE: i64 = 2
36const SJ_EXIT_REFUSED: i64 = 3
37const SJ_EXIT_NONE: i64 = 4
38const SJ_MB: i64 = 1048576
39
40func sj_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
41func sj_cat(d: *u8, off: i64, s: *u8) -> i64 {
42 var i: i64 = 0
43 while s[i] != (0 as u8) { d[off + i] = s[i]; i = i + 1 }
44 return off + i
45}
46func sj_num(d: *u8, off: i64, v: i64) -> i64 {
47 var m: i64 = v
48 var o: i64 = off
49 if m < 0 { m = 0 - m; d[o] = 45 as u8; o = o + 1 }
50 let t: *u8 = sys_mmap(32)
51 var k: i64 = 0
52 if m == 0 { t[0] = 48 as u8; k = 1 }
53 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
54 var i: i64 = 0
55 while i < k { d[o + i] = t[k - 1 - i]; i = i + 1 }
56 sys_munmap(t, 32)
57 return o + k
58}
59// file size via fstatat (st_size at +48), -1 if unstatable
60func sj_size(path: *u8) -> i64 {
61 let st: *u8 = sys_mmap(SJ_STAT)
62 if sys_fstatat(path, st) != 0 { sys_munmap(st, SJ_STAT); return 0 - 1 }
63 let p: *i64 = (st as i64 + SJ_STAT_SIZE_OFF) as *i64
64 let v: i64 = p[0]
65 sys_munmap(st, SJ_STAT)
66 return v
67}
68
69func main(argc: i64, argv: *i64) -> i64 {
70 let msg: *u8 = sys_mmap(SJ_MSG)
71 if argc < 2 {
72 var u: i64 = sj_cat(msg, 0, "usage: nx_store_janitor <knowledge/store/prefix-> [apply]\n DRY RUN by default. Moves segments absent from the plane manifest into knowledge/store/retired/.\n" as *u8)
73 sys_write(SJ_STDOUT, msg, u); return SJ_EXIT_USAGE
74 }
75 let prefix: *u8 = argv[1] as *u8
76 var doapply: i64 = 0
77 if argc >= 3 {
78 let a: *u8 = argv[2] as *u8
79 if a[0] == (97 as u8) { doapply = 1 }
80 }
81 let plen: i64 = sj_len(prefix)
82
83 // split prefix into dir and basename (everything after the last '/')
84 var cut: i64 = 0 - 1
85 var i0: i64 = 0
86 while i0 < plen { if prefix[i0] == (SJ_SLASH as u8) { cut = i0 } i0 = i0 + 1 }
87 let dir: *u8 = sys_mmap(SJ_PATH)
88 let base: *u8 = sys_mmap(SJ_PATH)
89 var di: i64 = 0
90 while di < cut { dir[di] = prefix[di]; di = di + 1 }
91 dir[di] = 0 as u8
92 var bi: i64 = 0
93 while cut + 1 + bi < plen { base[bi] = prefix[cut + 1 + bi]; bi = bi + 1 }
94 base[bi] = 0 as u8
95 let blen: i64 = bi
96
97 // ---- READ THE MANIFEST. This is the definition of live; without it there is no safe set.
98 let mpath: *u8 = sys_mmap(SJ_PATH)
99 var mo: i64 = sj_cat(mpath, 0, prefix)
100 mo = sj_cat(mpath, mo, "manifest.txt" as *u8)
101 mpath[mo] = 0 as u8
102 let mbuf: *u8 = sys_mmap(SJ_MANI)
103 var mn: i64 = 0
104 let mfd: i64 = sys_openat_rd(mpath)
105 if mfd >= 0 { let r: i64 = sys_read(mfd, mbuf, SJ_MANI); if r > 0 { mn = r } sys_close(mfd) }
106 // index live segment names (one per line)
107 let loff: *i64 = sys_mmap(8 * SJ_MAXLIVE) as *i64
108 let llen: *i64 = sys_mmap(8 * SJ_MAXLIVE) as *i64
109 var nlive: i64 = 0
110 var ls: i64 = 0
111 var ci: i64 = 0
112 while ci <= mn {
113 var atend: i64 = 0
114 if ci == mn { atend = 1 } else { if mbuf[ci] == (SJ_NL as u8) { atend = 1 } }
115 if atend == 1 {
116 let ll: i64 = ci - ls
117 if ll > 0 { if nlive < SJ_MAXLIVE { loff[nlive] = ls; llen[nlive] = ll; nlive = nlive + 1 } }
118 ls = ci + 1
119 }
120 ci = ci + 1
121 }
122 var o: i64 = sj_cat(msg, 0, "NX-STORE-JANITOR prefix=" as *u8)
123 o = sj_cat(msg, o, prefix)
124 o = sj_cat(msg, o, " live_segments=" as *u8); o = sj_num(msg, o, nlive)
125 if nlive == 0 {
126 o = sj_cat(msg, o, " verdict=REFUSED-manifest-unreadable-or-empty-UNKNOWN-LIVE-SET-nothing-touched\n" as *u8)
127 sys_write(SJ_STDOUT, msg, o); return SJ_EXIT_REFUSED
128 }
129
130 // ---- walk the store dir; any <base>seg-<id>.<ext> whose seg-<id> is absent from the manifest is dead
131 let dbuf: *u8 = sys_mmap(SJ_DBUF)
132 let fpath: *u8 = sys_mmap(SJ_PATH)
133 let tpath: *u8 = sys_mmap(SJ_PATH)
134 var dead: i64 = 0
135 var deadbytes: i64 = 0
136 var moved: i64 = 0
137 var failed: i64 = 0
138 let fd: i64 = sys_openat_rd(dir)
139 if fd < 0 {
140 o = sj_cat(msg, o, " verdict=REFUSED-cannot-open-dir\n" as *u8)
141 sys_write(SJ_STDOUT, msg, o); return SJ_EXIT_REFUSED
142 }
143 var go: i64 = 1
144 while go == 1 {
145 let nb: i64 = sys_getdents64(fd, dbuf, SJ_DBUF)
146 if nb <= 0 { go = 0 } else {
147 var off: i64 = 0
148 while off < nb {
149 let rec: *u8 = (dbuf as i64 + off) as *u8
150 let rl: i64 = dirent_reclen(rec)
151 if rl <= 0 { off = nb } else {
152 let nm: *u8 = dirent_name(rec)
153 // must start with base, then "seg-"
154 var pfxok: i64 = 1
155 var k: i64 = 0
156 while k < blen { if nm[k] != base[k] { pfxok = 0; k = blen } else { k = k + 1 } }
157 if pfxok == 1 {
158 if nm[blen] == (115 as u8) { if nm[blen+1] == (101 as u8) { if nm[blen+2] == (103 as u8) { if nm[blen+3] == (45 as u8) {
159 // segment token = from blen up to the first '.'
160 var e: i64 = blen
161 while nm[e] != (0 as u8) { if nm[e] == (SJ_DOT as u8) { break } e = e + 1 }
162 let seglen: i64 = e - blen
163 var islive: i64 = 0
164 var q: i64 = 0
165 while q < nlive {
166 if llen[q] == seglen {
167 var same: i64 = 1
168 var z: i64 = 0
169 while z < seglen { if mbuf[loff[q] + z] != nm[blen + z] { same = 0; z = seglen } else { z = z + 1 } }
170 if same == 1 { islive = 1; q = nlive } else { q = q + 1 }
171 } else { q = q + 1 }
172 }
173 if islive == 0 {
174 var fo: i64 = sj_cat(fpath, 0, dir)
175 fpath[fo] = SJ_SLASH as u8; fo = fo + 1
176 fo = sj_cat(fpath, fo, nm)
177 fpath[fo] = 0 as u8
178 let sz: i64 = sj_size(fpath)
179 dead = dead + 1
180 if sz > 0 { deadbytes = deadbytes + sz }
181 if doapply == 1 {
182 var to: i64 = sj_cat(tpath, 0, dir)
183 to = sj_cat(tpath, to, "/retired/" as *u8)
184 to = sj_cat(tpath, to, nm)
185 tpath[to] = 0 as u8
186 if sys_renameat(fpath, tpath) == 0 { moved = moved + 1 } else { failed = failed + 1 }
187 }
188 }
189 } } } }
190 }
191 off = off + rl
192 }
193 }
194 }
195 }
196 sys_close(fd)
197
198 // ★★★POST-SWEEP LIVE-SET VERIFICATION (closes the commit race). The manifest was read at the START; a
199 // writer committing a NEW segment mid-walk would leave that segment absent from our stale live set and
200 // it would be MOVED -- silently taking the plane's newest generation. So after moving, RE-READ the
201 // manifest as it is NOW and confirm every segment it lists still has its .docs on disk. If one is
202 // missing we moved a live file: rename it straight back from retired/ and fail LOUD.
203 // A JANITOR THAT CAN RACE A WRITER MUST CHECK THE WRITER'S ANSWER AFTER IT ACTS, NOT ONLY BEFORE.
204 var restored: i64 = 0
205 var lost: i64 = 0
206 if doapply == 1 {
207 let m2: *u8 = sys_mmap(SJ_MANI)
208 var m2n: i64 = 0
209 let f2: i64 = sys_openat_rd(mpath)
210 if f2 >= 0 { let r2: i64 = sys_read(f2, m2, SJ_MANI); if r2 > 0 { m2n = r2 } sys_close(f2) }
211 var s2: i64 = 0
212 var c2: i64 = 0
213 while c2 <= m2n {
214 var at2: i64 = 0
215 if c2 == m2n { at2 = 1 } else { if m2[c2] == (SJ_NL as u8) { at2 = 1 } }
216 if at2 == 1 {
217 let l2: i64 = c2 - s2
218 if l2 > 0 {
219 var fo2: i64 = sj_cat(fpath, 0, prefix)
220 var z2: i64 = 0
221 while z2 < l2 { fpath[fo2 + z2] = m2[s2 + z2]; z2 = z2 + 1 }
222 fo2 = fo2 + l2
223 fo2 = sj_cat(fpath, fo2, ".docs" as *u8)
224 fpath[fo2] = 0 as u8
225 if sj_size(fpath) < 0 {
226 // it is live NOW but gone from disk => we moved it. Put it back.
227 var to2: i64 = sj_cat(tpath, 0, dir)
228 to2 = sj_cat(tpath, to2, "/retired/" as *u8)
229 to2 = sj_cat(tpath, to2, base)
230 var z3: i64 = 0
231 while z3 < l2 { tpath[to2 + z3] = m2[s2 + z3]; z3 = z3 + 1 }
232 to2 = to2 + l2
233 to2 = sj_cat(tpath, to2, ".docs" as *u8)
234 tpath[to2] = 0 as u8
235 if sys_renameat(tpath, fpath) == 0 { restored = restored + 1 } else { lost = lost + 1 }
236 }
237 }
238 s2 = c2 + 1
239 }
240 c2 = c2 + 1
241 }
242 sys_munmap(m2, SJ_MANI)
243 }
244 o = sj_cat(msg, o, " superseded_files=" as *u8); o = sj_num(msg, o, dead)
245 if restored > 0 { o = sj_cat(msg, o, " RACE-RESTORED=" as *u8); o = sj_num(msg, o, restored) }
246 if lost > 0 { o = sj_cat(msg, o, " **LOST-LIVE-SEGMENT=" as *u8); o = sj_num(msg, o, lost) }
247 o = sj_cat(msg, o, " reclaimable_MB=" as *u8); o = sj_num(msg, o, deadbytes / SJ_MB)
248 if doapply == 1 {
249 o = sj_cat(msg, o, " MOVED=" as *u8); o = sj_num(msg, o, moved)
250 o = sj_cat(msg, o, " failed=" as *u8); o = sj_num(msg, o, failed)
251 o = sj_cat(msg, o, " mode=APPLY-moved-to-retired-NOT-deleted" as *u8)
252 } else {
253 o = sj_cat(msg, o, " mode=DRY-RUN-nothing-touched-pass-apply-to-move" as *u8)
254 }
255 var rc: i64 = 0
256 if dead == 0 { rc = SJ_EXIT_NONE }
257 if failed > 0 { rc = SJ_EXIT_REFUSED }
258 msg[o] = SJ_NL as u8; o = o + 1
259 sys_write(SJ_STDOUT, msg, o)
260 return rc
261}