code wiki / _hdl_build / nx_janitor_sprawl.nx
nx_janitor_sprawl.nx source
↩ module page · 303 lines · 15695 B
1// nx_janitor_sprawl.nx -- the JANITOR's SPRAWL/JUNK-CLEANUP half (X-JAN-002, the getdents-walk
2// TODO noted in nx_janitor.nx). Finds ORPHANED registry .tsv files -- ones referenced by ZERO
3// .nx organs anywhere under the scan root -- and QUARANTINES them (reversible MOVE, never delete).
4//
5// NO-FALSE-POSITIVE LAW (cardinal): a registry .tsv is "orphaned" ONLY IF its basename appears in
6// ZERO .nx files under the scan root. ANY mention counts as PROTECTED -- a real read, a path const,
7// or even a bare mention inside a // comment. When in doubt, protect. The detector therefore uses
8// SUBSTRING matching of the literal basename "<name>.tsv" (so it catches bare names, full paths, and
9// "\x00"-terminated path literals alike), gated by a LEFT word-boundary so that "census.tsv" is not
10// spuriously matched as a substring of "math_census.tsv" -- the boundary only ever REJECTS a match
11// that is genuinely part of a LONGER different filename, so it can never cause a false orphan.
12//
13// REVERSIBLE: the action is QUARANTINE = sys_renameat (atomic move) into the quarantine dir, preserving
14// the filename. We NEVER unlink/delete registry data (CLAUDE.md #13: additive / soft-delete; history
15// is sacred). Re-running is idempotent: a file already moved out of the registry is simply gone from
16// the registry scan, so it is neither re-detected nor re-moved.
17//
18// PARAMETERIZED for testability (the gate points it at /tmp fixtures, never the real tree):
19// nx_janitor_sprawl [scan_root] [registry_dir] [quarantine_dir] [logpath]
20// Defaults (no argv) operate on the REAL tree:
21// scan_root = runtime
22// registry_dir = knowledge/registry
23// quarantine_dir = knowledge/_quarantine/registry
24// logpath = knowledge/status/janitor_sprawl.log
25// Durable: a JANITOR-SPRAWL row -> the logpath. Exit 0. Slow forward is fine.
26// license_tier: ORIGINAL
27import "nx_syscalls.nx"
28
29// ---- sizing constants (named; no magic numbers) ----
30const JS_PATH_SLOT: i64 = 768 // bytes per queued directory path
31const JS_QUEUE_CAP: i64 = 16384 // max directories in the BFS queue
32const JS_DENT_BUF: i64 = 65536 // getdents64 read buffer
33const JS_FILE_CAP: i64 = 4194304 // per-.nx read buffer (reused)
34const JS_NAME_CAP: i64 = 256 // max registry-basename length
35const JS_REG_MAX: i64 = 4096 // max registry .tsv files tracked
36const JS_DIR_MODE: i64 = 0x1ed // 0755 for the quarantine dir
37const JS_LOG_MODE: i64 = 0x1a4 // 0644 for the log
38const JS_DT_OFF: i64 = 16 // d_reclen offset in linux_dirent64
39const JS_NAME_OFF: i64 = 19 // d_name offset in linux_dirent64
40
41func js_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
42func js_fputs(fd: i64, s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(fd,s,n); return 0 }
43func js_num(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=(48 as u8);k=1}; while m>0{t[k]=((48+(m%10)) as u8);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(1,bb,k); return 0 }
44func js_fnum(fd: i64, v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(fd,"-" as *u8,1)}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=(48 as u8);k=1}; while m>0{t[k]=((48+(m%10)) as u8);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(fd,bb,k); return 0 }
45func js_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
46func js_eq(a: *u8, b: *u8) -> i64 { var i: i64=0; while a[i]!=(0 as u8){ if a[i]!=b[i]{return 0} i=i+1 } if b[i]!=(0 as u8){return 0} return 1 }
47
48// does path (len pl) end with ext (NUL-terminated)?
49func js_ends(path: *u8, pl: i64, ext: *u8) -> i64 {
50 let el: i64=js_slen(ext); if el>pl { return 0 }
51 var i: i64=0; while i<el { if path[pl-el+i]!=ext[i] { return 0 } i=i+1 } return 1
52}
53
54// is c an identifier char [A-Za-z0-9_]? Used for the LEFT word-boundary guard.
55func js_is_ident(c: i64) -> i64 {
56 if c>=48 { if c<=57 { return 1 } } // 0-9
57 if c>=65 { if c<=90 { return 1 } } // A-Z
58 if c>=97 { if c<=122 { return 1 } } // a-z
59 if c==95 { return 1 } // _
60 return 0
61}
62
63// Is the basename `pat` (NUL-terminated, e.g. "foo.tsv") REFERENCED inside buf[0..n)?
64// SUBSTRING match with a LEFT word-boundary: the char immediately before a hit must NOT be an
65// identifier char (so "census.tsv" is not matched inside "math_census.tsv"). No right-boundary
66// check -> "foo.tsv.bak"/"foo.tsv\x00"/"foo.tsv " all still PROTECT "foo.tsv" (over-protect = safe).
67// Returns 1 if referenced (PROTECTED), 0 if not.
68func js_referenced(buf: *u8, n: i64, pat: *u8) -> i64 {
69 let pl: i64=js_slen(pat); if pl==0 { return 0 }
70 var i: i64=0
71 while i+pl<=n {
72 var j: i64=0; var ok: i64=1
73 while j<pl { if buf[i+j]!=pat[j] { ok=0; j=pl } else { j=j+1 } }
74 if ok==1 {
75 // left boundary: position i must be the start of a filename token
76 var boundary: i64=1
77 if i>0 { if js_is_ident(buf[i-1] as i64)==1 { boundary=0 } }
78 if boundary==1 { return 1 }
79 }
80 i=i+1
81 }
82 return 0
83}
84
85// join dir + "/" + name into dst (NUL-terminated); returns length
86func js_join(dst: *u8, dir: *u8, name: *u8) -> i64 {
87 var o: i64=0; var i: i64=0
88 while dir[i]!=(0 as u8){ dst[o]=dir[i]; o=o+1; i=i+1 }
89 dst[o]=47 as u8; o=o+1
90 i=0; while name[i]!=(0 as u8){ dst[o]=name[i]; o=o+1; i=i+1 }
91 dst[o]=0 as u8; return o
92}
93
94func js_getdents(fd: i64, buf: *u8, count: i64) -> i64 { return __syscall(SYS_GETDENTS64, fd, buf, count, 0, 0, 0) }
95
96// 1=dir, 0=not, -1=stat failed. newfstatat(AT_FDCWD); st_mode @ offset 24; S_IFDIR=0x4000.
97func js_isdir(path: *u8, st: *u8) -> i64 {
98 let r: i64=__syscall(262, 0-100, path, st, 0, 0, 0)
99 if r<0 { return 0-1 }
100 let mode: i64=(st[24] as i64) | ((st[25] as i64)<<8) | ((st[26] as i64)<<16) | ((st[27] as i64)<<24)
101 if (mode & 0xF000)==0x4000 { return 1 }
102 return 0
103}
104
105// bounded read of `path` into reused `buf` (NOT sys_read_file -- avoids the 4GB-per-call reserve)
106func js_read(path: *u8, buf: *u8, cap: i64) -> i64 {
107 let fd: i64=sys_openat_rd(path); if fd<0 { return 0-1 }
108 var total: i64=0; var nrd: i64=sys_read(fd, buf, cap)
109 while nrd>0 { total=total+nrd; if total>=cap { nrd=0 } else { nrd=sys_read(fd, ((buf as i64)+total) as *u8, cap-total) } }
110 sys_close(fd); return total
111}
112
113// copy NUL-terminated src into a freshly-mmap'd buffer; return it.
114func js_dup(src: *u8) -> *u8 {
115 let l: i64=js_slen(src)
116 let d: *u8=sys_mmap(l+1)
117 var i: i64=0; while i<l { d[i]=src[i]; i=i+1 } d[l]=0 as u8
118 return d
119}
120
121// ---- PROTECTED-SET BUILD: walk scan_root, read every .nx, and for each registry basename mark
122// referenced=1 if it appears in that file. This is the single source of truth for "is it used".
123// names[] = the registry basenames (NUL-terminated each), prot[] = parallel 0/1 referenced flags.
124func js_build_protected(scan_root: *u8, names: *i64, prot: *i64, ncount: i64) -> i64 {
125 let qbuf: *u8=sys_mmap(JS_PATH_SLOT*JS_QUEUE_CAP)
126 let gbuf: *u8=sys_mmap(JS_DENT_BUF)
127 let stbuf: *u8=sys_mmap(256)
128 let fbuf: *u8=sys_mmap(JS_FILE_CAP)
129 let child: *u8=sys_mmap(JS_PATH_SLOT)
130 var qhead: i64=0; var qtail: i64=0
131 // seed queue with scan_root
132 var ri: i64=0; while scan_root[ri]!=(0 as u8){ qbuf[ri]=scan_root[ri]; ri=ri+1 } qbuf[ri]=0 as u8; qtail=1
133
134 var nx_scanned: i64=0
135 while qhead<qtail {
136 let dir: *u8=((qbuf as i64)+qhead*JS_PATH_SLOT) as *u8; qhead=qhead+1
137 let fd: i64=sys_openat_rd(dir)
138 if fd<0 { continue }
139 var nread: i64=js_getdents(fd, gbuf, JS_DENT_BUF)
140 while nread>0 {
141 var off: i64=0
142 while off<nread {
143 let reclen: i64=(gbuf[off+JS_DT_OFF] as i64) | ((gbuf[off+JS_DT_OFF+1] as i64)<<8)
144 if reclen<=0 { off=nread } else {
145 let name: *u8=((gbuf as i64)+off+JS_NAME_OFF) as *u8
146 var skip: i64=0
147 if js_eq(name, "." as *u8)==1 { skip=1 }
148 if js_eq(name, ".." as *u8)==1 { skip=1 }
149 if skip==0 {
150 js_join(child, dir, name)
151 let isd: i64=js_isdir(child, stbuf)
152 if isd==1 {
153 // descend everything except vendored trees (bounded + honest)
154 var vendored: i64=0
155 if js_eq(name, "node_modules" as *u8)==1 { vendored=1 }
156 if js_eq(name, ".git" as *u8)==1 { vendored=1 }
157 if js_eq(name, ".alelane" as *u8)==1 { vendored=1 }
158 if vendored==0 { if qtail<JS_QUEUE_CAP { let dst: *u8=((qbuf as i64)+qtail*JS_PATH_SLOT) as *u8; var c: i64=0; while child[c]!=(0 as u8){ dst[c]=child[c]; c=c+1 } dst[c]=0 as u8; qtail=qtail+1 } }
159 } else { if isd==0 {
160 let pl: i64=js_slen(child)
161 if js_ends(child, pl, ".nx" as *u8)==1 {
162 let fn: i64=js_read(child, fbuf, JS_FILE_CAP)
163 if fn>0 {
164 nx_scanned=nx_scanned+1
165 var k: i64=0
166 while k<ncount {
167 if prot[k]==0 { if js_referenced(fbuf, fn, names[k] as *u8)==1 { prot[k]=1 } }
168 k=k+1
169 }
170 }
171 }
172 } }
173 }
174 off=off+reclen
175 }
176 }
177 nread=js_getdents(fd, gbuf, JS_DENT_BUF)
178 }
179 sys_close(fd)
180 }
181 sys_munmap(qbuf, JS_PATH_SLOT*JS_QUEUE_CAP)
182 sys_munmap(gbuf, JS_DENT_BUF)
183 sys_munmap(fbuf, JS_FILE_CAP)
184 return nx_scanned
185}
186
187// ---- the CLASSIFY + QUARANTINE core (param'd so the gate can point it at fixtures).
188// Lists registry_dir, builds the protected set from scan_root, and for each *.tsv NOT protected:
189// ensure quarantine_dir exists (mkdir), then sys_renameat(registry/file -> quarantine/file).
190// Writes a JANITOR-SPRAWL row to logpath. Returns the number quarantined (>=0), or -1 on a hard
191// error (registry unreadable). outs[0]=total_tsv, outs[1]=protected, outs[2]=quarantined.
192func js_run(scan_root: *u8, registry_dir: *u8, quarantine_dir: *u8, logpath: *u8, outs: *i64) -> i64 {
193 // 1) enumerate registry *.tsv basenames
194 let names: *i64=sys_mmap(8*JS_REG_MAX) as *i64
195 let prot: *i64=sys_mmap(8*JS_REG_MAX) as *i64
196 var ncount: i64=0
197 let gbuf: *u8=sys_mmap(JS_DENT_BUF)
198 let rfd: i64=sys_openat_rd(registry_dir)
199 if rfd<0 { return 0-1 }
200 var nread: i64=js_getdents(rfd, gbuf, JS_DENT_BUF)
201 while nread>0 {
202 var off: i64=0
203 while off<nread {
204 let reclen: i64=(gbuf[off+JS_DT_OFF] as i64) | ((gbuf[off+JS_DT_OFF+1] as i64)<<8)
205 if reclen<=0 { off=nread } else {
206 let name: *u8=((gbuf as i64)+off+JS_NAME_OFF) as *u8
207 let nl: i64=js_slen(name)
208 // only files whose name ends exactly in ".tsv" (NOT .tsv.bak / .tsv.tmp / .conf / .md)
209 if js_ends(name, nl, ".tsv" as *u8)==1 {
210 if ncount<JS_REG_MAX { names[ncount]=js_dup(name) as i64; prot[ncount]=0; ncount=ncount+1 }
211 }
212 off=off+reclen
213 }
214 }
215 nread=js_getdents(rfd, gbuf, JS_DENT_BUF)
216 }
217 sys_close(rfd)
218 sys_munmap(gbuf, JS_DENT_BUF)
219
220 // 2) build the protected set by walking scan_root's .nx graph
221 js_build_protected(scan_root, names, prot, ncount)
222
223 // 3) count protected; collect orphans
224 var protected: i64=0
225 var k: i64=0
226 while k<ncount { if prot[k]==1 { protected=protected+1 } k=k+1 }
227 let orphans: i64=ncount-protected
228
229 // 4) open the durable log (append) BEFORE we move anything
230 let lfd: i64=sys_openat_append(logpath, JS_LOG_MODE)
231 let now: i64=sys_now_realtime_sec()
232
233 // 5) quarantine each orphan (ensure dir, then atomic rename). Print + log every action.
234 js_puts("=== JANITOR-SPRAWL: orphaned registry .tsv detector + reversible quarantiner ===\n" as *u8)
235 js_puts(" scan_root=" as *u8); js_puts(scan_root)
236 js_puts(" registry=" as *u8); js_puts(registry_dir)
237 js_puts(" quarantine=" as *u8); js_puts(quarantine_dir); js_puts("\n" as *u8)
238 js_puts(" total_tsv=" as *u8); js_num(ncount)
239 js_puts(" protected=" as *u8); js_num(protected)
240 js_puts(" orphans=" as *u8); js_num(orphans); js_puts("\n" as *u8)
241
242 if lfd>=0 {
243 js_fputs(lfd, "JANITOR-SPRAWL epoch=" as *u8); js_fnum(lfd, now)
244 js_fputs(lfd, " scan_root=" as *u8); js_fputs(lfd, scan_root)
245 js_fputs(lfd, " total_tsv=" as *u8); js_fnum(lfd, ncount)
246 js_fputs(lfd, " protected=" as *u8); js_fnum(lfd, protected)
247 js_fputs(lfd, " quarantined=" as *u8); js_fnum(lfd, orphans)
248 js_fputs(lfd, " names=[" as *u8)
249 }
250
251 var moved: i64=0
252 var made_dir: i64=0
253 let src: *u8=sys_mmap(JS_PATH_SLOT)
254 let dst: *u8=sys_mmap(JS_PATH_SLOT)
255 k=0
256 while k<ncount {
257 if prot[k]==0 {
258 // lazily create the quarantine dir on first orphan only
259 if made_dir==0 { sys_mkdir(quarantine_dir, JS_DIR_MODE); made_dir=1 }
260 js_join(src, registry_dir, names[k] as *u8)
261 js_join(dst, quarantine_dir, names[k] as *u8)
262 let r: i64=sys_renameat(src, dst)
263 if r==0 {
264 moved=moved+1
265 js_puts(" QUARANTINED " as *u8); js_puts(names[k] as *u8)
266 js_puts(" -> " as *u8); js_puts(quarantine_dir); js_puts("\n" as *u8)
267 if lfd>=0 { if moved>1 { js_fputs(lfd, "," as *u8) } js_fputs(lfd, names[k] as *u8) }
268 } else {
269 // could not move -- report, never delete; the file stays put (safe)
270 js_puts(" WARN could-not-quarantine " as *u8); js_puts(names[k] as *u8)
271 js_puts(" rc=" as *u8); js_num(r); js_puts(" (left in place)\n" as *u8)
272 }
273 }
274 k=k+1
275 }
276
277 if lfd>=0 { js_fputs(lfd, "]\n" as *u8); sys_close(lfd) }
278
279 js_puts(" QUARANTINED-TOTAL=" as *u8); js_num(moved); js_puts("\n" as *u8)
280 outs[0]=ncount; outs[1]=protected; outs[2]=moved
281 return moved
282}
283
284func main(argc: i64, argv: *i64) -> i64 {
285 // REAL-TREE defaults; argv overrides let the gate target /tmp fixtures.
286 var scan_root: *u8 = "runtime" as *u8
287 var registry: *u8 = "knowledge/registry" as *u8
288 var quarantine: *u8 = "knowledge/_quarantine/registry" as *u8
289 var logpath: *u8 = "knowledge/status/janitor_sprawl.log" as *u8
290 if argc>=2 { scan_root = argv[1] as *u8 }
291 if argc>=3 { registry = argv[2] as *u8 }
292 if argc>=4 { quarantine = argv[3] as *u8 }
293 if argc>=5 { logpath = argv[4] as *u8 }
294
295 let outs: *i64=sys_mmap(64) as *i64
296 outs[0]=0; outs[1]=0; outs[2]=0
297 let rc: i64=js_run(scan_root, registry, quarantine, logpath, outs)
298 if rc<0 { js_puts(" ERROR: registry dir unreadable: " as *u8); js_puts(registry); js_puts("\n" as *u8); sys_exit(1); return 1 }
299 js_puts("JANITOR-SPRAWL-DONE total=" as *u8); js_num(outs[0]);
300 js_puts(" protected=" as *u8); js_num(outs[1]);
301 js_puts(" quarantined=" as *u8); js_num(outs[2]); js_puts("\n" as *u8)
302 sys_exit(0); return 0
303}