code wiki / _hdl_build / nx_datacurator.nx
nx_datacurator.nx source
↩ module page · 434 lines · 24799 B
1// nx_datacurator.nx -- SOVEREIGN storage curator (WizTree / TreeSize / ncdu-class, on the NAS). Recursively walks a
2// target dir, stats every file (one fstatat = mode+size), classifies by MEDIA TYPE, and reports size+count per type
3// + the biggest N files -- so the operator can SEE where the space is by media type and decide what to delete.
4//
5// READ-ONLY: stat only; it NEVER writes or deletes (deletion is the separate soft+confirmed `del` mode). Composes the
6// recursive getdents walk from nx_dr_tree, but is LEAK-SAFE at scale: the per-dir getdents buffer is munmap'd on
7// return (nx_dr_tree leaks it -- fine for one archive, fatal walking 142 TB), and NOTHING is mmap'd per FILE (one
8// reused statbuf threaded down). Bounded memory regardless of file count.
9//
10// Run: nx_datacurator <dir> [topN] (topN default 30, max 64)
11// license_tier: ORIGINAL
12import "nx_syscalls.nx"
13const DC_MAGIC_1099511627776: i64 = 1099511627776
14const DC_MAGIC_1073741824: i64 = 1073741824
15const DC_MAGIC_1048576: i64 = 1048576
16const DC_MAGIC_3900: i64 = 3900
17const DC_MAGIC_131072: i64 = 131072
18const DC_MAGIC_61440: i64 = 61440
19const DC_MAGIC_16384: i64 = 16384
20const DC_MAGIC_32768: i64 = 32768
21const DC_MAGIC_4096: i64 = 4096
22const DC_MAGIC_2048: i64 = 2048
23const DC_MAGIC_60000: i64 = 60000
24const DC_MAGIC_65536: i64 = 65536
25
26const DC_PATHW: i64 = 400 // bytes per stored biggest-file path
27const DC_TOPMAX: i64 = 64
28const DC_NCAT: i64 = 6 // 0=video 1=image 2=audio 3=doc 4=archive 5=other
29
30func dc_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
31func dc_w(s: *u8) -> i64 { sys_write(1, s, dc_slen(s)); return 0 }
32func dc_bcat(dst: *u8, o: i64, s: *u8) -> i64 { var x: i64 = o; var i: i64 = 0; while s[i] != (0 as u8) { dst[x] = s[i]; x = x + 1; i = i + 1 } return x }
33func dc_bcatn(dst: *u8, o: i64, v: i64) -> i64 {
34 var x: i64 = o; var m: i64 = v
35 if m < 0 { dst[x] = 45 as u8; x = x + 1; m = 0 - m }
36 if m == 0 { dst[x] = 48 as u8; return x + 1 }
37 let t: *u8 = sys_mmap(24); var k: i64 = 0
38 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
39 var j: i64 = 0; while j < k { dst[x] = t[k - 1 - j]; x = x + 1; j = j + 1 }
40 return x
41}
42// append a human size "X.Y UNIT" to dst; returns new offset.
43func dc_human(dst: *u8, o: i64, bytes: i64) -> i64 {
44 var oo: i64 = o
45 let tb: i64 = DC_MAGIC_1099511627776
46 let gb: i64 = DC_MAGIC_1073741824
47 let mb: i64 = DC_MAGIC_1048576
48 if bytes >= tb { let x: i64 = bytes * 10 / tb; oo = dc_bcatn(dst, oo, x / 10); dst[oo] = 46 as u8; oo = oo + 1; oo = dc_bcatn(dst, oo, x - (x / 10) * 10); return dc_bcat(dst, oo, " TB" as *u8) }
49 if bytes >= gb { let x: i64 = bytes * 10 / gb; oo = dc_bcatn(dst, oo, x / 10); dst[oo] = 46 as u8; oo = oo + 1; oo = dc_bcatn(dst, oo, x - (x / 10) * 10); return dc_bcat(dst, oo, " GB" as *u8) }
50 if bytes >= mb { let x: i64 = bytes * 10 / mb; oo = dc_bcatn(dst, oo, x / 10); dst[oo] = 46 as u8; oo = oo + 1; oo = dc_bcatn(dst, oo, x - (x / 10) * 10); return dc_bcat(dst, oo, " MB" as *u8) }
51 oo = dc_bcatn(dst, oo, bytes); return dc_bcat(dst, oo, " B" as *u8)
52}
53func dc_flush(buf: *u8, n: i64) -> i64 { sys_write(1, buf, n); return 0 }
54
55// append "/name" to buf at base_n (NOT NUL-terminated); returns new length.
56func dc_join(buf: *u8, base_n: i64, name: *u8) -> i64 {
57 buf[base_n] = 47 as u8; var o: i64 = base_n + 1; var i: i64 = 0
58 while name[i] != (0 as u8) { buf[o] = name[i]; o = o + 1; i = i + 1 }
59 return o
60}
61func dc_isdotdot(nm: *u8) -> i64 {
62 if nm[0] == (46 as u8) { if nm[1] == (0 as u8) { return 1 } if nm[1] == (46 as u8) { if nm[2] == (0 as u8) { return 1 } } }
63 return 0
64}
65func dc_lc(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c }
66// is `word` (nul-term, into the filename) a whitespace-delimited token in `list`? case-insensitive.
67func dc_in_ci(word: *u8, list: *u8) -> i64 {
68 let wl: i64 = dc_slen(word)
69 var i: i64 = 0
70 while list[i] != (0 as u8) {
71 var tstart: i64 = 0
72 if i == 0 { tstart = 1 } else { if list[i - 1] == (32 as u8) { tstart = 1 } }
73 if tstart == 1 {
74 var j: i64 = 0; var mt: i64 = 1
75 while j < wl { if dc_lc(list[i + j] as i64) != dc_lc(word[j] as i64) { mt = 0; j = wl } else { j = j + 1 } }
76 if mt == 1 {
77 let after: i64 = list[i + wl] as i64
78 if after == 32 { return 1 }
79 if after == 0 { return 1 }
80 }
81 }
82 i = i + 1
83 }
84 return 0
85}
86// classify a filename by extension -> category index (0..5). no allocation (ext is a pointer into nm).
87func dc_classify(nm: *u8) -> i64 {
88 var dot: i64 = 0 - 1; var i: i64 = 0
89 while nm[i] != (0 as u8) { if nm[i] == (46 as u8) { dot = i } i = i + 1 }
90 if dot < 0 { return 5 }
91 let ext: *u8 = (nm as i64 + dot + 1) as *u8
92 if dc_in_ci(ext, "mp4 mkv avi mov webm ts m4v wmv flv mpg mpeg m2ts vob 3gp" as *u8) == 1 { return 0 }
93 if dc_in_ci(ext, "jpg jpeg png gif webp heic bmp tiff tif raw cr2 nef dng arw svg" as *u8) == 1 { return 1 }
94 if dc_in_ci(ext, "mp3 flac wav m4a aac ogg wma opus aiff" as *u8) == 1 { return 2 }
95 if dc_in_ci(ext, "pdf txt doc docx epub mobi azw3 xls xlsx ppt pptx md rtf csv" as *u8) == 1 { return 3 }
96 if dc_in_ci(ext, "zip tar gz 7z rar bz2 xz iso img dmg part torrent" as *u8) == 1 { return 4 }
97 return 5
98}
99// consider a file for the top-N biggest list. bs=sizes[topN], bp=paths[topN*DC_PATHW], nkeep=topN.
100func dc_topn(bs: *i64, bp: *u8, nkeep: i64, path: *u8, pn: i64, sz: i64) -> i64 {
101 // find the current minimum slot
102 var mini: i64 = 0; var minv: i64 = bs[0]; var k: i64 = 1
103 while k < nkeep { if bs[k] < minv { minv = bs[k]; mini = k } k = k + 1 }
104 if sz <= minv { return 0 }
105 bs[mini] = sz
106 let dst: *u8 = (bp as i64 + mini * DC_PATHW) as *u8
107 var i: i64 = 0
108 while i < pn { if i < DC_PATHW - 1 { dst[i] = path[i] } i = i + 1 }
109 var t: i64 = pn; if t > DC_PATHW - 1 { t = DC_PATHW - 1 }
110 dst[t] = 0 as u8
111 return 0
112}
113// recursive walk. sb=reused 160B statbuf. cb/cc=per-cat bytes/count. bs/bp/nkeep=top-N. tot[0]=files tot[1]=bytes.
114func dc_walk(path: *u8, pn: i64, sb: *u8, cb: *i64, cc: *i64, bs: *i64, bp: *u8, nkeep: i64, tot: *i64) -> i64 {
115 if pn > DC_MAGIC_3900 { return 0 }
116 path[pn] = 0 as u8
117 let fd: i64 = sys_openat_rd(path)
118 if fd < 0 { return 0 }
119 let dbuf: *u8 = sys_mmap(DC_MAGIC_131072)
120 var go: i64 = 1
121 while go == 1 {
122 let nr: i64 = sys_getdents64(fd, dbuf, DC_MAGIC_131072)
123 if nr <= 0 { go = 0 } else {
124 var off: i64 = 0
125 while off < nr {
126 let rec: *u8 = (dbuf as i64 + off) as *u8
127 let nm: *u8 = dirent_name(rec)
128 if dc_isdotdot(nm) == 0 {
129 let cn: i64 = dc_join(path, pn, nm)
130 path[cn] = 0 as u8
131 if sys_fstatat(path, sb) == 0 {
132 let mode: i64 = (sb[24] as i64) | ((sb[25] as i64) << 8)
133 let sp: *i64 = sb as *i64
134 let sz: i64 = sp[6]
135 if (mode & DC_MAGIC_61440) == DC_MAGIC_16384 { // S_IFDIR (0x4000)
136 dc_walk(path, cn, sb, cb, cc, bs, bp, nkeep, tot)
137 } else {
138 if (mode & DC_MAGIC_61440) == DC_MAGIC_32768 { // S_IFREG (0x8000)
139 let cat: i64 = dc_classify(nm)
140 cb[cat] = cb[cat] + sz
141 cc[cat] = cc[cat] + 1
142 tot[0] = tot[0] + 1
143 tot[1] = tot[1] + sz
144 dc_topn(bs, bp, nkeep, path, cn, sz)
145 }
146 }
147 }
148 }
149 off = off + dirent_reclen(rec)
150 }
151 }
152 }
153 sys_close(fd)
154 sys_munmap(dbuf, DC_MAGIC_131072) // FREE per-dir buffer (leak-safe at scale)
155 return 0
156}
157
158func dc_streq(a: *u8, b: *u8) -> i64 { var i: i64 = 0; while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 } if b[i] != (0 as u8) { return 0 } return 1 }
159func dc_startswith(s: *u8, p: *u8) -> i64 { var i: i64 = 0; while p[i] != (0 as u8) { if s[i] != p[i] { return 0 } i = i + 1 } return 1 }
160func dc_contains(hay: *u8, needle: *u8) -> i64 {
161 let nl: i64 = dc_slen(needle); var i: i64 = 0
162 while hay[i] != (0 as u8) { var j: i64 = 0; var mt: i64 = 1; while j < nl { if hay[i + j] != needle[j] { mt = 0; j = nl } else { j = j + 1 } } if mt == 1 { return 1 } i = i + 1 }
163 return 0
164}
165// FAIL-CLOSED delete guard: only allow a path UNDER /volume1/<share>/<subpath> (depth>=2), never a bare share, an
166// @-system dir, or a path with '..'. So a typo can delete a handle-dir but NEVER a whole share or the system.
167func dc_pathguard(path: *u8) -> i64 {
168 if dc_startswith(path, "/volume1/" as *u8) == 0 { return 0 }
169 if dc_contains(path, "/@" as *u8) == 1 { return 0 }
170 if dc_contains(path, ".." as *u8) == 1 { return 0 }
171 var i: i64 = 9
172 var seg1_end: i64 = 0 - 1
173 var scan: i64 = 1
174 while scan == 1 {
175 if path[i] == (0 as u8) { scan = 0 } else { if path[i] == (47 as u8) { seg1_end = i; scan = 0 } else { i = i + 1 } }
176 }
177 if seg1_end < 0 { return 0 }
178 if path[seg1_end + 1] == (0 as u8) { return 0 }
179 return 1
180}
181// recursively delete the CONTENTS of path (post-order). exec=0 -> DRY-RUN (count only). st[0]=files st[1]=bytes st[2]=dirs.
182func dc_rm_contents(path: *u8, pn: i64, exec: i64, st: *i64, sb: *u8) -> i64 {
183 if pn > DC_MAGIC_3900 { return 0 }
184 path[pn] = 0 as u8
185 let fd: i64 = sys_openat_rd(path)
186 if fd < 0 { return 0 }
187 let dbuf: *u8 = sys_mmap(DC_MAGIC_131072)
188 var go: i64 = 1
189 while go == 1 {
190 let nr: i64 = sys_getdents64(fd, dbuf, DC_MAGIC_131072)
191 if nr <= 0 { go = 0 } else {
192 var off: i64 = 0
193 while off < nr {
194 let rec: *u8 = (dbuf as i64 + off) as *u8
195 let nm: *u8 = dirent_name(rec)
196 if dc_isdotdot(nm) == 0 {
197 let cn: i64 = dc_join(path, pn, nm)
198 path[cn] = 0 as u8
199 if sys_fstatat(path, sb) == 0 {
200 let mode: i64 = (sb[24] as i64) | ((sb[25] as i64) << 8)
201 let sp: *i64 = sb as *i64
202 let sz: i64 = sp[6]
203 if (mode & DC_MAGIC_61440) == DC_MAGIC_16384 {
204 dc_rm_contents(path, cn, exec, st, sb)
205 path[cn] = 0 as u8 // RESTORE: recursion mutated path beyond cn
206 if exec == 1 { __syscall(263, AT_FDCWD, path as i64, 512, 0, 0, 0) } // unlinkat AT_REMOVEDIR = rmdir
207 st[2] = st[2] + 1
208 } else {
209 st[0] = st[0] + 1; st[1] = st[1] + sz
210 if exec == 1 { __syscall(263, AT_FDCWD, path as i64, 0, 0, 0, 0) } // unlinkat = unlink file
211 }
212 }
213 }
214 off = off + dirent_reclen(rec)
215 }
216 }
217 }
218 sys_close(fd)
219 sys_munmap(dbuf, DC_MAGIC_131072)
220 return 0
221}
222func dc_del_main(argc: i64, argv: *i64) -> i64 {
223 if argc < 3 { dc_w("usage: nx_datacurator del <path-under-/volume1/share/> [YES-DELETE]\n" as *u8); return 2 }
224 let target: *u8 = argv[2] as *u8
225 if dc_pathguard(target) == 0 {
226 dc_w("REFUSED (fail-closed guard): only /volume1/<share>/<subpath> (depth>=2), no @system, no '..'\n path: " as *u8); dc_w(target); dc_w("\n" as *u8)
227 return 3
228 }
229 var exec: i64 = 0
230 if argc >= 4 { if dc_streq(argv[3] as *u8, "YES-DELETE" as *u8) == 1 { exec = 1 } }
231 let path: *u8 = sys_mmap(DC_MAGIC_4096); var pn: i64 = 0; var i: i64 = 0
232 while target[i] != (0 as u8) { path[pn] = target[i]; pn = pn + 1; i = i + 1 }
233 let sb: *u8 = sys_mmap(160)
234 let st: *i64 = sys_mmap(32) as *i64; st[0] = 0; st[1] = 0; st[2] = 0
235 if exec == 0 { dc_w("=== DRY-RUN -- nothing deleted. Would delete: ===\n " as *u8) } else { dc_w("=== DELETING (irreversible): ===\n " as *u8) }
236 dc_w(target); dc_w("\n" as *u8)
237 dc_rm_contents(path, pn, exec, st, sb)
238 path[pn] = 0 as u8 // RESTORE target path (recursion mutated it)
239 if exec == 1 { __syscall(263, AT_FDCWD, path as i64, 512, 0, 0, 0) } // rmdir the target itself
240 let rb: *u8 = sys_mmap(DC_MAGIC_2048); var o: i64 = 0
241 o = dc_bcat(rb, o, " files=" as *u8); o = dc_bcatn(rb, o, st[0])
242 o = dc_bcat(rb, o, " dirs=" as *u8); o = dc_bcatn(rb, o, st[2])
243 o = dc_bcat(rb, o, " size=" as *u8); o = dc_human(rb, o, st[1])
244 if exec == 1 { o = dc_bcat(rb, o, " ACTION=DELETED\n" as *u8) } else { o = dc_bcat(rb, o, " ACTION=DRYRUN (add YES-DELETE to execute)\n" as *u8) }
245 dc_flush(rb, o)
246 let afd: i64 = sys_openat_append("datacurator_audit.log" as *u8, 0x1a4)
247 if afd >= 0 {
248 var ao: i64 = 0
249 ao = dc_bcat(rb, ao, "DATACURATOR t=" as *u8); ao = dc_bcatn(rb, ao, sys_now_realtime_sec())
250 ao = dc_bcat(rb, ao, " files=" as *u8); ao = dc_bcatn(rb, ao, st[0])
251 ao = dc_bcat(rb, ao, " size_mb=" as *u8); ao = dc_bcatn(rb, ao, st[1] / DC_MAGIC_1048576)
252 if exec == 1 { ao = dc_bcat(rb, ao, " DELETED " as *u8) } else { ao = dc_bcat(rb, ao, " DRYRUN " as *u8) }
253 var ti: i64 = 0; while target[ti] != (0 as u8) { rb[ao] = target[ti]; ao = ao + 1; ti = ti + 1 }
254 rb[ao] = 10 as u8; ao = ao + 1
255 sys_write(afd, rb, ao); sys_close(afd)
256 }
257 return 0
258}
259
260// pointer to a filename's extension (after last '.'), or the whole name if none.
261func dc_ext(nm: *u8) -> *u8 {
262 var dot: i64 = 0 - 1; var i: i64 = 0
263 while nm[i] != (0 as u8) { if nm[i] == (46 as u8) { dot = i } i = i + 1 }
264 if dot < 0 { return nm }
265 return (nm as i64 + dot + 1) as *u8
266}
267// CHANNEL classification -> the native nishifamily channel/endpoint:
268// 0=media (SFW A/V -> media endpoint) 1=gallery (NSFW A/V -> /gallery opaque) 2=print3d (3D -> print endpoint)
269// 3=library (docs -> library/docportal + search index) 4=archive (WARC -> archive endpoint) 5=other
270// SFW vs NSFW is by PATH (the cam/scraped shares logs/logging/gallery/surveillance are NSFW; movies/music/photo are SFW).
271func dc_channel(nm: *u8, path: *u8) -> i64 {
272 let ext: *u8 = dc_ext(nm)
273 if dc_in_ci(ext, "stl 3mf obj gcode gco step stp ply amf scad" as *u8) == 1 { return 2 }
274 if dc_in_ci(ext, "warc arc wacz warc.gz" as *u8) == 1 { return 4 }
275 if dc_in_ci(ext, "pdf epub mobi azw3 doc docx txt md djvu rtf odt" as *u8) == 1 { return 3 }
276 if dc_in_ci(ext, "mp4 mkv avi mov webm ts m4v wmv flv mpg mpeg m2ts vob 3gp jpg jpeg png gif webp heic bmp tiff tif mp3 flac wav m4a aac ogg opus" as *u8) == 1 {
277 if dc_contains(path, "/logs/" as *u8) == 1 { return 1 }
278 if dc_contains(path, "/logging/" as *u8) == 1 { return 1 }
279 if dc_contains(path, "gallery" as *u8) == 1 { return 1 }
280 if dc_contains(path, "surveillance" as *u8) == 1 { return 1 }
281 return 0
282 }
283 return 5
284}
285func dc_chname(ch: i64) -> *u8 {
286 if ch == 0 { return "media " as *u8 }
287 if ch == 1 { return "gallery" as *u8 }
288 if ch == 2 { return "print3d" as *u8 }
289 if ch == 3 { return "library" as *u8 }
290 if ch == 4 { return "archive" as *u8 }
291 return "other " as *u8
292}
293// write one sortable catalog line per file: <size>\t<mtime>\t<channel>\t<path>\n (buffered). accumulate per-channel.
294func dc_catalog_walk(path: *u8, pn: i64, sb: *u8, fd: i64, wbuf: *u8, wo: *i64, chb: *i64, chc: *i64, tot: *i64) -> i64 {
295 if pn > DC_MAGIC_3900 { return 0 }
296 path[pn] = 0 as u8
297 let dfd: i64 = sys_openat_rd(path)
298 if dfd < 0 { return 0 }
299 let dbuf: *u8 = sys_mmap(DC_MAGIC_131072)
300 var go: i64 = 1
301 while go == 1 {
302 let nr: i64 = sys_getdents64(dfd, dbuf, DC_MAGIC_131072)
303 if nr <= 0 { go = 0 } else {
304 var off: i64 = 0
305 while off < nr {
306 let rec: *u8 = (dbuf as i64 + off) as *u8
307 let nm: *u8 = dirent_name(rec)
308 if dc_isdotdot(nm) == 0 {
309 let cn: i64 = dc_join(path, pn, nm)
310 path[cn] = 0 as u8
311 if sys_fstatat(path, sb) == 0 {
312 let mode: i64 = (sb[24] as i64) | ((sb[25] as i64) << 8)
313 let sp: *i64 = sb as *i64
314 if (mode & DC_MAGIC_61440) == DC_MAGIC_16384 {
315 dc_catalog_walk(path, cn, sb, fd, wbuf, wo, chb, chc, tot)
316 path[cn] = 0 as u8
317 } else {
318 if (mode & DC_MAGIC_61440) == DC_MAGIC_32768 {
319 let sz: i64 = sp[6]
320 let mt: i64 = sp[11]
321 let ch: i64 = dc_channel(nm, path)
322 chb[ch] = chb[ch] + sz; chc[ch] = chc[ch] + 1
323 tot[0] = tot[0] + 1; tot[1] = tot[1] + sz
324 if wo[0] > DC_MAGIC_60000 { sys_write(fd, wbuf, wo[0]); wo[0] = 0 }
325 var o: i64 = wo[0]
326 o = dc_bcatn(wbuf, o, sz); wbuf[o] = 9 as u8; o = o + 1
327 o = dc_bcatn(wbuf, o, mt); wbuf[o] = 9 as u8; o = o + 1
328 o = dc_bcat(wbuf, o, dc_chname(ch)); wbuf[o] = 9 as u8; o = o + 1
329 o = dc_bcat(wbuf, o, path); wbuf[o] = 10 as u8; o = o + 1
330 wo[0] = o
331 }
332 }
333 }
334 }
335 off = off + dirent_reclen(rec)
336 }
337 }
338 }
339 sys_close(dfd)
340 sys_munmap(dbuf, DC_MAGIC_131072)
341 return 0
342}
343func dc_catalog_main(argc: i64, argv: *i64) -> i64 {
344 if argc < 4 { dc_w("usage: nx_datacurator catalog <dir> <outfile.tsv>\n" as *u8); return 2 }
345 let dir: *u8 = argv[2] as *u8; let outf: *u8 = argv[3] as *u8
346 let path: *u8 = sys_mmap(DC_MAGIC_4096); var pn: i64 = 0; var i: i64 = 0
347 while dir[i] != (0 as u8) { path[pn] = dir[i]; pn = pn + 1; i = i + 1 }
348 let fd: i64 = sys_openat_wr(outf, 0x1a4)
349 if fd < 0 { dc_w("ERROR: cannot open outfile: " as *u8); dc_w(outf); dc_w("\n" as *u8); return 3 }
350 let sb: *u8 = sys_mmap(160)
351 let wbuf: *u8 = sys_mmap(DC_MAGIC_65536); let wo: *i64 = sys_mmap(16) as *i64; wo[0] = 0
352 let chb: *i64 = sys_mmap(DC_NCAT * 8) as *i64; let chc: *i64 = sys_mmap(DC_NCAT * 8) as *i64; let tot: *i64 = sys_mmap(16) as *i64
353 var c: i64 = 0; while c < DC_NCAT { chb[c] = 0; chc[c] = 0; c = c + 1 }
354 tot[0] = 0; tot[1] = 0
355 dc_w("=== nx_datacurator catalog: " as *u8); dc_w(dir); dc_w(" -> " as *u8); dc_w(outf); dc_w(" ===\n" as *u8)
356 dc_catalog_walk(path, pn, sb, fd, wbuf, wo, chb, chc, tot)
357 if wo[0] > 0 { sys_write(fd, wbuf, wo[0]) }
358 sys_close(fd)
359 let rb: *u8 = sys_mmap(DC_MAGIC_2048); var o: i64 = 0
360 o = dc_bcat(rb, o, "-- BY CHANNEL --\n" as *u8)
361 o = dc_bcat(rb, o, "media (SFW A/V) : " as *u8); o = dc_human(rb, o, chb[0]); o = dc_bcat(rb, o, " (" as *u8); o = dc_bcatn(rb, o, chc[0]); o = dc_bcat(rb, o, ")\n" as *u8)
362 o = dc_bcat(rb, o, "gallery (NSFW A/V): " as *u8); o = dc_human(rb, o, chb[1]); o = dc_bcat(rb, o, " (" as *u8); o = dc_bcatn(rb, o, chc[1]); o = dc_bcat(rb, o, ")\n" as *u8)
363 o = dc_bcat(rb, o, "print3d (3D print): " as *u8); o = dc_human(rb, o, chb[2]); o = dc_bcat(rb, o, " (" as *u8); o = dc_bcatn(rb, o, chc[2]); o = dc_bcat(rb, o, ")\n" as *u8)
364 o = dc_bcat(rb, o, "library (docs) : " as *u8); o = dc_human(rb, o, chb[3]); o = dc_bcat(rb, o, " (" as *u8); o = dc_bcatn(rb, o, chc[3]); o = dc_bcat(rb, o, ")\n" as *u8)
365 o = dc_bcat(rb, o, "archive (WARC) : " as *u8); o = dc_human(rb, o, chb[4]); o = dc_bcat(rb, o, " (" as *u8); o = dc_bcatn(rb, o, chc[4]); o = dc_bcat(rb, o, ")\n" as *u8)
366 o = dc_bcat(rb, o, "other : " as *u8); o = dc_human(rb, o, chb[5]); o = dc_bcat(rb, o, " (" as *u8); o = dc_bcatn(rb, o, chc[5]); o = dc_bcat(rb, o, ")\n" as *u8)
367 o = dc_bcat(rb, o, "TOTAL: " as *u8); o = dc_human(rb, o, tot[1]); o = dc_bcat(rb, o, " (" as *u8); o = dc_bcatn(rb, o, tot[0]); o = dc_bcat(rb, o, " files) cataloged\n" as *u8)
368 o = dc_bcat(rb, o, "sort by size desc: sort -t'\t' -k1 -rn " as *u8); o = dc_bcat(rb, o, outf); o = dc_bcat(rb, o, " | head -40\n" as *u8)
369 o = dc_bcat(rb, o, "filter a channel : grep -P '\tgallery\t' " as *u8); o = dc_bcat(rb, o, outf); o = dc_bcat(rb, o, " | sort -t'\t' -k1 -rn | head\n" as *u8)
370 dc_flush(rb, o)
371 return 0
372}
373
374func main(argc: i64, argv: *i64) -> i64 {
375 if argc >= 2 { if dc_streq(argv[1] as *u8, "del" as *u8) == 1 { return dc_del_main(argc, argv) } }
376 if argc >= 2 { if dc_streq(argv[1] as *u8, "catalog" as *u8) == 1 { return dc_catalog_main(argc, argv) } }
377 if argc < 2 { dc_w("usage: nx_datacurator <dir> [topN] | catalog <dir> <outfile.tsv> | del <path> [YES-DELETE]\n" as *u8); return 2 }
378 var nkeep: i64 = 30
379 if argc >= 3 { var v: i64 = 0; var i: i64 = 0; let a2: *u8 = argv[2] as *u8; while a2[i] != (0 as u8) { let c: i64 = a2[i] as i64; if c >= 48 { if c <= 57 { v = v * 10 + (c - 48) } } i = i + 1 } if v > 0 { nkeep = v } }
380 if nkeep > DC_TOPMAX { nkeep = DC_TOPMAX }
381
382 let path: *u8 = sys_mmap(DC_MAGIC_4096)
383 var pn: i64 = 0; var i: i64 = 0; let a1: *u8 = argv[1] as *u8
384 while a1[i] != (0 as u8) { path[pn] = a1[i]; pn = pn + 1; i = i + 1 }
385
386 let sb: *u8 = sys_mmap(160)
387 let cb: *i64 = sys_mmap(DC_NCAT * 8) as *i64
388 let cc: *i64 = sys_mmap(DC_NCAT * 8) as *i64
389 let bs: *i64 = sys_mmap(DC_TOPMAX * 8) as *i64
390 let bp: *u8 = sys_mmap(DC_TOPMAX * DC_PATHW)
391 let tot: *i64 = sys_mmap(16) as *i64
392 var c: i64 = 0; while c < DC_NCAT { cb[c] = 0; cc[c] = 0; c = c + 1 }
393 var s: i64 = 0; while s < nkeep { bs[s] = 0; s = s + 1 }
394 tot[0] = 0; tot[1] = 0
395
396 dc_w("=== nx_datacurator: " as *u8); dc_w(a1); dc_w(" ===\n" as *u8)
397 dc_walk(path, pn, sb, cb, cc, bs, bp, nkeep, tot)
398
399 // by-type report
400 let rb: *u8 = sys_mmap(DC_MAGIC_65536); var o: i64 = 0
401 let names: *u8 = sys_mmap(64) // scratch not used; keep explicit labels below
402 o = dc_bcat(rb, o, "-- BY MEDIA TYPE --\n" as *u8)
403 o = dc_bcat(rb, o, "video : " as *u8); o = dc_human(rb, o, cb[0]); o = dc_bcat(rb, o, " (" as *u8); o = dc_bcatn(rb, o, cc[0]); o = dc_bcat(rb, o, " files)\n" as *u8)
404 o = dc_bcat(rb, o, "image : " as *u8); o = dc_human(rb, o, cb[1]); o = dc_bcat(rb, o, " (" as *u8); o = dc_bcatn(rb, o, cc[1]); o = dc_bcat(rb, o, " files)\n" as *u8)
405 o = dc_bcat(rb, o, "audio : " as *u8); o = dc_human(rb, o, cb[2]); o = dc_bcat(rb, o, " (" as *u8); o = dc_bcatn(rb, o, cc[2]); o = dc_bcat(rb, o, " files)\n" as *u8)
406 o = dc_bcat(rb, o, "doc : " as *u8); o = dc_human(rb, o, cb[3]); o = dc_bcat(rb, o, " (" as *u8); o = dc_bcatn(rb, o, cc[3]); o = dc_bcat(rb, o, " files)\n" as *u8)
407 o = dc_bcat(rb, o, "archive : " as *u8); o = dc_human(rb, o, cb[4]); o = dc_bcat(rb, o, " (" as *u8); o = dc_bcatn(rb, o, cc[4]); o = dc_bcat(rb, o, " files)\n" as *u8)
408 o = dc_bcat(rb, o, "other : " as *u8); o = dc_human(rb, o, cb[5]); o = dc_bcat(rb, o, " (" as *u8); o = dc_bcatn(rb, o, cc[5]); o = dc_bcat(rb, o, " files)\n" as *u8)
409 o = dc_bcat(rb, o, "TOTAL : " as *u8); o = dc_human(rb, o, tot[1]); o = dc_bcat(rb, o, " (" as *u8); o = dc_bcatn(rb, o, tot[0]); o = dc_bcat(rb, o, " files)\n" as *u8)
410 dc_flush(rb, o)
411
412 // biggest N (selection-sort desc, in place)
413 dc_w("-- BIGGEST FILES --\n" as *u8)
414 var a: i64 = 0
415 while a < nkeep {
416 var bi: i64 = a; var bv: i64 = bs[a]; var k: i64 = a + 1
417 while k < nkeep { if bs[k] > bv { bv = bs[k]; bi = k } k = k + 1 }
418 if bi != a {
419 let ts: i64 = bs[a]; bs[a] = bs[bi]; bs[bi] = ts
420 let pa: *u8 = (bp as i64 + a * DC_PATHW) as *u8
421 let pbi: *u8 = (bp as i64 + bi * DC_PATHW) as *u8
422 var z: i64 = 0; while z < DC_PATHW { let tmp: i64 = pa[z] as i64; pa[z] = pbi[z]; pbi[z] = tmp as u8; z = z + 1 }
423 }
424 if bs[a] > 0 {
425 var ro: i64 = 0
426 ro = dc_human(rb, ro, bs[a]); rb[ro] = 9 as u8; ro = ro + 1
427 let pp: *u8 = (bp as i64 + a * DC_PATHW) as *u8
428 ro = dc_bcat(rb, ro, pp); rb[ro] = 10 as u8; ro = ro + 1
429 dc_flush(rb, ro)
430 }
431 a = a + 1
432 }
433 return 0
434}