nx_varcensus.nx source
↩ module page · 1365 lines · 63035 B
1// nx_varcensus.nx -- THE VaM CORPUS CENSUS SURFACE (laptop-local lane, 2026-09-11).
2//
3// nx_varcensus names <file.var> one row per central-directory record (the package's own manifest)
4// nx_varcensus meta <file.var> meta.json's DIRECT top-level members: key, kind, size; short strings
5// nx_varcensus census <dir> <outdir> <rules.conf> EVERY .var under <dir>, recursively, into TSV reports + a summary
6//
7// WHY A SECOND CLI BESIDE nx_varfacts: nx_varfacts is the refcorpus WRITER (it imports nx_refcorpus for rc_put_row and the
8// NAS plane); this surface runs where the corpus lives (the laptop) and writes reports. Both consume the ONE walk in
9// nx_varfacts_lib (vf_cd_locate / vf_cd_entry / vf_member_data / vf_json_*): no second zip reader and no second JSON reader
10// exist here, and the classifier is DATA (nx_varcensus_lib rules). ASSET BYTES ARE NEVER OPENED: the only member this organ
11// reads is meta.json, the package describing itself; images and video are never decoded, viewed or copied out.
12//
13// THE CENSUS CONTRACT (each asserted, each printed with its sum):
14// packages walked = ok + no_eocd + zip64_eocd + window + short + read_fail + record_refused + unopenable
15// for the OK packages: members = sum over role classes = sum over format classes; meta parsed + absent + unreadable +
16// unparsed = ok packages; licence label rows sum to ok packages. A package that is not OK is named in unreadable.tsv.
17// A meta string longer than VC_SHOW_STR_MAX prints its LENGTH only in the meta verb -- free text is not needed to learn a
18// schema, and the bound is announced on every row it applies to (shown=0), never silent.
19// license_tier: ORIGINAL No hw writes (Rule 26). expect_exit: 0
20import "nx_syscalls.nx"
21import "nx_varfacts_lib.nx"
22import "nx_varcensus_lib.nx"
23
24const VC_FD_OUT: i64 = 1
25const VC_FD_ERR: i64 = 2
26const VC_EXIT_OK: i64 = 0
27const VC_EXIT_IO: i64 = 1
28const VC_EXIT_USAGE: i64 = 2
29const VC_EXIT_MISMATCH: i64 = 3
30const VC_EXIT_REFUSED: i64 = 4
31const VC_SEEK_SET: i64 = 0
32const VC_SEEK_END: i64 = 2
33const VC_DEC: i64 = 10
34const VC_ASCII_0: i64 = 48
35const VC_ASCII_MINUS: i64 = 45
36const VC_PRINT_LO: i64 = 32
37const VC_DEL: i64 = 127
38const VC_QMARK: i64 = 63
39const VC_PAD: i64 = 16
40const VC_LINE_ROOM: i64 = 1024
41const VC_ST_N: i64 = 4
42const VC_ST_SIZE: i64 = 0
43const VC_ST_LOC: i64 = 1
44const VC_ST_WIN: i64 = 2
45const VC_ST_READFAIL: i64 = 9
46const VC_SHOW_STR_MAX: i64 = 80
47const VC_SHOW_ELEMS: i64 = 5
48const VC_MB_LEN: i64 = 0
49const VC_MB_STATE: i64 = 1
50const VC_MB_FOUND: i64 = 2
51const VC_MB_METHOD: i64 = 3
52const VC_MB_N: i64 = 4
53const VC_META_NAME: *u8 = "meta.json"
54const VC_DEPS_KEY: *u8 = "dependencies"
55const VC_CONTENT_KEY: *u8 = "contentList"
56const VC_LIC_KEY: *u8 = "licenseType"
57const VC_CREATOR_KEY: *u8 = "creatorName"
58const VC_PACKAGE_KEY: *u8 = "packageName"
59const VC_PROGVER_KEY: *u8 = "programVersion"
60
61// census bounds that are OS or format facts, not tunables
62const VC_PATH_MAX: i64 = 4096 // Linux PATH_MAX: a longer path cannot be opened by name at all (counted, never skipped silently)
63const VC_DENT_BUF: i64 = 65536 // one getdents64 batch; the walk loops until the call returns 0
64const VC_NAME_BUF: i64 = 65536 // a ZIP member name length is a u16, so no member name is longer
65const VC_STAT_BUF: i64 = 144
66const VC_S_MODE: i64 = 24
67const VC_S_IFMT: i64 = 0xf000
68const VC_S_IFDIR: i64 = 0x4000
69const VC_S_IFREG: i64 = 0x8000
70const VC_DT_UNKNOWN: i64 = 0
71const VC_FLAG_UTF8: i64 = 2048
72const VC_LEGACY_LIC: i64 = 64
73const VC_NLETTERS: i64 = 26
74const VC_ASCII_UPPER_A: i64 = 65
75const VC_TAB: i64 = 9
76const VC_LF: i64 = 10
77const VC_SLASH: i64 = 47
78const VC_DOT: i64 = 46
79const VC_ROW_SEP: i64 = 44
80const VC_EQ: i64 = 61
81// package states: the partition of the packages walked
82const VC_P_OK: i64 = 0
83const VC_P_NOEOCD: i64 = 1
84const VC_P_ZIP64: i64 = 2
85const VC_P_WINDOW: i64 = 3
86const VC_P_SHORT: i64 = 4
87const VC_P_READFAIL: i64 = 5
88const VC_P_RECORD: i64 = 6
89const VC_P_UNOPENABLE: i64 = 7
90// a NO-EOCD package is split by what the file IS -- MEASURED 2026-09-11: the first census read 206 NO-EOCD rows and the first
91// 40 were all size=0. EMPTY is a zero-byte file; NO-EOCD-PK-HEAD carries a ZIP local header at byte 0 but no EOCD in its
92// tail window (an incomplete or damaged archive); NO-EOCD-OTHER-HEAD is not a ZIP at all. Plain NO-EOCD stays in the
93// partition and must read 0 once the split runs.
94const VC_P_EMPTY: i64 = 8
95const VC_P_NOEOCD_PK: i64 = 9
96const VC_P_NOEOCD_OTHER: i64 = 10
97const VC_P_N: i64 = 11
98const VC_HEAD_LEN: i64 = 4
99// meta states: the partition of the OK packages
100const VC_M_PARSED: i64 = 0
101const VC_M_ABSENT: i64 = 1
102const VC_M_UNREADABLE: i64 = 2
103const VC_M_UNPARSED: i64 = 3
104const VC_M_N: i64 = 4
105// walk stats
106const VC_W_DIRS: i64 = 0
107const VC_W_DIRS_UNREADABLE: i64 = 1
108const VC_W_VARS: i64 = 2
109const VC_W_VARS_TOP: i64 = 3
110const VC_W_OTHER_FILES: i64 = 4
111const VC_W_SYMLINKS: i64 = 5
112const VC_W_OTHER_TYPES: i64 = 6
113const VC_W_TOO_LONG: i64 = 7
114const VC_W_N: i64 = 8
115// licence reading agreement (legacy first-occurrence reader vs the depth-aware direct member)
116const VC_L_AGREE: i64 = 0
117const VC_L_DISAGREE: i64 = 1
118const VC_L_LEGACY_ONLY: i64 = 2
119const VC_L_DIRECT_ONLY: i64 = 3
120const VC_L_NEITHER: i64 = 4
121const VC_L_N: i64 = 5
122
123func vc_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
124func vc_streq(a: *u8, b: *u8) -> i64 {
125 var i: i64 = 0
126 while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 }
127 if b[i] != (0 as u8) { return 0 }
128 return 1
129}
130func vc_wn(fd: i64, s: *u8, n: i64) -> i64 {
131 var done: i64 = 0
132 var run: i64 = 1
133 while run == 1 {
134 if done >= n { run = 0 } else {
135 let k: i64 = sys_write(fd, ((s as i64) + done) as *u8, n - done)
136 if k <= 0 { run = 0 } else { done = done + k }
137 }
138 }
139 return done
140}
141func vc_w(fd: i64, s: *u8) -> i64 { return vc_wn(fd, s, vc_slen(s)) }
142func vc_cat(b: *u8, o: i64, s: *u8) -> i64 {
143 var i: i64 = 0
144 while s[i] != (0 as u8) { b[o+i] = s[i]; i = i + 1 }
145 return o + i
146}
147func vc_catn(b: *u8, o: i64, v: i64) -> i64 {
148 var p: i64 = o
149 var m: i64 = v
150 if m < 0 { b[p] = VC_ASCII_MINUS as u8; p = p + 1; m = 0 - m }
151 if m == 0 { b[p] = VC_ASCII_0 as u8; return p + 1 }
152 let s: i64 = p
153 while m > 0 { b[p] = (VC_ASCII_0 + (m % VC_DEC)) as u8; m = m / VC_DEC; p = p + 1 }
154 var i: i64 = s
155 var j: i64 = p - 1
156 while i < j { let t: u8 = b[i]; b[i] = b[j]; b[j] = t; i = i + 1; j = j - 1 }
157 return p
158}
159// copy n bytes of src starting at s; control bytes and DEL become '?' so a name can never tear a row
160func vc_catb(b: *u8, o: i64, src: *u8, s: i64, n: i64) -> i64 {
161 var i: i64 = 0
162 while i < n {
163 let c: i64 = src[s+i] as i64 & 0xff
164 var ok: i64 = 1
165 if c < VC_PRINT_LO { ok = 0 }
166 if c == VC_DEL { ok = 0 }
167 if ok == 1 { b[o+i] = c as u8 } else { b[o+i] = VC_QMARK as u8 }
168 i = i + 1
169 }
170 return o + n
171}
172
173func vc_pread(fd: i64, off: i64, len: i64, buf: *u8) -> i64 {
174 if sys_lseek(fd, off, VC_SEEK_SET) != off { return 0 - 1 }
175 var got: i64 = 0
176 var run: i64 = 1
177 while run == 1 {
178 if got >= len { run = 0 } else {
179 let k: i64 = sys_read(fd, ((buf as i64) + got) as *u8, len - got)
180 if k <= 0 { run = 0 } else { got = got + k }
181 }
182 }
183 return got
184}
185
186func vc_state_name(s: i64) -> *u8 {
187 if s == VF_LOC_OK { return "OK" as *u8 }
188 if s == VF_LOC_NOEOCD { return "NO-EOCD" as *u8 }
189 if s == VF_LOC_ZIP64 { return "ZIP64" as *u8 }
190 if s == VF_LOC_WINDOW { return "WINDOW" as *u8 }
191 if s == VF_LOC_SHORT { return "SHORT" as *u8 }
192 if s == VC_ST_READFAIL { return "READ-FAIL" as *u8 }
193 return "UNNAMED" as *u8
194}
195func vc_pstate_name(s: i64) -> *u8 {
196 if s == VC_P_OK { return "OK" as *u8 }
197 if s == VC_P_NOEOCD { return "NO-EOCD" as *u8 }
198 if s == VC_P_ZIP64 { return "ZIP64-EOCD" as *u8 }
199 if s == VC_P_WINDOW { return "WINDOW" as *u8 }
200 if s == VC_P_SHORT { return "SHORT" as *u8 }
201 if s == VC_P_READFAIL { return "READ-FAIL" as *u8 }
202 if s == VC_P_RECORD { return "RECORD-REFUSED" as *u8 }
203 if s == VC_P_UNOPENABLE { return "UNOPENABLE" as *u8 }
204 if s == VC_P_EMPTY { return "EMPTY" as *u8 }
205 if s == VC_P_NOEOCD_PK { return "NO-EOCD-PK-HEAD" as *u8 }
206 if s == VC_P_NOEOCD_OTHER { return "NO-EOCD-OTHER-HEAD" as *u8 }
207 return "UNNAMED" as *u8
208}
209
210// does counter entry x rank before entry y? the higher count first, then the byte-smaller key
211func vc_rank_before(c: *i64, x: i64, y: i64) -> i64 {
212 let cx: i64 = vcl_counter_count(c, x)
213 let cy: i64 = vcl_counter_count(c, y)
214 if cx > cy { return 1 }
215 if cx < cy { return 0 }
216 if vc_cmp(vcl_counter_key(c, x), vcl_counter_key(c, y)) < 0 { return 1 }
217 return 0
218}
219// EVERY entry of a counter as TSV rows ranked by count (Shell sort of the entry indices): the whole table ranked, never a
220// top-N cut, so a reader taking the head of the file is reading a labelled head of a complete population
221func vc_counter_sorted_tsv(a: *i64, c: *i64) -> i64 {
222 let n: i64 = vcl_counter_n(c)
223 if n == 0 { return 0 }
224 let idx: *i64 = sys_mmap(n * 8) as *i64
225 var i: i64 = 0
226 while i < n { idx[i] = i; i = i + 1 }
227 var gap: i64 = n / 2
228 while gap > 0 {
229 var k: i64 = gap
230 while k < n {
231 let t: i64 = idx[k]
232 var j: i64 = k
233 var go: i64 = 1
234 while go == 1 {
235 if j < gap { go = 0 } else {
236 if vc_rank_before(c, t, idx[j - gap]) == 1 { idx[j] = idx[j - gap]; j = j - gap } else { go = 0 }
237 }
238 }
239 idx[j] = t
240 k = k + 1
241 }
242 gap = gap / 2
243 }
244 i = 0
245 while i < n {
246 vcl_appclean(a, vcl_counter_key(c, idx[i]), 0, vcl_counter_klen(c, idx[i]))
247 vcl_appc(a, VC_TAB)
248 vcl_appn(a, vcl_counter_count(c, idx[i]))
249 vcl_appc(a, VC_LF)
250 i = i + 1
251 }
252 sys_munmap(idx as *u8, n * 8)
253 return n
254}
255func vc_pstate_of(loc: i64) -> i64 {
256 if loc == VF_LOC_OK { return VC_P_OK }
257 if loc == VF_LOC_NOEOCD { return VC_P_NOEOCD }
258 if loc == VF_LOC_ZIP64 { return VC_P_ZIP64 }
259 if loc == VF_LOC_WINDOW { return VC_P_WINDOW }
260 if loc == VF_LOC_SHORT { return VC_P_SHORT }
261 return VC_P_READFAIL
262}
263func vc_md_name(s: i64) -> *u8 {
264 if s == VF_MD_STORED { return "STORED" as *u8 }
265 if s == VF_MD_INFLATED { return "INFLATED" as *u8 }
266 if s == VF_MD_UNSUPPORTED { return "UNSUPPORTED" as *u8 }
267 return "FAILED" as *u8
268}
269func vc_kind_name(k: i64) -> *u8 {
270 if k == VF_JK_STRING { return "string" as *u8 }
271 if k == VF_JK_OBJECT { return "object" as *u8 }
272 if k == VF_JK_ARRAY { return "array" as *u8 }
273 if k == VF_JK_SCALAR { return "scalar" as *u8 }
274 return "bad" as *u8
275}
276
277// THE WINDOW: read the EOCD window from the end; when the central directory starts before it, read exactly
278// [cd_abs, size) -- a window derived from the archive, never a guessed tail. st: size, locate state, window length.
279// Returns the mapping (release with sys_munmap(win, st[VC_ST_WIN] + VC_PAD)) or 0.
280func vc_window(fd: i64, size: i64, box: *i64, st: *i64) -> *u8 {
281 st[VC_ST_SIZE] = size
282 st[VC_ST_LOC] = VC_ST_READFAIL
283 st[VC_ST_WIN] = 0
284 let w0: i64 = vf_eocd_window(size)
285 if w0 <= 0 { st[VC_ST_LOC] = VF_LOC_NOEOCD; return 0 as *u8 }
286 let b0: *u8 = sys_mmap(w0 + VC_PAD)
287 if vc_pread(fd, size - w0, w0, b0) != w0 { sys_munmap(b0, w0 + VC_PAD); return 0 as *u8 }
288 let s0: i64 = vf_cd_locate(b0, w0, size, box)
289 st[VC_ST_LOC] = s0
290 st[VC_ST_WIN] = w0
291 if s0 != VF_LOC_WINDOW { return b0 }
292 let need: i64 = size - box[VF_L_CDABS]
293 sys_munmap(b0, w0 + VC_PAD)
294 st[VC_ST_WIN] = 0
295 if need <= w0 { st[VC_ST_LOC] = VF_LOC_SHORT; return 0 as *u8 }
296 if need > size { st[VC_ST_LOC] = VF_LOC_SHORT; return 0 as *u8 }
297 let b1: *u8 = sys_mmap(need + VC_PAD)
298 if vc_pread(fd, size - need, need, b1) != need { st[VC_ST_LOC] = VC_ST_READFAIL; sys_munmap(b1, need + VC_PAD); return 0 as *u8 }
299 st[VC_ST_LOC] = vf_cd_locate(b1, need, size, box)
300 st[VC_ST_WIN] = need
301 return b1
302}
303
304// meta.json of the package whose CD window is win. mbox: decoded length, vf_member_data state, found (1 found / 0 absent /
305// -1 unreadable), method. Returns the decoded bytes or 0. A member whose sizes could not be resolved (ZIP64 extra missing)
306// is UNREADABLE by name, never read at a 4 GiB size.
307func vc_meta_read(fd: i64, size: i64, win: *u8, winn: i64, box: *i64, mbox: *i64) -> *u8 {
308 mbox[VC_MB_LEN] = 0
309 mbox[VC_MB_STATE] = VF_MD_FAILED
310 mbox[VC_MB_FOUND] = 0
311 mbox[VC_MB_METHOD] = 0 - 1
312 let rec: *i64 = sys_mmap(VF_E_N * 8) as *i64
313 let entries: i64 = box[VF_L_ENTRIES]
314 var off: i64 = box[VF_L_CD]
315 var k: i64 = 0
316 var lho: i64 = 0 - 1
317 var method: i64 = 0 - 1
318 var csize: i64 = 0
319 var usize: i64 = 0
320 var run: i64 = 1
321 while run == 1 {
322 if k >= entries { run = 0 } else {
323 if vf_cd_entry(win, winn, off, rec) != 1 { mbox[VC_MB_FOUND] = 0 - 1; return 0 as *u8 }
324 let ns: i64 = rec[VF_E_NAMEOFF]
325 if vf_span_is(win, ns, ns + rec[VF_E_NAMELEN], VC_META_NAME) == 1 {
326 if rec[VF_E_Z64] == VF_Z64_UNRESOLVED { mbox[VC_MB_FOUND] = 0 - 1; return 0 as *u8 }
327 lho = rec[VF_E_LHO]
328 method = rec[VF_E_METHOD]
329 csize = rec[VF_E_CSIZE]
330 usize = rec[VF_E_USIZE]
331 run = 0
332 } else {
333 off = rec[VF_E_NEXT]
334 k = k + 1
335 }
336 }
337 }
338 if lho < 0 { return 0 as *u8 }
339 mbox[VC_MB_FOUND] = 1
340 mbox[VC_MB_METHOD] = method
341 let hdr: *u8 = sys_mmap(VF_LOCAL_LEN + VC_PAD)
342 if vc_pread(fd, lho, VF_LOCAL_LEN, hdr) != VF_LOCAL_LEN { mbox[VC_MB_FOUND] = 0 - 1; return 0 as *u8 }
343 if vf_u32(hdr, 0) != VF_SIG_LOCAL { mbox[VC_MB_FOUND] = 0 - 1; return 0 as *u8 }
344 let doff: i64 = lho + VF_LOCAL_LEN + vf_u16(hdr, VF_LOCAL_FNL) + vf_u16(hdr, VF_LOCAL_EXL)
345 if doff + csize > size { mbox[VC_MB_FOUND] = 0 - 1; return 0 as *u8 }
346 let body: *u8 = sys_mmap(csize + VC_PAD)
347 if vc_pread(fd, doff, csize, body) != csize { mbox[VC_MB_FOUND] = 0 - 1; sys_munmap(body, csize + VC_PAD); return 0 as *u8 }
348 let mdo: *i64 = sys_mmap(16) as *i64
349 let data: *u8 = vf_member_data(body, csize, method, usize, mdo)
350 mbox[VC_MB_LEN] = mdo[0]
351 mbox[VC_MB_STATE] = mdo[1]
352 if mdo[1] == VF_MD_INFLATED { sys_munmap(body, csize + VC_PAD) }
353 return data
354}
355
356func vc_names(path: *u8) -> i64 {
357 let fd: i64 = sys_openat_rd(path)
358 if fd < 0 { vc_w(VC_FD_OUT, "VARCENSUS-NAMES state=UNOPENABLE\n" as *u8); return VC_EXIT_IO }
359 let size: i64 = sys_lseek(fd, 0, VC_SEEK_END)
360 let box: *i64 = sys_mmap(VF_L_N * 8) as *i64
361 let st: *i64 = sys_mmap(VC_ST_N * 8) as *i64
362 let win: *u8 = vc_window(fd, size, box, st)
363 sys_close(fd)
364 let line: *u8 = sys_mmap(VF_U16_ALL + VC_LINE_ROOM)
365 if st[VC_ST_LOC] != VF_LOC_OK {
366 var o: i64 = vc_cat(line, 0, "VARCENSUS-NAMES state=" as *u8)
367 o = vc_cat(line, o, vc_state_name(st[VC_ST_LOC]))
368 o = vc_cat(line, o, " size=" as *u8)
369 o = vc_catn(line, o, size)
370 o = vc_cat(line, o, "\n" as *u8)
371 vc_wn(VC_FD_OUT, line, o)
372 return VC_EXIT_REFUSED
373 }
374 let rec: *i64 = sys_mmap(VF_E_N * 8) as *i64
375 let entries: i64 = box[VF_L_ENTRIES]
376 var off: i64 = box[VF_L_CD]
377 var k: i64 = 0
378 var usum: i64 = 0
379 var csum: i64 = 0
380 var z64r: i64 = 0
381 var z64u: i64 = 0
382 while k < entries {
383 if vf_cd_entry(win, st[VC_ST_WIN], off, rec) != 1 {
384 var o3: i64 = vc_cat(line, 0, "VARCENSUS-NAMES state=RECORD-REFUSED at=" as *u8)
385 o3 = vc_catn(line, o3, k)
386 o3 = vc_cat(line, o3, "\n" as *u8)
387 vc_wn(VC_FD_OUT, line, o3)
388 return VC_EXIT_REFUSED
389 }
390 var o2: i64 = vc_cat(line, 0, "M " as *u8)
391 o2 = vc_catn(line, o2, k)
392 o2 = vc_cat(line, o2, " m=" as *u8)
393 o2 = vc_catn(line, o2, rec[VF_E_METHOD])
394 o2 = vc_cat(line, o2, " c=" as *u8)
395 o2 = vc_catn(line, o2, rec[VF_E_CSIZE])
396 o2 = vc_cat(line, o2, " u=" as *u8)
397 o2 = vc_catn(line, o2, rec[VF_E_USIZE])
398 o2 = vc_cat(line, o2, " f=" as *u8)
399 o2 = vc_catn(line, o2, rec[VF_E_FLAGS])
400 o2 = vc_cat(line, o2, " z64=" as *u8)
401 o2 = vc_catn(line, o2, rec[VF_E_Z64])
402 o2 = vc_cat(line, o2, " " as *u8)
403 o2 = vc_catb(line, o2, win, rec[VF_E_NAMEOFF], rec[VF_E_NAMELEN])
404 o2 = vc_cat(line, o2, "\n" as *u8)
405 vc_wn(VC_FD_OUT, line, o2)
406 if rec[VF_E_Z64] == VF_Z64_RESOLVED { z64r = z64r + 1 }
407 if rec[VF_E_Z64] == VF_Z64_UNRESOLVED { z64u = z64u + 1 } else {
408 usum = usum + rec[VF_E_USIZE]
409 csum = csum + rec[VF_E_CSIZE]
410 }
411 off = rec[VF_E_NEXT]
412 k = k + 1
413 }
414 var o4: i64 = vc_cat(line, 0, "VARCENSUS-NAMES state=OK entries=" as *u8)
415 o4 = vc_catn(line, o4, entries)
416 o4 = vc_cat(line, o4, " size=" as *u8)
417 o4 = vc_catn(line, o4, size)
418 o4 = vc_cat(line, o4, " window=" as *u8)
419 o4 = vc_catn(line, o4, st[VC_ST_WIN])
420 o4 = vc_cat(line, o4, " cd_abs=" as *u8)
421 o4 = vc_catn(line, o4, box[VF_L_CDABS])
422 o4 = vc_cat(line, o4, " z64_resolved=" as *u8)
423 o4 = vc_catn(line, o4, z64r)
424 o4 = vc_cat(line, o4, " z64_unresolved=" as *u8)
425 o4 = vc_catn(line, o4, z64u)
426 o4 = vc_cat(line, o4, " usize_sum=" as *u8)
427 o4 = vc_catn(line, o4, usum)
428 o4 = vc_cat(line, o4, " csize_sum=" as *u8)
429 o4 = vc_catn(line, o4, csum)
430 o4 = vc_cat(line, o4, "\n" as *u8)
431 vc_wn(VC_FD_OUT, line, o4)
432 sys_munmap(win, st[VC_ST_WIN] + VC_PAD)
433 return VC_EXIT_OK
434}
435
436func vc_meta(path: *u8) -> i64 {
437 let fd: i64 = sys_openat_rd(path)
438 if fd < 0 { vc_w(VC_FD_OUT, "VARCENSUS-META state=UNOPENABLE\n" as *u8); return VC_EXIT_IO }
439 let size: i64 = sys_lseek(fd, 0, VC_SEEK_END)
440 let box: *i64 = sys_mmap(VF_L_N * 8) as *i64
441 let st: *i64 = sys_mmap(VC_ST_N * 8) as *i64
442 let win: *u8 = vc_window(fd, size, box, st)
443 let line: *u8 = sys_mmap(VF_U16_ALL + VC_LINE_ROOM)
444 if st[VC_ST_LOC] != VF_LOC_OK {
445 sys_close(fd)
446 var o: i64 = vc_cat(line, 0, "VARCENSUS-META window=" as *u8)
447 o = vc_cat(line, o, vc_state_name(st[VC_ST_LOC]))
448 o = vc_cat(line, o, "\n" as *u8)
449 vc_wn(VC_FD_OUT, line, o)
450 return VC_EXIT_REFUSED
451 }
452 let mbox: *i64 = sys_mmap(VC_MB_N * 8) as *i64
453 let data: *u8 = vc_meta_read(fd, size, win, st[VC_ST_WIN], box, mbox)
454 sys_close(fd)
455 let dn: i64 = mbox[VC_MB_LEN]
456 var o1: i64 = vc_cat(line, 0, "VARCENSUS-META found=" as *u8)
457 o1 = vc_catn(line, o1, mbox[VC_MB_FOUND])
458 o1 = vc_cat(line, o1, " method=" as *u8)
459 o1 = vc_catn(line, o1, mbox[VC_MB_METHOD])
460 o1 = vc_cat(line, o1, " md=" as *u8)
461 o1 = vc_cat(line, o1, vc_md_name(mbox[VC_MB_STATE]))
462 o1 = vc_cat(line, o1, " bytes=" as *u8)
463 o1 = vc_catn(line, o1, dn)
464 o1 = vc_cat(line, o1, "\n" as *u8)
465 vc_wn(VC_FD_OUT, line, o1)
466 if (data as i64) == 0 { return VC_EXIT_REFUSED }
467 let top: i64 = vf_json_skip(data, dn, 0)
468 if vf_json_kind(data, dn, top) != VF_JK_OBJECT { vc_w(VC_FD_OUT, "VARCENSUS-META json=NOT-OBJECT\n" as *u8); return VC_EXIT_REFUSED }
469 let end: i64 = vf_json_value_end(data, dn, top)
470 if end < 0 { vc_w(VC_FD_OUT, "VARCENSUS-META json=UNTERMINATED\n" as *u8); return VC_EXIT_REFUSED }
471 let mb: *i64 = sys_mmap(VF_JM_N * 8) as *i64
472 let db: *i64 = sys_mmap(VF_JM_N * 8) as *i64
473 let lenb: *i64 = sys_mmap(16) as *i64
474 var p: i64 = vf_json_skip(data, end, top + 1)
475 var members: i64 = 0
476 var run: i64 = 1
477 while run == 1 {
478 if p >= end - 1 { run = 0 } else {
479 let nx: i64 = vf_json_member(data, end, p, mb)
480 if nx < 0 { run = 0; vc_w(VC_FD_OUT, "VARCENSUS-META json=MALFORMED-MEMBER\n" as *u8) } else {
481 members = members + 1
482 let vs: i64 = mb[VF_JM_VS]
483 let ve: i64 = mb[VF_JM_VE]
484 let kind: i64 = vf_json_kind(data, dn, vs)
485 var o2: i64 = vc_cat(line, 0, "K " as *u8)
486 o2 = vc_catb(line, o2, data, mb[VF_JM_KS], mb[VF_JM_KE] - mb[VF_JM_KS])
487 o2 = vc_cat(line, o2, " kind=" as *u8)
488 o2 = vc_cat(line, o2, vc_kind_name(kind))
489 o2 = vc_cat(line, o2, " span=" as *u8)
490 o2 = vc_catn(line, o2, ve - vs)
491 if kind == VF_JK_STRING {
492 let sb: *u8 = sys_mmap(ve - vs + 1)
493 let sl: i64 = vf_json_decode(data, vs + 1, ve - 1, sb)
494 o2 = vc_cat(line, o2, " len=" as *u8)
495 o2 = vc_catn(line, o2, sl)
496 if sl >= 0 { if sl <= VC_SHOW_STR_MAX { o2 = vc_cat(line, o2, " value=" as *u8); o2 = vc_catb(line, o2, sb, 0, sl) } else { o2 = vc_cat(line, o2, " shown=0" as *u8) } }
497 }
498 if kind == VF_JK_ARRAY {
499 o2 = vc_cat(line, o2, " elems=" as *u8)
500 o2 = vc_catn(line, o2, vf_json_count_elems(data, dn, vs))
501 }
502 if kind == VF_JK_OBJECT {
503 o2 = vc_cat(line, o2, " members=" as *u8)
504 o2 = vc_catn(line, o2, vf_json_count_members(data, dn, vs))
505 }
506 if kind == VF_JK_SCALAR {
507 o2 = vc_cat(line, o2, " raw=" as *u8)
508 o2 = vc_catb(line, o2, data, vs, ve - vs)
509 }
510 o2 = vc_cat(line, o2, "\n" as *u8)
511 vc_wn(VC_FD_OUT, line, o2)
512 if kind == VF_JK_OBJECT { if vf_span_is(data, mb[VF_JM_KS], mb[VF_JM_KE], VC_DEPS_KEY) == 1 {
513 let dend: i64 = ve
514 var q: i64 = vf_json_skip(data, dend, vs + 1)
515 var drun: i64 = 1
516 while drun == 1 {
517 if q >= dend - 1 { drun = 0 } else {
518 let qn: i64 = vf_json_member(data, dend, q, db)
519 if qn < 0 { drun = 0 } else {
520 var o5: i64 = vc_cat(line, 0, " D " as *u8)
521 o5 = vc_catb(line, o5, data, db[VF_JM_KS], db[VF_JM_KE] - db[VF_JM_KS])
522 let dk: i64 = vf_json_kind(data, dn, db[VF_JM_VS])
523 o5 = vc_cat(line, o5, " kind=" as *u8)
524 o5 = vc_cat(line, o5, vc_kind_name(dk))
525 if dk == VF_JK_OBJECT {
526 o5 = vc_cat(line, o5, " members=" as *u8)
527 o5 = vc_catn(line, o5, vf_json_count_members(data, dn, db[VF_JM_VS]))
528 let lic: *u8 = vf_json_get_str(data, dn, db[VF_JM_VS], VC_LIC_KEY, lenb)
529 if (lic as i64) != 0 { if lenb[0] <= VC_SHOW_STR_MAX { o5 = vc_cat(line, o5, " dep_licenseType=" as *u8); o5 = vc_catb(line, o5, lic, 0, lenb[0]) } }
530 }
531 o5 = vc_cat(line, o5, "\n" as *u8)
532 vc_wn(VC_FD_OUT, line, o5)
533 q = qn
534 }
535 }
536 }
537 } }
538 if kind == VF_JK_ARRAY { if vf_span_is(data, mb[VF_JM_KS], mb[VF_JM_KE], VC_CONTENT_KEY) == 1 {
539 let aend: i64 = ve
540 var r: i64 = vf_json_skip(data, aend, vs + 1)
541 var shown: i64 = 0
542 var arun: i64 = 1
543 while arun == 1 {
544 if r >= aend - 1 { arun = 0 } else { if shown >= VC_SHOW_ELEMS { arun = 0 } else {
545 let re: i64 = vf_json_value_end(data, aend, r)
546 if re < 0 { arun = 0 } else {
547 var o6: i64 = vc_cat(line, 0, " C " as *u8)
548 if vf_json_kind(data, dn, r) == VF_JK_STRING {
549 let cb: *u8 = sys_mmap(re - r + 1)
550 let cl: i64 = vf_json_decode(data, r + 1, re - 1, cb)
551 if cl >= 0 { o6 = vc_catb(line, o6, cb, 0, cl) }
552 } else { o6 = vc_cat(line, o6, "(non-string)" as *u8) }
553 o6 = vc_cat(line, o6, "\n" as *u8)
554 vc_wn(VC_FD_OUT, line, o6)
555 shown = shown + 1
556 var rq: i64 = vf_json_skip(data, aend, re)
557 if rq < aend { if (data[rq] as i64 & 0xff) == VF_J_COMMA { rq = vf_json_skip(data, aend, rq + 1) } }
558 if rq <= r { arun = 0 } else { r = rq }
559 }
560 } }
561 }
562 } }
563 p = nx
564 }
565 }
566 }
567 let legacy: *u8 = sys_mmap(VC_LEGACY_LIC)
568 let lr: i64 = vf_license(data, dn, 0, dn, legacy)
569 let dlen: *i64 = sys_mmap(16) as *i64
570 let dlic: *u8 = vf_json_get_str(data, dn, top, VC_LIC_KEY, dlen)
571 var o7: i64 = vc_cat(line, 0, "VARCENSUS-META members=" as *u8)
572 o7 = vc_catn(line, o7, members)
573 o7 = vc_cat(line, o7, " legacy_license=" as *u8)
574 if lr > 0 { o7 = vc_cat(line, o7, legacy) } else { o7 = vc_cat(line, o7, "unparsed" as *u8) }
575 o7 = vc_cat(line, o7, " direct_license=" as *u8)
576 if (dlic as i64) != 0 { if dlen[0] <= VC_SHOW_STR_MAX { o7 = vc_catb(line, o7, dlic, 0, dlen[0]) } } else { o7 = vc_cat(line, o7, "absent" as *u8) }
577 o7 = vc_cat(line, o7, "\n" as *u8)
578 vc_wn(VC_FD_OUT, line, o7)
579 return VC_EXIT_OK
580}
581
582// ---------------------------------------------------------------- census ----------------------------------------------------------------
583
584func vc_cmp(a: *u8, b: *u8) -> i64 {
585 var i: i64 = 0
586 var run: i64 = 1
587 while run == 1 {
588 let ca: i64 = a[i] as i64 & 0xff
589 let cb: i64 = b[i] as i64 & 0xff
590 if ca != cb { if ca < cb { return 0 - 1 } return 1 }
591 if ca == 0 { run = 0 } else { i = i + 1 }
592 }
593 return 0
594}
595// Shell sort of the path offsets by the bytes of their paths: the report reads in one deterministic order on every run
596func vc_sort(paths: *i64, offs: *i64) -> i64 {
597 let n: i64 = vcl_vec_n(offs)
598 let v: *i64 = offs[VCL_A_PTR] as *i64
599 var gap: i64 = n / 2
600 while gap > 0 {
601 var i: i64 = gap
602 while i < n {
603 let t: i64 = v[i]
604 var j: i64 = i
605 var go: i64 = 1
606 while go == 1 {
607 if j < gap { go = 0 } else {
608 if vc_cmp(vcl_at(paths, v[j - gap]), vcl_at(paths, t)) > 0 { v[j] = v[j - gap]; j = j - gap } else { go = 0 }
609 }
610 }
611 v[j] = t
612 i = i + 1
613 }
614 gap = gap / 2
615 }
616 return n
617}
618
619// the lowercased extension of the basename of s[0..n): the bytes after its last dot, or "(none)"
620func vc_ext_key(s: *u8, n: i64, out: *u8) -> i64 {
621 var slash: i64 = 0 - 1
622 var dot: i64 = 0 - 1
623 var i: i64 = 0
624 while i < n {
625 let c: i64 = s[i] as i64 & 0xff
626 if c == VC_SLASH { slash = i; dot = 0 - 1 }
627 if c == VC_DOT { dot = i }
628 i = i + 1
629 }
630 if dot < 0 { return vc_cat(out, 0, "(none)" as *u8) }
631 if dot <= slash { return vc_cat(out, 0, "(none)" as *u8) }
632 var k: i64 = 0
633 var j: i64 = dot
634 while j < n { out[k] = vcl_lc(s[j] as i64 & 0xff) as u8; k = k + 1; j = j + 1 }
635 out[k] = 0 as u8
636 return k
637}
638// the first two path segments of a lowercased member name ("(root)" for a member with no slash): the residual worklist key
639func vc_prefix2_key(s: *u8, n: i64, out: *u8) -> i64 {
640 var first: i64 = 0 - 1
641 var second: i64 = 0 - 1
642 var i: i64 = 0
643 while i < n {
644 if (s[i] as i64 & 0xff) == VC_SLASH { if first < 0 { first = i } else { if second < 0 { second = i } } }
645 i = i + 1
646 }
647 if first < 0 { return vc_cat(out, 0, "(root)" as *u8) }
648 var e: i64 = n
649 if second >= 0 { e = second + 1 }
650 var k: i64 = 0
651 while k < e { out[k] = s[k]; k = k + 1 }
652 out[k] = 0 as u8
653 return k
654}
655
656// walk root recursively: every .var path (case-insensitive suffix) into paths/offs; other files counted by extension;
657// directories that cannot be opened, symlinks, other node types and over-long paths are COUNTED and named, never dropped
658func vc_walk(root: *u8, paths: *i64, offs: *i64, exts: *i64, ws: *i64, unread: *i64) -> i64 {
659 let dirs: *i64 = vcl_arena_new(VCL_ARENA_INIT)
660 let dq: *i64 = vcl_arena_new(VCL_ARENA_INIT)
661 let rl: i64 = vc_slen(root)
662 vcl_push(dq, vcl_put(dirs, root, 0, rl))
663 vcl_push(dq, 0)
664 let cur: *u8 = sys_mmap(VC_PATH_MAX + VC_PAD)
665 let child: *u8 = sys_mmap(VC_PATH_MAX + VC_PAD)
666 let dbuf: *u8 = sys_mmap(VC_DENT_BUF + VC_PAD)
667 let stbuf: *u8 = sys_mmap(VC_STAT_BUF + VC_PAD)
668 let extb: *u8 = sys_mmap(VC_PATH_MAX + VC_PAD)
669 var qi: i64 = 0
670 while qi < vcl_vec_n(dq) / 2 {
671 let doff: i64 = vcl_vec_at(dq, qi * 2)
672 let depth: i64 = vcl_vec_at(dq, qi * 2 + 1)
673 let dp: *u8 = vcl_at(dirs, doff)
674 let cl: i64 = vc_slen(dp)
675 var ci: i64 = 0
676 while ci < cl { cur[ci] = dp[ci]; ci = ci + 1 }
677 cur[cl] = 0 as u8
678 let fd: i64 = sys_openat_directory(cur)
679 if fd < 0 {
680 ws[VC_W_DIRS_UNREADABLE] = ws[VC_W_DIRS_UNREADABLE] + 1
681 vcl_apps(unread, "DIR-UNREADABLE" as *u8)
682 vcl_appc(unread, VC_TAB)
683 vcl_appclean(unread, cur, 0, cl)
684 vcl_appc(unread, VC_TAB)
685 vcl_appn(unread, fd)
686 vcl_appc(unread, VC_LF)
687 } else {
688 ws[VC_W_DIRS] = ws[VC_W_DIRS] + 1
689 var run: i64 = 1
690 while run == 1 {
691 let nb: i64 = sys_getdents64(fd, dbuf, VC_DENT_BUF)
692 if nb <= 0 { run = 0 } else {
693 var off: i64 = 0
694 while off < nb {
695 let rec: *u8 = ((dbuf as i64) + off) as *u8
696 let reclen: i64 = dirent_reclen(rec)
697 if reclen <= 0 { off = nb } else {
698 let nm: *u8 = dirent_name(rec)
699 let nml: i64 = vc_slen(nm)
700 var dot: i64 = 0
701 if vc_streq(nm, "." as *u8) == 1 { dot = 1 }
702 if vc_streq(nm, ".." as *u8) == 1 { dot = 1 }
703 if dot == 0 {
704 var co: i64 = cl
705 var ci2: i64 = 0
706 while ci2 < cl { child[ci2] = cur[ci2]; ci2 = ci2 + 1 }
707 var need_slash: i64 = 1
708 if cl > 0 { if (cur[cl-1] as i64 & 0xff) == VC_SLASH { need_slash = 0 } }
709 if co + need_slash + nml >= VC_PATH_MAX {
710 ws[VC_W_TOO_LONG] = ws[VC_W_TOO_LONG] + 1
711 vcl_apps(unread, "PATH-TOO-LONG" as *u8)
712 vcl_appc(unread, VC_TAB)
713 vcl_appclean(unread, cur, 0, cl)
714 vcl_appc(unread, VC_LF)
715 } else {
716 if need_slash == 1 { child[co] = VC_SLASH as u8; co = co + 1 }
717 var cj: i64 = 0
718 while cj < nml { child[co + cj] = nm[cj]; cj = cj + 1 }
719 co = co + nml
720 child[co] = 0 as u8
721 var t: i64 = dirent_type(rec)
722 if t == VC_DT_UNKNOWN {
723 if sys_fstatat(child, stbuf) == 0 {
724 let mode: i64 = vf_u32(stbuf, VC_S_MODE) & VC_S_IFMT
725 if mode == VC_S_IFDIR { t = DT_DIR }
726 if mode == VC_S_IFREG { t = DT_REG }
727 }
728 }
729 if t == DT_DIR {
730 vcl_push(dq, vcl_put(dirs, child, 0, co))
731 vcl_push(dq, depth + 1)
732 } else {
733 if t == DT_REG {
734 var isvar: i64 = 0
735 if nml >= 4 {
736 if (nm[nml-4] as i64 & 0xff) == VC_DOT {
737 if vcl_lc(nm[nml-3] as i64 & 0xff) == 118 { if vcl_lc(nm[nml-2] as i64 & 0xff) == 97 { if vcl_lc(nm[nml-1] as i64 & 0xff) == 114 { isvar = 1 } } }
738 }
739 }
740 if isvar == 1 {
741 vcl_push(offs, vcl_put(paths, child, 0, co))
742 ws[VC_W_VARS] = ws[VC_W_VARS] + 1
743 if depth == 0 { ws[VC_W_VARS_TOP] = ws[VC_W_VARS_TOP] + 1 }
744 } else {
745 ws[VC_W_OTHER_FILES] = ws[VC_W_OTHER_FILES] + 1
746 let el: i64 = vc_ext_key(nm, nml, extb)
747 vcl_counter_add(exts, extb, 0, el, 1)
748 }
749 } else {
750 if t == DT_LNK { ws[VC_W_SYMLINKS] = ws[VC_W_SYMLINKS] + 1 } else { ws[VC_W_OTHER_TYPES] = ws[VC_W_OTHER_TYPES] + 1 }
751 }
752 }
753 }
754 }
755 off = off + reclen
756 }
757 }
758 }
759 }
760 sys_close(fd)
761 }
762 qi = qi + 1
763 }
764 return ws[VC_W_VARS]
765}
766
767func vc_write_out(dir: *u8, name: *u8, a: *i64, total: *i64) -> i64 {
768 let p: *u8 = sys_mmap(VC_PATH_MAX + VC_PAD)
769 let dl: i64 = vc_slen(dir)
770 if dl + vc_slen(name) + 2 >= VC_PATH_MAX { return 0 - 1 }
771 var o: i64 = vc_cat(p, 0, dir)
772 if o > 0 { if (p[o-1] as i64 & 0xff) != VC_SLASH { p[o] = VC_SLASH as u8; o = o + 1 } }
773 o = vc_cat(p, o, name)
774 p[o] = 0 as u8
775 let fd: i64 = sys_openat_wr(p, MODE_0644)
776 if fd < 0 { return 0 - 1 }
777 let w: i64 = vc_wn(fd, vcl_at(a, 0), vcl_used(a))
778 sys_close(fd)
779 total[0] = total[0] + w
780 if w != vcl_used(a) { return 0 - 1 }
781 return w
782}
783
784func vc_kv(a: *i64, k: *u8, v: i64) -> i64 {
785 vcl_appc(a, 32)
786 vcl_apps(a, k)
787 vcl_appc(a, VC_EQ)
788 vcl_appn(a, v)
789 return 0
790}
791
792// decode the DIRECT string member key of the object at obj into out (caller sizes it >= n + 1); length or -1
793func vc_get_str(data: *u8, n: i64, obj: i64, key: *u8, jm: *i64, out: *u8) -> i64 {
794 out[0] = 0 as u8
795 let vs: i64 = vf_json_find(data, n, obj, key, jm)
796 if vs < 0 { return 0 - 1 }
797 if vf_json_kind(data, n, vs) != VF_JK_STRING { return 0 - 1 }
798 return vf_json_decode(data, vs + 1, jm[VF_JM_VE] - 1, out)
799}
800
801// append "name=count" for every class of an axis with a non-zero count, comma separated ("-" when none)
802func vc_app_counts(a: *i64, rs: *i64, axis: i64, counts: *i64, nclass: i64) -> i64 {
803 var first: i64 = 1
804 var c: i64 = 0
805 while c < nclass {
806 if counts[c] > 0 {
807 if first == 0 { vcl_appc(a, VC_ROW_SEP) }
808 vcl_apps(a, vcl_class_name(rs, axis, c))
809 vcl_appc(a, VC_EQ)
810 vcl_appn(a, counts[c])
811 first = 0
812 }
813 c = c + 1
814 }
815 if first == 1 { vcl_appc(a, VC_ASCII_MINUS) }
816 return 0
817}
818
819func vc_counter_tsv(a: *i64, c: *i64, prefix: *u8) -> i64 {
820 var i: i64 = 0
821 while i < vcl_counter_n(c) {
822 if vc_slen(prefix) > 0 { vcl_apps(a, prefix); vcl_appc(a, VC_TAB) }
823 vcl_appclean(a, vcl_counter_key(c, i), 0, vcl_counter_klen(c, i))
824 vcl_appc(a, VC_TAB)
825 vcl_appn(a, vcl_counter_count(c, i))
826 vcl_appc(a, VC_LF)
827 i = i + 1
828 }
829 return i
830}
831
832func vc_census(root: *u8, outdir: *u8, rulespath: *u8) -> i64 {
833 let rlen: *i64 = sys_mmap(16) as *i64
834 let rtext: *u8 = sys_read_file(rulespath, rlen)
835 if (rtext as i64) == 0 { vc_w(VC_FD_OUT, "VARCENSUS state=RULES-UNREADABLE\n" as *u8); return VC_EXIT_IO }
836 let rs: *i64 = vcl_rules_parse(rtext, rlen[0])
837 let sm: *i64 = vcl_arena_new(VCL_ARENA_INIT)
838 if rs[VCL_RS_ERRORS] > 0 {
839 vcl_apps(sm, "VARCENSUS state=RULES-MALFORMED" as *u8)
840 vc_kv(sm, "errors" as *u8, rs[VCL_RS_ERRORS])
841 vc_kv(sm, "first_line" as *u8, rs[VCL_RS_FIRST_ERR_LINE])
842 vcl_appc(sm, VC_LF)
843 vc_wn(VC_FD_OUT, vcl_at(sm, 0), vcl_used(sm))
844 return VC_EXIT_REFUSED
845 }
846 sys_mkdir(outdir, MODE_0755)
847 let nrole: i64 = vcl_nclasses(rs, VCL_AX_ROLE)
848 let nfmt: i64 = vcl_nclasses(rs, VCL_AX_FORMAT)
849 let tags: *i64 = rs[VCL_RS_TAGS] as *i64
850 let ntags: i64 = vcl_counter_n(tags)
851 let other_role: i64 = rs[VCL_RS_OTHER_ROLE]
852 let other_fmt: i64 = rs[VCL_RS_OTHER_FORMAT]
853
854 // ---- walk ----
855 let paths: *i64 = vcl_arena_new(VCL_ARENA_INIT)
856 let offs: *i64 = vcl_arena_new(VCL_ARENA_INIT)
857 let exts: *i64 = vcl_counter_new()
858 let ws: *i64 = sys_mmap(VC_W_N * 8) as *i64
859 let ur: *i64 = vcl_arena_new(VCL_ARENA_INIT)
860 vcl_apps(ur, "state\tfile\tdetail\n" as *u8)
861 vc_walk(root, paths, offs, exts, ws, ur)
862 let npk: i64 = vc_sort(paths, offs)
863 let rootl: i64 = vc_slen(root)
864
865 // ---- outputs ----
866 let pk: *i64 = vcl_arena_new(VCL_ARENA_INIT)
867 vcl_apps(pk, "file\tsize\tstate\tentries\tdirs\tz64_members\tz64_unresolved\tutf8_members\tmeta\tlicense\tcreator\tpackage\tprogram_version\tcontent_elems\tdeps_direct\tdeptags\trp_screen\tusize_sum\trole_counts\tformat_counts\n" as *u8)
868 let dpa: *i64 = vcl_arena_new(VCL_ARENA_INIT)
869 vcl_apps(dpa, "from\tto\tto_license\tdeptag\n" as *u8)
870 let rpa: *i64 = vcl_arena_new(VCL_ARENA_INIT)
871 vcl_apps(rpa, "file\treasons\tlicense\n" as *u8)
872 let labels: *i64 = vcl_counter_new()
873 let creators: *i64 = vcl_counter_new()
874 let targets: *i64 = vcl_counter_new()
875 let resrole: *i64 = vcl_counter_new()
876 let resfmt: *i64 = vcl_counter_new()
877
878 let role_members: *i64 = sys_mmap((nrole + 1) * 8) as *i64
879 let role_pkgs: *i64 = sys_mmap((nrole + 1) * 8) as *i64
880 let role_usize: *i64 = sys_mmap((nrole + 1) * 8) as *i64
881 let fmt_members: *i64 = sys_mmap((nfmt + 1) * 8) as *i64
882 let fmt_pkgs: *i64 = sys_mmap((nfmt + 1) * 8) as *i64
883 let fmt_usize: *i64 = sys_mmap((nfmt + 1) * 8) as *i64
884 let prole: *i64 = sys_mmap((nrole + 1) * 8) as *i64
885 let pfmt: *i64 = sys_mmap((nfmt + 1) * 8) as *i64
886 let pstates: *i64 = sys_mmap(VC_P_N * 8) as *i64
887 let mstates: *i64 = sys_mmap(VC_M_N * 8) as *i64
888 let lagree: *i64 = sys_mmap(VC_L_N * 8) as *i64
889 let tag_pkgs: *i64 = sys_mmap((ntags + 1) * 8) as *i64
890 let tag_edges: *i64 = sys_mmap((ntags + 1) * 8) as *i64
891 let ptag: *i64 = sys_mmap((ntags + 1) * 8) as *i64
892 let reason_pkgs: *i64 = sys_mmap(VC_NLETTERS * 8) as *i64
893 var t_members: i64 = 0
894 var t_dirs: i64 = 0
895 var t_z64: i64 = 0
896 var t_z64u: i64 = 0
897 var t_pkgs_z64: i64 = 0
898 var t_utf8: i64 = 0
899 var t_usize: i64 = 0
900 var t_edges: i64 = 0
901 var t_pkgs_deps: i64 = 0
902 var t_deps_malformed: i64 = 0
903 var t_flagged: i64 = 0
904 var t_role_sum: i64 = 0
905 var t_fmt_sum: i64 = 0
906
907 let box: *i64 = sys_mmap(VF_L_N * 8) as *i64
908 let st: *i64 = sys_mmap(VC_ST_N * 8) as *i64
909 let rec: *i64 = sys_mmap(VF_E_N * 8) as *i64
910 let namebuf: *u8 = sys_mmap(VC_NAME_BUF + VC_PAD)
911 let keyb: *u8 = sys_mmap(VC_NAME_BUF + VC_PAD)
912 let idlc: *u8 = sys_mmap(VC_PATH_MAX + VC_PAD)
913 let mbox: *i64 = sys_mmap(VC_MB_N * 8) as *i64
914 let jm: *i64 = sys_mmap(VF_JM_N * 8) as *i64
915 let jm2: *i64 = sys_mmap(VF_JM_N * 8) as *i64
916 let headb: *u8 = sys_mmap(VC_HEAD_LEN + VC_PAD)
917 let db: *i64 = sys_mmap(VF_JM_N * 8) as *i64
918 let legacy: *u8 = sys_mmap(VC_LEGACY_LIC)
919
920 var i: i64 = 0
921 while i < npk {
922 let path: *u8 = vcl_at(paths, vcl_vec_at(offs, i))
923 let pl: i64 = vc_slen(path)
924 var rel: i64 = rootl
925 if rel < pl { if (path[rel] as i64 & 0xff) == VC_SLASH { rel = rel + 1 } }
926 // the package id: the relative path's basename without its 4-byte .var suffix
927 var bs: i64 = rel
928 var bi: i64 = rel
929 while bi < pl { if (path[bi] as i64 & 0xff) == VC_SLASH { bs = bi + 1 } bi = bi + 1 }
930 let idl: i64 = pl - 4 - bs
931 vcl_lower(((path as i64) + bs) as *u8, idl, idlc)
932
933 var state: i64 = VC_P_OK
934 var size: i64 = 0 - 1
935 var win: *u8 = 0 as *u8
936 let fd: i64 = sys_openat_rd(path)
937 if fd < 0 { state = VC_P_UNOPENABLE } else {
938 size = sys_lseek(fd, 0, VC_SEEK_END)
939 win = vc_window(fd, size, box, st)
940 state = vc_pstate_of(st[VC_ST_LOC])
941 // split NO-EOCD by what the file IS: zero bytes, a ZIP local header with no EOCD, or not a ZIP at all
942 if state == VC_P_NOEOCD {
943 if size == 0 { state = VC_P_EMPTY } else {
944 state = VC_P_NOEOCD_OTHER
945 if size >= VC_HEAD_LEN { if vc_pread(fd, 0, VC_HEAD_LEN, headb) == VC_HEAD_LEN { if vf_u32(headb, 0) == VF_SIG_LOCAL { state = VC_P_NOEOCD_PK } } }
946 }
947 }
948 }
949 var c: i64 = 0
950 while c < nrole { prole[c] = 0; c = c + 1 }
951 c = 0
952 while c < nfmt { pfmt[c] = 0; c = c + 1 }
953 var p_entries: i64 = 0
954 var p_dirs: i64 = 0
955 var p_z64: i64 = 0
956 var p_z64u: i64 = 0
957 var p_utf8: i64 = 0
958 var p_usize: i64 = 0
959 if state == VC_P_OK {
960 p_entries = box[VF_L_ENTRIES]
961 var off: i64 = box[VF_L_CD]
962 var k: i64 = 0
963 var go: i64 = 1
964 while go == 1 {
965 if k >= p_entries { go = 0 } else {
966 if vf_cd_entry(win, st[VC_ST_WIN], off, rec) != 1 { state = VC_P_RECORD; go = 0 } else {
967 let nl: i64 = rec[VF_E_NAMELEN]
968 vcl_lower(((win as i64) + rec[VF_E_NAMEOFF]) as *u8, nl, namebuf)
969 if rec[VF_E_Z64] == VF_Z64_RESOLVED { p_z64 = p_z64 + 1 }
970 if (rec[VF_E_FLAGS] / VC_FLAG_UTF8) % 2 == 1 { p_utf8 = p_utf8 + 1 }
971 if nl > 0 { if (namebuf[nl-1] as i64 & 0xff) == VC_SLASH { p_dirs = p_dirs + 1 } }
972 let r: i64 = vcl_classify(rs, VCL_AX_ROLE, namebuf, nl)
973 let f: i64 = vcl_classify(rs, VCL_AX_FORMAT, namebuf, nl)
974 prole[r] = prole[r] + 1
975 pfmt[f] = pfmt[f] + 1
976 if rec[VF_E_Z64] == VF_Z64_UNRESOLVED { p_z64u = p_z64u + 1 } else {
977 p_usize = p_usize + rec[VF_E_USIZE]
978 role_usize[r] = role_usize[r] + rec[VF_E_USIZE]
979 fmt_usize[f] = fmt_usize[f] + rec[VF_E_USIZE]
980 }
981 if r == other_role { let kl: i64 = vc_prefix2_key(namebuf, nl, keyb); vcl_counter_add(resrole, keyb, 0, kl, 1) }
982 if f == other_fmt { let kl2: i64 = vc_ext_key(namebuf, nl, keyb); vcl_counter_add(resfmt, keyb, 0, kl2, 1) }
983 off = rec[VF_E_NEXT]
984 k = k + 1
985 }
986 }
987 }
988 }
989 pstates[state] = pstates[state] + 1
990 if state != VC_P_OK {
991 vcl_apps(ur, vc_pstate_name(state))
992 vcl_appc(ur, VC_TAB)
993 vcl_appclean(ur, path, rel, pl - rel)
994 vcl_appc(ur, VC_TAB)
995 vcl_apps(ur, "size=" as *u8)
996 vcl_appn(ur, size)
997 vcl_appc(ur, VC_LF)
998 if (win as i64) != 0 { sys_munmap(win, st[VC_ST_WIN] + VC_PAD) }
999 if fd >= 0 { sys_close(fd) }
1000 } else {
1001 // fold the package into the corpus only once its whole central directory read cleanly
1002 t_members = t_members + p_entries
1003 t_dirs = t_dirs + p_dirs
1004 t_z64 = t_z64 + p_z64
1005 t_z64u = t_z64u + p_z64u
1006 if p_z64 > 0 { t_pkgs_z64 = t_pkgs_z64 + 1 }
1007 t_utf8 = t_utf8 + p_utf8
1008 t_usize = t_usize + p_usize
1009 c = 0
1010 while c < nrole { role_members[c] = role_members[c] + prole[c]; if prole[c] > 0 { role_pkgs[c] = role_pkgs[c] + 1 } t_role_sum = t_role_sum + prole[c]; c = c + 1 }
1011 c = 0
1012 while c < nfmt { fmt_members[c] = fmt_members[c] + pfmt[c]; if pfmt[c] > 0 { fmt_pkgs[c] = fmt_pkgs[c] + 1 } t_fmt_sum = t_fmt_sum + pfmt[c]; c = c + 1 }
1013
1014 let data: *u8 = vc_meta_read(fd, size, win, st[VC_ST_WIN], box, mbox)
1015 sys_close(fd)
1016 let dn: i64 = mbox[VC_MB_LEN]
1017 var mstate: i64 = VC_M_ABSENT
1018 if mbox[VC_MB_FOUND] < 0 { mstate = VC_M_UNREADABLE }
1019 if mbox[VC_MB_FOUND] == 1 { if (data as i64) == 0 { mstate = VC_M_UNREADABLE } }
1020 var top: i64 = 0 - 1
1021 if (data as i64) != 0 {
1022 top = vf_json_skip(data, dn, 0)
1023 mstate = VC_M_UNPARSED
1024 if vf_json_kind(data, dn, top) == VF_JK_OBJECT { if vf_json_value_end(data, dn, top) >= 0 { mstate = VC_M_PARSED } }
1025 }
1026 mstates[mstate] = mstates[mstate] + 1
1027 var fbuf: *u8 = 0 as *u8
1028 var fcell: i64 = 0
1029 if mstate == VC_M_PARSED { fcell = dn + 1; fbuf = sys_mmap(fcell * 6 + VC_PAD) }
1030 let lb: *u8 = ((fbuf as i64) + 0) as *u8
1031 let cbuf: *u8 = ((fbuf as i64) + fcell) as *u8
1032 let pbuf: *u8 = ((fbuf as i64) + fcell * 2) as *u8
1033 let vbuf: *u8 = ((fbuf as i64) + fcell * 3) as *u8
1034 let kbuf: *u8 = ((fbuf as i64) + fcell * 4) as *u8
1035 let dlbuf: *u8 = ((fbuf as i64) + fcell * 5) as *u8
1036 var ll: i64 = 0 - 1
1037 var crl: i64 = 0 - 1
1038 var pnl: i64 = 0 - 1
1039 var pvl: i64 = 0 - 1
1040 var celems: i64 = 0 - 1
1041 var pdeps: i64 = 0
1042 c = 0
1043 while c < ntags { ptag[c] = 0; c = c + 1 }
1044 if mstate == VC_M_PARSED {
1045 ll = vc_get_str(data, dn, top, VC_LIC_KEY, jm, lb)
1046 crl = vc_get_str(data, dn, top, VC_CREATOR_KEY, jm, cbuf)
1047 pnl = vc_get_str(data, dn, top, VC_PACKAGE_KEY, jm, pbuf)
1048 pvl = vc_get_str(data, dn, top, VC_PROGVER_KEY, jm, vbuf)
1049 let cv: i64 = vf_json_find(data, dn, top, VC_CONTENT_KEY, jm)
1050 if cv >= 0 { celems = vf_json_count_elems(data, dn, cv) }
1051 // legacy first-occurrence licence reader against the depth-aware direct member: agreement MEASURED
1052 let lr: i64 = vf_license(data, dn, 0, dn, legacy)
1053 var li: i64 = VC_L_NEITHER
1054 if lr > 0 { if ll >= 0 {
1055 li = VC_L_DISAGREE
1056 if lr == ll { if vcl_eq(legacy, 0, lb, 0, ll) == 1 { li = VC_L_AGREE } }
1057 } else { li = VC_L_LEGACY_ONLY } } else { if ll >= 0 { li = VC_L_DIRECT_ONLY } }
1058 lagree[li] = lagree[li] + 1
1059 // direct dependency edges
1060 let dv: i64 = vf_json_find(data, dn, top, VC_DEPS_KEY, jm)
1061 if dv >= 0 { if vf_json_kind(data, dn, dv) == VF_JK_OBJECT {
1062 let dend: i64 = jm[VF_JM_VE]
1063 var q: i64 = vf_json_skip(data, dend, dv + 1)
1064 var drun: i64 = 1
1065 while drun == 1 {
1066 if q >= dend - 1 { drun = 0 } else {
1067 let qn: i64 = vf_json_member(data, dend, q, db)
1068 if qn < 0 { drun = 0; t_deps_malformed = t_deps_malformed + 1 } else {
1069 let kl3: i64 = vf_json_decode(data, db[VF_JM_KS], db[VF_JM_KE], kbuf)
1070 if kl3 >= 0 {
1071 pdeps = pdeps + 1
1072 vcl_lower(kbuf, kl3, keyb)
1073 let tg: i64 = vcl_deptag(rs, keyb, kl3)
1074 var dll: i64 = 0 - 1
1075 if vf_json_kind(data, dn, db[VF_JM_VS]) == VF_JK_OBJECT { dll = vc_get_str(data, dn, db[VF_JM_VS], VC_LIC_KEY, jm2, dlbuf) }
1076 vcl_appclean(dpa, path, bs, idl)
1077 vcl_appc(dpa, VC_TAB)
1078 vcl_appclean(dpa, kbuf, 0, kl3)
1079 vcl_appc(dpa, VC_TAB)
1080 if dll >= 0 { vcl_appclean(dpa, dlbuf, 0, dll) } else { vcl_appc(dpa, VC_ASCII_MINUS) }
1081 vcl_appc(dpa, VC_TAB)
1082 if tg >= 0 { vcl_apps(dpa, vcl_counter_key(tags, tg)); ptag[tg] = 1; tag_edges[tg] = tag_edges[tg] + 1 } else { vcl_appc(dpa, VC_ASCII_MINUS) }
1083 vcl_appc(dpa, VC_LF)
1084 vcl_counter_add(targets, keyb, 0, kl3, 1)
1085 t_edges = t_edges + 1
1086 }
1087 q = qn
1088 }
1089 }
1090 }
1091 } }
1092 }
1093 if pdeps > 0 { t_pkgs_deps = t_pkgs_deps + 1 }
1094 c = 0
1095 while c < ntags { if ptag[c] == 1 { tag_pkgs[c] = tag_pkgs[c] + 1 } c = c + 1 }
1096 // the licence label partition (every OK package lands in exactly one row)
1097 if mstate == VC_M_PARSED {
1098 if ll > 0 { vcl_counter_add(labels, lb, 0, ll, 1) } else { if ll == 0 { vcl_counter_add(labels, "(empty)" as *u8, 0, 7, 1) } else { vcl_counter_add(labels, "(no-licenseType)" as *u8, 0, 16, 1) } }
1099 if crl > 0 { vcl_counter_add(creators, cbuf, 0, crl, 1) } else { vcl_counter_add(creators, "(no-creatorName)" as *u8, 0, 16, 1) }
1100 } else {
1101 if mstate == VC_M_ABSENT { vcl_counter_add(labels, "(meta-absent)" as *u8, 0, 13, 1) }
1102 if mstate == VC_M_UNREADABLE { vcl_counter_add(labels, "(meta-unreadable)" as *u8, 0, 17, 1) }
1103 if mstate == VC_M_UNPARSED { vcl_counter_add(labels, "(meta-unparsed)" as *u8, 0, 15, 1) }
1104 }
1105 // the real-person screen: reasons from role classes and the package id, never adjudicated here
1106 let mask: i64 = vcl_screen(rs, prole, idlc, idl)
1107 if mask != 0 {
1108 t_flagged = t_flagged + 1
1109 var m: i64 = mask
1110 var bit: i64 = 0
1111 while bit < VC_NLETTERS { if m % 2 == 1 { reason_pkgs[bit] = reason_pkgs[bit] + 1 } m = m / 2; bit = bit + 1 }
1112 vcl_appclean(rpa, path, rel, pl - rel)
1113 vcl_appc(rpa, VC_TAB)
1114 vcl_app_reasons(rpa, mask)
1115 vcl_appc(rpa, VC_TAB)
1116 if ll > 0 { vcl_appclean(rpa, lb, 0, ll) } else { vcl_appc(rpa, VC_ASCII_MINUS) }
1117 vcl_appc(rpa, VC_LF)
1118 }
1119 // the package row
1120 vcl_appclean(pk, path, rel, pl - rel)
1121 vcl_appc(pk, VC_TAB)
1122 vcl_appn(pk, size)
1123 vcl_appc(pk, VC_TAB)
1124 vcl_apps(pk, "OK" as *u8)
1125 vcl_appc(pk, VC_TAB)
1126 vcl_appn(pk, p_entries)
1127 vcl_appc(pk, VC_TAB)
1128 vcl_appn(pk, p_dirs)
1129 vcl_appc(pk, VC_TAB)
1130 vcl_appn(pk, p_z64)
1131 vcl_appc(pk, VC_TAB)
1132 vcl_appn(pk, p_z64u)
1133 vcl_appc(pk, VC_TAB)
1134 vcl_appn(pk, p_utf8)
1135 vcl_appc(pk, VC_TAB)
1136 if mstate == VC_M_PARSED { vcl_apps(pk, "parsed" as *u8) }
1137 if mstate == VC_M_ABSENT { vcl_apps(pk, "absent" as *u8) }
1138 if mstate == VC_M_UNREADABLE { vcl_apps(pk, "unreadable" as *u8) }
1139 if mstate == VC_M_UNPARSED { vcl_apps(pk, "unparsed" as *u8) }
1140 vcl_appc(pk, VC_TAB)
1141 if ll > 0 { vcl_appclean(pk, lb, 0, ll) } else { vcl_appc(pk, VC_ASCII_MINUS) }
1142 vcl_appc(pk, VC_TAB)
1143 if crl > 0 { vcl_appclean(pk, cbuf, 0, crl) } else { vcl_appc(pk, VC_ASCII_MINUS) }
1144 vcl_appc(pk, VC_TAB)
1145 if pnl > 0 { vcl_appclean(pk, pbuf, 0, pnl) } else { vcl_appc(pk, VC_ASCII_MINUS) }
1146 vcl_appc(pk, VC_TAB)
1147 if pvl > 0 { vcl_appclean(pk, vbuf, 0, pvl) } else { vcl_appc(pk, VC_ASCII_MINUS) }
1148 vcl_appc(pk, VC_TAB)
1149 vcl_appn(pk, celems)
1150 vcl_appc(pk, VC_TAB)
1151 vcl_appn(pk, pdeps)
1152 vcl_appc(pk, VC_TAB)
1153 var tfirst: i64 = 1
1154 c = 0
1155 while c < ntags { if ptag[c] == 1 { if tfirst == 0 { vcl_appc(pk, VC_ROW_SEP) } vcl_apps(pk, vcl_counter_key(tags, c)); tfirst = 0 } c = c + 1 }
1156 if tfirst == 1 { vcl_appc(pk, VC_ASCII_MINUS) }
1157 vcl_appc(pk, VC_TAB)
1158 vcl_app_reasons(pk, mask)
1159 vcl_appc(pk, VC_TAB)
1160 vcl_appn(pk, p_usize)
1161 vcl_appc(pk, VC_TAB)
1162 vc_app_counts(pk, rs, VCL_AX_ROLE, prole, nrole)
1163 vcl_appc(pk, VC_TAB)
1164 vc_app_counts(pk, rs, VCL_AX_FORMAT, pfmt, nfmt)
1165 vcl_appc(pk, VC_LF)
1166 if fcell > 0 { sys_munmap(fbuf, fcell * 6 + VC_PAD) }
1167 sys_munmap(win, st[VC_ST_WIN] + VC_PAD)
1168 }
1169 i = i + 1
1170 }
1171
1172 // ---- aggregate tables ----
1173 let cls: *i64 = vcl_arena_new(VCL_ARENA_INIT)
1174 vcl_apps(cls, "axis\tclass\tmembers\tpackages\tusize_sum\n" as *u8)
1175 c = 0
1176 while c < nrole {
1177 vcl_apps(cls, "role\t" as *u8); vcl_apps(cls, vcl_class_name(rs, VCL_AX_ROLE, c)); vcl_appc(cls, VC_TAB)
1178 vcl_appn(cls, role_members[c]); vcl_appc(cls, VC_TAB); vcl_appn(cls, role_pkgs[c]); vcl_appc(cls, VC_TAB); vcl_appn(cls, role_usize[c]); vcl_appc(cls, VC_LF)
1179 c = c + 1
1180 }
1181 c = 0
1182 while c < nfmt {
1183 vcl_apps(cls, "format\t" as *u8); vcl_apps(cls, vcl_class_name(rs, VCL_AX_FORMAT, c)); vcl_appc(cls, VC_TAB)
1184 vcl_appn(cls, fmt_members[c]); vcl_appc(cls, VC_TAB); vcl_appn(cls, fmt_pkgs[c]); vcl_appc(cls, VC_TAB); vcl_appn(cls, fmt_usize[c]); vcl_appc(cls, VC_LF)
1185 c = c + 1
1186 }
1187 let lba: *i64 = vcl_arena_new(VCL_ARENA_INIT)
1188 vcl_apps(lba, "label\tpackages\n" as *u8)
1189 vc_counter_sorted_tsv(lba, labels)
1190 let cra: *i64 = vcl_arena_new(VCL_ARENA_INIT)
1191 vcl_apps(cra, "creator\tpackages\n" as *u8)
1192 vc_counter_sorted_tsv(cra, creators)
1193 let tga: *i64 = vcl_arena_new(VCL_ARENA_INIT)
1194 vcl_apps(tga, "dependency_lowercased\tedges\n" as *u8)
1195 vc_counter_sorted_tsv(tga, targets)
1196 let rsa: *i64 = vcl_arena_new(VCL_ARENA_INIT)
1197 vcl_apps(rsa, "axis\tkey\tmembers\n" as *u8)
1198 vc_counter_tsv(rsa, resrole, "role-other" as *u8)
1199 vc_counter_tsv(rsa, resfmt, "format-other" as *u8)
1200 let wxa: *i64 = vcl_arena_new(VCL_ARENA_INIT)
1201 vcl_apps(wxa, "extension\tfiles\n" as *u8)
1202 vc_counter_tsv(wxa, exts, "" as *u8)
1203
1204 // ---- the reconciliations ----
1205 var psum: i64 = 0
1206 c = 0
1207 while c < VC_P_N { psum = psum + pstates[c]; c = c + 1 }
1208 var msum: i64 = 0
1209 c = 0
1210 while c < VC_M_N { msum = msum + mstates[c]; c = c + 1 }
1211 var lsum: i64 = 0
1212 c = 0
1213 while c < vcl_counter_n(labels) { lsum = lsum + vcl_counter_count(labels, c); c = c + 1 }
1214 var ok_pk: i64 = 1
1215 if psum != npk { ok_pk = 0 }
1216 if npk != ws[VC_W_VARS] { ok_pk = 0 }
1217 var ok_role: i64 = 0
1218 if t_role_sum == t_members { ok_role = 1 }
1219 var ok_fmt: i64 = 0
1220 if t_fmt_sum == t_members { ok_fmt = 1 }
1221 var ok_meta: i64 = 0
1222 if msum == pstates[VC_P_OK] { ok_meta = 1 }
1223 var ok_lab: i64 = 0
1224 if lsum == pstates[VC_P_OK] { ok_lab = 1 }
1225
1226 vcl_apps(sm, "VARCENSUS-WALK" as *u8)
1227 vc_kv(sm, "dirs" as *u8, ws[VC_W_DIRS])
1228 vc_kv(sm, "dirs_unreadable" as *u8, ws[VC_W_DIRS_UNREADABLE])
1229 vc_kv(sm, "var_files" as *u8, ws[VC_W_VARS])
1230 vc_kv(sm, "var_top_level" as *u8, ws[VC_W_VARS_TOP])
1231 vc_kv(sm, "var_nested" as *u8, ws[VC_W_VARS] - ws[VC_W_VARS_TOP])
1232 vc_kv(sm, "other_files" as *u8, ws[VC_W_OTHER_FILES])
1233 vc_kv(sm, "symlinks" as *u8, ws[VC_W_SYMLINKS])
1234 vc_kv(sm, "other_types" as *u8, ws[VC_W_OTHER_TYPES])
1235 vc_kv(sm, "paths_too_long" as *u8, ws[VC_W_TOO_LONG])
1236 vcl_appc(sm, VC_LF)
1237 vcl_apps(sm, "VARCENSUS-PACKAGES" as *u8)
1238 vc_kv(sm, "walked" as *u8, npk)
1239 c = 0
1240 while c < VC_P_N {
1241 vcl_appc(sm, 32)
1242 vcl_apps(sm, vc_pstate_name(c))
1243 vcl_appc(sm, VC_EQ)
1244 vcl_appn(sm, pstates[c])
1245 c = c + 1
1246 }
1247 vc_kv(sm, "sum" as *u8, psum)
1248 vc_kv(sm, "reconciles" as *u8, ok_pk)
1249 vcl_appc(sm, VC_LF)
1250 vcl_apps(sm, "VARCENSUS-MEMBERS" as *u8)
1251 vc_kv(sm, "members" as *u8, t_members)
1252 vc_kv(sm, "dir_entries" as *u8, t_dirs)
1253 vc_kv(sm, "file_members" as *u8, t_members - t_dirs)
1254 vc_kv(sm, "z64_extra_members" as *u8, t_z64)
1255 vc_kv(sm, "z64_unresolved" as *u8, t_z64u)
1256 vc_kv(sm, "packages_with_z64_extra" as *u8, t_pkgs_z64)
1257 vc_kv(sm, "utf8_named_members" as *u8, t_utf8)
1258 vc_kv(sm, "usize_sum" as *u8, t_usize)
1259 vcl_appc(sm, VC_LF)
1260 vcl_apps(sm, "VARCENSUS-ROLE" as *u8)
1261 vc_kv(sm, "classes" as *u8, nrole)
1262 vc_kv(sm, "sum" as *u8, t_role_sum)
1263 vc_kv(sm, "other" as *u8, role_members[other_role])
1264 vc_kv(sm, "reconciles" as *u8, ok_role)
1265 vcl_appc(sm, VC_LF)
1266 vcl_apps(sm, "VARCENSUS-FORMAT" as *u8)
1267 vc_kv(sm, "classes" as *u8, nfmt)
1268 vc_kv(sm, "sum" as *u8, t_fmt_sum)
1269 vc_kv(sm, "other" as *u8, fmt_members[other_fmt])
1270 vc_kv(sm, "reconciles" as *u8, ok_fmt)
1271 vcl_appc(sm, VC_LF)
1272 vcl_apps(sm, "VARCENSUS-META" as *u8)
1273 vc_kv(sm, "parsed" as *u8, mstates[VC_M_PARSED])
1274 vc_kv(sm, "absent" as *u8, mstates[VC_M_ABSENT])
1275 vc_kv(sm, "unreadable" as *u8, mstates[VC_M_UNREADABLE])
1276 vc_kv(sm, "unparsed" as *u8, mstates[VC_M_UNPARSED])
1277 vc_kv(sm, "sum" as *u8, msum)
1278 vc_kv(sm, "reconciles" as *u8, ok_meta)
1279 vcl_appc(sm, VC_LF)
1280 vcl_apps(sm, "VARCENSUS-LICENSE" as *u8)
1281 vc_kv(sm, "distinct_labels" as *u8, vcl_counter_n(labels))
1282 vc_kv(sm, "label_rows_sum" as *u8, lsum)
1283 vc_kv(sm, "reconciles" as *u8, ok_lab)
1284 vc_kv(sm, "legacy_agree" as *u8, lagree[VC_L_AGREE])
1285 vc_kv(sm, "legacy_disagree" as *u8, lagree[VC_L_DISAGREE])
1286 vc_kv(sm, "legacy_only" as *u8, lagree[VC_L_LEGACY_ONLY])
1287 vc_kv(sm, "direct_only" as *u8, lagree[VC_L_DIRECT_ONLY])
1288 vc_kv(sm, "neither" as *u8, lagree[VC_L_NEITHER])
1289 vcl_appc(sm, VC_LF)
1290 vcl_apps(sm, "VARCENSUS-DEPS" as *u8)
1291 vc_kv(sm, "edges" as *u8, t_edges)
1292 vc_kv(sm, "distinct_targets" as *u8, vcl_counter_n(targets))
1293 vc_kv(sm, "packages_with_deps" as *u8, t_pkgs_deps)
1294 vc_kv(sm, "malformed_dependency_objects" as *u8, t_deps_malformed)
1295 c = 0
1296 while c < ntags {
1297 vcl_appc(sm, 32); vcl_apps(sm, vcl_counter_key(tags, c)); vcl_apps(sm, "_packages=" as *u8); vcl_appn(sm, tag_pkgs[c])
1298 vcl_appc(sm, 32); vcl_apps(sm, vcl_counter_key(tags, c)); vcl_apps(sm, "_edges=" as *u8); vcl_appn(sm, tag_edges[c])
1299 c = c + 1
1300 }
1301 vcl_appc(sm, VC_LF)
1302 vcl_apps(sm, "VARCENSUS-CREATORS" as *u8)
1303 vc_kv(sm, "distinct" as *u8, vcl_counter_n(creators))
1304 vcl_appc(sm, VC_LF)
1305 vcl_apps(sm, "VARCENSUS-SCREEN" as *u8)
1306 vc_kv(sm, "flagged" as *u8, t_flagged)
1307 c = 0
1308 while c < VC_NLETTERS {
1309 if reason_pkgs[c] > 0 { vcl_appc(sm, 32); vcl_appc(sm, VC_ASCII_UPPER_A + c); vcl_appc(sm, VC_EQ); vcl_appn(sm, reason_pkgs[c]) }
1310 c = c + 1
1311 }
1312 vcl_appc(sm, VC_LF)
1313
1314 let tot: *i64 = sys_mmap(16) as *i64
1315 tot[0] = 0
1316 var wfail: i64 = 0
1317 if vc_write_out(outdir, "packages.tsv" as *u8, pk, tot) < 0 { wfail = wfail + 1 }
1318 if vc_write_out(outdir, "deps.tsv" as *u8, dpa, tot) < 0 { wfail = wfail + 1 }
1319 if vc_write_out(outdir, "rpscreen.tsv" as *u8, rpa, tot) < 0 { wfail = wfail + 1 }
1320 if vc_write_out(outdir, "unreadable.tsv" as *u8, ur, tot) < 0 { wfail = wfail + 1 }
1321 if vc_write_out(outdir, "classes.tsv" as *u8, cls, tot) < 0 { wfail = wfail + 1 }
1322 if vc_write_out(outdir, "labels.tsv" as *u8, lba, tot) < 0 { wfail = wfail + 1 }
1323 if vc_write_out(outdir, "creators.tsv" as *u8, cra, tot) < 0 { wfail = wfail + 1 }
1324 if vc_write_out(outdir, "dep_targets.tsv" as *u8, tga, tot) < 0 { wfail = wfail + 1 }
1325 if vc_write_out(outdir, "residual.tsv" as *u8, rsa, tot) < 0 { wfail = wfail + 1 }
1326 if vc_write_out(outdir, "walk_extensions.tsv" as *u8, wxa, tot) < 0 { wfail = wfail + 1 }
1327 vcl_apps(sm, "VARCENSUS-OUT" as *u8)
1328 vc_kv(sm, "files" as *u8, 11)
1329 vc_kv(sm, "write_failures" as *u8, wfail)
1330 vc_kv(sm, "tsv_bytes" as *u8, tot[0])
1331 vcl_appc(sm, VC_LF)
1332 var verdict_ok: i64 = 1
1333 if ok_pk == 0 { verdict_ok = 0 }
1334 if ok_role == 0 { verdict_ok = 0 }
1335 if ok_fmt == 0 { verdict_ok = 0 }
1336 if ok_meta == 0 { verdict_ok = 0 }
1337 if ok_lab == 0 { verdict_ok = 0 }
1338 if wfail > 0 { verdict_ok = 0 }
1339 if verdict_ok == 1 { vcl_apps(sm, "VARCENSUS verdict=RECONCILED\n" as *u8) } else { vcl_apps(sm, "VARCENSUS verdict=MISMATCH\n" as *u8) }
1340 vc_write_out(outdir, "summary.txt" as *u8, sm, tot)
1341 vc_wn(VC_FD_OUT, vcl_at(sm, 0), vcl_used(sm))
1342 if verdict_ok == 1 { return VC_EXIT_OK }
1343 return VC_EXIT_MISMATCH
1344}
1345
1346func main(argc: i64, argv: *i64) -> i64 {
1347 if argc < 3 {
1348 vc_w(VC_FD_ERR, "usage: nx_varcensus names <file.var> | meta <file.var> | census <dir> <outdir> <rules.conf>\n" as *u8)
1349 vc_w(VC_FD_ERR, " manifest and meta.json only; asset bytes are never opened.\n" as *u8)
1350 sys_exit(VC_EXIT_USAGE)
1351 return VC_EXIT_USAGE
1352 }
1353 let verb: *u8 = argv[1] as *u8
1354 let arg: *u8 = argv[2] as *u8
1355 var rc: i64 = VC_EXIT_USAGE
1356 if vc_streq(verb, "names" as *u8) == 1 { rc = vc_names(arg) } else {
1357 if vc_streq(verb, "meta" as *u8) == 1 { rc = vc_meta(arg) } else {
1358 if vc_streq(verb, "census" as *u8) == 1 {
1359 if argc < 5 { vc_w(VC_FD_ERR, "census needs <dir> <outdir> <rules.conf>\n" as *u8) } else { rc = vc_census(arg, argv[3] as *u8, argv[4] as *u8) }
1360 } else { vc_w(VC_FD_ERR, "nx_varcensus: unknown verb\n" as *u8) }
1361 }
1362 }
1363 sys_exit(rc)
1364 return rc
1365}