nx_varcensus_candidate_t139.nx source

1// nx_varcensus.nx -- THE VaM CORPUS CENSUS SURFACE (laptop-local lane, 2026-09-11). 2// 3// nx_varcensus names <file.var> one row per central-directory record (the package's own manifest) 4// nx_varcensus meta <file.var> meta.json's DIRECT top-level members: key, kind, size; short strings 5// nx_varcensus census <dir> <outdir> <rules.conf> EVERY .var under <dir>, recursively, into TSV reports + a summary 6// 7// WHY A SECOND CLI BESIDE nx_varfacts: nx_varfacts is the refcorpus WRITER (it imports nx_refcorpus for rc_put_row and the 8// NAS plane); this surface runs where the corpus lives (the laptop) and writes reports. Both consume the ONE walk in 9// nx_varfacts_lib (vf_cd_locate / vf_cd_entry / vf_member_data / vf_json_*): no second zip reader and no second JSON reader 10// exist here, and the classifier is DATA (nx_varcensus_lib rules). Legacy census/meta reads meta.json only. 11// Private member <file.var> <exact-member-name> <maxbytes> reads one selected member for bounded integrity/schema 12// measurements; member content is never emitted, interpreted as executable code, or published. 13// 14// THE CENSUS CONTRACT (each asserted, each printed with its sum): 15// packages walked = ok + no_eocd + zip64_eocd + window + short + read_fail + record_refused + unopenable 16// for the OK packages: members = sum over role classes = sum over format classes; meta parsed + absent + unreadable + 17// unparsed = ok packages; licence label rows sum to ok packages. A package that is not OK is named in unreadable.tsv. 18// A meta string longer than VC_SHOW_STR_MAX prints its LENGTH only in the meta verb -- free text is not needed to learn a 19// schema, and the bound is announced on every row it applies to (shown=0), never silent. 20// license_tier: ORIGINAL No hw writes (Rule 26). expect_exit: 0 21import "nx_syscalls.nx" 22import "nx_varfacts_candidate_t139.nx" 23import "nx_varcensus_lib.nx" 24 25const VC_FD_OUT: i64 = 1 26const VC_FD_ERR: i64 = 2 27const VC_EXIT_OK: i64 = 0 28const VC_EXIT_IO: i64 = 1 29const VC_EXIT_USAGE: i64 = 2 30const VC_EXIT_MISMATCH: i64 = 3 31const VC_EXIT_REFUSED: i64 = 4 32const VC_SEEK_SET: i64 = 0 33const VC_SEEK_END: i64 = 2 34const VC_DEC: i64 = 10 35const VC_ASCII_0: i64 = 48 36const VC_ASCII_MINUS: i64 = 45 37const VC_PRINT_LO: i64 = 32 38const VC_DEL: i64 = 127 39const VC_QMARK: i64 = 63 40const VC_PAD: i64 = 16 41const VC_LINE_ROOM: i64 = 1024 42const VC_ST_N: i64 = 4 43const VC_ST_SIZE: i64 = 0 44const VC_ST_LOC: i64 = 1 45const VC_ST_WIN: i64 = 2 46const VC_ST_READFAIL: i64 = 9 47const VC_SHOW_STR_MAX: i64 = 80 48const VC_SHOW_ELEMS: i64 = 5 49const VC_MB_LEN: i64 = 0 50const VC_MB_STATE: i64 = 1 51const VC_MB_FOUND: i64 = 2 52const VC_MB_METHOD: i64 = 3 53const VC_MB_CSIZE: i64 = 4 54const VC_MB_USIZE: i64 = 5 55const VC_MB_CRC_EXPECTED: i64 = 6 56const VC_MB_CRC_ACTUAL: i64 = 7 57const VC_MB_N: i64 = 8 58const VC_MD_DUPLICATE: i64 = 6 59const VC_MD_LIMIT: i64 = 7 60const VC_MD_ENCRYPTED: i64 = 8 61const VC_META_NAME: *u8 = "meta.json" 62const VC_DEPS_KEY: *u8 = "dependencies" 63const VC_CONTENT_KEY: *u8 = "contentList" 64const VC_LIC_KEY: *u8 = "licenseType" 65const VC_CREATOR_KEY: *u8 = "creatorName" 66const VC_PACKAGE_KEY: *u8 = "packageName" 67const VC_PROGVER_KEY: *u8 = "programVersion" 68 69// census bounds that are OS or format facts, not tunables 70const VC_PATH_MAX: i64 = 4096 // Linux PATH_MAX: a longer path cannot be opened by name at all (counted, never skipped silently) 71const VC_DENT_BUF: i64 = 65536 // one getdents64 batch; the walk loops until the call returns 0 72const VC_NAME_BUF: i64 = 65536 // a ZIP member name length is a u16, so no member name is longer 73const VC_STAT_BUF: i64 = 144 74const VC_S_MODE: i64 = 24 75const VC_S_IFMT: i64 = 0xf000 76const VC_S_IFDIR: i64 = 0x4000 77const VC_S_IFREG: i64 = 0x8000 78const VC_DT_UNKNOWN: i64 = 0 79const VC_FLAG_UTF8: i64 = 2048 80const VC_LEGACY_LIC: i64 = 64 81const VC_NLETTERS: i64 = 26 82const VC_ASCII_UPPER_A: i64 = 65 83const VC_TAB: i64 = 9 84const VC_LF: i64 = 10 85const VC_SLASH: i64 = 47 86const VC_DOT: i64 = 46 87const VC_ROW_SEP: i64 = 44 88const VC_EQ: i64 = 61 89// package states: the partition of the packages walked 90const VC_P_OK: i64 = 0 91const VC_P_NOEOCD: i64 = 1 92const VC_P_ZIP64: i64 = 2 93const VC_P_WINDOW: i64 = 3 94const VC_P_SHORT: i64 = 4 95const VC_P_READFAIL: i64 = 5 96const VC_P_RECORD: i64 = 6 97const VC_P_UNOPENABLE: i64 = 7 98// a NO-EOCD package is split by what the file IS -- MEASURED 2026-09-11: the first census read 206 NO-EOCD rows and the first 99// 40 were all size=0. EMPTY is a zero-byte file; NO-EOCD-PK-HEAD carries a ZIP local header at byte 0 but no EOCD in its 100// tail window (an incomplete or damaged archive); NO-EOCD-OTHER-HEAD is not a ZIP at all. Plain NO-EOCD stays in the 101// partition and must read 0 once the split runs. 102const VC_P_EMPTY: i64 = 8 103const VC_P_NOEOCD_PK: i64 = 9 104const VC_P_NOEOCD_OTHER: i64 = 10 105const VC_P_N: i64 = 11 106const VC_HEAD_LEN: i64 = 4 107// meta states: the partition of the OK packages 108const VC_M_PARSED: i64 = 0 109const VC_M_ABSENT: i64 = 1 110const VC_M_UNREADABLE: i64 = 2 111const VC_M_UNPARSED: i64 = 3 112const VC_M_N: i64 = 4 113// walk stats 114const VC_W_DIRS: i64 = 0 115const VC_W_DIRS_UNREADABLE: i64 = 1 116const VC_W_VARS: i64 = 2 117const VC_W_VARS_TOP: i64 = 3 118const VC_W_OTHER_FILES: i64 = 4 119const VC_W_SYMLINKS: i64 = 5 120const VC_W_OTHER_TYPES: i64 = 6 121const VC_W_TOO_LONG: i64 = 7 122const VC_W_N: i64 = 8 123// licence reading agreement (legacy first-occurrence reader vs the depth-aware direct member) 124const VC_L_AGREE: i64 = 0 125const VC_L_DISAGREE: i64 = 1 126const VC_L_LEGACY_ONLY: i64 = 2 127const VC_L_DIRECT_ONLY: i64 = 3 128const VC_L_NEITHER: i64 = 4 129const VC_L_N: i64 = 5 130 131func vc_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n } 132func vc_streq(a: *u8, b: *u8) -> i64 { 133 var i: i64 = 0 134 while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 } 135 if b[i] != (0 as u8) { return 0 } 136 return 1 137} 138func vc_wn(fd: i64, s: *u8, n: i64) -> i64 { 139 var done: i64 = 0 140 var run: i64 = 1 141 while run == 1 { 142 if done >= n { run = 0 } else { 143 let k: i64 = sys_write(fd, ((s as i64) + done) as *u8, n - done) 144 if k <= 0 { run = 0 } else { done = done + k } 145 } 146 } 147 return done 148} 149func vc_w(fd: i64, s: *u8) -> i64 { return vc_wn(fd, s, vc_slen(s)) } 150func vc_cat(b: *u8, o: i64, s: *u8) -> i64 { 151 var i: i64 = 0 152 while s[i] != (0 as u8) { b[o+i] = s[i]; i = i + 1 } 153 return o + i 154} 155func vc_catn(b: *u8, o: i64, v: i64) -> i64 { 156 var p: i64 = o 157 var m: i64 = v 158 if m < 0 { b[p] = VC_ASCII_MINUS as u8; p = p + 1; m = 0 - m } 159 if m == 0 { b[p] = VC_ASCII_0 as u8; return p + 1 } 160 let s: i64 = p 161 while m > 0 { b[p] = (VC_ASCII_0 + (m % VC_DEC)) as u8; m = m / VC_DEC; p = p + 1 } 162 var i: i64 = s 163 var j: i64 = p - 1 164 while i < j { let t: u8 = b[i]; b[i] = b[j]; b[j] = t; i = i + 1; j = j - 1 } 165 return p 166} 167// copy n bytes of src starting at s; control bytes and DEL become '?' so a name can never tear a row 168func vc_catb(b: *u8, o: i64, src: *u8, s: i64, n: i64) -> i64 { 169 var i: i64 = 0 170 while i < n { 171 let c: i64 = src[s+i] as i64 & 0xff 172 var ok: i64 = 1 173 if c < VC_PRINT_LO { ok = 0 } 174 if c == VC_DEL { ok = 0 } 175 if ok == 1 { b[o+i] = c as u8 } else { b[o+i] = VC_QMARK as u8 } 176 i = i + 1 177 } 178 return o + n 179} 180 181func vc_pread(fd: i64, off: i64, len: i64, buf: *u8) -> i64 { 182 if sys_lseek(fd, off, VC_SEEK_SET) != off { return 0 - 1 } 183 var got: i64 = 0 184 var run: i64 = 1 185 while run == 1 { 186 if got >= len { run = 0 } else { 187 let k: i64 = sys_read(fd, ((buf as i64) + got) as *u8, len - got) 188 if k <= 0 { run = 0 } else { got = got + k } 189 } 190 } 191 return got 192} 193 194func vc_state_name(s: i64) -> *u8 { 195 if s == VF_LOC_OK { return "OK" as *u8 } 196 if s == VF_LOC_NOEOCD { return "NO-EOCD" as *u8 } 197 if s == VF_LOC_ZIP64 { return "ZIP64" as *u8 } 198 if s == VF_LOC_WINDOW { return "WINDOW" as *u8 } 199 if s == VF_LOC_SHORT { return "SHORT" as *u8 } 200 if s == VC_ST_READFAIL { return "READ-FAIL" as *u8 } 201 return "UNNAMED" as *u8 202} 203func vc_pstate_name(s: i64) -> *u8 { 204 if s == VC_P_OK { return "OK" as *u8 } 205 if s == VC_P_NOEOCD { return "NO-EOCD" as *u8 } 206 if s == VC_P_ZIP64 { return "ZIP64-EOCD" as *u8 } 207 if s == VC_P_WINDOW { return "WINDOW" as *u8 } 208 if s == VC_P_SHORT { return "SHORT" as *u8 } 209 if s == VC_P_READFAIL { return "READ-FAIL" as *u8 } 210 if s == VC_P_RECORD { return "RECORD-REFUSED" as *u8 } 211 if s == VC_P_UNOPENABLE { return "UNOPENABLE" as *u8 } 212 if s == VC_P_EMPTY { return "EMPTY" as *u8 } 213 if s == VC_P_NOEOCD_PK { return "NO-EOCD-PK-HEAD" as *u8 } 214 if s == VC_P_NOEOCD_OTHER { return "NO-EOCD-OTHER-HEAD" as *u8 } 215 return "UNNAMED" as *u8 216} 217 218// does counter entry x rank before entry y? the higher count first, then the byte-smaller key 219func vc_rank_before(c: *i64, x: i64, y: i64) -> i64 { 220 let cx: i64 = vcl_counter_count(c, x) 221 let cy: i64 = vcl_counter_count(c, y) 222 if cx > cy { return 1 } 223 if cx < cy { return 0 } 224 if vc_cmp(vcl_counter_key(c, x), vcl_counter_key(c, y)) < 0 { return 1 } 225 return 0 226} 227// EVERY entry of a counter as TSV rows ranked by count (Shell sort of the entry indices): the whole table ranked, never a 228// top-N cut, so a reader taking the head of the file is reading a labelled head of a complete population 229func vc_counter_sorted_tsv(a: *i64, c: *i64) -> i64 { 230 let n: i64 = vcl_counter_n(c) 231 if n == 0 { return 0 } 232 let idx: *i64 = sys_mmap(n * 8) as *i64 233 var i: i64 = 0 234 while i < n { idx[i] = i; i = i + 1 } 235 var gap: i64 = n / 2 236 while gap > 0 { 237 var k: i64 = gap 238 while k < n { 239 let t: i64 = idx[k] 240 var j: i64 = k 241 var go: i64 = 1 242 while go == 1 { 243 if j < gap { go = 0 } else { 244 if vc_rank_before(c, t, idx[j - gap]) == 1 { idx[j] = idx[j - gap]; j = j - gap } else { go = 0 } 245 } 246 } 247 idx[j] = t 248 k = k + 1 249 } 250 gap = gap / 2 251 } 252 i = 0 253 while i < n { 254 vcl_appclean(a, vcl_counter_key(c, idx[i]), 0, vcl_counter_klen(c, idx[i])) 255 vcl_appc(a, VC_TAB) 256 vcl_appn(a, vcl_counter_count(c, idx[i])) 257 vcl_appc(a, VC_LF) 258 i = i + 1 259 } 260 sys_munmap(idx as *u8, n * 8) 261 return n 262} 263func vc_pstate_of(loc: i64) -> i64 { 264 if loc == VF_LOC_OK { return VC_P_OK } 265 if loc == VF_LOC_NOEOCD { return VC_P_NOEOCD } 266 if loc == VF_LOC_ZIP64 { return VC_P_ZIP64 } 267 if loc == VF_LOC_WINDOW { return VC_P_WINDOW } 268 if loc == VF_LOC_SHORT { return VC_P_SHORT } 269 return VC_P_READFAIL 270} 271func vc_md_name(s: i64) -> *u8 { 272 if s == VF_MD_STORED { return "STORED" as *u8 } 273 if s == VF_MD_INFLATED { return "INFLATED" as *u8 } 274 if s == VF_MD_UNSUPPORTED { return "UNSUPPORTED" as *u8 } 275 if s == VF_MD_SIZE_MISMATCH { return "SIZE-MISMATCH" as *u8 } 276 if s == VF_MD_CRC_MISMATCH { return "CRC-MISMATCH" as *u8 } 277 if s == VC_MD_DUPLICATE { return "DUPLICATE-MEMBER" as *u8 } 278 if s == VC_MD_LIMIT { return "MEMBER-LIMIT" as *u8 } 279 if s == VC_MD_ENCRYPTED { return "ENCRYPTED" as *u8 } 280 return "FAILED" as *u8 281} 282func vc_kind_name(k: i64) -> *u8 { 283 if k == VF_JK_STRING { return "string" as *u8 } 284 if k == VF_JK_OBJECT { return "object" as *u8 } 285 if k == VF_JK_ARRAY { return "array" as *u8 } 286 if k == VF_JK_SCALAR { return "scalar" as *u8 } 287 return "bad" as *u8 288} 289 290// THE WINDOW: read the EOCD window from the end; when the central directory starts before it, read exactly 291// [cd_abs, size) -- a window derived from the archive, never a guessed tail. st: size, locate state, window length. 292// Returns the mapping (release with sys_munmap(win, st[VC_ST_WIN] + VC_PAD)) or 0. 293func vc_window(fd: i64, size: i64, box: *i64, st: *i64) -> *u8 { 294 st[VC_ST_SIZE] = size 295 st[VC_ST_LOC] = VC_ST_READFAIL 296 st[VC_ST_WIN] = 0 297 let w0: i64 = vf_eocd_window(size) 298 if w0 <= 0 { st[VC_ST_LOC] = VF_LOC_NOEOCD; return 0 as *u8 } 299 let b0: *u8 = sys_mmap(w0 + VC_PAD) 300 if vc_pread(fd, size - w0, w0, b0) != w0 { sys_munmap(b0, w0 + VC_PAD); return 0 as *u8 } 301 let s0: i64 = vf_cd_locate(b0, w0, size, box) 302 st[VC_ST_LOC] = s0 303 st[VC_ST_WIN] = w0 304 if s0 != VF_LOC_WINDOW { return b0 } 305 let need: i64 = size - box[VF_L_CDABS] 306 sys_munmap(b0, w0 + VC_PAD) 307 st[VC_ST_WIN] = 0 308 if need <= w0 { st[VC_ST_LOC] = VF_LOC_SHORT; return 0 as *u8 } 309 if need > size { st[VC_ST_LOC] = VF_LOC_SHORT; return 0 as *u8 } 310 let b1: *u8 = sys_mmap(need + VC_PAD) 311 if vc_pread(fd, size - need, need, b1) != need { st[VC_ST_LOC] = VC_ST_READFAIL; sys_munmap(b1, need + VC_PAD); return 0 as *u8 } 312 st[VC_ST_LOC] = vf_cd_locate(b1, need, size, box) 313 st[VC_ST_WIN] = need 314 return b1 315} 316 317// meta.json of the package whose CD window is win. mbox: decoded length, vf_member_data state, found (1 found / 0 absent / 318// -1 unreadable), method. Returns the decoded bytes or 0. A member whose sizes could not be resolved (ZIP64 extra missing) 319// is UNREADABLE by name, never read at a 4 GiB size. 320// Single selected-member walk shared by meta and private measurement. No member bytes are emitted. 321func vc_member_read(fd: i64, size: i64, win: *u8, winn: i64, box: *i64, selected: *u8, maxbytes: i64, mbox: *i64) -> *u8 { 322 var i: i64 = 0 323 while i < VC_MB_N { mbox[i] = 0 - 1; i = i + 1 } 324 mbox[VC_MB_LEN] = 0 325 mbox[VC_MB_STATE] = VF_MD_FAILED 326 mbox[VC_MB_FOUND] = 0 327 let rec: *i64 = sys_mmap(VF_E_N * 8) as *i64 328 let chosen: *i64 = sys_mmap(VF_E_N * 8) as *i64 329 var off: i64 = box[VF_L_CD] 330 var k: i64 = 0 331 while k < box[VF_L_ENTRIES] { 332 if vf_cd_entry(win, winn, off, rec) != 1 { mbox[VC_MB_FOUND] = 0 - 1; return 0 as *u8 } 333 let ns: i64 = rec[VF_E_NAMEOFF] 334 if vf_span_is(win, ns, ns + rec[VF_E_NAMELEN], selected) == 1 { 335 if mbox[VC_MB_FOUND] == 1 { mbox[VC_MB_STATE] = VC_MD_DUPLICATE; return 0 as *u8 } 336 mbox[VC_MB_FOUND] = 1 337 i = 0 338 while i < VF_E_N { chosen[i] = rec[i]; i = i + 1 } 339 } 340 off = rec[VF_E_NEXT] 341 k = k + 1 342 } 343 if mbox[VC_MB_FOUND] != 1 { return 0 as *u8 } 344 let lho: i64 = chosen[VF_E_LHO] 345 let method: i64 = chosen[VF_E_METHOD] 346 let csize: i64 = chosen[VF_E_CSIZE] 347 let usize: i64 = chosen[VF_E_USIZE] 348 mbox[VC_MB_METHOD] = method 349 mbox[VC_MB_CSIZE] = csize 350 mbox[VC_MB_USIZE] = usize 351 mbox[VC_MB_CRC_EXPECTED] = chosen[VF_E_CRC] 352 if chosen[VF_E_Z64] == VF_Z64_UNRESOLVED { return 0 as *u8 } 353 if csize < 0 { return 0 as *u8 } 354 if usize < 0 { return 0 as *u8 } 355 if maxbytes > 0 { if csize > maxbytes { mbox[VC_MB_STATE] = VC_MD_LIMIT; return 0 as *u8 } if usize > maxbytes { mbox[VC_MB_STATE] = VC_MD_LIMIT; return 0 as *u8 } } 356 if (chosen[VF_E_FLAGS] & 1) != 0 { mbox[VC_MB_STATE] = VC_MD_ENCRYPTED; return 0 as *u8 } 357 if lho < 0 { return 0 as *u8 } 358 if lho > size - VF_LOCAL_LEN { return 0 as *u8 } 359 let hdr: *u8 = sys_mmap(VF_LOCAL_LEN + VC_PAD) 360 if vc_pread(fd, lho, VF_LOCAL_LEN, hdr) != VF_LOCAL_LEN { return 0 as *u8 } 361 if vf_u32(hdr, 0) != VF_SIG_LOCAL { return 0 as *u8 } 362 if vf_u16(hdr, 8) != method { return 0 as *u8 } 363 let fnl: i64 = vf_u16(hdr, VF_LOCAL_FNL) 364 if fnl != chosen[VF_E_NAMELEN] { return 0 as *u8 } 365 let nm: *u8 = sys_mmap(fnl + VC_PAD) 366 if vc_pread(fd, lho + VF_LOCAL_LEN, fnl, nm) != fnl { return 0 as *u8 } 367 if vf_span_is(nm, 0, fnl, selected) != 1 { return 0 as *u8 } 368 let doff: i64 = lho + VF_LOCAL_LEN + fnl + vf_u16(hdr, VF_LOCAL_EXL) 369 if doff > size { return 0 as *u8 } 370 if csize > size - doff { return 0 as *u8 } 371 let body: *u8 = sys_mmap(csize + VC_PAD) 372 if vc_pread(fd, doff, csize, body) != csize { sys_munmap(body, csize + VC_PAD); return 0 as *u8 } 373 let mdo: *i64 = sys_mmap(32) as *i64 374 let data: *u8 = vf_member_checked(body, csize, method, usize, chosen[VF_E_CRC], mdo) 375 mbox[VC_MB_LEN] = mdo[0] 376 mbox[VC_MB_STATE] = mdo[1] 377 mbox[VC_MB_CRC_ACTUAL] = mdo[3] 378 if mdo[1] == VF_MD_INFLATED { sys_munmap(body, csize + VC_PAD) } 379 return data 380} 381func vc_meta_read(fd: i64, size: i64, win: *u8, winn: i64, box: *i64, mbox: *i64) -> *u8 { 382 return vc_member_read(fd, size, win, winn, box, VC_META_NAME, 0, mbox) 383} 384 385func vc_names(path: *u8) -> i64 { 386 let fd: i64 = sys_openat_rd(path) 387 if fd < 0 { vc_w(VC_FD_OUT, "VARCENSUS-NAMES state=UNOPENABLE\n" as *u8); return VC_EXIT_IO } 388 let size: i64 = sys_lseek(fd, 0, VC_SEEK_END) 389 let box: *i64 = sys_mmap(VF_L_N * 8) as *i64 390 let st: *i64 = sys_mmap(VC_ST_N * 8) as *i64 391 let win: *u8 = vc_window(fd, size, box, st) 392 sys_close(fd) 393 let line: *u8 = sys_mmap(VF_U16_ALL + VC_LINE_ROOM) 394 if st[VC_ST_LOC] != VF_LOC_OK { 395 var o: i64 = vc_cat(line, 0, "VARCENSUS-NAMES state=" as *u8) 396 o = vc_cat(line, o, vc_state_name(st[VC_ST_LOC])) 397 o = vc_cat(line, o, " size=" as *u8) 398 o = vc_catn(line, o, size) 399 o = vc_cat(line, o, "\n" as *u8) 400 vc_wn(VC_FD_OUT, line, o) 401 return VC_EXIT_REFUSED 402 } 403 let rec: *i64 = sys_mmap(VF_E_N * 8) as *i64 404 let entries: i64 = box[VF_L_ENTRIES] 405 var off: i64 = box[VF_L_CD] 406 var k: i64 = 0 407 var usum: i64 = 0 408 var csum: i64 = 0 409 var z64r: i64 = 0 410 var z64u: i64 = 0 411 while k < entries { 412 if vf_cd_entry(win, st[VC_ST_WIN], off, rec) != 1 { 413 var o3: i64 = vc_cat(line, 0, "VARCENSUS-NAMES state=RECORD-REFUSED at=" as *u8) 414 o3 = vc_catn(line, o3, k) 415 o3 = vc_cat(line, o3, "\n" as *u8) 416 vc_wn(VC_FD_OUT, line, o3) 417 return VC_EXIT_REFUSED 418 } 419 var o2: i64 = vc_cat(line, 0, "M " as *u8) 420 o2 = vc_catn(line, o2, k) 421 o2 = vc_cat(line, o2, " m=" as *u8) 422 o2 = vc_catn(line, o2, rec[VF_E_METHOD]) 423 o2 = vc_cat(line, o2, " c=" as *u8) 424 o2 = vc_catn(line, o2, rec[VF_E_CSIZE]) 425 o2 = vc_cat(line, o2, " u=" as *u8) 426 o2 = vc_catn(line, o2, rec[VF_E_USIZE]) 427 o2 = vc_cat(line, o2, " f=" as *u8) 428 o2 = vc_catn(line, o2, rec[VF_E_FLAGS]) 429 o2 = vc_cat(line, o2, " z64=" as *u8) 430 o2 = vc_catn(line, o2, rec[VF_E_Z64]) 431 o2 = vc_cat(line, o2, " " as *u8) 432 o2 = vc_catb(line, o2, win, rec[VF_E_NAMEOFF], rec[VF_E_NAMELEN]) 433 o2 = vc_cat(line, o2, "\n" as *u8) 434 vc_wn(VC_FD_OUT, line, o2) 435 if rec[VF_E_Z64] == VF_Z64_RESOLVED { z64r = z64r + 1 } 436 if rec[VF_E_Z64] == VF_Z64_UNRESOLVED { z64u = z64u + 1 } else { 437 usum = usum + rec[VF_E_USIZE] 438 csum = csum + rec[VF_E_CSIZE] 439 } 440 off = rec[VF_E_NEXT] 441 k = k + 1 442 } 443 var o4: i64 = vc_cat(line, 0, "VARCENSUS-NAMES state=OK entries=" as *u8) 444 o4 = vc_catn(line, o4, entries) 445 o4 = vc_cat(line, o4, " size=" as *u8) 446 o4 = vc_catn(line, o4, size) 447 o4 = vc_cat(line, o4, " window=" as *u8) 448 o4 = vc_catn(line, o4, st[VC_ST_WIN]) 449 o4 = vc_cat(line, o4, " cd_abs=" as *u8) 450 o4 = vc_catn(line, o4, box[VF_L_CDABS]) 451 o4 = vc_cat(line, o4, " z64_resolved=" as *u8) 452 o4 = vc_catn(line, o4, z64r) 453 o4 = vc_cat(line, o4, " z64_unresolved=" as *u8) 454 o4 = vc_catn(line, o4, z64u) 455 o4 = vc_cat(line, o4, " usize_sum=" as *u8) 456 o4 = vc_catn(line, o4, usum) 457 o4 = vc_cat(line, o4, " csize_sum=" as *u8) 458 o4 = vc_catn(line, o4, csum) 459 o4 = vc_cat(line, o4, "\n" as *u8) 460 vc_wn(VC_FD_OUT, line, o4) 461 sys_munmap(win, st[VC_ST_WIN] + VC_PAD) 462 return VC_EXIT_OK 463} 464 465func vc_meta(path: *u8) -> i64 { 466 let fd: i64 = sys_openat_rd(path) 467 if fd < 0 { vc_w(VC_FD_OUT, "VARCENSUS-META state=UNOPENABLE\n" as *u8); return VC_EXIT_IO } 468 let size: i64 = sys_lseek(fd, 0, VC_SEEK_END) 469 let box: *i64 = sys_mmap(VF_L_N * 8) as *i64 470 let st: *i64 = sys_mmap(VC_ST_N * 8) as *i64 471 let win: *u8 = vc_window(fd, size, box, st) 472 let line: *u8 = sys_mmap(VF_U16_ALL + VC_LINE_ROOM) 473 if st[VC_ST_LOC] != VF_LOC_OK { 474 sys_close(fd) 475 var o: i64 = vc_cat(line, 0, "VARCENSUS-META window=" as *u8) 476 o = vc_cat(line, o, vc_state_name(st[VC_ST_LOC])) 477 o = vc_cat(line, o, "\n" as *u8) 478 vc_wn(VC_FD_OUT, line, o) 479 return VC_EXIT_REFUSED 480 } 481 let mbox: *i64 = sys_mmap(VC_MB_N * 8) as *i64 482 let data: *u8 = vc_meta_read(fd, size, win, st[VC_ST_WIN], box, mbox) 483 sys_close(fd) 484 let dn: i64 = mbox[VC_MB_LEN] 485 var o1: i64 = vc_cat(line, 0, "VARCENSUS-META found=" as *u8) 486 o1 = vc_catn(line, o1, mbox[VC_MB_FOUND]) 487 o1 = vc_cat(line, o1, " method=" as *u8) 488 o1 = vc_catn(line, o1, mbox[VC_MB_METHOD]) 489 o1 = vc_cat(line, o1, " md=" as *u8) 490 o1 = vc_cat(line, o1, vc_md_name(mbox[VC_MB_STATE])) 491 o1 = vc_cat(line, o1, " bytes=" as *u8) 492 o1 = vc_catn(line, o1, dn) 493 o1 = vc_cat(line, o1, "\n" as *u8) 494 vc_wn(VC_FD_OUT, line, o1) 495 if (data as i64) == 0 { return VC_EXIT_REFUSED } 496 let top: i64 = vf_json_skip(data, dn, 0) 497 if vf_json_kind(data, dn, top) != VF_JK_OBJECT { vc_w(VC_FD_OUT, "VARCENSUS-META json=NOT-OBJECT\n" as *u8); return VC_EXIT_REFUSED } 498 let end: i64 = vf_json_value_end(data, dn, top) 499 if end < 0 { vc_w(VC_FD_OUT, "VARCENSUS-META json=UNTERMINATED\n" as *u8); return VC_EXIT_REFUSED } 500 let mb: *i64 = sys_mmap(VF_JM_N * 8) as *i64 501 let db: *i64 = sys_mmap(VF_JM_N * 8) as *i64 502 let lenb: *i64 = sys_mmap(16) as *i64 503 var p: i64 = vf_json_skip(data, end, top + 1) 504 var members: i64 = 0 505 var run: i64 = 1 506 while run == 1 { 507 if p >= end - 1 { run = 0 } else { 508 let nx: i64 = vf_json_member(data, end, p, mb) 509 if nx < 0 { run = 0; vc_w(VC_FD_OUT, "VARCENSUS-META json=MALFORMED-MEMBER\n" as *u8) } else { 510 members = members + 1 511 let vs: i64 = mb[VF_JM_VS] 512 let ve: i64 = mb[VF_JM_VE] 513 let kind: i64 = vf_json_kind(data, dn, vs) 514 var o2: i64 = vc_cat(line, 0, "K " as *u8) 515 o2 = vc_catb(line, o2, data, mb[VF_JM_KS], mb[VF_JM_KE] - mb[VF_JM_KS]) 516 o2 = vc_cat(line, o2, " kind=" as *u8) 517 o2 = vc_cat(line, o2, vc_kind_name(kind)) 518 o2 = vc_cat(line, o2, " span=" as *u8) 519 o2 = vc_catn(line, o2, ve - vs) 520 if kind == VF_JK_STRING { 521 let sb: *u8 = sys_mmap(ve - vs + 1) 522 let sl: i64 = vf_json_decode(data, vs + 1, ve - 1, sb) 523 o2 = vc_cat(line, o2, " len=" as *u8) 524 o2 = vc_catn(line, o2, sl) 525 if sl >= 0 { if sl <= VC_SHOW_STR_MAX { o2 = vc_cat(line, o2, " value=" as *u8); o2 = vc_catb(line, o2, sb, 0, sl) } else { o2 = vc_cat(line, o2, " shown=0" as *u8) } } 526 } 527 if kind == VF_JK_ARRAY { 528 o2 = vc_cat(line, o2, " elems=" as *u8) 529 o2 = vc_catn(line, o2, vf_json_count_elems(data, dn, vs)) 530 } 531 if kind == VF_JK_OBJECT { 532 o2 = vc_cat(line, o2, " members=" as *u8) 533 o2 = vc_catn(line, o2, vf_json_count_members(data, dn, vs)) 534 } 535 if kind == VF_JK_SCALAR { 536 o2 = vc_cat(line, o2, " raw=" as *u8) 537 o2 = vc_catb(line, o2, data, vs, ve - vs) 538 } 539 o2 = vc_cat(line, o2, "\n" as *u8) 540 vc_wn(VC_FD_OUT, line, o2) 541 if kind == VF_JK_OBJECT { if vf_span_is(data, mb[VF_JM_KS], mb[VF_JM_KE], VC_DEPS_KEY) == 1 { 542 let dend: i64 = ve 543 var q: i64 = vf_json_skip(data, dend, vs + 1) 544 var drun: i64 = 1 545 while drun == 1 { 546 if q >= dend - 1 { drun = 0 } else { 547 let qn: i64 = vf_json_member(data, dend, q, db) 548 if qn < 0 { drun = 0 } else { 549 var o5: i64 = vc_cat(line, 0, " D " as *u8) 550 o5 = vc_catb(line, o5, data, db[VF_JM_KS], db[VF_JM_KE] - db[VF_JM_KS]) 551 let dk: i64 = vf_json_kind(data, dn, db[VF_JM_VS]) 552 o5 = vc_cat(line, o5, " kind=" as *u8) 553 o5 = vc_cat(line, o5, vc_kind_name(dk)) 554 if dk == VF_JK_OBJECT { 555 o5 = vc_cat(line, o5, " members=" as *u8) 556 o5 = vc_catn(line, o5, vf_json_count_members(data, dn, db[VF_JM_VS])) 557 let lic: *u8 = vf_json_get_str(data, dn, db[VF_JM_VS], VC_LIC_KEY, lenb) 558 if (lic as i64) != 0 { if lenb[0] <= VC_SHOW_STR_MAX { o5 = vc_cat(line, o5, " dep_licenseType=" as *u8); o5 = vc_catb(line, o5, lic, 0, lenb[0]) } } 559 } 560 o5 = vc_cat(line, o5, "\n" as *u8) 561 vc_wn(VC_FD_OUT, line, o5) 562 q = qn 563 } 564 } 565 } 566 } } 567 if kind == VF_JK_ARRAY { if vf_span_is(data, mb[VF_JM_KS], mb[VF_JM_KE], VC_CONTENT_KEY) == 1 { 568 let aend: i64 = ve 569 var r: i64 = vf_json_skip(data, aend, vs + 1) 570 var shown: i64 = 0 571 var arun: i64 = 1 572 while arun == 1 { 573 if r >= aend - 1 { arun = 0 } else { if shown >= VC_SHOW_ELEMS { arun = 0 } else { 574 let re: i64 = vf_json_value_end(data, aend, r) 575 if re < 0 { arun = 0 } else { 576 var o6: i64 = vc_cat(line, 0, " C " as *u8) 577 if vf_json_kind(data, dn, r) == VF_JK_STRING { 578 let cb: *u8 = sys_mmap(re - r + 1) 579 let cl: i64 = vf_json_decode(data, r + 1, re - 1, cb) 580 if cl >= 0 { o6 = vc_catb(line, o6, cb, 0, cl) } 581 } else { o6 = vc_cat(line, o6, "(non-string)" as *u8) } 582 o6 = vc_cat(line, o6, "\n" as *u8) 583 vc_wn(VC_FD_OUT, line, o6) 584 shown = shown + 1 585 var rq: i64 = vf_json_skip(data, aend, re) 586 if rq < aend { if (data[rq] as i64 & 0xff) == VF_J_COMMA { rq = vf_json_skip(data, aend, rq + 1) } } 587 if rq <= r { arun = 0 } else { r = rq } 588 } 589 } } 590 } 591 } } 592 p = nx 593 } 594 } 595 } 596 let legacy: *u8 = sys_mmap(VC_LEGACY_LIC) 597 let lr: i64 = vf_license(data, dn, 0, dn, legacy) 598 let dlen: *i64 = sys_mmap(16) as *i64 599 let dlic: *u8 = vf_json_get_str(data, dn, top, VC_LIC_KEY, dlen) 600 var o7: i64 = vc_cat(line, 0, "VARCENSUS-META members=" as *u8) 601 o7 = vc_catn(line, o7, members) 602 o7 = vc_cat(line, o7, " legacy_license=" as *u8) 603 if lr > 0 { o7 = vc_cat(line, o7, legacy) } else { o7 = vc_cat(line, o7, "unparsed" as *u8) } 604 o7 = vc_cat(line, o7, " direct_license=" as *u8) 605 if (dlic as i64) != 0 { if dlen[0] <= VC_SHOW_STR_MAX { o7 = vc_catb(line, o7, dlic, 0, dlen[0]) } } else { o7 = vc_cat(line, o7, "absent" as *u8) } 606 o7 = vc_cat(line, o7, "\n" as *u8) 607 vc_wn(VC_FD_OUT, line, o7) 608 return VC_EXIT_OK 609} 610 611// ---------------------------------------------------------------- census ---------------------------------------------------------------- 612 613func vc_cmp(a: *u8, b: *u8) -> i64 { 614 var i: i64 = 0 615 var run: i64 = 1 616 while run == 1 { 617 let ca: i64 = a[i] as i64 & 0xff 618 let cb: i64 = b[i] as i64 & 0xff 619 if ca != cb { if ca < cb { return 0 - 1 } return 1 } 620 if ca == 0 { run = 0 } else { i = i + 1 } 621 } 622 return 0 623} 624// Shell sort of the path offsets by the bytes of their paths: the report reads in one deterministic order on every run 625func vc_sort(paths: *i64, offs: *i64) -> i64 { 626 let n: i64 = vcl_vec_n(offs) 627 let v: *i64 = offs[VCL_A_PTR] as *i64 628 var gap: i64 = n / 2 629 while gap > 0 { 630 var i: i64 = gap 631 while i < n { 632 let t: i64 = v[i] 633 var j: i64 = i 634 var go: i64 = 1 635 while go == 1 { 636 if j < gap { go = 0 } else { 637 if vc_cmp(vcl_at(paths, v[j - gap]), vcl_at(paths, t)) > 0 { v[j] = v[j - gap]; j = j - gap } else { go = 0 } 638 } 639 } 640 v[j] = t 641 i = i + 1 642 } 643 gap = gap / 2 644 } 645 return n 646} 647 648// the lowercased extension of the basename of s[0..n): the bytes after its last dot, or "(none)" 649func vc_ext_key(s: *u8, n: i64, out: *u8) -> i64 { 650 var slash: i64 = 0 - 1 651 var dot: i64 = 0 - 1 652 var i: i64 = 0 653 while i < n { 654 let c: i64 = s[i] as i64 & 0xff 655 if c == VC_SLASH { slash = i; dot = 0 - 1 } 656 if c == VC_DOT { dot = i } 657 i = i + 1 658 } 659 if dot < 0 { return vc_cat(out, 0, "(none)" as *u8) } 660 if dot <= slash { return vc_cat(out, 0, "(none)" as *u8) } 661 var k: i64 = 0 662 var j: i64 = dot 663 while j < n { out[k] = vcl_lc(s[j] as i64 & 0xff) as u8; k = k + 1; j = j + 1 } 664 out[k] = 0 as u8 665 return k 666} 667// the first two path segments of a lowercased member name ("(root)" for a member with no slash): the residual worklist key 668func vc_prefix2_key(s: *u8, n: i64, out: *u8) -> i64 { 669 var first: i64 = 0 - 1 670 var second: i64 = 0 - 1 671 var i: i64 = 0 672 while i < n { 673 if (s[i] as i64 & 0xff) == VC_SLASH { if first < 0 { first = i } else { if second < 0 { second = i } } } 674 i = i + 1 675 } 676 if first < 0 { return vc_cat(out, 0, "(root)" as *u8) } 677 var e: i64 = n 678 if second >= 0 { e = second + 1 } 679 var k: i64 = 0 680 while k < e { out[k] = s[k]; k = k + 1 } 681 out[k] = 0 as u8 682 return k 683} 684 685// walk root recursively: every .var path (case-insensitive suffix) into paths/offs; other files counted by extension; 686// directories that cannot be opened, symlinks, other node types and over-long paths are COUNTED and named, never dropped 687func vc_walk(root: *u8, paths: *i64, offs: *i64, exts: *i64, ws: *i64, unread: *i64) -> i64 { 688 let dirs: *i64 = vcl_arena_new(VCL_ARENA_INIT) 689 let dq: *i64 = vcl_arena_new(VCL_ARENA_INIT) 690 let rl: i64 = vc_slen(root) 691 vcl_push(dq, vcl_put(dirs, root, 0, rl)) 692 vcl_push(dq, 0) 693 let cur: *u8 = sys_mmap(VC_PATH_MAX + VC_PAD) 694 let child: *u8 = sys_mmap(VC_PATH_MAX + VC_PAD) 695 let dbuf: *u8 = sys_mmap(VC_DENT_BUF + VC_PAD) 696 let stbuf: *u8 = sys_mmap(VC_STAT_BUF + VC_PAD) 697 let extb: *u8 = sys_mmap(VC_PATH_MAX + VC_PAD) 698 var qi: i64 = 0 699 while qi < vcl_vec_n(dq) / 2 { 700 let doff: i64 = vcl_vec_at(dq, qi * 2) 701 let depth: i64 = vcl_vec_at(dq, qi * 2 + 1) 702 let dp: *u8 = vcl_at(dirs, doff) 703 let cl: i64 = vc_slen(dp) 704 var ci: i64 = 0 705 while ci < cl { cur[ci] = dp[ci]; ci = ci + 1 } 706 cur[cl] = 0 as u8 707 let fd: i64 = sys_openat_directory(cur) 708 if fd < 0 { 709 ws[VC_W_DIRS_UNREADABLE] = ws[VC_W_DIRS_UNREADABLE] + 1 710 vcl_apps(unread, "DIR-UNREADABLE" as *u8) 711 vcl_appc(unread, VC_TAB) 712 vcl_appclean(unread, cur, 0, cl) 713 vcl_appc(unread, VC_TAB) 714 vcl_appn(unread, fd) 715 vcl_appc(unread, VC_LF) 716 } else { 717 ws[VC_W_DIRS] = ws[VC_W_DIRS] + 1 718 var run: i64 = 1 719 while run == 1 { 720 let nb: i64 = sys_getdents64(fd, dbuf, VC_DENT_BUF) 721 if nb <= 0 { run = 0 } else { 722 var off: i64 = 0 723 while off < nb { 724 let rec: *u8 = ((dbuf as i64) + off) as *u8 725 let reclen: i64 = dirent_reclen(rec) 726 if reclen <= 0 { off = nb } else { 727 let nm: *u8 = dirent_name(rec) 728 let nml: i64 = vc_slen(nm) 729 var dot: i64 = 0 730 if vc_streq(nm, "." as *u8) == 1 { dot = 1 } 731 if vc_streq(nm, ".." as *u8) == 1 { dot = 1 } 732 if dot == 0 { 733 var co: i64 = cl 734 var ci2: i64 = 0 735 while ci2 < cl { child[ci2] = cur[ci2]; ci2 = ci2 + 1 } 736 var need_slash: i64 = 1 737 if cl > 0 { if (cur[cl-1] as i64 & 0xff) == VC_SLASH { need_slash = 0 } } 738 if co + need_slash + nml >= VC_PATH_MAX { 739 ws[VC_W_TOO_LONG] = ws[VC_W_TOO_LONG] + 1 740 vcl_apps(unread, "PATH-TOO-LONG" as *u8) 741 vcl_appc(unread, VC_TAB) 742 vcl_appclean(unread, cur, 0, cl) 743 vcl_appc(unread, VC_LF) 744 } else { 745 if need_slash == 1 { child[co] = VC_SLASH as u8; co = co + 1 } 746 var cj: i64 = 0 747 while cj < nml { child[co + cj] = nm[cj]; cj = cj + 1 } 748 co = co + nml 749 child[co] = 0 as u8 750 var t: i64 = dirent_type(rec) 751 if t == VC_DT_UNKNOWN { 752 if sys_fstatat(child, stbuf) == 0 { 753 let mode: i64 = vf_u32(stbuf, VC_S_MODE) & VC_S_IFMT 754 if mode == VC_S_IFDIR { t = DT_DIR } 755 if mode == VC_S_IFREG { t = DT_REG } 756 } 757 } 758 if t == DT_DIR { 759 vcl_push(dq, vcl_put(dirs, child, 0, co)) 760 vcl_push(dq, depth + 1) 761 } else { 762 if t == DT_REG { 763 var isvar: i64 = 0 764 if nml >= 4 { 765 if (nm[nml-4] as i64 & 0xff) == VC_DOT { 766 if vcl_lc(nm[nml-3] as i64 & 0xff) == 118 { if vcl_lc(nm[nml-2] as i64 & 0xff) == 97 { if vcl_lc(nm[nml-1] as i64 & 0xff) == 114 { isvar = 1 } } } 767 } 768 } 769 if isvar == 1 { 770 vcl_push(offs, vcl_put(paths, child, 0, co)) 771 ws[VC_W_VARS] = ws[VC_W_VARS] + 1 772 if depth == 0 { ws[VC_W_VARS_TOP] = ws[VC_W_VARS_TOP] + 1 } 773 } else { 774 ws[VC_W_OTHER_FILES] = ws[VC_W_OTHER_FILES] + 1 775 let el: i64 = vc_ext_key(nm, nml, extb) 776 vcl_counter_add(exts, extb, 0, el, 1) 777 } 778 } else { 779 if t == DT_LNK { ws[VC_W_SYMLINKS] = ws[VC_W_SYMLINKS] + 1 } else { ws[VC_W_OTHER_TYPES] = ws[VC_W_OTHER_TYPES] + 1 } 780 } 781 } 782 } 783 } 784 off = off + reclen 785 } 786 } 787 } 788 } 789 sys_close(fd) 790 } 791 qi = qi + 1 792 } 793 return ws[VC_W_VARS] 794} 795 796func vc_write_out(dir: *u8, name: *u8, a: *i64, total: *i64) -> i64 { 797 let p: *u8 = sys_mmap(VC_PATH_MAX + VC_PAD) 798 let dl: i64 = vc_slen(dir) 799 if dl + vc_slen(name) + 2 >= VC_PATH_MAX { return 0 - 1 } 800 var o: i64 = vc_cat(p, 0, dir) 801 if o > 0 { if (p[o-1] as i64 & 0xff) != VC_SLASH { p[o] = VC_SLASH as u8; o = o + 1 } } 802 o = vc_cat(p, o, name) 803 p[o] = 0 as u8 804 let fd: i64 = sys_openat_wr(p, MODE_0644) 805 if fd < 0 { return 0 - 1 } 806 let w: i64 = vc_wn(fd, vcl_at(a, 0), vcl_used(a)) 807 sys_close(fd) 808 total[0] = total[0] + w 809 if w != vcl_used(a) { return 0 - 1 } 810 return w 811} 812 813func vc_kv(a: *i64, k: *u8, v: i64) -> i64 { 814 vcl_appc(a, 32) 815 vcl_apps(a, k) 816 vcl_appc(a, VC_EQ) 817 vcl_appn(a, v) 818 return 0 819} 820 821// decode the DIRECT string member key of the object at obj into out (caller sizes it >= n + 1); length or -1 822func vc_get_str(data: *u8, n: i64, obj: i64, key: *u8, jm: *i64, out: *u8) -> i64 { 823 out[0] = 0 as u8 824 let vs: i64 = vf_json_find(data, n, obj, key, jm) 825 if vs < 0 { return 0 - 1 } 826 if vf_json_kind(data, n, vs) != VF_JK_STRING { return 0 - 1 } 827 return vf_json_decode(data, vs + 1, jm[VF_JM_VE] - 1, out) 828} 829 830// append "name=count" for every class of an axis with a non-zero count, comma separated ("-" when none) 831func vc_app_counts(a: *i64, rs: *i64, axis: i64, counts: *i64, nclass: i64) -> i64 { 832 var first: i64 = 1 833 var c: i64 = 0 834 while c < nclass { 835 if counts[c] > 0 { 836 if first == 0 { vcl_appc(a, VC_ROW_SEP) } 837 vcl_apps(a, vcl_class_name(rs, axis, c)) 838 vcl_appc(a, VC_EQ) 839 vcl_appn(a, counts[c]) 840 first = 0 841 } 842 c = c + 1 843 } 844 if first == 1 { vcl_appc(a, VC_ASCII_MINUS) } 845 return 0 846} 847 848func vc_counter_tsv(a: *i64, c: *i64, prefix: *u8) -> i64 { 849 var i: i64 = 0 850 while i < vcl_counter_n(c) { 851 if vc_slen(prefix) > 0 { vcl_apps(a, prefix); vcl_appc(a, VC_TAB) } 852 vcl_appclean(a, vcl_counter_key(c, i), 0, vcl_counter_klen(c, i)) 853 vcl_appc(a, VC_TAB) 854 vcl_appn(a, vcl_counter_count(c, i)) 855 vcl_appc(a, VC_LF) 856 i = i + 1 857 } 858 return i 859} 860 861func vc_census(root: *u8, outdir: *u8, rulespath: *u8) -> i64 { 862 let rlen: *i64 = sys_mmap(16) as *i64 863 let rtext: *u8 = sys_read_file(rulespath, rlen) 864 if (rtext as i64) == 0 { vc_w(VC_FD_OUT, "VARCENSUS state=RULES-UNREADABLE\n" as *u8); return VC_EXIT_IO } 865 let rs: *i64 = vcl_rules_parse(rtext, rlen[0]) 866 let sm: *i64 = vcl_arena_new(VCL_ARENA_INIT) 867 if rs[VCL_RS_ERRORS] > 0 { 868 vcl_apps(sm, "VARCENSUS state=RULES-MALFORMED" as *u8) 869 vc_kv(sm, "errors" as *u8, rs[VCL_RS_ERRORS]) 870 vc_kv(sm, "first_line" as *u8, rs[VCL_RS_FIRST_ERR_LINE]) 871 vcl_appc(sm, VC_LF) 872 vc_wn(VC_FD_OUT, vcl_at(sm, 0), vcl_used(sm)) 873 return VC_EXIT_REFUSED 874 } 875 sys_mkdir(outdir, MODE_0755) 876 let nrole: i64 = vcl_nclasses(rs, VCL_AX_ROLE) 877 let nfmt: i64 = vcl_nclasses(rs, VCL_AX_FORMAT) 878 let tags: *i64 = rs[VCL_RS_TAGS] as *i64 879 let ntags: i64 = vcl_counter_n(tags) 880 let other_role: i64 = rs[VCL_RS_OTHER_ROLE] 881 let other_fmt: i64 = rs[VCL_RS_OTHER_FORMAT] 882 883 // ---- walk ---- 884 let paths: *i64 = vcl_arena_new(VCL_ARENA_INIT) 885 let offs: *i64 = vcl_arena_new(VCL_ARENA_INIT) 886 let exts: *i64 = vcl_counter_new() 887 let ws: *i64 = sys_mmap(VC_W_N * 8) as *i64 888 let ur: *i64 = vcl_arena_new(VCL_ARENA_INIT) 889 vcl_apps(ur, "state\tfile\tdetail\n" as *u8) 890 vc_walk(root, paths, offs, exts, ws, ur) 891 let npk: i64 = vc_sort(paths, offs) 892 let rootl: i64 = vc_slen(root) 893 894 // ---- outputs ---- 895 let pk: *i64 = vcl_arena_new(VCL_ARENA_INIT) 896 vcl_apps(pk, "file\tsize\tstate\tentries\tdirs\tz64_members\tz64_unresolved\tutf8_members\tmeta\tlicense\tcreator\tpackage\tprogram_version\tcontent_elems\tdeps_direct\tdeptags\trp_screen\tusize_sum\trole_counts\tformat_counts\n" as *u8) 897 let dpa: *i64 = vcl_arena_new(VCL_ARENA_INIT) 898 vcl_apps(dpa, "from\tto\tto_license\tdeptag\n" as *u8) 899 let rpa: *i64 = vcl_arena_new(VCL_ARENA_INIT) 900 vcl_apps(rpa, "file\treasons\tlicense\n" as *u8) 901 let labels: *i64 = vcl_counter_new() 902 let creators: *i64 = vcl_counter_new() 903 let targets: *i64 = vcl_counter_new() 904 let resrole: *i64 = vcl_counter_new() 905 let resfmt: *i64 = vcl_counter_new() 906 907 let role_members: *i64 = sys_mmap((nrole + 1) * 8) as *i64 908 let role_pkgs: *i64 = sys_mmap((nrole + 1) * 8) as *i64 909 let role_usize: *i64 = sys_mmap((nrole + 1) * 8) as *i64 910 let fmt_members: *i64 = sys_mmap((nfmt + 1) * 8) as *i64 911 let fmt_pkgs: *i64 = sys_mmap((nfmt + 1) * 8) as *i64 912 let fmt_usize: *i64 = sys_mmap((nfmt + 1) * 8) as *i64 913 let prole: *i64 = sys_mmap((nrole + 1) * 8) as *i64 914 let pfmt: *i64 = sys_mmap((nfmt + 1) * 8) as *i64 915 let pstates: *i64 = sys_mmap(VC_P_N * 8) as *i64 916 let mstates: *i64 = sys_mmap(VC_M_N * 8) as *i64 917 let lagree: *i64 = sys_mmap(VC_L_N * 8) as *i64 918 let tag_pkgs: *i64 = sys_mmap((ntags + 1) * 8) as *i64 919 let tag_edges: *i64 = sys_mmap((ntags + 1) * 8) as *i64 920 let ptag: *i64 = sys_mmap((ntags + 1) * 8) as *i64 921 let reason_pkgs: *i64 = sys_mmap(VC_NLETTERS * 8) as *i64 922 var t_members: i64 = 0 923 var t_dirs: i64 = 0 924 var t_z64: i64 = 0 925 var t_z64u: i64 = 0 926 var t_pkgs_z64: i64 = 0 927 var t_utf8: i64 = 0 928 var t_usize: i64 = 0 929 var t_edges: i64 = 0 930 var t_pkgs_deps: i64 = 0 931 var t_deps_malformed: i64 = 0 932 var t_flagged: i64 = 0 933 var t_role_sum: i64 = 0 934 var t_fmt_sum: i64 = 0 935 936 let box: *i64 = sys_mmap(VF_L_N * 8) as *i64 937 let st: *i64 = sys_mmap(VC_ST_N * 8) as *i64 938 let rec: *i64 = sys_mmap(VF_E_N * 8) as *i64 939 let namebuf: *u8 = sys_mmap(VC_NAME_BUF + VC_PAD) 940 let keyb: *u8 = sys_mmap(VC_NAME_BUF + VC_PAD) 941 let idlc: *u8 = sys_mmap(VC_PATH_MAX + VC_PAD) 942 let mbox: *i64 = sys_mmap(VC_MB_N * 8) as *i64 943 let jm: *i64 = sys_mmap(VF_JM_N * 8) as *i64 944 let jm2: *i64 = sys_mmap(VF_JM_N * 8) as *i64 945 let headb: *u8 = sys_mmap(VC_HEAD_LEN + VC_PAD) 946 let db: *i64 = sys_mmap(VF_JM_N * 8) as *i64 947 let legacy: *u8 = sys_mmap(VC_LEGACY_LIC) 948 949 var i: i64 = 0 950 while i < npk { 951 let path: *u8 = vcl_at(paths, vcl_vec_at(offs, i)) 952 let pl: i64 = vc_slen(path) 953 var rel: i64 = rootl 954 if rel < pl { if (path[rel] as i64 & 0xff) == VC_SLASH { rel = rel + 1 } } 955 // the package id: the relative path's basename without its 4-byte .var suffix 956 var bs: i64 = rel 957 var bi: i64 = rel 958 while bi < pl { if (path[bi] as i64 & 0xff) == VC_SLASH { bs = bi + 1 } bi = bi + 1 } 959 let idl: i64 = pl - 4 - bs 960 vcl_lower(((path as i64) + bs) as *u8, idl, idlc) 961 962 var state: i64 = VC_P_OK 963 var size: i64 = 0 - 1 964 var win: *u8 = 0 as *u8 965 let fd: i64 = sys_openat_rd(path) 966 if fd < 0 { state = VC_P_UNOPENABLE } else { 967 size = sys_lseek(fd, 0, VC_SEEK_END) 968 win = vc_window(fd, size, box, st) 969 state = vc_pstate_of(st[VC_ST_LOC]) 970 // split NO-EOCD by what the file IS: zero bytes, a ZIP local header with no EOCD, or not a ZIP at all 971 if state == VC_P_NOEOCD { 972 if size == 0 { state = VC_P_EMPTY } else { 973 state = VC_P_NOEOCD_OTHER 974 if size >= VC_HEAD_LEN { if vc_pread(fd, 0, VC_HEAD_LEN, headb) == VC_HEAD_LEN { if vf_u32(headb, 0) == VF_SIG_LOCAL { state = VC_P_NOEOCD_PK } } } 975 } 976 } 977 } 978 var c: i64 = 0 979 while c < nrole { prole[c] = 0; c = c + 1 } 980 c = 0 981 while c < nfmt { pfmt[c] = 0; c = c + 1 } 982 var p_entries: i64 = 0 983 var p_dirs: i64 = 0 984 var p_z64: i64 = 0 985 var p_z64u: i64 = 0 986 var p_utf8: i64 = 0 987 var p_usize: i64 = 0 988 if state == VC_P_OK { 989 p_entries = box[VF_L_ENTRIES] 990 var off: i64 = box[VF_L_CD] 991 var k: i64 = 0 992 var go: i64 = 1 993 while go == 1 { 994 if k >= p_entries { go = 0 } else { 995 if vf_cd_entry(win, st[VC_ST_WIN], off, rec) != 1 { state = VC_P_RECORD; go = 0 } else { 996 let nl: i64 = rec[VF_E_NAMELEN] 997 vcl_lower(((win as i64) + rec[VF_E_NAMEOFF]) as *u8, nl, namebuf) 998 if rec[VF_E_Z64] == VF_Z64_RESOLVED { p_z64 = p_z64 + 1 } 999 if (rec[VF_E_FLAGS] / VC_FLAG_UTF8) % 2 == 1 { p_utf8 = p_utf8 + 1 } 1000 if nl > 0 { if (namebuf[nl-1] as i64 & 0xff) == VC_SLASH { p_dirs = p_dirs + 1 } } 1001 let r: i64 = vcl_classify(rs, VCL_AX_ROLE, namebuf, nl) 1002 let f: i64 = vcl_classify(rs, VCL_AX_FORMAT, namebuf, nl) 1003 prole[r] = prole[r] + 1 1004 pfmt[f] = pfmt[f] + 1 1005 if rec[VF_E_Z64] == VF_Z64_UNRESOLVED { p_z64u = p_z64u + 1 } else { 1006 p_usize = p_usize + rec[VF_E_USIZE] 1007 role_usize[r] = role_usize[r] + rec[VF_E_USIZE] 1008 fmt_usize[f] = fmt_usize[f] + rec[VF_E_USIZE] 1009 } 1010 if r == other_role { let kl: i64 = vc_prefix2_key(namebuf, nl, keyb); vcl_counter_add(resrole, keyb, 0, kl, 1) } 1011 if f == other_fmt { let kl2: i64 = vc_ext_key(namebuf, nl, keyb); vcl_counter_add(resfmt, keyb, 0, kl2, 1) } 1012 off = rec[VF_E_NEXT] 1013 k = k + 1 1014 } 1015 } 1016 } 1017 } 1018 pstates[state] = pstates[state] + 1 1019 if state != VC_P_OK { 1020 vcl_apps(ur, vc_pstate_name(state)) 1021 vcl_appc(ur, VC_TAB) 1022 vcl_appclean(ur, path, rel, pl - rel) 1023 vcl_appc(ur, VC_TAB) 1024 vcl_apps(ur, "size=" as *u8) 1025 vcl_appn(ur, size) 1026 vcl_appc(ur, VC_LF) 1027 if (win as i64) != 0 { sys_munmap(win, st[VC_ST_WIN] + VC_PAD) } 1028 if fd >= 0 { sys_close(fd) } 1029 } else { 1030 // fold the package into the corpus only once its whole central directory read cleanly 1031 t_members = t_members + p_entries 1032 t_dirs = t_dirs + p_dirs 1033 t_z64 = t_z64 + p_z64 1034 t_z64u = t_z64u + p_z64u 1035 if p_z64 > 0 { t_pkgs_z64 = t_pkgs_z64 + 1 } 1036 t_utf8 = t_utf8 + p_utf8 1037 t_usize = t_usize + p_usize 1038 c = 0 1039 while c < nrole { role_members[c] = role_members[c] + prole[c]; if prole[c] > 0 { role_pkgs[c] = role_pkgs[c] + 1 } t_role_sum = t_role_sum + prole[c]; c = c + 1 } 1040 c = 0 1041 while c < nfmt { fmt_members[c] = fmt_members[c] + pfmt[c]; if pfmt[c] > 0 { fmt_pkgs[c] = fmt_pkgs[c] + 1 } t_fmt_sum = t_fmt_sum + pfmt[c]; c = c + 1 } 1042 1043 let data: *u8 = vc_meta_read(fd, size, win, st[VC_ST_WIN], box, mbox) 1044 sys_close(fd) 1045 let dn: i64 = mbox[VC_MB_LEN] 1046 var mstate: i64 = VC_M_ABSENT 1047 if mbox[VC_MB_FOUND] < 0 { mstate = VC_M_UNREADABLE } 1048 if mbox[VC_MB_FOUND] == 1 { if (data as i64) == 0 { mstate = VC_M_UNREADABLE } } 1049 var top: i64 = 0 - 1 1050 if (data as i64) != 0 { 1051 top = vf_json_skip(data, dn, 0) 1052 mstate = VC_M_UNPARSED 1053 if vf_json_kind(data, dn, top) == VF_JK_OBJECT { if vf_json_value_end(data, dn, top) >= 0 { mstate = VC_M_PARSED } } 1054 } 1055 mstates[mstate] = mstates[mstate] + 1 1056 var fbuf: *u8 = 0 as *u8 1057 var fcell: i64 = 0 1058 if mstate == VC_M_PARSED { fcell = dn + 1; fbuf = sys_mmap(fcell * 6 + VC_PAD) } 1059 let lb: *u8 = ((fbuf as i64) + 0) as *u8 1060 let cbuf: *u8 = ((fbuf as i64) + fcell) as *u8 1061 let pbuf: *u8 = ((fbuf as i64) + fcell * 2) as *u8 1062 let vbuf: *u8 = ((fbuf as i64) + fcell * 3) as *u8 1063 let kbuf: *u8 = ((fbuf as i64) + fcell * 4) as *u8 1064 let dlbuf: *u8 = ((fbuf as i64) + fcell * 5) as *u8 1065 var ll: i64 = 0 - 1 1066 var crl: i64 = 0 - 1 1067 var pnl: i64 = 0 - 1 1068 var pvl: i64 = 0 - 1 1069 var celems: i64 = 0 - 1 1070 var pdeps: i64 = 0 1071 c = 0 1072 while c < ntags { ptag[c] = 0; c = c + 1 } 1073 if mstate == VC_M_PARSED { 1074 ll = vc_get_str(data, dn, top, VC_LIC_KEY, jm, lb) 1075 crl = vc_get_str(data, dn, top, VC_CREATOR_KEY, jm, cbuf) 1076 pnl = vc_get_str(data, dn, top, VC_PACKAGE_KEY, jm, pbuf) 1077 pvl = vc_get_str(data, dn, top, VC_PROGVER_KEY, jm, vbuf) 1078 let cv: i64 = vf_json_find(data, dn, top, VC_CONTENT_KEY, jm) 1079 if cv >= 0 { celems = vf_json_count_elems(data, dn, cv) } 1080 // legacy first-occurrence licence reader against the depth-aware direct member: agreement MEASURED 1081 let lr: i64 = vf_license(data, dn, 0, dn, legacy) 1082 var li: i64 = VC_L_NEITHER 1083 if lr > 0 { if ll >= 0 { 1084 li = VC_L_DISAGREE 1085 if lr == ll { if vcl_eq(legacy, 0, lb, 0, ll) == 1 { li = VC_L_AGREE } } 1086 } else { li = VC_L_LEGACY_ONLY } } else { if ll >= 0 { li = VC_L_DIRECT_ONLY } } 1087 lagree[li] = lagree[li] + 1 1088 // direct dependency edges 1089 let dv: i64 = vf_json_find(data, dn, top, VC_DEPS_KEY, jm) 1090 if dv >= 0 { if vf_json_kind(data, dn, dv) == VF_JK_OBJECT { 1091 let dend: i64 = jm[VF_JM_VE] 1092 var q: i64 = vf_json_skip(data, dend, dv + 1) 1093 var drun: i64 = 1 1094 while drun == 1 { 1095 if q >= dend - 1 { drun = 0 } else { 1096 let qn: i64 = vf_json_member(data, dend, q, db) 1097 if qn < 0 { drun = 0; t_deps_malformed = t_deps_malformed + 1 } else { 1098 let kl3: i64 = vf_json_decode(data, db[VF_JM_KS], db[VF_JM_KE], kbuf) 1099 if kl3 >= 0 { 1100 pdeps = pdeps + 1 1101 vcl_lower(kbuf, kl3, keyb) 1102 let tg: i64 = vcl_deptag(rs, keyb, kl3) 1103 var dll: i64 = 0 - 1 1104 if vf_json_kind(data, dn, db[VF_JM_VS]) == VF_JK_OBJECT { dll = vc_get_str(data, dn, db[VF_JM_VS], VC_LIC_KEY, jm2, dlbuf) } 1105 vcl_appclean(dpa, path, bs, idl) 1106 vcl_appc(dpa, VC_TAB) 1107 vcl_appclean(dpa, kbuf, 0, kl3) 1108 vcl_appc(dpa, VC_TAB) 1109 if dll >= 0 { vcl_appclean(dpa, dlbuf, 0, dll) } else { vcl_appc(dpa, VC_ASCII_MINUS) } 1110 vcl_appc(dpa, VC_TAB) 1111 if tg >= 0 { vcl_apps(dpa, vcl_counter_key(tags, tg)); ptag[tg] = 1; tag_edges[tg] = tag_edges[tg] + 1 } else { vcl_appc(dpa, VC_ASCII_MINUS) } 1112 vcl_appc(dpa, VC_LF) 1113 vcl_counter_add(targets, keyb, 0, kl3, 1) 1114 t_edges = t_edges + 1 1115 } 1116 q = qn 1117 } 1118 } 1119 } 1120 } } 1121 } 1122 if pdeps > 0 { t_pkgs_deps = t_pkgs_deps + 1 } 1123 c = 0 1124 while c < ntags { if ptag[c] == 1 { tag_pkgs[c] = tag_pkgs[c] + 1 } c = c + 1 } 1125 // the licence label partition (every OK package lands in exactly one row) 1126 if mstate == VC_M_PARSED { 1127 if ll > 0 { vcl_counter_add(labels, lb, 0, ll, 1) } else { if ll == 0 { vcl_counter_add(labels, "(empty)" as *u8, 0, 7, 1) } else { vcl_counter_add(labels, "(no-licenseType)" as *u8, 0, 16, 1) } } 1128 if crl > 0 { vcl_counter_add(creators, cbuf, 0, crl, 1) } else { vcl_counter_add(creators, "(no-creatorName)" as *u8, 0, 16, 1) } 1129 } else { 1130 if mstate == VC_M_ABSENT { vcl_counter_add(labels, "(meta-absent)" as *u8, 0, 13, 1) } 1131 if mstate == VC_M_UNREADABLE { vcl_counter_add(labels, "(meta-unreadable)" as *u8, 0, 17, 1) } 1132 if mstate == VC_M_UNPARSED { vcl_counter_add(labels, "(meta-unparsed)" as *u8, 0, 15, 1) } 1133 } 1134 // the real-person screen: reasons from role classes and the package id, never adjudicated here 1135 let mask: i64 = vcl_screen(rs, prole, idlc, idl) 1136 if mask != 0 { 1137 t_flagged = t_flagged + 1 1138 var m: i64 = mask 1139 var bit: i64 = 0 1140 while bit < VC_NLETTERS { if m % 2 == 1 { reason_pkgs[bit] = reason_pkgs[bit] + 1 } m = m / 2; bit = bit + 1 } 1141 vcl_appclean(rpa, path, rel, pl - rel) 1142 vcl_appc(rpa, VC_TAB) 1143 vcl_app_reasons(rpa, mask) 1144 vcl_appc(rpa, VC_TAB) 1145 if ll > 0 { vcl_appclean(rpa, lb, 0, ll) } else { vcl_appc(rpa, VC_ASCII_MINUS) } 1146 vcl_appc(rpa, VC_LF) 1147 } 1148 // the package row 1149 vcl_appclean(pk, path, rel, pl - rel) 1150 vcl_appc(pk, VC_TAB) 1151 vcl_appn(pk, size) 1152 vcl_appc(pk, VC_TAB) 1153 vcl_apps(pk, "OK" as *u8) 1154 vcl_appc(pk, VC_TAB) 1155 vcl_appn(pk, p_entries) 1156 vcl_appc(pk, VC_TAB) 1157 vcl_appn(pk, p_dirs) 1158 vcl_appc(pk, VC_TAB) 1159 vcl_appn(pk, p_z64) 1160 vcl_appc(pk, VC_TAB) 1161 vcl_appn(pk, p_z64u) 1162 vcl_appc(pk, VC_TAB) 1163 vcl_appn(pk, p_utf8) 1164 vcl_appc(pk, VC_TAB) 1165 if mstate == VC_M_PARSED { vcl_apps(pk, "parsed" as *u8) } 1166 if mstate == VC_M_ABSENT { vcl_apps(pk, "absent" as *u8) } 1167 if mstate == VC_M_UNREADABLE { vcl_apps(pk, "unreadable" as *u8) } 1168 if mstate == VC_M_UNPARSED { vcl_apps(pk, "unparsed" as *u8) } 1169 vcl_appc(pk, VC_TAB) 1170 if ll > 0 { vcl_appclean(pk, lb, 0, ll) } else { vcl_appc(pk, VC_ASCII_MINUS) } 1171 vcl_appc(pk, VC_TAB) 1172 if crl > 0 { vcl_appclean(pk, cbuf, 0, crl) } else { vcl_appc(pk, VC_ASCII_MINUS) } 1173 vcl_appc(pk, VC_TAB) 1174 if pnl > 0 { vcl_appclean(pk, pbuf, 0, pnl) } else { vcl_appc(pk, VC_ASCII_MINUS) } 1175 vcl_appc(pk, VC_TAB) 1176 if pvl > 0 { vcl_appclean(pk, vbuf, 0, pvl) } else { vcl_appc(pk, VC_ASCII_MINUS) } 1177 vcl_appc(pk, VC_TAB) 1178 vcl_appn(pk, celems) 1179 vcl_appc(pk, VC_TAB) 1180 vcl_appn(pk, pdeps) 1181 vcl_appc(pk, VC_TAB) 1182 var tfirst: i64 = 1 1183 c = 0 1184 while c < ntags { if ptag[c] == 1 { if tfirst == 0 { vcl_appc(pk, VC_ROW_SEP) } vcl_apps(pk, vcl_counter_key(tags, c)); tfirst = 0 } c = c + 1 } 1185 if tfirst == 1 { vcl_appc(pk, VC_ASCII_MINUS) } 1186 vcl_appc(pk, VC_TAB) 1187 vcl_app_reasons(pk, mask) 1188 vcl_appc(pk, VC_TAB) 1189 vcl_appn(pk, p_usize) 1190 vcl_appc(pk, VC_TAB) 1191 vc_app_counts(pk, rs, VCL_AX_ROLE, prole, nrole) 1192 vcl_appc(pk, VC_TAB) 1193 vc_app_counts(pk, rs, VCL_AX_FORMAT, pfmt, nfmt) 1194 vcl_appc(pk, VC_LF) 1195 if fcell > 0 { sys_munmap(fbuf, fcell * 6 + VC_PAD) } 1196 sys_munmap(win, st[VC_ST_WIN] + VC_PAD) 1197 } 1198 i = i + 1 1199 } 1200 1201 // ---- aggregate tables ---- 1202 let cls: *i64 = vcl_arena_new(VCL_ARENA_INIT) 1203 vcl_apps(cls, "axis\tclass\tmembers\tpackages\tusize_sum\n" as *u8) 1204 c = 0 1205 while c < nrole { 1206 vcl_apps(cls, "role\t" as *u8); vcl_apps(cls, vcl_class_name(rs, VCL_AX_ROLE, c)); vcl_appc(cls, VC_TAB) 1207 vcl_appn(cls, role_members[c]); vcl_appc(cls, VC_TAB); vcl_appn(cls, role_pkgs[c]); vcl_appc(cls, VC_TAB); vcl_appn(cls, role_usize[c]); vcl_appc(cls, VC_LF) 1208 c = c + 1 1209 } 1210 c = 0 1211 while c < nfmt { 1212 vcl_apps(cls, "format\t" as *u8); vcl_apps(cls, vcl_class_name(rs, VCL_AX_FORMAT, c)); vcl_appc(cls, VC_TAB) 1213 vcl_appn(cls, fmt_members[c]); vcl_appc(cls, VC_TAB); vcl_appn(cls, fmt_pkgs[c]); vcl_appc(cls, VC_TAB); vcl_appn(cls, fmt_usize[c]); vcl_appc(cls, VC_LF) 1214 c = c + 1 1215 } 1216 let lba: *i64 = vcl_arena_new(VCL_ARENA_INIT) 1217 vcl_apps(lba, "label\tpackages\n" as *u8) 1218 vc_counter_sorted_tsv(lba, labels) 1219 let cra: *i64 = vcl_arena_new(VCL_ARENA_INIT) 1220 vcl_apps(cra, "creator\tpackages\n" as *u8) 1221 vc_counter_sorted_tsv(cra, creators) 1222 let tga: *i64 = vcl_arena_new(VCL_ARENA_INIT) 1223 vcl_apps(tga, "dependency_lowercased\tedges\n" as *u8) 1224 vc_counter_sorted_tsv(tga, targets) 1225 let rsa: *i64 = vcl_arena_new(VCL_ARENA_INIT) 1226 vcl_apps(rsa, "axis\tkey\tmembers\n" as *u8) 1227 vc_counter_tsv(rsa, resrole, "role-other" as *u8) 1228 vc_counter_tsv(rsa, resfmt, "format-other" as *u8) 1229 let wxa: *i64 = vcl_arena_new(VCL_ARENA_INIT) 1230 vcl_apps(wxa, "extension\tfiles\n" as *u8) 1231 vc_counter_tsv(wxa, exts, "" as *u8) 1232 1233 // ---- the reconciliations ---- 1234 var psum: i64 = 0 1235 c = 0 1236 while c < VC_P_N { psum = psum + pstates[c]; c = c + 1 } 1237 var msum: i64 = 0 1238 c = 0 1239 while c < VC_M_N { msum = msum + mstates[c]; c = c + 1 } 1240 var lsum: i64 = 0 1241 c = 0 1242 while c < vcl_counter_n(labels) { lsum = lsum + vcl_counter_count(labels, c); c = c + 1 } 1243 var ok_pk: i64 = 1 1244 if psum != npk { ok_pk = 0 } 1245 if npk != ws[VC_W_VARS] { ok_pk = 0 } 1246 var ok_role: i64 = 0 1247 if t_role_sum == t_members { ok_role = 1 } 1248 var ok_fmt: i64 = 0 1249 if t_fmt_sum == t_members { ok_fmt = 1 } 1250 var ok_meta: i64 = 0 1251 if msum == pstates[VC_P_OK] { ok_meta = 1 } 1252 var ok_lab: i64 = 0 1253 if lsum == pstates[VC_P_OK] { ok_lab = 1 } 1254 1255 vcl_apps(sm, "VARCENSUS-WALK" as *u8) 1256 vc_kv(sm, "dirs" as *u8, ws[VC_W_DIRS]) 1257 vc_kv(sm, "dirs_unreadable" as *u8, ws[VC_W_DIRS_UNREADABLE]) 1258 vc_kv(sm, "var_files" as *u8, ws[VC_W_VARS]) 1259 vc_kv(sm, "var_top_level" as *u8, ws[VC_W_VARS_TOP]) 1260 vc_kv(sm, "var_nested" as *u8, ws[VC_W_VARS] - ws[VC_W_VARS_TOP]) 1261 vc_kv(sm, "other_files" as *u8, ws[VC_W_OTHER_FILES]) 1262 vc_kv(sm, "symlinks" as *u8, ws[VC_W_SYMLINKS]) 1263 vc_kv(sm, "other_types" as *u8, ws[VC_W_OTHER_TYPES]) 1264 vc_kv(sm, "paths_too_long" as *u8, ws[VC_W_TOO_LONG]) 1265 vcl_appc(sm, VC_LF) 1266 vcl_apps(sm, "VARCENSUS-PACKAGES" as *u8) 1267 vc_kv(sm, "walked" as *u8, npk) 1268 c = 0 1269 while c < VC_P_N { 1270 vcl_appc(sm, 32) 1271 vcl_apps(sm, vc_pstate_name(c)) 1272 vcl_appc(sm, VC_EQ) 1273 vcl_appn(sm, pstates[c]) 1274 c = c + 1 1275 } 1276 vc_kv(sm, "sum" as *u8, psum) 1277 vc_kv(sm, "reconciles" as *u8, ok_pk) 1278 vcl_appc(sm, VC_LF) 1279 vcl_apps(sm, "VARCENSUS-MEMBERS" as *u8) 1280 vc_kv(sm, "members" as *u8, t_members) 1281 vc_kv(sm, "dir_entries" as *u8, t_dirs) 1282 vc_kv(sm, "file_members" as *u8, t_members - t_dirs) 1283 vc_kv(sm, "z64_extra_members" as *u8, t_z64) 1284 vc_kv(sm, "z64_unresolved" as *u8, t_z64u) 1285 vc_kv(sm, "packages_with_z64_extra" as *u8, t_pkgs_z64) 1286 vc_kv(sm, "utf8_named_members" as *u8, t_utf8) 1287 vc_kv(sm, "usize_sum" as *u8, t_usize) 1288 vcl_appc(sm, VC_LF) 1289 vcl_apps(sm, "VARCENSUS-ROLE" as *u8) 1290 vc_kv(sm, "classes" as *u8, nrole) 1291 vc_kv(sm, "sum" as *u8, t_role_sum) 1292 vc_kv(sm, "other" as *u8, role_members[other_role]) 1293 vc_kv(sm, "reconciles" as *u8, ok_role) 1294 vcl_appc(sm, VC_LF) 1295 vcl_apps(sm, "VARCENSUS-FORMAT" as *u8) 1296 vc_kv(sm, "classes" as *u8, nfmt) 1297 vc_kv(sm, "sum" as *u8, t_fmt_sum) 1298 vc_kv(sm, "other" as *u8, fmt_members[other_fmt]) 1299 vc_kv(sm, "reconciles" as *u8, ok_fmt) 1300 vcl_appc(sm, VC_LF) 1301 vcl_apps(sm, "VARCENSUS-META" as *u8) 1302 vc_kv(sm, "parsed" as *u8, mstates[VC_M_PARSED]) 1303 vc_kv(sm, "absent" as *u8, mstates[VC_M_ABSENT]) 1304 vc_kv(sm, "unreadable" as *u8, mstates[VC_M_UNREADABLE]) 1305 vc_kv(sm, "unparsed" as *u8, mstates[VC_M_UNPARSED]) 1306 vc_kv(sm, "sum" as *u8, msum) 1307 vc_kv(sm, "reconciles" as *u8, ok_meta) 1308 vcl_appc(sm, VC_LF) 1309 vcl_apps(sm, "VARCENSUS-LICENSE" as *u8) 1310 vc_kv(sm, "distinct_labels" as *u8, vcl_counter_n(labels)) 1311 vc_kv(sm, "label_rows_sum" as *u8, lsum) 1312 vc_kv(sm, "reconciles" as *u8, ok_lab) 1313 vc_kv(sm, "legacy_agree" as *u8, lagree[VC_L_AGREE]) 1314 vc_kv(sm, "legacy_disagree" as *u8, lagree[VC_L_DISAGREE]) 1315 vc_kv(sm, "legacy_only" as *u8, lagree[VC_L_LEGACY_ONLY]) 1316 vc_kv(sm, "direct_only" as *u8, lagree[VC_L_DIRECT_ONLY]) 1317 vc_kv(sm, "neither" as *u8, lagree[VC_L_NEITHER]) 1318 vcl_appc(sm, VC_LF) 1319 vcl_apps(sm, "VARCENSUS-DEPS" as *u8) 1320 vc_kv(sm, "edges" as *u8, t_edges) 1321 vc_kv(sm, "distinct_targets" as *u8, vcl_counter_n(targets)) 1322 vc_kv(sm, "packages_with_deps" as *u8, t_pkgs_deps) 1323 vc_kv(sm, "malformed_dependency_objects" as *u8, t_deps_malformed) 1324 c = 0 1325 while c < ntags { 1326 vcl_appc(sm, 32); vcl_apps(sm, vcl_counter_key(tags, c)); vcl_apps(sm, "_packages=" as *u8); vcl_appn(sm, tag_pkgs[c]) 1327 vcl_appc(sm, 32); vcl_apps(sm, vcl_counter_key(tags, c)); vcl_apps(sm, "_edges=" as *u8); vcl_appn(sm, tag_edges[c]) 1328 c = c + 1 1329 } 1330 vcl_appc(sm, VC_LF) 1331 vcl_apps(sm, "VARCENSUS-CREATORS" as *u8) 1332 vc_kv(sm, "distinct" as *u8, vcl_counter_n(creators)) 1333 vcl_appc(sm, VC_LF) 1334 vcl_apps(sm, "VARCENSUS-SCREEN" as *u8) 1335 vc_kv(sm, "flagged" as *u8, t_flagged) 1336 c = 0 1337 while c < VC_NLETTERS { 1338 if reason_pkgs[c] > 0 { vcl_appc(sm, 32); vcl_appc(sm, VC_ASCII_UPPER_A + c); vcl_appc(sm, VC_EQ); vcl_appn(sm, reason_pkgs[c]) } 1339 c = c + 1 1340 } 1341 vcl_appc(sm, VC_LF) 1342 1343 let tot: *i64 = sys_mmap(16) as *i64 1344 tot[0] = 0 1345 var wfail: i64 = 0 1346 if vc_write_out(outdir, "packages.tsv" as *u8, pk, tot) < 0 { wfail = wfail + 1 } 1347 if vc_write_out(outdir, "deps.tsv" as *u8, dpa, tot) < 0 { wfail = wfail + 1 } 1348 if vc_write_out(outdir, "rpscreen.tsv" as *u8, rpa, tot) < 0 { wfail = wfail + 1 } 1349 if vc_write_out(outdir, "unreadable.tsv" as *u8, ur, tot) < 0 { wfail = wfail + 1 } 1350 if vc_write_out(outdir, "classes.tsv" as *u8, cls, tot) < 0 { wfail = wfail + 1 } 1351 if vc_write_out(outdir, "labels.tsv" as *u8, lba, tot) < 0 { wfail = wfail + 1 } 1352 if vc_write_out(outdir, "creators.tsv" as *u8, cra, tot) < 0 { wfail = wfail + 1 } 1353 if vc_write_out(outdir, "dep_targets.tsv" as *u8, tga, tot) < 0 { wfail = wfail + 1 } 1354 if vc_write_out(outdir, "residual.tsv" as *u8, rsa, tot) < 0 { wfail = wfail + 1 } 1355 if vc_write_out(outdir, "walk_extensions.tsv" as *u8, wxa, tot) < 0 { wfail = wfail + 1 } 1356 vcl_apps(sm, "VARCENSUS-OUT" as *u8) 1357 vc_kv(sm, "files" as *u8, 11) 1358 vc_kv(sm, "write_failures" as *u8, wfail) 1359 vc_kv(sm, "tsv_bytes" as *u8, tot[0]) 1360 vcl_appc(sm, VC_LF) 1361 var verdict_ok: i64 = 1 1362 if ok_pk == 0 { verdict_ok = 0 } 1363 if ok_role == 0 { verdict_ok = 0 } 1364 if ok_fmt == 0 { verdict_ok = 0 } 1365 if ok_meta == 0 { verdict_ok = 0 } 1366 if ok_lab == 0 { verdict_ok = 0 } 1367 if wfail > 0 { verdict_ok = 0 } 1368 if verdict_ok == 1 { vcl_apps(sm, "VARCENSUS verdict=RECONCILED\n" as *u8) } else { vcl_apps(sm, "VARCENSUS verdict=MISMATCH\n" as *u8) } 1369 vc_write_out(outdir, "summary.txt" as *u8, sm, tot) 1370 vc_wn(VC_FD_OUT, vcl_at(sm, 0), vcl_used(sm)) 1371 if verdict_ok == 1 { return VC_EXIT_OK } 1372 return VC_EXIT_MISMATCH 1373} 1374 1375// Reports source declarations, never claims runtime simulation from a preset flag. 1376func vc_member_measure(path: *u8, selected: *u8, maxbytes: i64) -> i64 { 1377 if maxbytes <= 0 { return VC_EXIT_USAGE } 1378 let fd: i64 = sys_openat_rd(path) 1379 if fd < 0 { vc_w(VC_FD_OUT, "VARCENSUS-MEMBER state=UNOPENABLE\n" as *u8); return VC_EXIT_IO } 1380 let size: i64 = sys_lseek(fd, 0, VC_SEEK_END) 1381 let box: *i64 = sys_mmap(VF_L_N * 8) as *i64 1382 let st: *i64 = sys_mmap(VC_ST_N * 8) as *i64 1383 let win: *u8 = vc_window(fd, size, box, st) 1384 if st[VC_ST_LOC] != VF_LOC_OK { sys_close(fd); vc_w(VC_FD_OUT, "VARCENSUS-MEMBER state=ARCHIVE-REFUSED\n" as *u8); return VC_EXIT_REFUSED } 1385 let mb: *i64 = sys_mmap(VC_MB_N * 8) as *i64 1386 let data: *u8 = vc_member_read(fd, size, win, st[VC_ST_WIN], box, selected, maxbytes, mb) 1387 sys_close(fd) 1388 let out: *i64 = vcl_arena_new(VCL_ARENA_INIT) 1389 vcl_apps(out, "VARCENSUS-MEMBER coverage=one-selected-member asset_payload_emitted=0 runtime_simulation=unknown\nmember=" as *u8) 1390 vcl_appclean(out, selected, 0, vc_slen(selected)) 1391 vcl_appc(out, VC_LF) 1392 vc_kv(out, "found" as *u8, mb[VC_MB_FOUND]) 1393 vc_kv(out, "state" as *u8, mb[VC_MB_STATE]) 1394 vcl_apps(out, " state_name=" as *u8) 1395 vcl_apps(out, vc_md_name(mb[VC_MB_STATE])) 1396 vc_kv(out, "method" as *u8, mb[VC_MB_METHOD]) 1397 vc_kv(out, "compressed_bytes" as *u8, mb[VC_MB_CSIZE]) 1398 vc_kv(out, "declared_bytes" as *u8, mb[VC_MB_USIZE]) 1399 vc_kv(out, "decoded_bytes" as *u8, mb[VC_MB_LEN]) 1400 vc_kv(out, "crc_expected" as *u8, mb[VC_MB_CRC_EXPECTED]) 1401 vc_kv(out, "crc_actual" as *u8, mb[VC_MB_CRC_ACTUAL]) 1402 vc_kv(out, "maxbytes" as *u8, maxbytes) 1403 vcl_appc(out, VC_LF) 1404 var inspected: i64 = 0 1405 var enabled: i64 = 0 1406 var disabled: i64 = 0 1407 var invalid: i64 = 0 1408 var missing: i64 = 0 1409 var schema: i64 = 0 1410 if (data as i64) != 0 { 1411 let n: i64 = mb[VC_MB_LEN] 1412 let jb: *i64 = sys_mmap(VF_JM_N * 8) as *i64 1413 let top: i64 = vf_json_skip(data, n, 0) 1414 if vf_json_kind(data, n, top) == VF_JK_OBJECT { 1415 let a: i64 = vf_json_find(data, n, top, "storables" as *u8, jb) 1416 if vf_json_kind(data, n, a) == VF_JK_ARRAY { 1417 let end: i64 = jb[VF_JM_VE] 1418 var p: i64 = vf_json_skip(data, n, a + 1) 1419 schema = 1 1420 while p < end - 1 { 1421 let ve: i64 = vf_json_value_end(data, n, p) 1422 if ve <= p { schema = 0; p = end } else { 1423 inspected = inspected + 1 1424 let flag: i64 = vf_json_find(data, n, p, "simEnabled" as *u8, jb) 1425 if flag < 0 { missing = missing + 1 } else { 1426 var fs: i64 = flag 1427 var fe: i64 = jb[VF_JM_VE] 1428 if vf_json_kind(data, n, flag) == VF_JK_STRING { fs = fs + 1; fe = fe - 1 } 1429 if vf_span_is(data, fs, fe, "true" as *u8) == 1 { enabled = enabled + 1 } else { 1430 if vf_span_is(data, fs, fe, "false" as *u8) == 1 { disabled = disabled + 1 } else { invalid = invalid + 1 } 1431 } 1432 } 1433 p = vf_json_skip(data, n, ve) 1434 if p < end - 1 { if (data[p] as i64) == VC_ROW_SEP { p = vf_json_skip(data, n, p + 1) } else { schema = 0; p = end } } 1435 } 1436 } 1437 } 1438 } 1439 } 1440 vc_kv(out, "storables_schema" as *u8, schema) 1441 vc_kv(out, "storables_inspected" as *u8, inspected) 1442 vc_kv(out, "sim_declared_true" as *u8, enabled) 1443 vc_kv(out, "sim_declared_false" as *u8, disabled) 1444 vc_kv(out, "sim_flag_invalid" as *u8, invalid) 1445 vc_kv(out, "storables_without_sim_flag" as *u8, missing) 1446 vcl_appc(out, VC_LF) 1447 vc_wn(VC_FD_OUT, vcl_at(out, 0), vcl_used(out)) 1448 if (data as i64) == 0 { return VC_EXIT_REFUSED } 1449 return VC_EXIT_OK 1450} 1451func vc_positive_limit(s: *u8) -> i64 { 1452 var n: i64 = 0 1453 var i: i64 = 0 1454 while s[i] != (0 as u8) { 1455 let d: i64 = s[i] as i64 - VC_ASCII_0 1456 if d < 0 { return 0 } 1457 if d >= VC_DEC { return 0 } 1458 if n > (0x7fffffffffffffff - d) / VC_DEC { return 0 } 1459 n = n * VC_DEC + d 1460 i = i + 1 1461 } 1462 return n 1463} 1464 1465func main(argc: i64, argv: *i64) -> i64 { 1466 if argc < 3 { 1467 vc_w(VC_FD_ERR, "usage: nx_varcensus names <file.var> | meta <file.var> | census <dir> <outdir> <rules.conf> | member <file.var> <name> <maxbytes>\n" as *u8) 1468 vc_w(VC_FD_ERR, " legacy verbs inspect manifest/meta.json; member reports bounded measurements only.\n" as *u8) 1469 sys_exit(VC_EXIT_USAGE) 1470 return VC_EXIT_USAGE 1471 } 1472 if vc_streq(argv[1] as *u8, "member" as *u8) == 1 { 1473 if argc != 5 { return VC_EXIT_USAGE } 1474 return vc_member_measure(argv[2] as *u8, argv[3] as *u8, vc_positive_limit(argv[4] as *u8)) 1475 } 1476 let verb: *u8 = argv[1] as *u8 1477 let arg: *u8 = argv[2] as *u8 1478 var rc: i64 = VC_EXIT_USAGE 1479 if vc_streq(verb, "names" as *u8) == 1 { rc = vc_names(arg) } else { 1480 if vc_streq(verb, "meta" as *u8) == 1 { rc = vc_meta(arg) } else { 1481 if vc_streq(verb, "census" as *u8) == 1 { 1482 if argc < 5 { vc_w(VC_FD_ERR, "census needs <dir> <outdir> <rules.conf>\n" as *u8) } else { rc = vc_census(arg, argv[3] as *u8, argv[4] as *u8) } 1483 } else { vc_w(VC_FD_ERR, "nx_varcensus: unknown verb\n" as *u8) } 1484 } 1485 } 1486 sys_exit(rc) 1487 return rc 1488}