code wiki / _hdl_build / nx_nas_book_census.nx
`nx_nas_book_census.nx` source

↩ module page · 388 lines · 20049 B
1// nx_nas_book_census.nx -- SOVEREIGN, TYPE-DRIVEN NAS book census (Reader-arc RUNG 0).
2//
3// MISSION: the old indexer reported 148 books -- a ~37x undercount. ROOT CAUSE: it (a) mounted
4// only 5 of 26 NAS shares, (b) classified books BY SHARE (nas_scans->"book") instead of by FILE
5// TYPE, (c) included #recycle junk. This organ walks ALL book-bearing mounts with getdents64 (its
6// OWN walk -- no find/grep/python/sqlite), classifies BY FILE EXTENSION across every share, excludes
7// Synology junk (#recycle / @eaDir), and AUTHORS the true book index + a census log. The count is
8// whatever the walk MEASURES -- nothing is hardcoded (no 5760, no 148).
9//
10// COMPOSITION (anti-reinvention): reuses sys_getdents64 + dirent_reclen/dirent_type/dirent_name from
11// nx_syscalls.nx, and the proven raw-offset getdents64 batch loop shape from _si_walk_probe2.nx:37-70
12// (struct-free; the NxDirent struct path crashes signal-11 under nx_cc_sovereign -- LANDMINE avoided).
13// The ONLY new logic is: (1) an EXPLICIT directory work-stack (the proven walkers are all depth-1; we
14// wrap the depth-1 loop in a bounded push/pop stack to descend the Calibre tree -- NOT self-recursion,
15// JPL bounded-loop discipline), (2) the type/extension classifier, (3) the junk-exclusion substring
16// filter. idx format mirrors nx_media_server.nx ms_handle_list exactly: one absolute path per line,
17// '\n'-terminated, no header/footer (server splits on \n, basename = after last '/').
18//
19// DEDUP: dedup is STRUCTURAL. The 10 roots are disjoint subtrees and /mnt/nas_home (the duplicate
20// Synology tree of /mnt/nas_homes) is deliberately EXCLUDED, so a single walk of distinct roots
21// cannot revisit any absolute path -- the walk itself is the dedup. Symlink loops are guarded by
22// skipping DT_LNK directories (never pushed) and a hard iteration cap (bounded loop).
23//
24// x86-ONLY lane: sys_getdents64 from nx_syscalls.nx gates on TARGET_X86_64 -> emits 217 (rv64 numbers
25// HANG on this native lane per nx_media_gather.nx). We never reimplement __syscall.
26// license_tier: ORIGINAL
27import "nx_syscalls.nx"
28import "nx_itoa_lib.nx"   // shared MSB-first emitter (zero-alloc)
29const K_MAGIC_268435456: i64 = 268435456
30const K_MAGIC_4000000: i64 = 4000000
31const K_MAGIC_1048640: i64 = 1048640
32const K_MAGIC_8192: i64 = 8192
33const K_MAGIC_1048576: i64 = 1048576
34const K_MAGIC_8000000: i64 = 8000000
35
36// ---- io helpers (mirror nx_sprawl_eval sp_* / nx_media_server ms_*) ----
37func bc_w(fd: i64, s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(fd,s,n); return 0 }
38func bc_wn(fd: i64, buf: *u8, n: i64) -> i64 { sys_write(fd, buf, n); return 0 }
39// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
40// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
41// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
42// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
43func bc_n(fd: i64, v: i64) -> i64 { nxi_fd(fd, v); return 0 }
44func bc_p(s: *u8) -> i64 { bc_w(1, s); return 0 }
45func bc_pn(v: i64) -> i64 { bc_n(1, v); return 0 }
46
47func bc_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
48
49// case-insensitive ASCII lower of a byte
50func bc_lc(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c }
51
52// substring scan over a NUL-terminated string (for #recycle / @eaDir exclusion)
53func bc_has(hay: *u8, needle: *u8) -> i64 {
  let hn: i64 = bc_slen(hay)
  let nl: i64 = bc_slen(needle)
  if nl == 0 { return 0 }
  var i: i64 = 0
  while i + nl <= hn {
      var k: i64 = 0
      var hit: i64 = 1
      while k < nl { if hay[i+k] != needle[k] { hit = 0; k = nl } else { k = k + 1 } }
      if hit == 1 { return 1 }
      i = i + 1
  }
  return 0
66}
67
68// ---- REFERENCE list of the canonical book-bearing shares (SUPERSEDED by runtime auto-discovery of
69//      /mnt/nas_* in main; kept as documentation of where books historically lived). ----
70const NROOT: i64 = 10
71func bc_root(i: i64) -> *u8 {
  if i == 0 { return "/mnt/nas_homes" as *u8 }
  if i == 1 { return "/mnt/nas_ai" as *u8 }
  if i == 2 { return "/mnt/nas_scans" as *u8 }
  if i == 3 { return "/mnt/nas_audiobooks" as *u8 }
  if i == 4 { return "/mnt/nas_share" as *u8 }
  if i == 5 { return "/mnt/nas_gatsby" as *u8 }
  if i == 6 { return "/mnt/nas_movies" as *u8 }
  if i == 7 { return "/mnt/nas_photo" as *u8 }
  if i == 8 { return "/mnt/nas_3d_printer" as *u8 }
  if i == 9 { return "/mnt/nas_music" as *u8 }
  return "" as *u8
83}
84
85// ---- BOOK EXTENSION SET (dispatch table; NUL-terminated literals, already lowercase) ----
86// returns extension index 0..NEXT-1 for a matched extension, else -1.  caller passes a
87// lowercased, NUL-terminated extension (no leading dot).
88const NEXT: i64 = 19
89func bc_ext_name(i: i64) -> *u8 {
  if i == 0  { return "epub" as *u8 }
  if i == 1  { return "mobi" as *u8 }
  if i == 2  { return "azw" as *u8 }
  if i == 3  { return "azw3" as *u8 }
  if i == 4  { return "azw4" as *u8 }
  if i == 5  { return "kfx" as *u8 }
  if i == 6  { return "cbz" as *u8 }
  if i == 7  { return "cbr" as *u8 }
  if i == 8  { return "cbx" as *u8 }
  if i == 9  { return "cb7" as *u8 }
  if i == 10 { return "djvu" as *u8 }
  if i == 11 { return "fb2" as *u8 }
  if i == 12 { return "lit" as *u8 }
  if i == 13 { return "prc" as *u8 }
  if i == 14 { return "pdb" as *u8 }
  if i == 15 { return "chm" as *u8 }
  if i == 16 { return "htmlz" as *u8 }
  if i == 17 { return "ibooks" as *u8 }
  if i == 18 { return "pdf" as *u8 }
  return "" as *u8
110}
111
112// exact (already-lowercased) string equality
113func bc_streq(a: *u8, b: *u8) -> i64 {
  var i: i64 = 0
  while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 }
  if b[i] != (0 as u8) { return 0 }
  return 1
118}
119
120// match lowercased extension against the book set -> ext id 0..18, else -1
121func bc_ext_id(extlc: *u8) -> i64 {
  var i: i64 = 0
  while i < NEXT { if bc_streq(extlc, bc_ext_name(i)) == 1 { return i } i = i + 1 }
  return 0 - 1
125}
126
127func main() -> i64 {
  bc_p("BOOK-CENSUS: start (sovereign type-driven NAS walk)\n" as *u8)
129
  // ---- output sinks (additive; idx truncates for idempotent re-run, log appends) ----
  let idxfd: i64 = sys_openat_wr("knowledge/staging/media/idx_book_true.wsl" as *u8, 0x1a4)
  if idxfd < 0 { bc_p("BOOK-CENSUS-FAIL idx-open\n" as *u8); sys_exit(1); return 1 }
  let logfd: i64 = sys_openat_append("knowledge/status/nas_book_census.log" as *u8, 0x1a4)
  if logfd < 0 { bc_p("BOOK-CENSUS-FAIL log-open\n" as *u8); sys_exit(1); return 1 }
135
  // ---- arenas (mmap; bounded) ----
  // path arena: store NUL-terminated directory paths for the work-stack.
  let DIR_ARENA: i64 = K_MAGIC_268435456      // 256 MiB of dir-path bytes
  let parena: *u8 = sys_mmap(DIR_ARENA + 64)
  var pused: i64 = 0
  // stack of (offset-into-parena, share-id) -- two parallel i64 arrays.
  let STKCAP: i64 = K_MAGIC_4000000           // up to 4M pending dirs (bounded)
  let stkoff: *i64 = sys_mmap(STKCAP * 8) as *i64
  let stkshr: *i64 = sys_mmap(STKCAP * 8) as *i64
  var sp: i64 = 0
  // scratch buffers
  let dirbuf: *u8 = sys_mmap(K_MAGIC_1048640) // getdents64 batch buffer (1 MiB)
  let childbuf: *u8 = sys_mmap(K_MAGIC_8192)  // assembled child absolute path
  let extbuf: *u8 = sys_mmap(64)      // lowercased extension scratch
  let linebuf: *u8 = sys_mmap(K_MAGIC_8192)   // idx line (path + '\n')
151
  // ---- counters ----
  var total: i64 = 0
  var dirs_walked: i64 = 0
  var excl_recycle: i64 = 0
  var excl_eadir: i64 = 0
  var open_fail: i64 = 0
  let extcnt: *i64 = sys_mmap(NEXT * 8) as *i64
  var ei: i64 = 0; while ei < NEXT { extcnt[ei] = 0; ei = ei + 1 }
  // per-share counters + root registry sized for AUTO-DISCOVERED roots (was the static NROOT=10)
  let MAXROOT: i64 = 64
  let shrcnt: *i64 = sys_mmap(MAXROOT * 8) as *i64
  var si: i64 = 0; while si < MAXROOT { shrcnt[si] = 0; si = si + 1 }
  let rootoff: *i64 = sys_mmap(MAXROOT * 8) as *i64   // parena offset of each discovered root path
  var nroots: i64 = 0
166
  // ---- AUTO-DISCOVER mount roots: every /mnt/nas_* dir EXCEPT the nas_home dup tree ----
  // A hardcoded root list is exactly what caused the original 37x undercount -- shares get silently
  // missed. The census now covers WHATEVER IS MOUNTED: mount a share, it is censused. nas_home is the
  // Synology double-mount of nas_homes and is structurally excluded so no absolute path is revisited.
  let mntfd: i64 = sys_openat_rd("/mnt" as *u8)
  if mntfd < 0 { bc_p("BOOK-CENSUS-FAIL mnt-open\n" as *u8); sys_exit(1); return 1 }
  var mdone: i64 = 0
  while mdone == 0 {
      let mnb: i64 = sys_getdents64(mntfd, dirbuf, K_MAGIC_1048576)
      if mnb <= 0 { mdone = 1 }
      else {
          var moff: i64 = 0
          while moff < mnb {
              let mrec: *u8 = (dirbuf as i64 + moff) as *u8
              let mrl: i64 = dirent_reclen(mrec)
              if mrl <= 0 { moff = mnb }
              else {
                  let mnm: *u8 = dirent_name(mrec)
                  // prefix match "nas_" (n=110 a=97 s=115 _=95)
                  var isnas: i64 = 0
                  if mnm[0] == (110 as u8) { if mnm[1] == (97 as u8) { if mnm[2] == (115 as u8) { if mnm[3] == (95 as u8) { isnas = 1 } } } }
                  // exclude EXACTLY "nas_home" (dup of nas_homes): h=104 o=111 m=109 e=101 then NUL@8
                  if isnas == 1 {
                      if mnm[4] == (104 as u8) { if mnm[5] == (111 as u8) { if mnm[6] == (109 as u8) { if mnm[7] == (101 as u8) { if mnm[8] == (0 as u8) { isnas = 0 } } } } }
                  }
                  if isnas == 1 {
                      if nroots < MAXROOT {
                          // assemble "/mnt/" + name into childbuf, copy into parena, push as a root
                          let pfx: *u8 = "/mnt/" as *u8
                          var w: i64 = 0
                          while pfx[w] != (0 as u8) { childbuf[w] = pfx[w]; w = w + 1 }
                          var z: i64 = 0
                          while mnm[z] != (0 as u8) { childbuf[w] = mnm[z]; w = w + 1; z = z + 1 }
                          childbuf[w] = 0 as u8
                          let rbase: i64 = pused
                          var q: i64 = 0
                          while q < w { parena[pused] = childbuf[q]; pused = pused + 1; q = q + 1 }
                          parena[pused] = 0 as u8; pused = pused + 1
                          rootoff[nroots] = rbase
                          stkoff[sp] = rbase
                          stkshr[sp] = nroots
                          sp = sp + 1
                          nroots = nroots + 1
                      }
                  }
                  moff = moff + mrl
              }
          }
      }
  }
  sys_close(mntfd)
  bc_p("BOOK-CENSUS: discovered roots=" as *u8); bc_pn(nroots); bc_p("\n" as *u8)
219
  // ---- bounded walk: pop a dir, getdents64 it, classify files, push child dirs ----
  let ITER_CAP: i64 = K_MAGIC_8000000          // hard upper bound on dirs processed (symlink-loop guard)
  var iters: i64 = 0
  while sp > 0 {
      if iters >= ITER_CAP { sp = 0 }
      else {
      iters = iters + 1
      sp = sp - 1
      let doff: i64 = stkoff[sp]
      let shr: i64 = stkshr[sp]
      let dpath: *u8 = (parena as i64 + doff) as *u8
      let dlen: i64 = bc_slen(dpath)
232
      let dfd: i64 = sys_openat_rd(dpath)
      if dfd < 0 { open_fail = open_fail + 1 }
      else {
          dirs_walked = dirs_walked + 1
          var done: i64 = 0
          while done == 0 {
              let nb: i64 = sys_getdents64(dfd, dirbuf, K_MAGIC_1048576)
              if nb <= 0 { done = 1 }
              else {
                  var off: i64 = 0
                  while off < nb {
                      let rec: *u8 = (dirbuf as i64 + off) as *u8
                      let rl: i64 = dirent_reclen(rec)
                      if rl <= 0 { off = nb }
                      else {
                          let nm: *u8 = dirent_name(rec)
                          let dt: i64 = dirent_type(rec)
                          // skip "." and ".." (dotlike)
                          var skip: i64 = 0
                          if nm[0] == (46 as u8) {
                              if nm[1] == (0 as u8) { skip = 1 }
                              else { if nm[1] == (46 as u8) { if nm[2] == (0 as u8) { skip = 1 } } }
                          }
                          if skip == 0 {
                              // assemble childpath = dpath + "/" + nm  (into childbuf)
                              var w: i64 = 0
                              var z: i64 = 0
                              while z < dlen { childbuf[w] = dpath[z]; w = w + 1; z = z + 1 }
                              childbuf[w] = 47 as u8; w = w + 1
                              var nl: i64 = 0
                              while nm[nl] != (0 as u8) { childbuf[w] = nm[nl]; w = w + 1; nl = nl + 1 }
                              childbuf[w] = 0 as u8
                              let clen: i64 = w
266
                              // junk exclusion (Synology): skip entirely -- no descend, no classify
                              var junk: i64 = 0
                              if bc_has(childbuf, "#recycle" as *u8) == 1 { junk = 1; excl_recycle = excl_recycle + 1 }
                              else { if bc_has(childbuf, "@eaDir" as *u8) == 1 { junk = 1; excl_eadir = excl_eadir + 1 } }
271
                              if junk == 0 {
                                  if dt == 4 {
                                      // DT_DIR: push child onto work-stack (descend later)
                                      if sp < STKCAP {
                                          if pused + clen + 1 < DIR_ARENA {
                                              let cbase: i64 = pused
                                              var q: i64 = 0
                                              while q < clen { parena[pused] = childbuf[q]; pused = pused + 1; q = q + 1 }
                                              parena[pused] = 0 as u8; pused = pused + 1
                                              stkoff[sp] = cbase
                                              stkshr[sp] = shr
                                              sp = sp + 1
                                          }
                                      }
                                  } else {
                                      if dt == 10 { } else {
                                          // DT_REG (8) OR DT_UNKNOWN (0; drvfs/9p often returns 0) -> classify by ext.
                                          // find last '.' in nm
                                          var dotp: i64 = 0 - 1
                                          var x: i64 = 0
                                          while nm[x] != (0 as u8) { if nm[x] == (46 as u8) { dotp = x } x = x + 1 }
                                          var matched: i64 = 0
                                          if dotp >= 0 {
                                              // lowercase the extension into extbuf
                                              var e: i64 = 0
                                              var y: i64 = dotp + 1
                                              while nm[y] != (0 as u8) { if e < 63 { extbuf[e] = bc_lc(nm[y] as i64) as u8; e = e + 1 } y = y + 1 }
                                              extbuf[e] = 0 as u8
                                              let id: i64 = bc_ext_id(extbuf)
                                              if id >= 0 {
                                                  matched = 1
                                                  extcnt[id] = extcnt[id] + 1
                                              } else {
                                                  // audiobooks-as-.zip: count .zip ONLY under /nas_audiobooks/
                                                  if bc_streq(extbuf, "zip" as *u8) == 1 {
                                                      if bc_has(childbuf, "/nas_audiobooks/" as *u8) == 1 { matched = 1 }
                                                  }
                                              }
                                          }
                                          if matched == 1 {
                                              total = total + 1
                                              shrcnt[shr] = shrcnt[shr] + 1
                                              // append absolute childpath + '\n' to idx
                                              var li: i64 = 0
                                              while li < clen { linebuf[li] = childbuf[li]; li = li + 1 }
                                              linebuf[clen] = 10 as u8
                                              bc_wn(idxfd, linebuf, clen + 1)
                                          }
                                      }
                                  }
                              }
                          }
                          off = off + rl
                      }
                  }
              }
          }
          sys_close(dfd)
      }
      }
  }
  sys_close(idxfd)
334
  // ---- walk-completion verdict: GREEN iff total>0 and the iteration cap did not abort the walk ----
  var green: i64 = 1
  if total <= 0 { green = 0 }
  if iters >= ITER_CAP { green = 0 }
339
  // ---- author the census log line BY the organ (additive append) ----
  bc_w(logfd, "NAS-BOOK-CENSUS ts=" as *u8); bc_n(logfd, sys_now_realtime_sec())
  bc_w(logfd, " total=" as *u8); bc_n(logfd, total)
  bc_w(logfd, " dirs_walked=" as *u8); bc_n(logfd, dirs_walked)
  bc_w(logfd, " open_fail=" as *u8); bc_n(logfd, open_fail)
  bc_w(logfd, " excluded_recycle=" as *u8); bc_n(logfd, excl_recycle)
  bc_w(logfd, " excluded_eadir=" as *u8); bc_n(logfd, excl_eadir)
  bc_w(logfd, " iters=" as *u8); bc_n(logfd, iters)
  // per-extension (nonzero only)
  bc_w(logfd, " ext{" as *u8)
  var pi: i64 = 0
  while pi < NEXT {
      if extcnt[pi] > 0 { bc_w(logfd, " " as *u8); bc_w(logfd, bc_ext_name(pi)); bc_w(logfd, "=" as *u8); bc_n(logfd, extcnt[pi]) }
      pi = pi + 1
  }
  bc_w(logfd, " }" as *u8)
  // per-share (nonzero only)
  bc_w(logfd, " share{" as *u8)
  var ki: i64 = 0
  while ki < nroots {
      if shrcnt[ki] > 0 { bc_w(logfd, " " as *u8); bc_w(logfd, (parena as i64 + rootoff[ki]) as *u8); bc_w(logfd, "=" as *u8); bc_n(logfd, shrcnt[ki]) }
      ki = ki + 1
  }
  bc_w(logfd, " }" as *u8)
  if green == 1 { bc_w(logfd, " verdict=GREEN\n" as *u8) } else { bc_w(logfd, " verdict=RED\n" as *u8) }
  sys_close(logfd)
366
  // ---- stdout markers for the gate ----
  bc_p("BOOK-CENSUS: total=" as *u8); bc_pn(total)
  bc_p(" dirs=" as *u8); bc_pn(dirs_walked)
  bc_p(" recycle_excl=" as *u8); bc_pn(excl_recycle)
  bc_p(" eadir_excl=" as *u8); bc_pn(excl_eadir)
  bc_p("\nBOOK-CENSUS ext:" as *u8)
  var qi: i64 = 0
  while qi < NEXT { if extcnt[qi] > 0 { bc_p(" " as *u8); bc_p(bc_ext_name(qi)); bc_p("=" as *u8); bc_pn(extcnt[qi]) } qi = qi + 1 }
  bc_p("\nBOOK-CENSUS share:" as *u8)
  var mi: i64 = 0
  while mi < nroots { if shrcnt[mi] > 0 { bc_p(" " as *u8); bc_p((parena as i64 + rootoff[mi]) as *u8); bc_p("=" as *u8); bc_pn(shrcnt[mi]) } mi = mi + 1 }
  bc_p("\n" as *u8)
379
  if green == 1 {
      bc_p("BOOK-CENSUS-OK\n" as *u8)
      sys_exit(0)
      return 0
  }
  bc_p("BOOK-CENSUS-FAIL walk-incomplete-or-empty\n" as *u8)
  sys_exit(1)
  return 1
388}
nx_nas_book_census.nx source

`nx_nas_book_census.nx` source
