`nx_lockwatch.nx` source

↩ module page · 329 lines · 14685 B
1// nx_lockwatch.nx -- WHO HOLDS THIS PLANE'S LOCK, AND WHAT ARE THEY DOING?
2//
3// WHY THIS EXISTS. On 2026-08-06 a gate went RED with an UNCHANGED binary. The cause was not in the
4// diff at all: one worker sat in kernel state D, wchan=wait_for_commit -- a filesystem journal commit
5// -- HOLDING a plane lock, while its siblings sat in wchan=locks_lock_inode_wait. Establishing that
6// took reading /proc/locks by hand, mapping an inode back to a lock file, then reading
7// /proc/<pid>/wchan and /proc/<pid>/fd for every candidate. Three separate investigations that day
8// ran the same manual procedure, and ss_plane_lock's own BUSY message now TELLS an operator to do it:
9//   "check /proc/<pid>/wchan: locks_lock_inode_wait = queued, wait_for_commit = wedged on disk"
10// ÃƒÂ¢Ã‹Å“Ã¢â‚¬Â¦TELLING SOMEONE TO RUN A PROCEDURE IS NOT THE SAME AS SHIPPING IT. A diagnostic that only exists
11// in a message is a diagnostic nobody runs at 3am. This is that procedure, as an organ.
12//
13// AND THE DISTINCTION IT DRAWS IS THE WHOLE POINT -- these look identical from outside and need
14// OPPOSITE responses:
15//   QUEUED  (wchan=locks_lock_inode_wait) -- healthy contention. Someone is working; wait.
16//   WEDGED  (state D, wchan=wait_for_commit / io_schedule / similar) -- the holder is stuck in the
17//           kernel on I/O. Waiting will not help; the disk is the problem, and every writer behind
18//           it is now blocked for as long as that lasts.
19//   STALE   -- a lock file whose recorded holder is GONE. flock releases on process death, so this
20//           should be impossible; if it ever prints, the assumption is wrong and that matters more
21//           than anything else on the report.
22//
23// READ-ONLY. Opens nothing but /proc and the lock files' metadata; writes nothing, locks nothing --
24// deliberately, because an instrument that took the lock it is diagnosing would be the defect.
25// usage: nx_lockwatch [store-dir]      default knowledge/store
26// exit: 0 = no held plane locks, 1 = at least one held, 2 = usage/unreadable
27// license_tier: ORIGINAL   No hw writes (Rule 26).
28import "nx_syscalls.nx"
29
30const LW_STORE: *u8 = "knowledge/store"
31const LW_LOCKS: *u8 = "/proc/locks"
32const LW_DIRBUF: i64 = 262144
33const LW_LOCKBUF: i64 = 262144
34const LW_PATH: i64 = 512
35const LW_SMALL: i64 = 256
36const LW_STATBUF: i64 = 256
37const LW_OFF_INO: i64 = 8          // x86-64 struct stat: st_dev@0, st_ino@8 (st_size@48 is the
                                 // offset the rest of the estate already relies on, same layout)
39const LW_MAXLOCKS: i64 = 4096
40const LW_ZERO: i64 = 48
41const LW_NINE: i64 = 57
42const LW_NL: i64 = 10
43const LW_COLON: i64 = 58
44const LW_SP: i64 = 32
45const LW_B10: i64 = 10
46const LW_EXIT_CLEAR: i64 = 0
47const LW_EXIT_HELD: i64 = 1
48const LW_EXIT_USAGE: i64 = 2
49
50func lw_w(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
51func lw_n(v: i64) -> i64 {
  if v == 0 { lw_w("0" as *u8); return 0 }
  var x: i64 = v
  if x < 0 { lw_w("-" as *u8); x = 0 - x }
  let b: *u8 = sys_mmap(32)
  var i: i64 = 0
  while x > 0 { b[i] = ((x % LW_B10) + LW_ZERO) as u8; x = x / LW_B10; i = i + 1 }
  while i > 0 { i = i - 1; sys_write(1, ((b as i64) + i) as *u8, 1) }
  return 0
60}
61func lw_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
62
63// /proc files report st_size 0, so a stat-then-read slurp returns nothing. Read until EOF instead.
64// ÃƒÂ¢Ã…Â¡Ã‚Â This is exactly the trap that makes /proc look empty to any helper written for regular files.
65func lw_slurp(path: *u8, buf: *u8, cap: i64) -> i64 {
  let fd: i64 = sys_openat_rd(path)
  if fd < 0 { return 0 - 1 }
  var off: i64 = 0
  var go: i64 = 1
  while go == 1 {
      let n: i64 = sys_read(fd, ((buf as i64) + off) as *u8, cap - off - 1)
      if n <= 0 { go = 0 } else {
          off = off + n
          if off >= cap - 1 { go = 0 }
      }
  }
  sys_close(fd)
  buf[off] = 0 as u8
  return off
80}
81
82// 1 if `nm` ends with `suf`
83func lw_ends(nm: *u8, suf: *u8) -> i64 {
  let ln: i64 = lw_len(nm)
  let ls: i64 = lw_len(suf)
  if ln < ls { return 0 }
  var i: i64 = 0
  while i < ls { if nm[ln - ls + i] != suf[i] { return 0 } i = i + 1 }
  return 1
90}
91
92// parse the decimal starting at buf[i]; advances nothing, returns the value (0 if none)
93func lw_num_at(buf: *u8, i: i64, n: i64) -> i64 {
  var v: i64 = 0
  var j: i64 = i
  var go: i64 = 1
  while go == 1 {
      if j >= n { go = 0 } else {
          let c: i64 = buf[j] as i64
          if c >= LW_ZERO { if c <= LW_NINE { v = v * LW_B10 + (c - LW_ZERO); j = j + 1 } else { go = 0 } } else { go = 0 }
      }
  }
  return v
104}
105
106// "FLOCK" at b[i..i+5)? Flat, one comparison per statement with an early return -- the shape the rest
107// of the estate uses. A five-deep nested-if with a multi-statement innermost block silently matched
108// NOTHING here while a bare byte compare in the same file found 61 'F' bytes: the primitive was fine,
109// the SHAPE was not. Do not re-nest this for brevity.
110// Start offset of whitespace-separated field `idx` (0-based) in [p, stop), or -1.
111// ONE loop, a state flag, no break-by-overshoot. NishiLang has no `break`, and faking one by setting
112// the loop variable past its bound is what corrupted the previous parser.
113func lw_field_start(b: *u8, p: i64, stop: i64, idx: i64) -> i64 {
  var i: i64 = p
  var f: i64 = 0 - 1
  var inword: i64 = 0
  var found: i64 = 0 - 1
  while i < stop {
      var issp: i64 = 0
      if b[i] == (32 as u8) { issp = 1 }
      if issp == 0 {
          if inword == 0 {
              inword = 1
              f = f + 1
              if f == idx { if found < 0 { found = i } }
          }
      }
      if issp == 1 { inword = 0 }
      i = i + 1
  }
  return found
132}
133
134// /proc/locks field 5 is "major:minor:inode" -- take the digits after the LAST colon.
135func lw_inode_of(b: *u8, fstart: i64, stop: i64) -> i64 {
  var i: i64 = fstart
  var lastc: i64 = 0 - 1
  var go: i64 = 1
  while go == 1 {
      if i >= stop { go = 0 } else {
          if b[i] == (32 as u8) { go = 0 } else {
              if b[i] == (58 as u8) { lastc = i }
              i = i + 1
          }
      }
  }
  if lastc < 0 { return 0 }
  return lw_num_at(b, lastc + 1, stop)
149}
150
151func lw_at_flock(b: *u8, i: i64) -> i64 {
  if b[i] != (70 as u8) { return 0 }
  if b[i + 1] != (76 as u8) { return 0 }
  if b[i + 2] != (79 as u8) { return 0 }
  if b[i + 3] != (67 as u8) { return 0 }
  if b[i + 4] != (75 as u8) { return 0 }
  return 1
158}
159
160func main(argc: i64, argv: *i64) -> i64 {
  var store: *u8 = LW_STORE
  if argc >= 2 { store = argv[1] as *u8 }
163
  let lbuf: *u8 = sys_mmap(LW_LOCKBUF)
  let ln: i64 = lw_slurp(LW_LOCKS, lbuf, LW_LOCKBUF)
  if ln <= 0 {
      lw_w("LOCKWATCH: cannot read /proc/locks -- no lock visibility on this host\n" as *u8)
      sys_exit(LW_EXIT_USAGE)
      return LW_EXIT_USAGE
  }
171
  // --- collect FLOCK (pid, inode) pairs ---
  let pids: *i64 = sys_mmap(8 * LW_MAXLOCKS) as *i64
  let inos: *i64 = sys_mmap(8 * LW_MAXLOCKS) as *i64
  var nlocks: i64 = 0
  var nflock: i64 = 0
  var nlines: i64 = 0
  var p: i64 = 0
  while p < ln {
      var stop: i64 = p
      var seek: i64 = 1
      while seek == 1 {
          if stop >= ln { seek = 0 } else {
              if lbuf[stop] == (LW_NL as u8) { seek = 0 } else { stop = stop + 1 }
          }
      }
      // is it a FLOCK row? scan the line for "FLOCK"
      var isf: i64 = 0
      var s: i64 = p
      let flim: i64 = stop - 5
      while s <= flim {
          if lw_at_flock(lbuf, s) == 1 { isf = 1 }
          s = s + 1
      }
      if isf == 1 {
          nflock = nflock + 1
          // FIELD EXTRACTION VIA FLAT HELPERS. The previous version walked fields with nested
          // while-loops that used "k = stop + 1" to break and then clamped back -- so it never
          // landed on a field start and every pid/inode came out 0. Detection said 18 FLOCK lines
          // while the report said 0 locks, because my own counter conflated DETECTED with PARSED.
          // >>A COUNTER THAT MEASURES TWO STAGES AT ONCE CANNOT TELL YOU WHICH ONE FAILED.<<
          let f4: i64 = lw_field_start(lbuf, p, stop, 4)
          let f5: i64 = lw_field_start(lbuf, p, stop, 5)
          var pidv: i64 = 0
          var inov: i64 = 0
          if f4 >= 0 { pidv = lw_num_at(lbuf, f4, stop) }
          if f5 >= 0 { inov = lw_inode_of(lbuf, f5, stop) }
          if pidv > 0 { if inov > 0 { if nlocks < LW_MAXLOCKS {
              pids[nlocks] = pidv
              inos[nlocks] = inov
              nlocks = nlocks + 1
          } } }
      }
      nlines = nlines + 1
      p = stop + 1
  }
217
  lw_w("=== nx_lockwatch -- who holds a plane lock, and what are they doing ===\n" as *u8)
  var fcount: i64 = 0
  var fi2: i64 = 0
  while fi2 < ln { if lbuf[fi2] == (70 as u8) { fcount = fcount + 1 } fi2 = fi2 + 1 }
  lw_w("  FLOCK rows visible in /proc/locks: " as *u8); lw_n(nlocks); lw_w("\n" as *u8)
223
  // --- walk the store for *slock / *plock and match by inode ---
  let fd: i64 = sys_openat_rd(store)
  if fd < 0 {
      lw_w("LOCKWATCH: cannot open " as *u8); lw_w(store)
      lw_w(" -- run from the nishihost CWD\n" as *u8)
      sys_exit(LW_EXIT_USAGE)
      return LW_EXIT_USAGE
  }
  let dbuf: *u8 = sys_mmap(LW_DIRBUF)
  let path: *u8 = sys_mmap(LW_PATH)
  let stbuf: *u8 = sys_mmap(LW_STATBUF)
  let wbuf: *u8 = sys_mmap(LW_SMALL)
  let cbuf: *u8 = sys_mmap(LW_SMALL)
  var files: i64 = 0
  var held: i64 = 0
  var wedged: i64 = 0
  var go: i64 = 1
  while go == 1 {
      let dn: i64 = sys_getdents64(fd, dbuf, LW_DIRBUF)
      if dn <= 0 { go = 0 } else {
          var off: i64 = 0
          while off < dn {
              let rec: *u8 = ((dbuf as i64) + off) as *u8
              let rl: i64 = dirent_reclen(rec)
              if rl <= 0 { off = dn } else {
                  let nm: *u8 = dirent_name(rec)
                  var islock: i64 = 0
                  if lw_ends(nm, "slock" as *u8) == 1 { islock = 1 }
                  if lw_ends(nm, "plock" as *u8) == 1 { islock = 1 }
                  if islock == 1 {
                      files = files + 1
                      var o: i64 = 0
                      var t: i64 = 0
                      while store[t] != (0 as u8) { path[o] = store[t]; o = o + 1; t = t + 1 }
                      path[o] = 47 as u8
                      o = o + 1
                      t = 0
                      while nm[t] != (0 as u8) { path[o] = nm[t]; o = o + 1; t = t + 1 }
                      path[o] = 0 as u8
                      let strc: i64 = sys_fstatat(path, stbuf)
                      if strc == 0 {
                          // ⚠NAMED POINTER, NOT AN INLINE CAST-AND-INDEX. `(stbuf as *i64)[n]` types
                          // as a POINTER in this compiler, so the inode silently read as an address and
                          // NEVER matched -- held=0 while /proc/locks plainly showed the lock held.
                          let stq2: *i64 = stbuf as *i64
                          let ino: i64 = stq2[LW_OFF_INO / 8]
                          var m: i64 = 0
                          while m < nlocks {
                              if inos[m] == ino {
                                  held = held + 1
                                  let hp: i64 = pids[m]
                                  // read the holder's wchan + comm; a missing wchan means the pid is GONE
                                  var wo: i64 = 0
                                  var wt: i64 = 0
                                  let pfx: *u8 = "/proc/" as *u8
                                  while pfx[wt] != (0 as u8) { path[wo] = pfx[wt]; wo = wo + 1; wt = wt + 1 }
                                  // itoa the pid inline
                                  var pv: i64 = hp
                                  let tmp: *u8 = sys_mmap(32)
                                  var ti: i64 = 0
                                  while pv > 0 { tmp[ti] = ((pv % LW_B10) + LW_ZERO) as u8; pv = pv / LW_B10; ti = ti + 1 }
                                  while ti > 0 { ti = ti - 1; path[wo] = tmp[ti]; wo = wo + 1 }
                                  let sfx: *u8 = "/wchan" as *u8
                                  wt = 0
                                  while sfx[wt] != (0 as u8) { path[wo] = sfx[wt]; wo = wo + 1; wt = wt + 1 }
                                  path[wo] = 0 as u8
                                  let wn: i64 = lw_slurp(path, wbuf, LW_SMALL)
                                  lw_w("  HELD   " as *u8); lw_w(nm)
                                  lw_w("  pid=" as *u8); lw_n(hp)
                                  lw_w("  wchan=" as *u8)
                                  if wn > 0 { lw_w(wbuf) } else { lw_w("<gone>" as *u8) }
                                  // classify: this is the judgement the manual procedure was for
                                  if wn > 0 {
                                      if lw_ends(wbuf, "commit" as *u8) == 1 {
                                          wedged = wedged + 1
                                          lw_w("   <-- WEDGED ON DISK: waiting will not help, every writer behind it is blocked" as *u8)
                                      }
                                  }
                                  if wn <= 0 {
                                      lw_w("   <-- STALE HOLDER: flock should release on process death; investigate, this should be impossible" as *u8)
                                  }
                                  lw_w("\n" as *u8)
                              }
                              m = m + 1
                          }
                      }
                  }
                  off = off + rl
              }
          }
      }
  }
  sys_close(fd)
317
  lw_w("NX-LOCKWATCH lock_files=" as *u8); lw_n(files)
  lw_w(" held=" as *u8); lw_n(held)
  lw_w(" wedged=" as *u8); lw_n(wedged)
  if held > 0 {
      lw_w(" verdict=HELD (a plane lock is currently held; queued waiters are normal, WEDGED is not)\n" as *u8)
      sys_exit(LW_EXIT_HELD)
      return LW_EXIT_HELD
  }
  lw_w(" verdict=CLEAR (no plane lock is held right now)\n" as *u8)
  sys_exit(LW_EXIT_CLEAR)
  return LW_EXIT_CLEAR
329}
nx_lockwatch.nx source

`nx_lockwatch.nx` source
