code wiki / _hdl_build / nx_mem_healer.nx
`nx_mem_healer.nx` source

↩ module page · 339 lines · 18404 B
1// nx_mem_healer.nx -- the swarm ADDRESS/PREVENT pillar: a periodic memory-cap AUTO-HEAL.
2//
3// Dispatched ARG-LESS by the sovereign clock (nx_clock_tickless, a root supervisor child) every ~60s. Reads
4// /proc/meminfo; if used% >= MH_THRESHOLD it scans /proc (getdents, bounded reads -- never sys_read_file) for the
5// TOP-RSS process, and if that process is a known guard-supervised daemon it restarts it via `./nx_hostctl
6// <kicksub>` (root -> kill by name; the hostctl guard respawns it fresh, reclaiming the leaked pages). This catches
7// ANY daemon memory leak BY CONSTRUCTION -- the docportal per-request-mmap class and any future one -- with zero
8// per-daemon code and NO edit to the supervisor. Composes the /proc idioms from nx_node_beacon + nx_zombie_audit.
9//
10// Run:
11//   nx_mem_healer                 (the live/clock path): sample mem; heal the top offender iff mem>=threshold.
12//                                 Building via nx_sov_build_run runs this once on WSL (mem<threshold -> OK -> exit 0).
13//   nx_mem_healer gate            self-gate: pure-logic KATs (threshold decision + comm->kicksub map), no /proc, no heal.
14// license_tier: ORIGINAL
15import "nx_syscalls.nx"
16import "nx_itoa_lib.nx"   // shared MSB-first emitter (zero-alloc)
17const MH_MAGIC_65536: i64 = 65536
18const MH_MAGIC_1048576: i64 = 1048576
19
20const MH_THRESHOLD: i64 = 88      // mem used% at/above which we heal. Below = report + no-op. Data-driven cap.
21const MH_BUF:       i64 = 65536
22const MH_LOGP:      *u8 = "mem_healer.log"   // cwd-relative (nishihost); append-only heal ledger
23const MH_HOSTCTL:   *u8 = "./nx_hostctl"     // consts (NOT inline literals) so they materialize as valid argv ptrs (.nx gotcha)
24const MH_PATHENV:   *u8 = "PATH=/usr/bin:/bin"
25const MH_RESTART:   *u8 = "restart"          // nx_hostctl restart <elf> = cmd_restart_known -> kill-by-cmdline -> guard respawns
26const MH_CORE_MIN:  i64 = 1073741824         // 1 GiB: reap Synology crash core-dumps (@*core.gz*) >= this from /volume1 (a 333 GB nx_hostctl core filled the volume)
27
28func mh_w(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
29// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
30// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
31// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
32// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
33func mh_wn(v: i64) -> i64 { nxi_out(v); return 0 }
34func mh_cat(dst: *u8, o: i64, s: *u8) -> i64 { var x: i64 = o; var i: i64 = 0; while s[i] != (0 as u8) { dst[x] = s[i]; x = x + 1; i = i + 1 } return x }
35func mh_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
36// substring test: does `hay` contain `needle`? bounded (needle length), safe past hay's NUL (0 != any needle byte).
37func mh_contains(hay: *u8, needle: *u8) -> i64 {
  let nl: i64 = mh_slen(needle)
  var i: i64 = 0
  while hay[i] != (0 as u8) {
      var j: i64 = 0; var ok: i64 = 1
      while j < nl { if hay[i + j] != needle[j] { ok = 0; j = nl } else { j = j + 1 } }
      if ok == 1 { return 1 }
      i = i + 1
  }
  return 0
47}
48
49// bounded /proc read (openat + one read + close). returns bytes, -1 on open fail.
50func mh_readproc(path: *u8, buf: *u8, cap: i64) -> i64 {
  let fd: i64 = sys_openat_rd(path)
  if fd < 0 { return 0 - 1 }
  let r: i64 = sys_read(fd, buf, cap - 1)
  sys_close(fd)
  if r > 0 { buf[r] = 0 as u8 } else { buf[0] = 0 as u8 }
  return r
57}
58func mh_skip_sp(buf: *u8, n: i64, p: i64) -> i64 {
  var i: i64 = p; var go: i64 = 1
  while go == 1 { if i >= n { go = 0 } else { if (buf[i] as i64) == 32 { i = i + 1 } else { go = 0 } } }
  return i
62}
63func mh_pdec(buf: *u8, n: i64, p: i64, pend: *i64) -> i64 {
  var i: i64 = p; var v: i64 = 0; var go: i64 = 1
  while go == 1 {
      if i >= n { go = 0 } else {
          let c: i64 = buf[i] as i64
          if c >= 48 { if c <= 57 { v = v * 10 + (c - 48); i = i + 1 } else { go = 0 } } else { go = 0 }
      }
  }
  pend[0] = i; return v
72}
73func mh_find_after(buf: *u8, n: i64, key: *u8, klen: i64) -> i64 {
  var i: i64 = 0; let last: i64 = n - klen
  while i <= last {
      var j: i64 = 0; var ok: i64 = 1
      while j < klen { if buf[i + j] != key[j] { ok = 0; j = klen } else { j = j + 1 } }
      if ok == 1 { return i + klen }
      i = i + 1
  }
  return 0 - 1
82}
83// memory used% = (MemTotal - MemAvailable) * 100 / MemTotal. 0 on failure (fail-safe: never heal on a bad read).
84func mh_mem_pct(buf: *u8) -> i64 {
  let r: i64 = mh_readproc("/proc/meminfo" as *u8, buf, MH_BUF)
  if r <= 0 { return 0 }
  let pend: *i64 = sys_mmap(16) as *i64
  let ta: i64 = mh_find_after(buf, r, "MemTotal:" as *u8, 9)
  let aa: i64 = mh_find_after(buf, r, "MemAvailable:" as *u8, 13)
  if ta < 0 { return 0 }
  if aa < 0 { return 0 }
  let tot: i64 = mh_pdec(buf, r, mh_skip_sp(buf, r, ta), pend)
  let av: i64 = mh_pdec(buf, r, mh_skip_sp(buf, r, aa), pend)
  if tot < 1 { return 0 }
  return (tot - av) * 100 / tot
96}
97// decimal into dst at off -> new off (path building for /proc/<pid>/...).
98func mh_n_into(dst: *u8, off: i64, v: i64) -> i64 {
  let t: *u8 = sys_mmap(24); var m: i64 = v; var k: i64 = 0
  if m == 0 { t[0] = 48 as u8; k = 1 }
  while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
  var i: i64 = 0; while i < k { dst[off + i] = t[k - 1 - i]; i = i + 1 }
  return off + k
104}
105func mh_pid_from_name(name: *u8) -> i64 {
  if name[0] == (0 as u8) { return 0 - 1 }
  var v: i64 = 0; var i: i64 = 0
  while name[i] != (0 as u8) { let c: i64 = name[i] as i64; if c < 48 { return 0 - 1 } if c > 57 { return 0 - 1 } v = v * 10 + (c - 48); i = i + 1 }
  return v
110}
111// resident pages from /proc/<pid>/statm (2nd field). 0 on fail. statm = "size RESIDENT shared ...".
112func mh_rss_pages(pid: i64, buf: *u8) -> i64 {
  let path: *u8 = sys_mmap(64); var o: i64 = 0
  let pf: *u8 = "/proc/" as *u8; while pf[o] != (0 as u8) { path[o] = pf[o]; o = o + 1 }
  o = mh_n_into(path, o, pid)
  let sf: *u8 = "/statm" as *u8; var j: i64 = 0; while sf[j] != (0 as u8) { path[o] = sf[j]; o = o + 1; j = j + 1 }
  path[o] = 0 as u8
  let r: i64 = mh_readproc(path, buf, 256)
  if r <= 0 { return 0 }
  let pend: *i64 = sys_mmap(16) as *i64
  mh_pdec(buf, r, 0, pend)                     // field 0 = total size
  let p2: i64 = mh_skip_sp(buf, r, pend[0])
  return mh_pdec(buf, r, p2, pend)             // field 1 = resident pages
124}
125// /proc/<pid>/comm (truncated 15-char name) into out. returns len.
126func mh_comm(pid: i64, out: *u8, cap: i64) -> i64 {
  let path: *u8 = sys_mmap(64); var o: i64 = 0
  let pf: *u8 = "/proc/" as *u8; while pf[o] != (0 as u8) { path[o] = pf[o]; o = o + 1 }
  o = mh_n_into(path, o, pid)
  let cf: *u8 = "/comm" as *u8; var j: i64 = 0; while cf[j] != (0 as u8) { path[o] = cf[j]; o = o + 1; j = j + 1 }
  path[o] = 0 as u8
  var r: i64 = mh_readproc(path, out, cap)
  var k: i64 = 0
  while k < r { if out[k] == (10 as u8) { out[k] = 0 as u8; r = k } k = k + 1 }   // strip trailing newline
  return r
136}
137func mh_streq(a: *u8, b: *u8) -> i64 {
  var i: i64 = 0
  while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 }
  if b[i] != (0 as u8) { return 0 }
  return 1
142}
143// ALLOWLIST: map a top-RSS process comm (kernel truncates to 15 chars) -> the hostctl kick sub. Fail-closed: an
144// unknown comm returns "" (we log but NEVER blind-kill an unrecognized process). Grows as we bless more daemons.
145func mh_setout(out: *u8, s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { out[i] = s[i]; i = i + 1 } out[i] = 0 as u8; return 1 }
146// map a top-RSS process comm (kernel truncates to 15 chars) -> the FULL elf name `nx_hostctl restart <elf>`
147// (cmd_restart_known) accepts. SAFE app-daemon set only -- EXCLUDES the control plane (nx_mgmt_api), the edge
148// (sites.elf), and the clock itself (auto-restarting those could cascade). Fail-closed: unknown comm -> "" ->
149// the healer LOGS but NEVER restarts an unrecognized process.
150func mh_kicksub(comm: *u8, out: *u8) -> i64 {
  if mh_streq(comm, "nx_docportal_ad" as *u8) == 1 { return mh_setout(out, "nx_docportal_admin_daemon.elf" as *u8) }
  if mh_streq(comm, "nx_media_server" as *u8) == 1 { return mh_setout(out, "nx_media_server_auth.elf" as *u8) }
  if mh_streq(comm, "nx_gallery_gate" as *u8) == 1 { return mh_setout(out, "nx_gallery_gateway.elf" as *u8) }
  if mh_streq(comm, "nx_gallery_serv" as *u8) == 1 { return mh_setout(out, "nx_gallery_serve.elf" as *u8) }
  if mh_streq(comm, "nx_wiki_gw.elf" as *u8) == 1 { return mh_setout(out, "nx_wiki_gw.elf" as *u8) }
  if mh_streq(comm, "nx_torrent_gw.e" as *u8) == 1 { return mh_setout(out, "nx_torrent_gw.elf" as *u8) }
  if mh_streq(comm, "nx_vroom_daemon" as *u8) == 1 { return mh_setout(out, "nx_vroom_daemon.elf" as *u8) }
  if mh_streq(comm, "nx_tools_api_se" as *u8) == 1 { return mh_setout(out, "nx_tools_api_serve.elf" as *u8) }
  if mh_streq(comm, "nx_fin_serve.el" as *u8) == 1 { return mh_setout(out, "nx_fin_serve.elf" as *u8) }
  if mh_streq(comm, "nx_mp_serve.elf" as *u8) == 1 { return mh_setout(out, "nx_mp_serve.elf" as *u8) }
  if mh_streq(comm, "nx_email_portal" as *u8) == 1 { return mh_setout(out, "nx_email_portal_daemon.elf" as *u8) }
  if mh_streq(comm, "nx_siteedit_dae" as *u8) == 1 { return mh_setout(out, "nx_siteedit_daemon.elf" as *u8) }
  if mh_streq(comm, "nx_health_eval." as *u8) == 1 { return mh_setout(out, "nx_health_eval.elf" as *u8) }
  out[0] = 0 as u8
  return 0
166}
167// append one line to the heal ledger (single write, O_APPEND).
168func mh_log(comm: *u8, pid: i64, rss_mb: i64, mem_pct: i64, sub: *u8, acted: i64) -> i64 {
  let b: *u8 = sys_mmap(512); var o: i64 = 0
  let p0: *u8 = "MEMHEAL t=" as *u8; var i: i64 = 0; while p0[i] != (0 as u8) { b[o] = p0[i]; o = o + 1; i = i + 1 }
  o = mh_n_into(b, o, sys_now_realtime_sec())
  let p1: *u8 = " mem_pct=" as *u8; i = 0; while p1[i] != (0 as u8) { b[o] = p1[i]; o = o + 1; i = i + 1 }
  o = mh_n_into(b, o, mem_pct)
  let p2: *u8 = " top=" as *u8; i = 0; while p2[i] != (0 as u8) { b[o] = p2[i]; o = o + 1; i = i + 1 }
  i = 0; while comm[i] != (0 as u8) { b[o] = comm[i]; o = o + 1; i = i + 1 }
  let p3: *u8 = " pid=" as *u8; i = 0; while p3[i] != (0 as u8) { b[o] = p3[i]; o = o + 1; i = i + 1 }
  o = mh_n_into(b, o, pid)
  let p4: *u8 = " rss_mb=" as *u8; i = 0; while p4[i] != (0 as u8) { b[o] = p4[i]; o = o + 1; i = i + 1 }
  o = mh_n_into(b, o, rss_mb)
  let p5: *u8 = " action=" as *u8; i = 0; while p5[i] != (0 as u8) { b[o] = p5[i]; o = o + 1; i = i + 1 }
  if acted == 1 { let a: *u8 = "RESTARTED " as *u8; i = 0; while a[i] != (0 as u8) { b[o] = a[i]; o = o + 1; i = i + 1 } i = 0; while sub[i] != (0 as u8) { b[o] = sub[i]; o = o + 1; i = i + 1 } }
  else { let a: *u8 = "none" as *u8; i = 0; while a[i] != (0 as u8) { b[o] = a[i]; o = o + 1; i = i + 1 } }
  b[o] = 10 as u8; o = o + 1
  let fd: i64 = sys_openat_append(MH_LOGP, 0x1a4)
  if fd >= 0 { sys_write(fd, b, o); sys_close(fd) }
  sys_write(1, b, o)
  return 0
188}
189// fork+exec `./nx_hostctl restart <elf>` (root under the clock -> kills the leaking daemon by cmdline; the
190// hostctl guard respawns it fresh, reclaiming the leaked pages).
191func mh_actuate(sub: *u8) -> i64 {
  let pid: i64 = sys_fork()
  if pid == 0 {
      let argv: *i64 = sys_mmap(32) as *i64
      argv[0] = MH_HOSTCTL as i64; argv[1] = MH_RESTART as i64; argv[2] = sub as i64; argv[3] = 0
      let envp: *i64 = sys_mmap(16) as *i64; envp[0] = MH_PATHENV as i64; envp[1] = 0
      sys_execve(MH_HOSTCTL, argv, envp)
      sys_exit(127)
  }
  let st: *i64 = sys_mmap(16) as *i64
  sys_wait4(pid, st, 0)
  return st[0]
203}
204
205// reap oversized Synology crash core-dumps from /volume1 (@*...core.gz*). A leaking daemon crash produces a HUGE
206// core (a 333 GB nx_hostctl dump was filling the volume); syno-dump-core.sh writes one on EVERY crash. Any core
207// >= MH_CORE_MIN is unlinked (root under the clock); small cores (<1 GiB) are kept for debugging. Runs every tick,
208// independent of memory -> the "don't worry" watchdog now self-heals DISK-filling cores too. Logged to the ledger.
209func mh_reap_cores() -> i64 {
  let fd: i64 = sys_openat_rd("/volume1" as *u8)
  if fd < 0 { return 0 }
  let gbuf: *u8 = sys_mmap(MH_MAGIC_65536)
  let stbuf: *u8 = sys_mmap(160)
  let path: *u8 = sys_mmap(512)
  var reaped: i64 = 0
  var go: i64 = 1
  while go == 1 {
      let n: i64 = sys_getdents64(fd, gbuf, MH_MAGIC_65536)
      if n <= 0 { go = 0 } else {
          var off: i64 = 0
          while off < n {
              let rec: *u8 = ((gbuf as i64) + off) as *u8
              let reclen: i64 = dirent_reclen(rec)
              let nm: *u8 = dirent_name(rec)
              if (nm[0] as i64) == 64 {
                  if mh_contains(nm, "core.gz" as *u8) == 1 {
                      var po: i64 = 0
                      let vp: *u8 = "/volume1/" as *u8
                      while vp[po] != (0 as u8) { path[po] = vp[po]; po = po + 1 }
                      var ni: i64 = 0
                      while nm[ni] != (0 as u8) { path[po] = nm[ni]; po = po + 1; ni = ni + 1 }
                      path[po] = 0 as u8
                      if sys_fstatat(path, stbuf) == 0 {
                          let sp: *i64 = stbuf as *i64
                          let sz: i64 = sp[6]                       // st_size @ byte offset 48 = i64 index 6
                          if sz >= MH_CORE_MIN {
                              __syscall(263, AT_FDCWD, path as i64, 0, 0, 0, 0)   // unlinkat(AT_FDCWD, path, 0)
                              reaped = reaped + 1
                              let lb: *u8 = sys_mmap(256); var lo: i64 = 0
                              lo = mh_cat(lb, lo, "MEMHEAL t=" as *u8); lo = mh_n_into(lb, lo, sys_now_realtime_sec())
                              lo = mh_cat(lb, lo, " action=REAPED_CORE size_mb=" as *u8); lo = mh_n_into(lb, lo, sz / MH_MAGIC_1048576)
                              lo = mh_cat(lb, lo, " name=" as *u8)
                              var xi: i64 = 0
                              while nm[xi] != (0 as u8) { lb[lo] = nm[xi]; lo = lo + 1; xi = xi + 1 }
                              lb[lo] = 10 as u8; lo = lo + 1
                              let lfd: i64 = sys_openat_append(MH_LOGP, 0x1a4)
                              if lfd >= 0 { sys_write(lfd, lb, lo); sys_close(lfd) }
                              sys_write(1, lb, lo)
                          }
                      }
                  }
              }
              if reclen <= 0 { off = n } else { off = off + reclen }
          }
      }
  }
  sys_close(fd)
  return reaped
259}
260
261func mh_gate() -> i64 {
  var ok: i64 = 1
  // threshold decision: >=88 heals, <88 does not
  if 88 < MH_THRESHOLD { ok = 0 }
  if 87 >= MH_THRESHOLD { ok = 0 }
  if 99 < MH_THRESHOLD { ok = 0 }
  // comm -> kicksub map
  let sub: *u8 = sys_mmap(64)
  if mh_kicksub("nx_docportal_ad" as *u8, sub) != 1 { ok = 0 }
  if mh_streq(sub, "nx_docportal_admin_daemon.elf" as *u8) != 1 { ok = 0 }
  if mh_kicksub("nx_gallery_gate" as *u8, sub) != 1 { ok = 0 }
  if mh_kicksub("nx_tools_api_se" as *u8, sub) != 1 { ok = 0 }
  if mh_kicksub("gitea" as *u8, sub) == 1 { ok = 0 }          // fail-closed: unknown process NOT restarted
  if mh_kicksub("dockerd" as *u8, sub) == 1 { ok = 0 }
  if mh_kicksub("nx_mgmt_api.elf" as *u8, sub) == 1 { ok = 0 }   // control plane EXCLUDED (never auto-restart)
  if mh_kicksub("sites.elf" as *u8, sub) == 1 { ok = 0 }         // edge EXCLUDED
  if ok == 1 { mh_w("MEMHEALGATE verdict=GREEN\n" as *u8) } else { mh_w("MEMHEALGATE verdict=RED\n" as *u8) }
  return 0
279}
280
281func main(argc: i64, argv: *i64) -> i64 {
  if argc >= 2 { if mh_streq(argv[1] as *u8, "gate" as *u8) == 1 { mh_gate(); return 0 } }
283
  // DISK self-heal: reap oversized crash core-dumps EVERY run, independent of the memory check below.
  mh_reap_cores()
286
  let big: *u8 = sys_mmap(MH_BUF)
  let mem_pct: i64 = mh_mem_pct(big)
  if mem_pct < MH_THRESHOLD {
      // heartbeat: log EVERY run (not just heals) -> an audit trail proving the auto-heal is alive + what it saw.
      let lb: *u8 = sys_mmap(160); var lo: i64 = 0
      lo = mh_cat(lb, lo, "MEMHEAL t=" as *u8); lo = mh_n_into(lb, lo, sys_now_realtime_sec())
      lo = mh_cat(lb, lo, " mem_pct=" as *u8); lo = mh_n_into(lb, lo, mem_pct)
      lo = mh_cat(lb, lo, " threshold=" as *u8); lo = mh_n_into(lb, lo, MH_THRESHOLD)
      lo = mh_cat(lb, lo, " action=ok\n" as *u8)
      let fd: i64 = sys_openat_append(MH_LOGP, 0x1a4)
      if fd >= 0 { sys_write(fd, lb, lo); sys_close(fd) }
      sys_write(1, lb, lo)
      return 0
  }
301
  // CRITICAL: scan /proc for the top-RSS process.
  let fd: i64 = sys_openat_rd("/proc" as *u8)
  if fd < 0 { return 0 }
  let gbuf: *u8 = sys_mmap(MH_BUF)
  let sbuf: *u8 = sys_mmap(512)
  let top_comm: *u8 = sys_mmap(64)
  var top_pid: i64 = 0
  var top_rss: i64 = 0
  var go: i64 = 1
  while go == 1 {
      let n: i64 = sys_getdents64(fd, gbuf, MH_BUF)
      if n <= 0 { go = 0 } else {
          var off: i64 = 0
          while off < n {
              let rec: *u8 = ((gbuf as i64) + off) as *u8
              let reclen: i64 = dirent_reclen(rec)
              let pid: i64 = mh_pid_from_name(dirent_name(rec))
              if pid > 0 {
                  let rss: i64 = mh_rss_pages(pid, sbuf)
                  if rss > top_rss { top_rss = rss; top_pid = pid; mh_comm(pid, top_comm, 64) }
              }
              if reclen <= 0 { off = n } else { off = off + reclen }
          }
      }
  }
  sys_close(fd)
328
  let rss_mb: i64 = top_rss / 256          // pages(4KB) -> MB
  let sub: *u8 = sys_mmap(64)
  let known: i64 = mh_kicksub(top_comm, sub)
  if known == 1 {
      mh_actuate(sub)
      mh_log(top_comm, top_pid, rss_mb, mem_pct, sub, 1)
  } else {
      mh_log(top_comm, top_pid, rss_mb, mem_pct, sub, 0)   // top offender not in the allowlist -> report only
  }
  return 0
339}
nx_mem_healer.nx source

`nx_mem_healer.nx` source
