code wiki / _hdl_build / nx_system_triage.nx
`nx_system_triage.nx` source

↩ module page · 517 lines · 25397 B
1// nx_system_triage.nx -- post-event situational awareness: WHAT IS RUNNING, WHAT IS CRASHED,
2// WHAT IS RECOVERABLE, reported live + durable. Built for the reboot/crash case: /tmp organs die,
3// daemons die, and until now nothing in the team could SAY so -- the operator had to ask Claude.
4//   (1) RUNNING      -- /proc sweep: every live process whose cmdline is team-ish (nx_/.sov.elf), pid+cmdline
5//   (2) DAEMONS      -- conf-listed expected daemons (D rows) matched against the same sweep -> RUNNING/DOWN
6//   (3) ORGANS       -- conf-listed expected elfs (E rows): /tmp/<n>.sov.elf|.elf present?
7//                       missing + source present = RECOVERABLE (rebuild lane named: S=sov_build_run W=wheeler)
8//                       missing + source missing = UNRECOVERABLE (loud)
9//   (4) CRASH-SIGNALS-- RUNTIME-CRASH count in knowledge/status/issues_durable.log (filed history)
10// Verdict: GREEN (all present, daemons up) / RECOVER (only rebuildable losses) / ATTENTION (unrecoverable
11// or expected daemon down). Durable TRIAGE line -> knowledge/status/system_triage.log; exit 0/2/1.
12// Expected set is DATA-DRIVEN: knowledge/status/triage_expected.conf (add a row = widen coverage).
13// Sovereign (syscalls only, no .sh). license_tier: ORIGINAL
14import "nx_syscalls.nx"
15import "nx_framed_append.nx"   // torn-write fix: the TRIAGE record = ONE locked fa_appendz (was ~34 sequential _fp/_fn)
16const ST_MAGIC_1073741824: i64 = 1073741824
17const ST_MAGIC_65536: i64 = 65536
18const ST_MAGIC_65520: i64 = 65520
19const ST_MAGIC_16384: i64 = 16384
20const ST_MAGIC_1048592: i64 = 1048592
21const ST_MAGIC_1048576: i64 = 1048576
22const ST_CAP: i64 = 1024       // bounded TRIAGE record (>= the full field set + verdict)
23func _p(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
24func _pn(v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48;k=1}; while m>0{t[k]=48+(m%10);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(1,bb,k); return 0 }
25func _fp(fd: i64, s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(fd,s,n); return 0 }
26func _fn(fd: i64, v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48;k=1}; while m>0{t[k]=48+(m%10);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(fd,bb,k); return 0 }
27func st_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
28func st_cat(dst: *u8, off: i64, s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { dst[off+i] = s[i]; i = i + 1 } return off + i }
29// does file at path open for read? (presence probe)
30func st_exists(path: *u8) -> i64 {
  let fd: i64 = sys_openat_rd(path)
  if fd < 0 { return 0 }
  sys_close(fd)
  return 1
35}
36// read up to cap bytes of path into buf; return byte count (0 if absent)
37func st_read_all(path: *u8, buf: *u8, cap: i64) -> i64 {
  let fd: i64 = sys_openat_rd(path)
  if fd < 0 { return 0 }
  var n: i64 = 0
  var go: i64 = 1
  while go == 1 { let base: i64 = buf as i64; let r: i64 = sys_read(fd, (base + n) as *u8, cap - n); if r <= 0 { go = 0 } else { n = n + r } if n >= cap { go = 0 } }
  sys_close(fd)
  return n
45}
46// substring scan over a raw byte buffer (hay may contain NULs; pat is NUL-terminated, NUL-free)
47func st_contains(hay: *u8, n: i64, pat: *u8) -> i64 {
  let pl: i64 = st_len(pat)
  if pl == 0 { return 0 }
  var i: i64 = 0
  while i + pl <= n {
      var k: i64 = 0
      var hit: i64 = 1
      while k < pl { if hay[i+k] != pat[k] { hit = 0; k = pl } else { k = k + 1 } }
      if hit == 1 { return 1 }
      i = i + 1
  }
  return 0
59}
60// count occurrences of pat in buffer
61func st_count_pat(hay: *u8, n: i64, pat: *u8) -> i64 {
  let pl: i64 = st_len(pat)
  if pl == 0 { return 0 }
  var cnt: i64 = 0
  var i: i64 = 0
  while i + pl <= n {
      var k: i64 = 0
      var hit: i64 = 1
      while k < pl { if hay[i+k] != pat[k] { hit = 0; k = pl } else { k = k + 1 } }
      if hit == 1 { cnt = cnt + 1 }
      i = i + 1
  }
  return cnt
74}
75// is dirent name all digits (= a pid dir under /proc)?
76func st_is_pid(name: *u8) -> i64 {
  var i: i64 = 0
  if name[0] == (0 as u8) { return 0 }
  while name[i] != (0 as u8) {
      if name[i] < (48 as u8) { return 0 }
      if name[i] > (57 as u8) { return 0 }
      i = i + 1
  }
  return 1
85}
86// organ presence: /tmp/<name>.sov.elf OR /tmp/<name>.elf OR _offc/<name>.elf
87// (_offc = the DURABLE home; 2026-06-10 the vault/audit helpers moved there so a
88// /tmp wipe can never kill them again -- durable presence counts as present)
89func tr_organ_present(name: *u8) -> i64 {
  let p: *u8 = sys_mmap(256)
  var o: i64 = 0
  o = st_cat(p, o, "/tmp/" as *u8); o = st_cat(p, o, name); o = st_cat(p, o, ".sov.elf" as *u8); p[o] = 0 as u8
  if st_exists(p) == 1 { return 1 }
  o = 0
  o = st_cat(p, o, "/tmp/" as *u8); o = st_cat(p, o, name); o = st_cat(p, o, ".elf" as *u8); p[o] = 0 as u8
  if st_exists(p) == 1 { return 1 }
  o = 0
  o = st_cat(p, o, "_offc/" as *u8); o = st_cat(p, o, name); o = st_cat(p, o, ".elf" as *u8); p[o] = 0 as u8
  return st_exists(p)
100}
101// organ source present: runtime/_hdl_build/<n>.nx OR runtime/<n>.nx OR nxasm/<n>.nx (mirrors runner fallback)
102func tr_source_present(name: *u8) -> i64 {
  let p: *u8 = sys_mmap(256)
  var o: i64 = 0
  o = st_cat(p, o, "runtime/_hdl_build/" as *u8); o = st_cat(p, o, name); o = st_cat(p, o, ".nx" as *u8); p[o] = 0 as u8
  if st_exists(p) == 1 { return 1 }
  o = 0
  o = st_cat(p, o, "runtime/" as *u8); o = st_cat(p, o, name); o = st_cat(p, o, ".nx" as *u8); p[o] = 0 as u8
  if st_exists(p) == 1 { return 1 }
  o = 0
  o = st_cat(p, o, "nxasm/" as *u8); o = st_cat(p, o, name); o = st_cat(p, o, ".nx" as *u8); p[o] = 0 as u8
  return st_exists(p)
113}
114// free GB on the filesystem holding path. x86 statfs=137 called DIRECT (probe-proven 2026-06-10:
115// rv64 43 returns -9 through the translation table; 137 raw matches df exactly). -1 = statfs failed.
116func tr_free_gb(path: *u8) -> i64 {
  let buf: *i64 = sys_mmap(144) as *i64
  let rc: i64 = __syscall(137, path, buf, 0, 0, 0, 0)
  if rc != 0 { return 0 - 1 }
  return (buf[1] * buf[4]) / ST_MAGIC_1073741824
121}
122// fork+exec elf with up to 2 args, output muted; exit code (128+sig on crash)
123func tr_run2(elf: *u8, a1: *u8, a2: *u8) -> i64 {
  let pid: i64 = sys_fork()
  if pid == 0 {
      let argv: *i64 = sys_mmap(64) as *i64
      argv[0] = elf as i64
      var i: i64 = 1
      if (a1 as i64) != 0 { argv[i] = a1 as i64; i = i + 1 }
      if (a2 as i64) != 0 { argv[i] = a2 as i64; i = i + 1 }
      argv[i] = 0
      let envp: *i64 = sys_mmap(16) as *i64; envp[0] = 0
      let dn: i64 = sys_openat_wr("/dev/null" as *u8, 0x1a4)
      if dn >= 0 { sys_dup3(dn, 1, 0); sys_dup3(dn, 2, 0) }
      sys_execve(elf, argv, envp)
      sys_exit(127)
  }
  let st: *i64 = sys_mmap(16) as *i64
  sys_wait4(pid, st, 0)
  let sig: i64 = st[0] & 0x7f
  if sig != 0 { return 128 + sig }
  return (st[0] >> 8) & 0xff
143}
144// one /proc/<pid> entry: read cmdline; if team-ish print pid+cmdline; mark any matched D rows.
145// returns 1 if team-ish (counted as RUNNING).  6 args (register path, no stack args).
146func tr_check_pid(nm: *u8, cmdl: *u8, ppath: *u8, dnames: *i64, dfound: *i64, nd: i64) -> i64 {
  var po: i64 = 0
  po = st_cat(ppath, po, "/proc/" as *u8); po = st_cat(ppath, po, nm); po = st_cat(ppath, po, "/cmdline" as *u8); ppath[po] = 0 as u8
  let cl: i64 = st_read_all(ppath, cmdl, 500)
  if cl <= 0 { return 0 }
  var dk: i64 = 0
  while dk < nd { if st_contains(cmdl, cl, dnames[dk] as *u8) == 1 { dfound[dk] = 1 } dk = dk + 1 }
  var teamish: i64 = st_contains(cmdl, cl, "nx_" as *u8)
  if teamish == 0 { teamish = st_contains(cmdl, cl, ".sov.elf" as *u8) }
  if teamish == 0 { return 0 }
  var b: i64 = 0
  while b < cl { if cmdl[b] == (0 as u8) { cmdl[b] = 32 as u8 } b = b + 1 }
  _p("    pid " as *u8); _p(nm); _p("  " as *u8); sys_write(1, cmdl, cl); _p("\n" as *u8)
  return 1
160}
161// LAST number following pat in buffer (digits parsed immediately after the final match; -1 = no match)
162func st_last_num_after(hay: *u8, n: i64, pat: *u8) -> i64 {
  let pl: i64 = st_len(pat)
  if pl == 0 { return 0 - 1 }
  var best: i64 = 0 - 1
  var i: i64 = 0
  while i + pl <= n {
      var k: i64 = 0
      var hit: i64 = 1
      while k < pl { if hay[i+k] != pat[k] { hit = 0; k = pl } else { k = k + 1 } }
      if hit == 1 { best = i + pl }
      i = i + 1
  }
  if best < 0 { return 0 - 1 }
  var v: i64 = 0
  var q: i64 = best
  var any: i64 = 0
  while q < n {
      if hay[q] >= (48 as u8) { if hay[q] <= (57 as u8) { v = v * 10 + ((hay[q] as i64) - 48); any = 1; q = q + 1 } else { q = n } } else { q = n }
  }
  if any == 0 { return 0 - 1 }
  return v
183}
184// argv[1] = conf path override, argv[2] = durable log override (defaults below) -- lets the
185// triage GATE run synthetic fixtures without polluting the real trend log (site-gate pattern)
186// argv[3] = beat-log override, argv[4] = boot-journal override (CR5 grace fixtures).
187// CR5 boot-grace (crash-recovery ladder, 2026-06-10): a stale beat/evidence verdict caused by
188// a JOURNALED recent boot (boot_journal.log BOOT row within the conf G window) grades RECOVER,
189// not ATTENTION -- the crash is already measured + revived; a wedged-while-up daemon has no
190// recent BOOT row and stays red. G row absent/0 = grace off.
191func main(argc: i64, argv: *i64) -> i64 {
  _p("=== SYSTEM TRIAGE: running / crashed / recoverable (post-event situational awareness) ===\n" as *u8)
  var confpath: *u8 = "knowledge/status/triage_expected.conf" as *u8
  if argc >= 2 { confpath = argv[1] as *u8 }
  var logpath: *u8 = "knowledge/status/system_triage.log" as *u8
  if argc >= 3 { logpath = argv[2] as *u8 }
  var beatpath: *u8 = "knowledge/status/pulse_daemon.log" as *u8
  if argc >= 4 { beatpath = argv[3] as *u8 }
  var journalpath: *u8 = "knowledge/status/boot_journal.log" as *u8
  if argc >= 5 { journalpath = argv[4] as *u8 }
  // ---- parse the expected-set conf (data-driven) ----
  let conf: *u8 = sys_mmap(ST_MAGIC_65536)
  let cn: i64 = st_read_all(confpath, conf, ST_MAGIC_65520)
  if cn <= 0 { _p(" TRIAGE: conf MISSING (" as *u8); _p(confpath); _p(") -- fail loud\n" as *u8); sys_exit(1); return 1 }
  let pool: *u8 = sys_mmap(ST_MAGIC_16384)
  var poff: i64 = 0
  let enames: *i64 = sys_mmap(64*8) as *i64
  let elanes: *u8 = sys_mmap(64)
  var ne: i64 = 0
  let dnames: *i64 = sys_mmap(16*8) as *i64
  var nd: i64 = 0
  var mingb: i64 = 0
  var hmax: i64 = 0
  var grace: i64 = 0
  let fbud: *i64 = sys_mmap(16*8) as *i64
  let fpath: *i64 = sys_mmap(16*8) as *i64
  var nf: i64 = 0
  let rhost: *i64 = sys_mmap(8*8) as *i64
  let rport: *i64 = sys_mmap(8*8) as *i64
  var nr: i64 = 0
  var ci: i64 = 0
  while ci < cn {
      // find end of this line (flag loop, no sentinels)
      var le: i64 = ci
      var scan: i64 = 1
      while scan == 1 {
          if le >= cn { scan = 0 }
          else { if conf[le] == (10 as u8) { scan = 0 } else { le = le + 1 } }
      }
      // 'E <name> <lane>'
      if conf[ci] == (69 as u8) {
          var te: i64 = ci + 2
          var s2: i64 = 1
          while s2 == 1 {
              if te >= le { s2 = 0 }
              else { if conf[te] == (32 as u8) { s2 = 0 } else { te = te + 1 } }
          }
          let nptr: i64 = (pool as i64) + poff
          var q: i64 = ci + 2
          while q < te { pool[poff] = conf[q]; poff = poff + 1; q = q + 1 }
          pool[poff] = 0 as u8; poff = poff + 1
          if ne < 64 {
              enames[ne] = nptr
              elanes[ne] = 83 as u8
              if te + 1 < le { elanes[ne] = conf[te+1] }
              ne = ne + 1
          }
      }
      // 'T <min-free-GB>' (disk budget; 0/absent = report-only, no gating)
      if conf[ci] == (84 as u8) {
          var tv: i64 = 0
          var q3: i64 = ci + 2
          while q3 < le {
              if conf[q3] >= (48 as u8) { if conf[q3] <= (57 as u8) { tv = tv * 10 + ((conf[q3] as i64) - 48) } }
              q3 = q3 + 1
          }
          mingb = tv
      }
      // 'H <max-beat-age-sec>' (heartbeat budget: last DAEMON-BEAT must be younger; 0/absent = skip)
      if conf[ci] == (72 as u8) {
          var hv: i64 = 0
          var q4: i64 = ci + 2
          while q4 < le {
              if conf[q4] >= (48 as u8) { if conf[q4] <= (57 as u8) { hv = hv * 10 + ((conf[q4] as i64) - 48) } }
              q4 = q4 + 1
          }
          hmax = hv
      }
      // 'G <grace-sec>' (CR5 boot-grace window after a journaled boot; 0/absent = off)
      if conf[ci] == (71 as u8) {
          var gv: i64 = 0
          var q9: i64 = ci + 2
          while q9 < le {
              if conf[q9] >= (48 as u8) { if conf[q9] <= (57 as u8) { gv = gv * 10 + ((conf[q9] as i64) - 48) } }
              q9 = q9 + 1
          }
          grace = gv
      }
      // 'F <max-age-sec> <logpath>' (evidence freshness: last epoch= in that log must be younger)
      if conf[ci] == (70 as u8) {
          var fv: i64 = 0
          var q5: i64 = ci + 2
          var s5: i64 = 1
          while s5 == 1 {
              if q5 >= le { s5 = 0 }
              else {
                  if conf[q5] == (32 as u8) { s5 = 0 }
                  else {
                      if conf[q5] >= (48 as u8) { if conf[q5] <= (57 as u8) { fv = fv * 10 + ((conf[q5] as i64) - 48) } }
                      q5 = q5 + 1
                  }
              }
          }
          let fptr: i64 = (pool as i64) + poff
          var q6: i64 = q5 + 1
          while q6 < le { pool[poff] = conf[q6]; poff = poff + 1; q6 = q6 + 1 }
          pool[poff] = 0 as u8; poff = poff + 1
          if nf < 16 { fbud[nf] = fv; fpath[nf] = fptr; nf = nf + 1 }
      }
      // 'R <host-or-ip> <port>' (remote reachability via nx_reach_probe; both tokens kept as strings)
      if conf[ci] == (82 as u8) {
          var t7: i64 = ci + 2
          var s7: i64 = 1
          while s7 == 1 {
              if t7 >= le { s7 = 0 }
              else { if conf[t7] == (32 as u8) { s7 = 0 } else { t7 = t7 + 1 } }
          }
          let hptr: i64 = (pool as i64) + poff
          var q7: i64 = ci + 2
          while q7 < t7 { pool[poff] = conf[q7]; poff = poff + 1; q7 = q7 + 1 }
          pool[poff] = 0 as u8; poff = poff + 1
          let pptr: i64 = (pool as i64) + poff
          var q8: i64 = t7 + 1
          while q8 < le { pool[poff] = conf[q8]; poff = poff + 1; q8 = q8 + 1 }
          pool[poff] = 0 as u8; poff = poff + 1
          if nr < 8 { rhost[nr] = hptr; rport[nr] = pptr; nr = nr + 1 }
      }
      // 'D <substr>'
      if conf[ci] == (68 as u8) {
          let dptr: i64 = (pool as i64) + poff
          var q2: i64 = ci + 2
          while q2 < le { pool[poff] = conf[q2]; poff = poff + 1; q2 = q2 + 1 }
          pool[poff] = 0 as u8; poff = poff + 1
          if nd < 16 { dnames[nd] = dptr; nd = nd + 1 }
      }
      ci = le + 1
  }
  // ---- /proc sweep: RUNNING team-ish processes + daemon matching ----
  let dfound: *i64 = sys_mmap(16*8) as *i64
  var di: i64 = 0
  while di < nd { dfound[di] = 0; di = di + 1 }
  var nproc: i64 = 0
  _p(" -- RUNNING (team-ish processes in /proc) --\n" as *u8)
  let pfd: i64 = sys_openat_rd("/proc" as *u8)
  if pfd >= 0 {
      let dirbuf: *u8 = sys_mmap(ST_MAGIC_65536)
      let cmdl: *u8 = sys_mmap(512)
      let ppath: *u8 = sys_mmap(256)
      var done: i64 = 0
      while done == 0 {
          let nb: i64 = sys_getdents64(pfd, dirbuf, ST_MAGIC_65536)
          if nb <= 0 { done = 1 }
          else {
              var off: i64 = 0
              while off < nb {
                  let dbase: i64 = dirbuf as i64
                  let rec: *u8 = (dbase + off) as *u8
                  let rl: i64 = dirent_reclen(rec)
                  if rl <= 0 { off = nb }
                  else {
                      let nm: *u8 = dirent_name(rec)
                      if st_is_pid(nm) == 1 { nproc = nproc + tr_check_pid(nm, cmdl, ppath, dnames, dfound, nd) }
                      off = off + rl
                  }
              }
          }
      }
      sys_close(pfd)
  }
  if nproc == 0 { _p("    (none)\n" as *u8) }
  // ---- expected daemons verdicts ----
  var ddown: i64 = 0
  if nd > 0 {
      _p(" -- EXPECTED DAEMONS --\n" as *u8)
      var dv: i64 = 0
      while dv < nd {
          _p("    " as *u8); _p(dnames[dv] as *u8); _p(": " as *u8)
          if dfound[dv] == 1 { _p("RUNNING\n" as *u8) } else { _p("DOWN\n" as *u8); ddown = ddown + 1 }
          dv = dv + 1
      }
  }
  // ---- expected organs: present / recoverable / unrecoverable ----
  _p(" -- EXPECTED ORGANS (/tmp elfs) --\n" as *u8)
  var present: i64 = 0
  var recov: i64 = 0
  var unrec: i64 = 0
  var ei: i64 = 0
  while ei < ne {
      let en: *u8 = enames[ei] as *u8
      _p("    " as *u8); _p(en); _p(": " as *u8)
      if tr_organ_present(en) == 1 { _p("PRESENT\n" as *u8); present = present + 1 }
      else {
          if tr_source_present(en) == 1 {
              recov = recov + 1
              _p("RECOVERABLE via " as *u8)
              if elanes[ei] == (87 as u8) { _p("wheeler-bootstrap lane\n" as *u8) } else { _p("nx_sov_build_run\n" as *u8) }
          } else { _p("UNRECOVERABLE (no source!)\n" as *u8); unrec = unrec + 1 }
      }
      ei = ei + 1
  }
  // ---- remote reachability (the NAS/live-site eye; probe is watchdog-bounded at 10s/row) ----
  var rdown: i64 = 0
  if nr > 0 {
      _p(" -- REACHABILITY (remote, via own DNS+TCP probe) --\n" as *u8)
      let probe2: i64 = sys_openat_rd("/tmp/nx_reach_probe.sov.elf" as *u8)
      if probe2 >= 0 { sys_close(probe2) }
      else { tr_run2("_offc/nx_sov_build_run.elf" as *u8, "nx_reach_probe" as *u8, 0 as *u8) }
      var ri: i64 = 0
      while ri < nr {
          _p("    " as *u8); _p(rhost[ri] as *u8); _p(":" as *u8); _p(rport[ri] as *u8); _p(" " as *u8)
          let rrc: i64 = tr_run2("/tmp/nx_reach_probe.sov.elf" as *u8, rhost[ri] as *u8, rport[ri] as *u8)
          if rrc == 0 { _p("REACHABLE\n" as *u8) } else { _p("UNREACHABLE rc=" as *u8); _pn(rrc); _p("\n" as *u8); rdown = rdown + 1 }
          ri = ri + 1
      }
  }
  // ---- crash signals filed (history; does not gate the verdict) ----
  let ibuf: *u8 = sys_mmap(ST_MAGIC_1048592)
  let inn: i64 = st_read_all("knowledge/status/issues_durable.log" as *u8, ibuf, ST_MAGIC_1048576)
  let ncrash: i64 = st_count_pat(ibuf, inn, "RUNTIME-CRASH" as *u8)
  _p(" -- CRASH SIGNALS filed (issues_durable.log): " as *u8); _pn(ncrash); _p(" RUNTIME-CRASH lines --\n" as *u8)
  // ---- disk budget (the 2026-06-10 lesson: 147GB of crash dumps emptied C: and NOTHING said so;
  // T row in the conf = the threshold; 0/absent = report-only) ----
  let repo_gb: i64 = tr_free_gb("." as *u8)
  let tmp_gb: i64 = tr_free_gb("/tmp" as *u8)
  var dlow: i64 = 0
  _p(" -- DISK: repo-disk free " as *u8); _pn(repo_gb); _p("GB, /tmp free " as *u8); _pn(tmp_gb)
  _p("GB (budget min " as *u8); _pn(mingb); _p("GB) --\n" as *u8)
  if mingb > 0 {
      if repo_gb >= 0 { if repo_gb < mingb { dlow = dlow + 1; _p("    DISK-LOW: repo disk below budget!\n" as *u8) } }
      if tmp_gb >= 0 { if tmp_gb < mingb { dlow = dlow + 1; _p("    DISK-LOW: /tmp below budget!\n" as *u8) } }
  }
  // ---- heartbeat staleness (a daemon can be ALIVE in /proc yet wedged -- the beat log is the
  // proof of life; H row = max age budget) ----
  var stale: i64 = 0
  var beat_age: i64 = 0 - 1
  if hmax > 0 {
      let hbuf: *u8 = sys_mmap(ST_MAGIC_1048592)
      let hn: i64 = st_read_all(beatpath, hbuf, ST_MAGIC_1048576)
      let bepoch: i64 = st_last_num_after(hbuf, hn, "DAEMON-BEAT epoch=" as *u8)
      if bepoch > 0 { beat_age = sys_now_realtime_sec() - bepoch }
      _p(" -- HEARTBEAT: last DAEMON-BEAT age " as *u8); _pn(beat_age); _p("s (budget " as *u8); _pn(hmax); _p("s) --\n" as *u8)
      if beat_age < 0 { stale = 1; _p("    BEAT-STALE: no DAEMON-BEAT line found!\n" as *u8) }
      if beat_age > hmax { stale = 1; _p("    BEAT-STALE: daemon exists but is not beating!\n" as *u8) }
  }
  // ---- evidence freshness (claims EXPIRE: a green verdict from days ago is not a grade;
  // each F row = one durable log whose last epoch= must be within budget) ----
  var fstale: i64 = 0
  if nf > 0 {
      _p(" -- EVIDENCE FRESHNESS (claims expire; per-log budgets) --\n" as *u8)
      let fb: *u8 = sys_mmap(ST_MAGIC_1048592)
      var fi: i64 = 0
      while fi < nf {
          let fp2: *u8 = fpath[fi] as *u8
          let fn2: i64 = st_read_all(fp2, fb, ST_MAGIC_1048576)
          let fe: i64 = st_last_num_after(fb, fn2, "epoch=" as *u8)
          var age2: i64 = 0 - 1
          if fe > 0 { age2 = sys_now_realtime_sec() - fe }
          _p("    " as *u8); _p(fp2); _p(": age " as *u8); _pn(age2); _p("s (budget " as *u8); _pn(fbud[fi]); _p("s) " as *u8)
          var rowok: i64 = 1
          if age2 < 0 { rowok = 0 }
          if age2 > fbud[fi] { rowok = 0 }
          if rowok == 1 { _p("FRESH\n" as *u8) } else { _p("STALE\n" as *u8); fstale = fstale + 1 }
          fi = fi + 1
      }
  }
  // ---- CR5 boot-grace: is current staleness explained by a JOURNALED recent boot? ----
  var graced: i64 = 0
  var boot_age: i64 = 0 - 1
  if grace > 0 {
      if stale + fstale > 0 {
          let jb: *u8 = sys_mmap(ST_MAGIC_1048592)
          let jn2: i64 = st_read_all(journalpath, jb, ST_MAGIC_1048576)
          let be2: i64 = st_last_num_after(jb, jn2, "BOOT boot_epoch=" as *u8)
          if be2 > 0 {
              boot_age = sys_now_realtime_sec() - be2
              if boot_age >= 0 { if boot_age <= grace { graced = stale + fstale } }
          }
      }
  }
  if graced > 0 {
      _p(" -- BOOT-GRACE: staleness is explained by a journaled boot " as *u8); _pn(boot_age)
      _p("s ago (window " as *u8); _pn(grace); _p("s) -> RECOVERING, not wedged --\n" as *u8)
  }
  // ---- verdict + durable line ----
  var verdict: i64 = 0                     // 0 GREEN / 2 RECOVER / 1 ATTENTION
  if fstale > 0 { verdict = 1 }
  if stale > 0 { verdict = 1 }
  if graced > 0 { if verdict == 1 { verdict = 2 } }   // staleness covered by the journaled boot
  if recov > 0 { verdict = 2 }
  if unrec > 0 { verdict = 1 }
  if ddown > 0 { verdict = 1 }
  if dlow > 0 { verdict = 1 }
  if rdown > 0 { verdict = 1 }
  let lfd: i64 = sys_openat_append(logpath, 0x1a4)
  if lfd >= 0 {
      sys_close(lfd)   // writability check; the TRIAGE record below is ONE atomic fa_appendz write
      let tb: *u8 = sys_mmap(ST_CAP + 16)
      var o: i64 = 0
      o = fa_cat(tb, o, "TRIAGE epoch=" as *u8); o = fa_catn(tb, o, sys_now_realtime_sec())
      o = fa_cat(tb, o, " procs_running=" as *u8); o = fa_catn(tb, o, nproc)
      o = fa_cat(tb, o, " daemons_down=" as *u8); o = fa_catn(tb, o, ddown)
      o = fa_cat(tb, o, " organs_present=" as *u8); o = fa_catn(tb, o, present)
      o = fa_cat(tb, o, " organs_recoverable=" as *u8); o = fa_catn(tb, o, recov)
      o = fa_cat(tb, o, " organs_unrecoverable=" as *u8); o = fa_catn(tb, o, unrec)
      o = fa_cat(tb, o, " crash_signals=" as *u8); o = fa_catn(tb, o, ncrash)
      o = fa_cat(tb, o, " disk_repo_gb=" as *u8); o = fa_catn(tb, o, repo_gb)
      o = fa_cat(tb, o, " disk_tmp_gb=" as *u8); o = fa_catn(tb, o, tmp_gb)
      o = fa_cat(tb, o, " disk_low=" as *u8); o = fa_catn(tb, o, dlow)
      o = fa_cat(tb, o, " beat_age=" as *u8); o = fa_catn(tb, o, beat_age)
      o = fa_cat(tb, o, " beat_stale=" as *u8); o = fa_catn(tb, o, stale)
      o = fa_cat(tb, o, " evidence_stale=" as *u8); o = fa_catn(tb, o, fstale)
      o = fa_cat(tb, o, " boot_age=" as *u8); o = fa_catn(tb, o, boot_age)
      o = fa_cat(tb, o, " graced=" as *u8); o = fa_catn(tb, o, graced)
      o = fa_cat(tb, o, " reach_down=" as *u8); o = fa_catn(tb, o, rdown)
      if verdict == 0 { o = fa_cat(tb, o, " verdict=GREEN" as *u8) }
      if verdict == 2 { o = fa_cat(tb, o, " verdict=RECOVER" as *u8) }
      if verdict == 1 { o = fa_cat(tb, o, " verdict=ATTENTION" as *u8) }
      tb[o] = 0 as u8
      fa_appendz(logpath, tb, ST_CAP)
  }
  _p(" durable: " as *u8); _p(logpath); _p("\n" as *u8)
  if verdict == 0 { _p(" SYSTEM TRIAGE: GREEN (everything expected is alive or present)\n" as *u8); sys_exit(0); return 0 }
  if verdict == 2 { _p(" SYSTEM TRIAGE: RECOVER (losses are rebuildable -- run the named lanes)\n" as *u8); sys_exit(2); return 2 }
  _p(" SYSTEM TRIAGE: ATTENTION (unrecoverable organ, daemon down, disk below budget, or beat stale)\n" as *u8)
  sys_exit(1)
  return 1
517}
nx_system_triage.nx source

`nx_system_triage.nx` source
