code wiki / _hdl_build / nx_raidwatch.nx
`nx_raidwatch.nx` source

↩ module page · 457 lines · 22670 B
1// nx_raidwatch.nx -- ARRAY + KERNEL-IO HEALTH BEAT. The estate had NONE.
2//
3// WHY THIS EXISTS (2026-08-07). A grep across every organ for mdstat, smartctl or raid returned only
4// substring false positives (a bip39 wordlist, a chemistry table). Nothing in a 140 TB estate was
5// watching whether its arrays were intact. Two live faults were sitting there unwatched:
6//   md3 (raid1, the NVMe mirror) DEGRADED [2/1] [_U] -- one member gone, /dev/nvme0n1p1 has no
7//     device node at all, so it cannot even be re-added without physical intervention. It was
8//     recorded in a memory file and NOTHING CHECKED IT, which is the difference between a note and
9//     a monitor.
10//   sata8 throwing repeated UNC (uncorrectable) read errors on ONE sector, 840 kernel records over
11//     four hours, while the array it backs still reports [8/8] because raid5 keeps reconstructing
12//     the block from parity. A disk can be failing loudly while every array metric reads healthy.
13//
14// ROOT-FREE BY CONSTRUCTION, because the beat runs as the estate user:
15//   /sys/block/<md>/md/degraded, raid_disks, level, mismatch_cnt -- structured integers, no text
16//     parsing of /proc/mdstat, so this cannot be broken by a format change.
17//   /dev/kmsg -- readable here, opened O_NONBLOCK so the final read returns EAGAIN instead of
18//     BLOCKING FOREVER at the end of the buffer. A monitor that can hang is not a monitor.
19//   smartctl is present but needs root (open of /dev/sata8 = Permission denied, measured), so
20//     per-disk SMART is deliberately OUT OF SCOPE here rather than silently half-working.
21//
22// REFUSES if it can read neither surface: ignorance is not health, and a GREEN produced by an
23// unreadable sysfs would be worse than no beat at all.
24import "nx_syscalls.nx"
25
26const RW_MAXMD:        i64 = 16
27const RW_MAXSECT:      i64 = 256
28const RW_KMSG_RECS:    i64 = 8192
29const RW_RECBUF:       i64 = 8192
30const RW_SMALL:        i64 = 64
31const RW_PATH:         i64 = 256
32const RW_O_NONBLOCK:   i64 = 2048
33const RW_AT_FDCWD:     i64 = 0 - 100
34const RW_SYS_OPENAT:   i64 = 257
35const RW_EXIT_RED:     i64 = 6
36const RW_EXIT_AMBER:   i64 = 4
37const RW_EXIT_NODATA:  i64 = 5
38const RW_DIRBUF:       i64 = 65536   // getdents64 batch for the per-member walk
39const RW_RECLEN_OFF:   i64 = 16      // linux_dirent64.d_reclen
40const RW_NAME_OFF:     i64 = 19      // linux_dirent64.d_name
41
42// ---- DURABLE VERDICT (2026-09-03). THIS MONITOR HAS BEEN RIGHT AND UNHEARD FOR 25 DAYS.
43// It sits on an hourly clock row with NO output redirection and wrote no artifact, so a RED verdict --
44// md3 running with zero redundancy since 2026-08-09 -- was computed and thrown away every hour, and
45// nothing anywhere consumed it. That is the estate's own law firing on the one monitor that matters
46// most: A CRON ROW THAT DISCARDS ITS OUTPUT MAKES ITS OWN FAILURE UNOBSERVABLE. The console line is
47// unchanged byte for byte -- anything already parsing stdout keeps working -- and the same numbers now
48// also land in a TRUNCATE-written status file with the verdict LAST, which is the shape a watch row and
49// gv_last_line can consume.
50const RW_STATUS:  *u8 = "knowledge/status/raidwatch.status"
51const RW_STATBUF: i64 = 1024
52// THE VERDICT LADDER, COMPUTED ONCE. The status file reads THIS; the printed branches below keep their
53// own prose because each names a DIFFERENT cause, but the machine-readable answer has exactly one
54// definition. Two copies of a decision agree the day they are written and drift on the next edit.
55func rw_verdict_code(degraded: i64, ioerr: i64, memberbad: i64, mismatched: i64) -> i64 {
  if degraded > 0 { return RW_EXIT_RED }
  if ioerr > 0 { return RW_EXIT_RED }
  if memberbad > 0 { return RW_EXIT_AMBER }
  if mismatched > 0 { return RW_EXIT_AMBER }
  return 0
61}
62func rw(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
63func rwe(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(2, s, n); return 0 }
64func rwn(v: i64) -> i64 {
  var m: i64 = v
  if m < 0 { rw("-" as *u8); m = 0 - m }
  let t: *u8 = sys_mmap(28)
  var k: i64 = 0
  if m == 0 { t[0] = 48 as u8; k = 1 }
  while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
  let o: *u8 = sys_mmap(28)
  var i: i64 = 0
  while i < k { o[i] = t[k - 1 - i]; i = i + 1 }
  sys_write(1, o, k)
  sys_munmap(t, 28)
  sys_munmap(o, 28)
  return 0
78}
79
80// append a NUL-terminated literal at offset o
81func rw_cat(d: *u8, o: i64, s: *u8) -> i64 { var i: i64 = 0; while s[i] != (0 as u8) { d[o + i] = s[i]; i = i + 1 } return o + i }
82// append a small non-negative integer
83func rw_catn(d: *u8, o: i64, v: i64) -> i64 {
  if v < 0 { d[o] = 45 as u8; return rw_catn(d, o + 1, 0 - v) }
  // NEGATIVES (2026-08-07). Without this the `while m > 0` loop below never runs for a
  // negative value and this function emits ZERO CHARACTERS, silently corrupting whatever
  // format it is writing into. Handled AT THE SIGNATURE so it is independent of which
  // cursor variable the body happens to use. Non-negative input is byte-identical (rule 19).
  if v < 0 { d[o] = 45 as u8; return rw_catn(d, o + 1, 0 - v) }
  if v == 0 { d[o] = 48 as u8; return o + 1 }
  let t: *u8 = sys_mmap(28)
  var m: i64 = v
  var k: i64 = 0
  while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
  var i: i64 = 0
  while i < k { d[o + i] = t[k - 1 - i]; i = i + 1 }
  sys_munmap(t, 28)
  return o + k
99}
100
101// read a small sysfs file as an integer. -1 = unreadable, which is NOT the same as 0.
102func rw_sysint(path: *u8) -> i64 {
  let fd: i64 = sys_openat_rd(path)
  if fd < 0 { return 0 - 1 }
  let b: *u8 = sys_mmap(RW_SMALL)
  let n: i64 = sys_read(fd, b, RW_SMALL - 1)
  sys_close(fd)
  if n <= 0 { sys_munmap(b, RW_SMALL); return 0 - 1 }
  var v: i64 = 0
  var any: i64 = 0
  var i: i64 = 0
  while i < n {
      let c: i64 = b[i] as i64
      if c >= 48 { if c <= 57 { v = v * 10 + (c - 48); any = 1 } }
      i = i + 1
  }
  sys_munmap(b, RW_SMALL)
  if any == 0 { return 0 - 1 }
  return v
120}
121
122// copy a small sysfs string (level name) into dst, NUL-terminated; 0 on success, -1 unreadable
123func rw_sysstr(path: *u8, dst: *u8, cap: i64) -> i64 {
  let fd: i64 = sys_openat_rd(path)
  if fd < 0 { return 0 - 1 }
  var n: i64 = sys_read(fd, dst, cap - 1)
  sys_close(fd)
  if n <= 0 { return 0 - 1 }
  while n > 0 { if dst[n - 1] == (10 as u8) { n = n - 1 } else { dst[n] = 0 as u8; return 0 } }
  dst[0] = 0 as u8
  return 0
132}
133
134// first integer following the NUL-terminated literal key within buf[0,n). -1 = key absent.
135func rw_after(buf: *u8, n: i64, key: *u8) -> i64 {
  var kl: i64 = 0
  while key[kl] != (0 as u8) { kl = kl + 1 }
  var i: i64 = 0
  while i + kl <= n {
      var k: i64 = 0
      var ok: i64 = 1
      while k < kl { if buf[i + k] != key[k] { ok = 0; k = kl } else { k = k + 1 } }
      if ok == 1 {
          var j: i64 = i + kl
          var v: i64 = 0
          var any: i64 = 0
          var go: i64 = 1
          while go == 1 {
              go = 0
              if j < n {
                  let c: i64 = buf[j] as i64
                  if c >= 48 { if c <= 57 { v = v * 10 + (c - 48); any = 1; j = j + 1; go = 1 } }
              }
          }
          if any == 0 { return 0 - 1 }
          return v
      }
      i = i + 1
  }
  return 0 - 1
161}
162
163// does [s,e) of buf contain the NUL-terminated literal lit?
164func rw_has(buf: *u8, n: i64, lit: *u8) -> i64 {
  var ll: i64 = 0
  while lit[ll] != (0 as u8) { ll = ll + 1 }
  if ll == 0 { return 0 }
  var i: i64 = 0
  while i + ll <= n {
      var k: i64 = 0
      var ok: i64 = 1
      while k < ll { if buf[i + k] != lit[k] { ok = 0; k = ll } else { k = k + 1 } }
      if ok == 1 { return 1 }
      i = i + 1
  }
  return 0
177}
178
179// ---- PER-MEMBER HEALTH: the gauge the array-level numbers structurally cannot show -----------------
180// THE BLIND SPOT THIS CLOSES, and it is the one this organ own header describes: md2 reports
181// degraded=0 raid_disks=8 mismatch_cnt=0 array_state=clean -- PERFECT -- while a member disk is visibly
182// failing, because raid5 reconstructs every failed read from parity. So an array-level monitor reports
183// health during exactly the failure it exists to catch. mdraid exposes the missing number, root-free, as
184// a structured integer: /sys/block/<md>/md/dev-<name>/errors = read errors detected on THAT member and
185// corrected without evicting it.
186// MEASURED 2026-08-07 the moment this was read: sata6p5=808  sata8p5=56  every other member 0.
187// The kmsg investigation that opened this had named sata8 as the single failing device -- true of the
188// RING-BUFFER WINDOW it could see, but sata6 carries 14x the cumulative count and nothing had ever read
189// it. ★A RING BUFFER REPORTS THE RECENT, NOT THE WORST -- AND A CUMULATIVE COUNTER IS THE ONLY THING
190// THAT CAN NAME THE WORST.
191// ★AN ARRAY-LEVEL METRIC IS A CLAIM ABOUT REDUNDANCY, NOT ABOUT DISK HEALTH; RAID EXISTS PRECISELY TO
192// MAKE THE SECOND INVISIBLE IN THE FIRST.
193// FULL SCOPE, NO SAMPLING: every md, every member, EVERY nonzero member NAMED -- no cap, no "shown=N of
194// M". A capped list is a sample, and a sample must never be published as a population fact.
195// out[0]=members seen  out[1]=sum errors  out[2]=members with errors>0  out[3]=max errors
196func rw_member_scan(mdnum: i64, out: *i64) -> i64 {
  out[0] = 0
  out[1] = 0
  out[2] = 0
  out[3] = 0
  let dp: *u8 = sys_mmap(RW_PATH)
  var od: i64 = rw_cat(dp, 0, "/sys/block/md" as *u8)
  od = rw_catn(dp, od, mdnum)
  od = rw_cat(dp, od, "/md" as *u8)
  dp[od] = 0 as u8
  let fd: i64 = sys_openat_rd(dp)
  if fd < 0 { return 0 - 1 }
  let db: *u8 = sys_mmap(RW_DIRBUF)
  let ep: *u8 = sys_mmap(RW_PATH)
  var n: i64 = sys_getdents64(fd, db, RW_DIRBUF)
  while n > 0 {
      var q: i64 = 0
      while q < n {
          let rl: i64 = (db[q + RW_RECLEN_OFF] as i64) + ((db[q + RW_RECLEN_OFF + 1] as i64) * 256)
          if rl <= 0 { q = n }
          else {
              let nm: *u8 = ((db as i64) + q + RW_NAME_OFF) as *u8
              if rw_has(nm, 4, "dev-" as *u8) == 1 {
                  out[0] = out[0] + 1
                  var oe: i64 = rw_cat(ep, 0, dp)
                  oe = rw_cat(ep, oe, "/" as *u8)
                  oe = rw_cat(ep, oe, nm)
                  oe = rw_cat(ep, oe, "/errors" as *u8)
                  ep[oe] = 0 as u8
                  let ec: i64 = rw_sysint(ep)
                  if ec > 0 {
                      out[1] = out[1] + ec
                      out[2] = out[2] + 1
                      if ec > out[3] { out[3] = ec }
                      rw("    MEMBER " as *u8); rw(nm)
                      rw(" errors=" as *u8); rwn(ec)
                      rw(" (corrected reads -- this member is damaged; the ARRAY still reads healthy)" as *u8)
                      rw("\n" as *u8)
                  }
              }
              q = q + rl
          }
      }
      n = sys_getdents64(fd, db, RW_DIRBUF)
  }
  sys_close(fd)
  return 0
243}
244
245func main(argc: i64, argv: *i64) -> i64 {
  let p: *u8 = sys_mmap(RW_PATH)
  let lvl: *u8 = sys_mmap(RW_SMALL)
  var seen: i64 = 0
  var degraded: i64 = 0
  var mismatched: i64 = 0
  var memberbad: i64 = 0     // members with errors>0, across EVERY array
  var membersum: i64 = 0
  var membermax: i64 = 0
254
  rw("=== nx_raidwatch -- md array integrity + kernel IO errors (root-free) ===\n" as *u8)
256
  var m: i64 = 0
  while m < RW_MAXMD {
      var o: i64 = rw_cat(p, 0, "/sys/block/md" as *u8)
      o = rw_catn(p, o, m)
      o = rw_cat(p, o, "/md/raid_disks" as *u8)
      p[o] = 0 as u8
      let rd: i64 = rw_sysint(p)
      if rd > 0 {
          seen = seen + 1
          var o2: i64 = rw_cat(p, 0, "/sys/block/md" as *u8)
          o2 = rw_catn(p, o2, m)
          o2 = rw_cat(p, o2, "/md/degraded" as *u8)
          p[o2] = 0 as u8
          let dg: i64 = rw_sysint(p)
          var o3: i64 = rw_cat(p, 0, "/sys/block/md" as *u8)
          o3 = rw_catn(p, o3, m)
          o3 = rw_cat(p, o3, "/md/mismatch_cnt" as *u8)
          p[o3] = 0 as u8
          let mc: i64 = rw_sysint(p)
          var o4: i64 = rw_cat(p, 0, "/sys/block/md" as *u8)
          o4 = rw_catn(p, o4, m)
          o4 = rw_cat(p, o4, "/md/level" as *u8)
          p[o4] = 0 as u8
          lvl[0] = 0 as u8
          rw_sysstr(p, lvl, RW_SMALL)
282
          rw("  md" as *u8); rwn(m)
          rw(" level=" as *u8); rw(lvl)
          rw(" members=" as *u8); rwn(rd)
          rw(" degraded=" as *u8); rwn(dg)
          rw(" mismatch=" as *u8); rwn(mc)
          if dg > 0 {
              degraded = degraded + 1
              rw(" *** DEGRADED: " as *u8); rwn(dg)
              rw(" of " as *u8); rwn(rd)
              rw(" members missing -- NO REDUNDANCY, a second failure loses the array ***" as *u8)
          }
          if mc > 0 { mismatched = mismatched + 1; rw(" *** MISMATCH: parity disagrees with data ***" as *u8) }
          rw("\n" as *u8)
          // PER-MEMBER immediately AFTER the array line, so the contrast sits in one screen: the array
          // says clean, the member says 808. Reading them apart is how the fault stayed invisible.
          let mout: *i64 = sys_mmap(64) as *i64
          rw_member_scan(m, mout)
          if mout[2] > 0 {
              memberbad = memberbad + mout[2]
              membersum = membersum + mout[1]
              if mout[3] > membermax { membermax = mout[3] }
          }
      }
      m = m + 1
  }
308
  // ---- kernel IO errors. O_NONBLOCK so the last read returns EAGAIN instead of blocking. ----
  // COUNTED SEPARATELY, because one physical failure emits a PAIR of records: an ATA line carrying
  // { UNC } and a block layer line carrying I/O error. Summing them reported 168 for 84 real events
  // -- twice the apparent severity. Cross-checked against dmesg: 84 and 84, union 168, zero false
  // positives on the loose UNC substring (every match was the ATA form).
  // DISTINCT SECTORS is the discriminator that actually decides what to do: 84 failures on ONE
  // sector is a single bad block, recoverable and remappable; 84 failures across 84 sectors is a
  // disk coming apart. The same error count means opposite things.
  var ioerr: i64 = 0
  var unc: i64 = 0
  var recs: i64 = 0
  var nsect: i64 = 0
  var corrected: i64 = 0
  var slo: i64 = 0 - 1
  var shi: i64 = 0 - 1
  let sect: *i64 = sys_mmap(8 * RW_MAXSECT + 16) as *i64
  let kfd: i64 = __syscall(RW_SYS_OPENAT, RW_AT_FDCWD, "/dev/kmsg" as i64, RW_O_NONBLOCK, 0, 0, 0)
  if kfd >= 0 {
      let rb: *u8 = sys_mmap(RW_RECBUF)
      var go: i64 = 1
      while go == 1 {
          let r: i64 = sys_read(kfd, rb, RW_RECBUF - 1)
          if r <= 0 { go = 0 } else {
              recs = recs + 1
              rb[r] = 0 as u8
              var hit: i64 = 0
              if rw_has(rb, r, "I/O error" as *u8) == 1 { ioerr = ioerr + 1; hit = 1 }
              if rw_has(rb, r, "{ UNC }" as *u8) == 1 { unc = unc + 1; hit = 1 }
              if rw_has(rb, r, "medium error" as *u8) == 1 { hit = 1 }
              // "read error corrected" is the ARRAY COMPENSATING: raid5 rebuilt the block from
              // parity and the caller never saw a failure. It is the difference between "a disk is
              // damaged" and "data was lost", and it is why every array metric can read healthy
              // while a member is visibly failing. Counted, never treated as an error.
              if rw_has(rb, r, "read error corrected" as *u8) == 1 { corrected = corrected + 1 }
              if hit == 1 {
                  var s: i64 = rw_after(rb, r, "sector in range " as *u8)
                  if s < 0 { s = rw_after(rb, r, "sector " as *u8) }
                  if s >= 0 {
                      var q: i64 = 0
                      var known: i64 = 0
                      while q < nsect { if sect[q] == s { known = 1; q = nsect } else { q = q + 1 } }
                      if known == 0 { if nsect < RW_MAXSECT { sect[nsect] = s; nsect = nsect + 1 } }
                      if slo < 0 { slo = s; shi = s }
                      if s < slo { slo = s }
                      if s > shi { shi = s }
                  }
              }
              if recs >= RW_KMSG_RECS { go = 0 }
          }
      }
      sys_close(kfd)
      sys_munmap(rb, RW_RECBUF)
  }
362
  rw("  kernel: records_scanned=" as *u8); rwn(recs)
  rw(" io_error=" as *u8); rwn(ioerr)
  rw(" unc=" as *u8); rwn(unc)
  rw(" distinct_sectors=" as *u8); rwn(nsect)
  rw(" corrected_by_parity=" as *u8); rwn(corrected)
  if nsect > 1 {
      rw(" span=" as *u8); rwn(shi - slo)
      rw(" sectors" as *u8)
  }
  // Deliberately NOT calling a verdict on the disk from a sector count. Three sectors inside a
  // 400 KB span is a LOCALISED defect; the same count spread over a whole platter is a different
  // failure. Report the count AND the span and let the reader see which one this is -- an earlier
  // draft of this line said "SPREAD across sectors -- this is a failing disk" for any count above
  // one, which would have overstated a 400 KB region as a disintegrating drive.
  if nsect == 1 { rw(" (ONE bad block -- remappable)" as *u8) }
  if nsect > 1 { rw(" (MULTIPLE bad blocks -- a growing defect; judge by the span, not the count)" as *u8) }
  if kfd < 0 { rw(" (kmsg unreadable -- IO errors NOT checked, this is ignorance not health)" as *u8) }
  rw("\n" as *u8)
381
  if seen == 0 {
      rwe("NX-RAIDWATCH REFUSED: no md array was readable under /sys/block -- refusing to report health it never measured\n" as *u8)
      sys_exit(RW_EXIT_NODATA)
      return RW_EXIT_NODATA
  }
387
  // PERSIST BEFORE PRINTING. Every verdict branch below sys_exit()s, so a write placed after them
  // would run on exactly ONE path -- and the path it would miss is RED.
  let rvc: i64 = rw_verdict_code(degraded, ioerr, memberbad, mismatched)
  let sb: *u8 = sys_mmap(RW_STATBUF)
  var so: i64 = 0
  so = rw_cat(sb, so, "arrays " as *u8); so = rw_catn(sb, so, seen)
  so = rw_cat(sb, so, "\ndegraded " as *u8); so = rw_catn(sb, so, degraded)
  so = rw_cat(sb, so, "\nmismatched " as *u8); so = rw_catn(sb, so, mismatched)
  so = rw_cat(sb, so, "\nio_error " as *u8); so = rw_catn(sb, so, ioerr)
  so = rw_cat(sb, so, "\ndistinct_sectors " as *u8); so = rw_catn(sb, so, nsect)
  so = rw_cat(sb, so, "\nmember_errors_sum " as *u8); so = rw_catn(sb, so, membersum)
  so = rw_cat(sb, so, "\nmembers_damaged " as *u8); so = rw_catn(sb, so, memberbad)
  so = rw_cat(sb, so, "\nworst_member_errors " as *u8); so = rw_catn(sb, so, membermax)
  so = rw_cat(sb, so, "\nexit_code " as *u8); so = rw_catn(sb, so, rvc)
  // THE ENVELOPE TRAVELS WITH THE NUMBERS. io_error, distinct_sectors and the sector span are read
  // from /dev/kmsg, a RING BUFFER: two readings days apart are two WINDOWS, not two points on a
  // trend. A seat compared them across 25 days in this very session and had to retract the
  // progression it published. member_errors_* are CUMULATIVE since array start and ARE comparable.
  so = rw_cat(sb, so, "\nkmsg_fields_are_ring_buffer_not_trend 1" as *u8)
  if rvc == RW_EXIT_RED { so = rw_cat(sb, so, "\nverdict=RED\n" as *u8) } else {
  if rvc == RW_EXIT_AMBER { so = rw_cat(sb, so, "\nverdict=AMBER\n" as *u8) } else {
      so = rw_cat(sb, so, "\nverdict=GREEN\n" as *u8)
  } }
  let sfd: i64 = sys_openat_wr(RW_STATUS, MODE_0644)
  if sfd >= 0 { sys_write(sfd, sb, so); sys_close(sfd) } else {
      rwe("NX-RAIDWATCH: status write FAILED -- this beat measured the arrays and published nothing durable, which from outside is indistinguishable from never running\n" as *u8)
  }
  rw("NX-RAIDWATCH arrays=" as *u8); rwn(seen)
  rw(" degraded=" as *u8); rwn(degraded)
  rw(" mismatched=" as *u8); rwn(mismatched)
  rw(" io_error=" as *u8); rwn(ioerr)
  rw(" distinct_sectors=" as *u8); rwn(nsect)
  rw(" member_errors_sum=" as *u8); rwn(membersum)
  rw(" members_damaged=" as *u8); rwn(memberbad)
  rw(" worst_member_errors=" as *u8); rwn(membermax)
  if degraded > 0 {
      rw(" verdict=RED (an array is running without redundancy; a second member failure is data loss)\n" as *u8)
      sys_exit(RW_EXIT_RED)
      return RW_EXIT_RED
  }
  if ioerr > 0 {
      rw(" verdict=RED (kernel is reporting uncorrectable IO; a disk can be failing while every array still reads healthy)\n" as *u8)
      sys_exit(RW_EXIT_RED)
      return RW_EXIT_RED
  }
  // MEMBER DAMAGE OUTLIVES THE RING BUFFER. This is the branch that makes the monitor survive its own
  // evidence expiring: io_error above is read from /dev/kmsg, so once the buffer rolls past the errors it
  // reports 0 and the organ would print GREEN on a fleet with a visibly damaged disk. The per-member
  // counter is CUMULATIVE since array start, so it cannot roll over and cannot be out-waited.
  // MEASURED 2026-08-07: sata6p5=808 sata8p5=56 while md2 read degraded=0 mismatch=0 array_state=clean.
  // AMBER not RED, deliberately: raid5 corrected every one of these reads, so nothing is lost YET -- but
  // md3 is already degraded, so the estate has less margin than 8/8 suggests, and a damaged member is a
  // REPLACEMENT DECISION, not an outage. Naming it AMBER keeps RED meaning "redundancy is gone".
  // ★A MONITOR WHOSE EVIDENCE EXPIRES REPORTS RECOVERY WHEN IT SHOULD REPORT AMNESIA.
  if memberbad > 0 {
      rw(" verdict=AMBER (a member disk is DAMAGED: " as *u8); rwn(membersum)
      rw(" corrected read errors across " as *u8); rwn(memberbad)
      rw(" member(s), worst " as *u8); rwn(membermax)
      rw(" -- every array still reads healthy because parity is masking it; plan replacement)\n" as *u8)
      sys_exit(RW_EXIT_AMBER)
      return RW_EXIT_AMBER
  }
  if mismatched > 0 {
      rw(" verdict=AMBER (parity mismatch recorded -- schedule a scrub)\n" as *u8)
      sys_exit(RW_EXIT_AMBER)
      return RW_EXIT_AMBER
  }
  rw(" verdict=GREEN (every array full, no parity mismatch, no uncorrectable IO in the kernel buffer, and ZERO cumulative per-member read errors -- the last clause is what makes this GREEN survive a rolled kmsg)\n" as *u8)
  return 0
457}
nx_raidwatch.nx source

`nx_raidwatch.nx` source
