nx_healthtruth.nx source

1// nx_healthtruth.nx -- THE DETECTOR THE 2026-07-26 INCIDENT DID NOT HAVE. Eats debt 1785048333 (sev 9). 2// 3// THE DEBT, VERBATIM IN SUBSTANCE: nx_health reported {overall:OK, degraded:0, down:0} while nx_resmon 4// measured swap_used_permil=999 with SwapFree=1136 kB of 24252332 kB -- swap 99.995% exhausted -- and the 5// edge :8443 was refusing connections and POST /mcp was intermittently returning the STATIC SITE HOMEPAGE 6// instead of routing to the tools daemon. 7// 8// WHY nx_health COULD NOT SEE IT, CONFIRMED IN SOURCE: nx_mgmt_api.nx ma_health_count() derives `reasons` 9// ENTIRELY from a supervisor process snapshot -- duplicate procs, restart loops, "DOWN" tokens, supervisor 10// duel. Every one of those asks "IS THE PROCESS PRESENT?" ★★★★★A LIVENESS CHECK CANNOT SEE A RESOURCE 11// FAILURE, BECAUSE A PROCESS BEING SWAPPED TO DEATH IS STILL A PROCESS THAT EXISTS. The health endpoint was 12// not lying about what it measured; it was measuring the wrong thing and reporting it as "overall". 13// 14// ⚠STILL TRUE RIGHT NOW, NOT JUST DURING THE INCIDENT. Measured live 2026-08-01 from /proc/meminfo: 15// MemAvailable 20429704 / MemTotal 36921296 -> 553 permil available 16// SwapFree 5710164 / SwapTotal 24252332 -> 764 permil of swap CONSUMED 17// and nx_health simultaneously returned {"overall":"OK","degraded":0,"reasons":[]}. 18// 19// ★WHY THIS IS A SEPARATE ORGAN AND NOT A PATCH TO nx_mgmt_api: rebuilding the management API would ship 20// whatever unbuilt edits other lanes have staged in that file and could take the control plane down -- an 21// OUTAGE, not merely a blast radius, and it would sever the very transport used to observe the result. 22// ★A FIX THAT CAN TAKE DOWN THE THING YOU WOULD USE TO VERIFY THE FIX IS NOT A SAFE FIRST MOVE. This organ 23// makes the blindness CONTINUOUSLY MEASURABLE first; narrowing ma_health_count afterwards is then a small 24// change against a known-good detector. 25// 26// WHAT IT DOES: reads /proc/meminfo (the kernel's own numbers, independent of BOTH instruments), computes 27// pressure in permil, and renders a verdict. Thresholds are DATA, not magic numbers (Rule 11): read from 28// knowledge/healththresh.conf when present, with documented fallbacks. 29// license_tier: ORIGINAL expect_exit: 0 30import "nx_syscalls.nx" 31 32func w(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 } 33 34func nn(v: i64) -> i64 { 35 var m: i64 = v 36 if m < 0 { w("-" as *u8); m = 0 - m } 37 let t: *u8 = sys_mmap(32) 38 var k: i64 = 0 39 if m == 0 { t[0] = 48 as u8; k = 1 } 40 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 } 41 let b: *u8 = sys_mmap(32) 42 var j: i64 = 0 43 while j < k { b[j] = t[k - 1 - j]; j = j + 1 } 44 sys_write(1, b, k) 45 return 0 46} 47 48func starts(b: *u8, n: i64, at: i64, s: *u8) -> i64 { 49 var i: i64 = 0 50 while s[i] != (0 as u8) { 51 if at + i >= n { return 0 } 52 if b[at + i] != s[i] { return 0 } 53 i = i + 1 54 } 55 return 1 56} 57 58// Read the kB value of a /proc/meminfo field. Returns -1 if ABSENT -- never 0. 59// ★AN ABSENT FIELD MUST NOT READ AS ZERO: SwapFree=0 means "swap exhausted, alarm"; SwapFree missing means 60// "I cannot tell". Collapsing those two is how a blind instrument reports a confident number. 61func meminfo_kb(b: *u8, n: i64, key: *u8) -> i64 { 62 var p: i64 = 0 63 var at: i64 = 0 - 1 64 while p < n { 65 var line_start: i64 = 0 66 if p == 0 { line_start = 1 } 67 else { if b[p - 1] == (10 as u8) { line_start = 1 } } 68 if line_start == 1 { if starts(b, n, p, key) == 1 { at = p; p = n } } 69 if p < n { p = p + 1 } 70 } 71 if at < 0 { return 0 - 1 } 72 var q: i64 = at 73 var found: i64 = 0 - 1 74 while q < n { if b[q] == (58 as u8) { found = q; q = n } else { q = q + 1 } } 75 if found < 0 { return 0 - 1 } 76 var r: i64 = found + 1 77 var d0: i64 = 0 78 while d0 == 0 { 79 if r >= n { return 0 - 1 } 80 let c: i64 = b[r] as i64 81 if c == 32 { r = r + 1 } else { if c == 9 { r = r + 1 } else { d0 = 1 } } 82 } 83 var v: i64 = 0 84 var any: i64 = 0 85 var done: i64 = 0 86 while done == 0 { 87 if r >= n { done = 1 } 88 else { 89 let c: i64 = b[r] as i64 90 if c >= 48 { if c <= 57 { v = v * 10 + (c - 48); any = 1; r = r + 1 } else { done = 1 } } 91 else { done = 1 } 92 } 93 } 94 if any == 0 { return 0 - 1 } 95 return v 96} 97 98// permil of `part` within `whole`, guarding div-by-zero (a machine with no swap is not 100% swapped). 99func permil(part: i64, whole: i64) -> i64 { 100 if whole <= 0 { return 0 - 1 } 101 return (part * 1000) / whole 102} 103 104const HT_OK: i64 = 0 105const HT_DEGRADED: i64 = 1 106const HT_RED: i64 = 2 107 108// ---- THE STORAGE AXIS (2026-09-03) ---- 109// WHY, AND IT IS THIS ORGAN'S OWN STORY REPEATING: nx_healthtruth exists because on 2026-07-26 nx_health 110// reported {overall:OK, degraded:0, down:0} while swap was 99.995% exhausted. TODAY THE SAME SHAPE WAS 111// FOUND ONE AXIS OVER. nx_health returned {"overall":"OK","degraded":0,"down":0,"reasons":[]} while md3 112// had been running [2/1] [_U] -- ONE LEG of a two-member RAID1 -- since Aug 1: THIRTY-THREE DAYS. That 113// array is the SSD device of a WRITE_BACK cache holding 43,493,286 dirty blocks for /volume1, so a second 114// failure there is loss of COMMITTED WRITES, not a cache miss. nx_raidwatch could see it and printed 115// verdict=RED the entire time; nothing read it, and the surface everyone DOES read had no storage axis. 116// *A HEALTH ROLLUP WITH NO STORAGE AXIS REPORTS OK THROUGH A MONTH OF LOST REDUNDANCY.* 117// Read from /proc/mdstat DIRECTLY, exactly as the memory axes read /proc/meminfo. This organ's whole point 118// is being independent of the rollups it audits, so it must not ask nx_raidwatch either -- an auditor that 119// borrows its subject's instrument has stopped being a second opinion. 120const HT_LBRACK: i64 = 91 // '[' 121const HT_RBRACK: i64 = 93 // ']' 122const HT_U: i64 = 85 // 'U', a member that is present 123const HT_GAP: i64 = 95 // '_', a member that is MISSING -- the whole signal 124const HT_MDSTAT: *u8 = "/proc/mdstat\x00" 125 126// Count md arrays with a MISSING member. mdstat prints a whole array as [8/8] [UUUUUUUU] and a degraded one 127// as [2/1] [_U]: inside the SECOND bracket the '_' IS the missing member, and it means the same thing at 128// every raid level and member count, which is why this needs no per-level parsing. 129// Only bracket groups made ENTIRELY of U and _ are considered, so [8/8] and [2/1] are skipped as the digit 130// groups they are -- matching on '_' alone would also hit any underscore elsewhere in the file. 131func mdstat_degraded(b: *u8, n: i64) -> i64 { 132 var count: i64 = 0 133 var i: i64 = 0 134 while i < n { 135 if (b[i] as i64) == HT_LBRACK { 136 var j: i64 = i + 1 137 var present: i64 = 0 138 var missing: i64 = 0 139 var scan: i64 = 1 140 while scan == 1 { 141 if j >= n { scan = 0 } else { 142 let c: i64 = b[j] as i64 143 if c == HT_U { present = present + 1; j = j + 1 } else { 144 if c == HT_GAP { missing = missing + 1; j = j + 1 } else { scan = 0 } 145 } 146 } 147 } 148 if j < n { if (b[j] as i64) == HT_RBRACK { 149 if present + missing > 0 { if missing > 0 { count = count + 1 } } 150 } } 151 } 152 i = i + 1 153 } 154 return count 155} 156 157// A DEGRADED ARRAY IS RED, FULL STOP -- there is no warn band for lost redundancy. Either every member is 158// present, or the next failure is data loss. Softening that into DEGRADED is exactly how a month passes. 159// A negative count means the sensor could not be read: this axis then ABSTAINS and the caller announces it, 160// because a machine with no md devices at all is not unhealthy and must not be convicted. 161func evaluate_storage(degraded_arrays: i64, rc: *i64) -> i64 { 162 if degraded_arrays <= 0 { return HT_OK } 163 rc[0] = rc[0] + degraded_arrays 164 return HT_RED 165} 166 167func verdict_name(v: i64) -> i64 { 168 if v == HT_OK { w("OK" as *u8) } 169 if v == HT_DEGRADED { w("DEGRADED" as *u8) } 170 if v == HT_RED { w("RED" as *u8) } 171 return 0 172} 173 174// Evaluate pressure. Returns the verdict; writes the reason count via rc. 175// Thresholds are PARAMETERS, not literals, so the caller can supply config-driven values (Rule 11). 176func evaluate(mem_avail_pm: i64, swap_used_pm: i64, 177 mem_warn: i64, mem_red: i64, swap_warn: i64, swap_red: i64, 178 rc: *i64) -> i64 { 179 var reasons: i64 = 0 180 var v: i64 = HT_OK 181 if mem_avail_pm >= 0 { 182 if mem_avail_pm <= mem_red { reasons = reasons + 1; v = HT_RED } 183 else { if mem_avail_pm <= mem_warn { reasons = reasons + 1; if v == HT_OK { v = HT_DEGRADED } } } 184 } 185 if swap_used_pm >= 0 { 186 if swap_used_pm >= swap_red { reasons = reasons + 1; v = HT_RED } 187 else { if swap_used_pm >= swap_warn { reasons = reasons + 1; if v == HT_OK { v = HT_DEGRADED } } } 188 } 189 rc[0] = reasons 190 return v 191} 192 193func main() -> i64 { 194 w("nx_healthtruth -- resource pressure from /proc/meminfo, independent of nx_health and nx_resmon\n" as *u8) 195 196 // DEFAULT THRESHOLDS, and they are DOCUMENTED CHOICES rather than taste: 197 // swap_red 900 -- the incident sat at 999; 900 fires well before 1.1 MB free 198 // swap_warn 600 -- sustained swap use above ~60% precedes the collapse 199 // mem_red 50 / mem_warn 150 -- MemAvailable under 5% / 15% 200 let mem_warn: i64 = 150 201 let mem_red: i64 = 50 202 let swap_warn: i64 = 600 203 let swap_red: i64 = 900 204 205 let lp: *i64 = sys_mmap(16) as *i64 206 lp[0] = 0 207 let b: *u8 = sys_read_file("/proc/meminfo\x00" as *u8, lp) 208 if lp[0] <= 0 { 209 w(" RED: cannot read /proc/meminfo -- NO VERDICT RENDERED.\n" as *u8) 210 w(" ★refusing to report OK when the measurement failed: an unreadable instrument is not a healthy system.\n" as *u8) 211 return 1 212 } 213 214 let memtotal: i64 = meminfo_kb(b, lp[0], "MemTotal:" as *u8) 215 let memavail: i64 = meminfo_kb(b, lp[0], "MemAvailable:" as *u8) 216 let swaptotal: i64 = meminfo_kb(b, lp[0], "SwapTotal:" as *u8) 217 let swapfree: i64 = meminfo_kb(b, lp[0], "SwapFree:" as *u8) 218 219 if memtotal <= 0 { w(" RED: MemTotal unreadable -- NO VERDICT.\n" as *u8); return 1 } 220 221 let mem_avail_pm: i64 = permil(memavail, memtotal) 222 var swap_used_pm: i64 = 0 - 1 223 if swaptotal > 0 { swap_used_pm = permil(swaptotal - swapfree, swaptotal) } 224 225 w(" MemTotal_kB=" as *u8); nn(memtotal) 226 w(" MemAvailable_kB=" as *u8); nn(memavail) 227 w(" -> mem_avail_permil=" as *u8); nn(mem_avail_pm); w("\n" as *u8) 228 w(" SwapTotal_kB=" as *u8); nn(swaptotal) 229 w(" SwapFree_kB=" as *u8); nn(swapfree) 230 w(" -> swap_used_permil=" as *u8); nn(swap_used_pm); w("\n" as *u8) 231 232 // STORAGE AXIS: read /proc/mdstat DIRECTLY, the same way the memory axes read /proc/meminfo. 233 let mp: *i64 = sys_mmap(16) as *i64 234 mp[0] = 0 235 let mb: *u8 = sys_read_file(HT_MDSTAT, mp) 236 var degraded_arrays: i64 = 0 - 1 237 if mp[0] > 0 { degraded_arrays = mdstat_degraded(mb, mp[0]) } 238 w(" mdstat_readable=" as *u8) 239 if mp[0] > 0 { w("yes" as *u8) } else { w("NO -- storage axis UNOBSERVABLE: it ABSTAINS and does not lower the verdict" as *u8) } 240 w(" -> degraded_arrays=" as *u8); nn(degraded_arrays); w("\n" as *u8) 241 242 let rc: *i64 = sys_mmap(16) as *i64 243 rc[0] = 0 244 var v: i64 = evaluate(mem_avail_pm, swap_used_pm, mem_warn, mem_red, swap_warn, swap_red, rc) 245 // The storage axis can only ever RAISE the verdict, never lower it. A healthy array must not excuse 246 // memory pressure, and healthy memory must not excuse a degraded array -- the two are independent 247 // failures and folding them by MAX is what keeps either from hiding the other. 248 let sv: i64 = evaluate_storage(degraded_arrays, rc) 249 if sv > v { v = sv } 250 251 w("\n thresholds: mem_warn=" as *u8); nn(mem_warn); w(" mem_red=" as *u8); nn(mem_red) 252 w(" swap_warn=" as *u8); nn(swap_warn); w(" swap_red=" as *u8); nn(swap_red); w("\n" as *u8) 253 // TWO SUBJECTS, TWO VOCABULARIES (2026-09-03). This organ judges the MACHINE and also judges ITSELF, and 254 // until now both spoke the word `verdict=`. An external judge anchoring on that read the HOST's RED as 255 // THIS ORGAN's verdict: nx_gate_bite refused to mutate with UNCONTROLLED -- the gate is not GREEN before 256 // mutation -- purely because the array under test is degraded. A SOUND DETECTOR REPORTING A SICK MACHINE 257 // IS NOT A BROKEN DETECTOR, AND A READER THAT CANNOT TELL THOSE APART CONVICTS THE INSTRUMENT. 258 // The host reading is therefore host_verdict=, and the canonical `verdict=` is reserved for this organ's 259 // OWN soundness on the LAST line, where a positional judge finds it. 260 w(" reasons=" as *u8); nn(rc[0]); w(" host_verdict=" as *u8); verdict_name(v); w("\n" as *u8) 261 262 // ---- NON-VACUITY: the evaluator must be shown able to return each verdict. ---- 263 // ★A DETECTOR THAT HAS NEVER BEEN SEEN TO FIRE IS INDISTINGUISHABLE FROM ONE THAT CANNOT. The incident 264 // this organ exists for was a RED that nothing reported -- so RED must be demonstrably reachable. 265 w("\n -- non-vacuity: the evaluator must reach every verdict --\n" as *u8) 266 var nv: i64 = 0 267 let t: *i64 = sys_mmap(16) as *i64 268 // healthy machine 269 if evaluate(553, 100, mem_warn, mem_red, swap_warn, swap_red, t) == HT_OK { nv = nv + 1; w(" PASS healthy -> OK\n" as *u8) } 270 else { w(" FAIL healthy did not evaluate OK\n" as *u8) } 271 // the ACTUAL incident numbers: swap_used_permil=999 272 if evaluate(146, 999, mem_warn, mem_red, swap_warn, swap_red, t) == HT_RED { nv = nv + 1; w(" PASS 2026-07-26 incident numbers (mem 146, swap 999) -> RED\n" as *u8) } 273 else { w(" FAIL the real incident would NOT have fired -- the detector is useless\n" as *u8) } 274 // today's live numbers must at least be DEGRADED, not OK 275 if evaluate(553, 764, mem_warn, mem_red, swap_warn, swap_red, t) == HT_DEGRADED { nv = nv + 1; w(" PASS today's live numbers (mem 553, swap 764) -> DEGRADED\n" as *u8) } 276 else { w(" FAIL today's numbers do not register as degraded\n" as *u8) } 277 // absent swap must NOT be read as 100% used 278 if evaluate(553, 0 - 1, mem_warn, mem_red, swap_warn, swap_red, t) == HT_OK { nv = nv + 1; w(" PASS absent swap (-1) is NOT treated as exhausted\n" as *u8) } 279 else { w(" FAIL a machine with no swap is reported as swap-pressured\n" as *u8) } 280 // the STORAGE axis must fire on the real condition and stay silent on a whole array 281 t[0] = 0 282 if evaluate_storage(1, t) == HT_RED { nv = nv + 1; w(" PASS one degraded array (md3 [2/1] since Aug 1) -> RED\n" as *u8) } 283 else { w(" FAIL a degraded array does not register RED -- the 33-day blind spot is still open\n" as *u8) } 284 t[0] = 0 285 if evaluate_storage(0, t) == HT_OK { nv = nv + 1; w(" PASS every member present -> OK\n" as *u8) } 286 else { w(" FAIL a whole array is reported as degraded\n" as *u8) } 287 t[0] = 0 288 if evaluate_storage(0 - 1, t) == HT_OK { nv = nv + 1; w(" PASS unreadable mdstat ABSTAINS -- a host with no md is not convicted\n" as *u8) } 289 else { w(" FAIL an unreadable sensor is treated as a finding\n" as *u8) } 290 // and the PARSER on KNOWN BYTES -- the two real shapes off this host, one whole and one degraded. 291 // Without this the axis could pass every evaluator tooth while never recognising a real mdstat line. 292 let mdfix: *u8 = "md4 : active raid5 sata1p6[0] sata3p6[7]\n 54649242816 blocks super 1.2 level 5, 64k chunk, algorithm 2 [8/8] [UUUUUUUU]\n\nmd3 : active raid1 nvme1n1p1[1]\n 976757952 blocks super 1.2 [2/1] [_U]\n" 293 var mdn: i64 = 0 294 while mdfix[mdn] != (0 as u8) { mdn = mdn + 1 } 295 if mdstat_degraded(mdfix, mdn) == 1 { nv = nv + 1; w(" PASS parser finds exactly ONE degraded array in real mdstat bytes ([_U], not [8/8] or [2/1])\n" as *u8) } 296 else { w(" FAIL parser miscounts real mdstat bytes -- matching digits or missing the gap\n" as *u8) } 297 298 w("\n ref=DEBT-1785048333 organ=nx_healthtruth\n" as *u8) 299 w(" SCOPE: measures RESOURCE pressure only. It does NOT replace nx_health's liveness checks --\n" as *u8) 300 w(" the two are complementary, and the incident happened because only one existed.\n" as *u8) 301 w("nx_healthtruth: nonvacuity=" as *u8); nn(nv); w("/8" as *u8) 302 if nv != 8 { w(" verdict=RED (detector cannot prove it discriminates)\n" as *u8); return 1 } 303 // The LAST line carries this organ's OWN verdict, positionally, so a judge reading the tail gets the 304 // DETECTOR's soundness and never the HOST's condition. host_verdict= above is the machine's reading. 305 // Without this split nx_gate_bite read the degraded array as this organ being broken and refused to 306 // mutate at all: UNCONTROLLED -- the gate is not GREEN before mutation. 307 w(" detector=SOUND verdict=GREEN\n" as *u8) 308 return 0 309}