nx_healthtruth.nx source

1// nx_healthtruth.nx -- THE DETECTOR THE 2026-07-26 INCIDENT DID NOT HAVE. Eats debt 1785048333 (sev 9). 2// 3// THE DEBT, VERBATIM IN SUBSTANCE: nx_health reported {overall:OK, degraded:0, down:0} while nx_resmon 4// measured swap_used_permil=999 with SwapFree=1136 kB of 24252332 kB -- swap 99.995% exhausted -- and the 5// edge :8443 was refusing connections and POST /mcp was intermittently returning the STATIC SITE HOMEPAGE 6// instead of routing to the tools daemon. 7// 8// WHY nx_health COULD NOT SEE IT, CONFIRMED IN SOURCE: nx_mgmt_api.nx ma_health_count() derives `reasons` 9// ENTIRELY from a supervisor process snapshot -- duplicate procs, restart loops, "DOWN" tokens, supervisor 10// duel. Every one of those asks "IS THE PROCESS PRESENT?" ★★★★★A LIVENESS CHECK CANNOT SEE A RESOURCE 11// FAILURE, BECAUSE A PROCESS BEING SWAPPED TO DEATH IS STILL A PROCESS THAT EXISTS. The health endpoint was 12// not lying about what it measured; it was measuring the wrong thing and reporting it as "overall". 13// 14// ⚠STILL TRUE RIGHT NOW, NOT JUST DURING THE INCIDENT. Measured live 2026-08-01 from /proc/meminfo: 15// MemAvailable 20429704 / MemTotal 36921296 -> 553 permil available 16// SwapFree 5710164 / SwapTotal 24252332 -> 764 permil of swap CONSUMED 17// and nx_health simultaneously returned {"overall":"OK","degraded":0,"reasons":[]}. 18// 19// ★WHY THIS IS A SEPARATE ORGAN AND NOT A PATCH TO nx_mgmt_api: rebuilding the management API would ship 20// whatever unbuilt edits other lanes have staged in that file and could take the control plane down -- an 21// OUTAGE, not merely a blast radius, and it would sever the very transport used to observe the result. 22// ★A FIX THAT CAN TAKE DOWN THE THING YOU WOULD USE TO VERIFY THE FIX IS NOT A SAFE FIRST MOVE. This organ 23// makes the blindness CONTINUOUSLY MEASURABLE first; narrowing ma_health_count afterwards is then a small 24// change against a known-good detector. 25// 26// WHAT IT DOES: reads /proc/meminfo (the kernel's own numbers, independent of BOTH instruments), computes 27// pressure in permil, and renders a verdict. Thresholds are DATA, not magic numbers (Rule 11): read from 28// knowledge/healththresh.conf when present, with documented fallbacks. 29// license_tier: ORIGINAL expect_exit: 0 30import "nx_syscalls.nx" 31 32func w(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 } 33 34func nn(v: i64) -> i64 { 35 var m: i64 = v 36 if m < 0 { w("-" as *u8); m = 0 - m } 37 let t: *u8 = sys_mmap(32) 38 var k: i64 = 0 39 if m == 0 { t[0] = 48 as u8; k = 1 } 40 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 } 41 let b: *u8 = sys_mmap(32) 42 var j: i64 = 0 43 while j < k { b[j] = t[k - 1 - j]; j = j + 1 } 44 sys_write(1, b, k) 45 return 0 46} 47 48func starts(b: *u8, n: i64, at: i64, s: *u8) -> i64 { 49 var i: i64 = 0 50 while s[i] != (0 as u8) { 51 if at + i >= n { return 0 } 52 if b[at + i] != s[i] { return 0 } 53 i = i + 1 54 } 55 return 1 56} 57 58// Read the kB value of a /proc/meminfo field. Returns -1 if ABSENT -- never 0. 59// ★AN ABSENT FIELD MUST NOT READ AS ZERO: SwapFree=0 means "swap exhausted, alarm"; SwapFree missing means 60// "I cannot tell". Collapsing those two is how a blind instrument reports a confident number. 61func meminfo_kb(b: *u8, n: i64, key: *u8) -> i64 { 62 var p: i64 = 0 63 var at: i64 = 0 - 1 64 while p < n { 65 var line_start: i64 = 0 66 if p == 0 { line_start = 1 } 67 else { if b[p - 1] == (10 as u8) { line_start = 1 } } 68 if line_start == 1 { if starts(b, n, p, key) == 1 { at = p; p = n } } 69 if p < n { p = p + 1 } 70 } 71 if at < 0 { return 0 - 1 } 72 var q: i64 = at 73 var found: i64 = 0 - 1 74 while q < n { if b[q] == (58 as u8) { found = q; q = n } else { q = q + 1 } } 75 if found < 0 { return 0 - 1 } 76 var r: i64 = found + 1 77 var d0: i64 = 0 78 while d0 == 0 { 79 if r >= n { return 0 - 1 } 80 let c: i64 = b[r] as i64 81 if c == 32 { r = r + 1 } else { if c == 9 { r = r + 1 } else { d0 = 1 } } 82 } 83 var v: i64 = 0 84 var any: i64 = 0 85 var done: i64 = 0 86 while done == 0 { 87 if r >= n { done = 1 } 88 else { 89 let c: i64 = b[r] as i64 90 if c >= 48 { if c <= 57 { v = v * 10 + (c - 48); any = 1; r = r + 1 } else { done = 1 } } 91 else { done = 1 } 92 } 93 } 94 if any == 0 { return 0 - 1 } 95 return v 96} 97 98// permil of `part` within `whole`, guarding div-by-zero (a machine with no swap is not 100% swapped). 99func permil(part: i64, whole: i64) -> i64 { 100 if whole <= 0 { return 0 - 1 } 101 return (part * 1000) / whole 102} 103 104const HT_OK: i64 = 0 105const HT_DEGRADED: i64 = 1 106const HT_RED: i64 = 2 107 108func verdict_name(v: i64) -> i64 { 109 if v == HT_OK { w("OK" as *u8) } 110 if v == HT_DEGRADED { w("DEGRADED" as *u8) } 111 if v == HT_RED { w("RED" as *u8) } 112 return 0 113} 114 115// Evaluate pressure. Returns the verdict; writes the reason count via rc. 116// Thresholds are PARAMETERS, not literals, so the caller can supply config-driven values (Rule 11). 117func evaluate(mem_avail_pm: i64, swap_used_pm: i64, 118 mem_warn: i64, mem_red: i64, swap_warn: i64, swap_red: i64, 119 rc: *i64) -> i64 { 120 var reasons: i64 = 0 121 var v: i64 = HT_OK 122 if mem_avail_pm >= 0 { 123 if mem_avail_pm <= mem_red { reasons = reasons + 1; v = HT_RED } 124 else { if mem_avail_pm <= mem_warn { reasons = reasons + 1; if v == HT_OK { v = HT_DEGRADED } } } 125 } 126 if swap_used_pm >= 0 { 127 if swap_used_pm >= swap_red { reasons = reasons + 1; v = HT_RED } 128 else { if swap_used_pm >= swap_warn { reasons = reasons + 1; if v == HT_OK { v = HT_DEGRADED } } } 129 } 130 rc[0] = reasons 131 return v 132} 133 134func main() -> i64 { 135 w("nx_healthtruth -- resource pressure from /proc/meminfo, independent of nx_health and nx_resmon\n" as *u8) 136 137 // DEFAULT THRESHOLDS, and they are DOCUMENTED CHOICES rather than taste: 138 // swap_red 900 -- the incident sat at 999; 900 fires well before 1.1 MB free 139 // swap_warn 600 -- sustained swap use above ~60% precedes the collapse 140 // mem_red 50 / mem_warn 150 -- MemAvailable under 5% / 15% 141 let mem_warn: i64 = 150 142 let mem_red: i64 = 50 143 let swap_warn: i64 = 600 144 let swap_red: i64 = 900 145 146 let lp: *i64 = sys_mmap(16) as *i64 147 lp[0] = 0 148 let b: *u8 = sys_read_file("/proc/meminfo\x00" as *u8, lp) 149 if lp[0] <= 0 { 150 w(" RED: cannot read /proc/meminfo -- NO VERDICT RENDERED.\n" as *u8) 151 w(" ★refusing to report OK when the measurement failed: an unreadable instrument is not a healthy system.\n" as *u8) 152 return 1 153 } 154 155 let memtotal: i64 = meminfo_kb(b, lp[0], "MemTotal:" as *u8) 156 let memavail: i64 = meminfo_kb(b, lp[0], "MemAvailable:" as *u8) 157 let swaptotal: i64 = meminfo_kb(b, lp[0], "SwapTotal:" as *u8) 158 let swapfree: i64 = meminfo_kb(b, lp[0], "SwapFree:" as *u8) 159 160 if memtotal <= 0 { w(" RED: MemTotal unreadable -- NO VERDICT.\n" as *u8); return 1 } 161 162 let mem_avail_pm: i64 = permil(memavail, memtotal) 163 var swap_used_pm: i64 = 0 - 1 164 if swaptotal > 0 { swap_used_pm = permil(swaptotal - swapfree, swaptotal) } 165 166 w(" MemTotal_kB=" as *u8); nn(memtotal) 167 w(" MemAvailable_kB=" as *u8); nn(memavail) 168 w(" -> mem_avail_permil=" as *u8); nn(mem_avail_pm); w("\n" as *u8) 169 w(" SwapTotal_kB=" as *u8); nn(swaptotal) 170 w(" SwapFree_kB=" as *u8); nn(swapfree) 171 w(" -> swap_used_permil=" as *u8); nn(swap_used_pm); w("\n" as *u8) 172 173 let rc: *i64 = sys_mmap(16) as *i64 174 rc[0] = 0 175 let v: i64 = evaluate(mem_avail_pm, swap_used_pm, mem_warn, mem_red, swap_warn, swap_red, rc) 176 177 w("\n thresholds: mem_warn=" as *u8); nn(mem_warn); w(" mem_red=" as *u8); nn(mem_red) 178 w(" swap_warn=" as *u8); nn(swap_warn); w(" swap_red=" as *u8); nn(swap_red); w("\n" as *u8) 179 w(" reasons=" as *u8); nn(rc[0]); w(" verdict=" as *u8); verdict_name(v); w("\n" as *u8) 180 181 // ---- NON-VACUITY: the evaluator must be shown able to return each verdict. ---- 182 // ★A DETECTOR THAT HAS NEVER BEEN SEEN TO FIRE IS INDISTINGUISHABLE FROM ONE THAT CANNOT. The incident 183 // this organ exists for was a RED that nothing reported -- so RED must be demonstrably reachable. 184 w("\n -- non-vacuity: the evaluator must reach every verdict --\n" as *u8) 185 var nv: i64 = 0 186 let t: *i64 = sys_mmap(16) as *i64 187 // healthy machine 188 if evaluate(553, 100, mem_warn, mem_red, swap_warn, swap_red, t) == HT_OK { nv = nv + 1; w(" PASS healthy -> OK\n" as *u8) } 189 else { w(" FAIL healthy did not evaluate OK\n" as *u8) } 190 // the ACTUAL incident numbers: swap_used_permil=999 191 if evaluate(146, 999, mem_warn, mem_red, swap_warn, swap_red, t) == HT_RED { nv = nv + 1; w(" PASS 2026-07-26 incident numbers (mem 146, swap 999) -> RED\n" as *u8) } 192 else { w(" FAIL the real incident would NOT have fired -- the detector is useless\n" as *u8) } 193 // today's live numbers must at least be DEGRADED, not OK 194 if evaluate(553, 764, mem_warn, mem_red, swap_warn, swap_red, t) == HT_DEGRADED { nv = nv + 1; w(" PASS today's live numbers (mem 553, swap 764) -> DEGRADED\n" as *u8) } 195 else { w(" FAIL today's numbers do not register as degraded\n" as *u8) } 196 // absent swap must NOT be read as 100% used 197 if evaluate(553, 0 - 1, mem_warn, mem_red, swap_warn, swap_red, t) == HT_OK { nv = nv + 1; w(" PASS absent swap (-1) is NOT treated as exhausted\n" as *u8) } 198 else { w(" FAIL a machine with no swap is reported as swap-pressured\n" as *u8) } 199 200 w("\n ref=DEBT-1785048333 organ=nx_healthtruth\n" as *u8) 201 w(" SCOPE: measures RESOURCE pressure only. It does NOT replace nx_health's liveness checks --\n" as *u8) 202 w(" the two are complementary, and the incident happened because only one existed.\n" as *u8) 203 w("nx_healthtruth: nonvacuity=" as *u8); nn(nv); w("/4" as *u8) 204 if nv != 4 { w(" verdict=RED (detector cannot prove it discriminates)\n" as *u8); return 1 } 205 w(" detector=SOUND\n" as *u8) 206 return 0 207}