nx_procchurn.nx source

1// nx_procchurn.nx -- the PROCESS-CHURN axis the health plane never had (debt seq1317 / seq1318). 2// 3// WHY THIS EXISTS: 2026-07-30, nx_health said {overall:OK, degraded:0, down:0} with all 14 services UP 4// while the NAS was forking 78.3 processes/sec sustained, burning 231 permil of an 8-core box in the 5// KERNEL against 98 permil in userspace (a 2.36x kernel:user inversion) and taking 91718 context 6// switches/sec. nx_resmon owns MEMORY, nx_netobs owns NETWORK -- the process-churn axis had NO organ, so 7// the largest CPU consumer on the platform was invisible to every instrument we owned and was found by 8// hand. This is that instrument. 9// LAW: liveness is not performance. A board that only asks "is it up" cannot see the machine burning. 10// 11// WHAT IT MEASURES (two /proc/stat samples bracketing a real window -- deltas, never since-boot totals, 12// because a since-boot average hides a storm that started an hour ago): 13// forks/sec -- `processes` delta. THE headline: fork+exec is the most expensive thing a kernel does. 14// ctxsw/sec -- `ctxt` delta. Rises with churn AND with thrash; corroborates, never diagnoses alone. 15// intr/sec -- `intr` delta, first field. Device pressure, reported for context (not thresholded). 16// kernel:user -- system:user+nice jiffies as permil. 1000 = parity. THE inversion detector. 17// busy/idle -- share of measured capacity, self-normalising (no HZ and no core count needed -- 18// every share is a fraction of the SAME jiffy total, so the arithmetic cannot drift 19// when the box changes size. That is what makes this instrument hardware-portable). 20// 21// Policy lives in nx_procchurn_lib.nx so the gate exercises the SAME code (rule 9 + rule 15); thresholds 22// are data-driven from knowledge/status/procchurn.conf (rule 11). Read-only. No hw writes (Rule 26). 23// Exit code IS the verdict so a caller can gate on it: 0=GREEN 1=AMBER 2=RED 3=UNMEASURED. 24// license_tier: ORIGINAL expect_exit: 0 25import "nx_procchurn_lib.nx" 26 27const PC_STATBUF: i64 = 8192 28const PC_CONFBUF: i64 = 4096 29const PC_FIELD_BYTES: i64 = 256 30const PC_NCPU_FIELDS: i64 = 10 31const PC_DEF_WINDOW_MS: i64 = 3000 32const PC_MIN_WINDOW_MS: i64 = 1000 33const PC_MAX_WINDOW_MS: i64 = 10000 34const PC_US_PER_MS: i64 = 1000 35const PC_DEF_F_AMBER: i64 = 20 36const PC_DEF_F_RED: i64 = 50 37const PC_DEF_R_AMBER: i64 = 1000 38const PC_DEF_R_RED: i64 = 1500 39const PC_DEF_C_AMBER: i64 = 20000 40const PC_DEF_C_RED: i64 = 50000 41const PC_EXIT_UNMEASURED: i64 = 3 42const PC_JBUF: i64 = 512 43const PC_MODE_644: i64 = 420 44 45// APPEND one frame to the trend journal. Append-only and never truncating: history is sacred (rule 13), 46// and the burst structure this organ measures is only visible ACROSS runs. Failure to journal is NOT 47// fatal -- a read-only or full filesystem must not cost us the live measurement -- but it IS reported, 48// because a silently-missing journal would make the trend look flat instead of absent. 49func pc_jappend(path: *u8, buf: *u8, n: i64) -> i64 { 50 let fd: i64 = sys_openat_append(path, PC_MODE_644) 51 if fd < 0 { return 0 - 1 } 52 let w: i64 = sys_write(fd, buf, n) 53 sys_close(fd) 54 return w 55} 56 57// read one scalar counter line ("processes 4153147") -- returns the value or -1 FAIL-CLOSED. 58func pc_scalar(buf: *u8, n: i64, key: *u8, out: *i64) -> i64 { 59 let c: i64 = pc_line_ints(buf, n, key, out, 1) 60 if c < 1 { return 0 - 1 } 61 return out[0] 62} 63 64func main() -> i64 { 65 let cbuf: *u8 = sys_mmap(PC_CONFBUF) 66 let cn: i64 = rm_read("knowledge/status/procchurn.conf" as *u8, cbuf, PC_CONFBUF) 67 var win_ms: i64 = rm_conf(cbuf, cn, "window-ms" as *u8, PC_DEF_WINDOW_MS) 68 if win_ms < PC_MIN_WINDOW_MS { win_ms = PC_MIN_WINDOW_MS } 69 if win_ms > PC_MAX_WINDOW_MS { win_ms = PC_MAX_WINDOW_MS } 70 let f_amber: i64 = rm_conf(cbuf, cn, "forks-per-sec-amber" as *u8, PC_DEF_F_AMBER) 71 let f_red: i64 = rm_conf(cbuf, cn, "forks-per-sec-red" as *u8, PC_DEF_F_RED) 72 let r_amber: i64 = rm_conf(cbuf, cn, "sysuser-permil-amber" as *u8, PC_DEF_R_AMBER) 73 let r_red: i64 = rm_conf(cbuf, cn, "sysuser-permil-red" as *u8, PC_DEF_R_RED) 74 let c_amber: i64 = rm_conf(cbuf, cn, "ctxsw-per-sec-amber" as *u8, PC_DEF_C_AMBER) 75 let c_red: i64 = rm_conf(cbuf, cn, "ctxsw-per-sec-red" as *u8, PC_DEF_C_RED) 76 77 let b1: *u8 = sys_mmap(PC_STATBUF) 78 let b2: *u8 = sys_mmap(PC_STATBUF) 79 let c1: *i64 = sys_mmap(PC_FIELD_BYTES) as *i64 80 let c2: *i64 = sys_mmap(PC_FIELD_BYTES) as *i64 81 let sc: *i64 = sys_mmap(PC_FIELD_BYTES) as *i64 82 83 // ---- SAMPLE 1 -> window -> SAMPLE 2. The window is measured, never assumed: sys_sleep_ms can 84 // ---- oversleep on a loaded box, and using the REQUESTED window would inflate every rate. 85 let t1: i64 = sys_now_us() 86 let n1: i64 = rm_read("/proc/stat" as *u8, b1, PC_STATBUF) 87 let p1: i64 = pc_scalar(b1, n1, "processes " as *u8, sc) 88 let x1: i64 = pc_scalar(b1, n1, "ctxt " as *u8, sc) 89 let i1: i64 = pc_scalar(b1, n1, "intr " as *u8, sc) 90 let nf1: i64 = pc_line_ints(b1, n1, "cpu " as *u8, c1, PC_NCPU_FIELDS) 91 sys_sleep_ms(win_ms) 92 let t2: i64 = sys_now_us() 93 let n2: i64 = rm_read("/proc/stat" as *u8, b2, PC_STATBUF) 94 let p2: i64 = pc_scalar(b2, n2, "processes " as *u8, sc) 95 let x2: i64 = pc_scalar(b2, n2, "ctxt " as *u8, sc) 96 let i2: i64 = pc_scalar(b2, n2, "intr " as *u8, sc) 97 let nf2: i64 = pc_line_ints(b2, n2, "cpu " as *u8, c2, PC_NCPU_FIELDS) 98 let elapsed_ms: i64 = (t2 - t1) / PC_US_PER_MS 99 100 var forks_ps: i64 = 0 - 1 101 if p1 >= 0 { if p2 >= 0 { forks_ps = pc_rate(p2 - p1, elapsed_ms) } } 102 var ctxsw_ps: i64 = 0 - 1 103 if x1 >= 0 { if x2 >= 0 { ctxsw_ps = pc_rate(x2 - x1, elapsed_ms) } } 104 var intr_ps: i64 = 0 - 1 105 if i1 >= 0 { if i2 >= 0 { intr_ps = pc_rate(i2 - i1, elapsed_ms) } } 106 107 var cpu_ok: i64 = 0 108 if nf1 >= PC_NCPU_FIELDS { if nf2 >= PC_NCPU_FIELDS { cpu_ok = 1 } } 109 var usr_d: i64 = 0 110 var sys_d: i64 = 0 111 var irq_d: i64 = 0 112 var idle_d: i64 = 0 113 var iow_d: i64 = 0 114 var tot_d: i64 = 0 115 if cpu_ok == 1 { 116 var k: i64 = 0 117 while k < PC_NCPU_FIELDS { tot_d = tot_d + (c2[k] - c1[k]); k = k + 1 } 118 usr_d = (c2[0] - c1[0]) + (c2[1] - c1[1]) 119 sys_d = c2[2] - c1[2] 120 idle_d = c2[3] - c1[3] 121 iow_d = c2[4] - c1[4] 122 irq_d = (c2[5] - c1[5]) + (c2[6] - c1[6]) 123 } 124 var ratio_pm: i64 = 0 - 1 125 if cpu_ok == 1 { ratio_pm = pc_ratio_permil(sys_d, usr_d) } 126 var usr_pm: i64 = 0 - 1 127 var sys_pm: i64 = 0 - 1 128 var irq_pm: i64 = 0 - 1 129 var busy_pm: i64 = 0 - 1 130 if tot_d > 0 { 131 usr_pm = (usr_d * 1000) / tot_d 132 sys_pm = (sys_d * 1000) / tot_d 133 irq_pm = (irq_d * 1000) / tot_d 134 busy_pm = ((tot_d - idle_d - iow_d) * 1000) / tot_d 135 } 136 137 // FAIL-CLOSED: a negative (unmeasurable) axis compares BELOW every threshold and would otherwise be 138 // published as a confident GREEN. That is the partial-as-complete defect the operator banned, so an 139 // incomplete sample gets its OWN exit code and prints no verdict at all. 140 var measured: i64 = 1 141 if forks_ps < 0 { measured = 0 } 142 if ctxsw_ps < 0 { measured = 0 } 143 if ratio_pm < 0 { measured = 0 } 144 if cpu_ok == 0 { measured = 0 } 145 if tot_d <= 0 { measured = 0 } 146 147 rm_puts("=== nx_procchurn -- the process-churn axis nx_health lacks (seq1317) ===\n" as *u8) 148 rm_puts("window_ms=" as *u8); rm_num(elapsed_ms) 149 rm_puts(" (requested " as *u8); rm_num(win_ms); rm_puts(", MEASURED not assumed)\n" as *u8) 150 if measured == 0 { 151 rm_puts("verdict=UNMEASURED sev=" as *u8); rm_num(PC_EXIT_UNMEASURED) 152 rm_puts(" why=a /proc/stat counter was absent or went backwards (wrap/reboot); NO verdict is" as *u8) 153 rm_puts(" published off a partial sample\n" as *u8) 154 return PC_EXIT_UNMEASURED 155 } 156 rm_puts("forks_per_sec=" as *u8); rm_num(forks_ps) 157 rm_puts(" (amber>=" as *u8); rm_num(f_amber); rm_puts(" red>=" as *u8); rm_num(f_red); rm_puts(")\n" as *u8) 158 rm_puts("ctxsw_per_sec=" as *u8); rm_num(ctxsw_ps) 159 rm_puts(" (amber>=" as *u8); rm_num(c_amber); rm_puts(" red>=" as *u8); rm_num(c_red); rm_puts(")\n" as *u8) 160 rm_puts("intr_per_sec=" as *u8); rm_num(intr_ps); rm_puts(" (context only, not thresholded)\n" as *u8) 161 rm_puts("kernel_user_permil=" as *u8); rm_num(ratio_pm) 162 rm_puts(" (1000=parity; amber>=" as *u8); rm_num(r_amber); rm_puts(" red>=" as *u8); rm_num(r_red); rm_puts(")\n" as *u8) 163 rm_puts("cpu_share_permil: user=" as *u8); rm_num(usr_pm) 164 rm_puts(" system=" as *u8); rm_num(sys_pm) 165 rm_puts(" irq_softirq=" as *u8); rm_num(irq_pm) 166 rm_puts(" busy=" as *u8); rm_num(busy_pm); rm_puts("\n" as *u8) 167 rm_puts("jiffies_delta: user+nice=" as *u8); rm_num(usr_d) 168 rm_puts(" system=" as *u8); rm_num(sys_d) 169 rm_puts(" total=" as *u8); rm_num(tot_d); rm_puts("\n" as *u8) 170 171 // QUEUE COMPOSITION (seq1555) -- the number that separates an I/O-blocked queue from a compute one, 172 // and it is FREE: /proc/stat already carries procs_running and procs_blocked, so this costs ZERO extra 173 // syscalls and needs NO admission gate (unlike a /proc walk, which is refused exactly when a deep 174 // queue makes it most interesting -- the priority-lane trap). 175 // WHY IT MATTERS: loadavg counts TASK_UNINTERRUPTIBLE as well as runnable, so a load of 14.94 against 176 // a 50.7pct-idle CPU says the QUEUE is deep but never says WHAT it waits on. iowait cannot answer 177 // either -- Linux only accrues it while a CPU is otherwise IDLE, so a D-state task on a busy core is 178 // invisible to it. procs_blocked counts those tasks DIRECTLY. 179 // READ IT AS: blocked >> running => the queue is I/O/lock-bound, and adding CPU would change nothing. 180 var blocked: i64 = 0 - 1 181 var running: i64 = 0 - 1 182 if pc_line_ints(b2, n2, "procs_blocked " as *u8, sc, 1) > 0 { blocked = sc[0] } 183 if pc_line_ints(b2, n2, "procs_running " as *u8, sc, 1) > 0 { running = sc[0] } 184 rm_puts("queue: procs_running=" as *u8); rm_num(running) 185 rm_puts(" procs_blocked=" as *u8); rm_num(blocked) 186 rm_puts(" (blocked >> running = I/O or lock bound, NOT compute; more CPU would not help)\n" as *u8) 187 188 let sev: i64 = pc_verdict(forks_ps, ratio_pm, ctxsw_ps, f_amber, f_red, r_amber, r_red, c_amber, c_red) 189 190 // ---- TREND: one append-only frame per run. A single verdict is a snapshot; the storm is bursty, 191 // ---- so only the journal can show whether a fix actually landed. 192 let jb: *u8 = sys_mmap(PC_JBUF) 193 let jn: i64 = pc_frame(jb, sys_now_realtime_sec(), elapsed_ms, forks_ps, ctxsw_ps, intr_ps, 194 ratio_pm, usr_pm, sys_pm, busy_pm, sev, running, blocked) 195 let jw: i64 = pc_jappend("knowledge/status/procchurn.jrnl" as *u8, jb, jn) 196 rm_puts("journal=knowledge/status/procchurn.jrnl " as *u8) 197 if jw < 0 { rm_puts("APPEND-FAILED (measurement still valid; the TREND is what was lost)\n" as *u8) } 198 if jw >= 0 { rm_puts("frame_bytes=" as *u8); rm_num(jw); rm_puts("\n" as *u8) } 199 200 rm_puts("verdict=" as *u8) 201 if sev == 0 { rm_puts("GREEN" as *u8) } 202 if sev == 1 { rm_puts("AMBER" as *u8) } 203 if sev == 2 { rm_puts("RED" as *u8) } 204 rm_puts(" sev=" as *u8); rm_num(sev); rm_puts("\n" as *u8) 205 return sev 206}