code wiki / _hdl_build / nx_perf_sentinel.nx
nx_perf_sentinel.nx source
↩ module page · 353 lines · 13650 B
1// nx_perf_sentinel.nx -- the team's BOTTLENECK/LIVENESS organ.
2//
3// Operator 2026-06-10: "slowness anywhere in our entire ecosystem is a
4// total failure, build the team to identify these bottlenecks and dead
5// ends and infinite loops etc and handle them."
6//
7// What it does (one re-runnable pass; the loop runs it on a beat):
8// 1. samples /proc twice (SENTINEL_WINDOW_MS apart) and computes REAL
9// per-process CPU%% from utime+stime deltas -- no vibes, kernel truth
10// 2. classifies each process against a BUDGET TABLE (config file, NO
11// magic numbers in code -- Cardinal 11):
12// SPIN cpu%% >= spin_pct (the nx_signaling 100%%-spin class,
13// bd.t/ovw multi-day runaways)
14// RUNAWAY cpu%% >= hot_pct AND age >= age_s (hot for days)
15// 3. HANDLES per the budget action: log -> sentinel log line;
16// escalate -> ESCALATE line (the PM/loop dispatcher's queue);
17// kill -> nx_kill(pid,9) ONLY when the budget line explicitly says so
18// (governed: default budgets never kill -- a human/PM line opts in)
19// 4. exit code = number of violations (0 = ecosystem clean) so it IS
20// its own gate and cron-able health check.
21//
22// Budget config (default /tmp/nishi_perf_budgets.conf, overridable argv[1]):
23// # comm-substring spin_pct hot_pct age_s action(log|escalate|kill)
24// * 97 80 86400 escalate
25// nx_signaling 90 50 3600 escalate
26// '*' = default row. First matching row wins; comm match = substring.
27//
28// Composes: nx_syscalls (getdents64/read/kill), the nx_loop_monitor
29// verdict->handle pattern, nx_sysload's /proc-truth doctrine.
30// license_tier: ORIGINAL
31
32import "nx_syscalls.nx"
33import "_hdl_build/nx_kill_portable.nx"
34const PS_MAGIC_32768: i64 = 32768
35
36const PS_WINDOW_MS: i64 = 2000
37const PS_MAX_PROCS: i64 = 2048
38const PS_HZ: i64 = 100 // kernel USER_HZ for utime/stime
39const PS_LOG_PATH: *u8 = "/tmp/nishi_perf_sentinel.log" as *u8
40const PS_CONF_DEF: *u8 = "/tmp/nishi_perf_budgets.conf" as *u8
41
42const PS_ACT_LOG: i64 = 1
43const PS_ACT_ESCALATE: i64 = 2
44const PS_ACT_KILL: i64 = 3
45
46func ps_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
47func ps_puts(s: *u8) -> i64 { sys_write(1, s, ps_slen(s)); return 0 }
48func ps_putn(v: i64) -> i64 {
49 let b: *u8 = sys_mmap(28)
50 if v == 0 { sys_write(1, "0" as *u8, 1); return 0 }
51 var x: i64 = v
52 if x < 0 { sys_write(1, "-" as *u8, 1); x = 0 - x }
53 var d: i64 = 0
54 var y: i64 = x
55 while y > 0 { d = d + 1; y = y / 10 }
56 var i: i64 = d - 1
57 y = x
58 while i >= 0 { b[i] = (48 + (y % 10)) as u8; y = y / 10; i = i - 1 }
59 sys_write(1, b, d)
60 return 0
61}
62func ps_append(dst: *u8, off: i64, s: *u8) -> i64 {
63 var i: i64 = 0
64 while s[i] != (0 as u8) { dst[off + i] = s[i]; i = i + 1 }
65 return off + i
66}
67func ps_append_n(dst: *u8, off: i64, v: i64) -> i64 {
68 var o: i64 = off
69 if v == 0 { dst[o] = 48; return o + 1 }
70 var x: i64 = v
71 if x < 0 { dst[o] = 45; o = o + 1; x = 0 - x }
72 var d: i64 = 0
73 var y: i64 = x
74 while y > 0 { d = d + 1; y = y / 10 }
75 var i: i64 = d - 1
76 y = x
77 while i >= 0 { dst[o + i] = (48 + (y % 10)) as u8; y = y / 10; i = i - 1 }
78 return o + d
79}
80
81// parse decimal at buf[i..n); returns value, writes end to ip[0]
82func ps_dec(buf: *u8, i: i64, n: i64, ip: *i64) -> i64 {
83 var k: i64 = i
84 var v: i64 = 0
85 var run: i64 = 1
86 while run == 1 {
87 if k >= n { run = 0 }
88 else {
89 let c: i64 = buf[k] as i64
90 var dig: i64 = 0
91 if c >= 48 { if c <= 57 { dig = 1 } }
92 if dig == 1 { v = v * 10 + (c - 48); k = k + 1 } else { run = 0 }
93 }
94 }
95 ip[0] = k
96 return v
97}
98
99// read /proc/<pid>/stat -> comm (out, NUL-term, <=30), state char, cpu
100// ticks (utime+stime), starttime ticks. returns 0 ok / -1 fail.
101func ps_read_stat(pid: i64, comm: *u8, statec: *u8, ticks: *i64, start: *i64) -> i64 {
102 let path: *u8 = sys_mmap(64)
103 var o: i64 = ps_append(path, 0, "/proc/" as *u8)
104 o = ps_append_n(path, o, pid)
105 o = ps_append(path, o, "/stat" as *u8)
106 path[o] = 0
107 let lenb: *i64 = sys_mmap(16) as *i64
108 lenb[0] = 0
109 let buf: *u8 = sys_read_file(path, lenb)
110 if (buf as i64) == 0 { return 0 - 1 }
111 let n: i64 = lenb[0]
112 if n < 20 { return 0 - 1 }
113 // comm = between first '(' and LAST ')'
114 var lp: i64 = 0 - 1
115 var rp: i64 = 0 - 1
116 var i: i64 = 0
117 while i < n {
118 if buf[i] == (40 as u8) { if lp < 0 { lp = i } }
119 if buf[i] == (41 as u8) { rp = i }
120 i = i + 1
121 }
122 if lp < 0 { return 0 - 1 }
123 if rp < lp { return 0 - 1 }
124 var cw: i64 = 0
125 var ci: i64 = lp + 1
126 while ci < rp { if cw < 30 { comm[cw] = buf[ci]; cw = cw + 1 } ci = ci + 1 }
127 comm[cw] = 0
128 // after ") " : field 3 = state, then fields 4..; utime=14 stime=15 starttime=22
129 var p: i64 = rp + 2
130 statec[0] = buf[p]
131 var field: i64 = 3
132 var ut: i64 = 0
133 var st: i64 = 0
134 var stt: i64 = 0
135 let ip: *i64 = sys_mmap(16) as *i64
136 while field < 22 {
137 // advance to next space then past it
138 while p < n { if buf[p] == (32 as u8) { p = p + 1; field = field + 1; break } else { p = p + 1 } }
139 if field == 14 { ut = ps_dec(buf, p, n, ip) }
140 if field == 15 { st = ps_dec(buf, p, n, ip) }
141 if field == 22 { stt = ps_dec(buf, p, n, ip) }
142 if p >= n { field = 22 }
143 }
144 ticks[0] = ut + st
145 start[0] = stt
146 return 0
147}
148
149// uptime seconds from /proc/uptime
150func ps_uptime_s() -> i64 {
151 let lenb: *i64 = sys_mmap(16) as *i64
152 lenb[0] = 0
153 let buf: *u8 = sys_read_file("/proc/uptime" as *u8, lenb)
154 if (buf as i64) == 0 { return 0 }
155 let ip: *i64 = sys_mmap(16) as *i64
156 return ps_dec(buf, 0, lenb[0], ip)
157}
158
159// enumerate numeric /proc entries into pids[]; returns count
160func ps_list_pids(pids: *i64) -> i64 {
161 let dfd: i64 = sys_openat_rd("/proc" as *u8)
162 if dfd < 0 { return 0 }
163 let dbuf: *u8 = sys_mmap(PS_MAGIC_32768)
164 var count: i64 = 0
165 var more: i64 = 1
166 while more == 1 {
167 let nread: i64 = sys_getdents64(dfd, dbuf, PS_MAGIC_32768)
168 if nread <= 0 { more = 0 }
169 else {
170 var off: i64 = 0
171 while off < nread {
172 let reclen: i64 = (dbuf[off + 16] as i64) | ((dbuf[off + 17] as i64) << 8)
173 let namep: i64 = off + 19
174 var pid: i64 = 0
175 var isnum: i64 = 1
176 var k: i64 = 0
177 var run: i64 = 1
178 while run == 1 {
179 let c: i64 = dbuf[namep + k] as i64
180 if c == 0 { run = 0 }
181 else {
182 var dig: i64 = 0
183 if c >= 48 { if c <= 57 { dig = 1 } }
184 if dig == 1 { pid = pid * 10 + (c - 48); k = k + 1 }
185 else { isnum = 0; run = 0 }
186 }
187 }
188 if k == 0 { isnum = 0 }
189 if isnum == 1 { if count < PS_MAX_PROCS { pids[count] = pid; count = count + 1 } }
190 if reclen <= 0 { off = nread } else { off = off + reclen }
191 }
192 }
193 }
194 sys_close(dfd)
195 return count
196}
197
198// substring match (case-sensitive; comm names are exact lowercase here)
199func ps_has(hay: *u8, needle: *u8) -> i64 {
200 let hn: i64 = ps_slen(hay)
201 let nn: i64 = ps_slen(needle)
202 if nn == 0 { return 1 }
203 if hn < nn { return 0 }
204 var i: i64 = 0
205 while i + nn <= hn {
206 var j: i64 = 0
207 var ok: i64 = 1
208 while j < nn { if hay[i + j] != needle[j] { ok = 0; j = nn } else { j = j + 1 } }
209 if ok == 1 { return 1 }
210 i = i + 1
211 }
212 return 0
213}
214
215// budget lookup: scan config lines, first row whose comm-substr matches
216// wins; '*' matches all. writes spin/hot/age/action; returns 1 found.
217func ps_budget(cfg: *u8, cfgn: i64, comm: *u8,
218 spin: *i64, hot: *i64, age: *i64, act: *i64) -> i64 {
219 var i: i64 = 0
220 while i < cfgn {
221 // skip spaces; '#' comment lines skipped flag-style (NO
222 // continue-in-nested-if -- the 2026-06-10 miscompile landmine)
223 while i < cfgn { if cfg[i] == (32 as u8) { i = i + 1 } else { break } }
224 var is_comment: i64 = 0
225 if i < cfgn { if cfg[i] == (35 as u8) { is_comment = 1 } }
226 if is_comment == 1 {
227 while i < cfgn { if cfg[i] == (10 as u8) { break } i = i + 1 }
228 i = i + 1
229 } else {
230 if i >= cfgn { return 0 }
231 // token 1: pattern
232 let pat: *u8 = sys_mmap(64)
233 var pw: i64 = 0
234 while i < cfgn {
235 let c: u8 = cfg[i]
236 if c == (32 as u8) { break }
237 if c == (10 as u8) { break }
238 if pw < 60 { pat[pw] = c; pw = pw + 1 }
239 i = i + 1
240 }
241 pat[pw] = 0
242 let ip: *i64 = sys_mmap(16) as *i64
243 while i < cfgn { if cfg[i] == (32 as u8) { i = i + 1 } else { break } }
244 let v1: i64 = ps_dec(cfg, i, cfgn, ip)
245 i = ip[0]
246 while i < cfgn { if cfg[i] == (32 as u8) { i = i + 1 } else { break } }
247 let v2: i64 = ps_dec(cfg, i, cfgn, ip)
248 i = ip[0]
249 while i < cfgn { if cfg[i] == (32 as u8) { i = i + 1 } else { break } }
250 let v3: i64 = ps_dec(cfg, i, cfgn, ip)
251 i = ip[0]
252 while i < cfgn { if cfg[i] == (32 as u8) { i = i + 1 } else { break } }
253 var action: i64 = PS_ACT_LOG
254 if i < cfgn { if cfg[i] == (101 as u8) { action = PS_ACT_ESCALATE } } // 'e'
255 if i < cfgn { if cfg[i] == (107 as u8) { action = PS_ACT_KILL } } // 'k'
256 while i < cfgn { if cfg[i] == (10 as u8) { break } i = i + 1 }
257 i = i + 1
258 var hit: i64 = 0
259 if pw == 1 { if pat[0] == (42 as u8) { hit = 1 } }
260 if hit == 0 { hit = ps_has(comm, pat) }
261 if hit == 1 {
262 spin[0] = v1; hot[0] = v2; age[0] = v3; act[0] = action
263 return 1
264 }
265 }
266 }
267 return 0
268}
269
270func main(argc: i64, argv: *i64) -> i64 {
271 var conf: *u8 = PS_CONF_DEF
272 if argc >= 2 { conf = argv[1] as *u8 }
273 let cfgl: *i64 = sys_mmap(16) as *i64
274 cfgl[0] = 0
275 var cfg: *u8 = sys_read_file(conf, cfgl)
276 var cfgn: i64 = 0
277 if (cfg as i64) != 0 { cfgn = cfgl[0] }
278 // graceful: no config -> built-in default row (* 97 80 86400 escalate)
279 if cfgn == 0 {
280 cfg = "* 97 80 86400 escalate\n" as *u8
281 cfgn = ps_slen(cfg)
282 }
283
284 let pids: *i64 = sys_mmap(PS_MAX_PROCS * 8) as *i64
285 let t0: *i64 = sys_mmap(PS_MAX_PROCS * 8) as *i64
286 let np: i64 = ps_list_pids(pids)
287
288 // sample 1
289 let comm: *u8 = sys_mmap(40)
290 let statec: *u8 = sys_mmap(8)
291 let ticks: *i64 = sys_mmap(16) as *i64
292 let start: *i64 = sys_mmap(16) as *i64
293 var i: i64 = 0
294 while i < np {
295 t0[i] = 0 - 1
296 if ps_read_stat(pids[i], comm, statec, ticks, start) == 0 { t0[i] = ticks[0] }
297 i = i + 1
298 }
299 sys_sleep_ms(PS_WINDOW_MS)
300
301 let uptime: i64 = ps_uptime_s()
302 let lfd: i64 = sys_openat_append(PS_LOG_PATH, 420)
303 let line: *u8 = sys_mmap(512)
304 let spin: *i64 = sys_mmap(16) as *i64
305 let hot: *i64 = sys_mmap(16) as *i64
306 let agemax: *i64 = sys_mmap(16) as *i64
307 let act: *i64 = sys_mmap(16) as *i64
308 var violations: i64 = 0
309
310 // sample 2 + classify
311 i = 0
312 while i < np {
313 if t0[i] >= 0 {
314 if ps_read_stat(pids[i], comm, statec, ticks, start) == 0 {
315 let dticks: i64 = ticks[0] - t0[i]
316 // cpu% = dticks/HZ / (window/1000) * 100
317 let cpu_pct: i64 = (dticks * 1000 * 100) / (PS_HZ * PS_WINDOW_MS)
318 let age_s: i64 = uptime - (start[0] / PS_HZ)
319 if ps_budget(cfg, cfgn, comm, spin, hot, agemax, act) == 1 {
320 var verdict: i64 = 0 // 0 none, 1 SPIN, 2 RUNAWAY
321 if cpu_pct >= spin[0] { verdict = 1 }
322 if verdict == 0 { if cpu_pct >= hot[0] { if age_s >= agemax[0] { verdict = 2 } } }
323 if verdict != 0 {
324 violations = violations + 1
325 var w: i64 = 0
326 if act[0] == PS_ACT_ESCALATE { w = ps_append(line, w, "PERF ESCALATE " as *u8) }
327 else { if act[0] == PS_ACT_KILL { w = ps_append(line, w, "PERF KILL " as *u8) }
328 else { w = ps_append(line, w, "PERF LOG " as *u8) } }
329 if verdict == 1 { w = ps_append(line, w, "SPIN pid=" as *u8) }
330 else { w = ps_append(line, w, "RUNAWAY pid=" as *u8) }
331 w = ps_append_n(line, w, pids[i])
332 w = ps_append(line, w, " comm=" as *u8)
333 w = ps_append(line, w, comm)
334 w = ps_append(line, w, " cpu_pct=" as *u8)
335 w = ps_append_n(line, w, cpu_pct)
336 w = ps_append(line, w, " age_s=" as *u8)
337 w = ps_append_n(line, w, age_s)
338 line[w] = 10
339 w = w + 1
340 sys_write(1, line, w)
341 if lfd >= 0 { sys_write(lfd, line, w) }
342 // HANDLE: kill only when the budget row opts in
343 if act[0] == PS_ACT_KILL { nxk_kill(pids[i], 9) }
344 }
345 }
346 }
347 }
348 i = i + 1
349 }
350 if lfd >= 0 { sys_close(lfd) }
351 ps_puts("sentinel: " as *u8); ps_putn(violations); ps_puts(" violation(s)\n" as *u8)
352 return violations
353}