nx_leak_check.nx source
↩ module page · 274 lines · 15434 B
1// nx_leak_check.nx -- FLEET LEAK-HEALTH CERTIFIER, CLI. Non-invasive (reads only /proc). Grounds every
2// verdict in a real time series + Mann-Kendall trend + Theil-Sen rate -- and PRINTS the numbers, so it is
3// never a bare assertion. Handles VARIABLE leaks (the sign-based trend test doesn't need a clean line).
4// series <pid> <samples> <interval_ms> deep single-process analysis; prints the raw series + tau + rate
5// fleet <samples> <interval_ms> [crit_sev] certify EVERY running sovereign organ (.elf); per-organ verdict.
6// crit_sev: 1=CHURN+ 2=SPIKE|LEAK (default, deep-dive) 3=LEAK-only (the STANDING-sweep invariant --
7// transient SPIKEs on bursty workers are printed evidence, not a standing failure). Organs that exit
8// mid-window are GONE-skipped (sample-validity guard), never convicted on poisoned series.
9// exit 0 = healthy / all-healthy, 1 = LEAK / some-leak. license_tier: ORIGINAL
10import "nx_leak_check_lib.nx"
11
12const LK_ARG_VERB: i64 = 1
13const LK_ARG_P1: i64 = 2 // series: pid ; fleet: samples
14const LK_ARG_P2: i64 = 3 // series: samples ; fleet: interval_ms
15const LK_ARG_P3: i64 = 4 // series: interval_ms
16const LK_ARGC_FLEET: i64 = 4 // fleet samples interval
17const LK_ARGC_SERIES: i64 = 5 // series pid samples interval
18const LK_RC_USAGE: i64 = 2
19const LK_RC_LEAK: i64 = 1
20const LK_MAX_S: i64 = 32 // max samples per series
21const LK_MAX_ORG: i64 = 256 // max organs in a fleet scan
22const LK_MAX_PROC: i64 = 4096 // /proc pid enumeration cap
23const LK_PAIRS: i64 = 496 // LK_MAX_S*(LK_MAX_S-1)/2 -- Theil-Sen pairwise-slope scratch
24const LK_WORD: i64 = 8
25const LK_MS_PER_S: i64 = 1000
26const LK_SEV_CHURN: i64 = 1 // severity ladder: HEALTHY 0 < CHURN 1 < SPIKE 2 < LEAK 3
27const LK_SEV_SPIKE: i64 = 2
28const LK_SEV_LEAK: i64 = 3
29const LK_SEV_CRIT: i64 = 2 // severity >= this (SPIKE|LEAK) = CRITICAL (fails the fleet exit)
30const LK_MIN_AGE_S: i64 = 120 // a LEAK verdict requires a process OLD ENOUGH to own a baseline: fork-per-
31 // connection children + fresh respawns ramp legitimately for seconds-minutes
32 // (live false-positive: an 8s-lived sites.elf child convicted at 12.8MB/min
33 // while serving the sweep's own edge probe). R3 replaces this floor with
34 // learned per-organ baselines from history.
35
36func lk_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
37func lk_putn(v: i64) -> i64 {
38 if v == 0 { sys_write(1, "0" as *u8, 1); return 0 }
39 var m: i64 = v; if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
40 let d: *u8 = sys_mmap(LC_NAME_CAP); var k: i64 = 0
41 while m > 0 { d[k] = (LC_ASCII_0 + (m % LC_DEC)) as u8; m = m / LC_DEC; k = k + 1 }
42 var i: i64 = 0
43 while i < k { sys_write(1, ((d as i64) + k - 1 - i) as *u8, 1); i = i + 1 }
44 sys_munmap(d, LC_NAME_CAP)
45 return 0
46}
47func lk_seq(a: *u8, b: *u8) -> i64 { var i: i64 = 0; while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 } if b[i] != (0 as u8) { return 0 } return 1 }
48func lk_atoi(s: *u8) -> i64 {
49 var v: i64 = 0; var i: i64 = 0
50 while s[i] != (0 as u8) { let c: i64 = s[i] as i64; if c < LC_ASCII_0 { return v } if c > LC_ASCII_0 + LC_DEC - 1 { return v } v = v * LC_DEC + (c - LC_ASCII_0); i = i + 1 }
51 return v
52}
53// clamp a sample count into [LC_MIN_N+LC_WARMUP, LK_MAX_S]
54func lk_clamp_n(n: i64) -> i64 {
55 var r: i64 = n
56 if r > LK_MAX_S { r = LK_MAX_S }
57 if r < LC_MIN_N + LC_WARMUP { r = LC_MIN_N + LC_WARMUP }
58 return r
59}
60// print the stat line + anomaly classification for a series already classified into out; return 1 if the
61// anomaly is CRITICAL (LEAK or SPIKE), 0 for CHURN (warning) / HEALTHY. Shows the numbers -- never a bare verdict.
62func lk_verdict_line(out: *i64) -> i64 {
63 lk_puts(" med=" as *u8); lk_putn(out[LC_O_MEDIAN]); lk_puts("kB tau=" as *u8); lk_putn(out[LC_O_TAU])
64 lk_puts("permille rate=" as *u8); lk_putn(out[LC_O_RATE]); lk_puts("kB/min mad=" as *u8); lk_putn(out[LC_O_MAD])
65 lk_puts("kB peak=" as *u8); lk_putn(out[LC_O_PEAK]); lk_puts("kB -> " as *u8)
66 let code: i64 = out[LC_O_CODE]
67 if code == LC_A_LEAK { lk_puts("LEAK\n" as *u8); return 1 }
68 if code == LC_A_SPIKE { lk_puts("SPIKE\n" as *u8); return 1 }
69 if code == LC_A_CHURN { lk_puts("CHURN\n" as *u8); return 0 }
70 lk_puts("HEALTHY\n" as *u8); return 0
71}
72
73// anomaly code -> name
74func lk_code_name(code: i64) -> *u8 {
75 if code == LC_A_LEAK { return "LEAK" as *u8 }
76 if code == LC_A_SPIKE { return "SPIKE" as *u8 }
77 if code == LC_A_CHURN { return "CHURN" as *u8 }
78 return "HEALTHY" as *u8
79}
80// severity for "worst meter" (HEALTHY 0 < CHURN 1 < SPIKE 2 < LEAK 3); CRITICAL = SPIKE|LEAK
81func lk_sev(code: i64) -> i64 {
82 if code == LC_A_LEAK { return LK_SEV_LEAK }
83 if code == LC_A_SPIKE { return LK_SEV_SPIKE }
84 if code == LC_A_CHURN { return LK_SEV_CHURN }
85 return 0
86}
87// print one meter's compact evidence: "<label> <CODE>(rate=<r>)"
88func lk_meter(label: *u8, out: *i64) -> i64 {
89 lk_puts(label); lk_puts(" " as *u8); lk_puts(lk_code_name(out[LC_O_CODE]))
90 lk_puts("(rate=" as *u8); lk_putn(out[LC_O_RATE]); lk_puts(")" as *u8)
91 return 0
92}
93// CPU: turn a cumulative-ticks row into the per-interval UTILIZATION series (permille of one core) + its time
94// axis. util[j] = (ticks[j+1]-ticks[j]) * 1e6 / (HZ * dt_ms). Returns the count of usable intervals (n-1).
95// (This is the delta transform that makes CPU just-another-meter -- no special detector, the rate/level/MAD
96// classifier runs on util[] exactly like memory runs on kB.)
97// process age in seconds (boot-relative: uptime - starttime/HZ); -1 if unreadable (gone). Composed from the
98// portable seam (osp_*) so the NishiOS backend inherits it unchanged.
99func lk_proc_age_s(pid: i64, hz: i64) -> i64 {
100 let st: i64 = osp_starttime_ticks(pid)
101 if st < 0 { return 0 - 1 }
102 return osp_uptime_s() - st / hz
103}
104func lk_cpu_util(cpu: *i64, coff: i64, ts: *i64, n: i64, util: *i64, utime: *i64, hz: i64) -> i64 {
105 var m: i64 = 0
106 var j: i64 = 0
107 while j < n - 1 {
108 let dt: i64 = ts[j+1] - ts[j]
109 if dt > 0 { util[m] = (cpu[coff + j + 1] - cpu[coff + j]) * LC_UTIL_SCALE / (hz * dt); utime[m] = ts[j+1]; m = m + 1 }
110 j = j + 1
111 }
112 return m
113}
114
115func main(argc: i64, argv: *i64) -> i64 {
116 if argc < LK_ARGC_FLEET { lk_puts("usage: nx_leak_check series <pid> <samples> <interval_ms> | fleet <samples> <interval_ms>\n" as *u8); return LK_RC_USAGE }
117 let verb: *u8 = argv[LK_ARG_VERB] as *u8
118 let scratch: *i64 = sys_mmap(LK_PAIRS * LK_WORD) as *i64
119 let out3: *i64 = sys_mmap(LC_O_SLOTS * LK_WORD) as *i64
120
121 if lk_seq(verb, "series" as *u8) == 1 {
122 if argc < LK_ARGC_SERIES { lk_puts("usage: nx_leak_check series <pid> <samples> <interval_ms>\n" as *u8); return LK_RC_USAGE }
123 let pid: i64 = lk_atoi(argv[LK_ARG_P1] as *u8)
124 let n: i64 = lk_clamp_n(lk_atoi(argv[LK_ARG_P2] as *u8))
125 let interval: i64 = lk_atoi(argv[LK_ARG_P3] as *u8)
126 let xs: *i64 = sys_mmap(LK_MAX_S * LK_WORD) as *i64
127 let cpu: *i64 = sys_mmap(LK_MAX_S * LK_WORD) as *i64
128 let util: *i64 = sys_mmap(LK_MAX_S * LK_WORD) as *i64
129 let utime: *i64 = sys_mmap(LK_MAX_S * LK_WORD) as *i64
130 let ts: *i64 = sys_mmap(LK_MAX_S * LK_WORD) as *i64
131 let outc: *i64 = sys_mmap(LC_O_SLOTS * LK_WORD) as *i64
132 var s: i64 = 0
133 while s < n {
134 ts[s] = lc_now_ms()
135 xs[s] = lc_vsz(pid)
136 cpu[s] = osp_cpu_ticks(pid)
137 s = s + 1
138 if s < n { sys_sleep_ms(interval) }
139 }
140 if lc_row_valid(xs, 0, n) == 0 {
141 lk_puts("=== NX-LEAK series pid=" as *u8); lk_putn(pid)
142 lk_puts(" GONE (absent/exited during sampling) -- nothing to certify ===\n" as *u8)
143 return 0
144 }
145 lk_puts("=== NX-LEAK series pid=" as *u8); lk_putn(pid); lk_puts(" VmSize kB over " as *u8); lk_putn(n); lk_puts(" samples ===\n data:" as *u8)
146 s = 0
147 while s < n { lk_puts(" " as *u8); lk_putn(xs[s]); s = s + 1 }
148 lk_puts("\n mem: " as *u8)
149 lc_classify(xs, 0, ts, 0, n, scratch, out3)
150 var critical: i64 = lk_verdict_line(out3)
151 // cpu% utilization series (the burn meter) -- prints the per-interval util + classification
152 let mu: i64 = lk_cpu_util(cpu, 0, ts, n, util, utime, osp_hz())
153 lk_puts(" cpu% util:" as *u8)
154 s = 0
155 while s < mu { lk_puts(" " as *u8); lk_putn(util[s]); s = s + 1 }
156 lk_puts("\n cpu: " as *u8)
157 lc_classify_t(util, 0, utime, 0, mu, scratch, outc, LC_CPU_RATE_MIN, LC_CPU_SPIKE_MIN, LC_CPU_BURN_MAX)
158 if lk_verdict_line(outc) == 1 { critical = 1 }
159 if critical == 1 { return LK_RC_LEAK }
160 return 0
161 }
162
163 if lk_seq(verb, "fleet" as *u8) == 1 {
164 let n: i64 = lk_clamp_n(lk_atoi(argv[LK_ARG_P1] as *u8))
165 let interval: i64 = lk_atoi(argv[LK_ARG_P2] as *u8)
166 // optional crit severity floor (argv[4]): 2 = SPIKE|LEAK fail (deep-dive default), 3 = LEAK-only --
167 // the STANDING-sweep contract (a sweep check asserts an INVARIANT; transient SPIKEs on bursty
168 // workers are printed evidence, not a standing failure). Garbage/omitted -> default.
169 var critsev: i64 = LK_SEV_CRIT
170 if argc > LK_ARG_P3 {
171 let cs: i64 = lk_atoi(argv[LK_ARG_P3] as *u8)
172 if cs >= LK_SEV_CHURN { if cs <= LK_SEV_LEAK { critsev = cs } }
173 }
174 let pids: *i64 = sys_mmap(LK_MAX_ORG * LK_WORD) as *i64
175 let allp: *i64 = sys_mmap(LK_MAX_PROC * LK_WORD) as *i64
176 let np: i64 = osp_list_pids(allp, LK_MAX_PROC)
177 let a0: *u8 = sys_mmap(LC_NAME_CAP)
178 let self: i64 = osp_selfpid()
179 var norg: i64 = 0
180 var i: i64 = 0
181 while i < np {
182 let p: i64 = allp[i]
183 if p != self { if norg < LK_MAX_ORG {
184 osp_cmd_argv0(p, a0, LC_NAME_CAP)
185 if lc_is_organ(a0) == 1 { if lc_vsz(p) > 0 { pids[norg] = p; norg = norg + 1 } }
186 } }
187 i = i + 1
188 }
189 // THREE-METER sample matrices, shared time axis ts[s]. A leak shows in whatever resource disappears:
190 // vsz = address space (kB) · rss = RESIDENT (kB -- heap leaks an arena hides from VSZ) · fd = descriptors
191 let vsz: *i64 = sys_mmap(LK_MAX_ORG * LK_MAX_S * LK_WORD) as *i64
192 let rss: *i64 = sys_mmap(LK_MAX_ORG * LK_MAX_S * LK_WORD) as *i64
193 let fdc: *i64 = sys_mmap(LK_MAX_ORG * LK_MAX_S * LK_WORD) as *i64
194 let cpu: *i64 = sys_mmap(LK_MAX_ORG * LK_MAX_S * LK_WORD) as *i64
195 let ts: *i64 = sys_mmap(LK_MAX_S * LK_WORD) as *i64
196 let util: *i64 = sys_mmap(LK_MAX_S * LK_WORD) as *i64
197 let utime: *i64 = sys_mmap(LK_MAX_S * LK_WORD) as *i64
198 let outr: *i64 = sys_mmap(LC_O_SLOTS * LK_WORD) as *i64
199 let outf: *i64 = sys_mmap(LC_O_SLOTS * LK_WORD) as *i64
200 let outc: *i64 = sys_mmap(LC_O_SLOTS * LK_WORD) as *i64
201 let hz: i64 = osp_hz()
202 lk_puts("=== NX-LEAK fleet: certifying " as *u8); lk_putn(norg); lk_puts(" organs x4 meters (VmSize+VmRSS+fd+cpu%), " as *u8); lk_putn(n)
203 lk_puts(" samples @ " as *u8); lk_putn(interval); lk_puts("ms (Mann-Kendall trend + Theil-Sen rate + MAD + level) ===\n" as *u8)
204 var s: i64 = 0
205 while s < n {
206 ts[s] = lc_now_ms()
207 var k: i64 = 0
208 while k < norg {
209 vsz[k * LK_MAX_S + s] = lc_vsz(pids[k])
210 rss[k * LK_MAX_S + s] = lc_rss(pids[k])
211 fdc[k * LK_MAX_S + s] = osp_fd_count(pids[k])
212 cpu[k * LK_MAX_S + s] = osp_cpu_ticks(pids[k])
213 k = k + 1
214 }
215 s = s + 1
216 if s < n { sys_sleep_ms(interval) }
217 }
218 let nm: *u8 = sys_mmap(LC_NAME_CAP)
219 let bn: *u8 = sys_mmap(LC_NAME_CAP)
220 var crit: i64 = 0
221 i = 0
222 while i < norg {
223 osp_cmd_argv0(pids[i], nm, LC_NAME_CAP); lc_basename(nm, bn)
224 lk_puts(" " as *u8); lk_puts(bn); lk_puts(" pid=" as *u8); lk_putn(pids[i])
225 // SAMPLE-VALIDITY GUARD: an organ that exited mid-window leaves -1 sentinels in its series --
226 // classifying that poison convicts a fake SPIKE (huge negative slope). GONE = skipped, honest.
227 if lc_row_valid(vsz, i * LK_MAX_S, n) == 0 {
228 lk_puts(" | GONE (exited mid-window) -- skipped, not certified\n" as *u8)
229 } else {
230 var young: i64 = 0
231 let age: i64 = lk_proc_age_s(pids[i], hz)
232 if age >= 0 { if age < LK_MIN_AGE_S { young = 1 } }
233 if young == 1 {
234 lk_puts(" | YOUNG (age=" as *u8); lk_putn(age)
235 lk_puts("s < " as *u8); lk_putn(LK_MIN_AGE_S)
236 lk_puts("s baseline) -- ramping allowed, not certified\n" as *u8)
237 } else {
238 lc_classify(vsz, i * LK_MAX_S, ts, 0, n, scratch, out3) // memory meter (kB thresholds)
239 var sev: i64 = lk_sev(out3[LC_O_CODE])
240 lk_puts(" | " as *u8)
241 lk_meter("mem" as *u8, out3)
242 lk_puts(" | " as *u8)
243 if lc_row_valid(rss, i * LK_MAX_S, n) == 1 {
244 lc_classify(rss, i * LK_MAX_S, ts, 0, n, scratch, outr) // rss meter (kB thresholds)
245 lk_meter("rss" as *u8, outr)
246 let sr: i64 = lk_sev(outr[LC_O_CODE]); if sr > sev { sev = sr }
247 } else { lk_puts("rss n/a" as *u8) }
248 lk_puts(" | " as *u8)
249 if lc_row_valid(fdc, i * LK_MAX_S, n) == 1 {
250 lc_classify_t(fdc, i * LK_MAX_S, ts, 0, n, scratch, outf, LC_FD_RATE_MIN, LC_FD_SPIKE_MIN, LC_LEVEL_OFF) // fd meter (descriptor thresholds)
251 lk_meter("fd" as *u8, outf)
252 let sf: i64 = lk_sev(outf[LC_O_CODE]); if sf > sev { sev = sf }
253 } else { lk_puts("fd n/a" as *u8) }
254 lk_puts(" | " as *u8)
255 if lc_row_valid(cpu, i * LK_MAX_S, n) == 1 {
256 let mu: i64 = lk_cpu_util(cpu, i * LK_MAX_S, ts, n, util, utime, hz) // cumulative ticks -> per-interval util
257 lc_classify_t(util, 0, utime, 0, mu, scratch, outc, LC_CPU_RATE_MIN, LC_CPU_SPIKE_MIN, LC_CPU_BURN_MAX) // cpu% (level check ON: sustained burn = LEAK)
258 lk_meter("cpu%" as *u8, outc)
259 let sc: i64 = lk_sev(outc[LC_O_CODE]); if sc > sev { sev = sc }
260 } else { lk_puts("cpu n/a" as *u8) }
261 lk_puts("\n" as *u8)
262 if sev >= critsev { crit = crit + 1 }
263 }
264 }
265 i = i + 1
266 }
267 lk_puts("NX-LEAK fleet verdict: " as *u8)
268 if crit == 0 { lk_puts("ALL HEALTHY (on the racing line -- no leaks/spikes/churn/burn across 4 meters)\n" as *u8); return 0 }
269 lk_puts("CRITICAL=" as *u8); lk_putn(crit); lk_puts(" organ(s) with a LEAK|SPIKE (mem/rss/fd/cpu) -- named above with the numbers\n" as *u8)
270 return LK_RC_LEAK
271 }
272 lk_puts("usage: nx_leak_check series <pid> <samples> <interval_ms> | fleet <samples> <interval_ms>\n" as *u8)
273 return LK_RC_USAGE
274}