nx_leak_check_lib.nx source
↩ module page · 271 lines · 14320 B
1// nx_leak_check_lib.nx -- FLEET LEAK-HEALTH CERTIFIER, library half. The operator's vision: the organs are
2// a family/community that must be TRULY HEALTHY, health PROVEN with real math, not asserted.
3//
4// ★A LEAK IS NOT "MONOTONE" AND NOT "A CLEAN LINE." A real leak -- like a water leak -- is VARIABLE, bursty,
5// intermittent, accelerating; the invariant is that the resource TRENDS UP over time amid the variation and
6// never returns to baseline (consumption that disappears). So the verdict is grounded in the environmental-
7// science trend toolkit (the same statistics hydrologists use for streamflow/groundwater trend + leak
8// detection), computed over a TIME SERIES of N samples -- never 2 points, never a linear-fit R2 that a
9// jagged leak would fail:
10// QUALITATIVE = MANN-KENDALL trend test (non-parametric, SIGN-based): S = sum over i<j of sign(x_j-x_i).
11// Detects a significant upward trend even when the data is noisy/variable. Significance via
12// the exact integer variance Var(S)=n(n-1)(2n+5)/18 vs a one-sided z=1.645 (95%) bar.
13// QUANTITATIVE = THEIL-SEN estimator: the MEDIAN of all pairwise slopes (x_j-x_i)/(t_j-t_i). A robust leak
14// RATE (kB/min) immune to outlier bursts (a spike moves the mean, not the median).
15// LEAK iff MK says "significant upward trend" AND Theil-Sen rate >= a magnitude floor. Direction AND rate.
16// All INTEGER (no-float doctrine). Pure stat funcs are gate-proven on SYNTHETIC series incl a variable leak.
17// Reuses the portable seams (nx_os_proc pid-enum + nx_vsz_watchdog_core VmSize read) -> couplings=0, portable.
18// license_tier: ORIGINAL
19import "nx_syscalls.nx"
20import "nx_os_proc.nx" // osp_list_pids / osp_cmd_argv0 / osp_selfpid -- portable process seam
21import "nx_vsz_watchdog_core.nx" // vw_vmsize_kb_of -- portable, leak-free VmSize read
22
23const LC_NAME_CAP: i64 = 256 // argv0 / pid-string buffer
24const LC_DEC: i64 = 10
25const LC_ASCII_0: i64 = 48
26const LC_SLASH: i64 = 47 // '/'
27const LC_PTR_CELL: i64 = 16 // 2-i64 timespec scratch
28const LC_MS_PER_S: i64 = 1000
29const LC_NS_PER_MS: i64 = 1000000
30const LC_MS_PER_MIN: i64 = 60000 // Theil-Sen slope units: kB per minute
31// Mann-Kendall variance Var(S) = n(n-1)(2n+5)/18 -- the coefficients are mathematical (not tunable)
32const LC_MKV_2: i64 = 2 // the '2' in (2n+5) and the pair/median divisor
33const LC_MKV_5: i64 = 5 // the '5' in (2n+5)
34const LC_MKV_DEN: i64 = 18 // the /18 in Var(S)
35const LC_Z2_X1000: i64 = 2706 // (1.645)^2 x 1000 -- one-sided 95% significance bar for MK
36const LC_TAU_SCALE: i64 = 1000 // Kendall tau reported in permille
37// verdict thresholds (data-driven policy, rule 11; roadmap: LEARN these per-organ from the family tree)
38const LC_RATE_MIN: i64 = 64 // kB/min: Theil-Sen median slope must exceed this to matter (leak floor)
39const LC_MIN_N: i64 = 3 // minimum samples for a meaningful trend
40const LC_WARMUP: i64 = 1 // drop the first sample (startup transient) before analysis
41const LC_SPIKE_MAD_X1000: i64 = 4448 // Hampel 3-sigma-equiv: excursion > 4.4478*MAD (1.4826*3) = a spike
42const LC_SPIKE_ABS_MIN: i64 = 256 // kB: a spike must ALSO be materially large (flat-series noise != spike)
43const LC_FD_RATE_MIN: i64 = 2 // fd/min: a sustained +2 fd/min descriptor climb is a leak
44const LC_FD_SPIKE_MIN: i64 = 16 // fds: a spike must be at least this many descriptors
45const LC_LEVEL_OFF: i64 = 0 // level_max sentinel: sustained-high-level check DISABLED (level = baseline = R3)
46// CPU meter (analyzes the per-interval UTILIZATION series, permille of one core; HZ ticks/sec)
47const LC_UTIL_SCALE: i64 = 1000000 // util_permille = delta_ticks * SCALE / (HZ * dt_ms)
48const LC_CPU_RATE_MIN: i64 = 200 // util permille/min: an ACCELERATING burn (worsening busy-loop)
49const LC_CPU_SPIKE_MIN: i64 = 500 // util permille: a compute burst (half a core in one interval)
50const LC_CPU_BURN_MAX: i64 = 500 // util permille: SUSTAINED median >= half a core = busy-loop (power disappearing)
51const LC_CHURN_PERMILLE: i64 = 300 // MAD/median >= 30% (robust CV) = thrashing = CHURN (inefficient, off the racing line)
52const LC_MAD_FLOOR: i64 = 1 // avoid divide-by-zero when the series is dead-flat
53// anomaly classification codes (the racing-line taxonomy)
54const LC_A_HEALTHY: i64 = 0 // on the racing line: flat baseline, bounded, no trend
55const LC_A_LEAK: i64 = 1 // sustained/variable upward trend (resource never returned) -- CRITICAL
56const LC_A_SPIKE: i64 = 2 // transient outlier excursion (a "reverse leak": bursts up then returns)
57const LC_A_CHURN: i64 = 3 // high relative volatility (thrashing / inefficient design)
58// out slots for lc_classify
59const LC_O_TAU: i64 = 0 // Kendall tau permille (trend direction/consistency)
60const LC_O_RATE: i64 = 1 // Theil-Sen rate kB/min (leak magnitude)
61const LC_O_MEDIAN: i64 = 2 // robust center kB
62const LC_O_MAD: i64 = 3 // median absolute deviation kB (robust spread/volatility)
63const LC_O_PEAK: i64 = 4 // peak upward excursion above median kB
64const LC_O_CODE: i64 = 5 // LC_A_*
65const LC_O_SLOTS: i64 = 6
66// legacy aliases (leak-only callers)
67const LC_HEALTHY: i64 = 0
68const LC_LEAK: i64 = 1
69const LC_O_VERDICT: i64 = 5 // == LC_O_CODE
70
71func lc_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
72
73// leak-free monotonic ms (munmaps its scratch -- a leak-checker that leaks would be absurd)
74func lc_now_ms() -> i64 {
75 let ts: *i64 = sys_mmap(LC_PTR_CELL) as *i64
76 sys_clock_gettime_mono(ts)
77 let r: i64 = ts[0] * LC_MS_PER_S + ts[1] / LC_NS_PER_MS
78 sys_munmap(ts as *u8, LC_PTR_CELL)
79 return r
80}
81// pid int -> decimal string (leak-free)
82func lc_itoa(v: i64, out: *u8) -> i64 {
83 if v == 0 { out[0] = LC_ASCII_0 as u8; out[1] = 0 as u8; return 1 }
84 let d: *u8 = sys_mmap(LC_NAME_CAP)
85 var m: i64 = v
86 var k: i64 = 0
87 while m > 0 { d[k] = (LC_ASCII_0 + (m % LC_DEC)) as u8; m = m / LC_DEC; k = k + 1 }
88 var i: i64 = 0
89 while i < k { out[i] = d[k-1-i]; i = i + 1 }
90 out[i] = 0 as u8
91 sys_munmap(d, LC_NAME_CAP)
92 return k
93}
94// VmSize kB for a numeric pid (bridges osp's i64 pid to vw's string API; leak-free). -1 unreadable/gone.
95func lc_vsz(pid: i64) -> i64 {
96 let s: *u8 = sys_mmap(LC_NAME_CAP)
97 lc_itoa(pid, s)
98 let r: i64 = vw_vmsize_kb_of(s)
99 sys_munmap(s, LC_NAME_CAP)
100 return r
101}
102// VmRSS kB for a numeric pid (resident set -- the heap-leak meter VmSize can hide in an arena). leak-free.
103func lc_rss(pid: i64) -> i64 {
104 let s: *u8 = sys_mmap(LC_NAME_CAP)
105 lc_itoa(pid, s)
106 let r: i64 = vw_rss_kb_of(s)
107 sys_munmap(s, LC_NAME_CAP)
108 return r
109}
110// does s end with suf? (organ = basename ends ".elf")
111func lc_ends(s: *u8, suf: *u8) -> i64 {
112 let sn: i64 = lc_slen(s)
113 let fn: i64 = lc_slen(suf)
114 if sn < fn { return 0 }
115 var i: i64 = 0
116 while i < fn { if s[sn - fn + i] != suf[i] { return 0 } i = i + 1 }
117 return 1
118}
119func lc_is_organ(argv0: *u8) -> i64 { return lc_ends(argv0, ".elf" as *u8) }
120func lc_basename(path: *u8, out: *u8) -> i64 {
121 let n: i64 = lc_slen(path)
122 var s: i64 = 0
123 var i: i64 = 0
124 while i < n { if path[i] == (LC_SLASH as u8) { s = i + 1 } i = i + 1 }
125 var o: i64 = 0
126 while s < n { if o < LC_NAME_CAP - 1 { out[o] = path[s]; o = o + 1 } s = s + 1 }
127 out[o] = 0 as u8
128 return o
129}
130
131// ============ REAL DATA-SCIENCE CORE (pure, integer, no mmap -> gate-testable on synthetic series) ========
132// sign(b - a): +1 up / 0 tie / -1 down
133func lc_sign(a: i64, b: i64) -> i64 { if b > a { return 1 } if b < a { return 0 - 1 } return 0 }
134
135// MANN-KENDALL S over xs[xoff .. xoff+n): sum of sign(x_j - x_i) for all time-ordered pairs i<j.
136// Robust to VARIABLE leaks: a jagged-but-rising series still has far more up-pairs than down-pairs.
137func lc_mk_s(xs: *i64, xoff: i64, n: i64) -> i64 {
138 var s: i64 = 0
139 var i: i64 = 0
140 while i < n {
141 var j: i64 = i + 1
142 while j < n { s = s + lc_sign(xs[xoff+i], xs[xoff+j]); j = j + 1 }
143 i = i + 1
144 }
145 return s
146}
147// Kendall tau in permille = S / (n(n-1)/2). In [-1000, 1000]. +1000 = every pair rose (pure climb).
148func lc_mk_tau_permille(s: i64, n: i64) -> i64 {
149 let np: i64 = n * (n - 1) / LC_MKV_2
150 if np == 0 { return 0 }
151 return s * LC_TAU_SCALE / np
152}
153// MANN-KENDALL significant UPWARD trend? (one-sided ~95%). S>0 AND S^2/Var(S) > z^2, Var=n(n-1)(2n+5)/18.
154// Integer form (no division/sqrt): 18*1000*S^2 > 2706 * n(n-1)(2n+5).
155func lc_mk_sig_up(s: i64, n: i64) -> i64 {
156 if s <= 0 { return 0 }
157 let lhs: i64 = LC_MKV_DEN * LC_TAU_SCALE * s * s
158 let rhs: i64 = LC_Z2_X1000 * n * (n - 1) * (LC_MKV_2 * n + LC_MKV_5)
159 if lhs > rhs { return 1 }
160 return 0
161}
162// insertion sort a[0..m) ascending (no break; m small -- pairwise-slope count)
163func lc_sort(a: *i64, m: i64) -> i64 {
164 var i: i64 = 1
165 while i < m {
166 let key: i64 = a[i]
167 var j: i64 = i - 1
168 var go: i64 = 1
169 while go == 1 {
170 if j >= 0 { if a[j] > key { a[j+1] = a[j]; j = j - 1 } else { go = 0 } } else { go = 0 }
171 }
172 a[j+1] = key
173 i = i + 1
174 }
175 return 0
176}
177// THEIL-SEN slope: MEDIAN of pairwise (x_j-x_i)*60000/(t_j-t_i) over i<j. Robust leak RATE in kB/min --
178// a spike moves the mean but not the median. scratch must hold >= n(n-1)/2 i64. Returns 0 if <2 usable pairs.
179func lc_theil_sen(xs: *i64, xoff: i64, ts: *i64, toff: i64, n: i64, scratch: *i64) -> i64 {
180 var m: i64 = 0
181 var i: i64 = 0
182 while i < n {
183 var j: i64 = i + 1
184 while j < n {
185 let dt: i64 = ts[toff+j] - ts[toff+i]
186 if dt > 0 { scratch[m] = (xs[xoff+j] - xs[xoff+i]) * LC_MS_PER_MIN / dt; m = m + 1 }
187 j = j + 1
188 }
189 i = i + 1
190 }
191 if m == 0 { return 0 }
192 lc_sort(scratch, m)
193 if m % LC_MKV_2 == 1 { return scratch[m / LC_MKV_2] }
194 return (scratch[m / LC_MKV_2 - 1] + scratch[m / LC_MKV_2]) / LC_MKV_2
195}
196func lc_abs(v: i64) -> i64 { if v < 0 { return 0 - v } return v }
197// SAMPLE VALIDITY: a /proc read that returns <0 means the process was GONE at that instant. A series holding
198// any invalid sample must NOT be classified -- a dying organ's collapse reads as a huge negative slope and
199// convicts as a fake SPIKE (live-fleet finding 2026-07-16: exited torrent workers at rate=-74M kB/min).
200// Callers skip the organ (GONE, exited mid-window), never certify or convict on poisoned data. Zeros ARE
201// valid (fd_count can legitimately be 0).
202func lc_row_valid(xs: *i64, off: i64, n: i64) -> i64 {
203 var k: i64 = 0
204 while k < n { if xs[off+k] < 0 { return 0 } k = k + 1 }
205 return 1
206}
207func lc_max(xs: *i64, off: i64, n: i64) -> i64 {
208 var m: i64 = xs[off]; var k: i64 = 1
209 while k < n { if xs[off+k] > m { m = xs[off+k] } k = k + 1 }
210 return m
211}
212// robust CENTER: median of xs[off..off+n) (copies into scratch, sorts -- xs untouched)
213func lc_median(xs: *i64, off: i64, n: i64, scratch: *i64) -> i64 {
214 var k: i64 = 0
215 while k < n { scratch[k] = xs[off+k]; k = k + 1 }
216 lc_sort(scratch, n)
217 if n % LC_MKV_2 == 1 { return scratch[n / LC_MKV_2] }
218 return (scratch[n / LC_MKV_2 - 1] + scratch[n / LC_MKV_2]) / LC_MKV_2
219}
220// robust SPREAD: MAD = median(|x - med|) -- volatility immune to outliers (the spike itself doesn't inflate it)
221func lc_mad(xs: *i64, off: i64, n: i64, med: i64, scratch: *i64) -> i64 {
222 var k: i64 = 0
223 while k < n { scratch[k] = lc_abs(xs[off+k] - med); k = k + 1 }
224 lc_sort(scratch, n)
225 if n % LC_MKV_2 == 1 { return scratch[n / LC_MKV_2] }
226 return (scratch[n / LC_MKV_2 - 1] + scratch[n / LC_MKV_2]) / LC_MKV_2
227}
228// FULL ANOMALY CLASSIFICATION on a series (warmup-trimmed). Fills out[tau,rate,median,mad,peak,code], returns
229// code. The racing-line taxonomy, priority LEAK > SPIKE > CHURN > HEALTHY:
230// LEAK = Mann-Kendall significant-up trend AND Theil-Sen rate >= floor (sustained/variable climb)
231// SPIKE = peak excursion is a robust (Hampel) outlier AND materially large (a "reverse leak" burst)
232// CHURN = MAD/median relative volatility >= floor (wild thrashing / inefficient design)
233// HEALTHY= none of the above (flat, bounded -- on the racing line)
234func lc_classify_t(xs: *i64, xoff: i64, ts: *i64, toff: i64, n_total: i64, scratch: *i64, out: *i64, rate_min: i64, spike_abs_min: i64, level_max: i64) -> i64 {
235 let n: i64 = n_total - LC_WARMUP
236 if n < LC_MIN_N {
237 out[LC_O_TAU] = 0; out[LC_O_RATE] = 0; out[LC_O_MEDIAN] = 0; out[LC_O_MAD] = 0; out[LC_O_PEAK] = 0
238 out[LC_O_CODE] = LC_A_HEALTHY; return LC_A_HEALTHY
239 }
240 let xo: i64 = xoff + LC_WARMUP
241 let to: i64 = toff + LC_WARMUP
242 let med: i64 = lc_median(xs, xo, n, scratch)
243 let mad: i64 = lc_mad(xs, xo, n, med, scratch)
244 let peak: i64 = lc_max(xs, xo, n) - med
245 let s: i64 = lc_mk_s(xs, xo, n)
246 let rate: i64 = lc_theil_sen(xs, xo, ts, to, n, scratch)
247 out[LC_O_TAU] = lc_mk_tau_permille(s, n)
248 out[LC_O_RATE] = rate
249 out[LC_O_MEDIAN] = med
250 out[LC_O_MAD] = mad
251 out[LC_O_PEAK] = peak
252 var code: i64 = LC_A_HEALTHY
253 var isleak: i64 = 0
254 if lc_mk_sig_up(s, n) == 1 { if rate >= rate_min { isleak = 1 } } // accelerating/trending consumption
255 if level_max > LC_LEVEL_OFF { if med >= level_max { isleak = 1 } } // SUSTAINED-HIGH level (busy-loop held high, no trend needed)
256 if isleak == 1 { code = LC_A_LEAK } else {
257 var madf: i64 = mad
258 if madf < LC_MAD_FLOOR { madf = LC_MAD_FLOOR }
259 var isspike: i64 = 0
260 if peak * LC_TAU_SCALE > LC_SPIKE_MAD_X1000 * madf { if peak >= spike_abs_min { isspike = 1 } }
261 if isspike == 1 { code = LC_A_SPIKE } else {
262 if med > 0 { if mad * LC_TAU_SCALE / med >= LC_CHURN_PERMILLE { code = LC_A_CHURN } }
263 }
264 }
265 out[LC_O_CODE] = code
266 return code
267}
268// memory-meter defaults (kB thresholds; level check OFF -- "how much is too much" is a baseline question = R3)
269func lc_classify(xs: *i64, xoff: i64, ts: *i64, toff: i64, n_total: i64, scratch: *i64, out: *i64) -> i64 {
270 return lc_classify_t(xs, xoff, ts, toff, n_total, scratch, out, LC_RATE_MIN, LC_SPIKE_ABS_MIN, LC_LEVEL_OFF)
271}