nx_resgov_core.nx source
↩ module page · 220 lines · 12946 B
1// nx_resgov_core.nx -- PURE DECISION CORE of the unified resource-governance regime (nx_resgov).
2//
3// WHY THIS SUPERSEDES THE VSZ WATCHDOG (operator 2026-08-04: "resource management should be proactive,
4// intelligent and SOTA, not watching the hardware get hammered"; debt 1785863639). The incumbent breaker
5// had three structural defects, each fixed here BY CONSTRUCTION:
6// 1. WRONG INSTRUMENT. It thresholded on VSZ. VSZ is reserved ADDRESS SPACE -- nx_wiki_gw sits at 47.4GB
7// VSZ on 355MB resident and is perfectly healthy; a naive VSZ rule kills it. The pressure that
8// actually degraded the estate on 08-04 was RESIDENT memory + SWAP. We meter VmRSS + VmSwap.
9// 2. COVERAGE BY ALLOW-LIST. Rows named 7 remembered processes; every grower nobody listed was
10// invisible (a transient 4GB nx_web_shard_co appeared and exited mid-session -- the population is
11// dynamic and unlistable by hand). Here the policy is INVERTED: a DEFAULT cap covers EVERY process
12// and exemptions are the explicit, auditable rows. ★A DETECTOR'S COVERAGE IS TWO SETS -- WHAT IT
13// MATCHES AND WHERE IT LOOKS; a default-deny breaker has no blind names.
14// 3. REACTIVE ONLY. It fired after a ceiling was crossed -- i.e. after the hardware was already being
15// hammered. Here the primary signal is PREDICTIVE: growth VELOCITY from persisted samples gives an
16// ETA to the ceiling, and the ladder engages while headroom still exists.
17//
18// ★ESCALATION, NEVER A BINARY KILL (rule 14, graceful degradation). Levels: OK -> WATCH (measured, logged)
19// -> WARN (predicted to breach inside the horizon) -> RENICE (deprioritise, let it finish) -> TERM (polite,
20// guard respawns fresh) -> KILL (last resort). A breaker whose only verb is SIGKILL cannot be trusted to
21// run every minute, so it gets disabled -- which is exactly how the incumbent came to be unwired.
22//
23// ★SYSTEM PRESSURE GATES ACTION. A 3GB process on an idle box is not a problem; the same process at 95%
24// memory is. Every destructive level requires BOTH a per-process finding AND real system scarcity, so a
25// healthy fleet is never disturbed.
26//
27// FAIL-SAFE BY CONSTRUCTION: unknown/unreadable measurement -> level OK (never act on ignorance);
28// exempt -> capped at OK; cap<1 -> inert row; cooldown suppresses storms; the CLI never touches pid<=300
29// or itself. Decisions live here (gate-locked, pure); the /proc walk, state and signals live in the CLI.
30// license_tier: ORIGINAL No hw writes (Rule 26).
31import "nx_syscalls.nx"
32const RG_MAGIC_1048576: i64 = 1048576
33
34// ---- levels ------------------------------------------------------------------------------------
35const RG_OK: i64 = 0
36const RG_WATCH: i64 = 1
37const RG_WARN: i64 = 2
38const RG_RENICE: i64 = 3
39const RG_TERM: i64 = 4
40const RG_KILL: i64 = 5
41
42// ---- system pressure ---------------------------------------------------------------------------
43// headroom in permil of total (1000 = all free). -1 when unmeasurable -> callers treat as NO pressure.
44func rg_headroom_permil(avail_kb: i64, total_kb: i64) -> i64 {
45 if total_kb <= 0 { return 0 - 1 }
46 if avail_kb < 0 { return 0 - 1 }
47 return avail_kb * 1000 / total_kb
48}
49// swap consumed in permil. -1 unmeasurable / no swap configured.
50func rg_swap_permil(used_kb: i64, total_kb: i64) -> i64 {
51 if total_kb <= 0 { return 0 - 1 }
52 if used_kb < 0 { return 0 - 1 }
53 return used_kb * 1000 / total_kb
54}
55// Is the SYSTEM actually scarce? 1 only when memory headroom is below its floor OR swap is above its
56// ceiling. Unmeasurable inputs (-1) are NOT scarcity -- ignorance must never authorise a kill.
57func rg_system_pressured(headroom_permil: i64, swap_permil: i64, head_floor: i64, swap_ceil: i64) -> i64 {
58 if headroom_permil >= 0 { if headroom_permil < head_floor { return 1 } }
59 if swap_permil >= 0 { if swap_permil > swap_ceil { return 1 } }
60 return 0
61}
62
63// ---- growth velocity (the proactive signal) ----------------------------------------------------
64// kB per second between two samples of the SAME process identity. 0 when no usable prior, when the
65// clock did not advance, or when the process SHRANK (a shrink is not negative growth to act on).
66// ⚠Identity is the caller's job: pid alone is reusable, so the CLI keys samples on pid+starttime.
67func rg_velocity_kb_s(prev_kb: i64, prev_t: i64, now_kb: i64, now_t: i64) -> i64 {
68 if prev_kb <= 0 { return 0 }
69 if prev_t <= 0 { return 0 }
70 if now_t <= prev_t { return 0 }
71 if now_kb <= prev_kb { return 0 }
72 return (now_kb - prev_kb) / (now_t - prev_t)
73}
74// seconds until `now_kb` reaches `ceil_kb` at `vel_kb_s`. -1 = not on course (not growing, or already
75// at/over the ceiling -- an already-breached process is the cap rule's business, not the forecast's).
76func rg_eta_s(now_kb: i64, ceil_kb: i64, vel_kb_s: i64) -> i64 {
77 if vel_kb_s <= 0 { return 0 - 1 }
78 if ceil_kb <= 0 { return 0 - 1 }
79 if now_kb >= ceil_kb { return 0 - 1 }
80 return (ceil_kb - now_kb) / vel_kb_s
81}
82
83// ---- the decision ------------------------------------------------------------------------------
84// footprint = the resource that actually hurts: resident + swapped-out pages of ONE process.
85func rg_footprint_kb(rss_kb: i64, swap_kb: i64) -> i64 {
86 var t: i64 = 0
87 if rss_kb > 0 { t = t + rss_kb }
88 if swap_kb > 0 { t = t + swap_kb }
89 return t
90}
91
92// THE level decision. Pure; every input measured, every branch fail-safe.
93// exempt=1 -> OK always (an exemption is a promise, honoured unconditionally)
94// cap_gb < 1 -> OK (inert row; never a 0-threshold kill-everything)
95// footprint unmeasurable (<=0) -> OK (never act on ignorance)
96// over cap + system pressured -> TERM, or KILL if it ALSO already had a TERM this window
97// over cap, system calm -> WATCH (recorded, not acted on -- big is not the same as harmful)
98// ETA inside act horizon + pressured -> RENICE (proactive: slow it while headroom remains)
99// ETA inside warn horizon -> WARN (logged forecast; the operator-visible early signal)
100func rg_level(exempt: i64, cap_gb: i64, rss_kb: i64, swap_kb: i64, vel_kb_s: i64,
101 system_pressured: i64, already_termed: i64,
102 eta_warn_s: i64, eta_act_s: i64) -> i64 {
103 if exempt == 1 { return RG_OK }
104 if cap_gb < 1 { return RG_OK }
105 let foot: i64 = rg_footprint_kb(rss_kb, swap_kb)
106 if foot <= 0 { return RG_OK }
107 let ceil_kb: i64 = cap_gb * RG_MAGIC_1048576
108 if foot > ceil_kb {
109 if system_pressured == 1 {
110 if already_termed == 1 { return RG_KILL }
111 return RG_TERM
112 }
113 return RG_WATCH
114 }
115 let eta: i64 = rg_eta_s(foot, ceil_kb, vel_kb_s)
116 if eta >= 0 {
117 if eta <= eta_act_s { if system_pressured == 1 { return RG_RENICE } return RG_WARN }
118 if eta <= eta_warn_s { return RG_WARN }
119 }
120 return RG_OK
121}
122
123// ---- AGGREGATE: the decision NO per-process rule can reach --------------------------------------
124// ***PRESSURE WITHOUT A NAMED CULPRIT PRODUCES NO ACTION.*** rg_level above is per-process, and EVERY
125// branch of it that acts requires that ONE process to be over ITS OWN cap. That is correct, and it is
126// not sufficient. MEASURED 2026-08-06: the estate sat at swap_permil=822 (94pc of a 23GB swap consumed)
127// with FOURTEEN leak suspects, each individually UNDER the 6GB default cap. Their sum was the whole
128// 23GB. Findings: ZERO. This governor scanned 504 processes and reported acted=0 every minute -- and it
129// was RIGHT every minute -- while the live edge stalled and nishifamily.com/writer TIMED OUT for the
130// operator. A verdict can be correct at every step and still be useless.
131// ***A GOVERNOR THAT ONLY ASKS "IS ANY ONE PROCESS TOO BIG?" IS BLIND TO DEATH BY A THOUSAND CUTS***,
132// and diffuse accumulation is exactly the shape a leak epidemic takes: debt 1941 measured 16942
133// unbalanced sys_mmap sites across 13.6pc of ALL functions in the ecosystem. The leak is everywhere,
134// so it is nowhere in particular, so no per-process cap can ever name it.
135// ***FORECASTING A FUTURE BREACH DOES NOTHING ABOUT MEMORY ALREADY PARKED IN SWAP.*** The predictive
136// ladder governs RATE. This governs accumulated DEBT. They are different quantities and the second one
137// had no verb at all -- which is why the box could sit at 94pc swap indefinitely, every process legal,
138// the governor reporting all-clear, until something user-visible broke.
139// THE RULE: when scarcity is MEASURED and enough suspects exist to explain it, name the largest
140// NON-EXEMPT holder and recycle it, even though it is individually legal. This is the industry answer
141// (Apache MaxRequestsPerChild, systemd RuntimeMaxSec, k8s memory-limit eviction): you do not wait for
142// a breach, you recycle accumulated debt on a policy.
143// FAIL-SAFE BY CONSTRUCTION: OK unless scarcity is MEASURED (never on -1); OK unless the suspect count
144// clears an explicit floor (so a single fat-but-fine process never trips it); OK unless the nominee is
145// itself above an absolute floor (never recycle something too small to matter); OK if the nominee is
146// exempt (an exemption is a promise, honoured unconditionally, aggregate or not). It nominates AT MOST
147// ONE process per sweep, so it can never cascade, and its scarcity bar is DELIBERATELY STRICTER than
148// the per-process one -- recycling is for real distress, not for ordinary busy.
149func rg_aggregate_level(headroom_permil: i64, swap_permil: i64,
150 agg_head_floor: i64, agg_swap_ceil: i64,
151 suspects: i64, suspect_floor: i64,
152 worst_foot_kb: i64, worst_exempt: i64, worst_recyclable: i64,
153 min_act_kb: i64, already_termed: i64) -> i64 {
154 if worst_exempt == 1 { return RG_OK }
155 if suspect_floor < 1 { return RG_OK }
156 if suspects < suspect_floor { return RG_OK }
157 if min_act_kb < 1 { return RG_OK }
158 if worst_foot_kb < min_act_kb { return RG_OK }
159 if rg_system_pressured(headroom_permil, swap_permil, agg_head_floor, agg_swap_ceil) == 0 { return RG_OK }
160 // ***NEVER TERMINATE WHAT NOTHING WILL RESTART.*** The estate banked this law in resgov.conf on
161 // 2026-08-04 over nx_hub_gw: it is the biggest slow leaker on the box, but daemons.reg registers it
162 // arm=WATCH, so nothing would bring it back -- A DEATH-DECIDER THAT OUTRUNS ITS RESPAWNER TURNS A
163 // LEAK INTO AN OUTAGE. Coverage-by-default is right for DETECTION and wrong for DESTRUCTION, so the
164 // two are split here: every non-exempt holder can be NAMED, only a declared restart-safe one can be
165 // recycled. An unguarded nominee returns WATCH -- reported loudly, every sweep, acted on never.
166 // ***EXEMPT FROM DESTRUCTION IS NOT EXEMPT FROM SCRUTINY***, and the inverse of a blind allow-list
167 // is not an unconditional licence to kill.
168 if worst_recyclable == 0 { return RG_WATCH }
169 if already_termed == 1 { return RG_KILL }
170 return RG_TERM
171}
172// cooldown: 1 = allowed to act again. A destructive level must never repeat inside its window.
173func rg_cooldown_ok(last_action_s: i64, now_s: i64, cooldown_s: i64) -> i64 {
174 if last_action_s <= 0 { return 1 }
175 if now_s - last_action_s >= cooldown_s { return 1 }
176 return 0
177}
178
179// level -> stable label (structured logging, rule 18: what happened and why)
180func rg_level_name(l: i64) -> *u8 {
181 if l == RG_OK { return "OK" as *u8 }
182 if l == RG_WATCH { return "WATCH" as *u8 }
183 if l == RG_WARN { return "WARN" as *u8 }
184 if l == RG_RENICE { return "RENICE" as *u8 }
185 if l == RG_TERM { return "TERM" as *u8 }
186 if l == RG_KILL { return "KILL" as *u8 }
187 return "?" as *u8
188}
189// is this level destructive (needs pressure + cooldown + a real signal)?
190func rg_is_destructive(l: i64) -> i64 {
191 if l == RG_TERM { return 1 }
192 if l == RG_KILL { return 1 }
193 return 0
194}
195
196// ---- /proc/meminfo parsing ---------------------------------------------------------------------
197// kB for a "<Label>:" row in a meminfo-shaped buffer. -1 absent (never 0 -- absent and zero differ).
198func rg_meminfo_kb(buf: *u8, n: i64, pat: *u8) -> i64 {
199 var pl: i64 = 0
200 while pat[pl] != (0 as u8) { pl = pl + 1 }
201 var i: i64 = 0
202 while i + pl <= n {
203 var k: i64 = 0
204 var ok: i64 = 1
205 while k < pl { if buf[i+k] != pat[k] { ok = 0; k = pl } k = k + 1 }
206 if ok == 1 {
207 var j: i64 = i + pl
208 var go: i64 = 1
209 while go == 1 { go = 0; if j < n { let c: i64 = buf[j] as i64; if c == 32 { j = j + 1; go = 1 } else { if c == 9 { j = j + 1; go = 1 } } } }
210 var v: i64 = 0
211 var any: i64 = 0
212 go = 1
213 while go == 1 { go = 0; if j < n { let c2: i64 = buf[j] as i64; if c2 >= 48 { if c2 <= 57 { v = v*10 + (c2-48); any = 1; j = j + 1; go = 1 } } } }
214 if any == 0 { return 0 - 1 }
215 return v
216 }
217 i = i + 1
218 }
219 return 0 - 1
220}