code wiki / _hdl_build / nx_build_admit.nx
nx_build_admit.nx source
↩ module page · 298 lines · 16307 B
1// nx_build_admit.nx -- BUILD ADMISSION CONTROL (the missing gate that let the fleet wedge the host).
2// INCIDENT 2026-07-20: ~10 concurrent sessions were compiling at once (registry grew 198->236 GREEN tools in
3// ONE session). Every /api/build forks nx_cc_sovereign (549KB) on a memory-limited Synology and NOTHING
4// serialized or admitted them -> the NAS went userspace-wedged (sshd could not complete a banner, DSM could
5// not serve a page, every sovereign daemon refused TCP) and then off the network entirely. Measured, not
6// guessed. nx_swarm_admit ALREADY documents itself as the gate "BEFORE any heavy launch (model decode, codec,
7// image-gen, BIG BUILD)" -- but /api/build never called it. A compile IS a heavy launch. This organ is the
8// build-shaped admission verdict, with ZERO deps beyond nx_syscalls so it can never itself be the thing that
9// fails under load.
10//
11// LIAR-KILLED: every number is read live from /proc on the host being protected -- no estimate, no cache.
12// MemAvailable from /proc/meminfo (the honest figure: reclaimable included, unlike MemFree)
13// 1-minute load from /proc/loadavg, carried as CENTI-load (integer; no floats, sovereign law)
14//
15// nx_build_admit check [floor_mb] [max_centiload]
16// exit 0 GRANT -- headroom exists, compile now
17// exit 3 DENY-LOAD -- below the memory floor; compiling now risks wedging the HOST, not just the build
18// exit 4 QUEUE -- load above the ceiling; hand to nx_orchestrate wait-for-opening and come back
19// exit 2 usage | 5 unreadable /proc (fail-CLOSED: cannot measure => cannot admit)
20//
21// COMPOSES (does not duplicate): the QUEUE verdict is designed to be absorbed by nx_orchestrate's
22// wait-for-opening deploy queue -- the same anti-collision primitive shipped this session, pointed at COMPUTE
23// instead of deploys. The structural half is mgmt-owned: ma_do_build must consult this BEFORE forking the
24// compiler (filed rung) -- until then this is the session-callable discipline half, same two-tier pattern as
25// nx_route_diff/nx_tooldiff (detector live, enforcement filed).
26// FAIL-CLOSED BY CONSTRUCTION: an unreadable /proc returns exit 5, never GRANT. A guard that cannot measure
27// must refuse, never wave through -- the same law as an instrument that must not score bytes it did not fetch.
28// ENVELOPE (declared in output): 64KB per /proc read; thresholds are NAMED CONSTS (rule-11) and argv-overridable.
29// license_tier: ORIGINAL No hw writes (Rule 26). expect_exit: 0
30import "nx_syscalls.nx"
31
32const BA_MEMINFO: *u8 = "/proc/meminfo" as *u8
33const BA_LOADAVG: *u8 = "/proc/loadavg" as *u8
34const BA_READCAP: i64 = 65536
35const BA_OUTCAP: i64 = 4096
36// DEFAULTS (rule-11: named, argv-overridable, never buried). Floor chosen from the incident: the host wedged
37// while userspace could not fork -- 512MB available is the point below which a 549KB compiler plus its
38// transient .s output stops being safe on this box.
39const BA_FLOOR_MB: i64 = 512
40const BA_MAX_CENTILOAD: i64 = 400
41// ---- HOST-DERIVED LOAD CEILING (seq1475, 2026-07-30) ------------------------------------------
42// WHY THIS EXISTS: the memory floor above is NECESSARY BUT NOT SUFFICIENT, measured. On 2026-07-30 the
43// hub flapped -- nx_tools_api_serve down, then nx_mgmt_api down, both self-recovering -- while this
44// detector was GRANTing at mem_available_mb=28437 against a 1024 floor. MEMORY WAS NEVER SCARCE, so
45// the seq708 "OOM" attribution is at best incomplete and a memory-only gate cannot see this failure.
46// The QUEUE verdict for load already existed and was DISABLED IN PRACTICE (callers passed a ceiling of
47// 100000 so memory was the sole gate) -- so the instrument could already measure the thing it was not
48// allowed to act on.
49//
50// ★A LOAD CEILING MUST BE HOST-DERIVED, NOT A CONSTANT (bounds-derived-adaptive-never-taste). "load 4"
51// means saturated on a 4-core box and idle on a 32-core one, so a baked number is wrong everywhere
52// except where it was written. Derive it from the CPU COUNT OF THE RUNNING HOST -- the same law as
53// deriving a supervisor's port table from the running system rather than from a comment.
54//
55// THE MULTIPLIER IS EVIDENCE-BRACKETED, NOT CHOSEN: this host has 8 CPUs (counted from /proc/stat).
56// FLAPPING was observed at load1 = 11.32 => 1.42 x ncpu
57// HEALTHY was observed at load1 = 3.88 => 0.49 x ncpu
58// 1.00 x ncpu sits between a measured-healthy point and a measured-flapping point, and is also the
59// classic saturation definition (runnable threads == cores). It is a BRACKET, not a guess; when more
60// flap/health samples land, tighten it with them and say so.
61const BA_STAT: *u8 = "/proc/stat" as *u8
62// Byte length of the needle "procs_running" -- the offset from the needle start to its integer field in
63// /proc/stat, same convention as the "MemAvailable:"+13 read above. Named, never buried (rule 11).
64const BA_PROCSRUN_OFF: i64 = 13
65const BA_CENTI_PER_CPU: i64 = 100 // 1.00 x ncpu, in centi-load
66const BA_NCPU_FALLBACK: i64 = 4 // only if /proc/stat is unreadable; conservative, never unbounded
67
68// count "cpuN " lines in /proc/stat (the aggregate "cpu " line has a SPACE at index 3, per-cpu lines a DIGIT)
69func ba_ncpu(buf: *u8, n: i64) -> i64 {
70 var c: i64 = 0
71 var i: i64 = 0
72 while i < n {
73 var bol: i64 = 0
74 if i == 0 { bol = 1 }
75 if i > 0 { if buf[i-1] == (10 as u8) { bol = 1 } }
76 if bol == 1 {
77 if i + 4 <= n {
78 if buf[i] == (99 as u8) {
79 if buf[i+1] == (112 as u8) {
80 if buf[i+2] == (117 as u8) {
81 let d: i64 = buf[i+3] as i64
82 if d >= 48 { if d <= 57 { c = c + 1 } }
83 }
84 }
85 }
86 }
87 }
88 i = i + 1
89 }
90 return c
91}
92const BA_KB_PER_MB: i64 = 1024
93const BA_CENTI: i64 = 100
94const BA_STDOUT: i64 = 1
95const BA_STDERR: i64 = 2
96const BA_ZERO: i64 = 48
97const BA_NINE: i64 = 57
98const BA_DOT: i64 = 46
99const BA_NL: i64 = 10
100const BA_SP: i64 = 32
101const BA_EXIT_USAGE: i64 = 2
102const BA_EXIT_DENY: i64 = 3
103const BA_EXIT_QUEUE: i64 = 4
104const BA_EXIT_UNREADABLE: i64 = 5
105
106// ---- PURE VERDICT (extracted 2026-07-30) ------------------------------------------------------
107// The whole admission DECISION as a pure function of already-measured numbers, extracted for exactly the
108// reason hc_keep_delay was: a policy welded into main() beside its own /proc reads CANNOT BE TESTED without
109// a host in the required state, and the two states that matter here -- CPU-saturated vs I/O-saturated --
110// cannot be summoned on demand on a shared box. As a pure function every quadrant is gateable with no /proc.
111// Returns the process exit code: 0 GRANT / BA_EXIT_DENY memory floor / BA_EXIT_QUEUE load ceiling.
112// ORDER IS DELIBERATE: the memory floor is the ONLY unconditional block (it is the measured 2026-07-20 wedge
113// cause, where userspace could not fork at all) so it is checked FIRST -- no run-queue reading may ever admit
114// a build below it. procs_run < 0 means the run-queue sensor was UNREADABLE, and that must NOT unlock the
115// gate: fail-closed, matching how the loadavg and meminfo reads above refuse rather than assume.
116func ba_verdict(avail_mb: i64, floor_mb: i64, load1: i64, max_load: i64, procs_run: i64, ncpu: i64) -> i64 {
117 if avail_mb < floor_mb { return BA_EXIT_DENY }
118 if load1 <= max_load { return 0 }
119 if procs_run >= 0 { if procs_run < ncpu { return 0 } }
120 return BA_EXIT_QUEUE
121}
122
123func ba_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
124func ba_werr(s: *u8) -> i64 { sys_write(BA_STDERR, s, ba_slen(s)); return 0 }
125func ba_puts(b: *u8, off: i64, s: *u8) -> i64 { var o: i64 = off; var j: i64 = 0; while s[j] != (0 as u8) { b[o] = s[j]; o = o + 1; j = j + 1 } return o }
126func ba_puti(b: *u8, off: i64, v: i64) -> i64 {
127 var o: i64 = off
128 var m: i64 = v
129 if m < 0 { b[o] = 45 as u8; o = o + 1; m = 0 - m }
130 let t: *u8 = sys_mmap(28)
131 var k: i64 = 0
132 if m == 0 { t[0] = BA_ZERO as u8; k = 1 }
133 while m > 0 { t[k] = (BA_ZERO + (m % 10)) as u8; m = m / 10; k = k + 1 }
134 var i: i64 = 0
135 while i < k { b[o] = t[k - 1 - i]; o = o + 1; i = i + 1 }
136 return o
137}
138func ba_read(path: *u8, buf: *u8, cap: i64) -> i64 {
139 let fd: i64 = sys_openat_rd(path)
140 if fd < 0 { return 0 - 1 }
141 var n: i64 = 0
142 var go: i64 = 1
143 while go == 1 { let r: i64 = sys_read(fd, (buf as i64 + n) as *u8, cap - n); if r <= 0 { go = 0 } else { n = n + r } if n >= cap { go = 0 } }
144 sys_close(fd)
145 return n
146}
147// index of NUL-term needle in buf[0,n), or -1
148func ba_find(buf: *u8, n: i64, needle: *u8) -> i64 {
149 let m: i64 = ba_slen(needle)
150 if m == 0 { return 0 - 1 }
151 var i: i64 = 0
152 while i + m <= n {
153 var j: i64 = 0
154 var ok: i64 = 1
155 while j < m { if buf[i+j] != needle[j] { ok = 0; j = m } else { j = j + 1 } }
156 if ok == 1 { return i }
157 i = i + 1
158 }
159 return 0 - 1
160}
161// parse the first unsigned integer at/after p; -1 if none before end-of-line
162func ba_uint_at(buf: *u8, p: i64, n: i64) -> i64 {
163 var i: i64 = p
164 while i < n {
165 let c: i64 = buf[i] as i64
166 if c == BA_NL { return 0 - 1 }
167 if c >= BA_ZERO { if c <= BA_NINE {
168 var v: i64 = 0
169 while i < n {
170 let d: i64 = buf[i] as i64
171 if d < BA_ZERO { i = n } else { if d > BA_NINE { i = n } else { v = v * 10 + (d - BA_ZERO); i = i + 1 } }
172 }
173 return v
174 } }
175 i = i + 1
176 }
177 return 0 - 1
178}
179// parse "N.MM" at the head of buf as centi-units (0.52 -> 52); -1 if unparseable
180func ba_centi_head(buf: *u8, n: i64) -> i64 {
181 if n <= 0 { return 0 - 1 }
182 var i: i64 = 0
183 var whole: i64 = 0
184 var got: i64 = 0
185 while i < n {
186 let c: i64 = buf[i] as i64
187 if c >= BA_ZERO { if c <= BA_NINE { whole = whole * 10 + (c - BA_ZERO); got = 1; i = i + 1 } else { i = n } } else { i = n }
188 }
189 if got == 0 { return 0 - 1 }
190 // re-scan for the fractional part (the loop above stopped at the dot or a space)
191 var frac: i64 = 0
192 var fi: i64 = 0
193 var k: i64 = 0
194 while k < n { if buf[k] == (BA_DOT as u8) { fi = k + 1; k = n } else { k = k + 1 } }
195 if fi > 0 {
196 var d1: i64 = 0
197 var d2: i64 = 0
198 if fi < n { let a: i64 = buf[fi] as i64; if a >= BA_ZERO { if a <= BA_NINE { d1 = a - BA_ZERO } } }
199 if fi + 1 < n { let b: i64 = buf[fi+1] as i64; if b >= BA_ZERO { if b <= BA_NINE { d2 = b - BA_ZERO } } }
200 frac = d1 * 10 + d2
201 }
202 return whole * BA_CENTI + frac
203}
204func main(argc: i64, argv: *i64) -> i64 {
205 var floor_mb: i64 = BA_FLOOR_MB
206 var max_load: i64 = BA_MAX_CENTILOAD
207 if argc < 2 { ba_werr("usage: nx_build_admit check [floor_mb] [max_centiload]\n" as *u8); sys_exit(BA_EXIT_USAGE); return BA_EXIT_USAGE }
208 let verb: *u8 = argv[1] as *u8
209 if verb[0] != (99 as u8) { ba_werr("usage: nx_build_admit check [floor_mb] [max_centiload]\n" as *u8); sys_exit(BA_EXIT_USAGE); return BA_EXIT_USAGE }
210 if argc > 2 { let a: *u8 = argv[2] as *u8; let v: i64 = ba_uint_at(a, 0, ba_slen(a)); if v > 0 { floor_mb = v } }
211 if argc > 3 { let b: *u8 = argv[3] as *u8; let v2: i64 = ba_uint_at(b, 0, ba_slen(b)); if v2 > 0 { max_load = v2 } }
212
213 let mbuf: *u8 = sys_mmap(BA_READCAP)
214 let mn: i64 = ba_read(BA_MEMINFO, mbuf, BA_READCAP)
215 if mn <= 0 {
216 ba_werr("BUILD-ADMIT verdict=UNREADABLE /proc/meminfo -- cannot measure, therefore REFUSING to admit (fail-closed)\n" as *u8)
217 sys_exit(BA_EXIT_UNREADABLE)
218 return BA_EXIT_UNREADABLE
219 }
220 let mi: i64 = ba_find(mbuf, mn, "MemAvailable:" as *u8)
221 var avail_mb: i64 = 0 - 1
222 if mi >= 0 { let kb: i64 = ba_uint_at(mbuf, mi + 13, mn); if kb >= 0 { avail_mb = kb / BA_KB_PER_MB } }
223 if avail_mb < 0 {
224 ba_werr("BUILD-ADMIT verdict=UNREADABLE MemAvailable absent -- fail-closed, not admitted\n" as *u8)
225 sys_exit(BA_EXIT_UNREADABLE)
226 return BA_EXIT_UNREADABLE
227 }
228 // HOST-DERIVED CEILING (seq1475): if the caller did not pass an explicit max_centiload, compute it
229 // from THIS host's CPU count instead of using the baked constant. An explicit argv override still
230 // wins -- a caller that deliberately states a ceiling is making a decision, and this must not
231 // silently overrule it. Unreadable /proc/stat falls back to a conservative ncpu, never to unbounded.
232 var ncpu: i64 = 0
233 let sbuf: *u8 = sys_mmap(BA_READCAP)
234 let sn: i64 = ba_read(BA_STAT, sbuf, BA_READCAP)
235 if sn > 0 { ncpu = ba_ncpu(sbuf, sn) }
236 if ncpu <= 0 { ncpu = BA_NCPU_FALLBACK }
237 if argc <= 3 { max_load = ncpu * BA_CENTI_PER_CPU }
238 // CPU-ONLY CONFIRMATION (2026-07-30, from measurement, not irritation). /proc/loadavg field 1 counts tasks
239 // in R *and* D state, so a disk-bound workload reads as CPU saturation and a CPU-bound compile is refused
240 // while cores sit idle. MEASURED LIVE ON THIS HOST: load1=8.33, which the OS itself split into IO=3.75 and
241 // CPU=4.58 across 8 cores -- 43% of the CPU idle, build refused by 0.33 of load that was disk wait.
242 // nx_procchurn agreed independently (procs_blocked>=1 with procs_running 3-6: "I/O or lock bound, NOT compute").
243 // procs_running from /proc/stat is the TRUE run queue (R only, D excluded), so a QUEUE verdict is now
244 // CONFIRMED against it before we refuse. This can only ever admit MORE builds, never fewer, and only when
245 // the run queue itself says the CPUs are free. The memory floor (DENY-MEM) is untouched -- that was the
246 // actual 2026-07-20 wedge cause and it remains the one unconditional block.
247 var procs_run: i64 = 0 - 1
248 if sn > 0 {
249 let pri: i64 = ba_find(sbuf, sn, "procs_running" as *u8)
250 if pri >= 0 { procs_run = ba_uint_at(sbuf, pri + BA_PROCSRUN_OFF, sn) }
251 }
252 let lbuf: *u8 = sys_mmap(BA_READCAP)
253 let ln: i64 = ba_read(BA_LOADAVG, lbuf, BA_READCAP)
254 var load1: i64 = 0 - 1
255 if ln > 0 {
256 var w: i64 = 0
257 while w < ln { if lbuf[w] == (BA_SP as u8) { w = ln } else { w = w + 1 } }
258 load1 = ba_centi_head(lbuf, w)
259 }
260 if load1 < 0 {
261 ba_werr("BUILD-ADMIT verdict=UNREADABLE /proc/loadavg -- fail-closed, not admitted\n" as *u8)
262 sys_exit(BA_EXIT_UNREADABLE)
263 return BA_EXIT_UNREADABLE
264 }
265 let out: *u8 = sys_mmap(BA_OUTCAP)
266 var o: i64 = 0
267 o = ba_puts(out, o, "BUILD-ADMIT mem_available_mb=" as *u8)
268 o = ba_puti(out, o, avail_mb)
269 o = ba_puts(out, o, " floor_mb=" as *u8)
270 o = ba_puti(out, o, floor_mb)
271 o = ba_puts(out, o, " load1_centi=" as *u8)
272 o = ba_puti(out, o, load1)
273 o = ba_puts(out, o, " max_centiload=" as *u8)
274 o = ba_puti(out, o, max_load)
275 o = ba_puts(out, o, " ncpu=" as *u8)
276 o = ba_puti(out, o, ncpu)
277 o = ba_puts(out, o, " procs_running=" as *u8)
278 o = ba_puti(out, o, procs_run)
279 o = ba_puts(out, o, " (envelope: 64KB per /proc read; every figure measured live from /proc on THIS host, thresholds are named consts and argv-overridable)\n" as *u8)
280 // ONE decision, ONE implementation -- main now only RENDERS what ba_verdict decided (gated above).
281 let verdict: i64 = ba_verdict(avail_mb, floor_mb, load1, max_load, procs_run, ncpu)
282 if verdict == BA_EXIT_DENY {
283 o = ba_puts(out, o, "VERDICT=DENY-LOAD below the memory floor -- forking the compiler now risks wedging the HOST, not merely failing the build (incident 2026-07-20: userspace could not fork, sshd and DSM both stopped serving). Wait, or raise the floor deliberately with evidence.\n" as *u8)
284 sys_write(BA_STDOUT, out, o)
285 sys_exit(BA_EXIT_DENY)
286 return BA_EXIT_DENY
287 }
288 if verdict == BA_EXIT_QUEUE {
289 o = ba_puts(out, o, "VERDICT=QUEUE load above the ceiling AND the run queue CONFIRMS CPU saturation (procs_running >= ncpu) -- hand this build to the nx_orchestrate wait-for-opening queue and let it fire when the opening appears. Do NOT spin-retry.\n" as *u8)
290 sys_write(BA_STDOUT, out, o)
291 sys_exit(BA_EXIT_QUEUE)
292 return BA_EXIT_QUEUE
293 }
294 o = ba_puts(out, o, "VERDICT=GRANT headroom present, compile now.\n" as *u8)
295 sys_write(BA_STDOUT, out, o)
296 sys_exit(0)
297 return 0
298}