nx_proc_snapshot.nx source
↩ module page · 246 lines · 12358 B
1// nx_proc_snapshot.nx -- ONE /proc walk answers EVERY liveness question (seq1318 root fix).
2//
3// THE DEFECT (measured 2026-07-30, source-grounded): nx_hostctl's proc_alive_by_name getdents-walks
4// all of /proc and reads every /proc/<pid>/cmdline -- and it runs ONCE PER GUARDED SERVICE PER POLL.
5// At ~25 guards x 951 processes that is ~23,775 cmdline reads per 15s poll (~1,585/s, ~4,750
6// syscalls/s) for LIVENESS ALONE, and it is a POSITIVE FEEDBACK LOOP: a fork storm inflates /proc,
7// which inflates the scan, which inflates kernel time. Measured context-switch rate 81k-93k/s.
8//
9// THE FIX IS STRUCTURAL, NOT A TUNING KNOB: walk /proc ONCE, concatenate every cmdline into one blob,
10// and answer all ~25 questions from memory with ZERO further syscalls. O(P) per poll, not O(S x P).
11// A second, free property matters as much as the speed: every guard now reads the SAME INSTANT, so
12// two guards can no longer disagree because they scanned at different times.
13//
14// THREE PROPERTIES MAKE IT SAFE BY CONSTRUCTION (not by caller discipline):
15// 1. AGE-BOUNDED. ps_alive rebuilds when the snapshot is older than PS_MAX_AGE_MS. No call site has
16// to remember to refresh; an answer can never be unboundedly stale even if a caller forgets.
17// 2. INVALIDATED ON MUTATION. ps_invalidate() after any spawn/kill forecloses the one DANGEROUS
18// staleness direction (snapshot says DEAD for a daemon that was just started -> a second spawn ->
19// EADDRINUSE churn). Stale-says-ALIVE merely defers a restart one poll, which is benign.
20// 3. FAIL-SAFE, NEVER FAIL-WRONG. Open failure or blob overflow returns PS_UNKNOWN, and the caller
21// falls back to ps_alive_direct -- the exact pre-existing scan. Worst case is today's cost; a
22// wrong answer is impossible. (A silent cap that answered "not alive" would kill live daemons.)
23//
24// ps_alive_direct is ALSO the gate's ORACLE: nx_proc_snapshot_gate proves snapshot == direct for every
25// probed name, so the fast path is verified against the slow one rather than asserted.
26// license_tier: ORIGINAL No hw writes (Rule 26). Importable (no main).
27import "nx_syscalls.nx"
28
29const PS_BLOBCAP: i64 = 4194304 // 4 MiB of cmdlines (~951 procs x ~100B measured = ~100KB; 40x headroom)
30const PS_DENTBUF: i64 = 65536 // getdents64 batch
31const PS_CLBUF: i64 = 8192 // per-pid cmdline read buffer (reused; never grows)
32const PS_PATHBUF: i64 = 256
33const PS_STATEN: i64 = 64 // state cells
34const PS_SEP: i64 = 1 // 0x01 record separator: not a legal byte of a process-name needle,
35 // so a match can NEVER span two cmdlines (span-safety by construction)
36const PS_MAX_AGE_MS: i64 = 5000 // rebuild if older (safety net; the poll refreshes explicitly)
37const PS_CONFIRM_AGE_MS: i64 = 250 // a NEGATIVE answer is only trusted from a snapshot this fresh
38const PS_UNKNOWN: i64 = 0 - 1 // "I cannot answer" -> caller MUST fall back (never guessed)
39const PS_ZERO: i64 = 48
40const PS_NINE: i64 = 57
41
42// state cells: [0]=blob len [1]=built_at_ms [2]=proc count [3]=valid(1/0) [4]=build counter
43static ps_blob: *u8
44static ps_st: *i64
45
46func ps_init() -> i64 {
47 if (ps_st as i64) == 0 {
48 ps_st = sys_mmap(PS_STATEN * 8) as *i64
49 ps_blob = sys_mmap(PS_BLOBCAP)
50 ps_st[0] = 0; ps_st[1] = 0; ps_st[2] = 0; ps_st[3] = 0; ps_st[4] = 0
51 }
52 return 0
53}
54func ps_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
55// byte-run search (same semantics as hostctl hc_contains: raw bytes incl. the cmdline's internal NULs)
56func ps_contains(hay: *u8, hn: i64, needle: *u8, nn: i64) -> i64 {
57 if nn == 0 { return 1 }
58 var i: i64 = 0
59 while i + nn <= hn {
60 var k: i64 = 0
61 var ok: i64 = 1
62 while k < nn { if hay[i+k] != needle[k] { ok = 0; k = nn } else { k = k + 1 } }
63 if ok == 1 { return 1 }
64 i = i + 1
65 }
66 return 0
67}
68// BOUNDED raw cmdline read into a caller-owned buffer. Deliberately NOT sys_read_file: that mmaps
69// 4 GiB per call and never frees, so walking /proc exhausted virtual memory and the next fork failed
70// ENOMEM -- the supervisor then could not respawn anything (the F-class root, banked in hostctl).
71func ps_read_cmdline(path: *u8, buf: *u8, cap: i64) -> i64 {
72 let fd: i64 = sys_openat_rd(path)
73 if fd < 0 { return 0 }
74 var total: i64 = 0
75 var n: i64 = sys_read(fd, buf, cap)
76 while n > 0 {
77 total = total + n
78 if total >= cap { n = 0 } else { n = sys_read(fd, (buf as i64 + total) as *u8, cap - total) }
79 }
80 sys_close(fd)
81 return total
82}
83
84// Walk /proc ONCE and fill the blob. Returns proc count, or PS_UNKNOWN on open-failure/overflow
85// (both leave valid=0 so every reader falls back to the direct scan -- fail-safe, never fail-wrong).
86func ps_build() -> i64 {
87 ps_init()
88 ps_st[3] = 0
89 let fd: i64 = sys_openat_rd("/proc" as *u8)
90 if fd < 0 { return PS_UNKNOWN }
91 let dbuf: *u8 = sys_mmap(PS_DENTBUF)
92 let path: *u8 = sys_mmap(PS_PATHBUF)
93 let clbuf: *u8 = sys_mmap(PS_CLBUF)
94 var blen: i64 = 0
95 var cnt: i64 = 0
96 var over: i64 = 0
97 var run: i64 = 1
98 while run == 1 {
99 let n: i64 = sys_getdents64(fd, dbuf, PS_DENTBUF)
100 if n <= 0 { run = 0 } else {
101 var off: i64 = 0
102 while off < n {
103 let rec: *u8 = ((dbuf as i64 + off) as *u8)
104 let reclen: i64 = dirent_reclen(rec)
105 if reclen <= 0 { off = n } else {
106 let name: *u8 = dirent_name(rec)
107 if name[0] >= (PS_ZERO as u8) { if name[0] <= (PS_NINE as u8) {
108 var p: i64 = 0
109 let pre: *u8 = "/proc/" as *u8
110 var a: i64 = 0
111 while pre[a] != (0 as u8) { path[p] = pre[a]; p = p + 1; a = a + 1 }
112 a = 0
113 while name[a] != (0 as u8) { path[p] = name[a]; p = p + 1; a = a + 1 }
114 let suf: *u8 = "/cmdline" as *u8
115 a = 0
116 while suf[a] != (0 as u8) { path[p] = suf[a]; p = p + 1; a = a + 1 }
117 path[p] = 0 as u8
118 let cln: i64 = ps_read_cmdline(path, clbuf, PS_CLBUF)
119 if cln > 0 {
120 if blen + cln + 1 > PS_BLOBCAP { over = 1; run = 0; off = n } else {
121 var c: i64 = 0
122 while c < cln { ps_blob[blen] = clbuf[c]; blen = blen + 1; c = c + 1 }
123 ps_blob[blen] = PS_SEP as u8
124 blen = blen + 1
125 cnt = cnt + 1
126 }
127 }
128 } }
129 off = off + reclen
130 }
131 }
132 }
133 }
134 sys_close(fd)
135 sys_munmap(dbuf, PS_DENTBUF)
136 sys_munmap(path, PS_PATHBUF)
137 sys_munmap(clbuf, PS_CLBUF)
138 if over == 1 { return PS_UNKNOWN }
139 ps_st[0] = blen
140 ps_st[1] = sys_now_realtime_ms()
141 ps_st[2] = cnt
142 ps_st[3] = 1
143 ps_st[4] = ps_st[4] + 1
144 return cnt
145}
146
147// Force the next question to rebuild. Call after ANY spawn/kill so a just-started daemon can never be
148// reported dead by a snapshot taken before it existed (the one dangerous staleness direction).
149func ps_invalidate() -> i64 { ps_init(); ps_st[3] = 0; return 0 }
150// Explicit per-poll refresh (the supervisor calls this once at the top of each poll).
151func ps_refresh() -> i64 { return ps_build() }
152func ps_valid() -> i64 { ps_init(); return ps_st[3] }
153func ps_count() -> i64 { ps_init(); return ps_st[2] }
154func ps_builds() -> i64 { ps_init(); return ps_st[4] } // build counter = the mechanical proof of the O(P) property
155func ps_blob_len() -> i64 { ps_init(); return ps_st[0] }
156func ps_age_ms() -> i64 {
157 ps_init()
158 if ps_st[3] == 0 { return PS_UNKNOWN }
159 return sys_now_realtime_ms() - ps_st[1]
160}
161// Rebuild iff invalid or older than max_age_ms. 1 = usable snapshot, 0 = not (caller falls back).
162func ps_fresh(max_age_ms: i64) -> i64 {
163 ps_init()
164 if ps_st[3] == 1 {
165 let age: i64 = sys_now_realtime_ms() - ps_st[1]
166 if age >= 0 { if age <= max_age_ms { return 1 } }
167 }
168 if ps_build() == PS_UNKNOWN { return 0 }
169 return 1
170}
171
172// THE SLOW ORACLE: the exact pre-existing per-name /proc scan. Kept deliberately -- it is both the
173// fallback when the snapshot cannot answer AND the gate's reference implementation.
174func ps_alive_direct(needle: *u8) -> i64 {
175 let nn: i64 = ps_slen(needle)
176 let fd: i64 = sys_openat_rd("/proc" as *u8)
177 if fd < 0 { return 0 }
178 let dbuf: *u8 = sys_mmap(PS_DENTBUF)
179 let path: *u8 = sys_mmap(PS_PATHBUF)
180 let clbuf: *u8 = sys_mmap(PS_CLBUF)
181 var alive: i64 = 0
182 var run: i64 = 1
183 while run == 1 {
184 let n: i64 = sys_getdents64(fd, dbuf, PS_DENTBUF)
185 if n <= 0 { run = 0 } else {
186 var off: i64 = 0
187 while off < n {
188 let rec: *u8 = ((dbuf as i64 + off) as *u8)
189 let reclen: i64 = dirent_reclen(rec)
190 if reclen <= 0 { off = n } else {
191 let name: *u8 = dirent_name(rec)
192 if name[0] >= (PS_ZERO as u8) { if name[0] <= (PS_NINE as u8) {
193 var p: i64 = 0
194 let pre: *u8 = "/proc/" as *u8
195 var a: i64 = 0
196 while pre[a] != (0 as u8) { path[p] = pre[a]; p = p + 1; a = a + 1 }
197 a = 0
198 while name[a] != (0 as u8) { path[p] = name[a]; p = p + 1; a = a + 1 }
199 let suf: *u8 = "/cmdline" as *u8
200 a = 0
201 while suf[a] != (0 as u8) { path[p] = suf[a]; p = p + 1; a = a + 1 }
202 path[p] = 0 as u8
203 let cln: i64 = ps_read_cmdline(path, clbuf, PS_CLBUF)
204 if cln > 0 { if ps_contains(clbuf, cln, needle, nn) == 1 { alive = 1; run = 0; off = n } }
205 } }
206 off = off + reclen
207 }
208 }
209 }
210 }
211 sys_close(fd)
212 sys_munmap(dbuf, PS_DENTBUF)
213 sys_munmap(path, PS_PATHBUF)
214 sys_munmap(clbuf, PS_CLBUF)
215 return alive
216}
217
218// Snapshot-backed liveness. PS_UNKNOWN = snapshot unusable, caller MUST fall back (ps_alive_or_direct
219// does exactly that and is what production calls).
220func ps_alive_snap(needle: *u8) -> i64 {
221 if ps_fresh(PS_MAX_AGE_MS) == 0 { return PS_UNKNOWN }
222 return ps_contains(ps_blob, ps_st[0], needle, ps_slen(needle))
223}
224// THE PRODUCTION ENTRY POINT -- ASYMMETRIC BY DESIGN, because the two answers carry very different risk:
225//
226// ALIVE is the common, cheap, SAFE answer. Acting on it means "do nothing". Even if it were a
227// moment stale, the worst case is deferring a restart by one 15s poll. Served straight from
228// the snapshot: ZERO syscalls. This is the case ~25 times per poll, so this is where the
229// whole O(S x P) -> O(P) win actually lands.
230// NOT-ALIVE is the RARE, DANGEROUS answer: it makes the supervisor KILL and SPAWN. Acting on a stale
231// negative could double-spawn a daemon that had just started (EADDRINUSE churn -- the exact
232// class this lane is trying to kill). So a negative is NEVER returned from an aged snapshot:
233// it is re-verified against one no older than PS_CONFIRM_AGE_MS.
234//
235// This is what makes the fix correct BY CONSTRUCTION rather than by an argument about call order --
236// no call site has to invalidate, and a future guard that asks twice, or a 38th spawn helper that
237// forgets ps_invalidate, still cannot be given a stale negative. Cost: the FIRST negative in a poll
238// forces one rebuild that every later negative in the same 250ms reuses -- so a poll costs ~1 walk
239// when everything is up and ~2 when something is down, versus ~25 before.
240func ps_alive_or_direct(needle: *u8) -> i64 {
241 let r: i64 = ps_alive_snap(needle)
242 if r == 1 { return 1 }
243 if r == PS_UNKNOWN { return ps_alive_direct(needle) }
244 if ps_fresh(PS_CONFIRM_AGE_MS) == 0 { return ps_alive_direct(needle) }
245 return ps_contains(ps_blob, ps_st[0], needle, ps_slen(needle))
246}