nx_health_probe.nx source
↩ module page · 234 lines · 13144 B
1// nx_health_probe.nx -- SERVING health probe (closes hosting_research gap #5 "health-probe-not-liveness",
2// 3/0 CONFIRMED: a service running-but-not-serving is worse than crashed; nx_hostctl only /proc-name-scans).
3// Black-box probe: connect + send a request + bounded recv. Distinguishes the 4 real states a supervisor
4// must act on differently -- crucially HUNG (process up + port bound but NOT responding), which PID-liveness
5// (systemd) and even a LISTEN check (nx_port_audit) both MISS. No daemon cooperation needed (unlike systemd
6// sd_notify WATCHDOG); same black-box idea as a k8s liveness probe but sovereign + composes with the
7// restart-guard + reap + listen audits into one supervision-health story.
8//
9// module: nishi-core.supervision.health_probe capability: DAEMON_ROBUSTNESS
10import "nx_syscalls.nx"
11import "nx_connect.nx" // bounded connect: a raw sys_connect hangs ~127s on a black-holed host
12const HP_MAGIC_4096: i64 = 4096
13const HP_MAGIC_4095: i64 = 4095
14const HP_MAGIC_8443: i64 = 8443
15const HP_MAGIC_7443: i64 = 7443
16const HP_MAGIC_18096: i64 = 18096
17const HP_MAGIC_18090: i64 = 18090
18const HP_MAGIC_18190: i64 = 18190
19const HP_MAGIC_11434: i64 = 11434
20const HP_MAGIC_18691: i64 = 18691
21const HP_MAGIC_18098: i64 = 18098
22const HP_MAGIC_18099: i64 = 18099
23const HP_MAGIC_8445: i64 = 8445
24
25const HP_SERVING: i64 = 1 // got an HTTP response -> the daemon is alive AND serving
26const HP_REFUSED: i64 = 2 // connect failed -> down / not listening (crashed or never bound)
27const HP_HUNG: i64 = 3 // connected but NO response within the timeout -> hung (the case PID/LISTEN miss)
28const HP_BADRESP: i64 = 4 // responded, but not a valid HTTP reply -> wrong/garbage process on the port
29
30func hp_name(v: i64) -> *u8 {
31 if v == HP_SERVING { return "SERVING" as *u8 }
32 if v == HP_REFUSED { return "REFUSED(down)" as *u8 }
33 if v == HP_HUNG { return "HUNG(up-but-not-serving)" as *u8 }
34 return "BADRESP" as *u8
35}
36// 16-byte sockaddr_in for 127.0.0.1:port (or a.b.c.d)
37func hp_sockaddr(buf: *u8, port: i64, a: i64, b: i64, c: i64, d: i64) -> i64 {
38 var i: i64 = 0; while i < 16 { buf[i] = 0 as u8; i = i + 1 }
39 buf[0] = 2 as u8 // AF_INET
40 buf[2] = ((port >> 8) & 0xff) as u8; buf[3] = (port & 0xff) as u8 // port, big-endian
41 buf[4] = a as u8; buf[5] = b as u8; buf[6] = c as u8; buf[7] = d as u8
42 return 0
43}
44// probe 127.0.0.1:port with timeout_sec. Returns a verdict.
45// Single-return + munmap on every path: this runs ~4x per supervisor poll forever, so a leaked dest(16)+
46// buf(4096) each call is exactly the kind of slow leak that segfaulted nx_hostctl (mmap -> -12 -> deref).
47func hp_probe(port: i64, timeout_sec: i64) -> i64 {
48 let fd: i64 = sys_socket(AF_INET, SOCK_STREAM, 0); if fd < 0 { return HP_REFUSED }
49 sys_set_socket_timeout(fd, timeout_sec)
50 let dest: *u8 = sys_mmap(16); hp_sockaddr(dest, port, 127, 0, 0, 1)
51 var verdict: i64 = HP_REFUSED
52 if nx_connect_bounded(fd, dest, 16, NX_CONN_DEFAULT_MS) == 0 {
53 let req: *u8 = "GET /healthz HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n" as *u8
54 var rl: i64 = 0; while req[rl] != (0 as u8) { rl = rl + 1 }
55 sys_write(fd, req, rl)
56 let buf: *u8 = sys_mmap(HP_MAGIC_4096); let r: i64 = sys_read(fd, buf, HP_MAGIC_4095)
57 if r <= 0 { verdict = HP_HUNG } // connected but silent within the timeout -> hung
58 else {
59 verdict = HP_BADRESP
60 if r >= 5 { if buf[0]==(72 as u8) { if buf[1]==(84 as u8) { if buf[2]==(84 as u8) { if buf[3]==(80 as u8) { if buf[4]==(47 as u8) { verdict = HP_SERVING } } } } } } // "HTTP/"
61 }
62 sys_munmap(buf, HP_MAGIC_4096)
63 }
64 sys_close(fd)
65 sys_munmap(dest, 16)
66 return verdict
67}
68
69// ---- CONNECT-ONLY readiness: is the LISTEN SOCKET still bound? (2026-07-30, seq1299) ----
70// THE FAILURE THIS EXISTS FOR, MEASURED: nx_tools_api_serve stayed PROCESS-ALIVE (parent Ss) while its
71// listen socket was GONE -- loopback connect REFUSED, every liveness-only guard read it as healthy, and
72// the entire MCP surface was dark until a human noticed. hp_probe ALREADY names that state (HP_REFUSED
73// while alive) but hc_needs_restart only acted on HP_HUNG, so the signal was measured and DISCARDED.
74// ★WHY CONNECT-ONLY: the retired serving-probe wrote `GET /healthz` and read the reply, which SIGPIPE'd
75// and hung-killed a WORKING redirect.elf every poll into a crash-loop (see the sites/redirect guard
76// notes in nx_hostctl). We never write a byte here, so this probe cannot damage a healthy daemon BY
77// CONSTRUCTION -- it answers exactly one question and refuses to guess about anything else.
78// derived: a loopback bind completes in microseconds; 500ms is ~3 orders of magnitude of margin, and it
79// is only ever paid on the rare REFUSED path (never in the healthy steady state).
80const HP_CONFIRM_GAP_MS: i64 = 500
81// derived: loopback connect either completes or resets immediately; 2s only bounds a pathological stall.
82const HP_CONNECT_TIMEOUT_S: i64 = 2
83
84// 1 = a listener is bound (connect accepted) · 0 = connect REFUSED (nothing listening).
85// FAIL-SAFE: if we cannot even create a socket (fd exhaustion in the SUPERVISOR, not a fault of the
86// daemon) we return 1 = "listening" so a probe failure can NEVER be used to accuse a healthy service.
87// A saturated backlog is still ACCEPTED by the kernel, so a merely BUSY daemon reads as 1 -- "busy" and
88// "unbound" are different states and only the second one is actionable.
89func hp_listening(port: i64) -> i64 {
90 let fd: i64 = sys_socket(AF_INET, SOCK_STREAM, 0)
91 if fd < 0 { return 1 }
92 sys_set_socket_timeout(fd, HP_CONNECT_TIMEOUT_S)
93 let dest: *u8 = sys_mmap(16)
94 hp_sockaddr(dest, port, 127, 0, 0, 1)
95 var ok: i64 = 1
96 if nx_connect_bounded(fd, dest, 16, NX_CONN_DEFAULT_MS) != 0 { ok = 0 }
97 sys_close(fd)
98 sys_munmap(dest, 16)
99 return ok
100}
101
102// 1 = the listen socket is CONFIRMED gone (two connect-only refusals HP_CONFIRM_GAP_MS apart).
103// A single refusal is NOT proof: a freshly spawned daemon is briefly alive-but-not-yet-bound, and
104// restarting it on that evidence would manufacture the very crash-loop this is meant to prevent.
105func hp_socket_dead(port: i64) -> i64 {
106 if hp_listening(port) == 1 { return 0 }
107 sys_sleep_ms(HP_CONFIRM_GAP_MS)
108 if hp_listening(port) == 1 { return 0 }
109 return 1
110}
111
112// ---- TLS responds-probe (2026-07-16) ----
113// hp_probe sends plain HTTP, so on a TLS port (sites :8443, sni-router :7443) it yields a FALSE verdict
114// (HUNG/BADRESP on a healthy edge) -- the supervisor could never own the edge with it. This probe speaks
115// the handshake's first step instead: send a canned TLS1.2 ClientHello, bounded-read the reply.
116// SERVING = any TLS record back (type 20..23: ServerHello/alert/ccs/appdata -- the accept loop AND the
117// handshake engine answered; the 2026-07-15 outage class was precisely "listening but HANGING
118// the handshake at CertificateVerify", which this catches as HUNG)
119// BADRESP = bytes back that are not a TLS record, or a clean close (wrong process on the port)
120// HUNG = connected but silent within the timeout ; REFUSED = connect failed (down)
121func hp_hexv(c: i64) -> i64 { if c >= 97 { return c - 87 } return c - 48 }
122// canned ClientHello, 118 bytes hex-encoded (lowercase): TLS1.2, SNI=b2b.loans, 10 common suites,
123// supported_groups + signature_algorithms + ec_point_formats. Decodes into out; returns byte length.
124func hp_build_ch(out: *u8) -> i64 {
125 let hx: *u8 = "16030100710100006d03034e582d544c532d50524f42452d323032362d30372d31362d6e69736869212121000014c02fc030c02bc02ccca8cca9009c009d002f0035010000300000000e000c0000096232622e6c6f616e73000a00080006001d00170018000d00080006040104030804000b00020100" as *u8
126 var i: i64 = 0
127 var go: i64 = 1
128 while go == 1 {
129 let j: i64 = i * 2
130 if hx[j] == (0 as u8) { go = 0 }
131 else {
132 out[i] = ((hp_hexv(hx[j] as i64) * 16) + hp_hexv(hx[j + 1] as i64)) as u8
133 i = i + 1
134 }
135 }
136 return i
137}
138// probe 127.0.0.1:port as a TLS endpoint. Single-return + munmap on every path (same bounded-VSZ
139// discipline as hp_probe -- this too runs every supervisor cycle forever).
140func hp_probe_tls(port: i64, timeout_sec: i64) -> i64 {
141 let fd: i64 = sys_socket(AF_INET, SOCK_STREAM, 0); if fd < 0 { return HP_REFUSED }
142 sys_set_socket_timeout(fd, timeout_sec)
143 let dest: *u8 = sys_mmap(16); hp_sockaddr(dest, port, 127, 0, 0, 1)
144 var verdict: i64 = HP_REFUSED
145 if nx_connect_bounded(fd, dest, 16, NX_CONN_DEFAULT_MS) == 0 {
146 let ch: *u8 = sys_mmap(256)
147 let cl: i64 = hp_build_ch(ch)
148 sys_write(fd, ch, cl)
149 let buf: *u8 = sys_mmap(HP_MAGIC_4096)
150 let r: i64 = sys_read(fd, buf, HP_MAGIC_4095)
151 if r > 0 {
152 verdict = HP_BADRESP
153 let b0: i64 = buf[0] as i64
154 if b0 >= 20 { if b0 <= 23 { verdict = HP_SERVING } }
155 }
156 else {
157 if r == 0 { verdict = HP_BADRESP } else { verdict = HP_HUNG }
158 }
159 sys_munmap(buf, HP_MAGIC_4096)
160 sys_munmap(ch, 256)
161 }
162 sys_close(fd)
163 sys_munmap(dest, 16)
164 return verdict
165}
166
167// WS responds-probe: a WebSocket-ONLY daemon (nx_signaling_v2 :8445) IGNORES plain HTTP by design, so
168// hp_probe reads a healthy one as HUNG -- a FALSE verdict that misled a live diagnosis 2026-07-16 (a
169// healthy signaling daemon got killed off it). This sends a real upgrade request; "HTTP/1.1 101" passes
170// the same "HTTP/" prefix check, so verdicts are honest. Registry probe type: ws.
171func hp_probe_ws(port: i64, timeout_sec: i64) -> i64 {
172 let fd: i64 = sys_socket(AF_INET, SOCK_STREAM, 0); if fd < 0 { return HP_REFUSED }
173 sys_set_socket_timeout(fd, timeout_sec)
174 let dest: *u8 = sys_mmap(16); hp_sockaddr(dest, port, 127, 0, 0, 1)
175 var verdict: i64 = HP_REFUSED
176 if nx_connect_bounded(fd, dest, 16, NX_CONN_DEFAULT_MS) == 0 {
177 let req: *u8 = "GET / HTTP/1.1\r\nHost: localhost\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Key: bnhfaGVhbHRoX3Byb2JlX2tleQ==\r\nSec-WebSocket-Version: 13\r\n\r\n" as *u8
178 var rl: i64 = 0; while req[rl] != (0 as u8) { rl = rl + 1 }
179 sys_write(fd, req, rl)
180 let buf: *u8 = sys_mmap(HP_MAGIC_4096); let r: i64 = sys_read(fd, buf, HP_MAGIC_4095)
181 if r <= 0 { verdict = HP_HUNG }
182 else {
183 verdict = HP_BADRESP
184 if r >= 5 { if buf[0]==(72 as u8) { if buf[1]==(84 as u8) { if buf[2]==(84 as u8) { if buf[3]==(80 as u8) { if buf[4]==(47 as u8) { verdict = HP_SERVING } } } } } }
185 }
186 sys_munmap(buf, HP_MAGIC_4096)
187 }
188 sys_close(fd)
189 sys_munmap(dest, 16)
190 return verdict
191}
192
193func hp_p(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
194func hp_n(v: i64) -> i64 { let t: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m} var k: i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} let b: *u8=sys_mmap(28); var i: i64=0; while i<k{b[i]=t[k-1-i];i=i+1} sys_write(1,b,k); return 0 }
195func hp_one(port: i64, name: *u8) -> i64 {
196 let v: i64 = hp_probe(port, 3)
197 hp_p(" :" as *u8); hp_n(port); hp_p(" " as *u8); hp_p(name); hp_p(" [http] -> " as *u8); hp_p(hp_name(v)); hp_p("\n" as *u8)
198 return v
199}
200func hp_one_tls(port: i64, name: *u8) -> i64 {
201 let v: i64 = hp_probe_tls(port, 3)
202 hp_p(" :" as *u8); hp_n(port); hp_p(" " as *u8); hp_p(name); hp_p(" [tls] -> " as *u8); hp_p(hp_name(v)); hp_p("\n" as *u8)
203 return v
204}
205func hp_one_ws(port: i64, name: *u8) -> i64 {
206 let v: i64 = hp_probe_ws(port, 3)
207 hp_p(" :" as *u8); hp_n(port); hp_p(" " as *u8); hp_p(name); hp_p(" [ws] -> " as *u8); hp_p(hp_name(v)); hp_p("\n" as *u8)
208 return v
209}
210// GATE battery: TLS probe must read the LIVE edge as SERVING, a dead port as REFUSED (neg-control),
211// and an HTTP port probed as TLS as BADRESP (discrimination). The plain-probe-on-TLS lines document
212// the false verdict this probe exists to fix.
213func main() -> i64 {
214 hp_p("=== NISHI HEALTH PROBE (serving, not just process-present) ===\n" as *u8)
215 hp_p("-- crown fleet, correct probe types --\n" as *u8)
216 hp_one_tls(HP_MAGIC_8443, "sites-edge" as *u8)
217 hp_one_tls(HP_MAGIC_7443, "sni-router" as *u8)
218 hp_one(HP_MAGIC_18096, "tools-api" as *u8)
219 hp_one(HP_MAGIC_18090, "gallery-serve" as *u8)
220 hp_one(HP_MAGIC_18190, "gallery-gw" as *u8)
221 hp_one(HP_MAGIC_11434, "llm-seat" as *u8)
222 hp_one(HP_MAGIC_18691, "sovgit" as *u8)
223 hp_one(HP_MAGIC_18098, "mgmt-api" as *u8)
224 hp_one(HP_MAGIC_18099, "studio" as *u8)
225 hp_one_ws(HP_MAGIC_8445, "signaling" as *u8)
226 hp_p("-- negative controls --\n" as *u8)
227 hp_one_tls(9, "dead-port" as *u8)
228 hp_one_tls(HP_MAGIC_18096, "http-as-tls" as *u8)
229 hp_p("-- the OLD false verdicts (wrong probe class on tls/ws ports; why typed probes exist) --\n" as *u8)
230 hp_one(HP_MAGIC_8443, "sites-edge" as *u8)
231 hp_one(HP_MAGIC_7443, "sni-router" as *u8)
232 hp_one(HP_MAGIC_8445, "signaling" as *u8)
233 sys_exit(0); return 0
234}