nx_health_probe.nx source
↩ module page · 243 lines · 14238 B
1// nx_health_probe.nx -- SERVING health probe (closes hosting_research gap #5 "health-probe-not-liveness",
2// 3/0 CONFIRMED: a service running-but-not-serving is worse than crashed; nx_hostctl only /proc-name-scans).
3// Black-box probe: connect + send a request + bounded recv. Distinguishes the 4 real states a supervisor
4// must act on differently -- crucially HUNG (process up + port bound but NOT responding), which PID-liveness
5// (systemd) and even a LISTEN check (nx_port_audit) both MISS. No daemon cooperation needed (unlike systemd
6// sd_notify WATCHDOG); same black-box idea as a k8s liveness probe but sovereign + composes with the
7// restart-guard + reap + listen audits into one supervision-health story.
8//
9// module: nishi-core.supervision.health_probe capability: DAEMON_ROBUSTNESS
10import "nx_syscalls.nx"
11import "nx_connect.nx" // bounded connect: a raw sys_connect hangs ~127s on a black-holed host
12const HP_MAGIC_4096: i64 = 4096
13const HP_MAGIC_4095: i64 = 4095
14const HP_MAGIC_8443: i64 = 8443
15const HP_MAGIC_7443: i64 = 7443
16const HP_MAGIC_18096: i64 = 18096
17const HP_MAGIC_18090: i64 = 18090
18const HP_MAGIC_18190: i64 = 18190
19const HP_MAGIC_11434: i64 = 11434
20const HP_MAGIC_18691: i64 = 18691
21const HP_MAGIC_18098: i64 = 18098
22const HP_MAGIC_18099: i64 = 18099
23const HP_MAGIC_8445: i64 = 8445
24
25const HP_SERVING: i64 = 1 // got an HTTP response -> the daemon is alive AND serving
26const HP_REFUSED: i64 = 2 // connect failed -> down / not listening (crashed or never bound)
27const HP_HUNG: i64 = 3 // connected but NO response within the timeout -> hung (the case PID/LISTEN miss)
28const HP_BADRESP: i64 = 4 // responded, but not a valid HTTP reply -> wrong/garbage process on the port
29
30func hp_name(v: i64) -> *u8 {
31 if v == HP_SERVING { return "SERVING" as *u8 }
32 if v == HP_REFUSED { return "REFUSED(down)" as *u8 }
33 if v == HP_HUNG { return "HUNG(up-but-not-serving)" as *u8 }
34 return "BADRESP" as *u8
35}
36// 16-byte sockaddr_in for 127.0.0.1:port (or a.b.c.d)
37func hp_sockaddr(buf: *u8, port: i64, a: i64, b: i64, c: i64, d: i64) -> i64 {
38 var i: i64 = 0; while i < 16 { buf[i] = 0 as u8; i = i + 1 }
39 buf[0] = 2 as u8 // AF_INET
40 buf[2] = ((port >> 8) & 0xff) as u8; buf[3] = (port & 0xff) as u8 // port, big-endian
41 buf[4] = a as u8; buf[5] = b as u8; buf[6] = c as u8; buf[7] = d as u8
42 return 0
43}
44// probe 127.0.0.1:port with timeout_sec. Returns a verdict.
45// Single-return + munmap on every path: this runs ~4x per supervisor poll forever, so a leaked dest(16)+
46// buf(4096) each call is exactly the kind of slow leak that segfaulted nx_hostctl (mmap -> -12 -> deref).
47func hp_probe(port: i64, timeout_sec: i64) -> i64 {
48 let fd: i64 = sys_socket(AF_INET, SOCK_STREAM, 0); if fd < 0 { return HP_REFUSED }
49 sys_set_socket_timeout(fd, timeout_sec)
50 let dest: *u8 = sys_mmap(16); hp_sockaddr(dest, port, 127, 0, 0, 1)
51 var verdict: i64 = HP_REFUSED
52 if nx_connect_bounded(fd, dest, 16, NX_CONN_DEFAULT_MS) == 0 {
53 let req: *u8 = "GET /healthz HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n" as *u8
54 var rl: i64 = 0; while req[rl] != (0 as u8) { rl = rl + 1 }
55 sys_write(fd, req, rl)
56 let buf: *u8 = sys_mmap(HP_MAGIC_4096); let r: i64 = sys_read(fd, buf, HP_MAGIC_4095)
57 if r <= 0 { verdict = HP_HUNG } // connected but silent within the timeout -> hung
58 else {
59 verdict = HP_BADRESP
60 if r >= 5 { if buf[0]==(72 as u8) { if buf[1]==(84 as u8) { if buf[2]==(84 as u8) { if buf[3]==(80 as u8) { if buf[4]==(47 as u8) { verdict = HP_SERVING } } } } } } // "HTTP/"
61 }
62 sys_munmap(buf, HP_MAGIC_4096)
63 }
64 sys_close(fd)
65 sys_munmap(dest, 16)
66 return verdict
67}
68
69// ---- CONNECT-ONLY readiness: is the LISTEN SOCKET still bound? (2026-07-30, seq1299) ----
70// THE FAILURE THIS EXISTS FOR, MEASURED: nx_tools_api_serve stayed PROCESS-ALIVE (parent Ss) while its
71// listen socket was GONE -- loopback connect REFUSED, every liveness-only guard read it as healthy, and
72// the entire MCP surface was dark until a human noticed. hp_probe ALREADY names that state (HP_REFUSED
73// while alive) but hc_needs_restart only acted on HP_HUNG, so the signal was measured and DISCARDED.
74// star WHY CONNECT-ONLY: the retired serving-probe wrote `GET /healthz` and read the reply, which SIGPIPE'd
75// and hung-killed a WORKING redirect.elf every poll into a crash-loop (see the sites/redirect guard
76// notes in nx_hostctl). We never write a byte here, so this probe cannot damage a healthy daemon BY
77// CONSTRUCTION -- it answers exactly one question and refuses to guess about anything else.
78// derived: a loopback bind completes in microseconds; 500ms is ~3 orders of magnitude of margin, and it
79// is only ever paid on the rare REFUSED path (never in the healthy steady state).
80const HP_CONFIRM_GAP_MS: i64 = 500
81// derived: loopback connect either completes or resets immediately; 2s only bounds a pathological stall.
82const HP_CONNECT_TIMEOUT_S: i64 = 2
83
84// 1 = a listener is bound (connect accepted) . 0 = connect REFUSED (nothing listening).
85// FAIL-SAFE: if we cannot even create a socket (fd exhaustion in the SUPERVISOR, not a fault of the
86// daemon) we return 1 = "listening" so a probe failure can NEVER be used to accuse a healthy service.
87// A saturated backlog is still ACCEPTED by the kernel, so a merely BUSY daemon reads as 1 -- "busy" and
88// "unbound" are different states and only the second one is actionable.
89func hp_listening(port: i64) -> i64 {
90 let fd: i64 = sys_socket(AF_INET, SOCK_STREAM, 0)
91 if fd < 0 { return 1 }
92 sys_set_socket_timeout(fd, HP_CONNECT_TIMEOUT_S)
93 let dest: *u8 = sys_mmap(16)
94 hp_sockaddr(dest, port, 127, 0, 0, 1)
95 var ok: i64 = 1
96 if nx_connect_bounded(fd, dest, 16, NX_CONN_DEFAULT_MS) != 0 { ok = 0 }
97 sys_close(fd)
98 sys_munmap(dest, 16)
99 return ok
100}
101
102// 1 = the listen socket is CONFIRMED gone (two connect-only refusals HP_CONFIRM_GAP_MS apart).
103// A single refusal is NOT proof: a freshly spawned daemon is briefly alive-but-not-yet-bound, and
104// restarting it on that evidence would manufacture the very crash-loop this is meant to prevent.
105func hp_socket_dead(port: i64) -> i64 {
106 if hp_listening(port) == 1 { return 0 }
107 sys_sleep_ms(HP_CONFIRM_GAP_MS)
108 if hp_listening(port) == 1 { return 0 }
109 return 1
110}
111
112// ---- TLS responds-probe (2026-07-16; ClientHello MODERNISED 2026-08-10) ----
113// hp_probe sends plain HTTP, so on a TLS port (sites :8443, sni-router :7443) it yields a FALSE verdict
114// (HUNG/BADRESP on a healthy edge) -- the supervisor could never own the edge with it. This probe speaks
115// the handshake's first step instead: send a real TLS1.3 ClientHello, bounded-read the reply.
116// SERVING = any TLS record back (type 20..23: ServerHello/alert/ccs/appdata -- the accept loop AND the
117// handshake engine answered; the 2026-07-15 outage class was precisely "listening but HANGING
118// the handshake at CertificateVerify", which this catches as HUNG)
119// BADRESP = bytes back that are not a TLS record, or a clean close (wrong process on the port)
120// HUNG = connected but silent within the timeout ; REFUSED = connect failed (down)
121// star 2026-08-10, MEASURED not assumed: the ORIGINAL 118-byte hello was a bare TLS1.2 ClientHello with NO
122// supported_versions and NO key_share, SNI=b2b.loans. Against the sovereign TLS1.3 edge (sites.elf) a
123// one-shot diagnostic (nx_edgeprobe_diag) proved it got read_r=0 -- a CLEAN CLOSE, no ServerHello, no
124// alert -- so hp_probe_tls returned BADRESP on a PROVEN-HEALTHY edge (nx_https_get got HTTP 401/200 through
125// the same port). That false BADRESP accumulated 47,000+ fails and fired WEDGE-KILL on the LIVE edge.
126// star A PROBE THAT SPEAKS A DIALECT THE SUBJECT NO LONGER ANSWERS MANUFACTURES THE OUTAGE IT REPORTS.
127// The hello below is a real TLS1.3 ClientHello; the SAME diagnostic then measured read_r=95, first byte
128// 22 (ServerHello) => SERVING. Re-arm edge_sites to guard only against this fixed probe.
129func hp_hexv(c: i64) -> i64 { if c >= 97 { return c - 87 } return c - 48 }
130// real TLS1.3 ClientHello, 159 bytes hex-encoded (lowercase): supported_versions[1.3], x25519 key_share
131// (fixed 32-byte pubkey -- we never complete the handshake, only elicit a ServerHello), SNI=nishifamily.com,
132// suites 1301/1302/1303. BITE-PROVEN 2026-08-10 vs the live edge -> ServerHello. Decodes into out; returns len.
133func hp_build_ch(out: *u8) -> i64 {
134 let hx: *u8 = "160301009a0100009603034e582d544c532d50524f42452d323032362d30372d31362d6e6973686921212100000813011302130300ff0100006500000014001200000f6e6973686966616d696c792e636f6d000a00060004001d0017000d00080006040308040401002b0003020304003300260024001d00204242424242424242424242424242424242424242424242424242424242424242000b00020100" as *u8
135 var i: i64 = 0
136 var go: i64 = 1
137 while go == 1 {
138 let j: i64 = i * 2
139 if hx[j] == (0 as u8) { go = 0 }
140 else {
141 out[i] = ((hp_hexv(hx[j] as i64) * 16) + hp_hexv(hx[j + 1] as i64)) as u8
142 i = i + 1
143 }
144 }
145 return i
146}
147// probe 127.0.0.1:port as a TLS endpoint. Single-return + munmap on every path (same bounded-VSZ
148// discipline as hp_probe -- this too runs every supervisor cycle forever).
149func hp_probe_tls(port: i64, timeout_sec: i64) -> i64 {
150 let fd: i64 = sys_socket(AF_INET, SOCK_STREAM, 0); if fd < 0 { return HP_REFUSED }
151 sys_set_socket_timeout(fd, timeout_sec)
152 let dest: *u8 = sys_mmap(16); hp_sockaddr(dest, port, 127, 0, 0, 1)
153 var verdict: i64 = HP_REFUSED
154 if nx_connect_bounded(fd, dest, 16, NX_CONN_DEFAULT_MS) == 0 {
155 let ch: *u8 = sys_mmap(256)
156 let cl: i64 = hp_build_ch(ch)
157 sys_write(fd, ch, cl)
158 let buf: *u8 = sys_mmap(HP_MAGIC_4096)
159 let r: i64 = sys_read(fd, buf, HP_MAGIC_4095)
160 if r > 0 {
161 verdict = HP_BADRESP
162 let b0: i64 = buf[0] as i64
163 if b0 >= 20 { if b0 <= 23 { verdict = HP_SERVING } }
164 }
165 else {
166 if r == 0 { verdict = HP_BADRESP } else { verdict = HP_HUNG }
167 }
168 sys_munmap(buf, HP_MAGIC_4096)
169 sys_munmap(ch, 256)
170 }
171 sys_close(fd)
172 sys_munmap(dest, 16)
173 return verdict
174}
175
176// WS responds-probe: a WebSocket-ONLY daemon (nx_signaling_v2 :8445) IGNORES plain HTTP by design, so
177// hp_probe reads a healthy one as HUNG -- a FALSE verdict that misled a live diagnosis 2026-07-16 (a
178// healthy signaling daemon got killed off it). This sends a real upgrade request; "HTTP/1.1 101" passes
179// the same "HTTP/" prefix check, so verdicts are honest. Registry probe type: ws.
180func hp_probe_ws(port: i64, timeout_sec: i64) -> i64 {
181 let fd: i64 = sys_socket(AF_INET, SOCK_STREAM, 0); if fd < 0 { return HP_REFUSED }
182 sys_set_socket_timeout(fd, timeout_sec)
183 let dest: *u8 = sys_mmap(16); hp_sockaddr(dest, port, 127, 0, 0, 1)
184 var verdict: i64 = HP_REFUSED
185 if nx_connect_bounded(fd, dest, 16, NX_CONN_DEFAULT_MS) == 0 {
186 let req: *u8 = "GET / HTTP/1.1\r\nHost: localhost\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Key: bnhfaGVhbHRoX3Byb2JlX2tleQ==\r\nSec-WebSocket-Version: 13\r\n\r\n" as *u8
187 var rl: i64 = 0; while req[rl] != (0 as u8) { rl = rl + 1 }
188 sys_write(fd, req, rl)
189 let buf: *u8 = sys_mmap(HP_MAGIC_4096); let r: i64 = sys_read(fd, buf, HP_MAGIC_4095)
190 if r <= 0 { verdict = HP_HUNG }
191 else {
192 verdict = HP_BADRESP
193 if r >= 5 { if buf[0]==(72 as u8) { if buf[1]==(84 as u8) { if buf[2]==(84 as u8) { if buf[3]==(80 as u8) { if buf[4]==(47 as u8) { verdict = HP_SERVING } } } } } }
194 }
195 sys_munmap(buf, HP_MAGIC_4096)
196 }
197 sys_close(fd)
198 sys_munmap(dest, 16)
199 return verdict
200}
201
202func hp_p(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
203func hp_n(v: i64) -> i64 { let t: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)} var k: i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} let b: *u8=sys_mmap(28); var i: i64=0; while i<k{b[i]=t[k-1-i];i=i+1} sys_write(1,b,k); return 0 }
204func hp_one(port: i64, name: *u8) -> i64 {
205 let v: i64 = hp_probe(port, 3)
206 hp_p(" :" as *u8); hp_n(port); hp_p(" " as *u8); hp_p(name); hp_p(" [http] -> " as *u8); hp_p(hp_name(v)); hp_p("\n" as *u8)
207 return v
208}
209func hp_one_tls(port: i64, name: *u8) -> i64 {
210 let v: i64 = hp_probe_tls(port, 3)
211 hp_p(" :" as *u8); hp_n(port); hp_p(" " as *u8); hp_p(name); hp_p(" [tls] -> " as *u8); hp_p(hp_name(v)); hp_p("\n" as *u8)
212 return v
213}
214func hp_one_ws(port: i64, name: *u8) -> i64 {
215 let v: i64 = hp_probe_ws(port, 3)
216 hp_p(" :" as *u8); hp_n(port); hp_p(" " as *u8); hp_p(name); hp_p(" [ws] -> " as *u8); hp_p(hp_name(v)); hp_p("\n" as *u8)
217 return v
218}
219// GATE battery: TLS probe must read the LIVE edge as SERVING, a dead port as REFUSED (neg-control),
220// and an HTTP port probed as TLS as BADRESP (discrimination). The plain-probe-on-TLS lines document
221// the false verdict this probe exists to fix.
222func main() -> i64 {
223 hp_p("=== NISHI HEALTH PROBE (serving, not just process-present) ===\n" as *u8)
224 hp_p("-- crown fleet, correct probe types --\n" as *u8)
225 hp_one_tls(HP_MAGIC_8443, "sites-edge" as *u8)
226 hp_one_tls(HP_MAGIC_7443, "sni-router" as *u8)
227 hp_one(HP_MAGIC_18096, "tools-api" as *u8)
228 hp_one(HP_MAGIC_18090, "gallery-serve" as *u8)
229 hp_one(HP_MAGIC_18190, "gallery-gw" as *u8)
230 hp_one(HP_MAGIC_11434, "llm-seat" as *u8)
231 hp_one(HP_MAGIC_18691, "sovgit" as *u8)
232 hp_one(HP_MAGIC_18098, "mgmt-api" as *u8)
233 hp_one(HP_MAGIC_18099, "studio" as *u8)
234 hp_one_ws(HP_MAGIC_8445, "signaling" as *u8)
235 hp_p("-- negative controls --\n" as *u8)
236 hp_one_tls(9, "dead-port" as *u8)
237 hp_one_tls(HP_MAGIC_18096, "http-as-tls" as *u8)
238 hp_p("-- the OLD false verdicts (wrong probe class on tls/ws ports; why typed probes exist) --\n" as *u8)
239 hp_one(HP_MAGIC_8443, "sites-edge" as *u8)
240 hp_one(HP_MAGIC_7443, "sni-router" as *u8)
241 hp_one(HP_MAGIC_8445, "signaling" as *u8)
242 sys_exit(0); return 0
243}