nx_daemon_health_audit.nx source
↩ module page · 180 lines · 9967 B
1// nx_daemon_health_audit.nx -- substrate-side audit of running daemons /
2// proxy chains / nginx-location-block coverage.
3//
4// module: nishi-core.perception.daemon_health_audit
5// depends: nishi-core.perception.profile + nishi-core.perception.instrument_diagnostician +
6// nishi-core.perception.perceptual_dataset + nishi-core.io.syscalls
7// disk_kb: 5
8// capability: PERCEPTION
9// wired_status: PARTIAL_WIRED
10//
11// MISSING_CAPABILITIES:
12// - REMOTE_PS_DISPATCH (issue ps + ss + lsof against remote host via SSH
13// and parse output; SSH primitive lands in queued nx_ssh_client arc)
14// - DEFUNCT_ZOMBIE_DETECTION -- ★ PROVIDED 2026-06-16 by runtime/nx_zombie_audit.nx (SOVEREIGN, no
15// ps/shell: scans /proc, parses /proc/<pid>/stat state 'Z', maps each to its parent pid+comm, reports
16// per-parent zombie counts, flags > threshold as a reap-LEAK). KAT-proven (nx_zombie_audit_gate 2/2:
17// fork a known 41-cluster -> detected -> reaped -> gone). FOUND REAL LEAKS this session: nx_mp_serve
18// (516, fixed) + nx_torrent_get workers (76, fixed) -- ALL the no-wait4 fork-hygiene class. TODO here:
19// call za_scan() in the morning-health-check (nx_daemon_audit_all_claimed) so the doctor auto-flags it.
20// - NGINX_CONFIG_AUDIT (parse nginx -T output for location blocks +
21// upstream proxy chains; verify EVERY claimed endpoint has a real
22// location block routing to a real listening upstream)
23// - LISTENING_VS_CLAIMED_MISMATCH -- ★ PROVIDED 2026-06-16 by runtime/nx_port_audit.nx (SOVEREIGN, no
24// ss/netstat: parses /proc/net/tcp + tcp6 for LISTEN sockets (st=0A), extracts local ports, checks each
25// known Nishi daemon port -> LISTENING or DOWN). FOUND a real outage this session: torrent/gallery/mp
26// LISTENING, but the WHOLE nishifamily stack DOWN -- sites-tls:8443 + redirect:8080 + vroom-video:8446
27// + translate:8447, AND the nx_hostctl supervisor not running (tmpfs-wipe on WSL/reboot, boot-supervise
28// didn't re-fire). Remediation = nx_hostctl supervise/takeover (REMEDIATION_HOOK, still supervised-only).
29// - REMEDIATION_HOOK (compose with nx_instrument_diagnostician + the
30// existing run_if_dead.sh pattern for auto-restart of dead daemons,
31// STAGE 1 SUPERVISED: substrate proposes, user ratifies first
32// few restarts)
33//
34// license_tier: PUBLIC_NISHI_SUBSTRATE
35// genealogy_id: feedback-substrate-does-heavy-lifting-user-is-partner-not-gate_2026 +
36// feedback-substrate-primitives-meta-not-one-off_2026 +
37// feedback-no-false-ok-substrate-honesty-audit +
38// feedback-launching-content-must-be-one-command-easy +
39// feedback-self-surfacing-intelligence-staged-autonomy
40//
41// Triggered by 2026-05-20 finding: nishifamily.com/video URL is serving
42// Synology DSM webman portal (NOT the user's bits-up nishi_video_room),
43// the nishi_video_room daemon has a defunct zombie child, and NO nginx
44// location block routes /video to the real upstream. This entire failure
45// mode was invisible to the existing in-NAS watchdog because the
46// watchdog reads nginx access log -- which logs DSM serving as 200 OK
47// just like the real room serving would. False-OK by construction.
48//
49// THIS primitive audits the chain end-to-end:
50// (1) is the claimed daemon actually running (not defunct / zombie / hung)?
51// (2) is it LISTENing on the port it claims?
52// (3) does nginx have a location block routing to that upstream?
53// (4) does the upstream-served URL actually return the daemon's expected
54// content signature (not "the server's default", which is the DSM
55// fallback that looked like 200-OK in the false-OK case)?
56//
57// Reuse set:
58// - nishifamily.com/video chain (PRIMARY for today)
59// - Every nx_*_room or nx_*_server on the NAS
60// - Production-line dog toys reporting health to substrate
61// - Livestock acoustic gateways reporting cluster health
62// - Any deployed Nishi instrument whose health is invisible from
63// just "process is running"
64
65import "nx_syscalls.nx"
66import "nx_perceptual_profile.nx"
67import "nx_instrument_diagnostician.nx"
68import "nx_perceptual_dataset.nx"
69
70// ===== Daemon health verdict sealed enum ==========================
71
72const NX_DAEMON_OK: i64 = 0
73const NX_DAEMON_RUNNING_BUT_HUNG: i64 = 1 // process alive, port
74 // listening, but no
75 // response to probe
76const NX_DAEMON_DEFUNCT_ZOMBIE: i64 = 2 // <defunct> in ps
77const NX_DAEMON_NOT_LISTENING_ON_CLAIMED: i64 = 3 // process up, no socket
78const NX_DAEMON_LISTENING_BUT_NO_ROUTE: i64 = 4 // bound but no nginx
79 // location block routes
80 // public URL to it
81const NX_DAEMON_RESPONDING_WRONG_CONTENT: i64 = 5 // upstream returned but
82 // content signature
83 // doesn't match expected
84 // (false-OK class)
85const NX_DAEMON_NOT_RUNNING_AT_ALL: i64 = 6
86const NX_DAEMON_DEPENDENCY_MISSING: i64 = 7 // PARTIAL_WIRED default
87
88func nx_daemon_verdict_name(v: i64) -> *u8 {
89 if v == NX_DAEMON_OK { return "OK" }
90 if v == NX_DAEMON_RUNNING_BUT_HUNG { return "RUNNING_BUT_HUNG" }
91 if v == NX_DAEMON_DEFUNCT_ZOMBIE { return "DEFUNCT_ZOMBIE" }
92 if v == NX_DAEMON_NOT_LISTENING_ON_CLAIMED { return "NOT_LISTENING_ON_CLAIMED" }
93 if v == NX_DAEMON_LISTENING_BUT_NO_ROUTE { return "LISTENING_BUT_NO_ROUTE" }
94 if v == NX_DAEMON_RESPONDING_WRONG_CONTENT { return "RESPONDING_WRONG_CONTENT" }
95 if v == NX_DAEMON_NOT_RUNNING_AT_ALL { return "NOT_RUNNING_AT_ALL" }
96 if v == NX_DAEMON_DEPENDENCY_MISSING { return "DEPENDENCY_MISSING" }
97 return "UNKNOWN_DAEMON_VERDICT"
98}
99
100// ===== Audit findings struct ======================================
101
102struct NxDaemonAuditFinding {
103 daemon_name_ptr: *u8
104 daemon_name_len: i64
105 public_url_ptr: *u8 // the URL the daemon CLAIMS to back
106 public_url_len: i64
107 expected_content_signature_ptr: *u8 // hash of expected response prefix
108 expected_content_signature_len: i64
109 verdict: i64 // NX_DAEMON_*
110 actual_listening_port: i64 // -1 if not listening
111 nginx_location_block_found: i64 // 0/1
112 nginx_upstream_target_matches: i64 // 0/1: does the nginx upstream
113 // actually point to the bound port?
114 content_signature_matches: i64 // 0/1
115 defunct_zombie_count: i64
116 suggested_fix_hash_ptr: *u8
117 suggested_fix_hash_len: i64
118}
119
120// ===== Top-level entry stubs ======================================
121
122// nx_daemon_audit_one -- check ONE claimed-daemon-public-URL pair end to
123// end and produce a NxDaemonAuditFinding. Substrate runs this on a
124// schedule + on every deploy.
125
126func nx_daemon_audit_one(daemon_name_ptr: *u8, daemon_name_len: i64,
127 public_url_ptr: *u8, public_url_len: i64,
128 expected_signature_ptr: *u8, expected_signature_len: i64,
129 finding_out_ptr: *NxDaemonAuditFinding) -> i64 {
130 if daemon_name_len <= 0 { return NX_DAEMON_DEPENDENCY_MISSING }
131 if public_url_len <= 0 { return NX_DAEMON_DEPENDENCY_MISSING }
132 if expected_signature_len <= 0 { return NX_DAEMON_DEPENDENCY_MISSING }
133 // PARTIAL_WIRED: remote ps + nginx -T + content-signature check queued.
134 return NX_DAEMON_DEPENDENCY_MISSING
135}
136
137// nx_daemon_audit_all_claimed -- iterate the registered list of
138// claimed-daemon-URL pairs + audit each. Used as the morning-health-check
139// + post-deploy verification. Surfaces a single summary verdict + count
140// of failures.
141
142func nx_daemon_audit_all_claimed(failures_count_out: *i64) -> i64 {
143 return NX_DAEMON_DEPENDENCY_MISSING
144}
145
146// nx_daemon_audit_register_pair -- caller registers an
147// expected daemon -> public URL -> content signature triple. Substrate
148// then audits this triple on every audit cycle. Refuses registration
149// without ALL THREE components -- false-OK prevention: you cannot register
150// "the daemon runs" without naming what its served content should look like.
151
152func nx_daemon_audit_register_pair(daemon_name_ptr: *u8, daemon_name_len: i64,
153 public_url_ptr: *u8, public_url_len: i64,
154 expected_content_signature_ptr: *u8,
155 expected_content_signature_len: i64) -> i64 {
156 if daemon_name_len <= 0 { return 0 }
157 if public_url_len <= 0 { return 0 }
158 if expected_content_signature_len <= 0 { return 0 } // refuse: false-OK prevention
159 return 0
160}
161
162// nx_daemon_audit_propose_remediation -- given an audit finding with
163// a non-OK verdict, compose with nx_instrument_diagnostician for a named
164// fix. E.g., DEFUNCT_ZOMBIE -> restart parent daemon; LISTENING_BUT_NO_ROUTE
165// -> add nginx location block + reload; RESPONDING_WRONG_CONTENT -> the
166// claimed daemon is not what's being served (a different daemon's default
167// is shadowing it).
168
169func nx_daemon_audit_propose_remediation(finding_ptr: *NxDaemonAuditFinding,
170 fix_text_buf_ptr: *u8,
171 fix_text_buf_cap: i64) -> i64 {
172 if fix_text_buf_cap <= 0 { return 0 }
173 return 0
174}
175
176// nx_daemon_audit_get_last_verdict -- inspector.
177
178func nx_daemon_audit_get_last_verdict() -> i64 {
179 return NX_DAEMON_DEPENDENCY_MISSING
180}