nx_restart_guard.nx source

1// nx_restart_guard.nx -- Prevents a crashing daemon from restarting indefinitely by enforcing burst limits and exponential backoff. 26123 nishihost/buildroot/runtime/nx_restart_guard.nx 3// nx_restart_guard.nx -- SUPERVISOR CRASH-LOOP GUARD (closes hosting_research gap #4 4// "watchdog-restart-loop-guard", 3/0 CONFIRMED -- the root cause of the live 2026-06-16 nx_hostctl 5// crash-loop). A bedrock supervision primitive any supervisor (nx_hostctl / nx_keeper / nx_torrent_up) 6// composes so a daemon that crashes immediately on start is NOT respawned forever. 7// 8// module: nishi-core.supervision.restart_guard 9// capability: DAEMON_ROBUSTNESS 10// 11// INCUMBENT (benchmark, sourced via the Nishi researcher -> hosting_research.tsv): systemd unit restart 12// policy -- Restart=on-failure + RestartSec (wait before restart, default 100ms) + StartLimitBurst (N, 13// default 5) within StartLimitIntervalSec (T, default 10s): more than N restarts in T -> unit enters 14// "failed", systemd stops trying. We MATCH that burst-limit semantics, and EXCEED it on two measured axes: 15// (1) DETERMINISTIC / REPLAYABLE -- given the same restart timestamps, the SAME verdict every run, so a 16// supervision session can be replayed bit-for-bit (systemd's behavior depends on wall-clock + monotonic 17// boot time and cannot be replayed). The caller passes `now`; nothing reads a hidden clock here. 18// (2) EXPONENTIAL BACKOFF -- rg_backoff_ms doubles the wait each restart (capped), vs systemd's FIXED 19// RestartSec, so a flapping daemon backs off instead of hammering at a constant rate. 20// 21// STATE per daemon (caller-owned, 2 words): ws[0] = window-start timestamp, cnt[0] = restarts in window. 22 23const RG_DEF_INTERVAL_MS: i64 = 10000 // systemd StartLimitIntervalSec default (10s) 24const RG_DEF_BURST: i64 = 5 // systemd StartLimitBurst default 25const RG_DEF_BASE_MS: i64 = 100 // systemd RestartSec default (100ms) = our backoff base 26const RG_DEF_MAX_MS: i64 = 30000 // backoff ceiling (30s) so it never waits unboundedly 27// HEALTH-RESET threshold: a daemon that ran healthy for >= this long since its last restart is NOT in a TIGHT 28// crash-loop -> reset its crash counter (auto-recovery). MUST be comfortably above the supervisor poll cadence 29// (nx_hostctl polls ~15s, so a dead daemon always shows elapsed>=~15s) -- set to 60s so only daemons that die 30// within ~1min REPEATEDLY are treated as crash-looping; anything that ran a full minute is "healthy -> reset". 31const RG_HEALTH_RESET_MS: i64 = 60000 32 33// S-CLASS crash-loop containment = Kubernetes CrashLoopBackOff + Erlang/OTP restart-intensity + CAPPED EXPONENTIAL 34// BACKOFF (grounded in the banked SOTA: knowledge/library/rel_erlang_otp.txt, rel_backoff.txt, rel_circuit_breaker.txt). 35// State per daemon (caller-owned, 2 words): ws[0] = last-restart timestamp (ms), cnt[0] = consecutive-crash count. 36// Called by the supervisor ONLY when the daemon is dead. `cap_ms` = backoff ceiling; `base_ms` = first backoff step. 37// (The hostctl reuses its two guard consts for these positionally.) 38// - HEALTH-RESET: ran healthily for >= RG_HEALTH_RESET_MS since the last restart -> cnt=0 (loop over; auto-recover). 39// - FIRST death (cnt==0): restart immediately (minimal downtime for a one-off crash). 40// - REPEATED crashes: WAIT an exponential backoff (base doubling, capped at cap_ms) before each restart, so a hard 41// crash-loop is CONTAINED to ~1 restart per cap_ms (e.g. 5 min) -- bounded CPU/log blast radius. This RETIRES the 42// old flat "5/window-reset-each-window" which allowed a slow INFINITE loop (5 restarts every 2 min forever). 43// returns 1 (restart NOW) or 0 (wait -- the loop is being contained by backoff). 44func rg_should_restart(ws: *i64, cnt: *i64, now: i64, cap_ms: i64, base_ms: i64) -> i64 { 45 let elapsed: i64 = now - ws[0] 46 if elapsed >= RG_HEALTH_RESET_MS { cnt[0] = 0 } // ran healthy >= reset thresh -> auto-recover 47 var backoff: i64 = 0 48 if cnt[0] > 0 { backoff = rg_backoff_ms(cnt[0], base_ms, cap_ms) } // exp backoff ONLY on repeated crashes 49 if elapsed < backoff { return 0 } // backoff not elapsed -> WAIT (contain the loop) 50 ws[0] = now 51 cnt[0] = cnt[0] + 1 52 return 1 53} 54 55// exponential backoff before the Nth restart: base * 2^(count-1), capped at max. count is 1-based. 56func rg_backoff_ms(restart_count: i64, base_ms: i64, max_ms: i64) -> i64 { 57 if restart_count <= 1 { return base_ms } 58 var v: i64 = base_ms; var i: i64 = 1 59 while i < restart_count { v = v * 2; if v >= max_ms { return max_ms } i = i + 1 } 60 return v 61} 62 63// how many restarts remain before the loop-guard trips (window-aware inspector). 64func rg_remaining(cnt: i64, burst: i64) -> i64 { let r: i64 = burst - cnt; if r < 0 { return 0 } return r } 65 66// ---- BIND-GRACE: alive-but-never-listening (2026-09-02, debt 1788361379) ---------------------------------------- 67// THE STATE PID-LIVENESS CANNOT SEE. MEASURED: a supervisor respawned nx_docportal_admin_daemon after a SIGSEGV and 68// the respawn spun 15.9h of CPU inside its shard pre-warm with its port never opened; proc_alive_by_name was 69// satisfied throughout, so /search answered 503 for ~16h while every health surface read UP. A daemon that opens 70// its port only AFTER a warm-up (10-15s here, once 6 min behind a registry lock) makes a single refusal worthless 71// as evidence and a kill on it a crash-loop generator; a refusal that PERSISTS past a grace is evidence. 72// PURE, like rg_should_restart: the supervisor supplies what it observed (alive, socket dead as CONFIRMED by two 73// connect-only refusals, how many consecutive polls the socket has read dead, the grace in polls) and this decides. 74// 1 = the grace is exhausted, kill the wedged husk and respawn. 0 = nothing to do (healthy, dead-by-PID which the 75// liveness path owns, or still inside the grace -- the caller keeps counting). Refereed by nx_restart_guard_gate. 76// ---- STALE-CYCLE LIVENESS (2026-09-02) -------------------------------------------------------------- 77// A process that is ALIVE BY NAME is not thereby DOING ITS JOB: nx_daemon_supervisor's /status page is 78// served by a forked child that outlives its parent, so name-liveness read ALIVE for a dead fleet 79// supervisor indefinitely (measured the day the name-only guard shipped). The parent proves itself by 80// ADVANCING a counter every loop; this predicate turns that into a decision a supervisor can act on and a 81// gate can pin. st[0] = last counter seen, st[1] = consecutive observations without an advance. cyc < 0 82// means "no observation" and resets the streak WITHOUT declaring staleness -- silence is not evidence of 83// death (the surface is a courtesy). Returns 1 exactly when the counter has been observed unchanged on 84// stale_polls CONSECUTIVE calls, and resets the streak so the caller acts once per episode, not on every 85// poll after. PURE: no I/O, so nx_restart_guard_gate pins every branch. 86func rg_stale_cycle(st: *i64, cyc: i64, stale_polls: i64) -> i64 { 87 if cyc < 0 { st[1] = 0; return 0 } 88 if cyc != st[0] { st[0] = cyc; st[1] = 0; return 0 } 89 st[1] = st[1] + 1 90 if st[1] < stale_polls { return 0 } 91 st[1] = 0 92 return 1 93} 94 95// SILENCE WITH A NAME IS THE ORPHAN'S SIGNATURE (2026-09-02). rg_stale_cycle deliberately treats a /status that does 96// not answer, or answers without a counter, as no evidence -- but an ORPHANED status child (its parent dead before 97// the first publish) answers every poll with an EMPTY body forever: name-alive, port answering, counter absent. 98// Measured that day on BOTH supervisor generations (an 83-byte body, no cycle line, for over forty minutes), while 99// the guard above correctly abstained and nothing revived the fleet supervisor. Silence that long with the name 100// alive is not a courtesy withheld; it is the one shape a dead parent leaves behind. st[2] = consecutive silent 101// observations. Returns 1 exactly on the silent_polls-th consecutive silent poll and resets the streak so the 102// caller acts once per episode; any counter observation (cyc >= 0) resets it; an unset bound never authorises a 103// kill. PURE: no I/O, so nx_restart_guard_gate pins every branch (T16-T19). 104func rg_silent_stale(st: *i64, cyc: i64, silent_polls: i64) -> i64 { 105 if cyc >= 0 { st[2] = 0; return 0 } 106 if silent_polls <= 0 { return 0 } 107 st[2] = st[2] + 1 108 if st[2] < silent_polls { return 0 } 109 st[2] = 0 110 return 1 111} 112 113func rg_bind_grace_expired(alive: i64, socket_dead: i64, dead_polls: i64, grace_polls: i64) -> i64 { 114 if alive != 1 { return 0 } 115 if socket_dead != 1 { return 0 } 116 if grace_polls <= 0 { return 0 } // an unset grace can never authorise a kill: abstain, never acquit-by-kill 117 if dead_polls < grace_polls { return 0 } 118 return 1 119}