nx_supervisor.nx source

1// nx_supervisor.nx -- Erlang-OTP-style restart-policy state machine. 2// 3// Closes the SSS-class "subsystem restart gap" named in 4// NISHI_HONEST_TRADE_OFFS.md row 2 + NISHI_PREEMPTIVE_BUG_ABSORPTION.md. 5// Per cardinal user-owns-every-bit: this is a STATE MACHINE primitive. 6// The substrate doesn't fork or exec anything -- caller does the 7// actual process management. Supervisor returns "what action to 8// take now"; caller obeys. 9// 10// Substrate's structural prevention of three known supervisor anti- 11// patterns: 12// 13// 1. Restart storms: max_restarts counter -> ESCALATE once exceeded 14// (don't burn CPU on infinite crash-restart loop) 15// 2. Split-brain restarts: caller is single-supervisor-per-child; 16// sealed enum state means "is this child being restarted right 17// now or not" is unambiguous 18// 3. State-loss across restart: snapshot-before-mark via 19// nx_module_cas integration (queued v2; v1 is stateless) 20// 21// Exponential backoff (RFC 7234-class): 22// 1st crash: base_backoff_ms wait 23// 2nd: 2 * base wait 24// 3rd: 4 * base wait 25// Nth: min(2^(N-1) * base, max_backoff_ms) 26// 27// Healthy-window reset: 28// if (now - last_restart_ms) > healthy_threshold_ms, counter 29// resets to 0. Per Erlang OTP: a child that's been up long 30// enough has "recovered" and the next crash isn't part of the 31// crash storm. 32// 33// Sealed enum: NxSupState (4 states): 34// HEALTHY child running normally (or never-started) 35// BACKING_OFF child crashed; waiting for backoff_ms before restart 36// ESCALATING exceeded max_restarts; parent supervisor should take over 37// TERMINATED explicit stop; no further restart 38// 39// Sealed enum: NxSupAction (4 decisions): 40// ACTION_NONE child healthy, no action needed 41// ACTION_RESTART restart child immediately 42// ACTION_WAIT still backing off, check again at next_check_ms 43// ACTION_ESCALATE give up; let parent supervisor handle 44// 45// nx_capability_claims: 46// needs: [sealed_enum, sys_now_ms, exp_backoff_math] 47// provides: [supervisor_state_machine, restart_policy_decision, 48// max_restarts_escalate, healthy_window_reset] 49// safety: [no_unchecked_deref, no_floating_point, 50// no_process_management_in_substrate (caller-driven)] 51// verdict: [sealed_enum_4_state + sealed_enum_4_action] 52// license: ORIGINAL 53// kind: racing_crew_specialist 54// layer: L3 (algorithm: state machine over time + counters) 55// sss: [S6 sealed verdicts; row-2 SSS issue closer] 56 57import "nx_syscalls_x86_64.nx" 58 59// ---- Sealed enum: supervisor state ------------------------------- 60 61const NX_SUP_HEALTHY: i64 = 0 62const NX_SUP_BACKING_OFF: i64 = 1 63const NX_SUP_ESCALATING: i64 = 2 64const NX_SUP_TERMINATED: i64 = 3 65const NX_SUP_STATE_N: i64 = 4 66 67func nx_supervisor_state_is_valid(s: i64) -> i64 { 68 if s < 0 { return 0 } 69 if s >= NX_SUP_STATE_N { return 0 } 70 return 1 71} 72 73func nx_supervisor_state_name(s: i64) -> *u8 { 74 if s == NX_SUP_HEALTHY { return "HEALTHY" as *u8 } 75 if s == NX_SUP_BACKING_OFF { return "BACKING_OFF" as *u8 } 76 if s == NX_SUP_ESCALATING { return "ESCALATING" as *u8 } 77 if s == NX_SUP_TERMINATED { return "TERMINATED" as *u8 } 78 return "INVALID" as *u8 79} 80 81// ---- Sealed enum: supervisor action ------------------------------ 82 83const NX_SUP_ACTION_NONE: i64 = 0 84const NX_SUP_ACTION_RESTART: i64 = 1 85const NX_SUP_ACTION_WAIT: i64 = 2 86const NX_SUP_ACTION_ESCALATE: i64 = 3 87const NX_SUP_ACTION_N: i64 = 4 88 89func nx_supervisor_action_is_valid(a: i64) -> i64 { 90 if a < 0 { return 0 } 91 if a >= NX_SUP_ACTION_N { return 0 } 92 return 1 93} 94 95func nx_supervisor_action_name(a: i64) -> *u8 { 96 if a == NX_SUP_ACTION_NONE { return "NONE" as *u8 } 97 if a == NX_SUP_ACTION_RESTART { return "RESTART" as *u8 } 98 if a == NX_SUP_ACTION_WAIT { return "WAIT" as *u8 } 99 if a == NX_SUP_ACTION_ESCALATE { return "ESCALATE" as *u8 } 100 return "INVALID" as *u8 101} 102 103// ---- Supervisor state struct ------------------------------------- 104 105struct NxSupervisor { 106 // Configuration (set at init; never modified after) 107 max_restarts: i64, // hard ceiling; over -> ESCALATING 108 base_backoff_ms: i64, // first crash wait 109 max_backoff_ms: i64, // cap on exponential growth 110 healthy_threshold_ms: i64, // uptime -> counter reset 111 112 // Mutable state 113 state: i64, // sealed enum NX_SUP_* 114 n_restarts: i64, // accumulated this storm 115 n_escalations: i64, // lifetime escalation count 116 last_crash_ms: i64, // when child last crashed 117 last_restart_ms: i64, // when caller last restarted 118 next_check_ms: i64, // earliest time WAIT becomes RESTART 119} 120 121const NX_SUPERVISOR_BYTES: i64 = 80 // 10 i64 fields 122 123// ---- Init / config ----------------------------------------------- 124 125// Initialize a caller-allocated NxSupervisor. Returns 0 or -1 on 126// BAD_ARG. 127func nx_supervisor_init(sup: *NxSupervisor, 128 max_restarts: i64, 129 base_backoff_ms: i64, 130 max_backoff_ms: i64, 131 healthy_threshold_ms: i64) -> i64 { 132 if sup == (0 as *NxSupervisor) { return -1 } 133 if max_restarts < 0 { return -1 } 134 if base_backoff_ms < 0 { return -1 } 135 if max_backoff_ms < base_backoff_ms { return -1 } 136 if healthy_threshold_ms < 0 { return -1 } 137 sup.max_restarts = max_restarts 138 sup.base_backoff_ms = base_backoff_ms 139 sup.max_backoff_ms = max_backoff_ms 140 sup.healthy_threshold_ms = healthy_threshold_ms 141 sup.state = NX_SUP_HEALTHY 142 sup.n_restarts = 0 143 sup.n_escalations = 0 144 sup.last_crash_ms = 0 145 sup.last_restart_ms = 0 146 sup.next_check_ms = 0 147 return 0 148} 149 150// ---- Exp backoff helper ------------------------------------------ 151 152// Compute backoff_ms = min(base * 2^(n_restarts), max_backoff). 153// n_restarts is the count of THIS storm (resets after healthy window). 154// Returns the backoff value (>=0); never panics on overflow. 155func nx_supervisor_backoff_ms(sup: *NxSupervisor) -> i64 { 156 if sup == (0 as *NxSupervisor) { return 0 } 157 if sup.n_restarts <= 0 { return sup.base_backoff_ms } 158 // Bound the shift to avoid overflow. base * 2^30 is plenty. 159 var shift: i64 = sup.n_restarts - 1 160 if shift > 30 { shift = 30 } 161 let mult: i64 = 1 << shift 162 let candidate: i64 = sup.base_backoff_ms * mult 163 if candidate > sup.max_backoff_ms { return sup.max_backoff_ms } 164 if candidate < 0 { return sup.max_backoff_ms } // overflow guard 165 return candidate 166} 167 168// ---- Event recorders --------------------------------------------- 169// 170// Caller calls these AT THE EVENT (not on the decision tick). 171 172// Record that the child crashed AT now_ms. Transitions HEALTHY -> 173// BACKING_OFF (with backoff computed from current counter). If 174// already in ESCALATING / TERMINATED, no-op. 175func nx_supervisor_record_crash(sup: *NxSupervisor, now_ms: i64) -> i64 { 176 if sup == (0 as *NxSupervisor) { return -1 } 177 if sup.state == NX_SUP_ESCALATING { return 0 } 178 if sup.state == NX_SUP_TERMINATED { return 0 } 179 // Healthy-window reset: if it's been a long time since the last 180 // crash, this isn't part of the same storm. 181 if sup.last_crash_ms > 0 { 182 let since: i64 = now_ms - sup.last_crash_ms 183 if since > sup.healthy_threshold_ms { 184 sup.n_restarts = 0 185 } 186 } 187 sup.last_crash_ms = now_ms 188 sup.n_restarts = sup.n_restarts + 1 189 // Exceeded the max -> escalate. 190 if sup.n_restarts > sup.max_restarts { 191 sup.state = NX_SUP_ESCALATING 192 sup.n_escalations = sup.n_escalations + 1 193 return 0 194 } 195 let backoff: i64 = nx_supervisor_backoff_ms(sup) 196 sup.next_check_ms = now_ms + backoff 197 sup.state = NX_SUP_BACKING_OFF 198 return 0 199} 200 201// Record that the caller successfully restarted the child AT now_ms. 202// Transitions BACKING_OFF -> HEALTHY. 203func nx_supervisor_record_start(sup: *NxSupervisor, now_ms: i64) -> i64 { 204 if sup == (0 as *NxSupervisor) { return -1 } 205 if sup.state == NX_SUP_ESCALATING { return 0 } 206 if sup.state == NX_SUP_TERMINATED { return 0 } 207 sup.last_restart_ms = now_ms 208 sup.state = NX_SUP_HEALTHY 209 return 0 210} 211 212// Explicit stop. Transitions any -> TERMINATED. Caller should 213// invoke before deliberate shutdown so decide() returns ACTION_NONE 214// rather than ACTION_RESTART after a clean exit. 215func nx_supervisor_terminate(sup: *NxSupervisor) -> i64 { 216 if sup == (0 as *NxSupervisor) { return -1 } 217 sup.state = NX_SUP_TERMINATED 218 return 0 219} 220 221// ---- Decision tick ----------------------------------------------- 222// 223// The single load-bearing entry point. Caller calls this on every 224// tick of its supervision loop with (child_alive, now_ms). 225// Returns one of NX_SUP_ACTION_*. 226 227func nx_supervisor_decide(sup: *NxSupervisor, 228 child_alive: i64, 229 now_ms: i64) -> i64 { 230 if sup == (0 as *NxSupervisor) { return NX_SUP_ACTION_ESCALATE } 231 232 // TERMINATED: never act again. 233 if sup.state == NX_SUP_TERMINATED { return NX_SUP_ACTION_NONE } 234 235 // ESCALATING: tell parent supervisor. 236 if sup.state == NX_SUP_ESCALATING { return NX_SUP_ACTION_ESCALATE } 237 238 // HEALTHY + alive: nothing to do. 239 if sup.state == NX_SUP_HEALTHY { 240 if child_alive == 1 { return NX_SUP_ACTION_NONE } 241 // Healthy but child died -- caller hasn't called record_crash yet. 242 // Treat this as the start of a backoff. (Defensive: caller 243 // should have called record_crash, but we don't trust.) 244 nx_supervisor_record_crash(sup, now_ms) 245 // After record_crash, state is BACKING_OFF or ESCALATING; fall 246 // through to BACKING_OFF logic below. 247 } 248 249 if sup.state == NX_SUP_ESCALATING { return NX_SUP_ACTION_ESCALATE } 250 251 // BACKING_OFF: time to restart? 252 if sup.state == NX_SUP_BACKING_OFF { 253 if now_ms < sup.next_check_ms { return NX_SUP_ACTION_WAIT } 254 return NX_SUP_ACTION_RESTART 255 } 256 257 return NX_SUP_ACTION_NONE 258} 259 260// ---- Observers --------------------------------------------------- 261 262func nx_supervisor_state(sup: *NxSupervisor) -> i64 { 263 if sup == (0 as *NxSupervisor) { return -1 } 264 return sup.state 265} 266 267func nx_supervisor_n_restarts(sup: *NxSupervisor) -> i64 { 268 if sup == (0 as *NxSupervisor) { return -1 } 269 return sup.n_restarts 270} 271 272func nx_supervisor_n_escalations(sup: *NxSupervisor) -> i64 { 273 if sup == (0 as *NxSupervisor) { return -1 } 274 return sup.n_escalations 275}