code wiki / _hdl_build / nx_restart_strategy.nx
`nx_restart_strategy.nx` source

1// nx_restart_strategy.nx -- S-CLASS supervision restart strategy (operator: "get those s class exceed stable ...
2// we arent s class stable compared to aws"). PURE decision logic (no syscalls -> unit-gateable offline), grounded
3// in the researched SOTA (knowledge/library/rel_erlang_otp.txt = OTP supervisor restart-INTENSITY; rel_backoff.txt
4// = exponential backoff + jitter; rel_circuit_breaker.txt = contain a failing component). THE GAP IT CLOSES: today
5// nx_hostctl restarts a crashing daemon FOREVER (the gallery/vroom crash-loops spin + spam logs + burn CPU). The
6// S-class rule (Erlang OTP): a child may restart at most MaxR times within MaxT seconds; exceed that and the
7// supervisor STOPS restarting it = QUARANTINE + escalate, so the failure is CONTAINED (bounded blast radius) and a
8// human/alert is engaged instead of an infinite loop. Restarts are spaced by EXPONENTIAL BACKOFF (capped + jittered)
9// so a flapping dependency gets time to recover and N daemons don't thunder-herd. license_tier: ORIGINAL
10
11const RS_RESTART:    i64 = 0   // within intensity -> restart (after the backoff delay)
12const RS_QUARANTINE: i64 = 1   // crash-loop: restart intensity exceeded -> STOP restarting, escalate (contain it)
13
14// count how many of the recorded restart timestamps (unix sec, ascending or not) fall within `window_sec` of `now`.
15// Old restarts age out of the window, so a daemon that crashed long ago but is now stable is NOT quarantined.
16func rs_restarts_in_window(times: *i64, n: i64, now: i64, window_sec: i64) -> i64 {
17    var c: i64 = 0
18    var i: i64 = 0
19    while i < n {
20        let age: i64 = now - times[i]
21        if age >= 0 { if age <= window_sec { c = c + 1 } }
22        i = i + 1
23    }
24    return c
25}
26
27// the OTP restart-intensity decision: > max_restarts within the window -> QUARANTINE, else RESTART.
28func rs_decide(restarts_in_window: i64, max_restarts: i64) -> i64 {
29    if restarts_in_window > max_restarts { return RS_QUARANTINE }
30    return RS_RESTART
31}
32
33// exponential backoff, CAPPED: delay = min(base_ms * 2^attempt, cap_ms). Doubling each consecutive failure gives a
34// flapping dependency room to recover instead of being hammered every poll.
35func rs_backoff_ms(attempt: i64, base_ms: i64, cap_ms: i64) -> i64 {
36    if attempt < 0 { return base_ms }
37    var d: i64 = base_ms
38    var i: i64 = 0
39    while i < attempt {
40        d = d * 2
41        if d >= cap_ms { return cap_ms }
42        i = i + 1
43    }
44    if d > cap_ms { return cap_ms }
45    return d
46}
47
48// bounded "decorrelated" jitter (deterministic from a seed -> gateable): returns a value in [d/2, d], so N daemons
49// restarting at once don't thunder-herd the same instant (rel_backoff: jitter prevents synchronized retries).
50func rs_jitter_ms(d: i64, seed: i64) -> i64 {
51    if d <= 1 { return d }
52    let half: i64 = d / 2
53    var s: i64 = seed
54    if s < 0 { s = 0 - s }
55    let j: i64 = s % (half + 1)
56    return d - j
57}
58
59// the FULL per-restart decision: given the restart history + now, RESTART (within intensity) or QUARANTINE.
60func rs_should_restart(times: *i64, n: i64, now: i64, window_sec: i64, max_restarts: i64) -> i64 {
61    return rs_decide(rs_restarts_in_window(times, n, now, window_sec), max_restarts)
62}
nx_restart_strategy.nx source

`nx_restart_strategy.nx` source
