nx_restart_guard_gate.nx source
↩ module page · 117 lines · 9034 B
1// nx_restart_guard_gate.nx -- KAT + measured-vs-systemd scorecard for the supervisor crash-loop guard.
2// MIGRATED 2026-09-02 onto nx_gate_verdict (D001): every tooth is a named gv_check, so /api/gate_run reads the verdict
3// from the exit code and harness.jrnl records a frame. Before this the gate returned N for the failing assertion and
4// printed its own verdict line, which /api/promote correctly refused. The teeth and their numbers are unchanged.
5// T1 healthy (restarts spread > interval) -> never trips
6// T2 crash-loop (6 restarts in 10s, burst=5) -> 1st-5th allowed, 6th TRIPS (give up)
7// T3 recovery (quiet > interval) -> window resets, restart allowed again
8// T4 exponential backoff (base*2^(n-1), capped) vs systemd's fixed RestartSec
9// T5 DETERMINISM (the exceed): same timestamps -> identical trip point on a re-run (replayable)
10// T6-T10 bind-grace: alive-but-never-listening (debt 1788361379, the 16 h /search outage class)
11// T11-T15 stale-cycle liveness: alive-by-NAME is not alive (the orphan /status child of a dead fleet supervisor)
12// license_tier: ORIGINAL No hw writes (Rule 26).
13import "nx_syscalls.nx"
14import "nx_restart_guard.nx"
15import "nx_gate_verdict.nx"
16
17// poll a DEAD daemon `n` times spaced `gap_ms` apart through FRESH state; count how many polls WAIT (return 0).
18func rg_waits(n: i64, gap_ms: i64, cap_ms: i64, base_ms: i64) -> i64 {
19 let ws: *i64 = sys_mmap(16) as *i64; let cnt: *i64 = sys_mmap(16) as *i64
20 ws[0] = 0; cnt[0] = 0
21 var waited: i64 = 0; var t: i64 = 0; var i: i64 = 0
22 while i < n { if rg_should_restart(ws, cnt, t, cap_ms, base_ms) == 0 { waited = waited + 1 } t = t + gap_ms; i = i + 1 }
23 return waited
24}
25
26func main() -> i64 {
27 let ctr: *i64 = gv_ctr()
28 gv_head("nx_restart_guard_gate -- crash-loop guard, bind-grace and stale-cycle liveness: PURE predicates, driven directly" as *u8)
29 let CAP: i64 = 10000 // cap_ms (backoff ceiling + health-reset threshold)
30 let BASE: i64 = 100 // base_ms (first backoff step)
31
32 // ---- T1: FIRST death (cnt=0) -> restart IMMEDIATELY (no backoff for a one-off crash) ----
33 let ws: *i64 = sys_mmap(16) as *i64; let cnt: *i64 = sys_mmap(16) as *i64
34 ws[0] = 0; cnt[0] = 0
35 let r1: i64 = rg_should_restart(ws, cnt, 0, CAP, BASE)
36 gv_puts("T1 first-death restart(1=immediate)=" as *u8); gv_num(r1); gv_puts("\n" as *u8)
37 gv_check("T1-first-death-restarts-immediately" as *u8, r1 == 1, ctr)
38
39 // ---- T2: CRASH-LOOP CONTAINMENT -- a hard loop (crashes every 50ms) MUST eventually WAIT (backoff contains it) ----
40 let w2: i64 = rg_waits(14, 50, CAP, BASE)
41 gv_puts("T2 hard-loop waits(>0 = contained)=" as *u8); gv_num(w2); gv_puts("\n" as *u8)
42 gv_check("T2-hard-crash-loop-is-contained-by-backoff" as *u8, w2 > 0, ctr)
43
44 // ---- T3: HEALTH-RESET / AUTO-RECOVERY -- ran healthy >= reset-threshold -> cnt resets -> restart allowed again ----
45 ws[0] = 0; cnt[0] = 5 // pretend it crash-looped 5x
46 let r3: i64 = rg_should_restart(ws, cnt, RG_HEALTH_RESET_MS + 1000, CAP, BASE)
47 gv_puts("T3 health-reset restart(1=recovered)=" as *u8); gv_num(r3); gv_puts(" cnt=" as *u8); gv_num(cnt[0]); gv_puts("\n" as *u8)
48 gv_check("T3-healthy-uptime-resets-the-window" as *u8, r3 == 1, ctr)
49 gv_check("T3-counter-reset-then-one-for-this-restart" as *u8, cnt[0] == 1, ctr)
50
51 // ---- T4: exponential backoff capped (the primitive, unchanged) ----
52 gv_check("T4-backoff-step1-is-base" as *u8, rg_backoff_ms(1, RG_DEF_BASE_MS, RG_DEF_MAX_MS) == 100, ctr)
53 gv_check("T4-backoff-step4-is-8x-base" as *u8, rg_backoff_ms(4, RG_DEF_BASE_MS, RG_DEF_MAX_MS) == 800, ctr)
54 gv_check("T4-backoff-step10-hits-the-ceiling" as *u8, rg_backoff_ms(10, RG_DEF_BASE_MS, RG_DEF_MAX_MS) == 30000, ctr)
55
56 // ---- T5: DETERMINISM -- replay the hard loop -> identical wait-count (replayable supervision) ----
57 let w5: i64 = rg_waits(14, 50, CAP, BASE)
58 gv_puts("T5 replay waits=" as *u8); gv_num(w5); gv_puts("\n" as *u8)
59 gv_check("T5-replay-is-deterministic-on-now-not-wall-clock" as *u8, w5 == w2, ctr)
60
61 // ---- T6-T10: BIND-GRACE (2026-09-02, debt 1788361379) -- alive-but-never-listening is the state PID-liveness
62 // cannot see; measured as a 16h /search outage while every surface read UP. ----
63 let G: i64 = 40 // grace polls, the supervisor's HC_ADMIN_BIND_GRACE_POLLS (40 x ~15s = the measured worst legit start with margin)
64 gv_check("T6-inside-the-grace-still-waits" as *u8, rg_bind_grace_expired(1, 1, G - 1, G) == 0, ctr)
65 gv_check("T7-exhausted-grace-restarts-the-16h-outage-tooth" as *u8, rg_bind_grace_expired(1, 1, G, G) == 1, ctr)
66 gv_check("neg-control-T8-a-listening-daemon-is-never-restarted" as *u8, rg_bind_grace_expired(1, 0, 1000000, G) == 0, ctr)
67 gv_check("neg-control-T9-dead-by-pid-belongs-to-the-liveness-path" as *u8, rg_bind_grace_expired(0, 1, 1000000, G) == 0, ctr)
68 gv_check("neg-control-T10-an-unset-grace-abstains" as *u8, rg_bind_grace_expired(1, 1, 1000000, 0) == 0, ctr)
69
70 // ---- T11-T15: STALE-CYCLE LIVENESS (2026-09-02) -- alive-by-NAME is not alive: nx_daemon_supervisor's /status is
71 // served by a forked child that outlives its parent, so a dead fleet supervisor read ALIVE by name for as long as
72 // the orphan lived. rg_stale_cycle decides from the counter the PARENT advances every loop. ----
73 let SP: i64 = 4 // the supervisor's HC_DSUP_STALE_POLLS (4 x 15 s poll = 60 s without an advance)
74 let st: *i64 = sys_mmap(32) as *i64
75 st[0] = 0; st[1] = 0
76 let f11: i64 = rg_stale_cycle(st, 7, SP)
77 gv_check("T11-first-observation-records-never-fires" as *u8, f11 == 0, ctr)
78 gv_check("T11-last-cycle-recorded" as *u8, st[0] == 7, ctr)
79 var k11: i64 = 1
80 var fired_early: i64 = 0
81 while k11 < SP { if rg_stale_cycle(st, 7, SP) != 0 { fired_early = 1 } k11 = k11 + 1 }
82 gv_check("T12a-inside-the-threshold-it-waits" as *u8, fired_early == 0, ctr)
83 gv_check("T12b-the-threshold-observation-fires-the-orphan-tooth" as *u8, rg_stale_cycle(st, 7, SP) == 1, ctr)
84 gv_check("T13-fires-once-per-episode-then-restarts-the-streak" as *u8, rg_stale_cycle(st, 7, SP) == 0, ctr)
85 st[0] = 0; st[1] = 0
86 var k14: i64 = 0
87 var fired_live: i64 = 0
88 while k14 < 100 { if rg_stale_cycle(st, 1000 + k14, SP) != 0 { fired_live = 1 } k14 = k14 + 1 }
89 gv_check("neg-control-T14-an-advancing-counter-never-fires" as *u8, fired_live == 0, ctr)
90 st[0] = 9; st[1] = SP - 1 // one poll from firing...
91 gv_check("neg-control-T15a-no-answer-is-not-death" as *u8, rg_stale_cycle(st, 0 - 1, SP) == 0, ctr)
92 gv_check("neg-control-T15b-silence-resets-the-streak" as *u8, st[1] == 0, ctr)
93 gv_check("neg-control-T15c-one-stale-poll-after-silence-waits" as *u8, rg_stale_cycle(st, 9, SP) == 0, ctr)
94
95 // ---- T16-T19: SILENCE WITH A NAME (2026-09-02) -- an orphaned status child answers every poll with an EMPTY body
96 // forever (measured on two supervisor generations: 83-byte body, no cycle line, 40+ minutes) while the stale-cycle
97 // tooth above correctly abstains. rg_silent_stale counts CONSECUTIVE silent polls in st[2]. ----
98 let SL: i64 = 5
99 st[2] = 0
100 var k16: i64 = 1
101 var fired_silent_early: i64 = 0
102 while k16 < SL { if rg_silent_stale(st, 0 - 1, SL) != 0 { fired_silent_early = 1 } k16 = k16 + 1 }
103 gv_check("T16-silence-below-the-bound-never-fires" as *u8, fired_silent_early == 0, ctr)
104 gv_check("T17-the-bound-th-consecutive-silent-poll-fires-the-orphan-tooth" as *u8, rg_silent_stale(st, 0 - 1, SL) == 1, ctr)
105 gv_check("T18-fires-once-per-episode-then-restarts-the-streak" as *u8, rg_silent_stale(st, 0 - 1, SL) == 0, ctr)
106 rg_silent_stale(st, 0 - 1, SL); rg_silent_stale(st, 0 - 1, SL) // two silent polls into a new streak...
107 gv_check("neg-control-T19a-a-counter-observation-resets-the-silence" as *u8, rg_silent_stale(st, 42, SL) == 0, ctr)
108 gv_check("neg-control-T19b-the-streak-is-zero-after-a-counter" as *u8, st[2] == 0, ctr)
109 gv_check("neg-control-T19c-an-unset-bound-never-authorises-a-kill" as *u8, rg_silent_stale(st, 0 - 1, 0) == 0, ctr)
110
111 gv_puts("--- vs Kubernetes CrashLoopBackOff / systemd (sourced: rel_erlang_otp + rel_backoff + rel_circuit_breaker) ---\n" as *u8)
112 gv_puts("containment: systemd flat RestartSec / old-guard 5-per-window-forever ; nishi CAPPED EXPONENTIAL (loop -> 1 restart per cap) -> EXCEEDS\n" as *u8)
113 gv_puts("auto-recovery: healthy uptime >= cap resets the counter (K8s CrashLoopBackOff parity) -> PARITY\n" as *u8)
114 gv_puts("replayable supervision: deterministic on `now` (not wall-clock) -> EXCEEDS\n" as *u8)
115 gv_puts("liveness: name-alive AND cycle-advancing (stale-cycle law) -- systemd watchdog parity via sd_notify, without the daemon opting in -> PARITY\n" as *u8)
116 return gv_verdict("nx_restart_guard_gate" as *u8, ctr, "crash-loop guard, bind-grace, stale-cycle and silent-name liveness predicates driven directly" as *u8)
117}