nx_supervisor_test.nx source
↩ module page · 156 lines · 7616 B
1// nx_supervisor_test.nx -- state-machine smoke for nx_supervisor.
2//
3// Exercises every transition + the exponential-backoff curve + the
4// healthy-window reset + the escalation gate.
5//
6// expect_exit: 0
7//
8// license_tier: ORIGINAL
9
10import "nx_syscalls_x86_64.nx"
11import "nx_supervisor.nx"
12
13func bytes_eq(a: *u8, b: *u8, n: i64) -> i64 {
14 var i: i64 = 0
15 while i < n {
16 if a[i] != b[i] { return 0 }
17 i = i + 1
18 }
19 return 1
20}
21
22func main() -> i64 {
23 // ---- State + action enum gates ----
24 if nx_supervisor_state_is_valid(NX_SUP_HEALTHY) != 1 { return 1 }
25 if nx_supervisor_state_is_valid(NX_SUP_ESCALATING) != 1 { return 2 }
26 if nx_supervisor_state_is_valid(NX_SUP_STATE_N) != 0 { return 3 }
27 if nx_supervisor_action_is_valid(NX_SUP_ACTION_RESTART) != 1 { return 4 }
28 if nx_supervisor_action_is_valid(NX_SUP_ACTION_N) != 0 { return 5 }
29
30 if bytes_eq(nx_supervisor_state_name(NX_SUP_HEALTHY), "HEALTHY" as *u8, 7) != 1 { return 10 }
31 if bytes_eq(nx_supervisor_state_name(NX_SUP_BACKING_OFF), "BACKING_OFF" as *u8, 11) != 1 { return 11 }
32 if bytes_eq(nx_supervisor_state_name(NX_SUP_ESCALATING), "ESCALATING" as *u8, 10) != 1 { return 12 }
33 if bytes_eq(nx_supervisor_action_name(NX_SUP_ACTION_ESCALATE), "ESCALATE" as *u8, 8) != 1 { return 13 }
34 if bytes_eq(nx_supervisor_action_name(NX_SUP_ACTION_RESTART), "RESTART" as *u8, 7) != 1 { return 14 }
35
36 // ---- Init ----
37 let sup_raw: *u8 = sys_mmap(NX_SUPERVISOR_BYTES)
38 let sup: *NxSupervisor = sup_raw as *NxSupervisor
39 // max=5 restarts, base=100ms, max=10000ms, healthy=60000ms
40 let rc: i64 = nx_supervisor_init(sup, 5, 100, 10000, 60000)
41 if rc != 0 { return 20 }
42
43 // Initial state is HEALTHY with 0 restarts.
44 if nx_supervisor_state(sup) != NX_SUP_HEALTHY { return 21 }
45 if nx_supervisor_n_restarts(sup) != 0 { return 22 }
46
47 // ---- BAD_ARG paths ----
48 if nx_supervisor_init(0 as *NxSupervisor, 5, 100, 10000, 60000) != -1 { return 30 }
49 if nx_supervisor_init(sup, -1, 100, 10000, 60000) != -1 { return 31 }
50 if nx_supervisor_init(sup, 5, -1, 10000, 60000) != -1 { return 32 }
51 if nx_supervisor_init(sup, 5, 1000, 100, 60000) != -1 { return 33 } // max<base
52 if nx_supervisor_init(sup, 5, 100, 10000, -1) != -1 { return 34 }
53 // Re-init for the test (previous attempts may have partial state).
54 nx_supervisor_init(sup, 5, 100, 10000, 60000)
55
56 // ---- decide() while HEALTHY + alive ----
57 if nx_supervisor_decide(sup, 1, 1000) != NX_SUP_ACTION_NONE { return 40 }
58
59 // ---- First crash -> BACKING_OFF with 100ms wait ----
60 let crash_t: i64 = 2000
61 nx_supervisor_record_crash(sup, crash_t)
62 if nx_supervisor_state(sup) != NX_SUP_BACKING_OFF { return 50 }
63 if nx_supervisor_n_restarts(sup) != 1 { return 51 }
64
65 // decide() at crash_t = WAIT (next_check = 2100)
66 if nx_supervisor_decide(sup, 0, crash_t) != NX_SUP_ACTION_WAIT { return 52 }
67 // decide() at 2099 = WAIT
68 if nx_supervisor_decide(sup, 0, 2099) != NX_SUP_ACTION_WAIT { return 53 }
69 // decide() at 2100 = RESTART
70 if nx_supervisor_decide(sup, 0, 2100) != NX_SUP_ACTION_RESTART { return 54 }
71 // decide() at 5000 = RESTART (still in backoff state until record_start)
72 if nx_supervisor_decide(sup, 0, 5000) != NX_SUP_ACTION_RESTART { return 55 }
73
74 // ---- record_start -> HEALTHY ----
75 nx_supervisor_record_start(sup, 5100)
76 if nx_supervisor_state(sup) != NX_SUP_HEALTHY { return 60 }
77 if nx_supervisor_decide(sup, 1, 6000) != NX_SUP_ACTION_NONE { return 61 }
78
79 // ---- Second crash within healthy window -> backoff 200ms ----
80 let crash2_t: i64 = 7000 // within 60000ms of crash1 -> same storm
81 nx_supervisor_record_crash(sup, crash2_t)
82 if nx_supervisor_n_restarts(sup) != 2 { return 70 }
83 // backoff = base * 2^(2-1) = 200ms
84 let bf: i64 = nx_supervisor_backoff_ms(sup)
85 if bf != 200 { return 71 }
86 // next_check = 7000 + 200 = 7200
87 if nx_supervisor_decide(sup, 0, 7199) != NX_SUP_ACTION_WAIT { return 72 }
88 if nx_supervisor_decide(sup, 0, 7200) != NX_SUP_ACTION_RESTART { return 73 }
89
90 // ---- Crash storm: drive past max_restarts=5 -> ESCALATING ----
91 nx_supervisor_record_start(sup, 7300)
92 nx_supervisor_record_crash(sup, 7400) // n=3, backoff=400
93 if nx_supervisor_n_restarts(sup) != 3 { return 80 }
94 if nx_supervisor_backoff_ms(sup) != 400 { return 81 }
95 nx_supervisor_record_start(sup, 7500)
96 nx_supervisor_record_crash(sup, 7600) // n=4, backoff=800
97 if nx_supervisor_n_restarts(sup) != 4 { return 82 }
98 if nx_supervisor_backoff_ms(sup) != 800 { return 83 }
99 nx_supervisor_record_start(sup, 7700)
100 nx_supervisor_record_crash(sup, 7800) // n=5, backoff=1600
101 if nx_supervisor_n_restarts(sup) != 5 { return 84 }
102 if nx_supervisor_backoff_ms(sup) != 1600 { return 85 }
103 nx_supervisor_record_start(sup, 7900)
104 nx_supervisor_record_crash(sup, 8000) // n=6 -> ESCALATING
105 if nx_supervisor_state(sup) != NX_SUP_ESCALATING { return 90 }
106 if nx_supervisor_n_escalations(sup) != 1 { return 91 }
107
108 // decide() in ESCALATING state always returns ESCALATE
109 if nx_supervisor_decide(sup, 0, 8100) != NX_SUP_ACTION_ESCALATE { return 92 }
110 if nx_supervisor_decide(sup, 1, 9000) != NX_SUP_ACTION_ESCALATE { return 93 }
111
112 // ---- Backoff cap: deep storm hits max_backoff_ms ----
113 // Re-init with small max_backoff to test cap.
114 let sup2_raw: *u8 = sys_mmap(NX_SUPERVISOR_BYTES)
115 let sup2: *NxSupervisor = sup2_raw as *NxSupervisor
116 nx_supervisor_init(sup2, 100, 100, 500, 60000) // base 100, max 500
117 nx_supervisor_record_crash(sup2, 1000) // n=1, backoff = 100
118 if nx_supervisor_backoff_ms(sup2) != 100 { return 100 }
119 nx_supervisor_record_start(sup2, 1100)
120 nx_supervisor_record_crash(sup2, 1200) // n=2, backoff = 200
121 if nx_supervisor_backoff_ms(sup2) != 200 { return 101 }
122 nx_supervisor_record_start(sup2, 1300)
123 nx_supervisor_record_crash(sup2, 1400) // n=3, backoff = 400
124 if nx_supervisor_backoff_ms(sup2) != 400 { return 102 }
125 nx_supervisor_record_start(sup2, 1500)
126 nx_supervisor_record_crash(sup2, 1600) // n=4, candidate=800 capped to 500
127 if nx_supervisor_backoff_ms(sup2) != 500 { return 103 }
128 nx_supervisor_record_start(sup2, 1700)
129 nx_supervisor_record_crash(sup2, 1800) // n=5, candidate=1600 capped to 500
130 if nx_supervisor_backoff_ms(sup2) != 500 { return 104 }
131
132 // ---- Healthy-window reset ----
133 let sup3_raw: *u8 = sys_mmap(NX_SUPERVISOR_BYTES)
134 let sup3: *NxSupervisor = sup3_raw as *NxSupervisor
135 nx_supervisor_init(sup3, 5, 100, 10000, 60000) // healthy=60s
136 nx_supervisor_record_crash(sup3, 1000) // n=1
137 if nx_supervisor_n_restarts(sup3) != 1 { return 110 }
138 nx_supervisor_record_start(sup3, 1100)
139 // Wait LONG past healthy_threshold -> next crash resets counter
140 nx_supervisor_record_crash(sup3, 1100 + 60000 + 1) // 61101 ms; n should reset to 1
141 if nx_supervisor_n_restarts(sup3) != 1 { return 111 }
142 // Within window: counter accumulates
143 nx_supervisor_record_start(sup3, 61200)
144 nx_supervisor_record_crash(sup3, 61300) // within 60000ms -> n=2
145 if nx_supervisor_n_restarts(sup3) != 2 { return 112 }
146
147 // ---- Terminate ----
148 nx_supervisor_terminate(sup3)
149 if nx_supervisor_state(sup3) != NX_SUP_TERMINATED { return 120 }
150 if nx_supervisor_decide(sup3, 0, 100000) != NX_SUP_ACTION_NONE { return 121 }
151 // record_crash after TERMINATED is no-op (state stays terminated)
152 nx_supervisor_record_crash(sup3, 100100)
153 if nx_supervisor_state(sup3) != NX_SUP_TERMINATED { return 122 }
154
155 return 0
156}