nx_circuit_breaker.nx source
↩ module page · 230 lines · 9045 B
1// nx_circuit_breaker.nx -- 3-state circuit breaker (Hystrix pattern).
2//
3// module: nishi-core.ingest.circuit_breaker
4// depends: nishi-core.io.syscalls, nishi-core.io.iso8601
5// disk_kb: 4
6// capability: CORE_IO
7//
8// license_tier: PUBLIC_NISHI_SUBSTRATE
9// genealogy_id: nygard_2007_release_it_stability_patterns +
10// netflix_hystrix_circuit_breaker_pattern +
11// resilience4j_circuit_breaker_states +
12// nishi_cardinal_14_graceful_degradation
13//
14// Three-state circuit breaker primitive. Substrate Cardinal 14
15// (graceful degradation) guard against runaway-failing upstreams:
16// when an upstream returns errors at high frequency, the breaker
17// trips OPEN and short-circuits subsequent calls without hitting
18// the upstream. After a cool-down window, the breaker transitions
19// to HALF_OPEN to test recovery; success closes it, failure re-trips.
20//
21// Per Pillar 5 equipment-health-verdict pattern applied at the
22// ingestion-substrate layer: every adapter gets a breaker
23// automatically; the substrate's poll-loop honors the verdict.
24//
25// ===== Why three states ===========================================
26//
27// CLOSED requests flow through; failures counted toward threshold
28// OPEN requests short-circuited; counter timer running
29// HALF_OPEN probe requests allowed; success → CLOSED, failure → OPEN
30
31// nx_safety_envelope:
32// intended_use: AUTO_APPLIED -- primitive-specific tuning queued
33// sil_target: SIL1
34// evidence: [bulk_applied_2026-05-16, see-file-comment-for-detail]
35// verdict: NOT_YET_EVALUATED
36
37import "nx_syscalls.nx"
38import "nx_iso8601.nx"
39
40// ===== State sealed enum ==========================================
41
42const NX_BREAKER_CLOSED: i64 = 1
43const NX_BREAKER_OPEN: i64 = 2
44const NX_BREAKER_HALF_OPEN: i64 = 3
45
46func nx_breaker_state_name(s: i64) -> *u8 {
47 if s == NX_BREAKER_CLOSED { return "CLOSED" }
48 if s == NX_BREAKER_OPEN { return "OPEN" }
49 if s == NX_BREAKER_HALF_OPEN { return "HALF_OPEN" }
50 return "UNKNOWN"
51}
52
53// ===== Verdict ====================================================
54
55const NX_BREAKER_VERDICT_PROCEED: i64 = 1
56const NX_BREAKER_VERDICT_SHORT_CIRCUIT: i64 = 2 // OPEN — don't call upstream
57const NX_BREAKER_VERDICT_PROBE: i64 = 3 // HALF_OPEN — single probe call
58
59func nx_breaker_verdict_name(v: i64) -> *u8 {
60 if v == NX_BREAKER_VERDICT_PROCEED { return "PROCEED" }
61 if v == NX_BREAKER_VERDICT_SHORT_CIRCUIT { return "SHORT_CIRCUIT" }
62 if v == NX_BREAKER_VERDICT_PROBE { return "PROBE" }
63 return "UNKNOWN"
64}
65
66// ===== CircuitBreaker struct ======================================
67
68struct CircuitBreaker {
69 breaker_hk: i64,
70 descriptor_hk: i64, // FK to SourceDescriptor
71 state: i64,
72 state_entered_unix: i64,
73 // CLOSED-state counters
74 consecutive_success_count: i64,
75 consecutive_failure_count: i64,
76 rolling_failure_count: i64, // failures in last `window_seconds`
77 rolling_request_count: i64, // requests in last `window_seconds`
78 // Thresholds (Cardinal 11: configurable, not magic)
79 failure_threshold_pct_q10: i64, // e.g. 51 = 50% failure rate (Q10)
80 min_requests_for_trip: i64, // need at least N requests in window to trip
81 cooldown_seconds: i64, // OPEN → HALF_OPEN cooldown
82 half_open_probe_count: i64, // probes allowed in HALF_OPEN
83 // Diagnostics
84 n_trips_total: i64,
85 last_trip_unix: i64,
86 last_recovery_unix: i64,
87}
88
89const NX_CIRCUIT_BREAKER_BYTES: i64 = 120 // 15 fields * 8 bytes
90
91// ===== Default thresholds =========================================
92
93const NX_BREAKER_DEFAULT_FAILURE_PCT_Q10: i64 = 512 // 50% in Q10
94const NX_BREAKER_DEFAULT_MIN_REQUESTS: i64 = 10
95const NX_BREAKER_DEFAULT_COOLDOWN_SEC: i64 = 60
96const NX_BREAKER_DEFAULT_HALF_OPEN_PROBES: i64 = 3
97
98// ===== Constructor ================================================
99
100func nx_circuit_breaker_new(descriptor_hk: i64, now_unix: i64) -> *CircuitBreaker {
101 let raw: *u8 = sys_mmap(NX_CIRCUIT_BREAKER_BYTES)
102 let b: *CircuitBreaker = raw as *CircuitBreaker
103 b.breaker_hk = 0
104 b.descriptor_hk = descriptor_hk
105 b.state = NX_BREAKER_CLOSED
106 b.state_entered_unix = now_unix
107 b.consecutive_success_count = 0
108 b.consecutive_failure_count = 0
109 b.rolling_failure_count = 0
110 b.rolling_request_count = 0
111 b.failure_threshold_pct_q10 = NX_BREAKER_DEFAULT_FAILURE_PCT_Q10
112 b.min_requests_for_trip = NX_BREAKER_DEFAULT_MIN_REQUESTS
113 b.cooldown_seconds = NX_BREAKER_DEFAULT_COOLDOWN_SEC
114 b.half_open_probe_count = NX_BREAKER_DEFAULT_HALF_OPEN_PROBES
115 b.n_trips_total = 0
116 b.last_trip_unix = 0
117 b.last_recovery_unix = 0
118 return b
119}
120
121// ===== Pre-call verdict (called before every upstream attempt) ===
122//
123// Returns the verdict telling the caller whether to proceed,
124// short-circuit, or attempt a probe.
125
126func nx_circuit_breaker_verdict(b: *CircuitBreaker, now_unix: i64) -> i64 {
127 if b == 0 as *CircuitBreaker { return NX_BREAKER_VERDICT_SHORT_CIRCUIT }
128
129 if b.state == NX_BREAKER_CLOSED { return NX_BREAKER_VERDICT_PROCEED }
130
131 if b.state == NX_BREAKER_OPEN {
132 // Cooldown check: transition to HALF_OPEN if window elapsed
133 let elapsed: i64 = now_unix - b.state_entered_unix
134 if elapsed >= b.cooldown_seconds {
135 b.state = NX_BREAKER_HALF_OPEN
136 b.state_entered_unix = now_unix
137 return NX_BREAKER_VERDICT_PROBE
138 }
139 return NX_BREAKER_VERDICT_SHORT_CIRCUIT
140 }
141
142 if b.state == NX_BREAKER_HALF_OPEN { return NX_BREAKER_VERDICT_PROBE }
143
144 return NX_BREAKER_VERDICT_SHORT_CIRCUIT
145}
146
147// ===== Post-call result handler ==================================
148//
149// Called by the adapter after every upstream attempt with the
150// outcome. Updates state machine.
151
152const NX_BREAKER_OUTCOME_SUCCESS: i64 = 1
153const NX_BREAKER_OUTCOME_FAILURE: i64 = 2
154
155func nx_circuit_breaker_record_outcome(b: *CircuitBreaker, outcome: i64, now_unix: i64) -> i64 {
156 if b == 0 as *CircuitBreaker { return -1 }
157
158 b.rolling_request_count = b.rolling_request_count + 1
159
160 if outcome == NX_BREAKER_OUTCOME_SUCCESS {
161 b.consecutive_success_count = b.consecutive_success_count + 1
162 b.consecutive_failure_count = 0
163 if b.state == NX_BREAKER_HALF_OPEN {
164 // Recovery — close the breaker
165 b.state = NX_BREAKER_CLOSED
166 b.state_entered_unix = now_unix
167 b.last_recovery_unix = now_unix
168 b.rolling_failure_count = 0
169 b.rolling_request_count = 0
170 return 0
171 }
172 }
173
174 if outcome == NX_BREAKER_OUTCOME_FAILURE {
175 b.consecutive_failure_count = b.consecutive_failure_count + 1
176 b.consecutive_success_count = 0
177 b.rolling_failure_count = b.rolling_failure_count + 1
178
179 if b.state == NX_BREAKER_HALF_OPEN {
180 // Probe failed — re-trip OPEN
181 b.state = NX_BREAKER_OPEN
182 b.state_entered_unix = now_unix
183 b.n_trips_total = b.n_trips_total + 1
184 b.last_trip_unix = now_unix
185 return 0
186 }
187 if b.state == NX_BREAKER_CLOSED {
188 // Check if failure rate exceeds threshold
189 if b.rolling_request_count >= b.min_requests_for_trip {
190 let failure_pct_q10: i64 = (b.rolling_failure_count * 1024) / b.rolling_request_count
191 if failure_pct_q10 >= b.failure_threshold_pct_q10 {
192 b.state = NX_BREAKER_OPEN
193 b.state_entered_unix = now_unix
194 b.n_trips_total = b.n_trips_total + 1
195 b.last_trip_unix = now_unix
196 return 0
197 }
198 }
199 }
200 }
201 return 0
202}
203
204// ===== Force trip (manual / admin override) =======================
205//
206// Operations team can manually trip the breaker (e.g. during a
207// known upstream-incident) without waiting for failure-threshold.
208
209func nx_circuit_breaker_force_trip(b: *CircuitBreaker, now_unix: i64) -> i64 {
210 if b == 0 as *CircuitBreaker { return -1 }
211 b.state = NX_BREAKER_OPEN
212 b.state_entered_unix = now_unix
213 b.n_trips_total = b.n_trips_total + 1
214 b.last_trip_unix = now_unix
215 return 0
216}
217
218// ===== Force reset (manual / admin override) ======================
219
220func nx_circuit_breaker_force_reset(b: *CircuitBreaker, now_unix: i64) -> i64 {
221 if b == 0 as *CircuitBreaker { return -1 }
222 b.state = NX_BREAKER_CLOSED
223 b.state_entered_unix = now_unix
224 b.consecutive_success_count = 0
225 b.consecutive_failure_count = 0
226 b.rolling_failure_count = 0
227 b.rolling_request_count = 0
228 b.last_recovery_unix = now_unix
229 return 0
230}