nx_retry_policy.nx source
↩ module page · 323 lines · 16195 B
1// nx_retry_policy.nx -- exponential backoff + jitter + max-attempts.
2//
3// module: nishi-core.ingest.retry_policy
4// depends: nishi-core.io.syscalls, nishi-core.io.iso8601
5// disk_kb: 5
6// capability: CORE_IO
7//
8// license_tier: PUBLIC_NISHI_SUBSTRATE
9// genealogy_id: aws_architecture_blog_exponential_backoff_jitter_2015 +
10// polly_microsoft_resilience_library +
11// google_sre_book_retry_amplification +
12// nishi_ingestion_s_class_cardinal_2026
13//
14// Retry primitive with exponential backoff + jitter + max-attempts.
15// Composes against nx_circuit_breaker (trips when retries exhausted).
16// Substrate Cardinal 14 graceful-degradation: every NX-INGEST adapter
17// invokes this primitive to decide whether to retry a failed
18// upstream call + how long to wait.
19//
20// ===== Why exponential backoff + jitter ===========================
21//
22// AWS Architectural Blog 2015: when many clients retry simultaneously
23// (thundering herd), they amplify the upstream load just as upstream
24// is recovering. Exponential backoff alone DOESN'T fix this — all
25// clients still retry at the same exponentially-spaced times.
26// JITTER (random offset within the backoff window) breaks the
27// synchronization.
28//
29// Google SRE book: retry amplification is the #1 cause of cascading
30// outages. Bounded max-attempts + jittered backoff prevents
31// substrate from contributing to upstream failure.
32//
33// ===== Backoff formula ============================================
34//
35// attempt_n: wait base * 2^n milliseconds, capped at max_wait_seconds
36// jitter: rs_jitter_ms bounds the delay to [d/2, d]
37// total: sum across attempts capped at attempts_cap
38//
39// ===== CORRECTED 2026-08-21 -- THIS BLOCK DESCRIBED A THING THE CODE DID NOT DO ====
40//
41// The line above used to read `multiply by random(0.5, 1.5) -- "full jitter" variant`, and it was wrong
42// twice over. (1) The code performed NO randomization at all: nx_retry_compute_wait_seconds returned the
43// bare capped exponential while nx_retry_policy_new set jitter_enabled = 1, so a caller reading that flag
44// believed the herd was decorrelated when every retry landed on the same instant. A FLAG THAT REPORTS A
45// CAPABILITY THE CODE DOES NOT PERFORM IS THE FABRICATED-CONSTANT DEFECT WEARING A FEATURE TOGGLE, AND IT
46// FAILS IN THE FLATTERING DIRECTION, SO NOBODY INVESTIGATES. (2) `random(0.5,1.5)` is not AWS's "full
47// jitter" (that is random(0, backoff)); naming the wrong scheme makes the next reader implement the wrong
48// thing. What ships now is the estate's incumbent, rs_jitter_ms: an EQUAL-jitter draw bounded to
49// [d/2, d], deterministic from a seed so it stays gateable offline.
50//
51// THE HONEST LIMIT, STATED WHERE IT CANNOT BE MISSED: jitter is applied in MILLISECONDS and BEFORE the
52// round-up to whole seconds, because rounding first destroys it. Even so, the SECONDS view cannot express
53// decorrelation below about a second -- at base 250 ms the first attempt draws from [125,250] ms and both
54// ends round up to 1 s. That is why next_wait_ms exists and is the field a caller should sleep on;
55// next_wait_seconds is a coarse view kept for reporting. Jittering only the seconds value would have been
56// a fix that is arithmetically inert for the first two attempts -- the same defect in a new place.
57//
58// Defaults:
59// base_ms = 250
60// max_wait_seconds = 60
61// max_attempts = 7 (totals ~3 min cumulative wait)
62
63// nx_safety_envelope:
64// intended_use: AUTO_APPLIED -- primitive-specific tuning queued
65// sil_target: SIL1
66// evidence: [bulk_applied_2026-05-16, see-file-comment-for-detail]
67// verdict: NOT_YET_EVALUATED
68
69import "nx_syscalls.nx"
70import "nx_iso8601.nx"
71// COMPOSE THE RULER, DO NOT RE-IMPLEMENT IT. rs_jitter_ms (nx_restart_strategy) is the estate's ONE
72// bounded-jitter function -- pure, deterministic from a seed, and already gate-proven by
73// nx_restart_strategy_gate T5 ("jitter bounded [d/2,d] (no thundering herd)", 6/6 GREEN). Writing a
74// second one here would be the duplicate-ruler defect, and the second copy is always the one that drifts.
75import "nx_restart_strategy.nx"
76
77// ===== Verdict ====================================================
78
79const NX_RETRY_PROCEED: i64 = 1 // try the upstream call
80const NX_RETRY_WAIT_AND_RETRY: i64 = 2 // wait then retry
81const NX_RETRY_EXHAUSTED: i64 = 3 // max attempts reached; give up
82const NX_RETRY_NON_RETRYABLE: i64 = 4 // error class not retryable (auth fail etc.)
83const NX_RETRY_CIRCUIT_OPEN: i64 = 5 // breaker says short-circuit
84
85func nx_retry_verdict_name(v: i64) -> *u8 {
86 if v == NX_RETRY_PROCEED { return "PROCEED" }
87 if v == NX_RETRY_WAIT_AND_RETRY { return "WAIT_AND_RETRY" }
88 if v == NX_RETRY_EXHAUSTED { return "EXHAUSTED" }
89 if v == NX_RETRY_NON_RETRYABLE { return "NON_RETRYABLE" }
90 if v == NX_RETRY_CIRCUIT_OPEN { return "CIRCUIT_OPEN" }
91 return "UNKNOWN"
92}
93
94// ===== Error-class sealed enum ====================================
95//
96// Different upstream-error classes have different retryability.
97// Substrate doesn't retry 4xx (caller error); does retry 5xx +
98// timeouts.
99
100const NX_RETRY_ERROR_TIMEOUT: i64 = 1 // retryable
101const NX_RETRY_ERROR_CONN_REFUSED: i64 = 2 // retryable
102const NX_RETRY_ERROR_5XX_SERVER: i64 = 3 // retryable (server-side)
103const NX_RETRY_ERROR_429_RATE_LIMIT: i64 = 4 // retryable with Retry-After
104const NX_RETRY_ERROR_4XX_CLIENT: i64 = 5 // NOT retryable (our request bad)
105const NX_RETRY_ERROR_401_AUTH: i64 = 6 // NOT retryable (need new creds)
106const NX_RETRY_ERROR_403_FORBIDDEN: i64 = 7 // NOT retryable
107const NX_RETRY_ERROR_404_NOT_FOUND: i64 = 8 // NOT retryable (doesn't exist)
108const NX_RETRY_ERROR_TLS_FAIL: i64 = 9 // retryable (transient TLS issue)
109const NX_RETRY_ERROR_DNS_FAIL: i64 = 10 // retryable
110const NX_RETRY_ERROR_NETWORK_OTHER: i64 = 11 // retryable conservatively
111
112func nx_retry_error_class_name(c: i64) -> *u8 {
113 if c == NX_RETRY_ERROR_TIMEOUT { return "TIMEOUT" }
114 if c == NX_RETRY_ERROR_CONN_REFUSED { return "CONN_REFUSED" }
115 if c == NX_RETRY_ERROR_5XX_SERVER { return "5XX_SERVER" }
116 if c == NX_RETRY_ERROR_429_RATE_LIMIT { return "429_RATE_LIMIT" }
117 if c == NX_RETRY_ERROR_4XX_CLIENT { return "4XX_CLIENT" }
118 if c == NX_RETRY_ERROR_401_AUTH { return "401_AUTH" }
119 if c == NX_RETRY_ERROR_403_FORBIDDEN { return "403_FORBIDDEN" }
120 if c == NX_RETRY_ERROR_404_NOT_FOUND { return "404_NOT_FOUND" }
121 if c == NX_RETRY_ERROR_TLS_FAIL { return "TLS_FAIL" }
122 if c == NX_RETRY_ERROR_DNS_FAIL { return "DNS_FAIL" }
123 if c == NX_RETRY_ERROR_NETWORK_OTHER { return "NETWORK_OTHER" }
124 return "UNKNOWN"
125}
126
127func nx_retry_error_is_retryable(c: i64) -> i64 {
128 if c == NX_RETRY_ERROR_4XX_CLIENT { return 0 }
129 if c == NX_RETRY_ERROR_401_AUTH { return 0 }
130 if c == NX_RETRY_ERROR_403_FORBIDDEN { return 0 }
131 if c == NX_RETRY_ERROR_404_NOT_FOUND { return 0 }
132 return 1
133}
134
135// ===== RetryPolicy struct =========================================
136
137struct RetryPolicy {
138 policy_hk: i64,
139 descriptor_hk: i64,
140 // Configuration (Cardinal 11: data-driven, not magic numbers)
141 base_wait_ms: i64, // initial wait window
142 max_wait_seconds: i64, // cap per-attempt wait
143 max_attempts: i64, // total attempts allowed (1 = no retry)
144 jitter_enabled: i64, // 1 = full jitter, 0 = deterministic
145 // Per-call state
146 current_attempt: i64, // 0-indexed
147 next_wait_ms: i64, // THE FIELD TO SLEEP ON -- jittered, millisecond resolution
148 next_wait_seconds: i64, // coarse view of next_wait_ms, rounded UP -- reporting only
149 total_wait_seconds: i64, // cumulative across attempts
150 // Diagnostics
151 n_total_invocations: i64,
152 n_total_retries: i64,
153 n_total_exhausted: i64,
154 last_error_class: i64,
155}
156
157// A HAND-COUNTED LENGTH BESIDE A STRUCT IS A SECOND COPY OF THAT STRUCT'S SHAPE, AND THE TWO DRIFT.
158// MEASURED 2026-08-21: this read `96 // 12 fields * 8 bytes` while RetryPolicy already carried THIRTEEN
159// fields, so nx_retry_policy_new mmap'd 96 bytes and the write to last_error_class landed 8 bytes past
160// the allocation. sys_mmap rounds to a page, so the overrun was benign in practice and had NO SYMPTOM --
161// which is exactly why it survived. A size constant that disagrees with its own struct is a defect
162// generator whose first real victim is whoever next changes the layout. The count is now stated ONCE and
163// the byte figure DERIVED from it, so adding a field is one edit instead of two that can disagree.
164const NX_RETRY_POLICY_FIELDS: i64 = 14
165const NX_RETRY_POLICY_BYTES: i64 = NX_RETRY_POLICY_FIELDS * 8
166
167// ===== Defaults ===================================================
168
169const NX_RETRY_DEFAULT_BASE_WAIT_MS: i64 = 250
170const NX_RETRY_DEFAULT_MAX_WAIT_SEC: i64 = 60
171const NX_RETRY_DEFAULT_MAX_ATTEMPTS: i64 = 7
172
173// ===== Constructor ================================================
174
175func nx_retry_policy_new(descriptor_hk: i64) -> *RetryPolicy {
176 let raw: *u8 = sys_mmap(NX_RETRY_POLICY_BYTES)
177 let p: *RetryPolicy = raw as *RetryPolicy
178 p.policy_hk = 0
179 p.descriptor_hk = descriptor_hk
180 p.base_wait_ms = NX_RETRY_DEFAULT_BASE_WAIT_MS
181 p.max_wait_seconds = NX_RETRY_DEFAULT_MAX_WAIT_SEC
182 p.max_attempts = NX_RETRY_DEFAULT_MAX_ATTEMPTS
183 p.jitter_enabled = 1
184 p.current_attempt = 0
185 p.next_wait_ms = 0
186 p.next_wait_seconds = 0
187 p.total_wait_seconds = 0
188 p.n_total_invocations = 0
189 p.n_total_retries = 0
190 p.n_total_exhausted = 0
191 p.last_error_class = 0
192 return p
193}
194
195// ===== Compute next wait (exponential backoff) ===================
196//
197// wait_ms = min(base * 2^attempt, max_wait_ms), then drawn down into [wait_ms/2, wait_ms] whenever
198// jitter_enabled is 1. The old "multiply by 1.0" v1 is GONE, not deferred.
199
200const NX_RETRY_MS_PER_SEC: i64 = 1000
201// 2^30 * base_wait_ms already dwarfs any sane max_wait_seconds; the clamp exists to keep the shift from
202// running off the end of an i64, not to express a policy.
203const NX_RETRY_SHIFT_CAP: i64 = 30
204
205// THE SEED. Jitter only decorrelates a herd if two DIFFERENT callers retrying at the SAME instant draw
206// DIFFERENT delays -- a seed of `now` alone moves every client together and changes nothing. descriptor_hk
207// supplies that per-caller identity; n_total_invocations makes successive attempts by one caller differ
208// too. Deterministic by construction, so the whole path stays unit-gateable offline with no clock and no
209// entropy source.
210// STATED IMPRECISION: this is a SUM, not a hash. Two callers whose (descriptor_hk + invocations) differ by
211// an exact multiple of the draw modulus still collide. That is acceptable HERE because the goal is
212// spreading a herd, not unpredictability -- and it is exactly why this must never be reused anywhere a
213// value has to be unguessable.
214func nx_retry_seed(p: *RetryPolicy, now_unix: i64) -> i64 {
215 if p == 0 as *RetryPolicy { return now_unix }
216 return now_unix + p.descriptor_hk + p.n_total_invocations
217}
218
219// Capped exponential backoff in MILLISECONDS, jittered. This is the primitive; the seconds view composes
220// it. Jitter is applied HERE, before any rounding, because rounding first destroys it.
221func nx_retry_compute_wait_ms(p: *RetryPolicy, seed: i64) -> i64 {
222 if p == 0 as *RetryPolicy { return 0 }
223 if p.current_attempt < 0 { return 0 }
224 var shift: i64 = p.current_attempt
225 if shift > NX_RETRY_SHIFT_CAP { shift = NX_RETRY_SHIFT_CAP }
226 let exp_ms: i64 = p.base_wait_ms << shift
227 let max_ms: i64 = p.max_wait_seconds * NX_RETRY_MS_PER_SEC
228 var wait_ms: i64 = exp_ms
229 if wait_ms > max_ms { wait_ms = max_ms }
230 if p.jitter_enabled == 1 { return rs_jitter_ms(wait_ms, seed) }
231 return wait_ms
232}
233
234// Coarse view: the jittered millisecond delay rounded UP to whole seconds, so a caller that can only
235// sleep in seconds never retries EARLIER than the backoff asked for.
236func nx_retry_compute_wait_seconds(p: *RetryPolicy, seed: i64) -> i64 {
237 let ms: i64 = nx_retry_compute_wait_ms(p, seed)
238 return (ms + NX_RETRY_MS_PER_SEC - 1) / NX_RETRY_MS_PER_SEC
239}
240
241// ===== Decide next action (the verdict entry-point) ==============
242//
243// Called after each upstream attempt fails. Caller passes the
244// error class; substrate returns verdict + populates
245// p.next_wait_seconds for the caller to sleep before retry.
246
247func nx_retry_policy_decide(
248 p: *RetryPolicy,
249 error_class: i64,
250 now_unix: i64
251) -> i64 {
252 if p == 0 as *RetryPolicy { return NX_RETRY_EXHAUSTED }
253 p.n_total_invocations = p.n_total_invocations + 1
254 p.last_error_class = error_class
255
256 // Non-retryable error classes
257 if nx_retry_error_is_retryable(error_class) == 0 {
258 return NX_RETRY_NON_RETRYABLE
259 }
260
261 // Increment attempt counter
262 p.current_attempt = p.current_attempt + 1
263
264 // Exhausted?
265 if p.current_attempt >= p.max_attempts {
266 p.n_total_exhausted = p.n_total_exhausted + 1
267 return NX_RETRY_EXHAUSTED
268 }
269
270 // Compute next wait. ONE draw feeds BOTH views, so next_wait_ms and next_wait_seconds can never
271 // disagree about which delay was actually chosen. Two independent draws here would be a second ruler,
272 // and the seconds field would then be rounding a number nobody ever slept on.
273 let seed: i64 = nx_retry_seed(p, now_unix)
274 p.next_wait_ms = nx_retry_compute_wait_ms(p, seed)
275 p.next_wait_seconds = (p.next_wait_ms + NX_RETRY_MS_PER_SEC - 1) / NX_RETRY_MS_PER_SEC
276 p.total_wait_seconds = p.total_wait_seconds + p.next_wait_seconds
277 p.n_total_retries = p.n_total_retries + 1
278 return NX_RETRY_WAIT_AND_RETRY
279}
280
281// ===== Reset on successful call ===================================
282//
283// After a successful upstream call, the retry policy state is reset
284// so the next failure starts the backoff sequence fresh.
285
286func nx_retry_policy_record_success(p: *RetryPolicy) -> i64 {
287 if p == 0 as *RetryPolicy { return -1 }
288 p.current_attempt = 0
289 p.next_wait_ms = 0
290 p.next_wait_seconds = 0
291 return 0
292}
293
294// ===== RFC 6585 Retry-After integration ==========================
295//
296// When upstream returns 429 with Retry-After header, that value
297// SUPERSEDES our exponential backoff (RFC mandates respecting it).
298// Caller passes the retry-after seconds; policy honors it for next
299// wait.
300
301// CORRECTED 2026-08-21: RFC 9110's Retry-After supersedes our backoff, and a BARE SCALAR SYNCHRONISES THE
302// HERD BY CONSTRUCTION -- every client told "60" wakes at the same instant, which is the stampede this
303// whole file exists to prevent. The header carries no range, no jitter and no accuracy obligation, so
304// honouring it verbatim reintroduces the defect at the one moment the server is already overloaded.
305// So the value is treated as a FLOOR and the jitter is added ABOVE it, NEVER below: retrying EARLIER than
306// a server asked would violate the header, so the direction of the draw matters and is inverted here on
307// purpose. The spread is the policy's own base_wait_ms -- DERIVED from configuration already present, not
308// a number invented at this line -- giving a wait in [retry_after, retry_after + base_wait_ms/2].
309func nx_retry_apply_retry_after(p: *RetryPolicy, retry_after_seconds: i64, seed: i64) -> i64 {
310 if p == 0 as *RetryPolicy { return -1 }
311 if retry_after_seconds <= 0 { return -1 }
312 let floor_ms: i64 = retry_after_seconds * NX_RETRY_MS_PER_SEC
313 var ms: i64 = floor_ms
314 if p.jitter_enabled == 1 {
315 // rs_jitter_ms draws DOWNWARD into [d/2, d]; its complement (d - draw) is an upward offset in
316 // [0, d/2], which is what a floor needs.
317 ms = floor_ms + (p.base_wait_ms - rs_jitter_ms(p.base_wait_ms, seed))
318 }
319 p.next_wait_ms = ms
320 p.next_wait_seconds = (ms + NX_RETRY_MS_PER_SEC - 1) / NX_RETRY_MS_PER_SEC
321 p.total_wait_seconds = p.total_wait_seconds + p.next_wait_seconds
322 return 0
323}