nx_retry_policy.nx source

1// nx_retry_policy.nx -- exponential backoff + jitter + max-attempts. 2// 3// module: nishi-core.ingest.retry_policy 4// depends: nishi-core.io.syscalls, nishi-core.io.iso8601 5// disk_kb: 5 6// capability: CORE_IO 7// 8// license_tier: PUBLIC_NISHI_SUBSTRATE 9// genealogy_id: aws_architecture_blog_exponential_backoff_jitter_2015 + 10// polly_microsoft_resilience_library + 11// google_sre_book_retry_amplification + 12// nishi_ingestion_s_class_cardinal_2026 13// 14// Retry primitive with exponential backoff + jitter + max-attempts. 15// Composes against nx_circuit_breaker (trips when retries exhausted). 16// Substrate Cardinal 14 graceful-degradation: every NX-INGEST adapter 17// invokes this primitive to decide whether to retry a failed 18// upstream call + how long to wait. 19// 20// ===== Why exponential backoff + jitter =========================== 21// 22// AWS Architectural Blog 2015: when many clients retry simultaneously 23// (thundering herd), they amplify the upstream load just as upstream 24// is recovering. Exponential backoff alone DOESN'T fix this — all 25// clients still retry at the same exponentially-spaced times. 26// JITTER (random offset within the backoff window) breaks the 27// synchronization. 28// 29// Google SRE book: retry amplification is the #1 cause of cascading 30// outages. Bounded max-attempts + jittered backoff prevents 31// substrate from contributing to upstream failure. 32// 33// ===== Backoff formula ============================================ 34// 35// attempt_n: wait base * 2^n seconds, capped at max_wait_seconds 36// jitter: multiply by random(0.5, 1.5) — "full jitter" variant 37// total: sum across attempts capped at attempts_cap 38// 39// Defaults: 40// base_ms = 250 41// max_wait_seconds = 60 42// max_attempts = 7 (totals ~3 min cumulative wait) 43 44// nx_safety_envelope: 45// intended_use: AUTO_APPLIED -- primitive-specific tuning queued 46// sil_target: SIL1 47// evidence: [bulk_applied_2026-05-16, see-file-comment-for-detail] 48// verdict: NOT_YET_EVALUATED 49 50import "nx_syscalls.nx" 51import "nx_iso8601.nx" 52 53// ===== Verdict ==================================================== 54 55const NX_RETRY_PROCEED: i64 = 1 // try the upstream call 56const NX_RETRY_WAIT_AND_RETRY: i64 = 2 // wait then retry 57const NX_RETRY_EXHAUSTED: i64 = 3 // max attempts reached; give up 58const NX_RETRY_NON_RETRYABLE: i64 = 4 // error class not retryable (auth fail etc.) 59const NX_RETRY_CIRCUIT_OPEN: i64 = 5 // breaker says short-circuit 60 61func nx_retry_verdict_name(v: i64) -> *u8 { 62 if v == NX_RETRY_PROCEED { return "PROCEED" } 63 if v == NX_RETRY_WAIT_AND_RETRY { return "WAIT_AND_RETRY" } 64 if v == NX_RETRY_EXHAUSTED { return "EXHAUSTED" } 65 if v == NX_RETRY_NON_RETRYABLE { return "NON_RETRYABLE" } 66 if v == NX_RETRY_CIRCUIT_OPEN { return "CIRCUIT_OPEN" } 67 return "UNKNOWN" 68} 69 70// ===== Error-class sealed enum ==================================== 71// 72// Different upstream-error classes have different retryability. 73// Substrate doesn't retry 4xx (caller error); does retry 5xx + 74// timeouts. 75 76const NX_RETRY_ERROR_TIMEOUT: i64 = 1 // retryable 77const NX_RETRY_ERROR_CONN_REFUSED: i64 = 2 // retryable 78const NX_RETRY_ERROR_5XX_SERVER: i64 = 3 // retryable (server-side) 79const NX_RETRY_ERROR_429_RATE_LIMIT: i64 = 4 // retryable with Retry-After 80const NX_RETRY_ERROR_4XX_CLIENT: i64 = 5 // NOT retryable (our request bad) 81const NX_RETRY_ERROR_401_AUTH: i64 = 6 // NOT retryable (need new creds) 82const NX_RETRY_ERROR_403_FORBIDDEN: i64 = 7 // NOT retryable 83const NX_RETRY_ERROR_404_NOT_FOUND: i64 = 8 // NOT retryable (doesn't exist) 84const NX_RETRY_ERROR_TLS_FAIL: i64 = 9 // retryable (transient TLS issue) 85const NX_RETRY_ERROR_DNS_FAIL: i64 = 10 // retryable 86const NX_RETRY_ERROR_NETWORK_OTHER: i64 = 11 // retryable conservatively 87 88func nx_retry_error_class_name(c: i64) -> *u8 { 89 if c == NX_RETRY_ERROR_TIMEOUT { return "TIMEOUT" } 90 if c == NX_RETRY_ERROR_CONN_REFUSED { return "CONN_REFUSED" } 91 if c == NX_RETRY_ERROR_5XX_SERVER { return "5XX_SERVER" } 92 if c == NX_RETRY_ERROR_429_RATE_LIMIT { return "429_RATE_LIMIT" } 93 if c == NX_RETRY_ERROR_4XX_CLIENT { return "4XX_CLIENT" } 94 if c == NX_RETRY_ERROR_401_AUTH { return "401_AUTH" } 95 if c == NX_RETRY_ERROR_403_FORBIDDEN { return "403_FORBIDDEN" } 96 if c == NX_RETRY_ERROR_404_NOT_FOUND { return "404_NOT_FOUND" } 97 if c == NX_RETRY_ERROR_TLS_FAIL { return "TLS_FAIL" } 98 if c == NX_RETRY_ERROR_DNS_FAIL { return "DNS_FAIL" } 99 if c == NX_RETRY_ERROR_NETWORK_OTHER { return "NETWORK_OTHER" } 100 return "UNKNOWN" 101} 102 103func nx_retry_error_is_retryable(c: i64) -> i64 { 104 if c == NX_RETRY_ERROR_4XX_CLIENT { return 0 } 105 if c == NX_RETRY_ERROR_401_AUTH { return 0 } 106 if c == NX_RETRY_ERROR_403_FORBIDDEN { return 0 } 107 if c == NX_RETRY_ERROR_404_NOT_FOUND { return 0 } 108 return 1 109} 110 111// ===== RetryPolicy struct ========================================= 112 113struct RetryPolicy { 114 policy_hk: i64, 115 descriptor_hk: i64, 116 // Configuration (Cardinal 11: data-driven, not magic numbers) 117 base_wait_ms: i64, // initial wait window 118 max_wait_seconds: i64, // cap per-attempt wait 119 max_attempts: i64, // total attempts allowed (1 = no retry) 120 jitter_enabled: i64, // 1 = full jitter, 0 = deterministic 121 // Per-call state 122 current_attempt: i64, // 0-indexed 123 next_wait_seconds: i64, // computed by last invocation 124 total_wait_seconds: i64, // cumulative across attempts 125 // Diagnostics 126 n_total_invocations: i64, 127 n_total_retries: i64, 128 n_total_exhausted: i64, 129 last_error_class: i64, 130} 131 132const NX_RETRY_POLICY_BYTES: i64 = 96 // 12 fields * 8 bytes 133 134// ===== Defaults =================================================== 135 136const NX_RETRY_DEFAULT_BASE_WAIT_MS: i64 = 250 137const NX_RETRY_DEFAULT_MAX_WAIT_SEC: i64 = 60 138const NX_RETRY_DEFAULT_MAX_ATTEMPTS: i64 = 7 139 140// ===== Constructor ================================================ 141 142func nx_retry_policy_new(descriptor_hk: i64) -> *RetryPolicy { 143 let raw: *u8 = sys_mmap(NX_RETRY_POLICY_BYTES) 144 let p: *RetryPolicy = raw as *RetryPolicy 145 p.policy_hk = 0 146 p.descriptor_hk = descriptor_hk 147 p.base_wait_ms = NX_RETRY_DEFAULT_BASE_WAIT_MS 148 p.max_wait_seconds = NX_RETRY_DEFAULT_MAX_WAIT_SEC 149 p.max_attempts = NX_RETRY_DEFAULT_MAX_ATTEMPTS 150 p.jitter_enabled = 1 151 p.current_attempt = 0 152 p.next_wait_seconds = 0 153 p.total_wait_seconds = 0 154 p.n_total_invocations = 0 155 p.n_total_retries = 0 156 p.n_total_exhausted = 0 157 p.last_error_class = 0 158 return p 159} 160 161// ===== Compute next wait (exponential backoff) =================== 162// 163// wait_ms = min(base * 2^attempt, max_wait_ms) 164// If jitter: wait_ms = wait_ms * random(0.5, 1.5) 165// 166// v1 uses deterministic mid-jitter (multiply by 1.0); v1.1 wires 167// against nx_prng for true full-jitter when nx_prng matures. 168 169func nx_retry_compute_wait_seconds(p: *RetryPolicy) -> i64 { 170 if p == 0 as *RetryPolicy { return 0 } 171 if p.current_attempt < 0 { return 0 } 172 // 2^attempt with cap at max_wait 173 var shift: i64 = p.current_attempt 174 if shift > 30 { shift = 30 } // avoid overflow on i64 175 let exp_ms: i64 = p.base_wait_ms << shift 176 let max_ms: i64 = p.max_wait_seconds * 1000 177 var wait_ms: i64 = exp_ms 178 if wait_ms > max_ms { wait_ms = max_ms } 179 // v1 deterministic; jitter via nx_prng in v1.1 180 return (wait_ms + 999) / 1000 // round up to whole seconds 181} 182 183// ===== Decide next action (the verdict entry-point) ============== 184// 185// Called after each upstream attempt fails. Caller passes the 186// error class; substrate returns verdict + populates 187// p.next_wait_seconds for the caller to sleep before retry. 188 189func nx_retry_policy_decide( 190 p: *RetryPolicy, 191 error_class: i64, 192 now_unix: i64 193) -> i64 { 194 if p == 0 as *RetryPolicy { return NX_RETRY_EXHAUSTED } 195 p.n_total_invocations = p.n_total_invocations + 1 196 p.last_error_class = error_class 197 198 // Non-retryable error classes 199 if nx_retry_error_is_retryable(error_class) == 0 { 200 return NX_RETRY_NON_RETRYABLE 201 } 202 203 // Increment attempt counter 204 p.current_attempt = p.current_attempt + 1 205 206 // Exhausted? 207 if p.current_attempt >= p.max_attempts { 208 p.n_total_exhausted = p.n_total_exhausted + 1 209 return NX_RETRY_EXHAUSTED 210 } 211 212 // Compute next wait 213 p.next_wait_seconds = nx_retry_compute_wait_seconds(p) 214 p.total_wait_seconds = p.total_wait_seconds + p.next_wait_seconds 215 p.n_total_retries = p.n_total_retries + 1 216 return NX_RETRY_WAIT_AND_RETRY 217} 218 219// ===== Reset on successful call =================================== 220// 221// After a successful upstream call, the retry policy state is reset 222// so the next failure starts the backoff sequence fresh. 223 224func nx_retry_policy_record_success(p: *RetryPolicy) -> i64 { 225 if p == 0 as *RetryPolicy { return -1 } 226 p.current_attempt = 0 227 p.next_wait_seconds = 0 228 return 0 229} 230 231// ===== RFC 6585 Retry-After integration ========================== 232// 233// When upstream returns 429 with Retry-After header, that value 234// SUPERSEDES our exponential backoff (RFC mandates respecting it). 235// Caller passes the retry-after seconds; policy honors it for next 236// wait. 237 238func nx_retry_apply_retry_after(p: *RetryPolicy, retry_after_seconds: i64) -> i64 { 239 if p == 0 as *RetryPolicy { return -1 } 240 if retry_after_seconds <= 0 { return -1 } 241 p.next_wait_seconds = retry_after_seconds 242 p.total_wait_seconds = p.total_wait_seconds + retry_after_seconds 243 return 0 244}