nx_hostctl_supervise_gate.nx source

1// nx_hostctl_supervise_gate.nx -- KAT for the upgraded supervisor decision logic (gaps #5 health-probe + 2// #4 restart-guard, wired into nx_hostctl.cmd_supervise). Proves: a dead daemon -> restart; an alive+SERVING 3// daemon -> leave it; an alive-but-HUNG daemon -> restart (the case PID-liveness misses); and the crash-loop 4// guard allows exactly BURST restarts per window then backs off. exit 0 = pass, N = assertion N failed. 5import "nx_hostctl.nx" 6import "nx_assert.nx" 7 8// bind+listen+fork a one-shot responder. mode 0 = reply 200 (serving); mode 1 = accept then hang (no reply). 9func hgg_mock(port: i64, mode: i64) -> i64 { 10 let lfd: i64 = sys_socket(AF_INET, SOCK_STREAM, 0); if lfd < 0 { return 0 - 1 } 11 let opt: *u8 = sys_mmap(4); opt[0]=1 as u8; sys_setsockopt(lfd, 1, 2, opt, 4) 12 let addr: *u8 = sys_mmap(16); hp_sockaddr(addr, port, 0, 0, 0, 0) 13 if sys_bind(lfd, addr, 16) != 0 { sys_close(lfd); return 0 - 1 } 14 sys_listen(lfd, 8) 15 let pid: i64 = sys_fork() 16 if pid == 0 { 17 let cfd: i64 = sys_accept(lfd) 18 if cfd >= 0 { 19 let rb: *u8 = sys_mmap(4096); sys_read(cfd, rb, 4095) 20 if mode == 0 { 21 let resp: *u8 = "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok" as *u8 22 var rl: i64 = 0; while resp[rl] != (0 as u8) { rl = rl + 1 } 23 sys_write(cfd, resp, rl) 24 } else { sys_sleep_ms(5000) } 25 sys_close(cfd) 26 } 27 sys_close(lfd); sys_exit(0) 28 } 29 sys_close(lfd); return pid 30} 31 32func main() -> i64 { 33 // --- pure liveness decisions (no probe) --- 34 if hc_needs_restart(0, 0) != 1 { return 1 } // dead, no port -> restart 35 if hc_needs_restart(0, 8080) != 1 { return 2 } // dead -> restart (port irrelevant when dead) 36 if hc_needs_restart(1, 0) != 0 { return 3 } // alive, PID-only (TLS) -> leave 37 38 // --- live serving-probe decisions --- 39 let ps: i64 = hgg_mock(19021, 0) // serving mock 40 let ph: i64 = hgg_mock(19022, 1) // hung mock 41 sys_sleep_ms(150) 42 let d_serv: i64 = hc_needs_restart(1, 19021) // alive + SERVING -> 0 43 let d_hung: i64 = hc_needs_restart(1, 19022) // alive + HUNG -> 1 44 let st: *i64 = sys_mmap(16) as *i64 45 nx_kill(ps, 9); sys_wait4(ps, st, 0); nx_kill(ph, 9); sys_wait4(ph, st, 0) 46 nx_puts_err("alive+serving decision="); nx_puti_err(d_serv) 47 nx_puts_err("alive+HUNG decision="); nx_puti_err(d_hung) 48 if d_serv != 0 { return 4 } 49 if d_hung != 1 { return 5 } // ★ the exceed: a wedged daemon is flagged when PID-liveness would not 50 51 // --- crash-loop guard = CAPPED EXPONENTIAL BACKOFF (deterministic, caller-supplied `now`). The guard was 52 // upgraded from the retired flat "BURST/window" model to Kubernetes-CrashLoopBackOff-style exponential 53 // backoff (rg_should_restart: first death restarts immediately; repeated deaths WAIT base*2^(n-1) capped; 54 // healthy >= RG_HEALTH_RESET_MS resets). hostctl passes HC_GUARD_INTERVAL_MS=cap_ms, HC_GUARD_BURST=base_ms 55 // POSITIONALLY. This KAT asserts that exponential-backoff contract (the old `allowed==BURST` test was stale). --- 56 let ws: *i64 = sys_mmap(16) as *i64; let cnt: *i64 = sys_mmap(16) as *i64 57 ws[0] = 0; cnt[0] = 0 58 let cap: i64 = HC_GUARD_INTERVAL_MS // cap_ms (backoff ceiling) 59 let base: i64 = HC_GUARD_BURST // base_ms (first backoff step) 60 // first death -> restart immediately (cnt 0->1) 61 if rg_should_restart(ws, cnt, 5000, cap, base) != 1 { return 6 } 62 // immediate repeat (same now) -> CONTAINED (base_ms backoff not elapsed) 63 if rg_should_restart(ws, cnt, 5000, cap, base) != 0 { return 6 } 64 // after base_ms elapses -> allowed (cnt 1->2) 65 if rg_should_restart(ws, cnt, 5000 + base, cap, base) != 1 { return 6 } 66 // only base_ms since -> still contained (this step's backoff is 2*base = exponential doubling) 67 if rg_should_restart(ws, cnt, 5000 + base + base, cap, base) != 0 { return 6 } 68 // after 2*base -> allowed (cnt 2->3): PROVES the wait DOUBLED 69 if rg_should_restart(ws, cnt, 5000 + base + 2 * base, cap, base) != 1 { return 6 } 70 // ran healthy >= RG_HEALTH_RESET_MS -> counter resets -> immediate restart (auto-recovery) 71 if rg_should_restart(ws, cnt, 5000 + base + 2 * base + RG_HEALTH_RESET_MS, cap, base) != 1 { return 6 } 72 nx_puts_err("guard: exp-backoff OK (immediate -> contain -> 2x doubling -> health-reset)\n") 73 74 nx_puts_err("--- vs systemd (PID-liveness + StartLimitBurst) / old nx_hostctl (/proc-name scan only) ---\n" as *u8) 75 nx_puts_err("HUNG daemon (up + bound, not serving): systemd & old supervisor leave it wedged (MISS) ; nishi restarts it -> EXCEEDS. crash-loop guard = PARITY w/ StartLimitBurst + deterministic.\n" as *u8) 76 nx_puts_err("nx_hostctl_supervise_gate verdict=GREEN pass=6\n" as *u8) 77 return 0 78}