nx_deploy_listen_gate.nx source

1// nx_deploy_listen_gate.nx -- THE REFEREE FOR THE DEPLOY LISTEN CHECK (debt 1786068492). 2// 3// WHAT IT PROVES, AND WHAT IT DELIBERATELY DOES NOT. The subject is md_probe_listen_edge_cfg: the 4// edge-triggered listener verification /api/deploy runs after it promotes and kills. The defect it exists to 5// catch is NOT "no check ran" -- a check ran for weeks -- it is that the check was LEVEL-TRIGGERED, so 6// "something is listening" was true both before and after the swap and a connect issued in the SIGKILL 7// teardown window greened on the OUTGOING process. This gate therefore does not merely ask "did it say 8// green"; it asks WHICH LISTENER the green describes. 9// 10// HERMETIC BY CONSTRUCTION: every fixture is a throwaway socket this gate binds itself, on a port it PROVES 11// free first, in a forked child it reaps. It never touches a serving daemon and writes NO scratch file at 12// all -- so it is idempotent by construction rather than by a teardown that a crash would skip. 13// license_tier: ORIGINAL No hw writes (Rule 26). 14import "nx_syscalls.nx" 15import "nx_gate_verdict.nx" 16import "_hdl_build/nx_mgmt_data.nx" 17 18// ---- fixture parameters. Rule 11: every number named, and every one DERIVED from a stated reason. ------- 19// The probe budget here is SYNTHETIC and deliberately tiny: the gate exercises the same code path at 20// millisecond timescales so the whole run costs ~1.5s instead of the production 30s. That is exactly why the 21// subject takes its budget as a PARAMETER. The PRODUCTION numbers are asserted separately, below, by the 22// derivation teeth -- so shrinking the fixture cannot quietly shrink what ships. 23const DLG_UP_TRIES: i64 = 6 // 6 x 100ms = 600ms up budget: 4x the bind delay below, so a healthy bind cannot lose on timing alone 24const DLG_UP_TICK_MS: i64 = 100 25const DLG_DOWN_TICK_MS: i64 = 20 // down window derives to 100/20 = 5 samples, mirroring the production derivation exactly 26const DLG_BIND_DELAY_MS: i64 = 150 // > one up-tick, so the probe MUST observe at least one REFUSED sample before the bind lands 27const DLG_HOLD_MS: i64 = 1200 // > the whole up budget, so the fixture cannot vanish mid-measurement 28const DLG_LINGER_MS: i64 = 1500 // fixture B outlives the probe: a child that exited early would free-run the port 29const DLG_PORT_BASE: i64 = 39701 // high, unassigned, above every port named in deploy_targets.conf 30const DLG_PORT_SCAN: i64 = 64 // bounded search; exhausting it is a PRECONDITION failure (SKIP), never a RED 31// BACKLOG SIZING IS LOAD-BEARING, NOT DECORATION -- measured 2026-08-21 as a 12s gate TIMEOUT. 32// md_tcp_alive CONNECTS and closes but never accept()s, and a closed client socket does NOT free its 33// slot in the accept queue -- the connection sits there until something accepts it. So once the queue 34// fills, every further connect gets its SYN dropped, nx_connect_bounded sees EINPROGRESS and polls out 35// the FULL NX_CONN_DEFAULT_MS (6000ms) per probe. At backlog=4 fixture C's 7 probes cost ~18s and the 36// gate died on the caller's deadline looking exactly like a hang in the subject. 37// DERIVED, not guessed: the worst fixture issues 1 condition-assert + (UP_TICK/DOWN_TICK) down-samples 38// + 1 up-sample against ONE listener. The tooth below binds this constant to that arithmetic so raising 39// a budget can never silently re-introduce the stall. 40const DLG_BACKLOG: i64 = 64 41 42// sockaddr_in field offsets. These are a STRUCT LAYOUT, not tunables -- naming them is what stops the 43// hand-counted-offset-beside-a-literal defect, where the layout and the indices drift apart silently. 44const DLG_SA_LEN: i64 = 16 45const DLG_SA_FAM_LO: i64 = 0 46const DLG_SA_FAM_HI: i64 = 1 47const DLG_SA_PORT_HI: i64 = 2 48const DLG_SA_PORT_LO: i64 = 3 49const DLG_SA_IP_A: i64 = 4 50const DLG_SA_IP_B: i64 = 5 51const DLG_SA_IP_C: i64 = 6 52const DLG_SA_IP_D: i64 = 7 53const DLG_SA_ADDR_END: i64 = 8 54const DLG_AF_INET_B: i64 = 2 55const DLG_BYTE_MASK: i64 = 0xff 56const DLG_PORT_SHIFT: i64 = 8 57const DLG_LOOPBACK_A: i64 = 127 58const DLG_LOOPBACK_D: i64 = 1 59 60const DLG_I64_BYTES: i64 = 8 // sizeof(i64): the slot table is MD_PC_SLOTS of these 61const DLG_WAIT_SCRATCH: i64 = 16 // wait4 status word scratch 62 63// the PRODUCTION defaults md_probe_conf must yield when no conf file is present 64const DLG_PROD_UP_TRIES: i64 = 10 65const DLG_PROD_UP_TICK_MS: i64 = 3000 66const DLG_PROD_DOWN_TICK_MS: i64 = 50 67// hc_deploy_one announces this bound to the operator in its own restart message ("guard respawns the NEW 68// binary ... <=15s"). The up budget must EXCEED it or healthy deploys roll back, so it is asserted as a 69// RELATIONSHIP below rather than as a duplicated literal. 70const DLG_GUARD_RESPAWN_BOUND_MS: i64 = 15000 71// the toolsapi health field from deploy_targets.conf. The port is spelled TWICE below -- once inside the 72// literal "port:18096" and once as this constant -- and that duplication is DELIBERATE: the tooth's whole 73// purpose is that the parser turns THAT EXACT TEXT into THAT NUMBER. Deriving the string from the constant 74// would make the parser agree with itself and the tooth would prove nothing. 75const DLG_TOOLSAPI_PORT: i64 = 18096 76 77// Bind+listen a throwaway on 127.0.0.1:<port>. Returns the fd, or -1. Mirrors md_tcp_alive's sockaddr layout 78// so the fixture and the ruler agree on the wire by construction. 79func dlg_listen(port: i64) -> i64 { 80 let fd: i64 = sys_socket(AF_INET, SOCK_STREAM, 0) 81 if fd < 0 { return 0 - 1 } 82 let sa: *u8 = sys_mmap(DLG_SA_LEN) 83 sa[DLG_SA_FAM_LO] = DLG_AF_INET_B as u8 84 sa[DLG_SA_FAM_HI] = 0 as u8 85 sa[DLG_SA_PORT_HI] = ((port >> DLG_PORT_SHIFT) & DLG_BYTE_MASK) as u8 86 sa[DLG_SA_PORT_LO] = (port & DLG_BYTE_MASK) as u8 87 sa[DLG_SA_IP_A] = DLG_LOOPBACK_A as u8 88 sa[DLG_SA_IP_B] = 0 as u8 89 sa[DLG_SA_IP_C] = 0 as u8 90 sa[DLG_SA_IP_D] = DLG_LOOPBACK_D as u8 91 var z: i64 = DLG_SA_ADDR_END 92 while z < DLG_SA_LEN { sa[z] = 0 as u8; z = z + 1 } 93 if sys_bind(fd, sa, DLG_SA_LEN) != 0 { sys_close(fd); return 0 - 1 } 94 if sys_listen(fd, DLG_BACKLOG) != 0 { sys_close(fd); return 0 - 1 } 95 return fd 96} 97 98// AAAAA A PROBE PORT YOU DID NOT VERIFY FREE IS NOT A CONTROL, IT IS A SECOND INSTANCE. Two independent 99// proofs are required and neither alone is sufficient: a CONNECT that is refused (nobody is accepting) AND a 100// BIND that succeeds (nobody holds the socket -- a bound-but-not-listening holder refuses connects too). 101func dlg_free_port(start: i64, tries: i64) -> i64 { 102 var p: i64 = start 103 var n: i64 = 0 104 while n < tries { 105 if md_tcp_alive(p) == 0 { 106 let fd: i64 = dlg_listen(p) 107 if fd >= 0 { sys_close(fd); return p } 108 } 109 p = p + 1 110 n = n + 1 111 } 112 return 0 - 1 113} 114 115func dlg_cfg(out: *i64, up_tries: i64, up_tick: i64, down_tick: i64) -> i64 { 116 out[MD_PC_UP_TRIES] = up_tries 117 out[MD_PC_UP_TICK_MS] = up_tick 118 out[MD_PC_DOWN_TICK_MS] = down_tick 119 return 0 120} 121func dlg_slots() -> *i64 { return sys_mmap(DLG_I64_BYTES * MD_PC_SLOTS) as *i64 } 122 123func main() -> i64 { 124 let ctr: *i64 = gv_ctr() 125 gv_head("nx_deploy_listen_gate -- the deploy listener check must name WHICH listener it saw (debt 1786068492)" as *u8) 126 127 let cfg: *i64 = dlg_slots() 128 dlg_cfg(cfg, DLG_UP_TRIES, DLG_UP_TICK_MS, DLG_DOWN_TICK_MS) 129 130 // ---- PRECONDITION: three genuinely free ports. Exhausting the scan says nothing about the subject, so 131 // it must SKIP, never RED -- an axis that cannot see must abstain, not acquit. 132 let pa: i64 = dlg_free_port(DLG_PORT_BASE, DLG_PORT_SCAN) 133 let pb: i64 = dlg_free_port(pa + 1, DLG_PORT_SCAN) 134 let pc: i64 = dlg_free_port(pb + 1, DLG_PORT_SCAN) 135 var gotports: i64 = 0 136 if pa > 0 { if pb > pa { if pc > pb { gotports = 1 } } } 137 if gv_need("three verified-free loopback ports for the fixtures" as *u8, gotports, ctr) == 0 { 138 let rcs: i64 = gv_verdict("DEPLOY-LISTEN-GATE" as *u8, ctr, "" as *u8) 139 sys_exit(rcs) 140 return rcs 141 } 142 143 // ---- FIXTURE A -- A REAL HANDOFF: port is down, then an incoming process binds it. Expect VERIFIED. 144 let kida: i64 = sys_fork() 145 if kida == 0 { 146 sys_sleep_ms(DLG_BIND_DELAY_MS) 147 let f: i64 = dlg_listen(pa) 148 sys_sleep_ms(DLG_HOLD_MS) 149 if f >= 0 { sys_close(f) } 150 sys_exit(0) 151 } 152 let a_down_at_start: i64 = md_tcp_alive(pa) // ASSERT THE FIXTURE REACHED ITS CONDITION, not just its outcome 153 let ra: i64 = md_probe_listen_edge_cfg(pa, cfg) 154 let a_up_at_end: i64 = md_tcp_alive(pa) 155 let wsa: *i64 = sys_mmap(DLG_WAIT_SCRATCH) as *i64 156 sys_wait4(kida, wsa, 0) 157 158 // ---- FIXTURE B -- THE EXACT OBSERVED FAILURE: the incoming process STARTS AND NEVER LISTENS. This is the 159 // neg-control the whole debt row describes: the new binary could not bind and exited, the guard respawned 160 // it, and the service stayed down with a perfectly healthy binary on disk. Expect NEVER_UP + rollback. 161 let kidb: i64 = sys_fork() 162 if kidb == 0 { 163 sys_sleep_ms(DLG_LINGER_MS) // alive as a PROCESS throughout, never a LISTENER: a process-existence check would have passed it 164 sys_exit(0) 165 } 166 let b_down_at_start: i64 = md_tcp_alive(pb) 167 let rb: i64 = md_probe_listen_edge_cfg(pb, cfg) 168 let wsb: *i64 = sys_mmap(DLG_WAIT_SCRATCH) as *i64 169 sys_wait4(kidb, wsb, 0) 170 171 // ---- FIXTURE C -- THE OVER-CLAIM CONTROL: a listener that is up the WHOLE time and never goes down. This 172 // is the outgoing process that never died, and it is the case the OLD level-triggered probe reported as a 173 // full green. Expect UNVERIFIED -- healthy-looking, provenance unproven -- and explicitly NOT VERIFIED. 174 let fc: i64 = dlg_listen(pc) 175 let c_up_at_start: i64 = md_tcp_alive(pc) 176 let rcc: i64 = md_probe_listen_edge_cfg(pc, cfg) 177 if fc >= 0 { sys_close(fc) } 178 179 // ---- ANTI-VACUITY FIRST. A probe that returns one constant for every input passes every single-outcome 180 // assertion below it. AAAAAA A CHECKER THAT ANSWERS THE SAME THING FOR EVERYTHING DISCRIMINATES NOTHING -- 181 // so the first tooth is that the three fixtures produced three DIFFERENT answers. 182 var distinct: i64 = 0 183 if ra != rb { if rb != rcc { if ra != rcc { distinct = 1 } } } 184 gv_check("anti-vacuity: three fixtures yield three DISTINCT outcomes" as *u8, distinct, ctr) 185 186 // ---- the fixtures genuinely reached their conditions (a fixture the defect cannot fail is not a test) 187 gv_check("fixture-A reached its condition: port was REFUSED before the incoming bind" as *u8, a_down_at_start == 0, ctr) 188 gv_check("fixture-A reached its condition: port was ACCEPTING when the probe returned" as *u8, a_up_at_end == 1, ctr) 189 gv_check("fixture-B reached its condition: port never accepted at any point" as *u8, b_down_at_start == 0, ctr) 190 gv_check("fixture-C reached its condition: a real listener was bound and accepting first" as *u8, c_up_at_start == 1, ctr) 191 192 // ---- the three outcomes themselves 193 gv_check("a witnessed handoff (down then up) reports VERIFIED" as *u8, ra == MD_HP_VERIFIED, ctr) 194 gv_check("a process that starts and never listens reports NEVER_UP -- the deploy rolls back" as *u8, rb == MD_HP_NEVER_UP, ctr) 195 gv_check("a listener that never dropped reports UNVERIFIED, NOT a full green" as *u8, rcc == MD_HP_UNVERIFIED, ctr) 196 gv_check("neg-control-never-claims-VERIFIED-without-witnessing-the-socket-change-hands" as *u8, rcc != MD_HP_VERIFIED, ctr) 197 gv_check("neg-control-a-healthy-deploy-is-never-rolled-back" as *u8, ra != MD_HP_NEVER_UP, ctr) 198 gv_check("neg-control-an-unwitnessed-handoff-is-never-rolled-back-either" as *u8, rcc != MD_HP_NEVER_UP, ctr) 199 200 // ---- BITE-PROVEN: fires on the bad input, silent on the good one. Without this pair a detector that 201 // refuses everything scores full marks on every negative test above. 202 gv_bite("neg-control-NEVER_UP-fires-only-on-a-process-that-never-bound" as *u8, rb == MD_HP_NEVER_UP, ra == MD_HP_NEVER_UP, ctr) 203 gv_bite("neg-control-UNVERIFIED-fires-only-when-no-handoff-was-witnessed" as *u8, rcc == MD_HP_UNVERIFIED, ra == MD_HP_UNVERIFIED, ctr) 204 205 // ---- THE PRODUCTION NUMBERS. The fixtures above run on a synthetic budget, so these teeth are what stop 206 // a shrunken fixture from quietly shrinking what ships. 207 let dflt: *i64 = dlg_slots() 208 md_probe_conf(dflt) 209 var prod_ok: i64 = 0 210 if dflt[MD_PC_UP_TRIES] > 0 { if dflt[MD_PC_UP_TICK_MS] > 0 { prod_ok = 1 } } 211 gv_check("the production up budget is positive on every axis (a zero tick spins, a zero try-count is an unconditional pass)" as *u8, prod_ok, ctr) 212 // THE REGRESSION TOOTH, earned the hard way 2026-08-21. The down-watch must default OFF: it costs the 213 // deploy watchdog ~one up-tick of LIFE before it can write any verdict, and measured on the live 214 // control plane the watchdog is KILLED inside that window -- which silently disarms the never-brick 215 // auto-rollback this whole path exists to provide. Off-by-default IS the safety property, so it gets a 216 // tooth rather than a comment. The fixtures above deliberately run with the watch ENABLED, so both 217 // states of the knob are exercised and neither is assumed. 218 gv_check("neg-control-the-down-watch-defaults-OFF-so-it-cannot-outlive-the-watchdog-verdict" as *u8, dflt[MD_PC_DOWN_TICK_MS] == 0, ctr) 219 let prod_budget_ms: i64 = dflt[MD_PC_UP_TRIES] * dflt[MD_PC_UP_TICK_MS] 220 gv_check("the up budget exceeds the guard-respawn bound hc_deploy_one announces" as *u8, prod_budget_ms > DLG_GUARD_RESPAWN_BOUND_MS, ctr) 221 222 // the down window is DERIVED as exactly one up-tick -- one calibrated quantity, never two that can drift 223 let synth: *i64 = dlg_slots() 224 dlg_cfg(synth, DLG_PROD_UP_TRIES, DLG_PROD_UP_TICK_MS, DLG_PROD_DOWN_TICK_MS) 225 gv_check("the down window derives to exactly one up-tick" as *u8, md_probe_down_tries(synth) == (DLG_PROD_UP_TICK_MS / DLG_PROD_DOWN_TICK_MS), ctr) 226 let zc: *i64 = dlg_slots() 227 dlg_cfg(zc, DLG_PROD_UP_TRIES, DLG_PROD_UP_TICK_MS, 0) 228 gv_check("neg-control-a-zero-down-tick-cannot-divide-by-zero-or-spin" as *u8, md_probe_down_tries(zc) == 1, ctr) 229 230 // The fixture backlog must exceed every probe one fixture issues against one listener, or the queue 231 // fills and each further connect polls out the whole connect budget -- a stall that reads as a hang in 232 // the SUBJECT, which is how this gate first failed. Bound to the ARITHMETIC, not a remembered number. 233 gv_check("the fixture backlog exceeds the probes one fixture issues against one listener" as *u8, DLG_BACKLOG > ((DLG_UP_TICK_MS / DLG_DOWN_TICK_MS) + DLG_UP_TRIES + 2), ctr) 234 235 // the port:<N> grammar the deploy row actually carries, and its rejection of anything else 236 gv_check("the port grammar parses the toolsapi row's health field" as *u8, md_url_port("port:18096" as *u8) == DLG_TOOLSAPI_PORT, ctr) 237 gv_check("neg-control-an-http-health-url-is-NOT-taken-as-a-port-row" as *u8, md_url_is_port("https://example.invalid/api/" as *u8) == 0, ctr) 238 gv_check("a port row IS recognised as one" as *u8, md_url_is_port("port:18096" as *u8) == 1, ctr) 239 240 let rc: i64 = gv_verdict("DEPLOY-LISTEN-GATE" as *u8, ctr, "the listener check names WHICH listener it saw; a start-but-never-listen deploy is caught and rolled back" as *u8) 241 sys_exit(rc) 242 return rc 243}