nx_ts_handoff_gate.nx source

1// nx_ts_handoff_gate.nx -- TS1: KEEP THE SOCKET, REPLACE THE PROCESS. 2// 3// /compare/trafficsafety rung TS1, accept rule taken VERBATIM from trafficsafety.plan and not 4// re-invented here: "a swap under a synthetic request stream drops ZERO connections, measured by a 5// client that counts refusals and resets, not by the daemon's own log. The neg-control is mandatory 6// -- the same harness against the CURRENT ordinary-listener path must show a non-zero drop, or the 7// test proves nothing." 8// 9// THE MEASUREMENT IS DROPPED CONNECTIONS, NOT BINDERS. nx_hotlisten_gate proves two processes can 10// co-bind a port, which is the NECESSARY condition and never the sufficient one -- LWN documents 11// SO_REUSEPORT as itself dropping connections during the three-way handshake when the listening set 12// changes. So nothing here counts binders. A client process drives real TCP connections, and every 13// connect that is refused and every exchange that does not complete is counted as a DROP. 14// 15// THE DECISIVE TOOTH IS THE ARRIVAL DURING THE GAP. Both arms run the same script: generation one 16// serves a stream, generation one EXITS AND IS REAPED so that zero servers exist, a client connects 17// at that exact instant, and only then is generation two started. With an owner outside the service 18// holding the listener the arrival completes its handshake into the kernel's accept queue and is 19// served microseconds later; with the current bind-per-process path there is nothing bound and the 20// arrival is refused. That difference is deterministic, which is why it can be a gate. 21// 22// SCOPE DECLARED UP FRONT RATHER THAN IMPLIED. This proves the STRUCTURAL property -- an arrival 23// during the swap is not refused. It is NOT a failure-RATE measurement: HAProxy's published figure 24// for the un-fixed race is 155 failures per million connections, and no run of this size could see 25// a rate like that. Measuring that would need a load generator and is a different instrument; the 26// ceiling is stated here rather than left for a reader to assume it was covered. 27// 28// SAFETY: loopback only, and every port is FOUND by a bind that would have failed if the port were 29// occupied -- a probe port you did not verify free is not a control, it is a second instance. No 30// serving daemon is touched. 31// expect_exit: 0 license_tier: ORIGINAL No hw writes (Rule 26). 32import "nx_syscalls.nx" 33import "nx_http_server.nx" 34import "nx_gate_emit_lib.nx" 35import "nx_gate_verdict.nx" 36 37const TH_NUM_SCRATCH: i64 = 24 38const TH_ASCII_ZERO: i64 = 48 39const TH_B10: i64 = 10 40const TH_ONE_BYTE: i64 = 1 41const TH_SA_IN_BYTES: i64 = 16 42const TH_BYTE_RADIX: i64 = 256 43const TH_OUT_SLOTS: i64 = 8 44const TH_OUT_BYTES: i64 = 64 45const TH_LOOPBACK_A: i64 = 127 46const TH_LOOPBACK_D: i64 = 1 47const TH_PORT_BASE: i64 = 39100 48const TH_PORT_TRIES: i64 = 64 49const TH_PORT_NONE: i64 = 0 - 1 50const TH_PING_BYTE: i64 = 80 51const TH_ECHO_BYTE: i64 = 90 52const TH_READY_BYTE: i64 = 82 53const TH_BOOL_TRUE: i64 = 1 54const TH_BOOL_FALSE: i64 = 0 55// The synthetic stream's size per generation is taken from the rendezvous backlog rather than 56// picked: it exercises the accept queue at the queue's own scale on both sides of the swap. The 57// decisive tooth does not depend on it -- the in-gap arrival is a single deterministic event -- so 58// this number sets how much "the server was really serving" evidence surrounds that event. 59const TH_REQS_PER_GEN: i64 = TS_RV_BACKLOG 60// Bounds exist only so a broken run fails LOUD instead of hanging. Reusing the shim's established 61// socket deadline rather than inventing a second budget for the same kind of wait. 62const TH_DEADLINE_S: i64 = ACCEPT_TMO_S 63const TH_CHILD_ALARM_S: i64 = ACCEPT_TMO_S * 2 64// Out-slot layout for ts_swap_run. 65const TH_O_ATTEMPTED: i64 = 0 66const TH_O_SERVED: i64 = 1 67const TH_O_GAP_RC: i64 = 2 68const TH_O_GAP_DONE: i64 = 3 69const TH_O_PORT: i64 = 4 70const TH_O_C1: i64 = 5 71const TH_O_C2: i64 = 6 72const TH_GAP_UNRUN: i64 = 0 - 999 73// Child exit codes -- each names WHICH step failed. 74const TH_C_OK: i64 = 0 75const TH_C_ACQUIRE: i64 = 2 76const TH_C_BIND: i64 = 3 77const TH_C_READY: i64 = 4 78const TH_C_ACCEPT: i64 = 5 79const TH_C_BADREQ: i64 = 6 80const TH_C_WRITE: i64 = 7 81const TH_SOCK_ARMED: *u8 = "/tmp/nx_ts_handoff_gate/armed.sock" as *u8 82const TH_SOCK_NEG: *u8 = "/tmp/nx_ts_handoff_gate/neg.sock" as *u8 83const TH_DIR: *u8 = "/tmp/nx_ts_handoff_gate" as *u8 84 85func g_putn(v: i64) -> i64 { 86 if v == 0 { sys_write(1, "0" as *u8, 1); return 0 } 87 var m: i64 = v 88 if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m } 89 let d: *u8 = sys_mmap(TH_NUM_SCRATCH); var k: i64 = 0 90 while m > 0 { d[k] = ((TH_ASCII_ZERO + (m - (m / TH_B10) * TH_B10)) as u8); m = m / TH_B10; k = k + 1 } 91 var j: i64 = k - 1 92 while j >= 0 { sys_write(1, ((d as i64)+j) as *u8, 1); j = j - 1 } 93 return 0 94} 95 96func gq(label: *u8, got: i64, want: i64, ctr: *i64) -> i64 { 97 var c: i64 = 0 98 if got == want { c = 1 } 99 let r: i64 = gv_check(label, c, ctr) 100 if c == 0 { 101 g_puts(" got=" as *u8); g_putn(got) 102 g_puts(" want=" as *u8); g_putn(want) 103 g_puts("\n" as *u8) 104 } 105 return r 106} 107 108func th_sa(sa: *u8, port: i64) -> i64 { 109 var i: i64 = 0 110 while i < TH_SA_IN_BYTES { sa[i] = 0; i = i + 1 } 111 sa[0] = AF_INET 112 sa[1] = 0 113 sa[2] = port / TH_BYTE_RADIX 114 sa[3] = port % TH_BYTE_RADIX 115 sa[4] = TH_LOOPBACK_A 116 sa[5] = 0 117 sa[6] = 0 118 sa[7] = TH_LOOPBACK_D 119 return 0 120} 121 122func th_put1(fd: i64, v: i64) -> i64 { 123 let b: *u8 = sys_mmap(TH_ONE_BYTE) 124 b[0] = v 125 return sys_write(fd, b, TH_ONE_BYTE) 126} 127func th_get1(fd: i64) -> i64 { 128 let b: *u8 = sys_mmap(TH_ONE_BYTE) 129 b[0] = 0 130 let r: i64 = sys_read(fd, b, TH_ONE_BYTE) 131 if r != TH_ONE_BYTE { return 0 - 1 } 132 return b[0] as i64 133} 134 135// Bind+listen loopback:<port>. reuse=1 sets SO_REUSEADDR. With reuse=0 the bind ITSELF is the 136// free-port check, which is why the port search below never sets it. 137func th_listen(port: i64, reuse: i64) -> i64 { 138 let fd: i64 = sys_socket(AF_INET, SOCK_STREAM, 0) 139 if fd < 0 { return fd } 140 if reuse == TH_BOOL_TRUE { 141 let ra: *u8 = sys_mmap(SCM_U32_BYTES) 142 ra[0] = 1 143 ra[1] = 0 144 ra[2] = 0 145 ra[3] = 0 146 sys_setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, ra, SCM_U32_BYTES) 147 } 148 let sa: *u8 = sys_mmap(TH_SA_IN_BYTES) 149 th_sa(sa, port) 150 if sys_bind(fd, sa, TH_SA_IN_BYTES) < 0 { sys_close(fd); return 0 - 1 } 151 if sys_listen(fd, TS_RV_BACKLOG) < 0 { sys_close(fd); return 0 - 1 } 152 return fd 153} 154 155// Find a free loopback port at or above `from` by BINDING it without SO_REUSEADDR, then releasing 156// it. A port that binds was free; a port that refuses the bind was not. 157func th_find_port(from: i64) -> i64 { 158 var t: i64 = 0 159 while t < TH_PORT_TRIES { 160 let cand: i64 = from + t 161 let fd: i64 = th_listen(cand, TH_BOOL_FALSE) 162 if fd >= 0 { sys_close(fd); return cand } 163 t = t + 1 164 } 165 return TH_PORT_NONE 166} 167 168// One client exchange on an already-connected socket. 1 = served, 0 = dropped. 169func th_exchange(fd: i64) -> i64 { 170 if th_put1(fd, TH_PING_BYTE) != TH_ONE_BYTE { return 0 } 171 if th_get1(fd) != TH_ECHO_BYTE { return 0 } 172 return 1 173} 174 175// One full client connection. 1 = served, 0 = dropped (refused, reset, or an incomplete exchange). 176func th_drive_one(port: i64) -> i64 { 177 let sa: *u8 = sys_mmap(TH_SA_IN_BYTES) 178 th_sa(sa, port) 179 let fd: i64 = sys_socket(AF_INET, SOCK_STREAM, 0) 180 if fd < 0 { return 0 } 181 sys_set_socket_timeout(fd, TH_DEADLINE_S) 182 if sys_connect(fd, sa, TH_SA_IN_BYTES) < 0 { sys_close(fd); return 0 } 183 let ok: i64 = th_exchange(fd) 184 sys_close(fd) 185 return ok 186} 187 188// A SERVER GENERATION. armed=1 acquires the listener from the owner and never binds; armed=0 is the 189// CURRENT path -- it binds the port itself and releases it on exit, which is the disease state. 190func th_generation(armed: i64, port: i64, sock_path: *u8, reqs: i64) -> i64 { 191 sys_alarm(TH_CHILD_ALARM_S) 192 let vb: *i64 = (sys_mmap(TH_OUT_BYTES)) as *i64 193 vb[0] = 0 194 var lfd: i64 = 0 - 1 195 if armed == TH_BOOL_TRUE { 196 lfd = ts_handoff_acquire(sock_path, vb) 197 if lfd < 0 { return TH_C_ACQUIRE } 198 } else { 199 lfd = th_listen(port, TH_BOOL_TRUE) 200 if lfd < 0 { return TH_C_BIND } 201 // Signal readiness over the SAME rendezvous the armed arm uses, so the two arms differ in 202 // exactly one thing -- where the listener came from -- and in nothing else. 203 let rc: i64 = sys_unix_connect_fd(sock_path) 204 if rc < 0 { return TH_C_READY } 205 if th_put1(rc, TH_READY_BYTE) != TH_ONE_BYTE { return TH_C_READY } 206 sys_close(rc) 207 } 208 sys_set_socket_timeout(lfd, TH_DEADLINE_S) 209 var i: i64 = 0 210 while i < reqs { 211 let acc: i64 = sys_accept(lfd) 212 if acc < 0 { return TH_C_ACCEPT } 213 sys_set_socket_timeout(acc, TH_DEADLINE_S) 214 if th_get1(acc) != TH_PING_BYTE { sys_close(acc); return TH_C_BADREQ } 215 if th_put1(acc, TH_ECHO_BYTE) != TH_ONE_BYTE { sys_close(acc); return TH_C_WRITE } 216 sys_close(acc) 217 i = i + 1 218 } 219 // armed: this releases only THIS generation's copy, and the owner still holds the socket. 220 // neg-control: this releases the port itself, which is the whole point of the control. 221 sys_close(lfd) 222 return TH_C_OK 223} 224 225// The owner's side of one generation start: hand over the listener (armed) or read the ready byte 226// (neg-control). Returns 1 on success. 227func th_serve_rv(rv: i64, listen_fd: i64, armed: i64) -> i64 { 228 if armed == TH_BOOL_TRUE { 229 let vb: *i64 = (sys_mmap(TH_OUT_BYTES)) as *i64 230 vb[0] = 0 231 if ts_handoff_publish(rv, listen_fd, vb) < 0 { return 0 } 232 return 1 233 } 234 let c: i64 = sys_accept(rv) 235 if c < 0 { return 0 } 236 sys_set_socket_timeout(c, TH_DEADLINE_S) 237 let b: i64 = th_get1(c) 238 sys_close(c) 239 if b != TH_READY_BYTE { return 0 } 240 return 1 241} 242 243// THE EXPERIMENT. Identical script for both arms; the ONLY difference is who owns the listener. 244// Returns the number of DROPPED connections, or -1 if the harness could not be set up at all -- 245// which is reported as UNOBSERVED, never as a pass. 246func ts_swap_run(armed: i64, port: i64, sock_path: *u8, out: *i64) -> i64 { 247 var k: i64 = 0 248 while k < TH_OUT_SLOTS { out[k] = 0; k = k + 1 } 249 out[TH_O_GAP_RC] = TH_GAP_UNRUN 250 out[TH_O_PORT] = port 251 let vb: *i64 = (sys_mmap(TH_OUT_BYTES)) as *i64 252 vb[0] = 0 253 let rv: i64 = ts_handoff_open(sock_path, vb) 254 if rv < 0 { return 0 - 1 } 255 sys_set_socket_timeout(rv, TH_DEADLINE_S) 256 var lsock: i64 = 0 - 1 257 if armed == TH_BOOL_TRUE { 258 lsock = th_listen(port, TH_BOOL_FALSE) 259 if lsock < 0 { sys_close(rv); sys_unlinkat(sock_path); return 0 - 1 } 260 } 261 var drops: i64 = 0 262 var attempted: i64 = 0 263 var served: i64 = 0 264 265 // ---- generation one 266 let c1: i64 = sys_fork() 267 if c1 == 0 { sys_exit(th_generation(armed, port, sock_path, TH_REQS_PER_GEN)) } 268 if th_serve_rv(rv, lsock, armed) == 0 { 269 sys_close(rv) 270 if lsock >= 0 { sys_close(lsock) } 271 sys_unlinkat(sock_path) 272 return 0 - 1 273 } 274 var i: i64 = 0 275 while i < TH_REQS_PER_GEN { 276 attempted = attempted + 1 277 let ok: i64 = th_drive_one(port) 278 if ok == 1 { served = served + 1 } else { drops = drops + 1 } 279 i = i + 1 280 } 281 let st: *i64 = (sys_mmap(TH_OUT_BYTES)) as *i64 282 st[0] = 0 283 sys_wait4(c1, st, 0) 284 out[TH_O_C1] = wait_status_rc(st[0]) 285 286 // ---- THE GAP. Generation one has exited and been reaped: at this instant ZERO server 287 // processes exist. An arrival now is the whole experiment. 288 let sa: *u8 = sys_mmap(TH_SA_IN_BYTES) 289 th_sa(sa, port) 290 let gap: i64 = sys_socket(AF_INET, SOCK_STREAM, 0) 291 sys_set_socket_timeout(gap, TH_DEADLINE_S) 292 let grc: i64 = sys_connect(gap, sa, TH_SA_IN_BYTES) 293 out[TH_O_GAP_RC] = grc 294 out[TH_O_GAP_DONE] = 1 295 attempted = attempted + 1 296 if grc < 0 { drops = drops + 1 } 297 298 // ---- generation two. It must also serve the in-gap arrival when that arrival survived. 299 var gen2: i64 = TH_REQS_PER_GEN 300 if grc == 0 { gen2 = gen2 + 1 } 301 let c2: i64 = sys_fork() 302 if c2 == 0 { sys_exit(th_generation(armed, port, sock_path, gen2)) } 303 let sv2: i64 = th_serve_rv(rv, lsock, armed) 304 if grc == 0 { 305 if sv2 == 1 { 306 if th_exchange(gap) == 1 { served = served + 1 } else { drops = drops + 1 } 307 } else { drops = drops + 1 } 308 } 309 sys_close(gap) 310 if sv2 == 1 { 311 var j: i64 = 0 312 while j < TH_REQS_PER_GEN { 313 attempted = attempted + 1 314 let ok2: i64 = th_drive_one(port) 315 if ok2 == 1 { served = served + 1 } else { drops = drops + 1 } 316 j = j + 1 317 } 318 } 319 st[0] = 0 320 sys_wait4(c2, st, 0) 321 out[TH_O_C2] = wait_status_rc(st[0]) 322 323 if lsock >= 0 { sys_close(lsock) } 324 sys_close(rv) 325 sys_unlinkat(sock_path) 326 out[TH_O_ATTEMPTED] = attempted 327 out[TH_O_SERVED] = served 328 return drops 329} 330 331func th_report(tag: *u8, drops: i64, out: *i64) -> i64 { 332 g_puts(" " as *u8); g_puts(tag) 333 g_puts(" port=" as *u8); g_putn(out[TH_O_PORT]) 334 g_puts(" attempted=" as *u8); g_putn(out[TH_O_ATTEMPTED]) 335 g_puts(" served=" as *u8); g_putn(out[TH_O_SERVED]) 336 g_puts(" drops=" as *u8); g_putn(drops) 337 g_puts(" in_gap_connect_rc=" as *u8); g_putn(out[TH_O_GAP_RC]) 338 g_puts(" gen1_exit=" as *u8); g_putn(out[TH_O_C1]) 339 g_puts(" gen2_exit=" as *u8); g_putn(out[TH_O_C2]) 340 g_puts("\n" as *u8) 341 return 0 342} 343 344func main() -> i64 { 345 let ctr: *i64 = gv_ctr() 346 gv_head("nx_ts_handoff_gate -- TS1: zero dropped connections across a process swap" as *u8) 347 348 sys_mkdir(TH_DIR, MODE_0755) 349 sys_unlinkat(TH_SOCK_ARMED) 350 sys_unlinkat(TH_SOCK_NEG) 351 352 let pa: i64 = th_find_port(TH_PORT_BASE) 353 var pn: i64 = TH_PORT_NONE 354 if pa != TH_PORT_NONE { pn = th_find_port(pa + 1) } 355 var ports_ok: i64 = 0 356 if pa != TH_PORT_NONE { if pn != TH_PORT_NONE { if pa != pn { ports_ok = 1 } } } 357 if gv_need("two DISTINCT loopback ports proven free by a bind without SO_REUSEADDR" as *u8, ports_ok, ctr) == 0 { 358 let rcs: i64 = gv_verdict("TS-HANDOFF-GATE" as *u8, ctr, "each tooth states its own strength above" as *u8) 359 sys_exit(rcs) 360 return rcs 361 } 362 gq("T01 the two arms run on DIFFERENT ports so neither can serve the other" as *u8, pa != pn, 1, ctr) 363 364 // ---- ARM 1: the owner outside the service holds the listener. 365 let oa: *i64 = (sys_mmap(TH_OUT_BYTES)) as *i64 366 let da: i64 = ts_swap_run(TH_BOOL_TRUE, pa, TH_SOCK_ARMED, oa) 367 th_report("ARMED " as *u8, da, oa) 368 var arm_ran: i64 = 0 369 if da >= 0 { if oa[TH_O_GAP_DONE] == 1 { arm_ran = 1 } } 370 if gv_need("the ARMED arm reached the swap with a live client stream" as *u8, arm_ran, ctr) == 1 { 371 // ANTI-VACUITY FIRST: a swap with nothing in flight is UNOBSERVED, never a pass. The 372 // denominator is asserted arithmetically before any verdict about drops is believed. 373 gq("T02 anti-vacuity the ARMED arm actually drove the whole stream" as *u8, 374 oa[TH_O_ATTEMPTED], TH_REQS_PER_GEN + TH_REQS_PER_GEN + 1, ctr) 375 gq("T03 anti-vacuity generation one really exited before the gap" as *u8, oa[TH_O_C1], TH_C_OK, ctr) 376 gq("T04 THE INVARIANT an arrival DURING the swap is not refused" as *u8, oa[TH_O_GAP_RC], 0, ctr) 377 gq("T05 THE ACCEPT RULE zero dropped connections across the swap" as *u8, da, 0, ctr) 378 gq("T06 and every attempted connection was served end to end" as *u8, oa[TH_O_SERVED], oa[TH_O_ATTEMPTED], ctr) 379 gq("T07 generation two completed its share including the in-gap arrival" as *u8, oa[TH_O_C2], TH_C_OK, ctr) 380 } 381 382 // ---- ARM 2: NEG-CONTROL. The same harness against the CURRENT bind-per-process path. If this 383 // does not drop, the armed arm's zero proves nothing at all. 384 let on: *i64 = (sys_mmap(TH_OUT_BYTES)) as *i64 385 let dn: i64 = ts_swap_run(TH_BOOL_FALSE, pn, TH_SOCK_NEG, on) 386 th_report("NEG-CTRL" as *u8, dn, on) 387 var neg_ran: i64 = 0 388 if dn >= 0 { if on[TH_O_GAP_DONE] == 1 { neg_ran = 1 } } 389 if gv_need("the NEG-CONTROL arm reached the swap with a live client stream" as *u8, neg_ran, ctr) == 1 { 390 gq("T08 anti-vacuity the neg-control drove the same number of connections" as *u8, 391 on[TH_O_ATTEMPTED], oa[TH_O_ATTEMPTED], ctr) 392 var refused: i64 = 0 393 if on[TH_O_GAP_RC] < 0 { refused = 1 } 394 gq("T09 neg-control-bind-per-process the in-gap arrival IS refused" as *u8, refused, 1, ctr) 395 var lost: i64 = 0 396 if dn > 0 { lost = 1 } 397 gq("T10 neg-control-drops the current path loses connections across a swap" as *u8, lost, 1, ctr) 398 // The control must fail for the RIGHT reason: the port went away, not the harness broke. 399 gq("T11 neg-control-attribution its generations both ran cleanly" as *u8, 400 on[TH_O_C1] + on[TH_O_C2], TH_C_OK, ctr) 401 // Both signals present at once: two arms that each isolate one signal do not prove the 402 // mechanism discriminates between them. 403 var better: i64 = 0 404 if da >= 0 { if dn > da { better = 1 } } 405 gq("T12 the handoff arm strictly beats the current path on the SAME harness" as *u8, better, 1, ctr) 406 } 407 408 sys_unlinkat(TH_SOCK_ARMED) 409 sys_unlinkat(TH_SOCK_NEG) 410 let rc: i64 = gv_verdict("TS-HANDOFF-GATE" as *u8, ctr, "each tooth states its own strength above" as *u8) 411 sys_exit(rc) 412 return rc 413}