code wiki / _hdl_build / nx_lease_wedge_gate.nx

nx_lease_wedge_gate.nx source

↩ module page · 441 lines · 21908 B

1// nx_lease_wedge_gate.nx -- THE LEASE-WEDGE DETECTOR (2026-09-04). 2// 3// WHY THIS EXISTS. On 2026-09-04 all four knowledge/lease/heavyio-slot-N.lock slots were found 4// PERMANENTLY WEDGED. The crawler AND the shard-compactor had been dead for 18 HOURS and NOTHING in 5// the estate noticed: /search kept answering 200, every daemon read UP, and the only symptom was a 6// status log whose mtime stopped advancing. The operator's report was "search still searches nothing". 7// 8// THE MECHANISM, read out of buildroot/runtime/nx_lease_lib.nx rather than guessed. A takeover is 9// arbitrated by mkdir on a monotonic generation: ls_acquire_root parses the stamp, and for a released 10// or expired lease calls ls_claim(dir, owner, ttl, g_nonce + 1, g_nonce + 1), whose FIRST act is 11// if ls_mkdir(dir + "/g" + gate) != 0 { return 0 } // 0 == LOST == "busy" 12// and whose stamp write is a SEPARATE, LATER, NON-ATOMIC step: 13// let tfd = sys_openat_wr(tp, LS_MODE) ... sys_write ... sys_renameat(tp, sp) 14// A process killed between the mkdir and the rename leaves g<N> ON DISK with the stamp still reading 15// N-1. Every later acquirer then re-derives the SAME gate from the SAME stamp, retries mkdir g<N>, 16// gets EEXIST, and correctly reports BUSY -- forever, with no recovery path and no diagnostic. 17// The host bugchecked 3x in 72 h (GPU driver), which is exactly the kill this window needs. 18// 19// THE TEST IS THE FAILING CALL ITSELF, NOT A PROXY. This gate does not ask "does maxdir exceed the 20// stamp"; it asks the one question that decides the outcome: DOES THE DIRECTORY THE NEXT TAKEOVER 21// WILL TRY TO CREATE ALREADY EXIST? That is stat of <lease>/g<stamp_gen + 1>. A proxy could be right 22// for the wrong reason; this cannot. 23// THE PROBE PATH IS NEVER CREATED. Calling mkdir on g<gen+1> to test for it WOULD ITSELF WEDGE THE 24// LEASE -- the detector would manufacture the defect it reports. Existence is read with fstatat and 25// nothing here writes to a live lease, ever. 26// 27// CHECK-BEFORE-BUILD (rule: "I didn't know it existed" is a retrieval failure). nx_spendgate reports 28// presubmit=FREE and nx_capsearch over 7,341 organs (corpus_complete=1) returns nx_lease and 29// nx_lease_probe -- the PRIMITIVE itself -- plus nx_lease_lib and nx_vault_lease. None censuses lease 30// HEALTH; there is no acquire/release verb that can even express "this lease can never be claimed 31// again". This is a new capability, not a duplicate ruler. 32// 33// SCOPE, STATED SO NOBODY TRUSTS IT AS EXACT: this detects the WEDGE. It does not detect a lease held 34// by a dead process (that is a TTL question and ls_acquire already expires it), and it says nothing 35// about whether the holder is making progress. 36// 37// OWED, DELIBERATELY NOT ATTEMPTED HERE: ls_claim mkdir-then-stamp has no crash recovery. Fixing a 38// shared estate-wide mutex is not a safe act under load, and that file own comments record TWO prior 39// attempts at exactly this which wedged it or double-admitted. A detector first is the right order. 40import "nx_gate_verdict.nx" 41import "nx_syscalls.nx" 42import "nx_lease_lib.nx" // THE REMEDY under test (2026-09-06): ls_claim_x reclaims an AGED orphan gate and refuses a fresh one 43 44const LW_ROOT: *u8 = "knowledge/lease" 45const LW_PATHCAP: i64 = 1024 46const LW_STAMPCAP: i64 = 512 47const LW_DENTBUF: i64 = 65536 48const LW_STATBUF: i64 = 160 49const LW_TAB: i64 = 9 50const LW_NL: i64 = 10 51const LW_D0: i64 = 48 52const LW_D9: i64 = 57 53const LW_B10: i64 = 10 54const LW_GEN_FIELD: i64 = 4 55const LW_DOT: i64 = 46 56const LW_SLASH: i64 = 47 57const LW_SCRATCH: i64 = 32 58const LW_SMALL: i64 = 64 59const LW_BOX: i64 = 16 60 61const LW_HEALTHY: i64 = 0 62const LW_WEDGED: i64 = 1 63const LW_UNREADABLE: i64 = 2 64 65func lw_cat(d: *u8, o: i64, s: *u8) -> i64 { 66 var i: i64 = 0 67 var p: i64 = o 68 while s[i] != (0 as u8) { d[p] = s[i]; p = p + 1; i = i + 1 } 69 return p 70} 71func lw_catn(d: *u8, o: i64, v: i64) -> i64 { 72 if v == 0 { d[o] = LW_D0 as u8; return o + 1 } 73 var tmp: i64 = v 74 var n: i64 = 0 75 let sc: *u8 = sys_mmap(LW_SCRATCH) 76 while tmp > 0 { sc[n] = (LW_D0 + (tmp % LW_B10)) as u8; n = n + 1; tmp = tmp / LW_B10 } 77 var p: i64 = o 78 var k: i64 = n 79 while k > 0 { k = k - 1; d[p] = sc[k]; p = p + 1 } 80 sys_munmap(sc, LW_SCRATCH) 81 return p 82} 83func lw_pos(v: i64) -> i64 { if v > 0 { return 1 } return 0 } 84// READ-ONLY existence. 1 if the path exists, 0 if not. NEVER creates. 85func lw_exists(path: *u8) -> i64 { 86 let sb: *u8 = sys_mmap(LW_STATBUF) 87 let r: i64 = sys_fstatat(path, sb) 88 sys_munmap(sb, LW_STATBUF) 89 if r == 0 { return 1 } 90 return 0 91} 92// Parse the stamp GENERATION (field 4, tab-separated). Returns the generation, or -1 when the stamp 93// cannot be read or does not carry five fields. A malformed stamp is UNREADABLE, never "generation 0": 94// defaulting here would let a corrupt stamp probe g1 and read HEALTHY on a wedged lease. 95func lw_stamp_gen(dir: *u8) -> i64 { 96 let sp: *u8 = sys_mmap(LW_PATHCAP) 97 var o: i64 = lw_cat(sp, 0, dir) 98 o = lw_cat(sp, o, "/stamp") 99 sp[o] = 0 as u8 100 let fd: i64 = sys_openat_rd(sp) 101 sys_munmap(sp, LW_PATHCAP) 102 if fd < 0 { return 0 - 1 } 103 let buf: *u8 = sys_mmap(LW_STAMPCAP) 104 let n: i64 = sys_read(fd, buf, LW_STAMPCAP - 1) 105 sys_close(fd) 106 if n <= 0 { sys_munmap(buf, LW_STAMPCAP); return 0 - 1 } 107 var field: i64 = 0 108 var v: i64 = 0 109 var seen: i64 = 0 110 var i: i64 = 0 111 while i < n { 112 let c: i64 = buf[i] as i64 113 if c == LW_TAB { 114 field = field + 1 115 v = 0 116 seen = 0 117 } else { 118 if c == LW_NL { i = n } else { 119 if field == LW_GEN_FIELD { 120 if c >= LW_D0 { if c <= LW_D9 { v = v * LW_B10 + (c - LW_D0); seen = 1 } } 121 } 122 } 123 } 124 i = i + 1 125 } 126 sys_munmap(buf, LW_STAMPCAP) 127 if field < LW_GEN_FIELD { return 0 - 1 } 128 if seen == 0 { return 0 - 1 } 129 return v 130} 131// THE CLASSIFIER. Returns LW_HEALTHY / LW_WEDGED / LW_UNREADABLE and writes the observed generation 132// into genout[0] (-1 when unreadable) so a caller prints evidence rather than a bare verdict. 133func lw_classify(dir: *u8, genout: *i64) -> i64 { 134 let g: i64 = lw_stamp_gen(dir) 135 genout[0] = g 136 if g < 0 { return LW_UNREADABLE } 137 let gp: *u8 = sys_mmap(LW_PATHCAP) 138 var o: i64 = lw_cat(gp, 0, dir) 139 o = lw_cat(gp, o, "/g") 140 o = lw_catn(gp, o, g + 1) 141 gp[o] = 0 as u8 142 let ex: i64 = lw_exists(gp) 143 sys_munmap(gp, LW_PATHCAP) 144 if ex == 1 { return LW_WEDGED } 145 return LW_HEALTHY 146} 147// Census every lease under root. counts[0]=healthy counts[1]=wedged counts[2]=unreadable 148// counts[3]=leases seen. Prints one line per NON-healthy lease: a count without a worklist is not 149// actionable, so every offender is NAMED with the exact path the next takeover will collide on. 150func lw_census(root: *u8, counts: *i64, verbose: i64) -> i64 { 151 counts[0] = 0 152 counts[1] = 0 153 counts[2] = 0 154 counts[3] = 0 155 let fd: i64 = sys_openat_rd(root) 156 if fd < 0 { return 0 - 1 } 157 let db: *u8 = sys_mmap(LW_DENTBUF) 158 let gen: *i64 = sys_mmap(LW_BOX) as *i64 159 let dp: *u8 = sys_mmap(LW_PATHCAP) 160 var more: i64 = 1 161 while more == 1 { 162 let nb: i64 = sys_getdents64(fd, db, LW_DENTBUF) 163 if nb <= 0 { more = 0 } else { 164 var off: i64 = 0 165 while off < nb { 166 let rec: *u8 = ((db as i64) + off) as *u8 167 let rl: i64 = dirent_reclen(rec) 168 if dirent_type(rec) == DT_DIR { 169 let nm: *u8 = dirent_name(rec) 170 var skip: i64 = 0 171 if nm[0] == (LW_DOT as u8) { skip = 1 } 172 if skip == 0 { 173 var o2: i64 = lw_cat(dp, 0, root) 174 dp[o2] = LW_SLASH as u8 175 o2 = o2 + 1 176 o2 = lw_cat(dp, o2, nm) 177 dp[o2] = 0 as u8 178 let cls: i64 = lw_classify(dp, gen) 179 counts[3] = counts[3] + 1 180 counts[cls] = counts[cls] + 1 181 if verbose == 1 { if cls != LW_HEALTHY { 182 if cls == LW_WEDGED { gv_puts(" WEDGED ") } else { gv_puts(" UNREADABLE ") } 183 gv_puts(dp) 184 gv_puts(" stamp_gen=") 185 gv_num(gen[0]) 186 gv_puts(" -- the next takeover mkdirs g") 187 gv_num(gen[0] + 1) 188 gv_puts(" and that path ALREADY EXISTS, so every acquire returns BUSY forever\n") 189 } } 190 } 191 } 192 if rl <= 0 { off = nb } else { off = off + rl } 193 } 194 } 195 } 196 sys_munmap(db, LW_DENTBUF) 197 sys_munmap(gen as *u8, LW_BOX) 198 sys_munmap(dp, LW_PATHCAP) 199 sys_close(fd) 200 return 0 201} 202// Fixtures are built at RUNTIME under /tmp and never share a path with a production beat: a gate that 203// shares a fixture with a live writer measures the writer, and its RED tracks the fixture not the code. 204// CLEAR BEFORE PLANT (2026-09-12): lw_mkfix is sys_mkdir-only and removes NOTHING, so every .lock and 205// every g<n> it has ever created survives between runs -- measured 7 dirs where the census tooth expects 206// 4, plus g0 dirs dated 2026-09-04 that no current call site can emit (every ls_claim gate is g_nonce+1, 207// whose floor is 1). The residue is NOT inert: a leftover g<n> IS the EEXIST the takeover under test must 208// not find, so the two neg-controls ALTERNATED on how long ago this gate last ran -- fresh.lock failed 209// once the residue aged past 3600 s, plain.lock failed while it had not, and both runs scored 21/25 with 210// a different pair red. A GATE WHOSE ONLY DISCRIMINATOR IS AGE MUST NOT LET THAT AGE ACCUMULATE ACROSS 211// RUNS. The bound is DERIVED BY ENUMERATION, never a guessed generation ceiling, so the fossils go too. 212// Files are unlinked as well as dirs removed: a surviving stamp makes the parent rmdir fail ENOTEMPTY. 213func lw_clearlock(dir: *u8) -> i64 { 214 let fd: i64 = sys_openat_rd(dir) 215 if fd < 0 { return 0 } 216 let db: *u8 = sys_mmap(LW_DENTBUF) 217 let dp: *u8 = sys_mmap(LW_PATHCAP) 218 var removed: i64 = 0 219 var more: i64 = 1 220 while more == 1 { 221 let nb: i64 = sys_getdents64(fd, db, LW_DENTBUF) 222 if nb <= 0 { more = 0 } else { 223 var off: i64 = 0 224 while off < nb { 225 let rec: *u8 = ((db as i64) + off) as *u8 226 let rl: i64 = dirent_reclen(rec) 227 let nm: *u8 = dirent_name(rec) 228 if nm[0] != (LW_DOT as u8) { 229 var o2: i64 = lw_cat(dp, 0, dir) 230 dp[o2] = LW_SLASH as u8 231 o2 = o2 + 1 232 o2 = lw_cat(dp, o2, nm) 233 dp[o2] = 0 as u8 234 if dirent_type(rec) == DT_DIR { if ls_rmdir(dp) == 0 { removed = removed + 1 } } else { if sys_unlinkat(dp) == 0 { removed = removed + 1 } } 235 } 236 if rl <= 0 { off = nb } else { off = off + rl } 237 } 238 } 239 } 240 sys_munmap(dp, LW_PATHCAP) 241 sys_munmap(db, LW_DENTBUF) 242 sys_close(fd) 243 return removed 244} 245// ONE pass over the root: clear each lock, then remove it. Returns how many locks went, because REMOVING 246// ENTRIES DURING A getdents WALK CAN SKIP ENTRIES -- one pass is a FLOOR, not a clear. The caller repeats 247// while a pass still removes something; that terminates by construction, since the root is finite and 248// every non-zero pass strictly shrinks it, so no pass ceiling is guessed here. 249func lw_clearroot(root: *u8) -> i64 { 250 let fd: i64 = sys_openat_rd(root) 251 if fd < 0 { return 0 } 252 let db: *u8 = sys_mmap(LW_DENTBUF) 253 let dp: *u8 = sys_mmap(LW_PATHCAP) 254 var removed: i64 = 0 255 var more: i64 = 1 256 while more == 1 { 257 let nb: i64 = sys_getdents64(fd, db, LW_DENTBUF) 258 if nb <= 0 { more = 0 } else { 259 var off: i64 = 0 260 while off < nb { 261 let rec: *u8 = ((db as i64) + off) as *u8 262 let rl: i64 = dirent_reclen(rec) 263 if dirent_type(rec) == DT_DIR { 264 let nm: *u8 = dirent_name(rec) 265 if nm[0] != (LW_DOT as u8) { 266 var o2: i64 = lw_cat(dp, 0, root) 267 dp[o2] = LW_SLASH as u8 268 o2 = o2 + 1 269 o2 = lw_cat(dp, o2, nm) 270 dp[o2] = 0 as u8 271 lw_clearlock(dp) 272 if ls_rmdir(dp) == 0 { removed = removed + 1 } 273 } 274 } 275 if rl <= 0 { off = nb } else { off = off + rl } 276 } 277 } 278 } 279 sys_munmap(dp, LW_PATHCAP) 280 sys_munmap(db, LW_DENTBUF) 281 sys_close(fd) 282 return removed 283} 284// The clear's OWN witness: entries under root, 0 when empty OR absent. A setup step that is not asserted 285// is a setup step nobody can tell ran -- and this one is the precondition of four teeth. 286func lw_rootcount(root: *u8) -> i64 { 287 let fd: i64 = sys_openat_rd(root) 288 if fd < 0 { return 0 } 289 let db: *u8 = sys_mmap(LW_DENTBUF) 290 var n: i64 = 0 291 var more: i64 = 1 292 while more == 1 { 293 let nb: i64 = sys_getdents64(fd, db, LW_DENTBUF) 294 if nb <= 0 { more = 0 } else { 295 var off: i64 = 0 296 while off < nb { 297 let rec: *u8 = ((db as i64) + off) as *u8 298 let rl: i64 = dirent_reclen(rec) 299 let nm: *u8 = dirent_name(rec) 300 if nm[0] != (LW_DOT as u8) { n = n + 1 } 301 if rl <= 0 { off = nb } else { off = off + rl } 302 } 303 } 304 } 305 sys_munmap(db, LW_DENTBUF) 306 sys_close(fd) 307 return n 308} 309func lw_mkfix(base: *u8, name: *u8, stamp: *u8, topgen: i64, out: *u8) -> i64 { 310 var o: i64 = lw_cat(out, 0, base) 311 out[o] = 0 as u8 312 sys_mkdir(out, MODE_0755) 313 o = lw_cat(out, o, "/") 314 o = lw_cat(out, o, name) 315 out[o] = 0 as u8 316 sys_mkdir(out, MODE_0755) 317 let sp: *u8 = sys_mmap(LW_PATHCAP) 318 var so: i64 = lw_cat(sp, 0, out) 319 so = lw_cat(sp, so, "/stamp") 320 sp[so] = 0 as u8 321 if stamp[0] != (0 as u8) { 322 let fd: i64 = sys_openat_wr(sp, MODE_0644) 323 if fd >= 0 { 324 var sl: i64 = 0 325 while stamp[sl] != (0 as u8) { sl = sl + 1 } 326 sys_write(fd, stamp, sl) 327 sys_close(fd) 328 } 329 } 330 sys_munmap(sp, LW_PATHCAP) 331 var k: i64 = 2 332 while k <= topgen { 333 let gp: *u8 = sys_mmap(LW_PATHCAP) 334 var go: i64 = lw_cat(gp, 0, out) 335 go = lw_cat(gp, go, "/g") 336 go = lw_catn(gp, go, k) 337 gp[go] = 0 as u8 338 sys_mkdir(gp, MODE_0755) 339 sys_munmap(gp, LW_PATHCAP) 340 k = k + 1 341 } 342 return 0 343} 344func main(argc: i64, argv: *i64) -> i64 { 345 gv_head("nx_lease_wedge_gate -- a lease whose next takeover cannot mkdir its generation is BUSY FOREVER") 346 let ctr: *i64 = gv_ctr() 347 let fx: *u8 = sys_mmap(LW_PATHCAP) 348 let gen: *i64 = sys_mmap(LW_BOX) as *i64 349 let counts: *i64 = sys_mmap(LW_SMALL) as *i64 350 let probe: *u8 = sys_mmap(LW_PATHCAP) 351 352 // SETUP CLEAR, BEFORE ANY FIXTURE IS PLANTED (2026-09-12). It CANNOT live inside lw_mkfix: the census 353 // below runs after only FOUR of the seven fixtures are planted, so remedy/fresh/plain would still be 354 // holding the PREVIOUS run's state when it counts -- measured 7 where the tooth expects 4, and healthy 355 // 4 where it expects 1. Repeat while a pass still removes something: deleting during a getdents walk 356 // can skip entries, and the loop terminates because the root is finite and every non-zero pass shrinks 357 // it. ASSERT THE CLEAR RATHER THAN ASSUMING IT -- it is the precondition of four teeth below. 358 var cleared: i64 = 1 359 while cleared > 0 { cleared = lw_clearroot("/tmp/nx_lease_wedge_gate") } 360 gv_check_eq("setup-cleared-the-fixture-root-so-no-run-inherits-its-predecessor", lw_rootcount("/tmp/nx_lease_wedge_gate"), 0, ctr) 361 362 // FIXTURE 1: the EXACT shape found live on 2026-09-04 -- stamp generation 26, g27 present. 363 // The fixture-reached-the-condition tooth fires BEFORE any outcome tooth: a fixture the defect 364 // cannot fail is not a test. 365 lw_mkfix("/tmp/nx_lease_wedge_gate", "wedged.lock", "released\tpid25751\t1788453609\t0\t26\n", 27, fx) 366 gv_check_eq("fixture-wedged-stamp-parsed-as-26", lw_stamp_gen(fx), 26, ctr) 367 var po: i64 = lw_cat(probe, 0, fx) 368 po = lw_cat(probe, po, "/g27") 369 probe[po] = 0 as u8 370 gv_check("fixture-reached-the-condition-g27-exists", lw_exists(probe), ctr) 371 gv_check_eq("wedged-lease-classified-WEDGED", lw_classify(fx, gen), LW_WEDGED, ctr) 372 gv_check_eq("wedged-lease-reports-its-generation", gen[0], 26, ctr) 373 374 // FIXTURE 2: the SAME shape one generation on -- stamp 27, g27 present, g28 absent. That is 375 // precisely the repair applied to production, so this GREEN is the repair own regression test. 376 lw_mkfix("/tmp/nx_lease_wedge_gate", "healthy.lock", "released\tpid25751\t1788453609\t0\t27\n", 27, fx) 377 var po2: i64 = lw_cat(probe, 0, fx) 378 po2 = lw_cat(probe, po2, "/g28") 379 probe[po2] = 0 as u8 380 gv_check("fixture-reached-the-condition-g28-absent", 1 - lw_exists(probe), ctr) 381 gv_check_eq("healthy-lease-classified-HEALTHY", lw_classify(fx, gen), LW_HEALTHY, ctr) 382 gv_check_eq("healthy-lease-reports-its-generation", gen[0], 27, ctr) 383 384 // NEG-CONTROL: a stampless directory is the crash-mid-acquire window. It must be UNREADABLE, NOT 385 // healthy: an instrument that cannot see must ABSTAIN, never ACQUIT. 386 lw_mkfix("/tmp/nx_lease_wedge_gate", "stampless.lock", "", 3, fx) 387 gv_check_eq("neg-control-stampless-abstains-UNREADABLE", lw_classify(fx, gen), LW_UNREADABLE, ctr) 388 gv_check_eq("neg-control-unreadable-reports-gen-minus-1", gen[0], 0 - 1, ctr) 389 390 // NEG-CONTROL: a truncated stamp carrying only four fields has NO generation. It must abstain 391 // rather than default to 0 -- a defaulted generation would probe g1 and read HEALTHY on a wedge. 392 lw_mkfix("/tmp/nx_lease_wedge_gate", "short.lock", "released\tpid1\t1788453609\t0\n", 9, fx) 393 gv_check_eq("neg-control-4-field-stamp-abstains", lw_classify(fx, gen), LW_UNREADABLE, ctr) 394 395 // FIXTURE CENSUS: prove the walk partitions. 4 fixtures = 1 wedged + 1 healthy + 2 unreadable, and 396 // the partition MUST SUM -- an unexplained residual is a leak, an explained one is a decision. 397 lw_census("/tmp/nx_lease_wedge_gate", counts, 0) 398 gv_check_eq("census-saw-all-4-fixture-leases", counts[3], 4, ctr) 399 gv_check_eq("census-wedged-count", counts[1], 1, ctr) 400 gv_check_eq("census-healthy-count", counts[0], 1, ctr) 401 gv_check_eq("census-unreadable-count", counts[2], 2, ctr) 402 gv_check_eq("census-partition-SUMS", counts[0] + counts[1] + counts[2], counts[3], ctr) 403 404 // THE REMEDY (2026-09-06, after the second wedge in two days): a takeover over a released stamp whose next 405 // generation gate already exists reclaims that gate IFF it is older than the orphan age -- an orphan of a dead 406 // takeover -- and refuses it when it is fresh (a live claim in flight). Planted here, never on the live root. 407 lw_mkfix("/tmp/nx_lease_wedge_gate", "remedy.lock", "released\tpid1\t1\t0\t26\n", 27, fx) 408 gv_check_eq("remedy-fixture-reached-the-condition-WEDGED (stamp 26, g27 present)", lw_classify(fx, gen), LW_WEDGED, ctr) 409 let rem1: i64 = ls_claim_x(fx, "pidremedy", 900, 27, 27, 0) 410 gv_check_eq("remedy-aged-orphan-gate-is-reclaimed-and-the-takeover-WINS (orphan age 0: any age qualifies)", rem1, 1, ctr) 411 gv_check_eq("remedy-stamp-advanced-to-the-reclaimed-generation (27)", lw_stamp_gen(fx), 27, ctr) 412 gv_check_eq("remedy-lease-reads-HEALTHY-afterwards (g28 absent)", lw_classify(fx, gen), LW_HEALTHY, ctr) 413 lw_mkfix("/tmp/nx_lease_wedge_gate", "fresh.lock", "released\tpid1\t1\t0\t26\n", 27, fx) 414 let rem2: i64 = ls_claim_x(fx, "pidremedy", 900, 27, 27, 3600) 415 gv_check_eq("neg-control-a-FRESH-gate-is-a-live-claim-in-flight-and-the-takeover-is-LOST (orphan age 3600 s)", rem2, 0, ctr) 416 gv_check_eq("neg-control-fresh-gate-left-in-place-and-stamp-untouched (still 26)", lw_stamp_gen(fx), 26, ctr) 417 lw_mkfix("/tmp/nx_lease_wedge_gate", "plain.lock", "released\tpid1\t1\t0\t27\n", 27, fx) 418 let rem3: i64 = ls_claim_x(fx, "pidremedy", 900, 28, 28, 3600) 419 gv_check_eq("neg-control-a-HEALTHY-lease-takes-over-normally (no gate to reclaim, orphan age irrelevant)", rem3, 1, ctr) 420 gv_check_eq("neg-control-healthy-takeover-installs-28", lw_stamp_gen(fx), 28, ctr) 421 gv_check_eq("remedy-floor-is-max(ttl, 30 s): ttl 900 -> 900, ttl 0 -> 30", ls_orphan_age_s(900) * 1000 + ls_orphan_age_s(0), 900030, ctr) 422 423 // LIVE CENSUS. This is the alarm: it would have been RED for the whole 18-hour outage. 424 gv_puts("\n-- LIVE LEASE CENSUS (knowledge/lease) --\n") 425 let live: *i64 = sys_mmap(LW_SMALL) as *i64 426 let rc: i64 = lw_census(LW_ROOT, live, 1) 427 gv_values_head() 428 gv_kv("live_leases_seen", live[3]) 429 gv_kv("live_healthy", live[0]) 430 gv_kv("live_wedged", live[1]) 431 gv_kv("live_unreadable", live[2]) 432 gv_kv("census_rc", rc) 433 gv_kv("fixture_wedged", counts[1]) 434 gv_kv("fixture_healthy", counts[0]) 435 gv_kv("fixture_unreadable", counts[2]) 436 // gv_need, not gv_check: if the lease root cannot be read this axis is UNOBSERVABLE and must SKIP 437 // rather than pass. A gate that acquits because it could not look is worse than no gate. 438 gv_need("live-lease-root-readable", 1 - lw_pos(0 - rc), ctr) 439 gv_check("live-no-wedged-lease", 1 - lw_pos(live[1]), ctr) 440 return gv_verdict("nx_lease_wedge_gate", ctr, "the wedge test IS the failing call: stat the g<stamp_gen+1> that the next takeover will mkdir. Fixtures are built under /tmp at runtime and the live census NEVER writes.") 441}