code wiki / _hdl_build / nx_lease_wedge_gate.nx
nx_lease_wedge_gate.nx source
↩ module page · 441 lines · 21908 B
1// nx_lease_wedge_gate.nx -- THE LEASE-WEDGE DETECTOR (2026-09-04).
2//
3// WHY THIS EXISTS. On 2026-09-04 all four knowledge/lease/heavyio-slot-N.lock slots were found
4// PERMANENTLY WEDGED. The crawler AND the shard-compactor had been dead for 18 HOURS and NOTHING in
5// the estate noticed: /search kept answering 200, every daemon read UP, and the only symptom was a
6// status log whose mtime stopped advancing. The operator's report was "search still searches nothing".
7//
8// THE MECHANISM, read out of buildroot/runtime/nx_lease_lib.nx rather than guessed. A takeover is
9// arbitrated by mkdir on a monotonic generation: ls_acquire_root parses the stamp, and for a released
10// or expired lease calls ls_claim(dir, owner, ttl, g_nonce + 1, g_nonce + 1), whose FIRST act is
11// if ls_mkdir(dir + "/g" + gate) != 0 { return 0 } // 0 == LOST == "busy"
12// and whose stamp write is a SEPARATE, LATER, NON-ATOMIC step:
13// let tfd = sys_openat_wr(tp, LS_MODE) ... sys_write ... sys_renameat(tp, sp)
14// A process killed between the mkdir and the rename leaves g<N> ON DISK with the stamp still reading
15// N-1. Every later acquirer then re-derives the SAME gate from the SAME stamp, retries mkdir g<N>,
16// gets EEXIST, and correctly reports BUSY -- forever, with no recovery path and no diagnostic.
17// The host bugchecked 3x in 72 h (GPU driver), which is exactly the kill this window needs.
18//
19// THE TEST IS THE FAILING CALL ITSELF, NOT A PROXY. This gate does not ask "does maxdir exceed the
20// stamp"; it asks the one question that decides the outcome: DOES THE DIRECTORY THE NEXT TAKEOVER
21// WILL TRY TO CREATE ALREADY EXIST? That is stat of <lease>/g<stamp_gen + 1>. A proxy could be right
22// for the wrong reason; this cannot.
23// THE PROBE PATH IS NEVER CREATED. Calling mkdir on g<gen+1> to test for it WOULD ITSELF WEDGE THE
24// LEASE -- the detector would manufacture the defect it reports. Existence is read with fstatat and
25// nothing here writes to a live lease, ever.
26//
27// CHECK-BEFORE-BUILD (rule: "I didn't know it existed" is a retrieval failure). nx_spendgate reports
28// presubmit=FREE and nx_capsearch over 7,341 organs (corpus_complete=1) returns nx_lease and
29// nx_lease_probe -- the PRIMITIVE itself -- plus nx_lease_lib and nx_vault_lease. None censuses lease
30// HEALTH; there is no acquire/release verb that can even express "this lease can never be claimed
31// again". This is a new capability, not a duplicate ruler.
32//
33// SCOPE, STATED SO NOBODY TRUSTS IT AS EXACT: this detects the WEDGE. It does not detect a lease held
34// by a dead process (that is a TTL question and ls_acquire already expires it), and it says nothing
35// about whether the holder is making progress.
36//
37// OWED, DELIBERATELY NOT ATTEMPTED HERE: ls_claim mkdir-then-stamp has no crash recovery. Fixing a
38// shared estate-wide mutex is not a safe act under load, and that file own comments record TWO prior
39// attempts at exactly this which wedged it or double-admitted. A detector first is the right order.
40import "nx_gate_verdict.nx"
41import "nx_syscalls.nx"
42import "nx_lease_lib.nx" // THE REMEDY under test (2026-09-06): ls_claim_x reclaims an AGED orphan gate and refuses a fresh one
43
44const LW_ROOT: *u8 = "knowledge/lease"
45const LW_PATHCAP: i64 = 1024
46const LW_STAMPCAP: i64 = 512
47const LW_DENTBUF: i64 = 65536
48const LW_STATBUF: i64 = 160
49const LW_TAB: i64 = 9
50const LW_NL: i64 = 10
51const LW_D0: i64 = 48
52const LW_D9: i64 = 57
53const LW_B10: i64 = 10
54const LW_GEN_FIELD: i64 = 4
55const LW_DOT: i64 = 46
56const LW_SLASH: i64 = 47
57const LW_SCRATCH: i64 = 32
58const LW_SMALL: i64 = 64
59const LW_BOX: i64 = 16
60
61const LW_HEALTHY: i64 = 0
62const LW_WEDGED: i64 = 1
63const LW_UNREADABLE: i64 = 2
64
65func lw_cat(d: *u8, o: i64, s: *u8) -> i64 {
66 var i: i64 = 0
67 var p: i64 = o
68 while s[i] != (0 as u8) { d[p] = s[i]; p = p + 1; i = i + 1 }
69 return p
70}
71func lw_catn(d: *u8, o: i64, v: i64) -> i64 {
72 if v == 0 { d[o] = LW_D0 as u8; return o + 1 }
73 var tmp: i64 = v
74 var n: i64 = 0
75 let sc: *u8 = sys_mmap(LW_SCRATCH)
76 while tmp > 0 { sc[n] = (LW_D0 + (tmp % LW_B10)) as u8; n = n + 1; tmp = tmp / LW_B10 }
77 var p: i64 = o
78 var k: i64 = n
79 while k > 0 { k = k - 1; d[p] = sc[k]; p = p + 1 }
80 sys_munmap(sc, LW_SCRATCH)
81 return p
82}
83func lw_pos(v: i64) -> i64 { if v > 0 { return 1 } return 0 }
84// READ-ONLY existence. 1 if the path exists, 0 if not. NEVER creates.
85func lw_exists(path: *u8) -> i64 {
86 let sb: *u8 = sys_mmap(LW_STATBUF)
87 let r: i64 = sys_fstatat(path, sb)
88 sys_munmap(sb, LW_STATBUF)
89 if r == 0 { return 1 }
90 return 0
91}
92// Parse the stamp GENERATION (field 4, tab-separated). Returns the generation, or -1 when the stamp
93// cannot be read or does not carry five fields. A malformed stamp is UNREADABLE, never "generation 0":
94// defaulting here would let a corrupt stamp probe g1 and read HEALTHY on a wedged lease.
95func lw_stamp_gen(dir: *u8) -> i64 {
96 let sp: *u8 = sys_mmap(LW_PATHCAP)
97 var o: i64 = lw_cat(sp, 0, dir)
98 o = lw_cat(sp, o, "/stamp")
99 sp[o] = 0 as u8
100 let fd: i64 = sys_openat_rd(sp)
101 sys_munmap(sp, LW_PATHCAP)
102 if fd < 0 { return 0 - 1 }
103 let buf: *u8 = sys_mmap(LW_STAMPCAP)
104 let n: i64 = sys_read(fd, buf, LW_STAMPCAP - 1)
105 sys_close(fd)
106 if n <= 0 { sys_munmap(buf, LW_STAMPCAP); return 0 - 1 }
107 var field: i64 = 0
108 var v: i64 = 0
109 var seen: i64 = 0
110 var i: i64 = 0
111 while i < n {
112 let c: i64 = buf[i] as i64
113 if c == LW_TAB {
114 field = field + 1
115 v = 0
116 seen = 0
117 } else {
118 if c == LW_NL { i = n } else {
119 if field == LW_GEN_FIELD {
120 if c >= LW_D0 { if c <= LW_D9 { v = v * LW_B10 + (c - LW_D0); seen = 1 } }
121 }
122 }
123 }
124 i = i + 1
125 }
126 sys_munmap(buf, LW_STAMPCAP)
127 if field < LW_GEN_FIELD { return 0 - 1 }
128 if seen == 0 { return 0 - 1 }
129 return v
130}
131// THE CLASSIFIER. Returns LW_HEALTHY / LW_WEDGED / LW_UNREADABLE and writes the observed generation
132// into genout[0] (-1 when unreadable) so a caller prints evidence rather than a bare verdict.
133func lw_classify(dir: *u8, genout: *i64) -> i64 {
134 let g: i64 = lw_stamp_gen(dir)
135 genout[0] = g
136 if g < 0 { return LW_UNREADABLE }
137 let gp: *u8 = sys_mmap(LW_PATHCAP)
138 var o: i64 = lw_cat(gp, 0, dir)
139 o = lw_cat(gp, o, "/g")
140 o = lw_catn(gp, o, g + 1)
141 gp[o] = 0 as u8
142 let ex: i64 = lw_exists(gp)
143 sys_munmap(gp, LW_PATHCAP)
144 if ex == 1 { return LW_WEDGED }
145 return LW_HEALTHY
146}
147// Census every lease under root. counts[0]=healthy counts[1]=wedged counts[2]=unreadable
148// counts[3]=leases seen. Prints one line per NON-healthy lease: a count without a worklist is not
149// actionable, so every offender is NAMED with the exact path the next takeover will collide on.
150func lw_census(root: *u8, counts: *i64, verbose: i64) -> i64 {
151 counts[0] = 0
152 counts[1] = 0
153 counts[2] = 0
154 counts[3] = 0
155 let fd: i64 = sys_openat_rd(root)
156 if fd < 0 { return 0 - 1 }
157 let db: *u8 = sys_mmap(LW_DENTBUF)
158 let gen: *i64 = sys_mmap(LW_BOX) as *i64
159 let dp: *u8 = sys_mmap(LW_PATHCAP)
160 var more: i64 = 1
161 while more == 1 {
162 let nb: i64 = sys_getdents64(fd, db, LW_DENTBUF)
163 if nb <= 0 { more = 0 } else {
164 var off: i64 = 0
165 while off < nb {
166 let rec: *u8 = ((db as i64) + off) as *u8
167 let rl: i64 = dirent_reclen(rec)
168 if dirent_type(rec) == DT_DIR {
169 let nm: *u8 = dirent_name(rec)
170 var skip: i64 = 0
171 if nm[0] == (LW_DOT as u8) { skip = 1 }
172 if skip == 0 {
173 var o2: i64 = lw_cat(dp, 0, root)
174 dp[o2] = LW_SLASH as u8
175 o2 = o2 + 1
176 o2 = lw_cat(dp, o2, nm)
177 dp[o2] = 0 as u8
178 let cls: i64 = lw_classify(dp, gen)
179 counts[3] = counts[3] + 1
180 counts[cls] = counts[cls] + 1
181 if verbose == 1 { if cls != LW_HEALTHY {
182 if cls == LW_WEDGED { gv_puts(" WEDGED ") } else { gv_puts(" UNREADABLE ") }
183 gv_puts(dp)
184 gv_puts(" stamp_gen=")
185 gv_num(gen[0])
186 gv_puts(" -- the next takeover mkdirs g")
187 gv_num(gen[0] + 1)
188 gv_puts(" and that path ALREADY EXISTS, so every acquire returns BUSY forever\n")
189 } }
190 }
191 }
192 if rl <= 0 { off = nb } else { off = off + rl }
193 }
194 }
195 }
196 sys_munmap(db, LW_DENTBUF)
197 sys_munmap(gen as *u8, LW_BOX)
198 sys_munmap(dp, LW_PATHCAP)
199 sys_close(fd)
200 return 0
201}
202// Fixtures are built at RUNTIME under /tmp and never share a path with a production beat: a gate that
203// shares a fixture with a live writer measures the writer, and its RED tracks the fixture not the code.
204// CLEAR BEFORE PLANT (2026-09-12): lw_mkfix is sys_mkdir-only and removes NOTHING, so every .lock and
205// every g<n> it has ever created survives between runs -- measured 7 dirs where the census tooth expects
206// 4, plus g0 dirs dated 2026-09-04 that no current call site can emit (every ls_claim gate is g_nonce+1,
207// whose floor is 1). The residue is NOT inert: a leftover g<n> IS the EEXIST the takeover under test must
208// not find, so the two neg-controls ALTERNATED on how long ago this gate last ran -- fresh.lock failed
209// once the residue aged past 3600 s, plain.lock failed while it had not, and both runs scored 21/25 with
210// a different pair red. A GATE WHOSE ONLY DISCRIMINATOR IS AGE MUST NOT LET THAT AGE ACCUMULATE ACROSS
211// RUNS. The bound is DERIVED BY ENUMERATION, never a guessed generation ceiling, so the fossils go too.
212// Files are unlinked as well as dirs removed: a surviving stamp makes the parent rmdir fail ENOTEMPTY.
213func lw_clearlock(dir: *u8) -> i64 {
214 let fd: i64 = sys_openat_rd(dir)
215 if fd < 0 { return 0 }
216 let db: *u8 = sys_mmap(LW_DENTBUF)
217 let dp: *u8 = sys_mmap(LW_PATHCAP)
218 var removed: i64 = 0
219 var more: i64 = 1
220 while more == 1 {
221 let nb: i64 = sys_getdents64(fd, db, LW_DENTBUF)
222 if nb <= 0 { more = 0 } else {
223 var off: i64 = 0
224 while off < nb {
225 let rec: *u8 = ((db as i64) + off) as *u8
226 let rl: i64 = dirent_reclen(rec)
227 let nm: *u8 = dirent_name(rec)
228 if nm[0] != (LW_DOT as u8) {
229 var o2: i64 = lw_cat(dp, 0, dir)
230 dp[o2] = LW_SLASH as u8
231 o2 = o2 + 1
232 o2 = lw_cat(dp, o2, nm)
233 dp[o2] = 0 as u8
234 if dirent_type(rec) == DT_DIR { if ls_rmdir(dp) == 0 { removed = removed + 1 } } else { if sys_unlinkat(dp) == 0 { removed = removed + 1 } }
235 }
236 if rl <= 0 { off = nb } else { off = off + rl }
237 }
238 }
239 }
240 sys_munmap(dp, LW_PATHCAP)
241 sys_munmap(db, LW_DENTBUF)
242 sys_close(fd)
243 return removed
244}
245// ONE pass over the root: clear each lock, then remove it. Returns how many locks went, because REMOVING
246// ENTRIES DURING A getdents WALK CAN SKIP ENTRIES -- one pass is a FLOOR, not a clear. The caller repeats
247// while a pass still removes something; that terminates by construction, since the root is finite and
248// every non-zero pass strictly shrinks it, so no pass ceiling is guessed here.
249func lw_clearroot(root: *u8) -> i64 {
250 let fd: i64 = sys_openat_rd(root)
251 if fd < 0 { return 0 }
252 let db: *u8 = sys_mmap(LW_DENTBUF)
253 let dp: *u8 = sys_mmap(LW_PATHCAP)
254 var removed: i64 = 0
255 var more: i64 = 1
256 while more == 1 {
257 let nb: i64 = sys_getdents64(fd, db, LW_DENTBUF)
258 if nb <= 0 { more = 0 } else {
259 var off: i64 = 0
260 while off < nb {
261 let rec: *u8 = ((db as i64) + off) as *u8
262 let rl: i64 = dirent_reclen(rec)
263 if dirent_type(rec) == DT_DIR {
264 let nm: *u8 = dirent_name(rec)
265 if nm[0] != (LW_DOT as u8) {
266 var o2: i64 = lw_cat(dp, 0, root)
267 dp[o2] = LW_SLASH as u8
268 o2 = o2 + 1
269 o2 = lw_cat(dp, o2, nm)
270 dp[o2] = 0 as u8
271 lw_clearlock(dp)
272 if ls_rmdir(dp) == 0 { removed = removed + 1 }
273 }
274 }
275 if rl <= 0 { off = nb } else { off = off + rl }
276 }
277 }
278 }
279 sys_munmap(dp, LW_PATHCAP)
280 sys_munmap(db, LW_DENTBUF)
281 sys_close(fd)
282 return removed
283}
284// The clear's OWN witness: entries under root, 0 when empty OR absent. A setup step that is not asserted
285// is a setup step nobody can tell ran -- and this one is the precondition of four teeth.
286func lw_rootcount(root: *u8) -> i64 {
287 let fd: i64 = sys_openat_rd(root)
288 if fd < 0 { return 0 }
289 let db: *u8 = sys_mmap(LW_DENTBUF)
290 var n: i64 = 0
291 var more: i64 = 1
292 while more == 1 {
293 let nb: i64 = sys_getdents64(fd, db, LW_DENTBUF)
294 if nb <= 0 { more = 0 } else {
295 var off: i64 = 0
296 while off < nb {
297 let rec: *u8 = ((db as i64) + off) as *u8
298 let rl: i64 = dirent_reclen(rec)
299 let nm: *u8 = dirent_name(rec)
300 if nm[0] != (LW_DOT as u8) { n = n + 1 }
301 if rl <= 0 { off = nb } else { off = off + rl }
302 }
303 }
304 }
305 sys_munmap(db, LW_DENTBUF)
306 sys_close(fd)
307 return n
308}
309func lw_mkfix(base: *u8, name: *u8, stamp: *u8, topgen: i64, out: *u8) -> i64 {
310 var o: i64 = lw_cat(out, 0, base)
311 out[o] = 0 as u8
312 sys_mkdir(out, MODE_0755)
313 o = lw_cat(out, o, "/")
314 o = lw_cat(out, o, name)
315 out[o] = 0 as u8
316 sys_mkdir(out, MODE_0755)
317 let sp: *u8 = sys_mmap(LW_PATHCAP)
318 var so: i64 = lw_cat(sp, 0, out)
319 so = lw_cat(sp, so, "/stamp")
320 sp[so] = 0 as u8
321 if stamp[0] != (0 as u8) {
322 let fd: i64 = sys_openat_wr(sp, MODE_0644)
323 if fd >= 0 {
324 var sl: i64 = 0
325 while stamp[sl] != (0 as u8) { sl = sl + 1 }
326 sys_write(fd, stamp, sl)
327 sys_close(fd)
328 }
329 }
330 sys_munmap(sp, LW_PATHCAP)
331 var k: i64 = 2
332 while k <= topgen {
333 let gp: *u8 = sys_mmap(LW_PATHCAP)
334 var go: i64 = lw_cat(gp, 0, out)
335 go = lw_cat(gp, go, "/g")
336 go = lw_catn(gp, go, k)
337 gp[go] = 0 as u8
338 sys_mkdir(gp, MODE_0755)
339 sys_munmap(gp, LW_PATHCAP)
340 k = k + 1
341 }
342 return 0
343}
344func main(argc: i64, argv: *i64) -> i64 {
345 gv_head("nx_lease_wedge_gate -- a lease whose next takeover cannot mkdir its generation is BUSY FOREVER")
346 let ctr: *i64 = gv_ctr()
347 let fx: *u8 = sys_mmap(LW_PATHCAP)
348 let gen: *i64 = sys_mmap(LW_BOX) as *i64
349 let counts: *i64 = sys_mmap(LW_SMALL) as *i64
350 let probe: *u8 = sys_mmap(LW_PATHCAP)
351
352 // SETUP CLEAR, BEFORE ANY FIXTURE IS PLANTED (2026-09-12). It CANNOT live inside lw_mkfix: the census
353 // below runs after only FOUR of the seven fixtures are planted, so remedy/fresh/plain would still be
354 // holding the PREVIOUS run's state when it counts -- measured 7 where the tooth expects 4, and healthy
355 // 4 where it expects 1. Repeat while a pass still removes something: deleting during a getdents walk
356 // can skip entries, and the loop terminates because the root is finite and every non-zero pass shrinks
357 // it. ASSERT THE CLEAR RATHER THAN ASSUMING IT -- it is the precondition of four teeth below.
358 var cleared: i64 = 1
359 while cleared > 0 { cleared = lw_clearroot("/tmp/nx_lease_wedge_gate") }
360 gv_check_eq("setup-cleared-the-fixture-root-so-no-run-inherits-its-predecessor", lw_rootcount("/tmp/nx_lease_wedge_gate"), 0, ctr)
361
362 // FIXTURE 1: the EXACT shape found live on 2026-09-04 -- stamp generation 26, g27 present.
363 // The fixture-reached-the-condition tooth fires BEFORE any outcome tooth: a fixture the defect
364 // cannot fail is not a test.
365 lw_mkfix("/tmp/nx_lease_wedge_gate", "wedged.lock", "released\tpid25751\t1788453609\t0\t26\n", 27, fx)
366 gv_check_eq("fixture-wedged-stamp-parsed-as-26", lw_stamp_gen(fx), 26, ctr)
367 var po: i64 = lw_cat(probe, 0, fx)
368 po = lw_cat(probe, po, "/g27")
369 probe[po] = 0 as u8
370 gv_check("fixture-reached-the-condition-g27-exists", lw_exists(probe), ctr)
371 gv_check_eq("wedged-lease-classified-WEDGED", lw_classify(fx, gen), LW_WEDGED, ctr)
372 gv_check_eq("wedged-lease-reports-its-generation", gen[0], 26, ctr)
373
374 // FIXTURE 2: the SAME shape one generation on -- stamp 27, g27 present, g28 absent. That is
375 // precisely the repair applied to production, so this GREEN is the repair own regression test.
376 lw_mkfix("/tmp/nx_lease_wedge_gate", "healthy.lock", "released\tpid25751\t1788453609\t0\t27\n", 27, fx)
377 var po2: i64 = lw_cat(probe, 0, fx)
378 po2 = lw_cat(probe, po2, "/g28")
379 probe[po2] = 0 as u8
380 gv_check("fixture-reached-the-condition-g28-absent", 1 - lw_exists(probe), ctr)
381 gv_check_eq("healthy-lease-classified-HEALTHY", lw_classify(fx, gen), LW_HEALTHY, ctr)
382 gv_check_eq("healthy-lease-reports-its-generation", gen[0], 27, ctr)
383
384 // NEG-CONTROL: a stampless directory is the crash-mid-acquire window. It must be UNREADABLE, NOT
385 // healthy: an instrument that cannot see must ABSTAIN, never ACQUIT.
386 lw_mkfix("/tmp/nx_lease_wedge_gate", "stampless.lock", "", 3, fx)
387 gv_check_eq("neg-control-stampless-abstains-UNREADABLE", lw_classify(fx, gen), LW_UNREADABLE, ctr)
388 gv_check_eq("neg-control-unreadable-reports-gen-minus-1", gen[0], 0 - 1, ctr)
389
390 // NEG-CONTROL: a truncated stamp carrying only four fields has NO generation. It must abstain
391 // rather than default to 0 -- a defaulted generation would probe g1 and read HEALTHY on a wedge.
392 lw_mkfix("/tmp/nx_lease_wedge_gate", "short.lock", "released\tpid1\t1788453609\t0\n", 9, fx)
393 gv_check_eq("neg-control-4-field-stamp-abstains", lw_classify(fx, gen), LW_UNREADABLE, ctr)
394
395 // FIXTURE CENSUS: prove the walk partitions. 4 fixtures = 1 wedged + 1 healthy + 2 unreadable, and
396 // the partition MUST SUM -- an unexplained residual is a leak, an explained one is a decision.
397 lw_census("/tmp/nx_lease_wedge_gate", counts, 0)
398 gv_check_eq("census-saw-all-4-fixture-leases", counts[3], 4, ctr)
399 gv_check_eq("census-wedged-count", counts[1], 1, ctr)
400 gv_check_eq("census-healthy-count", counts[0], 1, ctr)
401 gv_check_eq("census-unreadable-count", counts[2], 2, ctr)
402 gv_check_eq("census-partition-SUMS", counts[0] + counts[1] + counts[2], counts[3], ctr)
403
404 // THE REMEDY (2026-09-06, after the second wedge in two days): a takeover over a released stamp whose next
405 // generation gate already exists reclaims that gate IFF it is older than the orphan age -- an orphan of a dead
406 // takeover -- and refuses it when it is fresh (a live claim in flight). Planted here, never on the live root.
407 lw_mkfix("/tmp/nx_lease_wedge_gate", "remedy.lock", "released\tpid1\t1\t0\t26\n", 27, fx)
408 gv_check_eq("remedy-fixture-reached-the-condition-WEDGED (stamp 26, g27 present)", lw_classify(fx, gen), LW_WEDGED, ctr)
409 let rem1: i64 = ls_claim_x(fx, "pidremedy", 900, 27, 27, 0)
410 gv_check_eq("remedy-aged-orphan-gate-is-reclaimed-and-the-takeover-WINS (orphan age 0: any age qualifies)", rem1, 1, ctr)
411 gv_check_eq("remedy-stamp-advanced-to-the-reclaimed-generation (27)", lw_stamp_gen(fx), 27, ctr)
412 gv_check_eq("remedy-lease-reads-HEALTHY-afterwards (g28 absent)", lw_classify(fx, gen), LW_HEALTHY, ctr)
413 lw_mkfix("/tmp/nx_lease_wedge_gate", "fresh.lock", "released\tpid1\t1\t0\t26\n", 27, fx)
414 let rem2: i64 = ls_claim_x(fx, "pidremedy", 900, 27, 27, 3600)
415 gv_check_eq("neg-control-a-FRESH-gate-is-a-live-claim-in-flight-and-the-takeover-is-LOST (orphan age 3600 s)", rem2, 0, ctr)
416 gv_check_eq("neg-control-fresh-gate-left-in-place-and-stamp-untouched (still 26)", lw_stamp_gen(fx), 26, ctr)
417 lw_mkfix("/tmp/nx_lease_wedge_gate", "plain.lock", "released\tpid1\t1\t0\t27\n", 27, fx)
418 let rem3: i64 = ls_claim_x(fx, "pidremedy", 900, 28, 28, 3600)
419 gv_check_eq("neg-control-a-HEALTHY-lease-takes-over-normally (no gate to reclaim, orphan age irrelevant)", rem3, 1, ctr)
420 gv_check_eq("neg-control-healthy-takeover-installs-28", lw_stamp_gen(fx), 28, ctr)
421 gv_check_eq("remedy-floor-is-max(ttl, 30 s): ttl 900 -> 900, ttl 0 -> 30", ls_orphan_age_s(900) * 1000 + ls_orphan_age_s(0), 900030, ctr)
422
423 // LIVE CENSUS. This is the alarm: it would have been RED for the whole 18-hour outage.
424 gv_puts("\n-- LIVE LEASE CENSUS (knowledge/lease) --\n")
425 let live: *i64 = sys_mmap(LW_SMALL) as *i64
426 let rc: i64 = lw_census(LW_ROOT, live, 1)
427 gv_values_head()
428 gv_kv("live_leases_seen", live[3])
429 gv_kv("live_healthy", live[0])
430 gv_kv("live_wedged", live[1])
431 gv_kv("live_unreadable", live[2])
432 gv_kv("census_rc", rc)
433 gv_kv("fixture_wedged", counts[1])
434 gv_kv("fixture_healthy", counts[0])
435 gv_kv("fixture_unreadable", counts[2])
436 // gv_need, not gv_check: if the lease root cannot be read this axis is UNOBSERVABLE and must SKIP
437 // rather than pass. A gate that acquits because it could not look is worse than no gate.
438 gv_need("live-lease-root-readable", 1 - lw_pos(0 - rc), ctr)
439 gv_check("live-no-wedged-lease", 1 - lw_pos(live[1]), ctr)
440 return gv_verdict("nx_lease_wedge_gate", ctr, "the wedge test IS the failing call: stat the g<stamp_gen+1> that the next takeover will mkdir. Fixtures are built under /tmp at runtime and the live census NEVER writes.")
441}