code wiki / (root) / nx_joblost_gate.nx
`nx_joblost_gate.nx` source

↩ module page · 623 lines · 36156 B
1// nx_joblost_gate.nx -- WHICH ASYNC JOBS DIED, as distinct from which are still running.
2//
3// WHY THIS EXISTS, measured 2026-08-07. Under load (loadavg 18.18 on the 8-thread box,
4// nx_tools_api_se in D-state, /api/health 503 "upstream backend gave no response inside the edge
5// window") nx_tools_api claims a job id with O_CREAT|O_EXCL and can then die before the worker ever
6// produces output. The caller gets the bare JSON object `{}`. Two of my writes were destroyed that
7// way inside five minutes and I only noticed because I happened to read the row back. The estate's
8// standing rule for a dropped response -- VERIFY BY ARTEFACT -- fails silently here, because the
9// artefact was never written and the job record is EMPTY rather than ABSENT.
10//
11// ---------------------------------------------------------------------------------------------
12// v1 OF THIS GATE WAS WRONG AND SHIPPED GREEN. It counted every zero-byte .claim as a LOST job.
13// Reading nx_tools_api afterwards showed the claim is created EMPTY and only filled at completion
14// (ta_job_put -> sys_renameat), so a zero-byte claim is ALSO the normal representation of a job that
15// is still RUNNING. v1 therefore merged "in flight" with "died" under the alarming name.
16//   ★★★★★★ I WROTE "UNKNOWN IS ITS OWN BUCKET" INTO THIS FILE'S OWN HEADER AND THEN BROKE IT IN THE
17//   SAME HOUR. A LAW YOU CAN RECITE IS NOT A LAW YOU HAVE APPLIED.
18// v1 also declared the imprecision as unavoidable -- "separating in flight from wedged needs a clock
19// and this gate does not have one". IT HAD ONE ALL ALONG: the job id IS the epoch second, printed in
20// every filename.
21//   ★★★★★★ THE INFORMATION I DECLARED UNAVAILABLE WAS ENCODED IN THE NAME OF THE FILE I WAS ALREADY
22//   READING. "THIS CANNOT BE MEASURED" IS A CLAIM THAT MUST BE EARNED, EXACTLY LIKE ANY OTHER.
23//
24// FIVE BUCKETS, each with a different remedy, none folded into another:
25//   DONE     state=DONE was written        -- healthy, nothing to do
26//   INFLIGHT empty claim, no .out, young   -- probably running; NOT an alarm
27//   LOST     empty claim, no .out, OLD     -- the id was reserved and nothing ever came back
28//   ORPHAN   empty claim but .out EXISTS   -- the worker produced output and never renamed the claim,
29//                                             so a poller waiting on state=DONE waits forever
30//   REAPED   a tombstone was APPENDED      -- adjudicated and closed by nx_jobclaim_reap; the claim
31//                                             keeps its original marker, so only a reader that tests
32//                                             REAPED FIRST can see it (2026-08-20)
33// The age threshold is DECLARED and argv-overridable, and it is deliberately generous: jobs run with
34// tmo=0 (no timeout) precisely so they can outlive the request window, so a tight bound would accuse
35// healthy long jobs. ★ A HEURISTIC THAT GATES AN ALARM MUST BE WRONG IN THE DIRECTION OF SILENCE.
36//
37// license_tier: ORIGINAL  expect_exit: 0
38import "syscalls.nx"
39import "nx_gate_verdict.nx"
40// THE SHARED CONTRACT WITH THE REAPER. jr_state_of, the marker literals and the age threshold live
41// in ONE place so this READER and the WRITER (nx_jobclaim_reap) cannot disagree on the wire.
42import "nx_jobclaim_lib.nx"
43
44const JL_JOBS:     *u8 = "_jobs" as *u8
45const JL_RATCHET:  *u8 = "knowledge/status/joblost_ratchet.conf" as *u8
46const JL_DEFAULT_FLOOR: i64 = 0
47// THE AGE THRESHOLD IS NO LONGER A CONST HERE. It is read through nx_jobclaim_lib.jr_maxage, which
48// both this census and the reaper call, so the two hold ONE number: the reaper is only sound while
49// its guard is at least as generous as this bar, and sharing the ruler makes that hold by
50// construction instead of by two people remembering the same literal. The code default when the conf
51// is absent is JR_DEFAULT_MAX_AGE_SEC, and it is the same 3600 this line used to carry.
52// WARNING THIS LINE USED TO CARRY: it claimed argv[1] overrides, and main() never read argv at all.
53// A documented override that does not exist is worse than none. It exists now (argv > conf > code
54// default, rule 17) and it refuses any argv[1] that is not a positive integer, because the MCP
55// surface passes a VERB there and a lenient parse would make maxage 0 and report the whole plane LOST.
56const JL_DBUF:     i64 = 65536
57const JL_FBUF:     i64 = 4096
58const JL_PATH:     i64 = 1024
59const JL_MODE:     i64 = 420
60
61const JL_TOTAL:    i64 = 0
62const JL_DONE:     i64 = 1
63const JL_INFLIGHT: i64 = 2
64const JL_LOST:     i64 = 3
65const JL_ORPHAN:   i64 = 4
66const JL_UNKNOWN:  i64 = 5
67// REAPED, added 2026-08-20 with nx_jobclaim_reap. A dead claim is now retired by APPENDING a
68// tombstone rather than by an unlink -- rule 13, the reservation record is the evidence a later
69// investigation needs, and an unlink would destroy it. That makes REAPED a SIXTH state with its own
70// remedy (none: it is closed business), and it gets its own bucket rather than being folded into
71// DONE, which would claim the job finished, or left inside CLAIMED, which would keep counting it
72// LOST forever.
73//   A REAPER SHIPPED ALONE IS A NO-OP THAT LOOKS LIKE A FIX. This bucket and the REAPED-first order
74//   inside jr_state_of are the READER half of that change; without them the tombstone lands, this
75//   count does not move, and the partition tooth breaks.
76const JL_REAPED:   i64 = 6
77// The number of COUNTER buckets, so the zeroing loop is derived from the layout instead of carrying
78// a hand-typed 6 that silently stops zeroing the newest bucket. Must stay <= JL_IDBASE or the first
79// LOST id would be overwritten by a counter; T10 asserts exactly that.
80const JL_SLOTS:    i64 = 7
81
82// THE WORKLIST TRAVELS WITH THE COUNT. `LOST=4` and nothing else forces the reader to re-derive by hand
83// exactly the scan this organ just performed. The ids live in the SAME `out` block as the counters, past
84// JL_IDBASE, so no call site changes signature -- only the allocation grows, and it is DERIVED from
85// (JL_IDBASE + JL_LOSTCAP) rather than typed as a second copy of its own shape.
86const JL_IDBASE:   i64 = 8
87const JL_LOSTCAP:  i64 = 32
88
89func jl_slen(p: *u8) -> i64 {
  var n: i64 = 0
  while p[n] != (0 as u8) { n = n + 1 }
  return n
93}
94
95// jl_has WAS HERE and is now jr_has in nx_jobclaim_lib. It was deleted rather than left beside the
96// imported one: two substring searchers in one compilation unit is the duplicate-ruler defect, and a
97// function nothing calls is what nx_unwired counts. The marker LITERALS moved with it, which is the
98// point -- the writer and this reader now take them from the same place.
99
100func jl_ends_claim(nm: *u8, nl: i64) -> i64 {
  if nl < 7 { return 0 }
  let suf: *u8 = ".claim" as *u8
  var k: i64 = 0
  while k < 6 { if nm[nl-6+k] != suf[k] { return 0 } k = k + 1 }
  return 1
106}
107
108// "job_<digits>.claim" -> <digits>, or -1 when the name does not carry an id. A name we cannot parse
109// must NOT silently become age 0 (which would read as INFLIGHT and hide it) -- it becomes UNKNOWN.
110func jl_parse_id(nm: *u8, nl: i64) -> i64 {
  let pre: *u8 = "job_" as *u8
  if nl < 5 { return 0 - 1 }
  var k: i64 = 0
  while k < 4 { if nm[k] != pre[k] { return 0 - 1 } k = k + 1 }
  var v: i64 = 0
  var seen: i64 = 0
  var i: i64 = 4
  while i < nl {
      let c: i64 = nm[i] as i64
      if c >= 48 { if c <= 57 { v = v * 10 + (c - 48); seen = 1; i = i + 1 } else { i = nl } }
      else { i = nl }
  }
  if seen == 0 { return 0 - 1 }
  return v
125}
126
127func jl_read(path: *u8, buf: *u8, cap: i64) -> i64 {
  let fd: i64 = sys_openat_rd(path)
  if fd < 0 { return 0 - 1 }
  var tot: i64 = 0
  while tot < cap {
      let n: i64 = sys_read(fd, (buf as i64 + tot) as *u8, cap - tot)
      if n <= 0 { break }
      tot = tot + n
  }
  sys_close(fd)
  return tot
138}
139
140func jl_exists(path: *u8) -> i64 {
  let fd: i64 = sys_openat_rd(path)
  if fd < 0 { return 0 }
  sys_close(fd)
  return 1
145}
146
147// THE CLASSIFIER, a function over a DIRECTORY PATH so the fixtures below drive THIS code rather than a
148// copy of it -- a tooth that tests a re-implementation proves nothing about what ships.
149// Returns 0 ok, -1 if the directory could not be opened (a precondition failure, never a silent zero).
150func jl_scan(dir: *u8, now: i64, maxage: i64, out: *i64, dbuf: *u8, fbuf: *u8, pbuf: *u8) -> i64 {
  var z: i64 = 0
  while z < JL_SLOTS { out[z] = 0; z = z + 1 }
153
  let dfd: i64 = __syscall(257, 0-100, dir, 0x10000, 0, 0, 0)
  if dfd < 0 { return 0 - 1 }
156
  let dl: i64 = jl_slen(dir)
  var go: i64 = 1
  // ONE getdents64 CALL IS NOT A DIRECTORY LISTING -- loop until it returns 0, or a big directory is
  // read as a prefix and its total published as fact.
  while go == 1 {
      let nread: i64 = __syscall(217, dfd, dbuf, JL_DBUF, 0, 0, 0)
      if nread <= 0 { go = 0 } else {
          var pos: i64 = 0
          while pos < nread {
              let reclen: i64 = (dbuf[pos+16] as i64) | ((dbuf[pos+17] as i64) << 8)
              var nl: i64 = 0
              while dbuf[pos+19+nl] != (0 as u8) { nl = nl + 1 }
              let nmp: *u8 = ((dbuf as i64) + pos + 19) as *u8
              if jl_ends_claim(nmp, nl) == 1 {
                  if dl + nl + 8 < JL_PATH {
                      var c: i64 = 0
                      while c < dl { pbuf[c] = dir[c]; c = c + 1 }
                      pbuf[dl] = 47 as u8
                      c = 0
                      while c < nl { pbuf[dl+1+c] = nmp[c]; c = c + 1 }
                      pbuf[dl+1+nl] = 0 as u8
178
                      out[JL_TOTAL] = out[JL_TOTAL] + 1
                      let n: i64 = jl_read(pbuf, fbuf, JL_FBUF)
181
                      // PENDING = "claimed, not finished", in EITHER representation.
                      // Before 2026-08-07 that was an EMPTY claim; nx_tools_api now writes
                      // `state=CLAIMED ts=<id>` inside the O_EXCL reservation, so the same condition
                      // has two spellings while pre-fix claims remain on disk. Both route to the same
                      // age logic, because the QUESTION is identical: has it been too long?
                      //   ★ WHEN A REPRESENTATION CHANGES, MAP THE OLD AND NEW ONTO ONE STATE -- do
                      //   not let the new spelling fall through to UNKNOWN, which is exactly what this
                      //   gate did for a full deploy cycle after the producer was fixed.
                      // ONE CLASSIFIER, IMPORTED FROM THE LIB THE WRITER ALSO IMPORTS. The order --
                      // REAPED first, then DONE, then CLAIMED -- is jr_state_of contract, and it
                      // lives beside the marker nx_jobclaim_reap writes. It has to: the tombstone is
                      // APPENDED, so a reaped claim STILL contains state=CLAIMED, and a reader that
                      // tested CLAIMED first would report every reaped claim as still pending. The
                      // tombstone would land, this count would not move, and the reaper would read
                      // as a fix while being a no-op. Sharing the classifier makes that ordering
                      // impossible to get right in one organ and wrong in the other.
                      var pending: i64 = 0
                      // n < 0 = the file vanished between listing and reading (a live writer).
                      // A RACE IS NOT A FAILURE -- never fold it into the alarming bucket.
                      if n < 0 { out[JL_UNKNOWN] = out[JL_UNKNOWN] + 1 }
                      else {
                          let st: i64 = jr_state_of(fbuf, n)
                          if st == JR_ST_REAPED { out[JL_REAPED] = out[JL_REAPED] + 1 }
                          else {
                              if st == JR_ST_DONE { out[JL_DONE] = out[JL_DONE] + 1 }
                              else {
                                  if st == JR_ST_UNKNOWN { out[JL_UNKNOWN] = out[JL_UNKNOWN] + 1 }
                                  else { pending = 1 }
                              }
                          }
                      }
213
                      if pending == 1 {
                          let id: i64 = jl_parse_id(nmp, nl)
                          if id < 0 { out[JL_UNKNOWN] = out[JL_UNKNOWN] + 1 }
                          else {
                              // sibling ".out": swap the 6-char ".claim" tail for ".out"
                              var e: i64 = dl + 1 + nl - 6
                              pbuf[e] = 46 as u8
                              pbuf[e+1] = 111 as u8
                              pbuf[e+2] = 117 as u8
                              pbuf[e+3] = 116 as u8
                              pbuf[e+4] = 0 as u8
                              let hasout: i64 = jl_exists(pbuf)
                              if hasout == 1 { out[JL_ORPHAN] = out[JL_ORPHAN] + 1 }
                              else {
                                  if now - id > maxage {
                                      // NAME IT, do not merely count it. Store BEFORE the increment so
                                      // the slot index is the count-so-far. Overflow past JL_LOSTCAP is
                                      // DECLARED by the printer as a prefix, never dropped in silence.
                                      let k: i64 = out[JL_LOST]
                                      if k < JL_LOSTCAP { out[JL_IDBASE + k] = id }
                                      out[JL_LOST] = out[JL_LOST] + 1
                                  }
                                  else { out[JL_INFLIGHT] = out[JL_INFLIGHT] + 1 }
                              }
                          }
                      }
                  }
              }
              if reclen <= 0 { pos = nread } else { pos = pos + reclen }
          }
      }
  }
  sys_close(dfd)
  return 0
248}
249
250// Empty a fixture directory at SETUP. ★★★★★ A GATE THAT IS NOT IDEMPOTENT REPORTS ON ITS FIRST RUN
251// AND LIES ABOUT EVERY RUN AFTER. Cleaning belongs in SETUP and not in a teardown, because a teardown
252// does not run when a run crashes -- and the residue then becomes the next run's evidence.
253// MEASURED: v2 of this gate went RED on its first run because v1's fixture files (job_1.claim,
254// job_2.claim) were still sitting in these directories, so `total` read 4 where the teeth expected 2.
255// The gate caught it only because T2/T3 assert the fixture REACHED ITS CONDITION rather than merely
256// asserting the outcome.
257func jl_clean(dir: *u8, dbuf: *u8, pbuf: *u8) -> i64 {
  let dfd: i64 = __syscall(257, 0-100, dir, 0x10000, 0, 0, 0)
  if dfd < 0 { return 0 - 1 }
  let dl: i64 = jl_slen(dir)
  var removed: i64 = 0
  var go: i64 = 1
  while go == 1 {
      let nread: i64 = __syscall(217, dfd, dbuf, JL_DBUF, 0, 0, 0)
      if nread <= 0 { go = 0 } else {
          var pos: i64 = 0
          while pos < nread {
              let reclen: i64 = (dbuf[pos+16] as i64) | ((dbuf[pos+17] as i64) << 8)
              var nl: i64 = 0
              while dbuf[pos+19+nl] != (0 as u8) { nl = nl + 1 }
              let nmp: *u8 = ((dbuf as i64) + pos + 19) as *u8
              var skip: i64 = 0
              if nl == 1 { if nmp[0] == (46 as u8) { skip = 1 } }
              if nl == 2 { if nmp[0] == (46 as u8) { if nmp[1] == (46 as u8) { skip = 1 } } }
              if skip == 0 {
                  if dl + nl + 2 < JL_PATH {
                      var c: i64 = 0
                      while c < dl { pbuf[c] = dir[c]; c = c + 1 }
                      pbuf[dl] = 47 as u8
                      c = 0
                      while c < nl { pbuf[dl+1+c] = nmp[c]; c = c + 1 }
                      pbuf[dl+1+nl] = 0 as u8
                      sys_unlinkat(pbuf)
                      removed = removed + 1
                  }
              }
              if reclen <= 0 { pos = nread } else { pos = pos + reclen }
          }
      }
  }
  sys_close(dfd)
  return removed
293}
294
295func jl_touch(path: *u8, body: *u8) -> i64 {
  let fd: i64 = sys_openat_wr(path, JL_MODE)
  if fd < 0 { return 0 - 1 }
  let n: i64 = jl_slen(body)
  if n > 0 { sys_write(fd, body, n) }
  sys_close(fd)
  return 0
302}
303
304// Build "<dir>/job_<id>.<ext>" into buf.
305func jl_mkname(buf: *u8, dir: *u8, id: i64, ext: *u8) -> i64 {
  var p: i64 = 0
  var i: i64 = 0
  while dir[i] != (0 as u8) { buf[p] = dir[i]; p = p + 1; i = i + 1 }
  buf[p] = 47 as u8; p = p + 1
  let pre: *u8 = "job_" as *u8
  i = 0
  while pre[i] != (0 as u8) { buf[p] = pre[i]; p = p + 1; i = i + 1 }
  // decimal, most significant first
  var d: i64 = 1
  var t: i64 = id
  while t >= 10 { d = d * 10; t = t / 10 }
  while d > 0 { buf[p] = ((id / d) % 10 + 48) as u8; p = p + 1; d = d / 10 }
  i = 0
  while ext[i] != (0 as u8) { buf[p] = ext[i]; p = p + 1; i = i + 1 }
  buf[p] = 0 as u8
  return p
322}
323
324// ABSENT IS NOT ZERO: an absent conf yields the declared default and the caller is TOLD which it got.
325func jl_floor(defaulted: *i64, fbuf: *u8, seen: *i64) -> i64 {
  defaulted[0] = 1
  let n: i64 = jl_read(JL_RATCHET, fbuf, JL_FBUF)
  if n <= 0 { return JL_DEFAULT_FLOOR }
  // ONE decimal parser in this file, not two. jr_num_from carries exactly the semantics this
  // function used to hold inline (the first digit run on the first line, and a `seen` flag so an
  // unparseable conf stays DISTINGUISHABLE from a genuine 0). Writing a second copy here for the
  // shared age conf would have been the duplicate-ruler defect INSIDE one file -- a class this
  // estate has already measured and paid for.
  let v: i64 = jr_num_from(fbuf, n, seen)
  if seen[0] == 0 { return JL_DEFAULT_FLOOR }
  defaulted[0] = 0
  return v
338}
339
340func main(argc: i64, argv: *i64) -> i64 {
  let ctr: *i64 = gv_ctr()
  gv_head("=== NX-JOBLOST -- which async jobs DIED, as distinct from which are still running ===" as *u8)
343
  let dbuf: *u8 = sys_mmap(JL_DBUF)
  let fbuf: *u8 = sys_mmap(JL_FBUF)
  let pbuf: *u8 = sys_mmap(JL_PATH)
  let nbuf: *u8 = sys_mmap(JL_PATH)
  // SIZED FROM THE LAYOUT, NEVER HAND-TYPED: JL_IDBASE counter slots, then JL_LOSTCAP id slots.
  let outb: i64 = (JL_IDBASE + JL_LOSTCAP) * 8
  let real: *i64 = sys_mmap(outb) as *i64
  let bad:  *i64 = sys_mmap(outb) as *i64
  let good: *i64 = sys_mmap(outb) as *i64
  let dflt: *i64 = sys_mmap(16) as *i64
354
  let now: i64 = sys_now_realtime_sec()
  // THE THRESHOLD, RESOLVED THROUGH THE SHARED RULER (nx_jobclaim_lib), so this census and the
  // reaper that acts on its findings hold ONE number rather than two literals that drift. Priority
  // is argv > conf > code default (rule 17) and the SOURCE is PRINTED: a bar nobody can see is the
  // magic-number defect wearing a conf file.
  // argv[1] is accepted only as a POSITIVE integer. The MCP surface passes a VERB there, and a
  // lenient parse would turn it into maxage=0 and report the entire plane LOST -- an alarm made
  // entirely of the reader mis-parsing its own argument.
  let mflag: *i64 = sys_mmap(JR_SCRATCH) as *i64
  let mscr:  *i64 = sys_mmap(JR_SCRATCH) as *i64
  let nseen: *i64 = sys_mmap(JR_SCRATCH) as *i64
  var maxage: i64 = jr_maxage(mflag, mscr)
  var maxage_src: *u8 = JR_MAXAGE_CONF
  if mflag[0] == 1 { maxage_src = "the code default (conf absent or unparseable -- stated, not hidden)" as *u8 }
  if argc > 1 {
      let av: i64 = jr_atoi_strict(argv[1] as *u8)
      if av > 0 { maxage = av; maxage_src = "argv[1] (top of the configuration hierarchy)" as *u8 }
  }
373
  // ---- FIXTURES built at RUNTIME in /tmp/<gate>/, never sharing the production _jobs plane.
  // Their ids are computed FROM `now`, so the age axis under test is real rather than hard-coded:
  // a fixture with a frozen id would silently drift into "old" and the young case would stop existing.
  sys_mkdir("/tmp/nx_joblost_gate" as *u8, 493)
  sys_mkdir("/tmp/nx_joblost_gate/bad" as *u8, 493)
  sys_mkdir("/tmp/nx_joblost_gate/good" as *u8, 493)
  let wiped_bad:  i64 = jl_clean("/tmp/nx_joblost_gate/bad" as *u8,  dbuf, pbuf)
  let wiped_good: i64 = jl_clean("/tmp/nx_joblost_gate/good" as *u8, dbuf, pbuf)
382
  let old_id:   i64 = now - (maxage * 2)
  let young_id: i64 = now - 5
385
  jl_mkname(nbuf, "/tmp/nx_joblost_gate/bad" as *u8, old_id, ".claim" as *u8)
  jl_touch(nbuf, "" as *u8)                                     // OLD + empty + no .out  -> LOST
  jl_mkname(nbuf, "/tmp/nx_joblost_gate/bad" as *u8, now - 7, ".claim" as *u8)
  jl_touch(nbuf, "state=DONE rc=0 exit=0 bytes=12\n" as *u8)     // healthy alongside it
390
  jl_mkname(nbuf, "/tmp/nx_joblost_gate/good" as *u8, young_id, ".claim" as *u8)
  jl_touch(nbuf, "" as *u8)                                     // YOUNG + empty          -> INFLIGHT
  jl_mkname(nbuf, "/tmp/nx_joblost_gate/good" as *u8, now - 9, ".claim" as *u8)
  jl_touch(nbuf, "state=DONE rc=0 exit=0 bytes=12\n" as *u8)
395
  // The POST-FIX representation, both ages, so the new spelling is proven to reach the same two
  // answers as the old one. Without these the gate would silently sort every real in-flight job
  // into UNKNOWN the moment the producer started writing a marker.
  sys_mkdir("/tmp/nx_joblost_gate/claimed" as *u8, 493)
  let wiped_cl: i64 = jl_clean("/tmp/nx_joblost_gate/claimed" as *u8, dbuf, pbuf)
  jl_mkname(nbuf, "/tmp/nx_joblost_gate/claimed" as *u8, old_id, ".claim" as *u8)
  jl_touch(nbuf, "state=CLAIMED ts=1\n" as *u8)                 // OLD + CLAIMED   -> LOST
  jl_mkname(nbuf, "/tmp/nx_joblost_gate/claimed" as *u8, young_id, ".claim" as *u8)
  jl_touch(nbuf, "state=CLAIMED ts=2\n" as *u8)                 // YOUNG + CLAIMED -> INFLIGHT
405
  // THE TOMBSTONE FIXTURE, and it is a DISCRIMINATION test rather than a presence test. Both files
  // are OLD and both carry state=CLAIMED; only one of them also carries the tombstone -- and that
  // tombstone is written by jr_tombstone, THE REAL WRITER, not by a hand-typed copy of what it
  // emits, so this drives the actual producer/consumer pair end to end in one process.
  //   If jr_state_of ever tested CLAIMED before REAPED, BOTH files would read LOST and the reaper
  //   would be a silent no-op. If the REAPED test were too greedy, BOTH would read REAPED and the
  //   second file would stop being reported as LOST. Only the correct order gives 1 and 1, so this
  //   pair cannot be passed by either wrong answer.
  sys_mkdir("/tmp/nx_joblost_gate/reaped" as *u8, MODE_0755)
  let wiped_rp: i64 = jl_clean("/tmp/nx_joblost_gate/reaped" as *u8, dbuf, pbuf)
  let tline: *u8 = sys_mmap(JR_LINE)
  jl_mkname(nbuf, "/tmp/nx_joblost_gate/reaped" as *u8, old_id, ".claim" as *u8)
  jl_touch(nbuf, "state=CLAIMED ts=1\n" as *u8)
  let tomb_bytes: i64 = jr_tombstone(nbuf, now, now - old_id, 0, tline, JR_LINE)
  jl_mkname(nbuf, "/tmp/nx_joblost_gate/reaped" as *u8, old_id - 1, ".claim" as *u8)
  jl_touch(nbuf, "state=CLAIMED ts=2\n" as *u8)
422
  let clm:  *i64 = sys_mmap(outb) as *i64
  let rp:   *i64 = sys_mmap(outb) as *i64
  let rc_bad:  i64 = jl_scan("/tmp/nx_joblost_gate/bad" as *u8,  now, maxage, bad,  dbuf, fbuf, pbuf)
  let rc_good: i64 = jl_scan("/tmp/nx_joblost_gate/good" as *u8, now, maxage, good, dbuf, fbuf, pbuf)
  let rc_clm:  i64 = jl_scan("/tmp/nx_joblost_gate/claimed" as *u8, now, maxage, clm, dbuf, fbuf, pbuf)
  let rc_rp:   i64 = jl_scan("/tmp/nx_joblost_gate/reaped" as *u8, now, maxage, rp, dbuf, fbuf, pbuf)
  let rc_real: i64 = jl_scan(JL_JOBS, now, maxage, real, dbuf, fbuf, pbuf)
  let floor: i64 = jl_floor(dflt, fbuf, nseen)
431
  gv_puts("  age threshold=" as *u8); gv_num(maxage)
  gv_puts("s from " as *u8); gv_puts(maxage_src)
  gv_puts("   now=" as *u8); gv_num(now)
  gv_puts("   fixture residue wiped at setup: bad=" as *u8); gv_num(wiped_bad)
  gv_puts(" good=" as *u8); gv_num(wiped_good); gv_puts("\n" as *u8)
  gv_puts("  fixture bad : total=" as *u8); gv_num(bad[JL_TOTAL])
  gv_puts(" lost=" as *u8);     gv_num(bad[JL_LOST])
  gv_puts(" inflight=" as *u8); gv_num(bad[JL_INFLIGHT]);  gv_puts("\n" as *u8)
  gv_puts("  fixture good: total=" as *u8); gv_num(good[JL_TOTAL])
  gv_puts(" lost=" as *u8);     gv_num(good[JL_LOST])
  gv_puts(" inflight=" as *u8); gv_num(good[JL_INFLIGHT]); gv_puts("\n" as *u8)
  gv_puts("  LIVE _jobs  : total=" as *u8); gv_num(real[JL_TOTAL])
  gv_puts(" done=" as *u8);     gv_num(real[JL_DONE])
  gv_puts(" INFLIGHT=" as *u8); gv_num(real[JL_INFLIGHT])
  gv_puts(" LOST=" as *u8);     gv_num(real[JL_LOST])
  gv_puts(" ORPHAN=" as *u8);   gv_num(real[JL_ORPHAN])
  gv_puts(" REAPED=" as *u8);   gv_num(real[JL_REAPED])
  gv_puts(" unknown=" as *u8);  gv_num(real[JL_UNKNOWN]); gv_puts("\n" as *u8)
  gv_puts("  ratchet floor=" as *u8); gv_num(floor)
  if dflt[0] == 1 { gv_puts("  (DEFAULTED -- conf absent or unparseable, stated not hidden)" as *u8) }
  gv_puts("\n\n" as *u8)
453
  // ---- THE LOST WORKLIST. A count with no worklist is not actionable, and the reason is already in
  // hand at measure time: each row is a claim file that can be read, reaped or re-run without
  // re-deriving the scan. The list DECLARES itself a prefix when the cap binds.
  var shown: i64 = real[JL_LOST]
  if shown > JL_LOSTCAP { shown = JL_LOSTCAP }
  if real[JL_LOST] > 0 {
      gv_puts("  LOST WORKLIST -- each row is _jobs/job_<id>.claim with no .out and no state=DONE:\n" as *u8)
      var q: i64 = 0
      while q < shown {
          gv_puts("    LOST job_" as *u8); gv_num(real[JL_IDBASE + q])
          gv_puts("  age=" as *u8); gv_num(now - real[JL_IDBASE + q])
          gv_puts("s  over_threshold_by=" as *u8); gv_num(now - real[JL_IDBASE + q] - maxage)
          gv_puts("s\n" as *u8)
          q = q + 1
      }
      gv_puts("    listed=" as *u8); gv_num(shown); gv_puts(" of " as *u8); gv_num(real[JL_LOST])
      if shown < real[JL_LOST] { gv_puts("  <== THIS LIST IS A PREFIX OF ITS OWN COUNT" as *u8) }
      gv_puts("\n\n" as *u8)
  }
473
  var can_real: i64 = 0
  if rc_real == 0 { can_real = 1 }
  let looked: i64 = gv_need("the live _jobs directory is readable" as *u8, can_real, ctr)
477
  // ---- T1 THE BITE, and it is the AGE AXIS that is on trial. Both fixtures hold an empty claim with
  // no .out; ONLY the age differs. A detector that ignored age would fire on both and fail here --
  // which is exactly what v1 of this gate did.
  var fired_bad:  i64 = 0
  var fired_good: i64 = 0
  if bad[JL_LOST]  > 0 { fired_bad = 1 }
  if good[JL_LOST] > 0 { fired_good = 1 }
  gv_bite("T1 an OLD empty claim is LOST while a YOUNG one is not -- the two differ ONLY in age" as *u8,
          fired_bad, fired_good, ctr)
487
  // ---- T2 the young empty claim must land in INFLIGHT, not vanish. A bucket that quietly drops a row
  // makes the partition lie and the alarm look clean.
  var t2: i64 = 0
  if rc_good == 0 { if good[JL_TOTAL] == 2 { if good[JL_INFLIGHT] == 1 { if good[JL_DONE] == 1 { t2 = 1 } } } }
  gv_check("T2 the young empty claim is classified INFLIGHT (1 inflight + 1 done, nothing dropped)" as *u8, t2, ctr)
493
  // ---- T3 the bad fixture reached its condition: exactly 1 lost + 1 done, so T1's fire is not the
  // artefact of a half-read directory. ★ ASSERT THE FIXTURE REACHED THE CONDITION.
  var t3: i64 = 0
  if rc_bad == 0 { if bad[JL_TOTAL] == 2 { if bad[JL_LOST] == 1 { if bad[JL_DONE] == 1 { t3 = 1 } } } }
  gv_check("T3 the bad fixture partitions exactly 1 lost + 1 done (the fire is not a half-read)" as *u8, t3, ctr)
499
  // ---- T4 PARTITION SUMS over the live plane. A partition is a claim: check the parts add up.
  var t4: i64 = 0
  // REAPED joins the sum. A new bucket that is counted but left out of the partition is a leak
  // that reads as a healthy census, and the whole point of adding a state is that every claim lands
  // in exactly one of them.
  let sum: i64 = real[JL_DONE] + real[JL_INFLIGHT] + real[JL_LOST] + real[JL_ORPHAN] + real[JL_REAPED] + real[JL_UNKNOWN]
  if looked == 1 { if sum == real[JL_TOTAL] { t4 = 1 } }
  gv_puts("  partition: " as *u8); gv_num(sum); gv_puts(" of " as *u8); gv_num(real[JL_TOTAL]); gv_puts("\n" as *u8)
  gv_check("T4 done+inflight+lost+orphan+reaped+unknown == total (no row leaks out of the census)" as *u8, t4, ctr)
509
  var t5: i64 = 0
  if looked == 1 { if real[JL_TOTAL] > 0 { t5 = 1 } }
  gv_check("T5 the live _jobs census is non-vacuous (total > 0)" as *u8, t5, ctr)
513
  // ---- T6 NEG-CONTROL: a DONE claim is never counted lost no matter how old it is. Without this a
  // detector keyed on age alone would condemn the entire healthy history of the plane.
  var t6: i64 = 0
  if bad[JL_DONE] == 1 { if bad[JL_LOST] == 1 { t6 = 1 } }
  gv_check("T6 neg-control-an-old-DONE-claim-is-never-LOST (age alone must not condemn history)" as *u8, t6, ctr)
519
  // ---- T6b THE NEW REPRESENTATION reaches the SAME two answers as the old one. Both fixtures carry
  // `state=CLAIMED` and differ only in age; if the marker fell through to UNKNOWN this reads 0/0.
  gv_puts("  fixture claimed: total=" as *u8); gv_num(clm[JL_TOTAL])
  gv_puts(" lost=" as *u8);     gv_num(clm[JL_LOST])
  gv_puts(" inflight=" as *u8); gv_num(clm[JL_INFLIGHT])
  gv_puts(" unknown=" as *u8);  gv_num(clm[JL_UNKNOWN]); gv_puts("\n" as *u8)
  var t6b: i64 = 0
  if rc_clm == 0 { if clm[JL_TOTAL] == 2 { if clm[JL_LOST] == 1 { if clm[JL_INFLIGHT] == 1 { if clm[JL_UNKNOWN] == 0 { t6b = 1 } } } } }
  gv_check("T6b state=CLAIMED maps to the SAME states as an empty claim (1 lost + 1 inflight, 0 unknown)" as *u8, t6b, ctr)
529
  // ---- T6c THE TOMBSTONE, END TO END AND IN ONE PROCESS. jr_tombstone WROTE the marker and
  // jr_state_of READ it back, so this tooth exercises the real producer and the real consumer of
  // the wire rather than a description of them. It is a DISCRIMINATION: two OLD claims, both
  // carrying state=CLAIMED, and only the tombstoned one may move out of LOST.
  gv_puts("  fixture reaped: total=" as *u8); gv_num(rp[JL_TOTAL])
  gv_puts(" reaped=" as *u8);   gv_num(rp[JL_REAPED])
  gv_puts(" lost=" as *u8);     gv_num(rp[JL_LOST])
  gv_puts(" inflight=" as *u8); gv_num(rp[JL_INFLIGHT])
  gv_puts(" unknown=" as *u8);  gv_num(rp[JL_UNKNOWN])
  gv_puts("  tombstone_bytes=" as *u8); gv_num(tomb_bytes)
  gv_puts("  residue wiped at setup=" as *u8); gv_num(wiped_rp); gv_puts("\n" as *u8)
  var t6c: i64 = 0
  if rc_rp == 0 { if tomb_bytes > 0 { if rp[JL_TOTAL] == 2 { if rp[JL_REAPED] == 1 { if rp[JL_LOST] == 1 { if rp[JL_INFLIGHT] == 0 { if rp[JL_UNKNOWN] == 0 { t6c = 1 } } } } } } }
  gv_check("T6c a tombstoned OLD claim reads REAPED while its un-tombstoned OLD twin still reads LOST (the REAPED-first order, through the real writer)" as *u8, t6c, ctr)
544
  // ---- T6d NEG-CONTROL: THE TOMBSTONE IS ADDITIVE. Read the reaped fixture back and assert the
  // ORIGINAL marker survived. An overwriting reaper would pass T6c perfectly and quietly destroy
  // the record rule 13 exists to keep -- and no count-based tooth can tell the two apart, because
  // both produce identical bucket totals.
  jl_mkname(nbuf, "/tmp/nx_joblost_gate/reaped" as *u8, old_id, ".claim" as *u8)
  let rpn: i64 = jl_read(nbuf, fbuf, JL_FBUF)
  var kept_claimed: i64 = 0
  var kept_reaped:  i64 = 0
  if rpn > 0 {
      kept_claimed = jr_has(fbuf, rpn, JR_MARK_CLAIMED)
      kept_reaped  = jr_has(fbuf, rpn, JR_MARK_REAPED)
  }
  var t6d: i64 = 0
  if kept_claimed == 1 { if kept_reaped == 1 { t6d = 1 } }
  gv_puts("  reaped fixture bytes=" as *u8); gv_num(rpn)
  gv_puts(" still_holds_CLAIMED=" as *u8); gv_num(kept_claimed)
  gv_puts(" holds_REAPED=" as *u8); gv_num(kept_reaped); gv_puts("\n" as *u8)
  gv_check("T6d neg-control-the-tombstone-is-ADDITIVE: the reaped claim still holds state=CLAIMED beside state=REAPED" as *u8, t6d, ctr)
563
  var t7: i64 = 0
  if looked == 1 { if real[JL_LOST] <= floor { t7 = 1 } }
  gv_check("T7 lost jobs are at or below the ratchet floor" as *u8, t7, ctr)
567
  // ---- T8 THE WORKLIST NAMES WHAT IT COUNTS, and it is bound to the BAD FIXTURE where exactly one
  // claim was planted at a KNOWN id. A count-only implementation fails this by construction, and it
  // cannot pass on the empty set -- the trap every "no findings" tooth falls into.
  var t8: i64 = 0
  if bad[JL_LOST] == 1 { if bad[JL_IDBASE] == old_id { t8 = 1 } }
  gv_check("T8 the LOST worklist NAMES the id it counted (fixture: the planted old_id is reported back)" as *u8, t8, ctr)
574
  // ---- T9 the LIVE list accounts for every counted row: either it lists them all, or it is at the
  // cap AND said so. A silently truncated worklist is a smaller number that reads like better news.
  var t9: i64 = 0
  if looked == 1 {
      if shown == real[JL_LOST] { t9 = 1 }
      else { if shown == JL_LOSTCAP { t9 = 1 } }
  }
  gv_check("T9 the live LOST list accounts for every counted row (all listed, or capped and declared a prefix)" as *u8, t9, ctr)
583
  // ---- T10 THE LAYOUT INVARIANT, asserted rather than assumed. The counter buckets and the LOST id
  // list share ONE allocation: counters occupy 0..JL_SLOTS-1 and the ids start at JL_IDBASE. Adding
  // REAPED took the counters from 6 to 7 against a base of 8. A future eighth state would silently
  // overwrite the FIRST named LOST id, and the symptom would surface in T8 as a wrong id -- a place
  // nobody would think to look for a layout bug.
  var t10: i64 = 0
  if JL_SLOTS <= JL_IDBASE { t10 = 1 }
  gv_puts("  layout: counter buckets=" as *u8); gv_num(JL_SLOTS)
  gv_puts("  id list starts at " as *u8); gv_num(JL_IDBASE)
  gv_puts("  headroom=" as *u8); gv_num(JL_IDBASE - JL_SLOTS); gv_puts("\n" as *u8)
  gv_check("T10 the counter buckets fit below JL_IDBASE (a new state must never overwrite the first LOST id)" as *u8, t10, ctr)
  // ---- WHEN IT IS HONEST TO ADVISE TIGHTENING.
  // LOST is not a settled quantity while INFLIGHT is non-zero: every in-flight claim can become LOST
  // by the passage of time alone, with no new failure. Advising a tighter floor in that window would
  // recommend a bound the very next run breaks by itself.
  //   ★★★★★ A RATCHET ADVISED FROM A METRIC THAT CAN STILL RISE WITHOUT A NEW EVENT WILL ALWAYS
  //   ADVISE AT THE WRONG MOMENT -- WAIT UNTIL THE POPULATION HAS SETTLED.
  // MEASURED 2026-08-07: the run that first shipped this printed "lower the floor to 0" while a known
  // dead claim sat 1,300s old under a 3,600s threshold, i.e. it was counselling a floor that would
  // have gone RED forty minutes later on evidence already present.
  if real[JL_LOST] < floor {
      if real[JL_INFLIGHT] == 0 {
          gv_puts("  RATCHET EARNED: lower the floor in " as *u8); gv_puts(JL_RATCHET)
          gv_puts(" to " as *u8); gv_num(real[JL_LOST])
          gv_puts(" -- a ratchet that does not tighten when you improve is just a threshold.\n" as *u8)
      } else {
          gv_puts("  ratchet NOT advised yet: lost=" as *u8); gv_num(real[JL_LOST])
          gv_puts(" is below floor=" as *u8); gv_num(floor)
          gv_puts(" but " as *u8); gv_num(real[JL_INFLIGHT])
          gv_puts(" claim(s) are still in flight and can become LOST by age alone.\n" as *u8)
      }
  }
  if real[JL_ORPHAN] > 0 {
      gv_puts("  NOTE: ORPHAN>0 -- a worker wrote its .out and never renamed the claim, so anything\n" as *u8)
      gv_puts("  polling for state=DONE on those ids waits forever. Result present, signal missing.\n" as *u8)
  }
620
  return gv_verdict("NX-JOBLOST" as *u8, ctr,
      "an empty claim is RUNNING until it is old; LOST, INFLIGHT and ORPHAN are separate answers with separate remedies" as *u8)
623}
nx_joblost_gate.nx source

`nx_joblost_gate.nx` source
