code wiki / _hdl_build / nx_toolatency.nx
`nx_toolatency.nx` source

↩ module page · 559 lines · 29225 B
1// nx_toolatency.nx -- LATENCY CENSUS FOR THE TOOL SURFACE: the evidence behind async-lane routing.
2//
3// WHY THIS EXISTS (2026-08-07). The 503 class is a LATENCY problem. The edge closes its window at 15s
4// (NX_SD2_BACKEND_TIMEOUT_S) and the synchronous lane is budgeted at 14s (TEA_EXEC_TIMEOUT_SYNC_MS),
5// so a call that outruns the budget becomes a 503 WHOSE WORK MAY STILL HAVE LANDED -- the caller is
6// told it failed for a row that exists. async_only_tools.conf fixes that by forcing a tool onto the
7// detached job lane, but its membership was a HAND-WRITTEN LIST of tools somebody noticed failing.
8// You cannot route by a latency you never measured, and until today the daemon logged WHAT ran and
9// WHETHER it worked but never HOW LONG. ta_actlog now emits dur_ms (TOTAL HANDLER TIME, the quantity
10// the edge window actually applies -- exec time alone would understate it). This organ is the
11// consumer that turns those rows into a routing decision, so the lane stops being folklore.
12//
13// TAIL-ANCHORED BY CONSTRUCTION. actlog.jrnl is APPEND-ONLY, so a head-anchored capped read keeps the
14// OLDEST bytes and silently drops the NEWEST -- exactly backwards for a gauge whose job is to notice
15// what got slow LATELY. Measured cost of that mistake in this same estate: nx_wirecensus read this
16// very ledger head-anchored and was blind to 89 invocation records.
17//
18// REFUSES rather than reporting GREEN over data it never read: no-slow-tools and no-measurements are
19// different facts and must not share a representation.
20//
21// DURABLE + ROSTERABLE (2026-08-27, 503 lane): every run appends ONE row to
22// knowledge/status/toolatency.log (its own file, one writer -- the rm_trend_log law), so two runs are
23// a TREND and a fired beat is distinguishable from a dead one. The row carries `unrouted=` -- the
24// over-budget tools still on the sync lane -- so the count travels with its worklist. Refusals write
25// a NODATA row too: an abstention that leaves no artifact is indistinguishable from a beat that never
26// fired.
27import "nx_syscalls.nx"
28import "nx_toolatency_observation_lib.nx"
29
30const TL_TAIL_CAP:       i64 = 4194304
31const TL_MAXTOOLS:       i64 = 512
32const TL_NAMELEN:        i64 = 48
33const TL_SYNC_BUDGET_MS: i64 = 14000
34const TL_WARN_PERMIL:    i64 = 500
35const TL_CONFCAP:        i64 = 65536
36const TL_SEEK_SET:       i64 = 0
37const TL_SEEK_END:       i64 = 2
38const TL_EXIT_SHOULD:    i64 = 4
39const TL_EXIT_NODATA:    i64 = 5
40const TL_EXIT_OVER:      i64 = 6
41// A single call past the window is an OUTLIER; a tool that breaches repeatedly is BROKEN for that
42// lane. Rerouting changes the caller contract, so it must answer to a recurring breach and never to
43// one slow sample. Measured case that forced this: nx_mgmt showed max_ms=14002 -- TWO MILLISECONDS
44// past the budget -- across 69 calls averaging 1663 ms, i.e. one outlier, while nishi_compare_regen
45// breached on BOTH of its 2 calls at an average of 14854 ms. Max alone cannot tell those apart.
46const TL_MIN_BREACH:     i64 = 2
47// The beat's durable artifact (ONE WRITER, ONE LOG, ONE ROW) + the bounded worklist buffer for the
48// unrouted= field. 4096 covers ~150 tool names against a measured worst of 21 offenders; a fuller
49// buffer ANNOUNCES with a +MORE sentinel rather than truncating silently.
50const TL_JRNL: *u8 = "knowledge/status/toolatency.log" as *u8
51// THE RANK TABLE (2026-09-15). The tools API orders tools/list by the presented cap, then by THESE counts
52// (most-called first), then newest-registered first -- see nx_tools_api.nx ta_emit_tools, and the measured
53// reason: a seat's MCP client walked 20 pages of 64 and stopped, so the ~700 newest of 1,977 registered
54// tools were invisible to every seat. The path is TA_RANK_DIR + the plane's basename + TA_RANK_EXT for the
55// production plane knowledge/toolreg-, spelled here ONCE so the writer and the reader name the same file;
56// nx_tools_api_gate proves the reader on a fixture plane and never touches this file.
57const TL_RANK_PATH: *u8 = "knowledge/status/toolrank-toolreg-.tsv" as *u8
58const TL_RANK_TMP:  *u8 = "knowledge/status/toolrank-toolreg-.tsv.tmp" as *u8
59const TL_RANK_ROW_EXTRA: i64 = 48   // per row beyond the name: two decimals (<=20 digits each), two tabs, a newline, slack
60const TL_CSV_CAP: i64 = 4096
61// ACTION THRESHOLDS -- the pre-declared accept rule separating ROUTE (change the caller contract)
62// from TOLERATE (watch; the live sync-promote lane covers the tail). Without a rate/avg dimension
63// the census goes PERMANENTLY RED on high-volume fast tools (measured 2026-08-27: nx_fs, 19 breaches
64// in 12,141 calls = 1.6 permil -- rerouting it would tax twelve thousand fast reads with a poll to
65// save nineteen slow ones), and a permanently red detector is one everyone learns to ignore.
66// CONF IS THE SSOT (knowledge/toolatency.conf, rows act_permil= / act_avg_ms=, reasoning lives
67// there); these are the rule-17 bootstrap defaults, live only while the conf is absent.
68const TL_CONF: *u8 = "knowledge/toolatency.conf" as *u8
69const TL_ACT_PERMIL_DFLT: i64 = 40
70const TL_ACT_AVG_MS_DFLT: i64 = 2000
71// AUTO-ROUTE (2026-09-02). The accept rule below was applied BY HAND three times (08-27, 08-28, 08-29):
72// each time a seat copied this census into async_only_tools.conf while the OVER-BUDGET tools kept 503ing
73// for the day in between (measured 2026-09-02: nx_swarm_gpu, nx_frontier_put and nx_frontier_board sat
74// RED and unrouted across 20 consecutive hourly rows). A RULE APPLIED IDENTICALLY THREE TIMES BY A HUMAN
75// IS A RULE THAT BELONGS IN THE PATH. Conf-gated: knowledge/toolatency.conf auto_route=1 arms it; the
76// compiled default is OFF so the beat stays byte-identical until the operator turns the row on.
77const TL_AUTO_ROUTE_DFLT: i64 = 0
78const TL_ASYNC_CONF: *u8 = "async_only_tools.conf" as *u8
79const TL_CONF_MODE: i64 = 420   // 0644, the mode every other append-only conf/journal in the estate uses
80
81func tw(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
82func twe(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(2, s, n); return 0 }
83func twn(v: i64) -> i64 {
  var m: i64 = v
  if m < 0 { tw("-" as *u8); m = 0 - m }
  let t: *u8 = sys_mmap(28)
  var k: i64 = 0
  if m == 0 { t[0] = 48 as u8; k = 1 }
  while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
  let o: *u8 = sys_mmap(28)
  var i: i64 = 0
  while i < k { o[i] = t[k - 1 - i]; i = i + 1 }
  sys_write(1, o, k)
  sys_munmap(t, 28)
  sys_munmap(o, 28)
  return 0
97}
98
99// small bounded buffer cats, local ON PURPOSE: importing a string lib into a leaf census organ would
100// drag a module in for two 15-line helpers. Same shapes as ta_cat/ta_catn.
101func tl_cat(b: *u8, o: i64, s: *u8) -> i64 {
  var i: i64 = 0
  var p: i64 = o
  while s[i] != (0 as u8) { b[p] = s[i]; p = p + 1; i = i + 1 }
  return p
106}
107func tl_catn(b: *u8, o: i64, v: i64) -> i64 {
  var m: i64 = v
  var p: i64 = o
  if m < 0 { b[p] = 45 as u8; p = p + 1; m = 0 - m }
  let t: *u8 = sys_mmap(28)
  var k: i64 = 0
  if m == 0 { t[0] = 48 as u8; k = 1 }
  while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
  var i: i64 = 0
  while i < k { b[p + i] = t[k - 1 - i]; i = i + 1 }
  sys_munmap(t, 28)
  return p + k
119}
120// ONE WRITER, ONE LOG, ONE ROW (the rm_trend_log law, nx_resmon): the durable per-run artifact, so a
121// census whose verdict was never persisted stops being a private opinion and two rows become a trend.
122// CLOSED VOCABULARY (GREEN|AMBER|RED|NODATA) so no rollup ever meets a new dialect. unrouted= names
123// the over-budget-and-still-sync tools -- the row carries its own worklist, because a count without a
124// worklist is not actionable. An unwritable journal ANNOUNCES on stderr rather than vanishing.
125// auto_routed= (2026-09-02) rides BEFORE verdict= so verdict stays the LAST token (gv_last_line anchors by
126// position) and every key= scanner keeps working; it is what lets a reader of this journal tell "still
127// unrouted" from "routed on this very row", which the unrouted= list alone cannot.
128func tl_jrnl(rows: i64, measured: i64, ntools: i64, over: i64, atrisk: i64, routed: i64, unrouted: *u8, verdict: *u8) -> i64 {
  let fd: i64 = sys_openat_append(TL_JRNL, MODE_0644)
  if fd < 0 {
      twe("NX-TOOLATENCY jrnl-unwritable: " as *u8)
      twe(TL_JRNL)
      twe(" -- row lost, said out loud\n" as *u8)
      return 0 - 1
  }
  let b: *u8 = sys_mmap(TL_CSV_CAP + 256)
  var o: i64 = tl_cat(b, 0, "ts=" as *u8)
  o = tl_catn(b, o, sys_now_realtime_sec())
  o = tl_cat(b, o, " rows=" as *u8); o = tl_catn(b, o, rows)
  o = tl_cat(b, o, " measured=" as *u8); o = tl_catn(b, o, measured)
  o = tl_cat(b, o, " tools=" as *u8); o = tl_catn(b, o, ntools)
  o = tl_cat(b, o, " over_budget=" as *u8); o = tl_catn(b, o, over)
  o = tl_cat(b, o, " at_risk=" as *u8); o = tl_catn(b, o, atrisk)
  o = tl_cat(b, o, " unrouted=" as *u8)
  o = tl_cat(b, o, unrouted)
  o = tl_cat(b, o, " auto_routed=" as *u8); o = tl_catn(b, o, routed)
  o = tl_cat(b, o, " verdict=" as *u8)
  o = tl_cat(b, o, verdict)
  b[o] = 10 as u8
  sys_write(fd, b, o + 1)
  sys_close(fd)
  return 0
153}
154
155// TAIL-ANCHORED read: when the ledger exceeds the window, seek so the window ENDS at EOF.
156func tl_read_tail(path: *u8, buf: *u8, cap: i64) -> i64 {
  let fd: i64 = sys_openat_rd(path)
  if fd < 0 { return 0 - 1 }
  let sz: i64 = sys_lseek(fd, 0, TL_SEEK_END)
  if sz > cap { sys_lseek(fd, sz - cap, TL_SEEK_SET) } else { sys_lseek(fd, 0, TL_SEEK_SET) }
  let n: i64 = sys_read(fd, buf, cap)
  sys_close(fd)
  return n
164}
165
166func tl_eol(buf: *u8, n: i64, i: i64) -> i64 {
  var e: i64 = i
  var go: i64 = 1
  while go == 1 {
      if e >= n { go = 0 } else { if buf[e] == (10 as u8) { go = 0 } else { e = e + 1 } }
  }
  return e
173}
174
175// value following key= within [s,e). -1 = key absent, which means UNMEASURED and never 0.
176func tl_kv(buf: *u8, s: i64, e: i64, key: *u8) -> i64 {
  var kl: i64 = 0
  while key[kl] != (0 as u8) { kl = kl + 1 }
  var i: i64 = s
  while i + kl <= e {
      var k: i64 = 0
      var ok: i64 = 1
      while k < kl { if buf[i + k] != key[k] { ok = 0; k = kl } else { k = k + 1 } }
      if ok == 1 {
          var j: i64 = i + kl
          var v: i64 = 0
          var any: i64 = 0
          var go: i64 = 1
          while go == 1 {
              go = 0
              if j < e {
                  let c: i64 = buf[j] as i64
                  if c >= 48 { if c <= 57 { v = v * 10 + (c - 48); any = 1; j = j + 1; go = 1 } }
              }
          }
          if any == 0 { return 0 - 1 }
          return v
      }
      i = i + 1
  }
  return 0 - 1
202}
203
204// exact-line membership in async_only_tools.conf. CR-tolerant. MIRRORS THE ENFORCER: the daemon's
205// ta_name_in_lines (nx_tools_api.nx, seq1293) matches the WHOLE line against the tool name, so a
206// per-verb "NAME VERB" row can NEVER match there and is inert. This reporter used to also accept the
207// NAME-space prefix -- two readers of one conf with two grammars -- and so displayed lane=async for
208// tools the daemon demonstrably ran sync (measured 2026-08-27: the inert "nx_debt add" rows).
209// The reporter must speak the enforcer's grammar or it lies about lanes.
210func tl_on_lane(conf: *u8, n: i64, nm: *u8, nl: i64) -> i64 {
  var i: i64 = 0
  while i < n {
      let e: i64 = tl_eol(conf, n, i)
      var len: i64 = e - i
      if len > 0 { if conf[i + len - 1] == (13 as u8) { len = len - 1 } }
      if len >= nl {
          var k: i64 = 0
          var same: i64 = 1
          while k < nl { if conf[i + k] != nm[k] { same = 0; k = nl } else { k = k + 1 } }
          if same == 1 {
              if len == nl { return 1 }
          }
      }
      i = e + 1
  }
  return 0
227}
228
229// ---- AUTO-ROUTE writers (2026-09-02; see TL_AUTO_ROUTE_DFLT) --------------------------------------------
230// One append-row per routed tool: a comment carrying the measurement that admitted it (the same shape the
231// hand-adopted 08-28/08-29 rows carry, so the conf stays adjudicable without re-running the census), then
232// the bare name on its OWN line -- the enforcer's grammar is exact-whole-line (ta_name_in_lines), so the
233// name must stand alone. Returns the new offset; returns `o` unchanged (nothing appended) when the buffer
234// cannot take a whole row, so a half-written row can never be emitted.
235func tl_auto_route_row(b: *u8, o: i64, nm: *u8, nl: i64, breaches: i64, calls: i64, avg: i64) -> i64 {
  var p: i64 = o
  if p + nl + nl + 160 >= TL_CONFCAP { return o }
  p = tl_cat(b, p, "# AUTO-ROUTED " as *u8)
  p = tl_catn(b, p, sys_now_realtime_sec())
  p = tl_cat(b, p, " by nx_toolatency under the standing accept rule (breaches>=2 AND (permil>=act_permil OR avg>=act_avg_ms)): " as *u8)
  var k: i64 = 0
  while k < nl { b[p] = nm[k]; p = p + 1; k = k + 1 }
  p = tl_cat(b, p, "(b=" as *u8)
  p = tl_catn(b, p, breaches)
  b[p] = 47 as u8
  p = p + 1
  p = tl_catn(b, p, calls)
  p = tl_cat(b, p, ",avg=" as *u8)
  p = tl_catn(b, p, avg)
  p = tl_cat(b, p, ")\n" as *u8)
  k = 0
  while k < nl { b[p] = nm[k]; p = p + 1; k = k + 1 }
  b[p] = 10 as u8
  p = p + 1
  return p
256}
257// ONE open, ONE write, ONE close, after the census: the conf is hot-read per call by the daemon, so the
258// very next call of each routed tool is already on the job lane. An unwritable conf is ANNOUNCED with its
259// rc -- a silent no-op here would read on the next census as "still unrouted" with no cause.
260func tl_auto_route_apply(b: *u8, n: i64, routed: i64) -> i64 {
  let fd: i64 = sys_openat_append(TL_ASYNC_CONF, TL_CONF_MODE)
  if fd < 0 {
      tw("NX-TOOLATENCY auto_route=1 but " as *u8); tw(TL_ASYNC_CONF); tw(" is UNWRITABLE rc=" as *u8); twn(fd); tw(" -- NOTHING routed; the tools above stay on the sync lane\n" as *u8)
      return 0 - 1
  }
  let w: i64 = sys_write(fd, b, n)
  sys_close(fd)
  tw("NX-TOOLATENCY AUTO-ROUTED " as *u8); twn(routed); tw(" tool(s) -> " as *u8); tw(TL_ASYNC_CONF); tw(" (+" as *u8); twn(w); tw(" B, one append; the daemon hot-reads it, so the NEXT call of each is already on the job lane)\n" as *u8)
  return w
270}
271
272// Write the rank table ATOMICALLY (temp + rename): every tool this census saw, most-called first, ties by
273// first-seen order. The tools API reads it once per tools/list request, and a torn file would reorder a
274// seat's surface mid-page, so the rename is the publish. Returns rows written; -1 when the write was refused
275// (the previous table then stands untouched -- a stale rank beats a torn one).
276func tl_write_rank(names: *u8, cnt: *i64, ntools: i64, asof: i64) -> i64 {
  let rcap: i64 = TL_MAXTOOLS * (TL_NAMELEN + TL_RANK_ROW_EXTRA)
  let rb: *u8 = sys_mmap(rcap + 16)
  let done: *i64 = sys_mmap(8 * TL_MAXTOOLS + 16) as *i64
  var o: i64 = tl_cat(rb, 0, "# name<TAB>calls<TAB>asof -- written by nx_toolatency from actlog.jrnl; read by nx_tools_api ta_emit_tools (tier 2 of the list order)\n" as *u8)
  var rows: i64 = 0
  var more: i64 = 1
  while more == 1 {
      var best: i64 = 0 - 1
      var q: i64 = 0
      while q < ntools { if done[q] == 0 { if best < 0 { best = q } else { if cnt[q] > cnt[best] { best = q } } } q = q + 1 }
      if best < 0 { more = 0 } else {
          done[best] = 1
          let nmp: *u8 = ((names as i64) + best * TL_NAMELEN) as *u8
          if o + TL_NAMELEN + TL_RANK_ROW_EXTRA >= rcap { more = 0 } else {
              o = tl_cat(rb, o, nmp)
              rb[o] = 9 as u8; o = o + 1
              o = tl_catn(rb, o, cnt[best])
              rb[o] = 9 as u8; o = o + 1
              o = tl_catn(rb, o, asof)
              rb[o] = 10 as u8; o = o + 1
              rows = rows + 1
          }
      }
  }
  let fd: i64 = sys_openat_wr(TL_RANK_TMP, TL_CONF_MODE)
  if fd < 0 { return 0 - 1 }
  sys_write(fd, rb, o)
  sys_close(fd)
  if sys_renameat(TL_RANK_TMP, TL_RANK_PATH) != 0 { return 0 - 1 }
  return rows
307}
308
309func main(argc: i64, argv: *i64) -> i64 {
  if argc > 1 {
      let command: *u8=argv[1] as *u8
      if to_eq(command,0,to_len(command),"observations" as *u8) == 1 {
          if sys_ignore_sigpipe() != 0 { return TO_STATUS_OUTPUT }
          return to_observations(argc,argv)
      }
  }
  let buf: *u8 = sys_mmap(TL_TAIL_CAP + 16)
  let n: i64 = tl_read_tail("knowledge/status/actlog.jrnl" as *u8, buf, TL_TAIL_CAP)
  if n <= 0 {
      twe("NX-TOOLATENCY REFUSED: knowledge/status/actlog.jrnl unreadable or empty -- refusing to report GREEN over a ledger it never read\n" as *u8)
      tl_jrnl(0, 0, 0, 0, 0, 0, "-" as *u8, "NODATA" as *u8)
      sys_exit(TL_EXIT_NODATA)
      return TL_EXIT_NODATA
  }
  let names: *u8 = sys_mmap(TL_MAXTOOLS * TL_NAMELEN + 16)
  let cnt: *i64 = sys_mmap(8 * TL_MAXTOOLS + 16) as *i64
  let mx:  *i64 = sys_mmap(8 * TL_MAXTOOLS + 16) as *i64
  let sm:  *i64 = sys_mmap(8 * TL_MAXTOOLS + 16) as *i64
  let br:  *i64 = sys_mmap(8 * TL_MAXTOOLS + 16) as *i64
  var ntools: i64 = 0
  var rows: i64 = 0
  var measured: i64 = 0
333
  var i: i64 = 0
  if n >= TL_TAIL_CAP { i = tl_eol(buf, n, 0) + 1 }
336
  while i < n {
      let e: i64 = tl_eol(buf, n, i)
      var t2: i64 = 0 - 1
      var t3: i64 = 0 - 1
      var seen: i64 = 0
      var p: i64 = i
      while p < e {
          if buf[p] == (9 as u8) {
              seen = seen + 1
              if seen == 2 { t2 = p }
              if seen == 3 { t3 = p }
          }
          p = p + 1
      }
      if t2 >= 0 { if t3 > t2 {
          rows = rows + 1
          let d: i64 = tl_kv(buf, i, e, "dur_ms=" as *u8)
          if d >= 0 {
              measured = measured + 1
              var nl2: i64 = t3 - t2 - 1
              if nl2 > TL_NAMELEN - 1 { nl2 = TL_NAMELEN - 1 }
              var idx: i64 = 0 - 1
              var q: i64 = 0
              while q < ntools {
                  var k: i64 = 0
                  var same: i64 = 1
                  while k < nl2 { if names[q * TL_NAMELEN + k] != buf[t2 + 1 + k] { same = 0; k = nl2 } else { k = k + 1 } }
                  if same == 1 { if names[q * TL_NAMELEN + nl2] == (0 as u8) { idx = q; q = ntools } else { q = q + 1 } } else { q = q + 1 }
              }
              if idx < 0 {
                  if ntools < TL_MAXTOOLS {
                      idx = ntools
                      var k2: i64 = 0
                      while k2 < nl2 { names[idx * TL_NAMELEN + k2] = buf[t2 + 1 + k2]; k2 = k2 + 1 }
                      names[idx * TL_NAMELEN + nl2] = 0 as u8
                      cnt[idx] = 0
                      mx[idx] = 0
                      sm[idx] = 0
                      br[idx] = 0
                      ntools = ntools + 1
                  }
              }
              if idx >= 0 {
                  cnt[idx] = cnt[idx] + 1
                  sm[idx] = sm[idx] + d
                  if d > mx[idx] { mx[idx] = d }
                  if d >= TL_SYNC_BUDGET_MS { br[idx] = br[idx] + 1 }
              }
          }
      } }
      i = e + 1
  }
389
  let cf: *u8 = sys_mmap(TL_CONFCAP + 16)
  var cn: i64 = 0
  let cfd: i64 = sys_openat_rd("async_only_tools.conf" as *u8)
  if cfd >= 0 { cn = sys_read(cfd, cf, TL_CONFCAP); sys_close(cfd) }
394
  // action thresholds: conf first (SSOT), bootstrap defaults otherwise. tl_kv returns -1 for an
  // absent key, and -1 must never become a threshold -- absent means default, decided per key.
  var act_permil: i64 = TL_ACT_PERMIL_DFLT
  var act_avg: i64 = TL_ACT_AVG_MS_DFLT
  let tcf: *u8 = sys_mmap(TL_CONFCAP + 16)
  var tcn: i64 = 0
  let tfd: i64 = sys_openat_rd(TL_CONF)
  if tfd >= 0 { tcn = sys_read(tfd, tcf, TL_CONFCAP); sys_close(tfd) }
  if tcn > 0 {
      let cpm: i64 = tl_kv(tcf, 0, tcn, "act_permil=" as *u8)
      if cpm >= 0 { act_permil = cpm }
      let cav: i64 = tl_kv(tcf, 0, tcn, "act_avg_ms=" as *u8)
      if cav >= 0 { act_avg = cav }
  }
  // AUTO-ROUTE arm (see TL_AUTO_ROUTE_DFLT): conf row wins, compiled default OFF. tl_kv returns -1 for an
  // absent key and -1 must never become a mode -- absent means default, decided here.
  var auto_route: i64 = TL_AUTO_ROUTE_DFLT
  if tcn > 0 { let car: i64 = tl_kv(tcf, 0, tcn, "auto_route=" as *u8); if car >= 0 { auto_route = car } }
  let apb: *u8 = sys_mmap(TL_CONFCAP + 16)
  var apo: i64 = 0
  var routed: i64 = 0
416
  let warn_ms: i64 = TL_SYNC_BUDGET_MS * TL_WARN_PERMIL / 1000
  tw("=== nx_toolatency -- measured handler latency vs the sync budget ===\n" as *u8)
  tw("rows=" as *u8); twn(rows)
  tw(" measured=" as *u8); twn(measured)
  tw(" tools=" as *u8); twn(ntools)
  tw(" sync_budget_ms=" as *u8); twn(TL_SYNC_BUDGET_MS)
  tw(" warn_ms=" as *u8); twn(warn_ms)
  tw(" lane_conf_bytes=" as *u8); twn(cn)
  tw(" act_permil=" as *u8); twn(act_permil)
  tw(" act_avg_ms=" as *u8); twn(act_avg)
  if tcn > 0 { tw(" act_src=conf" as *u8) } else { tw(" act_src=default" as *u8) }
  tw("\n" as *u8)
429
  if measured == 0 {
      twe("NX-TOOLATENCY REFUSED: not one row carries dur_ms -- this ledger predates the telemetry, so GREEN here would mean UNMEASURED, not FAST\n" as *u8)
      tl_jrnl(rows, 0, ntools, 0, 0, 0, "-" as *u8, "NODATA" as *u8)
      sys_exit(TL_EXIT_NODATA)
      return TL_EXIT_NODATA
  }
436
  // THREE TIERS, arrived at by being wrong twice and letting the data correct the ruler.
  //   v1 flagged everything past HALF the budget as SHOULD-ASYNC -- that conflates an ALERT
  //      threshold with an ACTION threshold, and rerouting CHANGES THE CALLER CONTRACT (the caller
  //      gets a job id instead of a result), so it must never fire on a tool that still works.
  //   v2 acted on MAX past the full window -- but max cannot tell a systematic breach from one bad
  //      sample. It immediately flagged nx_mgmt, the tool that drives every build and deploy, on a
  //      max of 14002 ms: TWO MILLISECONDS over, across 69 calls averaging 1663. Rerouting the whole
  //      management API on that would have been an overreaction caused by the ruler, not the tool.
  //   v3 counts BREACHES. nishi_compare_regen breached on both of its 2 calls (avg 14854) -- that is
  //      a tool that does not fit the lane. nx_mgmt breached once in 69 -- that is an outlier.
  // OVER-BUDGET = repeated breaches, act. OUTLIER = crossed once, watch. AT-RISK = past half, watch.
  // the journal row's worklist: over-budget tools still on the sync lane, comma-joined, bounded.
  let unr: *u8 = sys_mmap(TL_CSV_CAP + 16)
  var uo: i64 = 0
  var utrunc: i64 = 0
  unr[0] = 0 as u8
  var over: i64 = 0
  var atrisk: i64 = 0
  var should: i64 = 0
  var q2: i64 = 0
  while q2 < ntools {
      let nmp: *u8 = ((names as i64) + q2 * TL_NAMELEN) as *u8
      var nl3: i64 = 0
      while nmp[nl3] != (0 as u8) { nl3 = nl3 + 1 }
      let onlane: i64 = tl_on_lane(cf, cn, nmp, nl3)
      tw("  " as *u8); tw(nmp)
      tw(" calls=" as *u8); twn(cnt[q2])
      tw(" max_ms=" as *u8); twn(mx[q2])
      tw(" avg_ms=" as *u8); twn(sm[q2] / cnt[q2])
      if onlane == 1 { tw(" lane=async" as *u8) } else { tw(" lane=sync" as *u8) }
      tw(" breaches=" as *u8); twn(br[q2])
      if onlane == 0 {
          // ROUTE vs TOLERATE: recurring breaches alone are not the action bar -- rerouting taxes
          // EVERY call of a tool with a poll to save its slow tail, so the rule demands the tail be
          // a real fraction of calls (act_permil) or the tool be slow even on average (act_avg_ms).
          // Same rule, same values, that admitted the 2026-08-27 async_only rows: ruler and decision
          // share arithmetic or they drift.
          var act: i64 = 0
          if br[q2] >= TL_MIN_BREACH {
              if (br[q2] * 1000) / cnt[q2] >= act_permil { act = 1 }
              if sm[q2] / cnt[q2] >= act_avg { act = 1 }
          }
          if act == 1 {
              over = over + 1
              should = should + 1
              // THE REASON TRAVELS WITH THE COUNT. A worklist of bare names is one step short: the
              // reader still has to re-run the whole census to learn WHY each name is on it, and the
              // numbers are already in hand right here. MEASURED CASE THAT FORCED THIS (2026-08-27):
              // the beat wrote `unrouted=nx_page_verify` and a hand-run minutes later showed that tool
              // TOLERATED at 3/88 = 34 permil, just under the 40 bar -- the tail window slides, so a
              // borderline tool oscillates across the bar and a bare name cannot say which side it was
              // on when the row was written. Emitting name(b=<breaches>/<calls>,avg=<ms>) makes the row
              // adjudicable on its own, with no re-run and no access to the census that produced it.
              // The 24-byte headroom covers the widest reason this can emit (two 10-digit counts plus
              // the punctuation); too little room marks utrunc rather than emitting a half-written one.
              if uo + nl3 + 24 < TL_CSV_CAP {
                  if uo > 0 { unr[uo] = 44 as u8; uo = uo + 1 }
                  var uk: i64 = 0
                  while uk < nl3 { unr[uo] = nmp[uk]; uo = uo + 1; uk = uk + 1 }
                  uo = tl_cat(unr, uo, "(b=" as *u8)
                  uo = tl_catn(unr, uo, br[q2])
                  unr[uo] = 47 as u8
                  uo = uo + 1
                  uo = tl_catn(unr, uo, cnt[q2])
                  uo = tl_cat(unr, uo, ",avg=" as *u8)
                  uo = tl_catn(unr, uo, sm[q2] / cnt[q2])
                  unr[uo] = 41 as u8
                  uo = uo + 1
                  unr[uo] = 0 as u8
              } else { utrunc = 1 }
              // AUTO-ROUTE: accumulate the row now, write ONCE after the loop (one open, one write, one close).
              if auto_route == 1 {
                  let apo2: i64 = tl_auto_route_row(apb, apo, nmp, nl3, br[q2], cnt[q2], sm[q2] / cnt[q2])
                  if apo2 > apo { apo = apo2; routed = routed + 1 }
              }
              tw(" *** OVER-BUDGET: breaches the sync window REPEATEDLY -- these calls 503 today, move to async_only_tools.conf" as *u8)
              if auto_route == 1 { tw(" (AUTO-ROUTING NOW -- see the AUTO-ROUTED line below)" as *u8) }
              tw(" ***" as *u8)
          } else { if br[q2] >= TL_MIN_BREACH {
              atrisk = atrisk + 1
              tw(" *** TOLERATED: repeated breaches below the action bar (rate under act_permil, avg under act_avg_ms) -- high-volume fast tool, the sync-promote lane covers this tail; deliberately NOT rerouted ***" as *u8)
          } else { if br[q2] > 0 {
              atrisk = atrisk + 1
              tw(" *** OUTLIER: crossed the window once -- watch it, do not change the caller contract on one sample ***" as *u8)
          } else {
              if mx[q2] >= warn_ms {
                  atrisk = atrisk + 1
                  tw(" *** AT-RISK: past half the sync budget -- watch, do not reroute yet (rerouting changes the caller contract) ***" as *u8)
              }
          } } }
      }
      tw("\n" as *u8)
      q2 = q2 + 1
  }
531
  // RANK TABLE for the tools API list order (2026-09-15): published on every census run, whatever the verdict.
  let rrows: i64 = tl_write_rank(names, cnt, ntools, sys_now_realtime_sec())
  tw("NX-TOOLATENCY rank_rows=" as *u8); twn(rrows); tw(" rank_path=" as *u8); tw(TL_RANK_PATH); tw("\n" as *u8)
  // AUTO-ROUTE: the accumulated rows land in ONE append, after the census and before the verdict.
  if routed > 0 { tl_auto_route_apply(apb, apo, routed) }
  tw("NX-TOOLATENCY tools=" as *u8); twn(ntools)
  tw(" over_budget=" as *u8); twn(over)
  tw(" at_risk=" as *u8); twn(atrisk)
  tw(" auto_route=" as *u8); twn(auto_route)
  tw(" auto_routed=" as *u8); twn(routed)
  if utrunc == 1 { uo = tl_cat(unr, uo, ",+MORE" as *u8); unr[uo] = 0 as u8 }
  if uo == 0 { uo = tl_cat(unr, 0, "-" as *u8); unr[uo] = 0 as u8 }
  if over > 0 {
      tw(" verdict=RED (the OVER-BUDGET tools are 503ing now; a 503 there loses a response for work that may have landed)\n" as *u8)
      tl_jrnl(rows, measured, ntools, over, atrisk, routed, unr, "RED" as *u8)
      sys_exit(TL_EXIT_OVER)
      return TL_EXIT_OVER
  }
  if atrisk > 0 {
      tw(" verdict=AMBER (nothing is failing yet; the AT-RISK tools are past half the window and worth watching)\n" as *u8)
      tl_jrnl(rows, measured, ntools, over, atrisk, routed, unr, "AMBER" as *u8)
      sys_exit(TL_EXIT_SHOULD)
      return TL_EXIT_SHOULD
  }
  tw(" verdict=GREEN (every measured tool sits inside half the sync budget, or is already on the job lane)\n" as *u8)
  tl_jrnl(rows, measured, ntools, over, atrisk, routed, unr, "GREEN" as *u8)
  return 0
559}
nx_toolatency.nx source

`nx_toolatency.nx` source
