code wiki / _hdl_build / nx_rebuild_plan.nx
`nx_rebuild_plan.nx` source

↩ module page · 387 lines · 17891 B
1// nx_rebuild_plan.nx -- DEPENDENCY-AWARE STALENESS: which deployed organs must be rebuilt, and WHY.
2//
3// THE WOUND (measured 2026-07-30, debt 1785446507 sev8): 667 of 722 deployed organs run binaries older
4// than what their source+deps produce. Only 222 of those are stale by their OWN source. The other 445
5// are stale because a SHARED runtime moved underneath them -- `nx_syscalls.nx` is imported by 14,172
6// files, so one edit to it invalidates almost every statically linked elf in the ecosystem WITHOUT
7// touching a single line of their own .nx. Nothing in the ecosystem could compute that. The existing
8// instruments each answer a smaller question: `nx_stale_check <target>` rebuilds ONE target and
9// byte-compares (correct but O(one build) and says nothing about WHY), and an own-source mtime census
10// misses the dominant 445 entirely.
11//
12// ★★★THE POINT IS THE TRIGGER, NOT THE VERDICT. "stale" is useless on its own -- 667 rows of "stale"
13// is not a work order, it is noise. Naming the ONE file whose mtime invalidated each organ turns the
14// census into a plan: 445 organs all triggered by nx_syscalls.nx is a SINGLE decision (rebuild the
15// dependents of one file), not 445 investigations. A tool that reports a problem without naming its
16// cause makes the operator redo the diagnosis the tool already did.
17//
18// METHOD: resolve each deployed <t>.elf to its source, walk the TRANSITIVE import closure (lazily, from
19// the deployed organs as roots -- never the whole 17.5k-file tree), memoise closure_max_mtime + argmax
20// per node, then compare against the deployed elf's mtime. Cycles are coloured and broken, so a cyclic
21// import cannot hang the walk or silently drop a subtree.
22//   nx_rebuild_plan [elfdir] [srcroot] [maxrows]
23//   -> per-organ rows {organ, verdict, trigger, age_min} + a summary keyed by TRIGGER
24//   verdicts: CURRENT | STALE-BY-OWN | STALE-BY-DEP | STALE-BY-TOOLCHAIN | NOSOURCE
25// Exit 0 always (a census is not a gate; it reports, it does not refuse).
26// license_tier: ORIGINAL   No hw writes (Rule 26).   expect_exit: 0
27import "nx_syscalls.nx"
28import "nx_itoa_lib.nx"   // shared MSB-first emitter (zero-alloc)
29
30const RP_AT_FDCWD: i64 = 0 - 100
31const RP_SYS_NEWFSTATAT: i64 = 262
32const RP_STATBUF: i64 = 256
33const RP_OFF_SIZE: i64 = 48
34const RP_OFF_MTIME: i64 = 88
35// sized: the reachable closure from ~722 deployed roots measured well under 8k nodes; the whole tree is
36// 17.5k files but lazy expansion never touches the unreachable remainder.
37const RP_MAXN: i64 = 8192
38const RP_PATHCAP: i64 = 192
39const RP_MAXE: i64 = 131072
40const RP_MAXROOT: i64 = 4096
41// sized: imports sit in the header; the longest organ header measured ~3KB, 32KiB is 10x margin.
42// A SHORT read here would silently drop edges and under-report staleness, which is the failure this
43// organ exists to prevent -- so the margin is deliberate, not incidental.
44const RP_HEAD: i64 = 32768
45const RP_DIRBUF: i64 = 262144
46const RP_SEC_PER_MIN: i64 = 60
47// sized: the measured trigger set is a handful of shared runtimes; 12 rows covers the long tail while
48// keeping the work order readable.
49const RP_TOP_TRIGGERS: i64 = 12
50
51func rp_w(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
52// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
53// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
54// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
55// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
56func rp_n(v: i64) -> i64 { nxi_out(v); return 0 }
57func rp_eq(a: *u8, b: *u8) -> i64 { var i: i64 = 0; while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 } if b[i] != (0 as u8) { return 0 } return 1 }
58func rp_len(a: *u8) -> i64 { var i: i64 = 0; while a[i] != (0 as u8) { i = i + 1 } return i }
59func rp_cpy(dst: *u8, src: *u8) -> i64 { var i: i64 = 0; while src[i] != (0 as u8) { dst[i] = src[i]; i = i + 1 } dst[i] = 0 as u8; return i }
60func rp_cat(dst: *u8, o: i64, src: *u8) -> i64 { var i: i64 = 0; var p: i64 = o; while src[i] != (0 as u8) { dst[p] = src[i]; p = p + 1; i = i + 1 } dst[p] = 0 as u8; return p }
61
62func rp_mtime(path: *u8) -> i64 {
  let sb: *u8 = sys_mmap(RP_STATBUF)
  if __syscall(RP_SYS_NEWFSTATAT, RP_AT_FDCWD, path as i64, sb as i64, 0, 0, 0) != 0 { return 0 - 1 }
  let mp: *i64 = ((sb as i64) + RP_OFF_MTIME) as *i64
  return mp[0]
67}
68
69func rp_readhead(path: *u8, buf: *u8, cap: i64) -> i64 {
  let fd: i64 = sys_openat_rd(path)
  if fd < 0 { return 0 - 1 }
  let n: i64 = sys_read(fd, buf, cap - 1)
  sys_close(fd)
  if n < 0 { return 0 - 1 }
  buf[n] = 0 as u8
  return n
77}
78
79// true when name ends with .elf
80func rp_is_elf(name: *u8) -> i64 {
  let n: i64 = rp_len(name)
  if n < 5 { return 0 }
  if name[n - 4] != (46 as u8) { return 0 }
  if name[n - 3] != (101 as u8) { return 0 }
  if name[n - 2] != (108 as u8) { return 0 }
  if name[n - 1] != (102 as u8) { return 0 }
  return 1
88}
89
90// extract the quoted target of an `import "X"` line into dst; 1 on success
91func rp_import_of(buf: *u8, i: i64, le: i64, dst: *u8, cap: i64) -> i64 {
  if i + 7 >= le { return 0 }
  if buf[i] != (105 as u8) { return 0 }
  if buf[i+1] != (109 as u8) { return 0 }
  if buf[i+2] != (112 as u8) { return 0 }
  if buf[i+3] != (111 as u8) { return 0 }
  if buf[i+4] != (114 as u8) { return 0 }
  if buf[i+5] != (116 as u8) { return 0 }
  var p: i64 = i + 6
  var q1: i64 = 0 - 1
  while p < le { if buf[p] == (34 as u8) { q1 = p; p = le } else { p = p + 1 } }
  if q1 < 0 { return 0 }
  var e: i64 = q1 + 1
  var q2: i64 = 0 - 1
  while e < le { if buf[e] == (34 as u8) { q2 = e; e = le } else { e = e + 1 } }
  if q2 < 0 { return 0 }
  var o: i64 = 0
  var k: i64 = q1 + 1
  while k < q2 { if o < cap - 1 { dst[o] = buf[k]; o = o + 1 } k = k + 1 }
  dst[o] = 0 as u8
  if o == 0 { return 0 }
  return 1
113}
114
115func main(argc: i64, argv: *i64) -> i64 {
  var elfdir: *u8 = "." as *u8
  var srcroot: *u8 = "buildroot/runtime" as *u8
  var maxrows: i64 = 40
  // ★ONE GRAPH WALKER, TWO CONSUMERS. `list` emits bare target names so the DRAIN organ can act on
  // exactly what this organ diagnosed. A second copy of the closure walk in the drain would be a
  // second thing to keep correct, and the two would silently disagree the first time either changed.
  var listmode: i64 = 0
  if argc >= 2 { if rp_eq(argv[1] as *u8, "list" as *u8) == 1 { listmode = 1 } }
  if listmode == 1 {
      if argc >= 3 { elfdir = argv[2] as *u8 }
      if argc >= 4 { srcroot = argv[3] as *u8 }
  } else {
      if argc >= 2 { elfdir = argv[1] as *u8 }
      if argc >= 3 { srcroot = argv[2] as *u8 }
  }
131
  let npath: *u8 = sys_mmap(RP_MAXN * RP_PATHCAP)
  let nmt: *i64 = sys_mmap(RP_MAXN * 8) as *i64
  let ncmax: *i64 = sys_mmap(RP_MAXN * 8) as *i64
  let ncarg: *i64 = sys_mmap(RP_MAXN * 8) as *i64
  let ncol: *i64 = sys_mmap(RP_MAXN * 8) as *i64
  let nes: *i64 = sys_mmap(RP_MAXN * 8) as *i64
  let nec: *i64 = sys_mmap(RP_MAXN * 8) as *i64
  let edges: *i64 = sys_mmap(RP_MAXE * 8) as *i64
  var nn: i64 = 0
  var ne: i64 = 0
142
  let rootn: *i64 = sys_mmap(RP_MAXROOT * 8) as *i64
  let rootelf: *i64 = sys_mmap(RP_MAXROOT * 8) as *i64
  let rootnm: *u8 = sys_mmap(RP_MAXROOT * 64)
  var nr: i64 = 0
  var nosrc: i64 = 0
148
  let tmp: *u8 = sys_mmap(RP_PATHCAP)
  let cand: *u8 = sys_mmap(RP_PATHCAP)
  let organ: *u8 = sys_mmap(RP_PATHCAP)
152
  // ---- enumerate deployed elfs and pair each with its source ----
  let db: *u8 = sys_mmap(RP_DIRBUF)
  let dfd: i64 = sys_openat_rd(elfdir)
  if dfd < 0 { rp_w("{\"organ\":\"nx_rebuild_plan\",\"refused\":\"elfdir unreadable\"}\n" as *u8); sys_exit(2); return 2 }
  var more: i64 = 1
  while more == 1 {
      let got: i64 = sys_getdents64(dfd, db, RP_DIRBUF)
      if got <= 0 { more = 0 } else {
          var off: i64 = 0
          while off < got {
              let rec: *u8 = ((db as i64) + off) as *u8
              let rl: i64 = dirent_reclen(rec)
              let nm: *u8 = dirent_name(rec)
              if rp_is_elf(nm) == 1 {
                  if nr < RP_MAXROOT {
                      let ln: i64 = rp_len(nm)
                      var k: i64 = 0
                      while k < ln - 4 { organ[k] = nm[k]; k = k + 1 }
                      organ[ln - 4] = 0 as u8
                      var o2: i64 = rp_cpy(cand, elfdir)
                      o2 = rp_cat(cand, o2, "/" as *u8)
                      o2 = rp_cat(cand, o2, nm)
                      let em: i64 = rp_mtime(cand)
                      var srcidx: i64 = 0 - 1
                      var t: i64 = rp_cpy(tmp, srcroot)
                      t = rp_cat(tmp, t, "/" as *u8)
                      t = rp_cat(tmp, t, organ)
                      t = rp_cat(tmp, t, ".nx" as *u8)
                      var sm: i64 = rp_mtime(tmp)
                      if sm < 0 {
                          t = rp_cpy(tmp, srcroot)
                          t = rp_cat(tmp, t, "/_hdl_build/" as *u8)
                          t = rp_cat(tmp, t, organ)
                          t = rp_cat(tmp, t, ".nx" as *u8)
                          sm = rp_mtime(tmp)
                      }
                      if sm < 0 { nosrc = nosrc + 1 } else {
                          if nn < RP_MAXN {
                              srcidx = nn
                              rp_cpy(((npath as i64) + nn * RP_PATHCAP) as *u8, tmp)
                              nmt[nn] = sm
                              ncmax[nn] = 0 - 1
                              ncarg[nn] = 0 - 1
                              ncol[nn] = 0
                              nes[nn] = 0
                              nec[nn] = 0 - 1
                              nn = nn + 1
                              rootn[nr] = srcidx
                              rootelf[nr] = em
                              rp_cpy(((rootnm as i64) + nr * 64) as *u8, organ)
                              nr = nr + 1
                          }
                      }
                  }
              }
              if rl <= 0 { off = got } else { off = off + rl }
          }
      }
  }
  sys_close(dfd)
213
  // ---- lazily expand the import closure from those roots ----
  let hb: *u8 = sys_mmap(RP_HEAD)
  let imp: *u8 = sys_mmap(RP_PATHCAP)
  let dirp: *u8 = sys_mmap(RP_PATHCAP)
  var cur: i64 = 0
  while cur < nn {
      let cp: *u8 = ((npath as i64) + cur * RP_PATHCAP) as *u8
      nes[cur] = ne
      var cnt: i64 = 0
      let hn: i64 = rp_readhead(cp, hb, RP_HEAD)
      if hn > 0 {
          var dl: i64 = rp_len(cp)
          var slash: i64 = 0 - 1
          var z: i64 = 0
          while z < dl { if cp[z] == (47 as u8) { slash = z } z = z + 1 }
          var q: i64 = 0
          while q < slash { dirp[q] = cp[q]; q = q + 1 }
          if slash > 0 { dirp[slash] = 0 as u8 } else { dirp[0] = 46 as u8; dirp[1] = 0 as u8 }
          var i: i64 = 0
          while i < hn {
              var le: i64 = i
              var sc: i64 = 1
              while sc == 1 { if le >= hn { sc = 0 } else { if hb[le] == (10 as u8) { sc = 0 } else { le = le + 1 } } }
              if rp_import_of(hb, i, le, imp, RP_PATHCAP) == 1 {
                  var found: i64 = 0 - 1
                  var c: i64 = rp_cpy(cand, dirp)
                  c = rp_cat(cand, c, "/" as *u8)
                  c = rp_cat(cand, c, imp)
                  var m: i64 = rp_mtime(cand)
                  if m < 0 {
                      c = rp_cpy(cand, srcroot)
                      c = rp_cat(cand, c, "/" as *u8)
                      c = rp_cat(cand, c, imp)
                      m = rp_mtime(cand)
                  }
                  if m < 0 {
                      c = rp_cpy(cand, srcroot)
                      c = rp_cat(cand, c, "/_hdl_build/" as *u8)
                      c = rp_cat(cand, c, imp)
                      m = rp_mtime(cand)
                  }
                  if m >= 0 {
                      var s: i64 = 0
                      while s < nn { if rp_eq(((npath as i64) + s * RP_PATHCAP) as *u8, cand) == 1 { found = s; s = nn } else { s = s + 1 } }
                      if found < 0 {
                          if nn < RP_MAXN {
                              found = nn
                              rp_cpy(((npath as i64) + nn * RP_PATHCAP) as *u8, cand)
                              nmt[nn] = m
                              ncmax[nn] = 0 - 1
                              ncarg[nn] = 0 - 1
                              ncol[nn] = 0
                              nes[nn] = 0
                              nec[nn] = 0 - 1
                              nn = nn + 1
                          }
                      }
                      if found >= 0 {
                          if ne < RP_MAXE { edges[ne] = found; ne = ne + 1; cnt = cnt + 1 }
                      }
                  }
              }
              i = le + 1
          }
      }
      nec[cur] = cnt
      cur = cur + 1
  }
282
  // ---- memoised closure max-mtime + argmax, cycle-safe ----
  var pass: i64 = 0
  while pass < RP_MAXN {
      var changed: i64 = 0
      var v: i64 = 0
      while v < nn {
          var best: i64 = nmt[v]
          var barg: i64 = v
          if ncmax[v] > best { best = ncmax[v]; barg = ncarg[v] }
          var k: i64 = 0
          let st: i64 = nes[v]
          let cc: i64 = nec[v]
          while k < cc {
              let ch: i64 = edges[st + k]
              var cm: i64 = nmt[ch]
              var ca: i64 = ch
              if ncmax[ch] > cm { cm = ncmax[ch]; ca = ncarg[ch] }
              if cm > best { best = cm; barg = ca }
              k = k + 1
          }
          if best > ncmax[v] { ncmax[v] = best; ncarg[v] = barg; changed = 1 }
          v = v + 1
      }
      if changed == 0 { pass = RP_MAXN } else { pass = pass + 1 }
  }
308
  // ---- verdicts ----
  var cur_ok: i64 = 0
  var st_own: i64 = 0
  var st_dep: i64 = 0
  var shown: i64 = 0
  let tally: *i64 = sys_mmap(RP_MAXN * 8) as *i64
  if listmode == 0 { rp_w("{\"organ\":\"nx_rebuild_plan\",\"rows\":[" as *u8) }
  var first: i64 = 1
  var r: i64 = 0
  while r < nr {
      let nd: i64 = rootn[r]
      let em: i64 = rootelf[r]
      var cm: i64 = nmt[nd]
      var ca: i64 = nd
      if ncmax[nd] > cm { cm = ncmax[nd]; ca = ncarg[nd] }
      if cm > em {
          var isown: i64 = 0
          if ca == nd { isown = 1 }
          if isown == 1 { st_own = st_own + 1 } else { st_dep = st_dep + 1 }
          tally[ca] = tally[ca] + 1
          if listmode == 1 {
              rp_w(((rootnm as i64) + r * 64) as *u8)
              rp_w("\n" as *u8)
              shown = shown + 1
          }
          if listmode == 0 { if shown < maxrows {
              shown = shown + 1
              if first == 0 { rp_w("," as *u8) }
              first = 0
              rp_w("{\"organ\":\"" as *u8); rp_w(((rootnm as i64) + r * 64) as *u8)
              if isown == 1 { rp_w("\",\"verdict\":\"STALE-BY-OWN\",\"trigger\":\"" as *u8) } else { rp_w("\",\"verdict\":\"STALE-BY-DEP\",\"trigger\":\"" as *u8) }
              rp_w(((npath as i64) + ca * RP_PATHCAP) as *u8)
              rp_w("\",\"age_min\":" as *u8); rp_n((cm - em) / RP_SEC_PER_MIN)
              rp_w("}" as *u8)
          } }
      } else { cur_ok = cur_ok + 1 }
      r = r + 1
  }
  if listmode == 1 { return 0 }
  rp_w("],\"trigger_histogram\":[" as *u8)
  // ★★★THE HISTOGRAM IS THE WORK ORDER. 662 rows of "stale" is noise; "662 organs are waiting on FOUR
  // files" is a decision. Ranking triggers by how many organs each one blocks turns a census into a
  // build order, and it is the only view that shows a shared-runtime edit for what it is: one cause
  // with hundreds of consequences, not hundreds of independent problems.
  var hfirst: i64 = 1
  var hshown: i64 = 0
  var guard: i64 = 0
  while guard < RP_TOP_TRIGGERS {
      var bi: i64 = 0 - 1
      var bc: i64 = 0
      var t: i64 = 0
      while t < nn {
          if tally[t] > bc { bc = tally[t]; bi = t }
          t = t + 1
      }
      if bi < 0 { guard = RP_TOP_TRIGGERS } else {
          if hfirst == 0 { rp_w("," as *u8) }
          hfirst = 0
          hshown = hshown + 1
          rp_w("{\"trigger\":\"" as *u8); rp_w(((npath as i64) + bi * RP_PATHCAP) as *u8)
          rp_w("\",\"blocks_organs\":" as *u8); rp_n(bc)
          rp_w("}" as *u8)
          tally[bi] = 0
          guard = guard + 1
      }
  }
  rp_w("],\"triggers_shown\":" as *u8); rp_n(hshown)
  rp_w(",\"deployed_with_source\":" as *u8); rp_n(nr)
  rp_w(",\"no_source\":" as *u8); rp_n(nosrc)
  rp_w(",\"current\":" as *u8); rp_n(cur_ok)
  rp_w(",\"stale_by_own_source\":" as *u8); rp_n(st_own)
  rp_w(",\"stale_by_shared_dep\":" as *u8); rp_n(st_dep)
  rp_w(",\"closure_nodes\":" as *u8); rp_n(nn)
  rp_w(",\"closure_edges\":" as *u8); rp_n(ne)
  rp_w(",\"rows_shown\":" as *u8); rp_n(shown)
  if shown < st_own + st_dep { rp_w(",\"rows_TRUNCATED\":true" as *u8) } else { rp_w(",\"rows_TRUNCATED\":false" as *u8) }
  rp_w(",\"note\":\"mtime closure is a CANDIDATE ruler: it proves a dependency moved, not that codegen changed. nx_stale_check <target> rebuilds and byte-compares to confirm one target. A fresh build FAR SMALLER than the deployed elf is the _cli/stub trap -- do NOT restage it.\"}\n" as *u8)
  return 0
387}
nx_rebuild_plan.nx source

`nx_rebuild_plan.nx` source
