code wiki / _hdl_build / nx_schema_backfill.nx
`nx_schema_backfill.nx` source

↩ module page · 322 lines · 16274 B
1// nx_schema_backfill.nx -- CAPABILITY DISCOVERABILITY HARVESTER (seq1282). Every tool_allowlist.conf row
2// lacking a knowledge/tool_schemas.conf row gets one AUTO-DERIVED from its source's own first header line
3// (the tree-wide `// <stem>.nx -- <desc>` convention; the codewiki maker beat writes missing headers daily,
4// so RE-RUNNING harvests newly documented organs). The register API keeps NEW tools discoverable by
5// construction (seq722 fix) -- this organ only drains history. HONEST: no source / no `-- ` header / desc
6// too short => SKIPPED + COUNTED, never guessed (blank beats bad). Append-if-absent via the API's own
7// ma_schema_has_name tri-state (UNKNOWN refuses the WHOLE run -- a truncated read can never duplicate).
8//   nx_schema_backfill        (JSON report: scanned/had_schema/filled/skipped_no_source/skipped_no_header)
9// license_tier: ORIGINAL   No hw writes (Rule 26).   expect_exit: 0
10import "nx_mgmt_api.nx"
11import "nx_toolsafety_lib.nx"
12import "nx_syscalls.nx"
13
14// sized: allowlist read matches md_allow_has_name's 256KiB envelope; refuse-on-full keeps the census honest
15const SB_ALCAP: i64 = 1 << 18
16// sized: only the FIRST source line is read; header docs are one-liners
17const SB_HDRCAP: i64 = 4096
18// sized: schema titles are one-line prose; hard cap keeps rows lean
19const SB_TITLECAP: i64 = 300
20// derived: shortest honest capability description in curated rows is ~20 bytes; below = not a title, skip
21const SB_MINDESC: i64 = 20
22// sized: tool names are sanitized [a-zA-Z0-9_] <= 120 by the register API
23const SB_NAMECAP: i64 = 128
24// sized: elf paths in the allowlist are absolute NAS paths well under this
25const SB_PATHCAP: i64 = 512
26
27func sbw(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
28
29func sb_ends(s: *u8, sl: i64, suf: *u8, fl: i64) -> i64 {
  if sl < fl { return 0 }
  var i: i64 = 0
  while i < fl { if s[sl-fl+i] != suf[i] { return 0 } i = i + 1 }
  return 1
34}
35
36// ---- SAFETY FLAGS: DERIVED, NOT ASSERTED (2026-08-06) ----------------------------------------
37// MEASURED BEFORE CHANGING ANYTHING: of the 389 rows this organ has auto-derived into
38// tool_schemas.conf, **389 carry the identical quad `0 1 0 1`** -- zero exceptions. Against ~852
39// total rows that is ~46% of the registry's safety metadata being ONE HARDCODED CONSTANT that was
40// never looked up. nx_prim_query serves it downstream as `safety:{read_only,destructive,
41// idempotent,open_world}`, so a consumer cannot tell a measured 0 from a constant 0.
42// ★★A CONSTANT WEARING THE SHAPE OF A MEASUREMENT IS WORSE THAN A BLANK: a blank invites a
43// lookup, a fabricated value ends the enquiry. This organ's own header says "blank beats bad" and
44// it honours that for the DESCRIPTION -- then guessed the most security-relevant field.
45// (In fairness the old constant erred CONSERVATIVE -- claiming destructive+open_world over-warns
46// rather than under-warns -- so this was an INFORMATION defect, not an unsafe one.)
47//
48// WHAT IS AND IS NOT DERIVABLE. read_only / destructive / open_world fall out of a static scan of
49// the organ's own source for the syscalls it actually reaches for. `idempotent` does NOT -- nothing
50// in the source declares it -- so it stays 0, the conservative direction (a caller will not blindly
51// re-run something marked non-idempotent). ★DERIVE WHAT IS DERIVABLE, DECLARE THE REST UNDECLARED,
52// AND SAY WHICH IS WHICH IN THE ROW ITSELF.
53// SB_SRCCAP and the private sb_has substring scanner were DELETED 2026-08-25, not merely bypassed:
54// a dead copy of a ruler beside the shared one is the duplicate-ruler defect waiting to be re-called.
55
56func sb_wflag(fd: i64, v: i64) -> i64 {
  if v == 1 { ma_write_str(fd, "1" as *u8) } else { ma_write_str(fd, "0" as *u8) }
  return 0
59}
60
61// box[0]=read_only box[1]=destructive box[2]=idempotent box[3]=open_world box[4]=derived(1/0)
62func sb_derive_flags(path: *u8, box: *i64) -> i64 {
  box[0] = 0
  box[1] = 1
  box[2] = 0
  box[3] = 1
  box[4] = 0
  // COMPOSED, NOT RE-IMPLEMENTED (2026-08-25). The single-file scan that used to live here is gone.
  // It was not wrong about what it looked at -- it was looking at the wrong SUBJECT. An organ that
  // writes through an imported helper reaches the write syscall in the LIB, so scanning only the
  // organ's own source published readOnly=1 for a writer: wrong in the UNSAFE direction, and stamped
  // with a provenance note confident enough that nx_tools_api's overlay deliberately left it alone.
  // MEASURED by nx_schemadrift over the whole population this organ had already written.
  // A MEASUREMENT OF THE WRONG SUBJECT IS STILL A MEASUREMENT, AND THAT IS EXACTLY WHY NOTHING
  // DOWNSTREAM COULD TELL IT FROM A GOOD ONE.
  // ts_quad is the ONE ruler now (nx_toolsafety_lib), shared with nx_toolflags and the drift control,
  // so the three cannot disagree: there is only one of it. It also removes the 256 KiB source cap --
  // the old scan returned NOT-DERIVED for any source past it -- because sys_read_file sizes from the
  // file and cannot short-read.
  let quad: *i64 = sys_mmap(64) as *i64
  let deep: *i64 = sys_mmap(64) as *i64
  let own: *i64 = sys_mmap(64) as *i64
  // ABSTAIN, DO NOT ASSERT. An inconclusive walk (unreadable subject, unreadable marker table, or a
  // budget hit) leaves the CAUTIOUS DEFAULT set above and box[4]=0, which makes sb_append stamp the
  // row NOT DERIVABLE. A DERIVER THAT FAILS TOWARD PERMISSIVE IS WORSE THAN ONE THAT DECLINES.
  if ts_quad(path, quad, deep, own) == 0 { return 0 }
  box[0] = quad[0]
  box[1] = quad[1]
  box[2] = quad[2]
  box[3] = quad[3]
  box[4] = 1
  return 1
93}
94
95// append one schema row -- SAME invariants as the API path (trailing-nl self-heal; caller scrubbed title)
96func sb_append(nm: *u8, ti: *u8, box: *i64) -> i64 {
  let sf: i64 = sys_openat_append("knowledge/tool_schemas.conf" as *u8, 420)
  if sf < 0 { return 0 }
  ma_ensure_trailing_nl("knowledge/tool_schemas.conf" as *u8, sf)
  ma_write_str(sf, nm)
  ma_write_str(sf, "\t" as *u8)
  ma_write_str(sf, ti)
  ma_write_str(sf, "\t" as *u8)
  sb_wflag(sf, box[0])
  ma_write_str(sf, "\t" as *u8)
  sb_wflag(sf, box[1])
  ma_write_str(sf, "\t" as *u8)
  sb_wflag(sf, box[2])
  ma_write_str(sf, "\t" as *u8)
  sb_wflag(sf, box[3])
  ma_write_str(sf, "\t" as *u8)
  ma_write_str(sf, nm)
  if box[4] == 1 {
      ma_write_str(sf, " output (auto-derived by nx_schema_backfill; safety flags DERIVED FROM THE TRANSITIVE IMPORT CLOSURE by nx_toolsafety_lib -- a write reached through an imported helper counts, idempotent UNDECLARED)\n" as *u8)
  } else {
      ma_write_str(sf, " output (auto-derived by nx_schema_backfill; safety flags NOT DERIVABLE -- conservative default, treat as UNKNOWN)\n" as *u8)
  }
  sys_close(sf)
  return 1
120}
121
122// read the FIRST line of path; copy the text after the first `-- ` into ti (scrubbed, trimmed, capped).
123// returns title length; 0 = header convention absent or desc too short; -1 = file unreadable.
124func sb_title_from_header(path: *u8, ti: *u8) -> i64 {
  let fd: i64 = sys_openat_rd(path)
  if fd < 0 { return 0 - 1 }
  let hb: *u8 = sys_mmap(SB_HDRCAP)
  let hn: i64 = sys_read(fd, hb, SB_HDRCAP - 1)
  sys_close(fd)
  if hn <= 2 { return 0 }
  // The header is NOT always line 1: many organs open with their `import` lines and put the
  // `// <stem>.nx -- <desc>` block AFTER them (measured: nx_gamebench_gate, nx_media_extract_gate
  // were reported as UNDOCUMENTED purely because this parser only looked at line 1 -- the organs were
  // documented all along). Scan the header window for the FIRST `//` line carrying `-- `; that is the
  // convention's actual shape. Declared limitation: a `//` line containing `-- ` above the real header
  // would win, so the rule is first-match and the emitted title is always verbatim source text.
  var ls: i64 = 0
  var ds: i64 = 0
  var le: i64 = 0
  var scan: i64 = 1
  while scan == 1 {
      if ls >= hn { scan = 0 } else {
          le = ls
          var sc: i64 = 1
          while sc == 1 { if le >= hn { sc = 0 } else { if hb[le] == (10 as u8) { sc = 0 } else { le = le + 1 } } }
          var iscmt: i64 = 0
          if le - ls >= 8 { if hb[ls] == (47 as u8) { if hb[ls+1] == (47 as u8) { iscmt = 1 } } }
          if iscmt == 1 {
              var p: i64 = ls + 2
              var s2: i64 = 1
              while s2 == 1 {
                  if p + 3 > le { s2 = 0 } else {
                      if hb[p] == (45 as u8) { if hb[p+1] == (45 as u8) { if hb[p+2] == (32 as u8) { ds = p + 3; s2 = 0 } } }
                      if ds == 0 { p = p + 1 }
                  }
              }
          }
          if ds > 0 { scan = 0 } else { ls = le + 1 }
      }
  }
  if ds == 0 { return 0 }
  var o: i64 = 0
  var q: i64 = ds
  while q < le {
      var c: i64 = hb[q] as i64
      if c < 32 { c = 32 }
      if c == 9 { c = 32 }
      if o < SB_TITLECAP - 1 { ti[o] = c as u8; o = o + 1 }
      q = q + 1
  }
  var tc: i64 = 1
  while tc == 1 { if o > 0 { if ti[o-1] == (32 as u8) { o = o - 1 } else { tc = 0 } } else { tc = 0 } }
  ti[o] = 0 as u8
  if o < SB_MINDESC { return 0 }
  return o
176}
177
178func main() -> i64 {
  let al: *u8 = sys_mmap(SB_ALCAP)
  let fd: i64 = sys_openat_rd("tool_allowlist.conf" as *u8)
  if fd < 0 { sbw("{\"organ\":\"nx_schema_backfill\",\"refused\":\"tool_allowlist.conf unreadable\"}\n" as *u8); sys_exit(2); return 2 }
  let n: i64 = sys_read(fd, al, SB_ALCAP - 1)
  sys_close(fd)
  if n <= 0 { sbw("{\"organ\":\"nx_schema_backfill\",\"refused\":\"tool_allowlist.conf empty\"}\n" as *u8); sys_exit(2); return 2 }
  if n >= SB_ALCAP - 1 { sbw("{\"organ\":\"nx_schema_backfill\",\"refused\":\"allowlist read filled the buffer (possible truncation) -- refusing a partial census\"}\n" as *u8); sys_exit(2); return 2 }
  al[n] = 0 as u8
  var scanned: i64 = 0
  var had: i64 = 0
  var filled: i64 = 0
  var nosrc: i64 = 0
  var nohdr: i64 = 0
  var derived: i64 = 0
  let nm: *u8 = sys_mmap(SB_NAMECAP)
  let elf: *u8 = sys_mmap(SB_PATHCAP)
  let ti: *u8 = sys_mmap(SB_TITLECAP)
  let pth: *u8 = sys_mmap(SB_PATHCAP)
  let fbox: *i64 = sys_mmap(64) as *i64
  var i: i64 = 0
  while i < n {
      var le: i64 = i
      var sc: i64 = 1
      while sc == 1 { if le >= n { sc = 0 } else { if al[le] == (10 as u8) { sc = 0 } else { le = le + 1 } } }
      var ok: i64 = 1
      if le <= i { ok = 0 }
      if ok == 1 { if al[i] == (35 as u8) { ok = 0 } }
      if ok == 1 {
          var nmn: i64 = 0
          var p: i64 = i
          var s2: i64 = 1
          while s2 == 1 {
              if p >= le { s2 = 0; ok = 0 } else {
                  if al[p] == (9 as u8) { s2 = 0 } else {
                      if nmn < SB_NAMECAP - 1 { nm[nmn] = al[p]; nmn = nmn + 1 }
                      p = p + 1
                  }
              }
          }
          nm[nmn] = 0 as u8
          var en: i64 = 0
          if ok == 1 {
              p = p + 1
              var s3: i64 = 1
              while s3 == 1 {
                  if p >= le { s3 = 0 } else {
                      if al[p] == (9 as u8) { s3 = 0 } else {
                          if en < SB_PATHCAP - 1 { elf[en] = al[p]; en = en + 1 }
                          p = p + 1
                      }
                  }
              }
          }
          elf[en] = 0 as u8
          if nmn == 0 { ok = 0 }
          if en == 0 { ok = 0 }
          if ok == 1 {
              scanned = scanned + 1
              let hs: i64 = ma_schema_has_name(nm)
              if hs == 1 { had = had + 1 }
              if hs < 0 {
                  sbw("{\"organ\":\"nx_schema_backfill\",\"refused\":\"tool_schemas.conf unreadable or possibly truncated -- refusing to append blind\"}\n" as *u8)
                  sys_exit(3)
                  return 3
              }
              if hs == 0 {
                  var bs: i64 = 0
                  var k: i64 = 0
                  while elf[k] != (0 as u8) { if elf[k] == (47 as u8) { bs = k + 1 } k = k + 1 }
                  var stl: i64 = k - bs
                  let base: *u8 = ((elf as i64) + bs) as *u8
                  if sb_ends(base, stl, ".sov.elf.new" as *u8, 12) == 1 { stl = stl - 12 } else {
                      if sb_ends(base, stl, ".elf.new" as *u8, 8) == 1 { stl = stl - 8 } else {
                          if sb_ends(base, stl, ".sov.elf" as *u8, 8) == 1 { stl = stl - 8 } else {
                              if sb_ends(base, stl, ".elf" as *u8, 4) == 1 { stl = stl - 4 } else {
                                  if sb_ends(base, stl, ".new" as *u8, 4) == 1 { stl = stl - 4 }
                              }
                          }
                      }
                  }
                  var r1: i64 = 0 - 1
                  var r2: i64 = 0 - 1
                  var tin: i64 = 0
                  if stl > 0 {
                      var o2: i64 = sd_cat(pth, 0, "buildroot/runtime/_hdl_build/" as *u8)
                      var c2: i64 = 0
                      while c2 < stl { pth[o2] = base[c2]; o2 = o2 + 1; c2 = c2 + 1 }
                      o2 = sd_cat(pth, o2, ".nx" as *u8)
                      pth[o2] = 0 as u8
                      r1 = sb_title_from_header(pth, ti)
                      if r1 > 0 { tin = r1 } else {
                          o2 = sd_cat(pth, 0, "buildroot/runtime/" as *u8)
                          c2 = 0
                          while c2 < stl { pth[o2] = base[c2]; o2 = o2 + 1; c2 = c2 + 1 }
                          o2 = sd_cat(pth, o2, ".nx" as *u8)
                          pth[o2] = 0 as u8
                          r2 = sb_title_from_header(pth, ti)
                          if r2 > 0 { tin = r2 }
                      }
                  }
                  if tin > 0 {
                      // pth still holds the source path that actually yielded the title, so the
                      // flags are derived from the SAME file the description came from.
                      sb_derive_flags(pth, fbox)
                      if sb_append(nm, ti, fbox) == 1 {
                          filled = filled + 1
                          if fbox[4] == 1 { derived = derived + 1 }
                      }
                  } else {
                      var nof: i64 = 1
                      if r1 >= 0 { nof = 0 }
                      if r2 >= 0 { nof = 0 }
                      // a COUNT is not a work item: name every skip on stderr so the residual is
                      // actionable (write the header, re-run, watch the number fall) instead of
                      // being an unfalsifiable "2 left" that nobody can pick up.
                      sys_write(2, "SKIP " as *u8, 5)
                      if nof == 1 { sys_write(2, "no-source   " as *u8, 12) } else { sys_write(2, "no-header   " as *u8, 12) }
                      sys_write(2, nm, nmn)
                      sys_write(2, "\n" as *u8, 1)
                      if nof == 1 { nosrc = nosrc + 1 } else { nohdr = nohdr + 1 }
                  }
              }
          }
      }
      i = le + 1
  }
  let out: *u8 = sys_mmap(SB_HDRCAP)
  var o3: i64 = sd_cat(out, 0, "{\"organ\":\"nx_schema_backfill\",\"scanned\":" as *u8)
  o3 = sd_catn(out, o3, scanned)
  o3 = sd_cat(out, o3, ",\"had_schema\":" as *u8)
  o3 = sd_catn(out, o3, had)
  o3 = sd_cat(out, o3, ",\"flags_static_derived\":" as *u8)
  o3 = sd_catn(out, o3, derived)
  o3 = sd_cat(out, o3, ",\"filled\":" as *u8)
  o3 = sd_catn(out, o3, filled)
  o3 = sd_cat(out, o3, ",\"skipped_no_source\":" as *u8)
  o3 = sd_catn(out, o3, nosrc)
  o3 = sd_cat(out, o3, ",\"skipped_no_header\":" as *u8)
  o3 = sd_catn(out, o3, nohdr)
  o3 = sd_cat(out, o3, ",\"note\":\"idempotent; re-run after maker-beat header writes; run nx_toolreg_reconcile to publish\"}\n" as *u8)
  out[o3] = 0 as u8
  sbw(out)
  return 0
322}
nx_schema_backfill.nx source

`nx_schema_backfill.nx` source
