`nx_docstage.nx` source

↩ module page · 363 lines · 18345 B
1// nx_docstage.nx -- DID THE THING WE SHIPPED GET WRITTEN DOWN ANYWHERE? (DM8, devmgmt rung.)
2// The adoption gap this closes is not "no docs" -- it is that NOBODY MEASURES whether a shipped
3// symbol ever reaches a surface another seat can retrieve. A capability that exists and is unfindable
4// is a retrieval failure, and the estate's own record says the same doctrine got written four times
5// because nobody looked first.
6//
7// DOCUMENTED = the symbol appears as an EXACT TOKEN in at least one file under the configured doc
8// roots (compare matrix / plan / watch rows, the doc library, or -- when a caller passes its own
9// roots -- a memory tree).
10//
11// EXACT TOKEN, NOT SUBSTRING, AND THAT IS THE WHOLE BAR. The accept rule is ZERO FALSE DOCUMENTED,
12// and a substring search reports `nx_dora` as documented because `nx_dora_page` is mentioned
13// somewhere. So a hit counts only when the character before AND after are non-identifier bytes.
14// This is deliberately NOT a fork of nx_absent: that organ answers "is this substring absent, and was
15// the search complete", which is a DIFFERENT QUESTION -- and a ruler whose subject does not match the
16// question it is asked is the defect this estate finds most often. The coverage half of nx_absent's
17// contract is reproduced here honestly instead: every cap sets a flag and the flag decides the verdict.
18//
19// THREE STATES, because presence and absence do not need the same evidence:
20//   DOCUMENTED   -- at least one exact-token witness. ONE WITNESS IS ENOUGH, so this verdict stands
21//                   even when coverage was partial.
22//   UNDOCUMENTED -- zero witnesses AND coverage_complete=1. An absence claim needs the whole corpus.
23//   UNMEASURED   -- zero witnesses and coverage was NOT complete, or no root could be read at all.
24//                   "I could not look" is not "it is not written down", and collapsing those two is
25//                   how a reporting stage starts lying in the safe-sounding direction.
26// Exit: 0 DOCUMENTED, 1 UNDOCUMENTED, 3 UNMEASURED. Callers that are report-only IGNORE the code and
27// print the line -- a pre-flight that can refuse gets disabled, one that only informs survives.
28//   nx_docstage <symbol> [roots-conf]
29// license_tier: ORIGINAL  Read-only. No hw writes (Rule 26).
30import "nx_sovjson_lib.nx"
31import "nx_syscalls.nx"
32
33const DS_DIRBUF: i64 = 65536      // getdents64 batch -- LOOPED until 0; ONE CALL IS NOT A LISTING
34const DS_PATH: i64 = 1024         // one path cell
35const DS_MAXDIRS: i64 = 8192      // directory stack; overflow sets the cap flag (never silent)
36const DS_MAXFILES: i64 = 131072   // file table; overflow sets the cap flag
37const DS_MAXROOTS: i64 = 64       // configured doc roots
38const DS_CONFCAP: i64 = 65536     // roots conf read reserve
39const DS_SHOW: i64 = 24           // hits PRINTED; the COUNT is always complete and says so
40// A PER-FILE WORK BUDGET, and it is NOT a buffer cap to tune. The corpus is whatever a caller points
41// this at, so its content is genuinely unknowable in advance: measured 2026-08-20, knowledge/library
42// holds 7,627+ top-level entries INCLUDING a video/ tree of mirrored media, and scanning that
43// byte-by-byte never returns -- the organ produced ZERO bytes and read exactly like a crash. So the
44// bound exists for ONE named purpose (keep a documentation scan proportional to documentation) and
45// its truncation ANNOUNCES: every skipped file is counted and forces coverage_complete=0, which makes
46// UNDOCUMENTED unclaimable over a partially-read corpus. DERIVATION, not a guess: the two real doc
47// corpora measured are buildroot/knowledge/compare (298 files / 1,816,322 B, mean 6,095) and
48// knowledge/compare (380 files / 14,920,470 B, mean 39,264). 4 MiB is ~107x the larger MEAN and more
49// than twice the ENTIRE smaller corpus, so no text documentation file reaches it while every mirrored
50// video and model exceeds it. Override per corpus with `max_file_bytes=<n>` in the roots conf.
51const DS_MAXFILE_DEFAULT: i64 = 4194304
52const DS_EQ: i64 = 61             // ASCII '=' -- a conf line with one is a SETTING, not a root
53const DS_OUT: i64 = 262144
54const DS_SLASH: i64 = 47          // ASCII '/'
55const DS_DOT: i64 = 46            // ASCII '.'
56const DS_NL: i64 = 10             // ASCII newline
57const DS_HASH: i64 = 35           // ASCII '#' -- conf comment
58const DS_SP: i64 = 32             // ASCII space
59const DS_TAB: i64 = 9             // ASCII tab
60const DS_UND: i64 = 95            // ASCII '_' -- an identifier byte, so NOT a token boundary
61const DS_A_UP: i64 = 65
62const DS_Z_UP: i64 = 90
63const DS_A_LO: i64 = 97
64const DS_Z_LO: i64 = 122
65const DS_D0: i64 = 48
66const DS_D9: i64 = 57
67const DS_DT_DIR: i64 = 4          // linux dirent DT_DIR
68const DS_DT_UNKNOWN: i64 = 0      // linux dirent DT_UNKNOWN -- probed, never assumed
69const DS_EXIT_DOC: i64 = 0
70const DS_EXIT_UNDOC: i64 = 1
71const DS_EXIT_UNMEASURED: i64 = 3
72
73func ds_cell(base: *u8, idx: i64, cell: i64) -> *u8 { return ((base as i64) + idx * cell) as *u8 }
74func ds_ident(c: i64) -> i64 {
  if c == DS_UND { return 1 }
  if c >= DS_A_UP { if c <= DS_Z_UP { return 1 } }
  if c >= DS_A_LO { if c <= DS_Z_LO { return 1 } }
  if c >= DS_D0 { if c <= DS_D9 { return 1 } }
  return 0
80}
81func ds_isdir(path: *u8, pbuf: *u8) -> i64 {
  let fd: i64 = sys_openat_rd(path)
  if fd < 0 { return 0 }
  let r: i64 = sys_getdents64(fd, pbuf, DS_DIRBUF)
  sys_close(fd)
  if r < 0 { return 0 }
  return 1
88}
89// wcx[0]=dirs wcx[1]=files wcx[2]=capped wcx[3]=unopenable_dirs
90func ds_walk(root: *u8, parena: *u8, nf0: i64, wcx: *i64) -> i64 {
  let stk: *u8 = sys_mmap(DS_MAXDIRS * DS_PATH)
  let dbuf: *u8 = sys_mmap(DS_DIRBUF)
  let pbuf: *u8 = sys_mmap(DS_DIRBUF)
  let cur: *u8 = sys_mmap(DS_PATH)
  let fp: *u8 = sys_mmap(DS_PATH)
  let c0: *u8 = ds_cell(stk, 0, DS_PATH)
  var z: i64 = sj_cat(c0, 0, root)
  c0[z] = 0 as u8
  var sn: i64 = 1
  var nf: i64 = nf0
  while sn > 0 {
      sn = sn - 1
      let dcell: *u8 = ds_cell(stk, sn, DS_PATH)
      var dl: i64 = sj_cat(cur, 0, dcell)
      cur[dl] = 0 as u8
      let fd: i64 = sys_openat_rd(cur)
      if fd < 0 { wcx[3] = wcx[3] + 1 } else {
          wcx[0] = wcx[0] + 1
          var go: i64 = 1
          while go == 1 {
              let nr: i64 = sys_getdents64(fd, dbuf, DS_DIRBUF)
              if nr <= 0 { go = 0 } else {
                  var off: i64 = 0
                  while off < nr {
                      let rec: *u8 = ((dbuf as i64) + off) as *u8
                      let nm: *u8 = dirent_name(rec)
                      let dt: i64 = dirent_type(rec)
                      var skip: i64 = 0
                      if sj_vlen(nm) == 1 { if nm[0] == (DS_DOT as u8) { skip = 1 } }
                      if sj_vlen(nm) == 2 { if nm[0] == (DS_DOT as u8) { if nm[1] == (DS_DOT as u8) { skip = 1 } } }
                      if skip == 0 {
                          var fo: i64 = sj_cat(fp, 0, cur)
                          if fo > 0 { if fp[fo-1] != (DS_SLASH as u8) { fp[fo] = DS_SLASH as u8; fo = fo + 1 } }
                          fo = sj_cat(fp, fo, nm)
                          fp[fo] = 0 as u8
                          var isd: i64 = 0
                          if dt == DS_DT_DIR { isd = 1 }
                          if dt == DS_DT_UNKNOWN { isd = ds_isdir(fp, pbuf) }
                          if isd == 1 {
                              if sn >= DS_MAXDIRS { wcx[2] = 1 } else {
                                  let sc: *u8 = ds_cell(stk, sn, DS_PATH)
                                  var q3: i64 = sj_cat(sc, 0, fp)
                                  sc[q3] = 0 as u8
                                  sn = sn + 1
                              }
                          } else {
                              if nf >= DS_MAXFILES { wcx[2] = 1 } else {
                                  let fc: *u8 = ds_cell(parena, nf, DS_PATH)
                                  var q2: i64 = sj_cat(fc, 0, fp)
                                  fc[q2] = 0 as u8
                                  nf = nf + 1
                              }
                          }
                      }
                      off = off + dirent_reclen(rec)
                  }
              }
          }
          sys_close(fd)
      }
  }
  wcx[1] = nf
  sys_munmap(stk, DS_MAXDIRS * DS_PATH)
  sys_munmap(dbuf, DS_DIRBUF)
  sys_munmap(pbuf, DS_DIRBUF)
  return nf
157}
158// cx[0]=hits cx[1]=files_with_hits cx[2]=files_scanned cx[3]=coverage_complete cx[4]=roots_used
159// returns report bytes.
160func ds_scan(sym: *u8, conf: *u8, d: *u8, cx: *i64) -> i64 {
  var ci: i64 = 0
  while ci < 8 { cx[ci] = 0; ci = ci + 1 }
  let sl: i64 = sj_vlen(sym)
  let roots: *u8 = sys_mmap(DS_MAXROOTS * DS_PATH)
  var nroots: i64 = 0
  var builtin: i64 = 0
  let maxfile: *i64 = sys_mmap(16) as *i64
  let maxsrc: *i64 = sys_mmap(16) as *i64
  let rootcap: *i64 = sys_mmap(16) as *i64
  maxfile[0] = DS_MAXFILE_DEFAULT
  maxsrc[0] = 0
  rootcap[0] = 0
  let cbuf: *u8 = sys_mmap(DS_CONFCAP)
  var cn: i64 = 0
  let cfd: i64 = sys_openat_rd(conf)
  if cfd >= 0 {
      var go: i64 = 1
      while go == 1 { let r: i64 = sys_read(cfd, ((cbuf as i64)+cn) as *u8, DS_CONFCAP - 8 - cn); if r <= 0 { go = 0 } else { cn = cn + r } if cn >= DS_CONFCAP - 8 { go = 0 } }
      sys_close(cfd)
  }
  var i: i64 = 0
  while i < cn {
      let le: i64 = sj_le(cbuf, i, cn)
      var s: i64 = i
      var g: i64 = 1
      while g == 1 { if s >= le { g = 0 } else { if (cbuf[s] as i64) == DS_SP { s = s + 1 } else { if (cbuf[s] as i64) == DS_TAB { s = s + 1 } else { g = 0 } } } }
      var e: i64 = le
      var g2: i64 = 1
      while g2 == 1 { if e <= s { g2 = 0 } else { let c2: i64 = cbuf[e-1] as i64; if c2 == DS_SP { e = e - 1 } else { if c2 == DS_TAB { e = e - 1 } else { g2 = 0 } } } }
      // a line carrying '=' is a SETTING, not a root: a corpus of media needs a different budget
      // from a corpus of prose, and that belongs in config rather than in this file (rule 17).
      var iseq: i64 = 0
      var qq: i64 = s
      while qq < e { if (cbuf[qq] as i64) == DS_EQ { iseq = 1; qq = e } else { qq = qq + 1 } }
      if e > s { if (cbuf[s] as i64) != DS_HASH { if iseq == 1 {
          if sj_lit_eq(cbuf, s, s + sj_vlen("max_file_bytes=" as *u8), "max_file_bytes=" as *u8) == 1 {
              let mv: i64 = sj_atoi_span(cbuf, s + sj_vlen("max_file_bytes=" as *u8), e)
              if mv > 0 { maxfile[0] = mv; maxsrc[0] = 1 }
          }
      } } }
      if e > s { if (cbuf[s] as i64) != DS_HASH { if iseq == 0 {
          if nroots < DS_MAXROOTS {
              let rc: *u8 = ds_cell(roots, nroots, DS_PATH)
              var k: i64 = 0
              while k < e - s { rc[k] = cbuf[s+k]; k = k + 1 }
              rc[e-s] = 0 as u8
              nroots = nroots + 1
          } else { rootcap[0] = 1 }
      } } }
      i = le + 1
  }
  // RULE 17: config wins; the code default is the LOWEST rung of the hierarchy and it ANNOUNCES
  // itself, because a default nobody was told about reads exactly like a measurement.
  if nroots == 0 {
      let rc0: *u8 = ds_cell(roots, 0, DS_PATH)
      // buildroot/, NOT the nishihost twin: THERE ARE TWO KNOWLEDGE TREES and the compare generator
      // reads the buildroot one. Measured 2026-08-20 -- pointed at the twin, this organ reported
      // `rm_lostset` UNDOCUMENTED over 380 files while the plan row naming it sat in the other tree.
      var z0: i64 = sj_cat(rc0, 0, "buildroot/knowledge/compare" as *u8)
      rc0[z0] = 0 as u8
      nroots = 1
      builtin = 1
  }
  let parena: *u8 = sys_mmap(DS_MAXFILES * DS_PATH)
  let wcx: *i64 = sys_mmap(8 * 8) as *i64
  var nf: i64 = 0
  var r: i64 = 0
  var roots_ok: i64 = 0
  while r < nroots {
      let before: i64 = wcx[0]
      nf = ds_walk(ds_cell(roots, r, DS_PATH), parena, nf, wcx)
      if wcx[0] > before { roots_ok = roots_ok + 1 }
      r = r + 1
  }
  var p: i64 = 0
  p = sj_cat(d, p, "# NX-DOCSTAGE v=1 symbol=" as *u8); p = sj_cat(d, p, sym)
  p = sj_cat(d, p, " roots_conf=" as *u8); p = sj_cat(d, p, conf)
  p = sj_cat(d, p, " roots_declared=" as *u8); p = sj_catn(d, p, nroots)
  p = sj_cat(d, p, " roots_walked=" as *u8); p = sj_catn(d, p, roots_ok)
  p = sj_cat(d, p, " roots_source=" as *u8)
  if builtin == 1 { p = sj_cat(d, p, "builtin-default" as *u8) } else { p = sj_cat(d, p, "conf" as *u8) }
  d[p] = DS_NL as u8; p = p + 1
  var hits: i64 = 0
  var fhits: i64 = 0
  var scanned: i64 = 0
  var unreadable: i64 = 0
  var oversize: i64 = 0
  var bytes: i64 = 0
  var shown: i64 = 0
  let flen: *i64 = sys_mmap(16) as *i64
  var fi: i64 = 0
  while fi < nf {
      let path: *u8 = ds_cell(parena, fi, DS_PATH)
      let q: *u8 = sys_map_file(path, flen)
      if (q as i64) == 0 { unreadable = unreadable + 1 } else {
      if flen[0] > maxfile[0] {
          // OVER BUDGET: skipped, counted, and it costs the run its completeness claim.
          oversize = oversize + 1
          sys_munmap(q, flen[0])
      } else {
          let n: i64 = flen[0]
          scanned = scanned + 1
          bytes = bytes + n
          var line: i64 = 1
          var fh: i64 = 0
          var j: i64 = 0
          while j < n {
              if (q[j] as i64) == DS_NL { line = line + 1 } else {
                  if q[j] == sym[0] {
                      if j + sl <= n {
                          var m: i64 = 0
                          var eq: i64 = 1
                          while m < sl { if q[j+m] != sym[m] { eq = 0; m = sl } else { m = m + 1 } }
                          if eq == 1 {
                              // THE BAR: a hit is a hit only at a token boundary on BOTH sides.
                              var tok: i64 = 1
                              if j > 0 { if ds_ident(q[j-1] as i64) == 1 { tok = 0 } }
                              if j + sl < n { if ds_ident(q[j+sl] as i64) == 1 { tok = 0 } }
                              if tok == 1 {
                                  hits = hits + 1
                                  if fh == 0 { fh = 1; fhits = fhits + 1 }
                                  if shown < DS_SHOW {
                                      p = sj_cat(d, p, "DOC-HIT " as *u8)
                                      p = sj_cat(d, p, path)
                                      p = sj_cat(d, p, " line=" as *u8); p = sj_catn(d, p, line)
                                      d[p] = DS_NL as u8; p = p + 1
                                      shown = shown + 1
                                  }
                                  j = j + sl - 1
                              }
                          }
                      }
                  }
              }
              j = j + 1
          }
          sys_munmap(q, n)
      } }
      fi = fi + 1
  }
  if shown < hits {
      p = sj_cat(d, p, "# NOTE the list above is a PREFIX of its own count: shown=" as *u8); p = sj_catn(d, p, shown)
      p = sj_cat(d, p, " of hits=" as *u8); p = sj_catn(d, p, hits)
      d[p] = DS_NL as u8; p = p + 1
  }
  var cov: i64 = 1
  if wcx[2] == 1 { cov = 0 }
  if unreadable > 0 { cov = 0 }
  if roots_ok < nroots { cov = 0 }
  if oversize > 0 { cov = 0 }
  if rootcap[0] == 1 { cov = 0 }
  p = sj_cat(d, p, "# BUDGET max_file_bytes=" as *u8); p = sj_catn(d, p, maxfile[0])
  p = sj_cat(d, p, " budget_source=" as *u8)
  if maxsrc[0] == 1 { p = sj_cat(d, p, "conf" as *u8) } else { p = sj_cat(d, p, "builtin-derived" as *u8) }
  p = sj_cat(d, p, " files_skipped_oversize=" as *u8); p = sj_catn(d, p, oversize)
  p = sj_cat(d, p, " roots_table_capped=" as *u8); p = sj_catn(d, p, rootcap[0])
  d[p] = DS_NL as u8; p = p + 1
  p = sj_cat(d, p, "# ENVELOPE dirs=" as *u8); p = sj_catn(d, p, wcx[0])
  p = sj_cat(d, p, " files_found=" as *u8); p = sj_catn(d, p, nf)
  p = sj_cat(d, p, " files_scanned=" as *u8); p = sj_catn(d, p, scanned)
  p = sj_cat(d, p, " files_unreadable=" as *u8); p = sj_catn(d, p, unreadable)
  p = sj_cat(d, p, " unopenable_dirs=" as *u8); p = sj_catn(d, p, wcx[3])
  p = sj_cat(d, p, " bytes=" as *u8); p = sj_catn(d, p, bytes)
  p = sj_cat(d, p, " table_capped=" as *u8); p = sj_catn(d, p, wcx[2])
  p = sj_cat(d, p, " coverage_complete=" as *u8); p = sj_catn(d, p, cov)
  d[p] = DS_NL as u8; p = p + 1
  // PRESENCE NEEDS ONE WITNESS; ABSENCE NEEDS THE WHOLE CORPUS. That asymmetry is the verdict rule,
  // not a caveat attached to it.
  var verdict: *u8 = "UNMEASURED" as *u8
  var rc: i64 = DS_EXIT_UNMEASURED
  if hits > 0 { verdict = "DOCUMENTED" as *u8; rc = DS_EXIT_DOC } else {
      if cov == 1 { verdict = "UNDOCUMENTED" as *u8; rc = DS_EXIT_UNDOC }
  }
  p = sj_cat(d, p, "DOCSTAGE symbol=" as *u8); p = sj_cat(d, p, sym)
  p = sj_cat(d, p, " verdict=" as *u8); p = sj_cat(d, p, verdict)
  p = sj_cat(d, p, " hits=" as *u8); p = sj_catn(d, p, hits)
  p = sj_cat(d, p, " files_with_hits=" as *u8); p = sj_catn(d, p, fhits)
  p = sj_cat(d, p, " files_scanned=" as *u8); p = sj_catn(d, p, scanned)
  p = sj_cat(d, p, " coverage_complete=" as *u8); p = sj_catn(d, p, cov)
  p = sj_cat(d, p, " match=exact-token" as *u8)
  d[p] = DS_NL as u8; p = p + 1
  cx[0] = hits
  cx[1] = fhits
  cx[2] = scanned
  cx[3] = cov
  cx[4] = roots_ok
  cx[5] = rc
  return p
349}
350
351func main(argc: i64, argv: *i64) -> i64 {
  if argc < 2 { sj_puts("usage: nx_docstage <symbol> [roots-conf]   (exit 0 DOCUMENTED, 1 UNDOCUMENTED, 3 UNMEASURED)\n" as *u8); sys_exit(2); return 2 }
  let sym: *u8 = argv[1] as *u8
  if sj_vlen(sym) == 0 { sj_puts("REFUSED empty symbol -- an empty needle matches every boundary and would report everything DOCUMENTED\n" as *u8); sys_exit(2); return 2 }
  var conf: *u8 = "knowledge/docstage_roots.conf" as *u8
  if argc >= 3 { conf = argv[2] as *u8 }
  let d: *u8 = sys_mmap(DS_OUT)
  let cx: *i64 = sys_mmap(8 * 8) as *i64
  let dl: i64 = ds_scan(sym, conf, d, cx)
  sys_write(1, d, dl)
  sys_exit(cx[5])
  return cx[5]
363}
nx_docstage.nx source

`nx_docstage.nx` source
