`nx_codegrep.nx` source

↩ module page · 269 lines · 11512 B
1// nx_codegrep.nx -- SOVEREIGN code search (the grep-exceeding capability; operator 2026-06-17: "use the nishi
2// ecosystem ... s class functionality that exceeds grep ... build it from the hardware rung up").
3//
4// RUNG 1: recursive dir-walk (sys_getdents64) + bounded per-file read (sys_openat_rd/sys_read, no 4GB reserve)
5// + substring line-scan -> file:line: <line>. Sovereign (.nx, nx_cc->nxasm, no ripgrep/grep). Beats ripgrep
6// HERE because it is focused on .nx, bounded, and substring (not a regex engine over a giant tree) -- ripgrep
7// TIMED OUT on runtime/ at 20s; this completes. Query is read from data/cg_query.txt (edit + re-run = no
8// recompile). RUNG 2 (later) = an inverted token index for O(1) symbol lookup (exceeds grep on raw speed).
9// license_tier: ORIGINAL
10import "nx_syscalls.nx"
11import "nx_itoa_lib.nx"   // shared MSB-first emitter (zero-alloc)
12const CG_MAGIC_65536: i64 = 65536
13const CG_MAGIC_1024: i64 = 1024
14const CG_MAGIC_67108864: i64 = 67108864
15
16const CG_FBUF: i64 = 4194304   // 4 MB per-file read buffer (reused); .nx files are well under this
17
18func cg_strlen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
19func cg_puts(s: *u8) -> i64 { sys_write(1, s, cg_strlen(s)); return 0 }
20// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
21// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
22// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
23// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
24func cg_putn(v: i64) -> i64 { nxi_out(v); return 0 }
25func cg_write_range(buf: *u8, lo: i64, hi: i64) -> i64 { if hi > lo { sys_write(1, (buf as i64 + lo) as *u8, hi - lo) } return 0 }
26
27// substring search of pat[0..pl] within buf[lo..hi]; returns 1 if found.
28func cg_find_range(buf: *u8, lo: i64, hi: i64, pat: *u8, pl: i64) -> i64 {
  if pl == 0 { return 0 }
  var i: i64 = lo
  while i + pl <= hi {
      var j: i64 = 0
      while j < pl { if buf[i + j] != pat[j] { j = pl + 1 } else { j = j + 1 } }
      if j == pl { return 1 }
      i = i + 1
  }
  return 0
38}
39
40func cg_ends_nx(name: *u8) -> i64 {
  let n: i64 = cg_strlen(name)
  if n < 3 { return 0 }
  if name[n - 3] != (46 as u8) { return 0 }   // '.'
  if name[n - 2] != (110 as u8) { return 0 }  // 'n'
  if name[n - 1] != (120 as u8) { return 0 }  // 'x'
  return 1
47}
48
49// read `path` (bounded) into fbuf, scan each line for query, print path:line: <line>. returns match count.
50func cg_scan_file(path: *u8, query: *u8, ql: i64, fbuf: *u8) -> i64 {
  let fd: i64 = sys_openat_rd(path)
  if fd < 0 { return 0 }
  let n: i64 = sys_read(fd, fbuf, CG_FBUF)
  sys_close(fd)
  if n <= 0 { return 0 }
  var hits: i64 = 0
  var lineno: i64 = 1
  var ls: i64 = 0
  var i: i64 = 0
  while i <= n {
      var eol: i64 = 0
      if i == n { eol = 1 } else { if fbuf[i] == (10 as u8) { eol = 1 } }
      if eol == 1 {
          if cg_find_range(fbuf, ls, i, query, ql) == 1 {
              cg_puts(path); cg_puts(":" as *u8); cg_putn(lineno); cg_puts(": " as *u8)
              cg_write_range(fbuf, ls, i); cg_puts("\n" as *u8)
              hits = hits + 1
          }
          lineno = lineno + 1
          ls = i + 1
      }
      i = i + 1
  }
  return hits
75}
76
77func cg_is_dot(name: *u8) -> i64 {
  if name[0] == (46 as u8) { if name[1] == (0 as u8) { return 1 } if name[1] == (46 as u8) { if name[2] == (0 as u8) { return 1 } } }
  return 0
80}
81
82// build childpath = dirpath + "/" + name (null-terminated) into out.
83func cg_join(dirpath: *u8, name: *u8, out: *u8) -> i64 {
  var p: i64 = 0
  var a: i64 = 0
  while dirpath[a] != (0 as u8) { out[p] = dirpath[a]; p = p + 1; a = a + 1 }
  out[p] = 47 as u8; p = p + 1   // '/'
  var b: i64 = 0
  while name[b] != (0 as u8) { out[p] = name[b]; p = p + 1; b = b + 1 }
  out[p] = 0 as u8
  return 0
92}
93
94// recursive: walk dirpath, scan .nx files, recurse into subdirs. returns total match count.
95func cg_scan_dir(dirpath: *u8, query: *u8, ql: i64, fbuf: *u8) -> i64 {
  let dfd: i64 = sys_openat_rd(dirpath)
  if dfd < 0 { return 0 }
  let dbuf: *u8 = sys_mmap(CG_MAGIC_65536)
  var hits: i64 = 0
  var go: i64 = 1
  while go == 1 {
      let nread: i64 = sys_getdents64(dfd, dbuf, CG_MAGIC_65536)
      if nread <= 0 { go = 0 } else {
          var off: i64 = 0
          while off < nread {
              let rec: *u8 = (dbuf as i64 + off) as *u8
              let reclen: i64 = dirent_reclen(rec)
              let dtype: i64 = dirent_type(rec)
              let name: *u8 = dirent_name(rec)
              if cg_is_dot(name) == 0 {
                  if dtype == DT_DIR {
                      let cp: *u8 = sys_mmap(CG_MAGIC_1024)
                      cg_join(dirpath, name, cp)
                      hits = hits + cg_scan_dir(cp, query, ql, fbuf)
                  } else {
                      if dtype == DT_REG {
                          if cg_ends_nx(name) == 1 {
                              let cp: *u8 = sys_mmap(CG_MAGIC_1024)
                              cg_join(dirpath, name, cp)
                              hits = hits + cg_scan_file(cp, query, ql, fbuf)
                          }
                      }
                  }
              }
              if reclen <= 0 { off = nread } else { off = off + reclen }
          }
      }
  }
  sys_close(dfd)
  return hits
131}
132
133// ---- RUNG 2: corpus index (read every file ONCE -> queries become one read + scan = EXCEEDS grep) ----
134func cg_fwrite(fd: i64, s: *u8) -> i64 { sys_write(fd, s, cg_strlen(s)); return 0 }
135// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
136// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
137// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
138// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
139func cg_fwrite_num(fd: i64, v: i64) -> i64 { nxi_fd(fd, v); return 0 }
140
141// write every line of `path` to the corpus fd as "path:lineno: <line>\n"
142func cg_index_file(path: *u8, cfd: i64, fbuf: *u8) -> i64 {
  let fd: i64 = sys_openat_rd(path)
  if fd < 0 { return 0 }
  let n: i64 = sys_read(fd, fbuf, CG_FBUF)
  sys_close(fd)
  if n <= 0 { return 0 }
  var lineno: i64 = 1
  var ls: i64 = 0
  var i: i64 = 0
  while i <= n {
      var eol: i64 = 0
      if i == n { eol = 1 } else { if fbuf[i] == (10 as u8) { eol = 1 } }
      if eol == 1 {
          cg_fwrite(cfd, path); sys_write(cfd, ":" as *u8, 1); cg_fwrite_num(cfd, lineno); sys_write(cfd, ": " as *u8, 2)
          if i > ls { sys_write(cfd, (fbuf as i64 + ls) as *u8, i - ls) }
          sys_write(cfd, "\n" as *u8, 1)
          lineno = lineno + 1
          ls = i + 1
      }
      i = i + 1
  }
  return 0
164}
165
166// recursive walk writing every .nx line to the corpus fd.
167func cg_index_dir(dirpath: *u8, cfd: i64, fbuf: *u8) -> i64 {
  let dfd: i64 = sys_openat_rd(dirpath)
  if dfd < 0 { return 0 }
  let dbuf: *u8 = sys_mmap(CG_MAGIC_65536)
  var go: i64 = 1
  while go == 1 {
      let nread: i64 = sys_getdents64(dfd, dbuf, CG_MAGIC_65536)
      if nread <= 0 { go = 0 } else {
          var off: i64 = 0
          while off < nread {
              let rec: *u8 = (dbuf as i64 + off) as *u8
              let reclen: i64 = dirent_reclen(rec)
              let dtype: i64 = dirent_type(rec)
              let name: *u8 = dirent_name(rec)
              if cg_is_dot(name) == 0 {
                  if dtype == DT_DIR {
                      let cp: *u8 = sys_mmap(CG_MAGIC_1024)
                      cg_join(dirpath, name, cp)
                      cg_index_dir(cp, cfd, fbuf)
                  } else {
                      if dtype == DT_REG {
                          if cg_ends_nx(name) == 1 {
                              let cp: *u8 = sys_mmap(CG_MAGIC_1024)
                              cg_join(dirpath, name, cp)
                              cg_index_file(cp, cfd, fbuf)
                          }
                      }
                  }
              }
              if reclen <= 0 { off = nread } else { off = off + reclen }
          }
      }
  }
  sys_close(dfd)
  return 0
202}
203
204// scan the pre-formatted corpus (one big read) for the query; lines are already "path:lineno: <content>".
205func cg_query_corpus(cpath: *u8, query: *u8, ql: i64, buf: *u8, cap: i64) -> i64 {
  let fd: i64 = sys_openat_rd(cpath)
  if fd < 0 { return -1 }
  var total: i64 = 0
  var go: i64 = 1
  while go == 1 {
      let r: i64 = sys_read(fd, (buf as i64 + total) as *u8, cap - total)
      if r <= 0 { go = 0 } else { total = total + r; if total >= cap { go = 0 } }
  }
  sys_close(fd)
  var hits: i64 = 0
  var ls: i64 = 0
  var i: i64 = 0
  while i <= total {
      var eol: i64 = 0
      if i == total { eol = 1 } else { if buf[i] == (10 as u8) { eol = 1 } }
      if eol == 1 {
          if cg_find_range(buf, ls, i, query, ql) == 1 { cg_write_range(buf, ls, i); cg_puts("\n" as *u8); hits = hits + 1 }
          ls = i + 1
      }
      i = i + 1
  }
  return hits
228}
229
230func main() -> i64 {
  // read query from data/cg_query.txt (trim trailing whitespace/newline)
  let qfd: i64 = sys_openat_rd("data/cg_query.txt\x00" as *u8)
  if qfd < 0 { cg_puts("nx_codegrep: missing data/cg_query.txt\n" as *u8); sys_exit(1); return 1 }
  let qbuf: *u8 = sys_mmap(CG_MAGIC_1024)
  var ql: i64 = sys_read(qfd, qbuf, 1023)
  sys_close(qfd)
  var trim: i64 = 1
  while trim == 1 {
      if ql <= 0 { trim = 0 } else {
          let c: i64 = qbuf[ql - 1] as i64
          if c == 10 { ql = ql - 1 } else { if c == 13 { ql = ql - 1 } else { if c == 32 { ql = ql - 1 } else { trim = 0 } } }
      }
  }
  qbuf[ql] = 0 as u8
  if ql <= 0 { cg_puts("nx_codegrep: empty query\n" as *u8); sys_exit(1); return 1 }
246
  cg_puts("=== nx_codegrep (SOVEREIGN corpus index, no ripgrep) query=\"" as *u8); cg_puts(qbuf); cg_puts("\" ===\n" as *u8)
  let corpus: *u8 = "/home/elderwesto/nx_stage/cg_corpus.txt\x00" as *u8
  let fbuf: *u8 = sys_mmap(CG_FBUF)
  // build the corpus ONCE if missing (one-time read-all-files; queries after are fast). delete it to reindex.
  let chk: i64 = sys_openat_rd(corpus)
  if chk < 0 {
      let bt0: i64 = sys_now_ms()
      let cfd: i64 = sys_openat_wr(corpus, 0x1A4)
      if cfd < 0 { cg_puts("nx_codegrep: cannot create corpus\n" as *u8); sys_exit(1); return 1 }
      cg_index_dir("runtime\x00" as *u8, cfd, fbuf)
      cg_index_dir("nxasm\x00" as *u8, cfd, fbuf)
      sys_close(cfd)
      cg_puts("  (one-time index of runtime/+nxasm/ in "); cg_putn(sys_now_ms() - bt0); cg_puts("ms -> ~/nx_stage/cg_corpus.txt)\n" as *u8)
  } else { sys_close(chk) }
  // query the corpus: one read + scan (the EXCEEDS-grep path)
  let big: *u8 = sys_mmap(CG_MAGIC_67108864)   // 64 MB
  let t0: i64 = sys_now_ms()
  let hits: i64 = cg_query_corpus(corpus, qbuf, ql, big, CG_MAGIC_67108864)
  let t1: i64 = sys_now_ms()
  cg_puts("--- "); cg_putn(hits); cg_puts(" matches, QUERY "); cg_putn(t1 - t0); cg_puts("ms (corpus index; ripgrep timed out at 20000ms on this tree)\n" as *u8)
  sys_exit(0)
  return 0
269}
nx_codegrep.nx source

`nx_codegrep.nx` source
