nx_shelltool.nx source
↩ module page · 363 lines · 17819 B
1// nx_shelltool.nx -- the INVOCABLE + MCP-READY surface for the shell-replacement tier (07-15
2// operator: "build our nishi grep and glob and search ... as full MCP API state of the art ...
3// available to these workstreams ... anything else that uses wsl when we could just use nishi").
4// The S1-S9 organs (nx_grep etc.) are LIBS (functions, no main) -> not callable as tools + not on
5// MCP. This is the ONE CLI that COMPOSES the canonical nx_grep (per NISHI_SHELL_REPLACEMENT_ROADMAP
6// "consumers compose nx_grep", no proliferation) into filesystem verbs a workstream can invoke over
7// MCP tools/call instead of the Grep/Glob/Bash builtins + WSL:
8// nx_shelltool grep <pattern> <dir> [ext] -> recursive; prints path:lineno:line for matches
9// nx_shelltool glob <glob-pat> <dir> -> recursive; prints paths whose NAME matches (* wildcard)
10// nx_shelltool find <substr> <dir> -> recursive; prints paths whose NAME contains substr
11// Composes nx_grep_any (canonical match). Read-only. Exit 0; 2 usage; 3 baddir; 5 budget-partial.
12//
13// SEQ1292 ROOT FIX (2026-07-30, incident 2026-07-29 sitewide outage): every scan now runs under a
14// SCAN BUDGET -- an unbounded walk over a multi-GB store starved the whole hub (and the old code
15// mmap'd 1MiB per file, never unmapped => RSS grew linearly with files scanned = the thrash vector).
16// Budget source: shelltool_budget.conf in CWD (keys max_bytes= deadline_ms= max_files=), else
17// derived defaults: max_bytes 512MiB (~1.4% of the 36GB hub, seconds of sequential NAS IO),
18// deadline_ms 20000 (the proven WC_FETCH_BUDGET_MS wall-clock precedent), max_files 200000.
19// Checked per file + per getdents batch so exceeding STOPS IO, prints partial counts + a structured
20// NX-SHELLTOOL BUDGET-EXCEEDED line, exit 5 -- never a silent truncation, never a host wedge.
21// Buffers: ONE reused 1MiB read buffer; per-level walk buffers munmap'd; overlong paths (>4095)
22// skipped LOUDLY via skipped_toolong= in the summary.
23// license_tier: ORIGINAL expect_exit: 0
24import "nx_syscalls.nx"
25import "nx_grep_rt.nx"
26
27const S_EXIT_BADDIR: i64 = 3 // <dir> not a readable directory (loud, distinct from usage=2)
28const S_EXIT_BUDGET: i64 = 5 // budget exceeded -> PARTIAL results (loud, distinct)
29const S_PROBE_CAP: i64 = 4096 // one-getdents probe buffer (dir-ness check)
30const S_DEF_MAX_BYTES: i64 = 536870912 // 512MiB derived: ~1.4% hub RAM, seconds of NAS IO (seq1292)
31const S_DEF_DEADLINE_MS: i64 = 20000 // proven wall-clock budget precedent (WC_FETCH_BUDGET_MS)
32const S_DEF_MAX_FILES: i64 = 200000 // > any legitimate single-tree scan observed to date
33// --- out= support (id 1785516061: the MCP stdout capture truncates at 160KiB with NO marker, so a
34// large scan's result was arriving CLIPPED MID-LINE with its honesty trailer deleted. Same shape as
35// the nx_treediff OUTFILE fix: results to disk, envelope on the wire, coverage bounded by the corpus
36// and never by a caller payload cap.)
37const S_FD_STDOUT: i64 = 1
38const S_MODE_644: i64 = 420
39const S_EXIT_BADOUT: i64 = 6 // out= refused/unwritable (loud, distinct from usage/baddir/budget)
40const S_NUMBUF: i64 = 24
41const S_OUTKEY_LEN: i64 = 4 // len("out=")
42const S_OUT_MINLEN: i64 = 5 // at least one name char + ".out"
43const S_DOT: i64 = 46
44const S_SLASH: i64 = 47
45const S_CH_O: i64 = 111
46const S_CH_U: i64 = 117
47const S_CH_T: i64 = 116
48const S_BUD_OFD: i64 = 9 // bud[] already threads the scan context to every emit site --
49const S_BUD_OBYTES: i64 = 10 // reusing 2 spare slots avoids changing 3 function signatures
50
51func s_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
52func s_write(b: *u8, n: i64) -> i64 { sys_write(1, b, n); return 0 }
53func s_putn(v: i64) -> i64 {
54 if v == 0 { sys_write(1, "0" as *u8, 1); return 0 }
55 var m: i64 = v; if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
56 let d: *u8 = sys_mmap(24); var k: i64 = 0
57 while m > 0 { d[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
58 let o: *u8 = sys_mmap(24); var w: i64 = 0
59 while w < k { o[w] = d[k-1-w]; w = w + 1 }
60 sys_write(1, o, k)
61 sys_munmap(d, 24); sys_munmap(o, 24)
62 return 0
63}
64func s_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
65func s_starts(s: *u8, pre: *u8) -> i64 { var i: i64 = 0; while pre[i] != (0 as u8) { if s[i] != pre[i] { return 0 } i = i + 1 } return 1 }
66// fd-aware emit (id 1785516061): results may go to a FILE so stdout carries only the envelope.
67// Both return bytes written, so the envelope can report out_bytes without a second stat.
68func s_fputs(fd: i64, s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(fd, s, n); return n }
69func s_fputn(fd: i64, v: i64) -> i64 {
70 if v == 0 { sys_write(fd, "0" as *u8, 1); return 1 }
71 var m: i64 = v
72 var w: i64 = 0
73 if m < 0 { sys_write(fd, "-" as *u8, 1); m = 0 - m; w = 1 }
74 let d: *u8 = sys_mmap(S_NUMBUF); var k: i64 = 0
75 while m > 0 { d[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
76 let o: *u8 = sys_mmap(S_NUMBUF); var q: i64 = 0
77 while q < k { o[q] = d[k-1-q]; q = q + 1 }
78 sys_write(fd, o, k)
79 sys_munmap(d, S_NUMBUF); sys_munmap(o, S_NUMBUF)
80 return w + k
81}
82// out= turns a READ-ONLY tool into a writer, so the target is constrained BY CONSTRUCTION rather
83// than by convention: no absolute path, no .. traversal, and the name MUST end in .out -- which
84// makes clobbering any source (.nx), registry (.tsv/.conf) or binary (.elf) structurally impossible.
85func s_out_ok(p: *u8) -> i64 {
86 let n: i64 = s_slen(p)
87 if n < S_OUT_MINLEN { return 0 }
88 if p[0] == (S_SLASH as u8) { return 0 }
89 var i: i64 = 0
90 while i + 1 < n { if p[i] == (S_DOT as u8) { if p[i+1] == (S_DOT as u8) { return 0 } } i = i + 1 }
91 if p[n-4] != (S_DOT as u8) { return 0 }
92 if p[n-3] != (S_CH_O as u8) { return 0 }
93 if p[n-2] != (S_CH_U as u8) { return 0 }
94 if p[n-1] != (S_CH_T as u8) { return 0 }
95 return 1
96}
97func s_streq(a: *u8, b: *u8) -> i64 { var i: i64 = 0; while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 } if b[i] != (0 as u8) { return 0 } return 1 }
98
99// bounded whole-file read
100func s_read(path: *u8, buf: *u8, cap: i64) -> i64 {
101 let fd: i64 = sys_openat_rd(path)
102 if fd < 0 { return 0 - 1 }
103 var tot: i64 = 0
104 var n: i64 = sys_read(fd, buf, cap)
105 while n > 0 { tot = tot + n; if tot >= cap { n = 0 } else { n = sys_read(fd, (buf as i64 + tot) as *u8, cap - tot) } }
106 sys_close(fd)
107 return tot
108}
109
110// first integer after <key> in buf (key includes the '='), -1 if absent/empty
111func s_num_after(buf: *u8, n: i64, key: *u8) -> i64 {
112 let kn: i64 = s_slen(key)
113 var i: i64 = 0
114 while i + kn <= n {
115 var j: i64 = 0
116 var ok: i64 = 1
117 while j < kn { if buf[i+j] != key[j] { ok = 0; j = kn } else { j = j + 1 } }
118 if ok == 1 {
119 var p: i64 = i + kn
120 var v: i64 = 0
121 var any: i64 = 0
122 var go: i64 = 1
123 while go == 1 {
124 go = 0
125 if p < n {
126 let c: i64 = buf[p] as i64
127 if c >= 48 { if c <= 57 { v = v * 10 + (c - 48); any = 1; p = p + 1; go = 1 } }
128 }
129 }
130 if any == 1 { return v }
131 return 0 - 1
132 }
133 i = i + 1
134 }
135 return 0 - 1
136}
137
138func s_ends_with(name: *u8, nn: i64, ext: *u8, en: i64) -> i64 {
139 if en == 0 { return 1 }
140 if en > nn { return 0 }
141 var i: i64 = 0
142 while i < en { if name[nn-en+i] != ext[i] { return 0 } i = i + 1 }
143 return 1
144}
145
146// iterative glob match with '*' (any sequence) + literal chars; backtracking. 1 if match.
147func s_glob(pat: *u8, pn: i64, str: *u8, sn: i64) -> i64 {
148 var pi: i64 = 0
149 var si: i64 = 0
150 var star: i64 = 0 - 1
151 var mark: i64 = 0
152 var run: i64 = 1
153 while run == 1 {
154 if si < sn {
155 var advanced: i64 = 0
156 if pi < pn { if pat[pi] == (42 as u8) { star = pi; mark = si; pi = pi + 1; advanced = 1 } }
157 if advanced == 0 { if pi < pn { if pat[pi] == str[si] { pi = pi + 1; si = si + 1; advanced = 1 } } }
158 if advanced == 0 { if star >= 0 { pi = star + 1; mark = mark + 1; si = mark; advanced = 1 } }
159 if advanced == 0 { return 0 }
160 } else { run = 0 }
161 }
162 while pi < pn { if pat[pi] == (42 as u8) { pi = pi + 1 } else { return 0 } }
163 return 1
164}
165
166// the sticky budget check: bud[0]=deadline_abs_us bud[1]=bytes_left bud[2]=files_left bud[3]=exceeded
167// bud[4]=bytes_read bud[5]=files_seen bud[6]=max_bytes_cfg bud[7]=deadline_ms_cfg bud[8]=skipped_toolong
168func s_bud_ok(bud: *i64) -> i64 {
169 if bud[3] == 1 { return 0 }
170 if bud[1] <= 0 { bud[3] = 1; return 0 }
171 if bud[2] <= 0 { bud[3] = 1; return 0 }
172 if sys_clock_now_us() > bud[0] { bud[3] = 1; return 0 }
173 return 1
174}
175
176// grep one file into the SHARED gbuf (no per-file mmap -- the seq1292 leak). Budget-accounted.
177// Returns matching-line count (the honesty count -- 0-matches must be distinguishable).
178func s_grep_file(path: *u8, pat: *u8, pn: i64, gbuf: *u8, bud: *i64) -> i64 {
179 if s_bud_ok(bud) == 0 { return 0 }
180 let n: i64 = s_read(path, gbuf, 1048575)
181 if n <= 0 { return 0 }
182 bud[1] = bud[1] - n
183 bud[4] = bud[4] + n
184 var hits: i64 = 0
185 var ln: i64 = 1
186 var ls: i64 = 0
187 while ls < n {
188 var le: i64 = ls
189 var go: i64 = 1
190 while go == 1 { go = 0; if le < n { if gbuf[le] != (10 as u8) { le = le + 1; go = 1 } } }
191 if nx_grep_any((gbuf as i64 + ls) as *u8, le - ls, pat, pn) == 1 {
192 let ofd: i64 = bud[S_BUD_OFD]
193 var wb: i64 = s_fputs(ofd, path)
194 sys_write(ofd, ":" as *u8, 1); wb = wb + 1
195 wb = wb + s_fputn(ofd, ln)
196 sys_write(ofd, ":" as *u8, 1); wb = wb + 1
197 sys_write(ofd, (gbuf as i64 + ls) as *u8, le - ls); wb = wb + (le - ls)
198 sys_write(ofd, "\n" as *u8, 1); wb = wb + 1
199 bud[S_BUD_OBYTES] = bud[S_BUD_OBYTES] + wb
200 hits = hits + 1
201 }
202 ln = ln + 1
203 ls = le + 1
204 }
205 return hits
206}
207
208// recursive walk under budget. mode: 0 grep, 1 glob, 2 find. Frees its own level buffers (munmap).
209// Returns the total match count across the subtree (files matched, or grep lines matched).
210func s_walk(dir: *u8, mode: i64, pat: *u8, pn: i64, ext: *u8, en: i64, gbuf: *u8, bud: *i64) -> i64 {
211 let fd: i64 = sys_openat_rd(dir)
212 if fd < 0 { return 0 }
213 let dbuf: *u8 = sys_mmap(65536)
214 let path: *u8 = sys_mmap(4096)
215 let dl: i64 = s_slen(dir)
216 var hits: i64 = 0
217 var run: i64 = 1
218 while run == 1 {
219 if s_bud_ok(bud) == 0 { run = 0 } else {
220 let nn: i64 = sys_getdents64(fd, dbuf, 65536)
221 if nn <= 0 { run = 0 } else {
222 var off: i64 = 0
223 while off < nn {
224 let rec: *u8 = ((dbuf as i64 + off) as *u8)
225 let reclen: i64 = dirent_reclen(rec)
226 if reclen <= 0 { off = nn } else {
227 if s_bud_ok(bud) == 0 { off = nn } else {
228 let name: *u8 = dirent_name(rec)
229 let dt: i64 = dirent_type(rec)
230 var skip: i64 = 0
231 if name[0] == (46 as u8) { if name[1] == (0 as u8) { skip = 1 } else { if name[1] == (46 as u8) { if name[2] == (0 as u8) { skip = 1 } } } }
232 if skip == 0 {
233 let nl: i64 = s_slen(name)
234 if dl + 1 + nl + 1 >= 4096 { bud[8] = bud[8] + 1 } else {
235 // build dir + "/" + name (bounds proven above)
236 var o: i64 = 0
237 var a: i64 = 0
238 while dir[a] != (0 as u8) { path[o] = dir[a]; o = o + 1; a = a + 1 }
239 path[o] = 47 as u8; o = o + 1
240 a = 0
241 while name[a] != (0 as u8) { path[o] = name[a]; o = o + 1; a = a + 1 }
242 path[o] = 0 as u8
243 if dt == 4 { hits = hits + s_walk(path, mode, pat, pn, ext, en, gbuf, bud) } else {
244 bud[2] = bud[2] - 1
245 bud[5] = bud[5] + 1
246 if mode == 0 { if s_ends_with(name, nl, ext, en) == 1 { hits = hits + s_grep_file(path, pat, pn, gbuf, bud) } }
247 if mode == 1 { if s_glob(pat, pn, name, nl) == 1 { let og: i64 = bud[S_BUD_OFD]; bud[S_BUD_OBYTES] = bud[S_BUD_OBYTES] + s_fputs(og, path) + 1; sys_write(og, "\n" as *u8, 1); hits = hits + 1 } }
248 if mode == 2 { if nx_grep_any(name, nl, pat, pn) == 1 { let of2: i64 = bud[S_BUD_OFD]; bud[S_BUD_OBYTES] = bud[S_BUD_OBYTES] + s_fputs(of2, path) + 1; sys_write(of2, "\n" as *u8, 1); hits = hits + 1 } }
249 }
250 }
251 }
252 off = off + reclen
253 }
254 }
255 }
256 }
257 }
258 }
259 sys_close(fd)
260 sys_munmap(dbuf, 65536)
261 sys_munmap(path, 4096)
262 return hits
263}
264
265// dir must open AND getdents -- a file or missing path is a LOUD error, never a silent empty
266// (the MCP capture is stdout, so the error goes to stdout + a distinct exit code).
267func s_dir_ok(dir: *u8) -> i64 {
268 let fd: i64 = sys_openat_rd(dir)
269 if fd < 0 { return 0 }
270 let probe: *u8 = sys_mmap(S_PROBE_CAP)
271 let nn: i64 = sys_getdents64(fd, probe, S_PROBE_CAP)
272 sys_close(fd)
273 sys_munmap(probe, S_PROBE_CAP)
274 if nn < 0 { return 0 }
275 return 1
276}
277
278func main(argc: i64, argv: *i64) -> i64 {
279 if argc < 4 { s_puts("usage: nx_shelltool grep <pattern> <dir> [ext] | glob <pat> <dir> | find <substr> <dir>\n" as *u8); return 2 }
280 let verb: *u8 = argv[1] as *u8
281 let a2: *u8 = argv[2] as *u8
282 let a3: *u8 = argv[3] as *u8
283 var mode: i64 = 0 - 1
284 if s_streq(verb, "grep" as *u8) == 1 { mode = 0 }
285 if s_streq(verb, "glob" as *u8) == 1 { mode = 1 }
286 if s_streq(verb, "find" as *u8) == 1 { mode = 2 }
287 if mode < 0 { s_puts("usage: nx_shelltool grep <pattern> <dir> [ext] | glob <pat> <dir> | find <substr> <dir>\n" as *u8); return 2 }
288 if s_dir_ok(a3) == 0 {
289 s_puts("NX-SHELLTOOL ERROR: not a readable directory: " as *u8)
290 s_puts(a3)
291 s_puts(" (verbs take <pattern> <dir>; to search ONE file pass its parent dir + ext)\n" as *u8)
292 return S_EXIT_BADDIR
293 }
294 // budget: shelltool_budget.conf in CWD (max_bytes= deadline_ms= max_files=) else derived defaults
295 var maxb: i64 = S_DEF_MAX_BYTES
296 var dlms: i64 = S_DEF_DEADLINE_MS
297 var maxf: i64 = S_DEF_MAX_FILES
298 let cbuf: *u8 = sys_mmap(256)
299 let cn: i64 = s_read("shelltool_budget.conf" as *u8, cbuf, 255)
300 if cn > 0 {
301 let v1: i64 = s_num_after(cbuf, cn, "max_bytes=" as *u8)
302 if v1 > 0 { maxb = v1 }
303 let v2: i64 = s_num_after(cbuf, cn, "deadline_ms=" as *u8)
304 if v2 > 0 { dlms = v2 }
305 let v3: i64 = s_num_after(cbuf, cn, "max_files=" as *u8)
306 if v3 > 0 { maxf = v3 }
307 }
308 sys_munmap(cbuf, 256)
309 let bud: *i64 = sys_mmap(96) as *i64
310 bud[0] = sys_clock_now_us() + dlms * 1000
311 bud[1] = maxb
312 bud[2] = maxf
313 bud[3] = 0
314 bud[4] = 0
315 bud[5] = 0
316 bud[6] = maxb
317 bud[7] = dlms
318 bud[8] = 0
319 var ext: *u8 = "" as *u8
320 var en: i64 = 0
321 if mode == 0 { if argc >= 5 { let c4: *u8 = argv[4] as *u8; if s_starts(c4, "out=" as *u8) == 0 { ext = c4; en = s_slen(ext) } } }
322 // out=<path>.out is a KEY=VALUE token accepted in ANY trailing position, so it can never be
323 // confused with grep's positional [ext] and all three verbs take it identically.
324 // Rule 19: absent out= is byte-for-byte the old stdout behaviour.
325 var outp: *u8 = 0 as *u8
326 var ai: i64 = 4
327 while ai < argc { let av: *u8 = argv[ai] as *u8; if s_starts(av, "out=" as *u8) == 1 { outp = ((av as i64) + S_OUTKEY_LEN) as *u8 } ai = ai + 1 }
328 var ofd: i64 = S_FD_STDOUT
329 if (outp as i64) != 0 {
330 if s_out_ok(outp) == 0 {
331 s_puts("NX-SHELLTOOL ERROR: out= refused: " as *u8); s_puts(outp)
332 s_puts(" (must be relative, contain no .., and end in .out -- so a scan can never clobber source, registry or binary)\n" as *u8)
333 return S_EXIT_BADOUT
334 }
335 ofd = sys_openat_wr(outp, S_MODE_644)
336 if ofd < 0 {
337 s_puts("NX-SHELLTOOL ERROR: out= unwritable: " as *u8); s_puts(outp); s_puts("\n" as *u8)
338 return S_EXIT_BADOUT
339 }
340 }
341 bud[S_BUD_OFD] = ofd
342 bud[S_BUD_OBYTES] = 0
343 let gbuf: *u8 = sys_mmap(1048576)
344 let hits: i64 = s_walk(a3, mode, a2, s_slen(a2), ext, en, gbuf, bud)
345 s_puts("-- matches=" as *u8); s_putn(hits)
346 s_puts(" bytes=" as *u8); s_putn(bud[4])
347 s_puts(" files=" as *u8); s_putn(bud[5])
348 if bud[8] > 0 { s_puts(" skipped_toolong=" as *u8); s_putn(bud[8]) }
349 if ofd != S_FD_STDOUT {
350 sys_close(ofd)
351 s_puts(" out=" as *u8); s_puts(outp)
352 s_puts(" out_bytes=" as *u8); s_putn(bud[S_BUD_OBYTES])
353 s_puts(" stdout_bounded=1" as *u8)
354 }
355 sys_write(1, "\n" as *u8, 1)
356 if bud[3] == 1 {
357 s_puts("NX-SHELLTOOL BUDGET-EXCEEDED partial=1 max_bytes=" as *u8); s_putn(bud[6])
358 s_puts(" deadline_ms=" as *u8); s_putn(bud[7])
359 s_puts(" (PARTIAL scan; raise shelltool_budget.conf deliberately, never remove the budget)\n" as *u8)
360 return S_EXIT_BUDGET
361 }
362 return 0
363}