nx_txtscan.nx source
↩ module page · 223 lines · 8657 B
1// nx_txtscan.nx -- CLI over nx_txtscan_lib: sovereign, BYTE-ORIENTED text search.
2//
3// The estate already had two sovereign greps (nx_codegrep, nx_shelltool grep) and both are
4// line-oriented, because both reimplemented grep's worldview rather than only its independence.
5// On knowledge/library/write_nsfwbot_flowgpt_2601_14324.txt -- 112,755 bytes, ONE line, which is
6// what scraped text normally looks like -- asking for a pattern returns the whole file as a single
7// "match". This organ reports OFFSETS, which every byte stream has, instead of lines, which only
8// some text has.
9//
10// nx_txtscan find <path> <pattern> [ctx] every hit: offset + a clamped context window
11// nx_txtscan count <path> <pattern> occurrences (overlapping)
12// nx_txtscan ctl <path> non-tab/LF/CR control bytes + run lengths
13//
14// `ctl` is not a bonus mode: it is the capability that found 211 corrupted sources in the local
15// tree, each carrying one 119-byte run of 0x0E where a migration should have rewritten a number
16// formatter. nx_cc accepts 0x0E as whitespace, so every one of those files compiles clean.
17//
18// u2605HONEST TRUNCATION IS PART OF THE CONTRACT. This organ prints `truncated=1` with the counts
19// whenever it did not see everything -- because a capped listing that reports absence is a silent
20// wrong answer, and that exact defect cost this session a job that sat enqueued and ignored while
21// a worker reported "queue empty".
22// license_tier: ORIGINAL
23// module: nishi-core.text.scan.cli
24// capability: TEXT_SCAN_BYTE_ORIENTED
25import "nx_estate_path.nx"
26import "nx_itoa_lib.nx"
27import "nx_txtscan_lib.nx"
28
29// Read budget. PROVENANCE, not a guess: the largest text artifact measured in this estate is the
30// 112,755-byte ruler corpus, and .nx sources run far smaller; 32 MiB is ~300x that in a single
31// mmap. Oversize input is REPORTED, never silently shrunk (the memfloor law: refuse or say so).
32const TS_BUF_BYTES: i64 = 33554432
33
34// How many hits are PRINTED. The total is always counted and always printed, so capping the
35// display can never be mistaken for a smaller result set -- the failure mode of the 200-entry
36// listing cap that hid an enqueued job earlier in this session (debt 1785932638).
37const TS_SHOW_MAX: i64 = 64
38
39const TS_CTX_DEFAULT: i64 = 60
40const TS_CTX_MAX: i64 = 400
41
42func tw(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
43func te(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(2, s, n); return 0 }
44
45// Print buf[lo,hi) with every control byte rendered as '.', so a binary window cannot mangle the
46// terminal or forge a newline that makes one hit look like several.
47func ts_show(buf: *u8, lo: i64, hi: i64) -> i64 {
48 let one: *u8 = sys_mmap(8)
49 var i: i64 = lo
50 while i < hi {
51 let c: i64 = buf[i] as i64
52 if tx_is_ctl(c) == 1 { one[0] = 46 as u8 } else { one[0] = buf[i] }
53 sys_write(1, one, 1)
54 i = i + 1
55 }
56 sys_munmap(one, 8)
57 return 0
58}
59
60// Slurp up to cap bytes. more[0] is set to 1 when the file had MORE than we read.
61func ts_slurp(path: *u8, buf: *u8, cap: i64, more: *i64) -> i64 {
62 more[0] = 0
63 let fd: i64 = ep_open_rd(path)
64 if fd < 0 { return 0 - 1 }
65 var t: i64 = 0
66 var go: i64 = 1
67 while go == 1 {
68 if t >= cap { go = 0 } else {
69 let r: i64 = sys_read(fd, ((buf as i64) + t) as *u8, cap - t)
70 if r <= 0 { go = 0 } else { t = t + r }
71 }
72 }
73 // Detect "there was more" WITHOUT a stat: one more read after the buffer is full.
74 if t >= cap {
75 let probe: *u8 = sys_mmap(8)
76 if sys_read(fd, probe, 1) > 0 { more[0] = 1 }
77 sys_munmap(probe, 8)
78 }
79 sys_close(fd)
80 return t
81}
82
83func ts_note_trunc(more: i64, n: i64) -> i64 {
84 if more == 1 {
85 tw(" truncated=1 scanned=" as *u8); nxi_out(n)
86 tw(" -- the file is LARGER than the read budget; a hit past this point was NOT looked for\n" as *u8)
87 }
88 return 0
89}
90
91func ts_find(path: *u8, pat: *u8, ctx: i64) -> i64 {
92 let buf: *u8 = sys_mmap(TS_BUF_BYTES)
93 let more: *i64 = sys_mmap(16) as *i64
94 let n: i64 = ts_slurp(path, buf, TS_BUF_BYTES, more)
95 if n < 0 { te("nx_txtscan: cannot open " as *u8); te(path); te("\n" as *u8); return 3 }
96 let m: i64 = tx_len(pat)
97 if m == 0 { te("nx_txtscan: empty pattern matches nothing (by design)\n" as *u8); return 4 }
98
99 var shown: i64 = 0
100 var total: i64 = 0
101 var p: i64 = 0
102 var go: i64 = 1
103 while go == 1 {
104 let h: i64 = tx_next(buf, n, p, pat, m, TX_CASE_INSENSITIVE)
105 if h == TX_NOT_FOUND { go = 0 } else {
106 total = total + 1
107 if shown < TS_SHOW_MAX {
108 shown = shown + 1
109 tw("off=" as *u8); nxi_out(h); tw(" " as *u8)
110 ts_show(buf, tx_ctx_lo(h, ctx), tx_ctx_hi(h, m, ctx, n))
111 tw("\n" as *u8)
112 }
113 p = h + 1
114 }
115 }
116 tw("-- shown=" as *u8); nxi_out(shown)
117 tw(" total=" as *u8); nxi_out(total)
118 tw(" bytes=" as *u8); nxi_out(n)
119 if total > shown { tw(" truncated_display=1" as *u8) }
120 tw("\n" as *u8)
121 ts_note_trunc(more[0], n)
122 if total == 0 { return 1 }
123 return 0
124}
125
126func ts_count(path: *u8, pat: *u8) -> i64 {
127 let buf: *u8 = sys_mmap(TS_BUF_BYTES)
128 let more: *i64 = sys_mmap(16) as *i64
129 let n: i64 = ts_slurp(path, buf, TS_BUF_BYTES, more)
130 if n < 0 { te("nx_txtscan: cannot open " as *u8); te(path); te("\n" as *u8); return 3 }
131 let c: i64 = tx_count(buf, n, pat, tx_len(pat), TX_CASE_INSENSITIVE, TX_STEP_OVERLAP)
132 tw("count=" as *u8); nxi_out(c)
133 tw(" bytes=" as *u8); nxi_out(n); tw("\n" as *u8)
134 ts_note_trunc(more[0], n)
135 if c == 0 { return 1 }
136 return 0
137}
138
139// Report every run of non-tab/LF/CR control bytes: offset, byte value, run length.
140func ts_ctl(path: *u8) -> i64 {
141 let buf: *u8 = sys_mmap(TS_BUF_BYTES)
142 let more: *i64 = sys_mmap(16) as *i64
143 let n: i64 = ts_slurp(path, buf, TS_BUF_BYTES, more)
144 if n < 0 { te("nx_txtscan: cannot open " as *u8); te(path); te("\n" as *u8); return 3 }
145
146 var runs: i64 = 0
147 var bytes: i64 = 0
148 var p: i64 = 0
149 var go: i64 = 1
150 while go == 1 {
151 let h: i64 = tx_next_ctl(buf, n, p)
152 if h == TX_NOT_FOUND { go = 0 } else {
153 let rl: i64 = tx_run_len(buf, n, h)
154 runs = runs + 1
155 bytes = bytes + rl
156 if runs <= TS_SHOW_MAX {
157 tw("ctl off=" as *u8); nxi_out(h)
158 tw(" byte=0x" as *u8)
159 let v: i64 = buf[h] as i64
160 let hexd: *u8 = "0123456789abcdef" as *u8
161 let o: *u8 = sys_mmap(8)
162 o[0] = hexd[(v / 16) % 16]; o[1] = hexd[v % 16]
163 sys_write(1, o, 2)
164 sys_munmap(o, 8)
165 tw(" runlen=" as *u8); nxi_out(rl); tw("\n" as *u8)
166 }
167 p = h + rl
168 }
169 }
170 tw("-- ctl_runs=" as *u8); nxi_out(runs)
171 tw(" ctl_bytes=" as *u8); nxi_out(bytes)
172 tw(" bytes=" as *u8); nxi_out(n)
173 if runs == 0 { tw(" verdict=CLEAN" as *u8) } else { tw(" verdict=CORRUPT" as *u8) }
174 tw("\n" as *u8)
175 ts_note_trunc(more[0], n)
176 if runs == 0 { return 0 }
177 return 1
178}
179
180func ts_streq(a: *u8, b: *u8) -> i64 {
181 var i: i64 = 0
182 while a[i] != (0 as u8) { if a[i] != b[i] { return 0 } i = i + 1 }
183 if b[i] != (0 as u8) { return 0 }
184 return 1
185}
186
187func ts_atoi(s: *u8) -> i64 {
188 var v: i64 = 0
189 var i: i64 = 0
190 while s[i] != (0 as u8) {
191 let c: i64 = s[i] as i64
192 if c < 48 { return v }
193 if c > 57 { return v }
194 v = v * 10 + (c - 48)
195 i = i + 1
196 }
197 return v
198}
199
200func main(argc: i64, argv: *i64) -> i64 {
201 if argc < 3 {
202 tw("usage: nx_txtscan find <path> <pattern> [ctx] | count <path> <pattern> | ctl <path>\n" as *u8)
203 tw(" offsets, not lines -- works on files with no newline at all\n" as *u8)
204 return 2
205 }
206 let cmd: *u8 = argv[1] as *u8
207 let path: *u8 = argv[2] as *u8
208
209 if ts_streq(cmd, "ctl" as *u8) == 1 { return ts_ctl(path) }
210 if argc < 4 { te("nx_txtscan: needs a pattern\n" as *u8); return 2 }
211 let pat: *u8 = argv[3] as *u8
212
213 if ts_streq(cmd, "count" as *u8) == 1 { return ts_count(path, pat) }
214 if ts_streq(cmd, "find" as *u8) == 1 {
215 var ctx: i64 = TS_CTX_DEFAULT
216 if argc > 4 { ctx = ts_atoi(argv[4] as *u8) }
217 if ctx < 0 { ctx = 0 }
218 if ctx > TS_CTX_MAX { ctx = TS_CTX_MAX }
219 return ts_find(path, pat, ctx)
220 }
221 te("nx_txtscan: unknown mode (find | count | ctl)\n" as *u8)
222 return 2
223}