nx_txtscan_lib.nx source
↩ module page · 139 lines · 6612 B
1// nx_txtscan_lib.nx -- BYTE-ORIENTED text search. The unit of a match is an OFFSET, never a line.
2//
3// u2605u2605u2605u2605u2605u2605WE DID NOT ONLY AVOID THE UNIX TOOLS, WE INHERITED THEIR UNIT OF MEANING. Every sovereign
4// search organ in this estate -- nx_codegrep, nx_shelltool grep -- scans for a substring and then
5// prints "the line that contains it". That is grep's worldview, reimplemented in NishiLang: it
6// assumes text is a sequence of newline-delimited records.
7//
8// MEASURED 2026-08-05, and this is the bug that motivated the organ: the ruler corpus
9// knowledge/library/write_nsfwbot_flowgpt_2601_14324.txt is 112,755 bytes on ONE line (scraped HTML,
10// as most fetched text is). Asking the sovereign grep for "greeting" returned the ENTIRE FILE as a
11// single match. A search that echoes 112KB for a 8-byte pattern has not searched, it has copied --
12// and it reports "1 match" while telling you nothing about WHERE.
13// u2605A TOOL THAT DEGRADES TO THE IDENTITY FUNCTION ON ITS HARDEST INPUT STILL REPORTS SUCCESS.
14//
15// So the fix is not a faster grep, it is a different RECORD: the offset. Offsets exist in every
16// byte stream -- minified JSON, scraped HTML, a firmware image, a file with no newline at all --
17// whereas lines are a property some text happens to have. This organ imposes its own record
18// structure on the input instead of borrowing the input's.
19//
20// PURE: no syscalls, no allocation, caller owns every buffer -- so the gate can prove the search
21// hermetically with in-memory fixtures and no filesystem at all. The CLI (nx_txtscan.nx) adds I/O.
22// license_tier: ORIGINAL
23// module: nishi-core.text.scan
24// capability: TEXT_SCAN_BYTE_ORIENTED
25
26const TX_NOT_FOUND: i64 = 0 - 1
27
28// Overlap policy, passed as `step` to tx_count. OVERLAP is the default everywhere in this organ.
29// u2605"aaa" OCCURS THREE TIMES IN "aaaaa", NOT ONCE. Disjoint counting silently under-reports, and an
30// under-report is indistinguishable from a clean file -- exactly the failure mode that lets a
31// corruption sweep come back empty. Disjoint remains available because a de-duplicating caller
32// (replace-all) genuinely wants it; it is never the default.
33const TX_STEP_OVERLAP: i64 = 1
34const TX_STEP_DISJOINT: i64 = 0
35
36const TX_CASE_SENSITIVE: i64 = 0
37const TX_CASE_INSENSITIVE: i64 = 1
38
39func tx_lower(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c }
40func tx_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
41
42// Does pat[0..m) sit exactly at buf[i..i+m)? The caller guarantees i+m <= n.
43func tx_match_at(buf: *u8, i: i64, pat: *u8, m: i64, ci: i64) -> i64 {
44 var j: i64 = 0
45 while j < m {
46 var a: i64 = buf[i + j] as i64
47 var b: i64 = pat[j] as i64
48 if ci == TX_CASE_INSENSITIVE { a = tx_lower(a); b = tx_lower(b) }
49 if a != b { return 0 }
50 j = j + 1
51 }
52 return 1
53}
54
55// FIRST occurrence at or after `from` within buf[0..n), else TX_NOT_FOUND.
56// u2605THE PRIMITIVE RETURNS WHERE, NOT WHETHER, AND IT MUST NOT DESTROY THE INDEX WHILE SEARCHING.
57// This estate has written the position-losing bug twice in one session (nx_cwdguard's parser set
58// q = rn + 1 and threw away the hit, reporting 1 registry row out of 775; then rp_name_shaped
59// repeated it an hour after the law was banked). The loop below advances a SEPARATE cursor and
60// returns i untouched.
61func tx_next(buf: *u8, n: i64, from: i64, pat: *u8, m: i64, ci: i64) -> i64 {
62 if m <= 0 { return TX_NOT_FOUND } // an empty needle matches everywhere = matches nothing useful
63 if n <= 0 { return TX_NOT_FOUND }
64 var i: i64 = from
65 if i < 0 { i = 0 }
66 while i + m <= n {
67 if tx_match_at(buf, i, pat, m, ci) == 1 { return i }
68 i = i + 1
69 }
70 return TX_NOT_FOUND
71}
72
73// Total occurrences in buf[0..n). See TX_STEP_OVERLAP for why overlap is the default.
74func tx_count(buf: *u8, n: i64, pat: *u8, m: i64, ci: i64, step: i64) -> i64 {
75 if m <= 0 { return 0 }
76 var c: i64 = 0
77 var p: i64 = 0
78 var go: i64 = 1
79 while go == 1 {
80 let h: i64 = tx_next(buf, n, p, pat, m, ci)
81 if h == TX_NOT_FOUND { go = 0 } else {
82 c = c + 1
83 if step == TX_STEP_OVERLAP { p = h + 1 } else { p = h + m }
84 }
85 }
86 return c
87}
88
89// ---------------------------------------------------------------------------------------------
90// BYTE-CLASS scanning. Same organ, same record (an offset) -- the predicate is what changes.
91//
92// WHY THIS LIVES HERE: it is what found the defect that prompted the whole rung. 211 .nx sources in
93// the local tree each carry exactly one 119-byte run of 0x0E (Shift Out) where a migration should
94// have rewritten a number-formatting loop. nx_cc accepts 0x0E as whitespace, so every one of those
95// files COMPILES CLEAN and the corruption is invisible to the toolchain that reads it.
96// u2605u2605u2605u2605u2605A COMPILER THAT SILENTLY ACCEPTS CONTROL BYTES MAKES CORRUPTION LOOK LIKE FORMATTING.
97// Tab/LF/CR are legitimate text; every other byte below 0x20 is not, in a source file.
98
99func tx_is_ctl(c: i64) -> i64 {
100 if c >= 32 { return 0 }
101 if c == 9 { return 0 } // tab
102 if c == 10 { return 0 } // LF
103 if c == 13 { return 0 } // CR
104 return 1
105}
106
107// FIRST control byte at or after `from`, else TX_NOT_FOUND.
108func tx_next_ctl(buf: *u8, n: i64, from: i64) -> i64 {
109 var i: i64 = from
110 if i < 0 { i = 0 }
111 while i < n {
112 if tx_is_ctl(buf[i] as i64) == 1 { return i }
113 i = i + 1
114 }
115 return TX_NOT_FOUND
116}
117
118// Length of the run of IDENTICAL bytes starting at `at`. A run length is the signature that
119// separates a stray byte from a mechanical overwrite: 211 files sharing one run length of exactly
120// 119 is a tool's fingerprint, not disk rot.
121func tx_run_len(buf: *u8, n: i64, at: i64) -> i64 {
122 if at < 0 { return 0 }
123 if at >= n { return 0 }
124 let v: i64 = buf[at] as i64
125 var i: i64 = at
126 while i < n { if (buf[i] as i64) != v { return i - at } i = i + 1 }
127 return n - at
128}
129
130// ---------------------------------------------------------------------------------------------
131// Context windows. Clamped to the buffer at BOTH ends: a match at offset 0 or at the final byte is
132// exactly where an unclamped window walks off the allocation, and those are the two offsets a
133// hand-written test is least likely to cover.
134func tx_ctx_lo(hit: i64, ctx: i64) -> i64 { let a: i64 = hit - ctx; if a < 0 { return 0 } return a }
135func tx_ctx_hi(hit: i64, m: i64, ctx: i64, n: i64) -> i64 {
136 let b: i64 = hit + m + ctx
137 if b > n { return n }
138 return b
139}