nx_textcut.nx source

1// nx_textcut.nx -- LIB: TOKEN-BOUNDARY-SAFE TEXT CUTTING. The estate's ONE truncation ruler. 2// 3// THE DEFECT THIS EXISTS TO REMOVE, MEASURED LIVE ON nishifamily.com/search 2026-08-25: 4// a renderer that truncates at a BYTE cap cuts mid-token. The stored doc 5// "NIST Reference Fluid Thermodynamic and Transport Properties Database (REFPROP) Version 9" 6// rendered a title ending "...Database (RE" (72-byte cap) and a snippet BEGINNING "FPROP) Version 9". 7// TWO independent caps in one renderer (title 72B, sentence 320B) both cut mid-word. 8// 9// WHY IT SURVIVED: the damage is INVISIBLE AT THE CUT SITE. A title ending "(RE" looks merely 10// truncated -- plausible. The mid-word START it creates surfaces in the NEXT span, which reads as 11// a snippet bug, so every investigation looked at the snippet code and found nothing wrong with it. 12// A CUT IS A CLAIM ABOUT WHERE A TOKEN ENDS; this lib makes that claim checkable and never 13// silently wrong -- every cut lands on a real boundary or ANNOUNCES that it could not. 14// 15// ALSO CLOSES A DECLARED GAP ELSEWHERE: nx_search_snippet_extract.nx names both 16// "UTF-8 codepoint awareness (no mid-codepoint truncation)" and "Word-boundary clipping" in its 17// V2 SCOPE (TODO) and is unwired. Treating every byte >= 128 as a token byte closes BOTH at once: 18// a multi-byte codepoint can never be split, because a cut inside one is a cut inside a token. 19// 20// NO THRESHOLDS. This lib is pure STRUCTURE (what a token is), not POLICY (how much to show), so 21// it carries no conf and nothing here is tunable -- the cap is always the CALLER's. 22// license_tier: ORIGINAL 23import "nx_syscalls.nx" 24 25// ASCII class boundaries -- character DEFINITIONS, named per rule 11 (house pattern: nx_block_density BD_LT/BD_GT). 26const TC_CH_TAB: i64 = 9 27const TC_CH_LF: i64 = 10 28const TC_CH_CR: i64 = 13 29const TC_CH_SP: i64 = 32 30const TC_CH_D0: i64 = 48 31const TC_CH_D9: i64 = 57 32const TC_CH_UA: i64 = 65 33const TC_CH_UZ: i64 = 90 34const TC_CH_US: i64 = 95 35const TC_CH_LA: i64 = 97 36const TC_CH_LZ: i64 = 122 37const TC_CH_HIGH: i64 = 128 // first non-ASCII byte: lead or continuation of a UTF-8 codepoint 38// bytes that OPEN or CONNECT -- see tc_is_dangling. Named rather than spelled as a literal set 39// because the nx lexer forbids '#' inside a string literal, so a character-class STRING is not even 40// expressible here; consts are the only honest way to write this set down. 41const TC_CH_DQUOTE: i64 = 34 42const TC_CH_HASH: i64 = 35 43const TC_CH_AMP: i64 = 38 44const TC_CH_SQUOTE: i64 = 39 45const TC_CH_LPAREN: i64 = 40 46const TC_CH_PLUS: i64 = 43 47const TC_CH_COMMA: i64 = 44 48const TC_CH_HYPHEN: i64 = 45 49const TC_CH_SLASH: i64 = 47 50const TC_CH_LT: i64 = 60 51const TC_CH_EQ: i64 = 61 52const TC_CH_AT: i64 = 64 53const TC_CH_LBRACK: i64 = 91 54const TC_CH_LBRACE: i64 = 123 55const TC_CH_PIPE: i64 = 124 56 57// ANNOUNCE, NEVER INFER: a consumer that wants to report its own imprecision reads these 58// instead of assuming every cut was clean. (Same contract as nx_block_density's bd_last_* stats.) 59static tc_cuts_g: i64 60static tc_hardcuts_g: i64 61 62func tc_stats_reset() -> i64 { tc_cuts_g = 0; tc_hardcuts_g = 0; return 0 } 63func tc_cuts() -> i64 { return tc_cuts_g } 64func tc_hardcuts() -> i64 { return tc_hardcuts_g } 65 66// A TOKEN BYTE. Alphanumeric, underscore, or ANY byte >= 128. 67// The >= 128 arm is load-bearing and is not a shortcut: UTF-8 lead and continuation bytes are all 68// >= 128, so refusing to cut between two of them makes a mid-codepoint cut structurally impossible. 69func tc_is_wordch(c: i64) -> i64 { 70 if c >= TC_CH_D0 { if c <= TC_CH_D9 { return 1 } } 71 if c >= TC_CH_UA { if c <= TC_CH_UZ { return 1 } } 72 if c >= TC_CH_LA { if c <= TC_CH_LZ { return 1 } } 73 if c == TC_CH_US { return 1 } 74 if c >= TC_CH_HIGH { return 1 } 75 return 0 76} 77 78func tc_is_ws(c: i64) -> i64 { 79 if c == TC_CH_SP { return 1 } 80 if c == TC_CH_LF { return 1 } 81 if c == TC_CH_CR { return 1 } 82 if c == TC_CH_TAB { return 1 } 83 return 0 84} 85 86// A UTF-8 CONTINUATION BYTE (10xxxxxx = 128..191). A cut landing ON one of these is a cut INSIDE a 87// codepoint, which mangles the final character. The token rule above normally makes this impossible, 88// but it CANNOT when the whole span is one unbroken token -- and that is exactly the hard-cut path. 89// Found by this lib's own gate on its first run: the fixture 'a','a',0xC3,0xA9,'a' at cap 3 has no 90// token boundary at all, so the hard cut returned cap and split the codepoint. 91const TC_CH_CONT_HI: i64 = 191 92func tc_is_utf8_cont(c: i64) -> i64 { 93 if c >= TC_CH_HIGH { if c <= TC_CH_CONT_HI { return 1 } } 94 return 0 95} 96 97// THE PREDICATE EVERYTHING ELSE IS BUILT ON: would cutting [0,n) at `at` split a token in half? 98// Cutting at 0 or at n is never a split -- those are the ends of the span, not a position inside a token. 99func tc_splits_token(txt: *u8, n: i64, at: i64) -> i64 { 100 if at <= 0 { return 0 } 101 if at >= n { return 0 } 102 if tc_is_wordch(txt[at - 1] as i64) == 0 { return 0 } 103 if tc_is_wordch(txt[at] as i64) == 0 { return 0 } 104 return 1 105} 106 107// A DANGLING BYTE: one that opens or connects and therefore cannot be the last byte of a span that 108// was TRUNCATED. Cutting "...Database (REFPROP)" at the longest non-splitting boundary yields 109// "...Database (" -- not mid-word, but still visibly broken. These are trimmed ONLY when the cut 110// actually truncated: an author's own trailing '.' or ')' on a span that fit is left exactly alone. 111func tc_is_dangling(c: i64) -> i64 { 112 if c == TC_CH_LPAREN { return 1 } 113 if c == TC_CH_LBRACK { return 1 } 114 if c == TC_CH_LBRACE { return 1 } 115 if c == TC_CH_LT { return 1 } 116 if c == TC_CH_DQUOTE { return 1 } 117 if c == TC_CH_SQUOTE { return 1 } 118 if c == TC_CH_HYPHEN { return 1 } 119 if c == TC_CH_SLASH { return 1 } 120 if c == TC_CH_AMP { return 1 } 121 if c == TC_CH_AT { return 1 } 122 if c == TC_CH_HASH { return 1 } 123 if c == TC_CH_EQ { return 1 } 124 if c == TC_CH_PLUS { return 1 } 125 if c == TC_CH_COMMA { return 1 } 126 if c == TC_CH_PIPE { return 1 } 127 return 0 128} 129 130// THE CUT. Returns the LONGEST length L <= min(n, cap) that does not split a token. 131// Longest, not prettiest: a rule that walked back to the previous SPACE would be simpler but would 132// throw away up to a whole cap's worth of text on a run with no spaces in it, and a shorter answer 133// is not a safer one -- it is just less of the document. 134// exactbox[0] = 1 -> the cut landed on a real boundary. 135// exactbox[0] = 0 -> the span is ONE unbroken token longer than cap (a 300-char URL, a hash, a 136// CJK run) and the cut HAD to be hard. DECLARED, never hidden: a caller that 137// cares can render an ellipsis, and the counters above make the rate measurable. 138func tc_cut(txt: *u8, n: i64, cap: i64, exactbox: *i64) -> i64 { 139 exactbox[0] = 1 140 if cap <= 0 { return 0 } 141 if n <= cap { return n } 142 tc_cuts_g = tc_cuts_g + 1 143 var l: i64 = cap 144 var go: i64 = 1 145 while go == 1 { 146 if l <= 0 { go = 0 } else { 147 if tc_splits_token(txt, n, l) == 0 { return l } 148 l = l - 1 149 } 150 } 151 // HARD CUT: no token boundary exists anywhere in [1, cap], so we MUST cut inside a token. We must 152 // still never cut inside a CHARACTER: back off any continuation byte so the cut lands on a codepoint 153 // boundary. This is a strictly WEAKER guarantee than the token rule, and it is declared as such -- 154 // exactbox stays 0 and the hardcut counter still fires, because the caller's text really was severed. 155 var hl: i64 = cap 156 var hg: i64 = 1 157 while hg == 1 { 158 if hl <= 0 { hg = 0 } else { 159 if tc_is_utf8_cont(txt[hl] as i64) == 1 { hl = hl - 1 } else { hg = 0 } 160 } 161 } 162 if hl <= 0 { hl = cap } 163 exactbox[0] = 0 164 tc_hardcuts_g = tc_hardcuts_g + 1 165 return hl 166} 167 168// tc_cut, then -- ONLY IF IT TRUNCATED -- drop trailing whitespace and dangling opener/connector 169// bytes. The truncation test is n > cap, checked BEFORE the cut, so a span that fit is returned 170// byte-for-byte and this function is a no-op on it. 171func tc_cut_trim(txt: *u8, n: i64, cap: i64, exactbox: *i64) -> i64 { 172 var truncated: i64 = 0 173 if n > cap { truncated = 1 } 174 var l: i64 = tc_cut(txt, n, cap, exactbox) 175 var go: i64 = 1 176 while go == 1 { 177 if l <= 0 { go = 0 } else { 178 let c: i64 = txt[l - 1] as i64 179 var drop: i64 = 0 180 if tc_is_ws(c) == 1 { drop = 1 } 181 if drop == 0 { if truncated == 1 { if tc_is_dangling(c) == 1 { drop = 1 } } } 182 if drop == 1 { l = l - 1 } else { go = 0 } 183 } 184 } 185 return l 186} 187 188// THE OTHER HALF OF THE DEFECT. Advance `off` to the start of a whole token: when off lands INSIDE 189// a token -- because some earlier cap cut there -- skip the remainder of that token, then skip 190// whitespace. This is what turns a snippet reading "FPROP) Version 9" into "Version 9". 191// Uses an explicit flag to leave each loop; never writes the exit sentinel into the cursor itself 192// (that idiom erases the answer, and nx_srclint hunts it). 193func tc_start(txt: *u8, n: i64, off: i64) -> i64 { 194 var i: i64 = off 195 if i < 0 { i = 0 } 196 if i >= n { return n } 197 var wassplit: i64 = 0 198 if tc_splits_token(txt, n, i) == 1 { wassplit = 1 } 199 if wassplit == 1 { 200 // skip the amputated remainder of the token we landed inside ("FPROP") 201 var g1: i64 = 1 202 while g1 == 1 { 203 if i >= n { g1 = 0 } else { 204 if tc_is_wordch(txt[i] as i64) == 1 { i = i + 1 } else { g1 = 0 } 205 } 206 } 207 // ...and then the punctuation that amputation ORPHANED (the ')' of a '(...)' whose '(' was cut 208 // away, giving ") Version 9"). This runs ONLY on the split path -- a span that began cleanly 209 // keeps its own leading quote or bracket, which is content and not an artifact. 210 var g2: i64 = 1 211 while g2 == 1 { 212 if i >= n { g2 = 0 } else { 213 if tc_is_wordch(txt[i] as i64) == 1 { g2 = 0 } else { i = i + 1 } 214 } 215 } 216 return i 217 } 218 return tc_skip_ws(txt, n, i) 219} 220 221// skip whitespace only (no token logic) -- the common prefix trim. 222func tc_skip_ws(txt: *u8, n: i64, off: i64) -> i64 { 223 var i: i64 = off 224 if i < 0 { i = 0 } 225 var go: i64 = 1 226 while go == 1 { 227 if i >= n { go = 0 } else { 228 if tc_is_ws(txt[i] as i64) == 1 { i = i + 1 } else { go = 0 } 229 } 230 } 231 return i 232}