nx_textnorm_lib.nx source

1// nx_textnorm_lib.nx -- ONE COPY OF THE PRE-SHINGLING NORMALISATION DECISION. 2// 3// module: nishi-core.search.textnorm 4// capability: CORE_COMPUTE (pure; reads no conf, touches no store, writes no file) 5// license_tier: ORIGINAL No hw writes (Rule 26). 6// 7// WHY THIS EXISTS, MEASURED 2026-08-25 BY THE GATE THAT MOTIVATED IT: 8// nx_sitededup_gate fingerprints a REAL capture pair -- one document stored twice, the second copy 9// carrying an HTTP capture header and a bumped revision line -- and measured their Hamming distance 10// at SIX bits against the published k=3 bar. The collapse pass therefore did NOT collapse the very 11// duplicate class it was built for. A fingerprint taken over un-normalised bytes cannot: the header 12// and the revision id are TOKENS, and tokens move bits. 13// 14// THE ESTABLISHED FIX, and it is not ours: normalise the text BEFORE shingling, and in particular 15// normalise DIGITS TO A CONSTANT -- the step that collapses counter, date, revision and price 16// near-duplicates, which is exactly the class our store holds. Charikar (STOC 2002) defines the 17// fingerprint and Manku/Jain/Das Sarma (WWW 2007) the k=3 bar for 64-bit fingerprints; NEITHER says 18// anything about what text you hand it, and that omission is where our duplicates were surviving. 19// The normalisation is the missing half of the published recipe, not an invention of ours. 20// 21// COMPOSES, NEVER RE-IMPLEMENTS: nx_simhash for the fingerprint (there must stay exactly ONE 22// fingerprint kernel in the estate) and nx_html_to_text for markup (there must stay exactly ONE 23// markup stripper). This file adds only what neither has: the capture-header strip and the digit fold. 24// 25// WHAT IT DELIBERATELY DOES NOT DO: it does not re-implement block-density boilerplate extraction. 26// bd_fit_text (nx_block_density) is the ruler for that and belongs to the ingest lane, which already 27// applies it FORWARD. The residue actually present in already-stored rows is the HTTP capture header, 28// which is what tn_body_offset removes -- precisely, by parsing the header grammar, not by a 29// heuristic that could eat the first paragraph of a document. 30// 31// FAILS TOWARD DOING NOTHING IS NOT AVAILABLE HERE, SO SAY SO PLAINLY: every step below can only make 32// two texts MORE similar, so the direction of any error is a FALSE POSITIVE -- a document wrongly 33// judged a duplicate and hidden from search. That is why the consumer carries a neg-control of 34// genuinely different documents and measures its false-positive rate against REAL corpus documents, 35// never against fixtures written by the same hand that chose the rule. 36import "nx_syscalls.nx" 37import "nx_simhash.nx" 38import "nx_html_to_text.nx" 39 40const TN_ZERO: i64 = 48 41const TN_NINE: i64 = 57 42const TN_LT: i64 = 60 43const TN_SLASH: i64 = 47 44const TN_NL: i64 = 10 45const TN_CR: i64 = 13 46const TN_COLON: i64 = 58 47const TN_HYPHEN: i64 = 45 48const TN_UA: i64 = 65 49const TN_UZ: i64 = 90 50const TN_LA: i64 = 97 51const TN_LZ: i64 = 122 52const TN_WORD: i64 = 8 53const TN_BOX: i64 = 8 54const TN_DEC: i64 = 10 55// The capture-header sentinel, spelled as its bytes: H T T P then the slash. 56const TN_H_H: i64 = 72 57const TN_H_T: i64 = 84 58const TN_H_P: i64 = 80 59 60func tn_is_alpha(c: i64) -> i64 { 61 if c >= TN_UA { if c <= TN_UZ { return 1 } } 62 if c >= TN_LA { if c <= TN_LZ { return 1 } } 63 return 0 64} 65func tn_is_digit(c: i64) -> i64 { 66 if c >= TN_ZERO { if c <= TN_NINE { return 1 } } 67 return 0 68} 69 70// index of the newline at or after i0, or n when the line is unterminated. 71func tn_line_end(src: *u8, n: i64, i0: i64) -> i64 { 72 var i: i64 = i0 73 while i < n { 74 if (src[i] as i64) == TN_NL { return i } 75 i = i + 1 76 } 77 return n 78} 79 80// Does the line beginning at i0 have field-line shape -- a run of at least one token character 81// (letters or hyphen) followed immediately by a colon? This is a grammar test, not a guess, which 82// is what lets the strip be exact rather than heuristic. 83func tn_is_header_line(src: *u8, n: i64, i0: i64) -> i64 { 84 var i: i64 = i0 85 var len: i64 = 0 86 var go: i64 = 1 87 while go == 1 { 88 go = 0 89 if i < n { 90 let c: i64 = src[i] as i64 91 var tok: i64 = 0 92 if tn_is_alpha(c) == 1 { tok = 1 } 93 if c == TN_HYPHEN { tok = 1 } 94 if tok == 1 { len = len + 1; i = i + 1; go = 1 } 95 } 96 } 97 if len == 0 { return 0 } 98 if i >= n { return 0 } 99 if (src[i] as i64) == TN_COLON { return 1 } 100 return 0 101} 102 103// Byte offset of the first BODY byte: 0 when there is no capture header to remove. 104// Fires only on a document that literally begins with the HTTP status line, and REFUSES to strip 105// when the header grammar would swallow the whole document -- an empty result is never an 106// improvement, and a normaliser that can empty its input is a corpus-eating defect. 107func tn_body_offset(src: *u8, n: i64) -> i64 { 108 if n < 5 { return 0 } 109 if (src[0] as i64) != TN_H_H { return 0 } 110 if (src[1] as i64) != TN_H_T { return 0 } 111 if (src[2] as i64) != TN_H_T { return 0 } 112 if (src[3] as i64) != TN_H_P { return 0 } 113 if (src[4] as i64) != TN_SLASH { return 0 } 114 var i: i64 = tn_line_end(src, n, 0) 115 if i < n { i = i + 1 } 116 var go: i64 = 1 117 while go == 1 { 118 go = 0 119 if i < n { 120 let c: i64 = src[i] as i64 121 if c == TN_NL { i = i + 1 } else { 122 var blank: i64 = 0 123 if c == TN_CR { if i + 1 < n { if (src[i + 1] as i64) == TN_NL { i = i + 2; blank = 1 } } } 124 if blank == 0 { 125 if tn_is_header_line(src, n, i) == 1 { 126 i = tn_line_end(src, n, i) 127 if i < n { i = i + 1 } 128 go = 1 129 } 130 } 131 } 132 } 133 } 134 if i >= n { return 0 } 135 return i 136} 137 138// Is there real markup here? An open angle bracket followed by a letter or a slash. A bare angle 139// bracket in prose (a comparison, an arrow) is not markup and must not send a plain-text research 140// document through an HTML extractor. 141func tn_has_markup(src: *u8, n: i64) -> i64 { 142 var i: i64 = 0 143 while i + 1 < n { 144 if (src[i] as i64) == TN_LT { 145 let c: i64 = src[i + 1] as i64 146 if tn_is_alpha(c) == 1 { return 1 } 147 if c == TN_SLASH { return 1 } 148 } 149 i = i + 1 150 } 151 return 0 152} 153 154// THE DECISIVE STEP. Every ASCII digit becomes the SAME constant byte, so "revision 4821" and 155// "revision 9137" shingle to an identical token and a date, a counter, a price or a build number 156// stops being a content difference. 157// LENGTH IS PRESERVED ON PURPOSE: folding a digit RUN to a single byte would additionally erase 158// magnitude, turning "SRD 17" and "SRD 147" into one token -- and this corpus contains exactly that, 159// four distinct NIST Standard Reference Databases separated only by their number. The conservative 160// reading of "normalise digits to a constant" is the one that keeps those four distinguishable. 161// Counts what it changed, because a normaliser that silently did nothing is indistinguishable from 162// one that was never wired in. 163func tn_fold_digits(src: *u8, n: i64, out: *u8, cap: i64, box: *i64) -> i64 { 164 var w: i64 = 0 165 var i: i64 = 0 166 var folded: i64 = 0 167 while i < n { 168 if w >= cap { i = n } else { 169 let c: i64 = src[i] as i64 170 if tn_is_digit(c) == 1 { out[w] = TN_ZERO as u8; folded = folded + 1 } else { out[w] = src[i] } 171 w = w + 1 172 i = i + 1 173 } 174 } 175 box[1] = folded 176 return w 177} 178 179// THE CONTRACT. box[0] = capture-header bytes removed, box[1] = digit bytes folded, 180// box[2] = 1 when the markup extractor ran. Returns bytes written to out. 181func tn_normalize(src: *u8, n: i64, out: *u8, cap: i64, box: *i64) -> i64 { 182 box[0] = 0 183 box[1] = 0 184 box[2] = 0 185 if n <= 0 { return 0 } 186 let off: i64 = tn_body_offset(src, n) 187 box[0] = off 188 let bsrc: *u8 = ((src as i64) + off) as *u8 189 let bn: i64 = n - off 190 if tn_has_markup(bsrc, bn) == 1 { 191 box[2] = 1 192 // A stripper output is never longer than its input, so this buffer is DERIVED from the input 193 // size rather than capped at a guessed ceiling that could truncate in silence. 194 let scratch: *u8 = sys_mmap(bn + 1) 195 let tn2: i64 = nx_html_to_text(bsrc, bn, scratch, bn + 1) 196 let w2: i64 = tn_fold_digits(scratch, tn2, out, cap, box) 197 sys_munmap(scratch, bn + 1) 198 box[0] = off 199 box[2] = 1 200 return w2 201 } 202 let w: i64 = tn_fold_digits(bsrc, bn, out, cap, box) 203 box[0] = off 204 return w 205} 206 207// The normalised fingerprint: normalise, then hand the result to the ONE fingerprint kernel. 208// Allocation is sized from the input and RELEASED, because this runs once per document over a whole 209// shard and a per-document leak is a resource defect even when the answer is right. 210func tn_fingerprint_box(src: *u8, n: i64, box: *i64) -> i64 { 211 box[0] = 0 212 box[1] = 0 213 box[2] = 0 214 if n <= 0 { return 0 } 215 let cap: i64 = n + 1 216 let buf: *u8 = sys_mmap(cap) 217 let w: i64 = tn_normalize(src, n, buf, cap, box) 218 let fp: i64 = nx_simhash_fingerprint(buf, w) 219 sys_munmap(buf, cap) 220 return fp 221} 222func tn_fingerprint(src: *u8, n: i64) -> i64 { 223 let box: *i64 = sys_mmap(TN_BOX * TN_WORD) as *i64 224 let fp: i64 = tn_fingerprint_box(src, n, box) 225 sys_munmap(box as *u8, TN_BOX * TN_WORD) 226 return fp 227} 228 229// "fpn:" plus the cid -- the NORMALISED fingerprint plane, DELIBERATELY a different keyspace from 230// the raw "fp:" plane that ci_mkfpkey writes. The defect this whole lane exists to repair is two 231// different hash functions writing one keyspace; persisting a normalised fingerprint under the raw 232// prefix would commit that same defect a second time, in the same store, with the same symptom. 233// An fpn: row and an fp: row for one cid are both correct and mean different things. 234// MSB-first, so no scratch buffer is allocated per key (ci_mkfpkey leaks 24 bytes per call). 235func tn_mkfpnkey(cid: i64, out: *u8) -> i64 { 236 out[0] = 102 as u8 237 out[1] = 112 as u8 238 out[2] = 110 as u8 239 out[3] = 58 as u8 240 var o: i64 = 4 241 if cid == 0 { out[o] = TN_ZERO as u8; o = o + 1; out[o] = 0 as u8; return o } 242 var pw: i64 = 1 243 var m: i64 = cid 244 while m / pw >= TN_DEC { pw = pw * TN_DEC } 245 while pw > 0 { 246 out[o] = (TN_ZERO + ((m / pw) % TN_DEC)) as u8 247 o = o + 1 248 pw = pw / TN_DEC 249 } 250 out[o] = 0 as u8 251 return o 252}