nx_sitededup_lib.nx source

1// nx_sitededup_lib.nx -- THE ONE COPY OF THE NEAR-DUPLICATE COLLAPSE DECISION. 2// 3// module: nishi-core.search.sitededup 4// capability: CORE_COMPUTE (pure; reads a conf, touches no store, writes no file) 5// license_tier: ORIGINAL No hw writes (Rule 26). 6// 7// WHY A LIB AND NOT A FUNCTION INSIDE THE ACTUATOR: the actuator suppresses rows in a LIVE 8// production shard and the gate proves the decision is correct. If each held its own copy of 9// the rule they could disagree, and the one that ships is the one nobody proved. There is 10// exactly one clusterer here, and both import it -- disagreement is impossible by construction. 11// 12// COMPOSES nx_simhash (Charikar STOC 2002 fingerprint, Manku WWW 2007 threshold). It does NOT 13// contain a second fingerprinting ruler and must never grow one. The TEXT handed to that 14// fingerprint is normalised by nx_textnorm_lib, which is likewise a single shared copy: the 15// clusterer never sees raw bytes and never decides what normalisation means. 16// 17// ---- THE TRANSITIVITY HAZARD, HANDLED RATHER THAN ASSUMED ------------------------------- 18// Near-duplication is NOT transitive: a~b and b~c does not give a~c. So a connected component 19// of the near-dup graph can be a CHAIN whose endpoints are genuinely different documents, and 20// collapsing it onto one survivor would suppress a document that is not a near-duplicate of 21// anything left searchable -- i.e. it would DELETE INFORMATION, which is the one outcome this 22// organ exists to avoid. 23// The fix is not to hope chains are rare. sdd_mark_chained MEASURES every component's diameter 24// (its widest internal pair) and any component wider than the threshold is marked UNSAFE and 25// left ENTIRELY ALONE -- every member stays searchable. The organ abstains on exactly the cases 26// it cannot justify, and reports how many. Abstain, never acquit; and here, abstain rather than 27// destroy. 28// The survivor of a safe component is therefore always within `k` bits of EVERY doc it replaces, 29// which is the property that makes suppression defensible at all. 30// 31// DETERMINISM: union-by-lowest-cid means a component's root is its globally minimum cid no 32// matter what order the unions happen in. Two runs over the same shard pick the same survivor, 33// so the pass is idempotent and re-runnable -- it can never flip-flop and eat the corpus. 34import "nx_syscalls.nx" 35import "nx_simhash.nx" 36import "nx_lineconf_lib.nx" 37 38// histogram buckets: distances 0..SDD_HIST-2 exactly, final bucket = "this far or further". 39const SDD_HIST: i64 = 17 40const SDD_BITS: i64 = 64 41 42// ---- TWO KNOWLEDGE TREES, ONE CONF --------------------------------------------------------- 43// A bare path resolves against the CWD, and this organ legitimately runs from two different ones: 44// the actuator runs at the nishihost root (the only place knowledge/store/ holds the shard it must 45// read) while the sovereign build lane anchors CWD to buildroot/. There is exactly ONE conf file, 46// at the nishihost root. The second path below is THAT SAME FILE seen from buildroot/ -- it is a 47// resolver, not a second copy, so there is nothing that can drift out of agreement. 48// MEASURED 2026-08-25: with only the first path the gate read hamming_max = LCF_MISS and went RED 49// on every conf-dependent tooth while the file it wanted sat one directory up. 50// A CONF THAT CANNOT BE FOUND IS INDISTINGUISHABLE FROM A CONF THAT WAS NEVER WRITTEN -- and the 51// value it failed to find is the safety threshold, so the failure had to be loud, not defaulted. 52func sdd_confpath() -> *u8 { return "knowledge/sitededup.conf" as *u8 } 53func sdd_confpath_up() -> *u8 { return "../knowledge/sitededup.conf" as *u8 } 54 55// The threshold is DATA. A missing row returns LCF_MISS from BOTH paths and every caller refuses 56// on it, so there is no compiled-in fallback that could quietly become the operating value. 57func sdd_conf_int(key: *u8) -> i64 { 58 let v: i64 = lcf_int_of(sdd_confpath(), key) 59 if v != LCF_MISS { return v } 60 return lcf_int_of(sdd_confpath_up(), key) 61} 62// WHICH path answered: 1 = CWD-relative, 2 = one directory up, 0 = neither. Provenance is printed 63// beside the value by every consumer, so no reader has to guess which file governed a run. 64func sdd_conf_which() -> i64 { 65 if lcf_int_of(sdd_confpath(), "hamming_max" as *u8) != LCF_MISS { return 1 } 66 if lcf_int_of(sdd_confpath_up(), "hamming_max" as *u8) != LCF_MISS { return 2 } 67 return 0 68} 69func sdd_conf_k() -> i64 { return sdd_conf_int("hamming_max" as *u8) } 70func sdd_conf_maxdocs() -> i64 { return sdd_conf_int("max_docs" as *u8) } 71func sdd_conf_showrows() -> i64 { return sdd_conf_int("show_rows" as *u8) } 72func sdd_conf_missing(v: i64) -> i64 { if v == LCF_MISS { return 1 } return 0 } 73 74// ---- distinct-cid set -------------------------------------------------------------------- 75// The shard walk visits EVERY entry in EVERY segment, and an append-only store holds the same 76// key in more than one segment. Feeding those repeats into the clusterer would pair a document 77// WITH ITSELF at distance 0 and report it as a duplicate -- inflating the rate and, far worse, 78// suppressing a document because it exists twice in the log. The set is what makes the census a 79// count of DOCUMENTS rather than of ROWS. 80// Open addressing, linear probe, slot value 0 = empty. No mixing constant is needed and none is 81// used: a cid IS already a hash (FNV-1a or the base-131 polynomial, both folded positive), so 82// its low bits are uniform and `cid & mask` is a sound slot function. 83func sdd_set_add(tab: *i64, mask: i64, cid: i64) -> i64 { 84 var s: i64 = cid & mask 85 var guard: i64 = 0 86 while guard <= mask { 87 if tab[s] == 0 { tab[s] = cid; return 1 } 88 if tab[s] == cid { return 0 } 89 s = (s + 1) & mask 90 guard = guard + 1 91 } 92 return 0 - 1 93} 94 95// ---- union-find, rooted at the lowest cid -------------------------------------------------- 96func sdd_find(parent: *i64, x0: i64) -> i64 { 97 var x: i64 = x0 98 while parent[x] != x { x = parent[x] } 99 return x 100} 101func sdd_union(parent: *i64, cids: *i64, a: i64, b: i64) -> i64 { 102 let ra: i64 = sdd_find(parent, a) 103 let rb: i64 = sdd_find(parent, b) 104 if ra == rb { return 0 } 105 if cids[ra] <= cids[rb] { parent[rb] = ra } else { parent[ra] = rb } 106 return 1 107} 108 109// ---- the sweep ----------------------------------------------------------------------------- 110// Fills parent[] with the near-dup components, eparent[] with the EXACT-fingerprint classes, and 111// hist[] with the FULL pairwise Hamming distribution. The histogram is not decoration: it is the 112// only evidence that the threshold separates anything on THIS corpus, and it costs nothing because 113// the distances are already being computed. Returns the near-dup edge count. 114// 115// eparent COSTS NOTHING EXTRA -- it is unioned inside the same O(n^2) sweep, from a distance that 116// has already been computed. A second pass over 173,166 pairs to learn something the first pass 117// had in its hand would be the duplicate-ruler defect wearing a performance cost. 118func sdd_cluster(cids: *i64, fps: *i64, n: i64, k: i64, parent: *i64, hist: *i64, eparent: *i64) -> i64 { 119 var i: i64 = 0 120 while i < n { parent[i] = i; eparent[i] = i; i = i + 1 } 121 var b0: i64 = 0 122 while b0 < SDD_HIST { hist[b0] = 0; b0 = b0 + 1 } 123 var edges: i64 = 0 124 i = 0 125 while i < n { 126 var j: i64 = 0 127 while j < i { 128 let d: i64 = nx_simhash_hamming(fps[i], fps[j]) 129 var b: i64 = d 130 if b >= SDD_HIST { b = SDD_HIST - 1 } 131 hist[b] = hist[b] + 1 132 if d <= k { 133 edges = edges + 1 134 sdd_union(parent, cids, i, j) 135 } 136 if d == 0 { sdd_union(eparent, cids, i, j) } 137 j = j + 1 138 } 139 i = i + 1 140 } 141 return edges 142} 143 144// Mark every member of a CHAINED component (diameter > k). Returns how many docs were marked. 145func sdd_mark_chained(fps: *i64, n: i64, k: i64, parent: *i64, unsafe_of: *i64) -> i64 { 146 var i: i64 = 0 147 while i < n { unsafe_of[i] = 0; i = i + 1 } 148 i = 0 149 while i < n { 150 var j: i64 = 0 151 while j < i { 152 let ri: i64 = sdd_find(parent, i) 153 if ri == sdd_find(parent, j) { 154 if nx_simhash_hamming(fps[i], fps[j]) > k { unsafe_of[ri] = 1 } 155 } 156 j = j + 1 157 } 158 i = i + 1 159 } 160 var nbad: i64 = 0 161 i = 0 162 while i < n { 163 let r: i64 = sdd_find(parent, i) 164 if unsafe_of[r] == 1 { unsafe_of[i] = 1 } 165 if unsafe_of[i] == 1 { nbad = nbad + 1 } 166 i = i + 1 167 } 168 return nbad 169} 170 171// THE VERDICT, one place. 172// 173// A doc in a SAFE component is suppressed iff it is not that component's survivor -- the survivor is 174// then within k of every document it replaces, which is what makes suppression defensible. 175// 176// ---- WHY A CHAINED COMPONENT IS NO LONGER ABSTAINED *ENTIRELY* ----------------------------------- 177// MEASURED LIVE 2026-08-25, and it is why the first commit changed the corpus without changing the 178// SERP: the four duplicate pairs visible on page one all measure Hamming ZERO -- byte-identical text 179// under two cids -- and every one of them survived, because each sat inside a WIDE component that 180// whole-component abstention then sheltered. One broad cluster of related VORTEX / RISC-V documents 181// was enough to protect every exact duplicate inside it. 182// ★ ABSTAINING ON A COMPONENT ABSTAINS ON ITS TIGHTEST PAIRS TOO, AND THOSE ARE THE ONES YOU WERE 183// MOST SURE ABOUT. 184// The refinement is not a loosened bar and it costs no safety: within a chained component we still 185// collapse the EXACT-FINGERPRINT classes. Fingerprint equality is an equivalence relation -- it IS 186// transitive, unlike near-duplication -- so such a class has diameter 0 BY CONSTRUCTION and its 187// survivor is at distance 0 from every member it replaces. That is a strictly stronger guarantee 188// than the one already accepted for safe components, so admitting it cannot widen the blast radius. 189// Everything in a chained component that is NOT an exact twin stays searchable, exactly as before. 190func sdd_is_suppressed(i: i64, parent: *i64, unsafe_of: *i64, eparent: *i64) -> i64 { 191 if unsafe_of[i] == 1 { 192 if sdd_find(eparent, i) == i { return 0 } 193 return 1 194 } 195 if sdd_find(parent, i) == i { return 0 } 196 return 1 197} 198 199// The document that REPLACES i, under whichever rule suppressed it. Callers must report the survivor 200// from here rather than assuming parent[], or a chained-component exact twin would be reported 201// against the wrong document and the worklist would be unauditable. 202func sdd_survivor_of(i: i64, parent: *i64, unsafe_of: *i64, eparent: *i64) -> i64 { 203 if unsafe_of[i] == 1 { return sdd_find(eparent, i) } 204 return sdd_find(parent, i) 205} 206 207// permil of documents suppressed. Integer-only; returns 0 for an empty population rather than 208// dividing by zero -- and callers must report the denominator beside it, because a rate without 209// its population is not a measurement. 210func sdd_permil(part: i64, whole: i64) -> i64 { 211 if whole <= 0 { return 0 } 212 return (part * 1000) / whole 213} 214 215// Clear the public-search consent bit while PRESERVING every other owner flag. Written as a 216// subtraction rather than a mask complement so it is obvious that exactly one bit moves: 217// AI-blog / AI-summary / share-external consent are the owner's and are not this organ's to 218// touch. DP_USE_PUB_SEARCH is bit 0 (nx_docportal_lib), DSS_POL_SEARCH agrees. 219func sdd_clear_search_bit(flags: i64) -> i64 { return flags - (flags & 1) } 220func sdd_search_allowed(flags: i64) -> i64 { if (flags & 1) == 1 { return 1 } return 0 }