code wiki / _hdl_build / nx_sitededup.nx
`nx_sitededup.nx` source

↩ module page · 532 lines · 28823 B
1// nx_sitededup.nx -- COLLAPSE DUPLICATE DOCUMENTS IN A SITE SEARCH SHARD.
2//
3// THE DEFECT, measured 2026-08-25 on a live scope=site SERP for "nishi search": of 30 results,
4// SIX were the SAME document under two different cids -- e.g. "VORTEX RISC-V GPGPU -- COMPLETE
5// PUBLICATION BIBLIOGRAPHY" as 7477184972746352550 AND 2839761502039440748, with IDENTICAL scores.
6// ROOT CAUSE: a cid is a content hash of the RAW STORED BYTES, and TWO DIFFERENT HASH FUNCTIONS write
7// the same doc:<cid> keyspace -- dp_cid (FNV-1a, nx_docportal_lib) and ci_hash (base-131 polynomial,
8// nx_corpus_ingest). Two captures of one page that differ only in a Date: header or a revision line
9// are therefore two different documents, and cross-path dedup is impossible by construction.
10// Ingest has since been fixed FORWARD (dp_ingest_policy strips capture headers and runs bd_fit_text
11// before hashing). That does nothing for rows already stored. This organ is the backward half.
12//
13// ---- WHY THE FIRST VERSION OF THIS ORGAN COULD NOT HAVE WORKED, AND HOW WE KNOW ----------------
14// The first cut fingerprinted the RAW stored bytes. Its own gate then measured the real capture pair
15// -- one document, stored twice, the second copy carrying an HTTP header and a bumped revision line
16// -- at SIX bits of Hamming distance against the published k=3 bar. It would have collapsed NOTHING.
17// The published recipe has a step the fingerprint papers do not cover: NORMALISE THE TEXT BEFORE
18// SHINGLING, and specifically normalise DIGITS TO A CONSTANT, which is what kills counter, date,
19// revision and price near-duplicates. That is precisely our duplicate class. nx_textnorm_lib is that
20// step, it is a SHARED lib so the gate proves the same normalisation the actuator ships, and this
21// organ now reports the raw-vs-normalised comparison ON THE WHOLE POPULATION so the claim
22// "normalisation is what made this work" is a measurement rather than an assertion.
23//
24// WHY SUPPRESSION AND NOT DELETION: rule 13, additive-only. Nothing is removed. A duplicate is marked
25// by appending a pol:<cid> row with the public-search consent bit cleared -- the SAME row the search
26// path ALREADY reads per candidate (dss_search, DSS_POL_SEARCH), so this needs no change to any serve
27// organ and takes effect on the next query. It is reversible by appending a row with the bit set.
28// Every other owner flag on that row is preserved (sdd_clear_search_bit).
29//
30// WHY DURABLE AND NOT AT RANK TIME: the alternative is fingerprinting every candidate on every query.
31// Fingerprinting is a full pass over the document TEXT, and the serve path deliberately does not read
32// doc bodies during candidate filtering (the site: filter's own comment: "Cheap: ss_hget per candidate,
33// no doc reads"). Paying a text scan per candidate per query to rediscover a fact that does not change
34// between queries is the wrong side of the trade. This pass pays it once.
35//
36// COMPOSES, NEVER RE-IMPLEMENTS: nx_simhash for the fingerprint (Charikar 2002 / Manku 2007),
37// nx_textnorm_lib for the ONE copy of the normalisation decision, nx_sitededup_lib for the ONE copy
38// of the collapse decision, dss_mkpolkey for the policy row, and ccz_cat_num as the integer emitter.
39//
40// THE FINGERPRINT PLANE IS DELIBERATELY A NEW KEYSPACE. Persisted fingerprints go to fpn:<cid>
41// (tn_mkfpnkey), NOT to the existing fp:<cid> that ci_mkfpkey writes. fp: rows hold RAW fingerprints
42// written by the crawler; an fpn: row holds a NORMALISED one. They are different rulers and mixing
43// them in one keyspace is EXACTLY the defect this organ exists to repair -- committing it a second
44// time, in the same store, would be indefensible. Both rows can coexist and both are correct.
45//
46// BOUNDED, RESUMABLE-BY-DESIGN, DRY BY DEFAULT: `apply` writes only when handed the literal "commit".
47// The population bound is a conf row, and reaching it REFUSES rather than truncating -- a census that
48// quietly stopped counting reports a smaller number that reads like better news.
49// license_tier: ORIGINAL   No hw writes (Rule 26).
50import "nx_corpus_ingest.nx"      // dss_prefix / dss_mkkey / dss_mkpolkey / ss_* primitives
51import "nx_sitededup_lib.nx"      // THE collapse decision -- shared with the gate
52import "nx_textnorm_lib.nx"       // THE normalisation decision -- shared with the gate
53import "nx_itoa_lib.nx"           // ccz_cat_num -- THE canonical integer emitter
54
55const SD_KEYCAP: i64 = 64
56const SD_VALCAP: i64 = 32
57const SD_WORD: i64 = 8
58const SD_MINUS: i64 = 45
59const SD_ZERO: i64 = 48
60const SD_NINE: i64 = 57
61const SD_DEC: i64 = 10
62const SD_URL_MINKEY: i64 = 5
63const SD_URL_MAXKEY: i64 = 60
64const SD_C_U: i64 = 117
65const SD_C_R: i64 = 114
66const SD_C_L: i64 = 108
67const SD_C_COLON: i64 = 58
68const SD_C_C: i64 = 99
69const SD_C_A: i64 = 97
70const SD_C_P: i64 = 112
71const SD_C_D: i64 = 100
72const SD_C_O: i64 = 111
73const SD_PREFIXCAP: i64 = 512
74const SD_BOX: i64 = 8
75
76func sd_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
77func sd_num(v: i64) -> i64 {
  let b: *u8 = sys_mmap(NXI_BUF)
  let n: i64 = ccz_cat_num(b, 0, v)
  sys_write(1, b, n)
  sys_munmap(b, NXI_BUF)
  return 0
83}
84func sd_atoi(s: *u8) -> i64 {
  var v: i64 = 0
  var i: i64 = 0
  while s[i] != (0 as u8) { let c: i64 = s[i] as i64; if c >= SD_ZERO { if c <= SD_NINE { v = v * SD_DEC + (c - SD_ZERO) } } i = i + 1 }
  return v
89}
90// SIGN-AWARE, because a 64-bit simhash with bit 63 set is a NEGATIVE i64 and ccz_cat_num emits it
91// with a leading '-'. A parser that skipped the sign would read a stored fingerprint back as its own
92// negation and silently place the document in the wrong cluster.
93func sd_parse_i64(p: *u8, n: i64) -> i64 {
  var v: i64 = 0
  var neg: i64 = 0
  var i: i64 = 0
  if n > 0 { if p[0] == (SD_MINUS as u8) { neg = 1; i = 1 } }
  while i < n {
      let c: i64 = p[i] as i64
      if c >= SD_ZERO { if c <= SD_NINE { v = v * SD_DEC + (c - SD_ZERO) } }
      i = i + 1
  }
  if neg == 1 { return 0 - v }
  return v
105}
106func sd_usage() -> i64 {
  sd_puts("usage: nx_sitededup census <domain> <max>            -- measure only, writes NOTHING\n" as *u8)
  sd_puts("       nx_sitededup apply  <domain> <max> [commit]   -- default is a DRY RUN\n" as *u8)
  sd_puts("       nx_sitededup pair   <domain> <cidA> <cidB>    -- RAW vs NORMALISED distance, on two real documents\n" as *u8)
  return 2
111}
112
113// Fingerprint one cid.
114//   box[0] = 1 when it had to be COMPUTED (so `apply` knows to persist it)
115//   box[1] = 1 when there is no readable doc row at all
116//   out[0] = the NORMALISED fingerprint -- the one the collapse decision uses
117//   out[1] = the RAW fingerprint, and out[2] = 1 when out[1] is valid
118// The raw fingerprint is a free by-product of having the text in hand, and it is the evidence for
119// the whole normalisation claim. It is NOT available when the normalised value came from the fpn:
120// cache, and the caller must then ABSTAIN from the raw comparison rather than compute it over a
121// biased subset -- a partial control is worse than none, because it reads like a full one.
122func sd_fp_of(h: *i64, cid: i64, fkey: *u8, dkey: *u8, ap: *i64, al: *i64, box: *i64, nb: *i64, out: *i64) -> i64 {
  box[0] = 0
  box[1] = 0
  out[0] = 0
  out[1] = 0
  out[2] = 0
  tn_mkfpnkey(cid, fkey)
  if ss_hget(h, fkey, ap, al) == 1 { if al[0] > 0 {
      out[0] = sd_parse_i64(ap[0] as *u8, al[0])
      return 0
  } }
  dss_mkkey(cid, dkey)
  if ss_hget(h, dkey, ap, al) == 1 { if al[0] > 0 {
      box[0] = 1
      let t: *u8 = ap[0] as *u8
      let tl: i64 = al[0]
      out[1] = nx_simhash_fingerprint(t, tl)
      out[2] = 1
      out[0] = tn_fingerprint_box(t, tl, nb)
      return 0
  } }
  box[1] = 1
  return 0
145}
146
147func main(argc: i64, argv: *i64) -> i64 {
  if argc < 4 { return sd_usage() }
  let verb: *u8 = argv[1] as *u8
  let domain: *u8 = argv[2] as *u8
151
  // ---- the threshold is DATA, and a missing row REFUSES ---------------------------------------
  let k: i64 = sdd_conf_k()
  let confmax: i64 = sdd_conf_maxdocs()
  let showrows: i64 = sdd_conf_showrows()
  let confwhich: i64 = sdd_conf_which()
  if sdd_conf_missing(k) == 1 { sd_puts("REFUSED: hamming_max absent from knowledge/sitededup.conf (looked in the CWD and one directory up) -- there is no compiled-in default\n" as *u8); return 4 }
  if sdd_conf_missing(confmax) == 1 { sd_puts("REFUSED: max_docs absent from knowledge/sitededup.conf\n" as *u8); return 4 }
  if sdd_conf_missing(showrows) == 1 { sd_puts("REFUSED: show_rows absent from knowledge/sitededup.conf\n" as *u8); return 4 }
160
  let prefix: *u8 = sys_mmap(SD_PREFIXCAP)
  dss_prefix(domain, prefix)
  let h: *i64 = ss_open(prefix)
  if (h as i64) == 0 { sd_puts("REFUSED: shard absent: " as *u8); sd_puts(prefix); sd_puts("\n" as *u8); return 3 }
165
  let fkey: *u8 = sys_mmap(SD_KEYCAP)
  let dkey: *u8 = sys_mmap(SD_KEYCAP)
  let box: *i64 = sys_mmap(SD_BOX * SD_WORD) as *i64
  let nb: *i64 = sys_mmap(SD_BOX * SD_WORD) as *i64
  let fout: *i64 = sys_mmap(SD_BOX * SD_WORD) as *i64
  let ap: *i64 = sys_mmap(SD_WORD * 2) as *i64
  let al: *i64 = sys_mmap(SD_WORD * 2) as *i64
173
  // ---- verb: pair -- THE REAL-CONTROL DIAGNOSTIC -----------------------------------------------
  // Feed it two cids read off a LIVE SERP and it prints their true distance under BOTH rulers.
  // This is how the threshold's false-positive rate gets measured against documents that actually
  // exist, rather than against fixtures written by the same hand that chose the bar -- and it is
  // the only way to see whether normalisation has pulled two GENUINELY DIFFERENT documents inside
  // the bar, which is the one direction in which this whole lane can do harm.
  if verb[0] == (SD_C_P as u8) {
      let ca: i64 = sd_atoi(argv[3] as *u8)
      var cb: i64 = 0
      if argc >= 5 { cb = sd_atoi(argv[4] as *u8) }
      sd_fp_of(h, ca, fkey, dkey, ap, al, box, nb, fout)
      let amiss: i64 = box[1]
      let na: i64 = fout[0]
      let ra: i64 = fout[1]
      let rav: i64 = fout[2]
      let ahdr: i64 = nb[0]
      let adig: i64 = nb[1]
      sd_fp_of(h, cb, fkey, dkey, ap, al, box, nb, fout)
      let bmiss: i64 = box[1]
      let nbf: i64 = fout[0]
      let rb: i64 = fout[1]
      let rbv: i64 = fout[2]
      let bhdr: i64 = nb[0]
      let bdig: i64 = nb[1]
      if amiss == 1 { sd_puts("NO-DOC cid=" as *u8); sd_num(ca); sd_puts("\n" as *u8); return 3 }
      if bmiss == 1 { sd_puts("NO-DOC cid=" as *u8); sd_num(cb); sd_puts("\n" as *u8); return 3 }
      let dn: i64 = nx_simhash_hamming(na, nbf)
      sd_puts("PAIR a=" as *u8); sd_num(ca)
      sd_puts(" b=" as *u8); sd_num(cb)
      sd_puts(" threshold=" as *u8); sd_num(k)
      sd_puts("\n  normalised: hamming=" as *u8); sd_num(dn)
      if dn <= k { sd_puts(" verdict=NEAR-DUPLICATE" as *u8) } else { sd_puts(" verdict=DISTINCT" as *u8) }
      sd_puts("\n" as *u8)
      if rav == 1 { if rbv == 1 {
          let dr: i64 = nx_simhash_hamming(ra, rb)
          sd_puts("  raw       : hamming=" as *u8); sd_num(dr)
          if dr <= k { sd_puts(" verdict=NEAR-DUPLICATE" as *u8) } else { sd_puts(" verdict=DISTINCT" as *u8) }
          sd_puts("\n" as *u8)
          if dr > k { if dn <= k { sd_puts("  NORMALISATION IS WHAT COLLAPSES THIS PAIR (raw missed it)\n" as *u8) } }
          if dr <= k { if dn <= k { sd_puts("  both rulers agree: duplicate\n" as *u8) } }
          if dn > k { sd_puts("  both rulers agree: distinct\n" as *u8) }
      } }
      if rav == 0 { sd_puts("  raw       : UNAVAILABLE (normalised value came from the fpn: cache; the raw comparison needs the text)\n" as *u8) }
      sd_puts("  normalisation did: a header_bytes_stripped=" as *u8); sd_num(ahdr)
      sd_puts(" digits_folded=" as *u8); sd_num(adig)
      sd_puts(" | b header_bytes_stripped=" as *u8); sd_num(bhdr)
      sd_puts(" digits_folded=" as *u8); sd_num(bdig)
      sd_puts("\n" as *u8)
      return 0
  }
224
  let want: i64 = sd_atoi(argv[3] as *u8)
  if want <= 0 { sd_puts("REFUSED: max must be > 0\n" as *u8); return 2 }
  if want > confmax { sd_puts("REFUSED: max exceeds max_docs in knowledge/sitededup.conf (" as *u8); sd_num(confmax); sd_puts(") -- raise the conf row deliberately, with its complexity budget\n" as *u8); return 2 }
  var docommit: i64 = 0
  if argc >= 5 { let a4: *u8 = argv[4] as *u8; if a4[0] == (SD_C_C as u8) { docommit = 1 } }
  var isapply: i64 = 0
  if verb[0] == (SD_C_A as u8) { isapply = 1 }
  if isapply == 0 { docommit = 0 }
233
  // ---- arrays ------------------------------------------------------------------------------
  let cids: *i64 = sys_mmap(want * SD_WORD) as *i64
  let fps: *i64 = sys_mmap(want * SD_WORD) as *i64
  let rawfps: *i64 = sys_mmap(want * SD_WORD) as *i64
  let parent: *i64 = sys_mmap(want * SD_WORD) as *i64
  let eparent: *i64 = sys_mmap(want * SD_WORD) as *i64
  let unsafe_of: *i64 = sys_mmap(want * SD_WORD) as *i64
  let computed: *i64 = sys_mmap(want * SD_WORD) as *i64
  let hist: *i64 = sys_mmap(SDD_HIST * SD_WORD) as *i64
  var tsz: i64 = 1
  while tsz < want * 2 { tsz = tsz * 2 }
  let tab: *i64 = sys_mmap(tsz * SD_WORD) as *i64
  let mask: i64 = tsz - 1
247
  // ---- walk every doc: row, collecting DISTINCT cids ----------------------------------------
  // The store is append-only, so one key can appear in several segments. Counting ROWS instead of
  // DOCUMENTS would pair a document with itself at distance 0 and then suppress it for existing
  // twice in the log. The partition below reconciles rows against documents so the gap is visible.
  //
  // ---- WHY doc: AND NOT url:, MEASURED LIVE 2026-08-25 -----------------------------------------
  // The first version enumerated the population from url:<cid> rows and reported 589 documents.
  // A single live query then returned total=611 MATCHING documents. A query cannot match more
  // documents than the corpus contains, so the enumeration axis was under-counting the population
  // -- documents reachable by search but invisible to the census, which is the worst possible
  // direction for a dedup pass: it cannot suppress what it cannot see, and it would have reported
  // a complete-looking duplicate rate over an incomplete corpus.
  // ★ AN ENUMERATION AXIS IS A CLAIM ABOUT COVERAGE, AND A CONSUMER THAT SEES MORE THAN THE CENSUS
  //   DOES IS A DISPROOF OF THAT CLAIM. doc:<cid> IS the document keyspace (dss_mkkey) and is what
  //   the serve path resolves per hit; url: is a secondary index that not every ingest path writes.
  // BOTH counts are printed below so the gap between the two axes stays visible instead of being
  // silently fixed and forgotten.
  var rows: i64 = 0
  var urlrows: i64 = 0
  var ndoc: i64 = 0
  var overflow: i64 = 0
  let ns: i64 = h[0]
  var s: i64 = 0
  while s < ns {
      let kb: *u8 = h[1 + 8 * s] as *u8
      if h[2 + 8 * s] >= 8 {
          let m9: i64 = ss_r32(kb, 4)
          var e9: i64 = 0
          while e9 < m9 {
              let eo: i64 = 8 + 4 * m9 + ss_r32(kb, 8 + 4 * e9)
              if (kb[eo] as i64) == 1 {
                  let kl9: i64 = ss_r32(kb, eo + 1)
                  if kl9 >= SD_URL_MINKEY { if kl9 < SD_URL_MAXKEY {
                      var isdoc: i64 = 0
                      var isurl: i64 = 0
                      if kb[eo + 8] == (SD_C_COLON as u8) {
                          if kb[eo + 5] == (SD_C_D as u8) { if kb[eo + 6] == (SD_C_O as u8) { if kb[eo + 7] == (SD_C_C as u8) { isdoc = 1 } } }
                          if kb[eo + 5] == (SD_C_U as u8) { if kb[eo + 6] == (SD_C_R as u8) { if kb[eo + 7] == (SD_C_L as u8) { isurl = 1 } } }
                      }
                      if isurl == 1 { urlrows = urlrows + 1 }
                      if isdoc == 1 {
                          rows = rows + 1
                          var cid: i64 = 0
                          var ki: i64 = 4
                          while ki < kl9 { let kc: i64 = kb[eo + 5 + ki] as i64; if kc >= SD_ZERO { if kc <= SD_NINE { cid = cid * SD_DEC + (kc - SD_ZERO) } } ki = ki + 1 }
                          if cid > 0 {
                              if overflow == 0 {
                                  let isnew: i64 = sdd_set_add(tab, mask, cid)
                                  if isnew < 0 { overflow = 2 }
                                  if isnew == 1 {
                                      if ndoc >= want { overflow = 1 } else { cids[ndoc] = cid; ndoc = ndoc + 1 }
                                  }
                              }
                          }
                      }
                  } }
              }
              e9 = e9 + 1
          }
      }
      s = s + 1
  }
  if overflow == 1 {
      sd_puts("REFUSED: distinct documents exceed max=" as *u8); sd_num(want)
      sd_puts(" -- this run measured NOTHING; re-run with a larger max (ceiling " as *u8); sd_num(confmax)
      sd_puts("). NOT truncated: a partial census published as a population is the defect this refuses to commit.\n" as *u8)
      return 2
  }
  if overflow == 2 { sd_puts("REFUSED: distinct-cid table full -- internal bound, raise max\n" as *u8); return 2 }
317
  // ---- fingerprint every document -----------------------------------------------------------
  var had_fp: i64 = 0
  var made_fp: i64 = 0
  var nodoc: i64 = 0
  var n: i64 = 0
  var nraw: i64 = 0
  var hdrstrip: i64 = 0
  var digfold: i64 = 0
  var mkupx: i64 = 0
  var i: i64 = 0
  while i < ndoc {
      sd_fp_of(h, cids[i], fkey, dkey, ap, al, box, nb, fout)
      if box[1] == 1 { nodoc = nodoc + 1 } else {
          if box[0] == 1 { made_fp = made_fp + 1 } else { had_fp = had_fp + 1 }
          if nb[0] > 0 { hdrstrip = hdrstrip + 1 }
          digfold = digfold + nb[1]
          if nb[2] == 1 { mkupx = mkupx + 1 }
          cids[n] = cids[i]
          fps[n] = fout[0]
          rawfps[n] = fout[1]
          if fout[2] == 1 { nraw = nraw + 1 }
          computed[n] = box[0]
          n = n + 1
      }
      i = i + 1
  }
344
  // ---- cluster + the chained-component safety measurement -----------------------------------
  let edges: i64 = sdd_cluster(cids, fps, n, k, parent, hist, eparent)
  let chained: i64 = sdd_mark_chained(fps, n, k, parent, unsafe_of)
348
  var kept: i64 = 0
  var supp: i64 = 0
  var supp_exact_in_chain: i64 = 0
  i = 0
  while i < n {
      if sdd_is_suppressed(i, parent, unsafe_of, eparent) == 1 {
          supp = supp + 1
          if unsafe_of[i] == 1 { supp_exact_in_chain = supp_exact_in_chain + 1 }
      } else { kept = kept + 1 }
      i = i + 1
  }
360
  // ---- report --------------------------------------------------------------------------------
  sd_puts("{\"tool\":\"nx_sitededup\",\"shard\":\"" as *u8); sd_puts(prefix)
  sd_puts("\",\"conf_resolved_from\":" as *u8); sd_num(confwhich)
  sd_puts(",\"doc_rows_walked\":" as *u8); sd_num(rows)
  sd_puts(",\"url_rows_walked\":" as *u8); sd_num(urlrows)
  sd_puts(",\"distinct_cids\":" as *u8); sd_num(ndoc)
  sd_puts(",\"no_doc_row\":" as *u8); sd_num(nodoc)
  sd_puts(",\"documents\":" as *u8); sd_num(n)
  sd_puts(",\"fpn_rows_present\":" as *u8); sd_num(had_fp)
  sd_puts(",\"fpn_computed_now\":" as *u8); sd_num(made_fp)
  sd_puts(",\"norm_docs_header_stripped\":" as *u8); sd_num(hdrstrip)
  sd_puts(",\"norm_digit_bytes_folded\":" as *u8); sd_num(digfold)
  sd_puts(",\"norm_docs_markup_extracted\":" as *u8); sd_num(mkupx)
  sd_puts(",\"threshold_bits\":" as *u8); sd_num(k)
  sd_puts(",\"neardup_edges\":" as *u8); sd_num(edges)
  sd_puts(",\"kept\":" as *u8); sd_num(kept)
  sd_puts(",\"suppressed\":" as *u8); sd_num(supp)
  sd_puts(",\"chained_documents\":" as *u8); sd_num(chained)
  sd_puts(",\"suppressed_exact_twin_inside_chain\":" as *u8); sd_num(supp_exact_in_chain)
  sd_puts(",\"chained_abstained\":" as *u8); sd_num(chained - supp_exact_in_chain)
  sd_puts(",\"dup_permil\":" as *u8); sd_num(sdd_permil(supp, n))
  sd_puts(",\"committed\":" as *u8); sd_num(docommit)
  sd_puts("}\n" as *u8)
384
  // PARTITIONS MUST SUM, AND THE SUM MUST BE PRINTED.
  if kept + supp == n { sd_puts("PARTITION-RECONCILES documents = kept + suppressed\n" as *u8) } else { sd_puts("PARTITION-LEAK -- do not trust these counts\n" as *u8) }
  if had_fp + made_fp + nodoc == ndoc { sd_puts("PARTITION-RECONCILES distinct_cids = fpn_present + fpn_computed + no_doc\n" as *u8) } else { sd_puts("PARTITION-LEAK (fingerprint sources)\n" as *u8) }
388
  // ---- THE NORMALISATION CONTROL, OVER THE WHOLE POPULATION ------------------------------------
  // Re-runs the identical clusterer over the RAW fingerprints of the SAME documents. The only thing
  // that differs between the two numbers is the normalisation step, so the gap between them IS the
  // measured value of normalising -- on this corpus, not on a fixture.
  // It ABSTAINS unless every document contributed a raw fingerprint. A control computed over the
  // subset that happened to miss the cache would be a biased sample wearing the name of a control.
  if nraw == n {
      if n > 0 {
          let rparent: *i64 = sys_mmap(n * SD_WORD) as *i64
          let reparent: *i64 = sys_mmap(n * SD_WORD) as *i64
          let runsafe: *i64 = sys_mmap(n * SD_WORD) as *i64
          let rhist: *i64 = sys_mmap(SDD_HIST * SD_WORD) as *i64
          let redges: i64 = sdd_cluster(cids, rawfps, n, k, rparent, rhist, reparent)
          sdd_mark_chained(rawfps, n, k, rparent, runsafe)
          var rsupp: i64 = 0
          i = 0
          while i < n {
              if sdd_is_suppressed(i, rparent, runsafe, reparent) == 1 { rsupp = rsupp + 1 }
              i = i + 1
          }
          sd_puts("NORMALISATION-CONTROL (same clusterer, same bar, same " as *u8); sd_num(n)
          sd_puts(" documents, raw text vs normalised text)\n" as *u8)
          sd_puts("  raw       : edges=" as *u8); sd_num(redges); sd_puts(" suppressed=" as *u8); sd_num(rsupp)
          sd_puts(" permil=" as *u8); sd_num(sdd_permil(rsupp, n)); sd_puts("\n" as *u8)
          sd_puts("  normalised: edges=" as *u8); sd_num(edges); sd_puts(" suppressed=" as *u8); sd_num(supp)
          sd_puts(" permil=" as *u8); sd_num(sdd_permil(supp, n)); sd_puts("\n" as *u8)
          sd_puts("  DELTA suppressed=" as *u8); sd_num(supp - rsupp)
          sd_puts(" -- duplicates that ONLY normalisation can see\n" as *u8)
          sys_munmap(rparent as *u8, n * SD_WORD)
          sys_munmap(reparent as *u8, n * SD_WORD)
          sys_munmap(runsafe as *u8, n * SD_WORD)
          sys_munmap(rhist as *u8, SDD_HIST * SD_WORD)
      }
  } else {
      sd_puts("NORMALISATION-CONTROL: ABSTAINED -- only " as *u8); sd_num(nraw)
      sd_puts(" of " as *u8); sd_num(n)
      sd_puts(" documents were fingerprinted from text this run (the rest came from the fpn: cache),\n" as *u8)
      sd_puts("  and a control over the subset that happened to miss the cache is a biased sample, not a control.\n" as *u8)
  }
428
  // The histogram is the calibration evidence: k is only defensible while a clear gap separates
  // the duplicate mass near 0 from the bulk of unrelated pairs.
  sd_puts("HAMMING-HISTOGRAM, NORMALISED (all " as *u8); sd_num((n * (n - 1)) / 2); sd_puts(" pairs; last bucket is >=)\n" as *u8)
  var b: i64 = 0
  while b < SDD_HIST {
      if hist[b] > 0 {
          sd_puts("  d=" as *u8); sd_num(b)
          if b == SDD_HIST - 1 { sd_puts("+" as *u8) }
          sd_puts(" pairs=" as *u8); sd_num(hist[b])
          sd_puts("\n" as *u8)
      }
      b = b + 1
  }
442
  if chained > 0 {
      sd_puts("CHAINED components hold " as *u8); sd_num(chained)
      sd_puts(" document(s) (component diameter > threshold). Near-duplication is not transitive,\n" as *u8)
      sd_puts("  so collapsing a whole chain could suppress a document that is NOT a near-duplicate of the\n" as *u8)
      sd_puts("  survivor. Of those, " as *u8); sd_num(supp_exact_in_chain)
      sd_puts(" are EXACT fingerprint twins and ARE collapsed: fingerprint equality IS\n" as *u8)
      sd_puts("  transitive, so such a class has diameter 0 and its survivor is at distance 0 from every\n" as *u8)
      sd_puts("  member it replaces -- a stronger guarantee than the one safe components already get.\n" as *u8)
      sd_puts("ABSTAINED on the remaining " as *u8); sd_num(chained - supp_exact_in_chain)
      sd_puts(" chained document(s). They stay searchable. Abstain, never destroy.\n" as *u8)
  }
454
  // ---- worklist + commit ---------------------------------------------------------------------
  let w: *i64 = ss_begin()
  let segsbox: *i64 = sys_mmap(SD_WORD * 2) as *i64
  segsbox[0] = 0
  let segidbox: *i64 = sys_mmap(SD_WORD * 2) as *i64
  segidbox[0] = ss_next_segid(prefix)
  let pkey: *u8 = sys_mmap(SD_KEYCAP)
  let pval: *u8 = sys_mmap(SD_VALCAP)
  let fval: *u8 = sys_mmap(SD_VALCAP)
  let qp: *i64 = sys_mmap(SD_WORD * 2) as *i64
  let ql: *i64 = sys_mmap(SD_WORD * 2) as *i64
466
  var wrote_pol: i64 = 0
  var wrote_fp: i64 = 0
  var already: i64 = 0
  var shown: i64 = 0
  sd_puts("SUPPRESSION WORKLIST (cid -> survivor, distance)\n" as *u8)
  i = 0
  while i < n {
      if sdd_is_suppressed(i, parent, unsafe_of, eparent) == 1 {
          let r: i64 = sdd_survivor_of(i, parent, unsafe_of, eparent)
          if shown < showrows {
              shown = shown + 1
              sd_puts("  SUPPRESS cid=" as *u8); sd_num(cids[i])
              sd_puts(" survivor=" as *u8); sd_num(cids[r])
              sd_puts(" d=" as *u8); sd_num(nx_simhash_hamming(fps[i], fps[r]))
              if unsafe_of[i] == 1 { sd_puts(" rule=EXACT-TWIN-INSIDE-CHAINED-COMPONENT" as *u8) }
              sd_puts("\n" as *u8)
          }
          // Read the CURRENT policy row. An absent row means the docportal default, which is
          // PUBLIC + searchable (dp_default_policy) -- exactly what the serve path assumes.
          dss_mkpolkey(cids[i], pkey)
          var flags: i64 = 1
          if ss_hget(h, pkey, qp, ql) == 1 { if ql[0] > 0 { flags = sd_parse_i64(qp[0] as *u8, ql[0]) } }
          if sdd_search_allowed(flags) == 0 { already = already + 1 } else {
              let nf: i64 = sdd_clear_search_bit(flags)
              let pn: i64 = ccz_cat_num(pval, 0, nf)
              wrote_pol = wrote_pol + 1
              if docommit == 1 {
                  if ss_add(w, 1, pkey, pval, pn) < 0 {
                      if ss_commit(prefix, w, segidbox[0]) == 0 { segsbox[0] = segsbox[0] + 1 }
                      segidbox[0] = segidbox[0] + 1
                      w[1] = 0
                      ss_add(w, 1, pkey, pval, pn)
                  }
              }
          }
      }
      // Persist the NORMALISED fingerprint we just computed, into the fpn: plane. Capture once, ask
      // forever: the next census reads the row instead of re-scanning the text.
      if computed[i] == 1 {
          tn_mkfpnkey(cids[i], fkey)
          let fn2: i64 = ccz_cat_num(fval, 0, fps[i])
          wrote_fp = wrote_fp + 1
          if docommit == 1 {
              if ss_add(w, 1, fkey, fval, fn2) < 0 {
                  if ss_commit(prefix, w, segidbox[0]) == 0 { segsbox[0] = segsbox[0] + 1 }
                  segidbox[0] = segidbox[0] + 1
                  w[1] = 0
                  ss_add(w, 1, fkey, fval, fn2)
              }
          }
      }
      i = i + 1
  }
  if docommit == 1 { if ss_commit(prefix, w, segidbox[0]) == 0 { segsbox[0] = segsbox[0] + 1 } }
521
  if supp > shown { sd_puts("  <== THIS LIST IS A PREFIX OF ITS OWN COUNT (" as *u8); sd_num(shown); sd_puts(" of " as *u8); sd_num(supp); sd_puts(" shown; raise show_rows)\n" as *u8) }
  sd_puts("POL-ROWS pending=" as *u8); sd_num(wrote_pol)
  sd_puts(" already_suppressed=" as *u8); sd_num(already)
  sd_puts(" FPN-ROWS pending=" as *u8); sd_num(wrote_fp)
  sd_puts(" segments_committed=" as *u8); sd_num(segsbox[0])
  sd_puts("\n" as *u8)
  if docommit == 0 { sd_puts("DRY RUN -- nothing was written. Re-run `apply <domain> <max> commit` to persist.\n" as *u8) }
  if docommit == 1 { sd_puts("REVERSIBLE: append a pol:<cid> row with bit 0 set to restore any document to search.\n" as *u8) }
  sd_puts("SITEDEDUP-OK\n" as *u8)
  return 0
532}
nx_sitededup.nx source

`nx_sitededup.nx` source
