code wiki / _hdl_build / nx_sitededup_gate.nx
`nx_sitededup_gate.nx` source

↩ module page · 544 lines · 35857 B
1// nx_sitededup_gate.nx -- proves the near-duplicate collapse decision COLLAPSES WHAT IT SHOULD AND
2// NOTHING ELSE, and that DIGIT NORMALISATION is what makes the first one possible.
3//
4// A dedup pass that collapses everything scores a perfect duplicate-removal rate and destroys the
5// corpus. So the load-bearing tooth here is not "did the duplicate collapse" -- it is the NEG-CONTROL:
6// genuinely different documents that must SURVIVE. Without it this gate would go green on an
7// implementation that suppressed every document but one. And because normalisation can only ever make
8// two texts MORE similar, the neg-control matters MORE after normalisation than before it, not less.
9//
10// ---- WHAT THIS GATE MEASURED THAT CHANGED THE DESIGN -------------------------------------------
11// The first version of this gate ran its fixtures through the RAW fingerprint and reported
12//     dup(A,A2) = 6   against a bar of k = 3
13// i.e. the collapse pass DID NOT COLLAPSE the exact production duplicate class it was written for.
14// That is why nx_textnorm_lib exists. The teeth below pin BOTH sides of that finding: the raw ruler
15// must MISS the pair (or the normalisation tooth proves nothing) and the normalised ruler must CATCH
16// it. A single tooth asserting only the second would pass just as happily on a ruler that collapses
17// the entire corpus.
18//
19// ---- AND WHAT THE SECOND RUN MEASURED, WHICH CHANGED THE FIXTURES -------------------------------
20// Two of the first fixtures were VACUOUS and this gate said so rather than being quietly believed:
21//   * the capture-header pair was a single string literal with NO NEWLINES, so the header block had
22//     no line structure to parse, the strip correctly refused, and the pair measured raw=6 vs
23//     normalised=7 -- normalisation made it WORSE. A capture header in the real store has line
24//     breaks; a fixture without them tests a document shape that cannot occur.
25//   * the digit-only pair was 607 bytes of prose carrying nine numbers, and its RAW distance was
26//     already 2 -- INSIDE the k=3 bar. It could never have shown that normalisation was necessary,
27//     because the un-normalised ruler already caught it.
28// Both are fixed below by BUILDING the fixtures at runtime, and the prose pair is kept -- with an
29// honest tooth of its own asserting the opposite property (normalisation must not LOSE a duplicate
30// the raw ruler already had). A fixture that cannot fail is not a test; a fixture that fails for a
31// reason other than the one under test is worse, because its red looks like evidence.
32//
33// The fixtures are REAL TEXT run through the REAL hashers and the REAL shared clusterer
34// (nx_sitededup_lib) -- not hand-picked fingerprints. The one place fingerprints ARE constructed by
35// bit-flipping is the CHAIN fixture, where the property under test is about exact distances and must
36// be exercised at exact distances.
37//
38// THE DIGIT FIXTURES ARE VERIFIED MECHANICALLY, NOT ASSERTED IN A COMMENT. sg_differs_only_in_digits
39// walks both texts byte by byte and requires every differing position to be a digit in BOTH and at
40// least one difference to exist. A fixture that merely CLAIMS to differ only in digits, and in fact
41// differs in a word, would prove something else entirely while reading exactly the same.
42//
43// FIXTURE LOCATION: entirely in-process. This gate writes NO file and touches NO seg-store, so it can
44// never share a fixture with a production beat, and it cannot leave state that poisons the next run.
45// license_tier: ORIGINAL   No hw writes (Rule 26).
46import "nx_gate_verdict.nx"
47import "nx_sitededup_lib.nx"
48import "nx_textnorm_lib.nx"
49
50const SG_N: i64 = 4
51const SG_WORD: i64 = 8
52const SG_CHAIN_N: i64 = 3
53// The chain-with-an-exact-twin fixture: the same three-document chain plus one byte-identical twin.
54const SG_CHAIN2_N: i64 = 4
55const SG_BOX: i64 = 8
56const SG_NL: i64 = 10
57// Headroom for the capture-header prefix built by sg_build_capture. The buffer is sized as
58// body length + this, so it is DERIVED from the fixture rather than being a guessed ceiling; the
59// prefix below is ~200 bytes and this is the next power of two above it.
60const SG_HDRCAP: i64 = 512
61// Distinct synthetic cids for the fixture docs. Deliberately NOT in ascending fixture order, so the
62// "survivor is the lowest cid" property is actually exercised rather than accidentally satisfied by
63// array position.
64const SG_CID_A2: i64 = 9100
65const SG_CID_A:  i64 = 4200
66const SG_CID_B:  i64 = 7300
67const SG_CID_C:  i64 = 5500
68
69func g_slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
70
71// Append a NUL-terminated string; returns the new write offset.
72func sg_cat(dst: *u8, o0: i64, s: *u8) -> i64 {
  var o: i64 = o0
  var i: i64 = 0
  while s[i] != (0 as u8) { dst[o] = s[i]; o = o + 1; i = i + 1 }
  return o
77}
78// The newline is CONSTRUCTED as its byte rather than written as a literal inside a string, because a
79// bare newline in a string literal is ambiguous to this lexer and has silently corrupted fixtures
80// before -- which is precisely how the previous version of this gate ended up with a header block
81// that had no lines in it.
82func sg_nl(dst: *u8, o: i64) -> i64 { dst[o] = SG_NL as u8; return o + 1 }
83
84// MECHANICAL fixture verification: returns the number of differing byte positions when the two texts
85// are the same length and EVERY difference is a digit on both sides, otherwise 0. Zero therefore
86// means "this fixture is not what it says it is" and the tooth that calls it fails loudly.
87func sg_differs_only_in_digits(a: *u8, b: *u8) -> i64 {
  let la: i64 = g_slen(a)
  let lb: i64 = g_slen(b)
  if la != lb { return 0 }
  var diffs: i64 = 0
  var i: i64 = 0
  while i < la {
      if a[i] != b[i] {
          diffs = diffs + 1
          if tn_is_digit(a[i] as i64) == 0 { return 0 }
          if tn_is_digit(b[i] as i64) == 0 { return 0 }
      }
      i = i + 1
  }
  return diffs
102}
103
104// ---- REAL FIXTURE TEXTS --------------------------------------------------------------------------
105// The shared body: one real harvested document, byte-identical in both captures of it.
106func sg_body() -> *u8 {
  return "VORTEX RISC-V GPGPU COMPLETE PUBLICATION BIBLIOGRAPHY. Source project Vortex, Georgia Institute of Technology, HPArch lab, principal investigator Hyesoon Kim. Project site vortex cc gatech edu. Harvested and verified for the Nishi sovereign library. The bibliography lists the MICRO 54 paper from 2021 describing the microarchitecture of an open source RISC V general purpose graphics processing unit, together with the arXiv preprint covering the ISA extensions for single instruction multiple thread execution, the warp scheduler, the shared memory hierarchy and the published performance measurements against comparable open hardware designs. Each entry records authors, venue, year and a verified retrieval link." as *u8
108}
109
110// TWO CAPTURES OF ONE PAGE, the live defect exactly: same body, different revision/date line, and
111// the second capture carries the HTTP response header the fetcher left in the stored bytes. The two
112// byte strings differ, so the content hash differs, so the store holds them as two documents.
113// hdr = 1 puts the capture header on; both get a revision line, and those differ only in digits.
114func sg_build_capture(dst: *u8, hdr: i64, revline: *u8) -> i64 {
  var o: i64 = 0
  if hdr == 1 {
      o = sg_cat(dst, o, "HTTP/1.1 200 OK" as *u8); o = sg_nl(dst, o)
      o = sg_cat(dst, o, "Date: Thu, 02 Jul 2026 17:07:07 GMT" as *u8); o = sg_nl(dst, o)
      o = sg_cat(dst, o, "Content-Type: text/html; charset=utf-8" as *u8); o = sg_nl(dst, o)
      o = sg_cat(dst, o, "Cache-Control: public, max-age=0, must-revalidate" as *u8); o = sg_nl(dst, o)
      o = sg_nl(dst, o)
  }
  o = sg_cat(dst, o, revline); o = sg_nl(dst, o)
  o = sg_cat(dst, o, sg_body())
  dst[o] = 0 as u8
  return o
127}
128func sg_rev_a()  -> *u8 { return "Revision 3907 captured 2026 07 14 at 09 12 44." as *u8 }
129func sg_rev_a2() -> *u8 { return "Revision 4821 captured 2026 08 02 at 17 07 07." as *u8 }
130
131// docB and docC are GENUINELY DIFFERENT documents on unrelated subjects. These are the neg-control.
132func sg_doc_b() -> *u8 {
  return "Lithium ion Battery Reclamation Business Plan. A safe, laundromat style shop that restores power tool, electric bicycle and cordless battery packs to specification, engineered so that a lithium ion fire cannot happen. Assembled by the Nishi finance member. The plan covers intake grading of returned packs, cell level capacity testing, spot welding of replacement nickel strip, thermal runaway containment in sand filled enclosures, insurance requirements, staffing, the retail pricing ladder and the projected monthly cash flow across the first three trading years, with a sensitivity table on cell purchase cost." as *u8
134}
135func sg_doc_c() -> *u8 {
  return "NIST Chemical Kinetics Database Standard Reference Database 17. National Institute of Standards and Technology, public standard reference data published as DCAT and NERDm catalogue records. The database compiles evaluated rate constants for gas phase chemical reactions, including bimolecular and termolecular processes, Arrhenius parameters, temperature ranges of validity, pressure dependence where measured, and the primary literature citation supporting every tabulated value. Records are versioned and each carries a persistent archival resource key identifier." as *u8
137}
138
139// ---- THE NUMERIC-DENSE DIGIT FIXTURE -------------------------------------------------------------
140// Two daily captures of one build-summary page: same words, same punctuation, same length, and every
141// number moved. This is the counter/date class the published normalisation step exists to collapse,
142// and it is a page shape a crawler genuinely re-fetches every day. It is numeric-DENSE on purpose:
143// the prose pair below carries only nine numbers in 607 bytes and its raw distance is already inside
144// the bar, so it cannot demonstrate that normalisation is NECESSARY -- only this one can.
145func sg_doc_num1() -> *u8 {
  return "Daily build ledger 2026 08 14. Commits 4821. Tests 9137 passed 9102 failed 35. Coverage 8734 permil. Artifact 993471 bytes across 12 segments. Peak memory 2048 MB. Wall clock 3617 seconds. Queue depth 47. Cache hits 88231 misses 4417. Nodes 16 workers 128. Checksum 8827301946. Warnings 213 errors 79." as *u8
147}
148func sg_doc_num2() -> *u8 {
  return "Daily build ledger 2026 03 29. Commits 5137. Tests 4482 passed 4451 failed 71. Coverage 9251 permil. Artifact 118264 bytes across 87 segments. Peak memory 4096 MB. Wall clock 8104 seconds. Queue depth 92. Cache hits 31775 misses 9903. Nodes 32 workers 256. Checksum 5510428733. Warnings 668 errors 31." as *u8
150}
151
152// ---- THE PROSE DIGIT FIXTURE ---------------------------------------------------------------------
153// KEPT DELIBERATELY, AND FOR THE OPPOSITE PROPERTY. Its raw distance is 2, already inside the k=3
154// bar, so it proves nothing about necessity -- but it does test something the numeric pair cannot:
155// that normalisation does not LOSE a duplicate the raw ruler had already caught. Folding digits could
156// in principle re-weight a document enough to push a true duplicate back OUT of the bar, and nothing
157// else here would notice.
158func sg_doc_pro1() -> *u8 {
  return "Sovereign build ledger for the nishi toolchain lane. Revision 4821 recorded on 2026 08 14 at 17 07 07 by the release lane. Artifact size 993471 bytes, segment count 12, rolling checksum 8827301946. The ledger records every promotion of the sovereign toolchain into the serving root, together with the operator who authorised it, the gate that proved it, and the byte size of the artifact that was installed. Entries are append only and are never rewritten once committed, so a reader can reconstruct the exact state of the serving root at any past instant by replaying the ledger forward from its first row." as *u8
160}
161func sg_doc_pro2() -> *u8 {
  return "Sovereign build ledger for the nishi toolchain lane. Revision 9137 recorded on 2026 03 29 at 04 51 22 by the release lane. Artifact size 118264 bytes, segment count 87, rolling checksum 5510428733. The ledger records every promotion of the sovereign toolchain into the serving root, together with the operator who authorised it, the gate that proved it, and the byte size of the artifact that was installed. Entries are append only and are never rewritten once committed, so a reader can reconstruct the exact state of the serving root at any past instant by replaying the ledger forward from its first row." as *u8
163}
164
165func main(argc: i64, argv: *i64) -> i64 {
  let ctr: *i64 = gv_ctr()
  gv_head("nx_sitededup_gate -- a near-duplicate must collapse, two different documents must NOT, and DIGIT NORMALISATION is what makes the first one possible" as *u8)
168
  // ---- T1: the threshold is DATA -------------------------------------------------------------
  let k: i64 = sdd_conf_k()
  let cw: i64 = sdd_conf_which()
  gv_puts("  conf hamming_max=" as *u8); gv_num(k)
  gv_puts(" resolved_from=" as *u8); gv_num(cw)
  gv_puts(" (1=cwd, 2=one directory up, 0=nowhere)\n" as *u8)
  var t1: i64 = 0
  if sdd_conf_missing(k) == 0 { if k >= 0 { if k <= SDD_BITS { t1 = 1 } } }
  gv_check("T1 threshold read from knowledge/sitededup.conf, in range (no compiled-in default)" as *u8, t1, ctr)
  if sdd_conf_missing(sdd_conf_maxdocs()) == 0 { gv_check("T2 population bound max_docs present in conf" as *u8, 1, ctr) } else { gv_check("T2 population bound max_docs present in conf" as *u8, 0, ctr) }
  var t2b: i64 = 0
  if cw > 0 { t2b = 1 }
  gv_check("T2b the conf RESOLVED from a real path and the run says which one (a conf nobody can find reads exactly like a conf nobody wrote)" as *u8, t2b, ctr)
182
  // ---- build the fixture with the REAL hashers -----------------------------------------------
  let cids: *i64 = sys_mmap(SG_N * SG_WORD) as *i64
  let fps: *i64 = sys_mmap(SG_N * SG_WORD) as *i64
  let rawfps: *i64 = sys_mmap(SG_N * SG_WORD) as *i64
  let parent: *i64 = sys_mmap(SG_N * SG_WORD) as *i64
  let unsafe_of: *i64 = sys_mmap(SG_N * SG_WORD) as *i64
  let eparent: *i64 = sys_mmap(SG_N * SG_WORD) as *i64
  let hist: *i64 = sys_mmap(SDD_HIST * SG_WORD) as *i64
  let nbox: *i64 = sys_mmap(SG_BOX * SG_WORD) as *i64
192
  let bodylen: i64 = g_slen(sg_body())
  let ta: *u8 = sys_mmap(bodylen + SG_HDRCAP)
  let ta2: *u8 = sys_mmap(bodylen + SG_HDRCAP)
  let la: i64 = sg_build_capture(ta, 0, sg_rev_a())
  let la2: i64 = sg_build_capture(ta2, 1, sg_rev_a2())
  let tb: *u8 = sg_doc_b()
  let tc: *u8 = sg_doc_c()
  gv_puts("  capture fixture built: A=" as *u8); gv_num(la)
  gv_puts(" bytes (no header), A2=" as *u8); gv_num(la2)
  gv_puts(" bytes (HTTP capture header + a different revision line)\n" as *u8)
203
  cids[0] = SG_CID_A2; fps[0] = tn_fingerprint(ta2, la2); rawfps[0] = nx_simhash_fingerprint(ta2, la2)
  cids[1] = SG_CID_A;  fps[1] = tn_fingerprint(ta,  la);  rawfps[1] = nx_simhash_fingerprint(ta,  la)
  cids[2] = SG_CID_B;  fps[2] = tn_fingerprint(tb, g_slen(tb)); rawfps[2] = nx_simhash_fingerprint(tb, g_slen(tb))
  cids[3] = SG_CID_C;  fps[3] = tn_fingerprint(tc, g_slen(tc)); rawfps[3] = nx_simhash_fingerprint(tc, g_slen(tc))
208
  gv_subjects("fixture documents fingerprinted by nx_textnorm + nx_simhash" as *u8, SG_N, ctr)
210
  let d_dup: i64 = nx_simhash_hamming(fps[0], fps[1])
  let d_dup_raw: i64 = nx_simhash_hamming(rawfps[0], rawfps[1])
  let d_ab: i64 = nx_simhash_hamming(fps[1], fps[2])
  let d_ac: i64 = nx_simhash_hamming(fps[1], fps[3])
  let d_bc: i64 = nx_simhash_hamming(fps[2], fps[3])
  gv_puts("  capture pair (A,A2): raw=" as *u8); gv_num(d_dup_raw)
  gv_puts(" normalised=" as *u8); gv_num(d_dup)
  gv_puts("   bar k=" as *u8); gv_num(k); gv_puts("\n" as *u8)
  gv_puts("  normalised distinct(A,B)=" as *u8); gv_num(d_ab)
  gv_puts(" distinct(A,C)=" as *u8); gv_num(d_ac)
  gv_puts(" distinct(B,C)=" as *u8); gv_num(d_bc)
  gv_puts("\n" as *u8)
223
  // ---- ASSERT THE FIXTURE REACHED THE CONDITION, BEFORE ASSERTING ANY OUTCOME ------------------
  var t3: i64 = 0
  if d_dup <= k { t3 = 1 }
  gv_check("T3 fixture-reached-the-condition: the capture pair is genuinely inside the bar once normalised" as *u8, t3, ctr)
  var t4: i64 = 0
  if d_ab > k { if d_ac > k { if d_bc > k { t4 = 1 } } }
  gv_check("T4 fixture-reached-the-condition: all three unrelated pairs are genuinely outside the bar" as *u8, t4, ctr)
  // THE FINDING THAT MOTIVATED THE WHOLE NORMALISATION LANE, PINNED AS A TOOTH. If the raw ruler
  // ever starts catching this pair on its own, T3 stops proving anything about normalisation and
  // this gate must say so instead of quietly reporting a green it has not earned.
  var t4b: i64 = 0
  if d_dup_raw > k { t4b = 1 }
  gv_check("T4b fixture-reached-the-condition: the RAW ruler MISSES the capture pair, so T3 is a fact about normalisation and not a coincidence" as *u8, t4b, ctr)
237
  // ---- cluster with the SHARED decision -------------------------------------------------------
  let edges: i64 = sdd_cluster(cids, fps, SG_N, k, parent, hist, eparent)
  let chained: i64 = sdd_mark_chained(fps, SG_N, k, parent, unsafe_of)
  var kept: i64 = 0
  var supp: i64 = 0
  var i: i64 = 0
  while i < SG_N {
      if sdd_is_suppressed(i, parent, unsafe_of, eparent) == 1 { supp = supp + 1 } else { kept = kept + 1 }
      i = i + 1
  }
  gv_puts("  edges=" as *u8); gv_num(edges)
  gv_puts(" kept=" as *u8); gv_num(kept)
  gv_puts(" suppressed=" as *u8); gv_num(supp)
  gv_puts(" chained_abstained=" as *u8); gv_num(chained)
  gv_puts("\n" as *u8)
253
  // ---- T5: the duplicate collapses ------------------------------------------------------------
  var t5: i64 = 0
  if sdd_find(parent, 0) == sdd_find(parent, 1) { t5 = 1 }
  gv_check("T5 the known near-duplicate pair COLLAPSES into one component" as *u8, t5, ctr)
258
  // ---- T6/T7: THE NEG-CONTROL -- different documents must survive ------------------------------
  // Without this a dedup that suppressed everything would score perfectly on T5. It is checked
  // against the NORMALISED fingerprints on purpose: normalisation only ever moves documents closer
  // together, so this is the direction in which the whole lane can do harm.
  var t6: i64 = 0
  if sdd_is_suppressed(2, parent, unsafe_of, eparent) == 0 { if sdd_is_suppressed(3, parent, unsafe_of, eparent) == 0 { t6 = 1 } }
  gv_check("T6 neg-control-two-genuinely-different-documents-both-SURVIVE-after-normalisation" as *u8, t6, ctr)
  var t7: i64 = 0
  if sdd_find(parent, 2) != sdd_find(parent, 3) { if sdd_find(parent, 2) != sdd_find(parent, 1) { t7 = 1 } }
  gv_check("T7 neg-control-different-documents-are-in-DIFFERENT-components" as *u8, t7, ctr)
269
  // ---- T8/T9: the partition SUMS ---------------------------------------------------------------
  var t8: i64 = 0
  if kept + supp == SG_N { t8 = 1 }
  gv_check("T8 partition sums: documents = kept + suppressed" as *u8, t8, ctr)
  var t9: i64 = 0
  if supp == 1 { t9 = 1 }
  gv_check("T9 exactly ONE document suppressed (not zero, not the whole corpus)" as *u8, t9, ctr)
277
  // ---- T10: the survivor is the LOWEST cid, so re-runs are idempotent --------------------------
  let surv: i64 = sdd_find(parent, 0)
  gv_puts("  survivor cid=" as *u8); gv_num(cids[surv]); gv_puts(" (A=4200 A2=9100)\n" as *u8)
  var t10: i64 = 0
  if cids[surv] == SG_CID_A { t10 = 1 }
  gv_check("T10 survivor is the LOWEST cid of its component (deterministic across re-runs)" as *u8, t10, ctr)
284
  // ---- T11: ORDER INDEPENDENCE -- the same survivor from a reversed input ----------------------
  let rc2: *i64 = sys_mmap(SG_N * SG_WORD) as *i64
  let rf2: *i64 = sys_mmap(SG_N * SG_WORD) as *i64
  let rp2: *i64 = sys_mmap(SG_N * SG_WORD) as *i64
  let ru2: *i64 = sys_mmap(SG_N * SG_WORD) as *i64
  let re2: *i64 = sys_mmap(SG_N * SG_WORD) as *i64
  let rh2: *i64 = sys_mmap(SDD_HIST * SG_WORD) as *i64
  i = 0
  while i < SG_N { rc2[i] = cids[SG_N - 1 - i]; rf2[i] = fps[SG_N - 1 - i]; i = i + 1 }
  sdd_cluster(rc2, rf2, SG_N, k, rp2, rh2, re2)
  sdd_mark_chained(rf2, SG_N, k, rp2, ru2)
  var rsupp: i64 = 0
  i = 0
  while i < SG_N { if sdd_is_suppressed(i, rp2, ru2, re2) == 1 { rsupp = rsupp + 1 } i = i + 1 }
  let rsurv: i64 = sdd_find(rp2, SG_N - 1)
  var t11: i64 = 0
  if rc2[rsurv] == cids[surv] { if rsupp == supp { t11 = 1 } }
  gv_check("T11 order-independent: reversed input yields the SAME survivor and the same count" as *u8, t11, ctr)
303
  // ---- T12: every suppressed doc is within k of the document that replaces it -------------------
  var t12: i64 = 1
  i = 0
  while i < SG_N {
      if sdd_is_suppressed(i, parent, unsafe_of, eparent) == 1 {
          let r: i64 = sdd_find(parent, i)
          if nx_simhash_hamming(fps[i], fps[r]) > k { t12 = 0 }
      }
      i = i + 1
  }
  gv_check("T12 every suppressed document is within the bar of its own survivor" as *u8, t12, ctr)
315
  // ---- T13: BITE -- the discriminator fires on the duplicate and is silent on the distinct ------
  var fired_bad: i64 = 0
  var fired_good: i64 = 0
  if nx_simhash_is_near_dup(fps[0], fps[1], k) == 1 { fired_bad = 1 }
  if nx_simhash_is_near_dup(fps[1], fps[2], k) == 1 { fired_good = 1 }
  gv_bite("T13 near-dup discriminator" as *u8, fired_bad, fired_good, ctr)
322
  // ================= THE DIGIT-NORMALISATION PROOF ==============================================
  // The published step this lane turns on is "normalise digits to a constant, which is what kills
  // counter/date/price near-dupes". These teeth are the whole claim, ordered so that each fixture
  // is proven to be what it says BEFORE any outcome is asserted about it.
  let n1: *u8 = sg_doc_num1()
  let n2: *u8 = sg_doc_num2()
  let ndig: i64 = sg_differs_only_in_digits(n1, n2)
  gv_puts("  numeric-dense fixture: differing byte positions, all digits on BOTH sides = " as *u8); gv_num(ndig)
  gv_puts(" (0 would mean it differs somewhere that is NOT a digit)\n" as *u8)
  var td_a: i64 = 0
  if ndig > 0 { td_a = 1 }
  gv_check("TD1 fixture-reached-the-condition: the numeric-dense pair differs ONLY in digit bytes, verified byte by byte and not merely claimed" as *u8, td_a, ctr)
335
  let dd_norm: i64 = nx_simhash_hamming(tn_fingerprint(n1, g_slen(n1)), tn_fingerprint(n2, g_slen(n2)))
  let dd_raw: i64 = nx_simhash_hamming(nx_simhash_fingerprint(n1, g_slen(n1)), nx_simhash_fingerprint(n2, g_slen(n2)))
  gv_puts("  numeric-dense pair: raw=" as *u8); gv_num(dd_raw)
  gv_puts(" normalised=" as *u8); gv_num(dd_norm)
  gv_puts("   bar k=" as *u8); gv_num(k); gv_puts("\n" as *u8)
341
  // The load-bearing pair. Either tooth alone is worthless: TD3 alone passes on a ruler that
  // collapses everything, TD2 alone passes on a ruler that collapses nothing.
  var td_b: i64 = 0
  if dd_raw > k { td_b = 1 }
  gv_check("TD2 WITHOUT normalisation the digit-only pair is NOT caught -- the raw ruler is outside the bar" as *u8, td_b, ctr)
  var td_c: i64 = 0
  if dd_norm <= k { td_c = 1 }
  gv_check("TD3 WITH normalisation the digit-only pair COLLAPSES -- this is the step that kills counter/date/revision near-dupes" as *u8, td_c, ctr)
350
  // A digit-only difference should normalise to byte-identical text, hence distance ZERO. Asserting
  // the strict value rather than "<= k" is what would catch a normaliser that folded only SOME
  // digits: such a ruler would still slip under the bar and TD3 would never notice.
  var td_d: i64 = 0
  if dd_norm == 0 { td_d = 1 }
  gv_check("TD4 the normalised fingerprints are IDENTICAL (distance 0), so every digit was folded and not merely most of them" as *u8, td_d, ctr)
357
  // ---- TD4b: normalisation must not LOSE a duplicate the raw ruler already had -------------------
  let p1: *u8 = sg_doc_pro1()
  let p2: *u8 = sg_doc_pro2()
  let pdig: i64 = sg_differs_only_in_digits(p1, p2)
  let pd_raw: i64 = nx_simhash_hamming(nx_simhash_fingerprint(p1, g_slen(p1)), nx_simhash_fingerprint(p2, g_slen(p2)))
  let pd_norm: i64 = nx_simhash_hamming(tn_fingerprint(p1, g_slen(p1)), tn_fingerprint(p2, g_slen(p2)))
  gv_puts("  prose pair (digit-only, sparse numbers): raw=" as *u8); gv_num(pd_raw)
  gv_puts(" normalised=" as *u8); gv_num(pd_norm)
  gv_puts("  differing-digit positions=" as *u8); gv_num(pdig); gv_puts("\n" as *u8)
  var td_pa: i64 = 0
  if pdig > 0 { if pd_raw <= k { td_pa = 1 } }
  gv_check("TD4a fixture-reached-the-condition: the prose pair differs only in digits AND the raw ruler already caught it (so it can only test preservation, never necessity)" as *u8, td_pa, ctr)
  var td_pb: i64 = 0
  if pd_norm <= k { td_pb = 1 }
  gv_check("TD4b normalisation does not LOSE a duplicate the raw ruler already had -- folding digits cannot push a true duplicate back OUT of the bar" as *u8, td_pb, ctr)
373
  // THE NEG-CONTROL FOR THE NORMALISER ITSELF. Digit folding must not drag two genuinely different
  // documents together. Measured on the normalised fingerprints of the unrelated fixture docs.
  var td_e: i64 = 0
  if nx_simhash_hamming(tn_fingerprint(tb, g_slen(tb)), tn_fingerprint(tc, g_slen(tc))) > k { td_e = 1 }
  gv_check("TD5 neg-control-digit-normalisation-does-NOT-collapse-two-unrelated-documents" as *u8, td_e, ctr)
379
  // PROVE THE NORMALISER ACTUALLY FIRED. A normaliser wired in but doing nothing would leave every
  // tooth above passing for the wrong reason on any corpus where the raw ruler already worked.
  let probe: i64 = tn_fingerprint_box(n1, g_slen(n1), nbox)
  gv_puts("  normaliser activity on the numeric fixture: header_bytes_stripped=" as *u8); gv_num(nbox[0])
  gv_puts(" digit_bytes_folded=" as *u8); gv_num(nbox[1])
  gv_puts(" markup_extractor_ran=" as *u8); gv_num(nbox[2]); gv_puts("\n" as *u8)
  var td_f: i64 = 0
  if nbox[1] > 0 { if probe != 0 { td_f = 1 } }
  gv_check("TD6 the normaliser REPORTS the digit bytes it folded (a normaliser that silently did nothing is indistinguishable from one never wired in)" as *u8, td_f, ctr)
389
  // THE CAPTURE-HEADER HALF, on the real capture pair. The header strip must fire on A2 and must
  // NOT fire on A -- a stripper that fires on everything would eat the first paragraph of every
  // plain document in the corpus, which is silent corruption, not dedup.
  tn_fingerprint_box(ta2, la2, nbox)
  let a2strip: i64 = nbox[0]
  tn_fingerprint_box(ta, la, nbox)
  let astrip: i64 = nbox[0]
  gv_puts("  capture-header strip: on A2 (has a header) = " as *u8); gv_num(a2strip)
  gv_puts(" bytes, on A (plain text) = " as *u8); gv_num(astrip); gv_puts(" bytes\n" as *u8)
  var a2fire: i64 = 0
  if a2strip > 0 { a2fire = 1 }
  var afire: i64 = 0
  if astrip > 0 { afire = 1 }
  gv_bite("TD7 capture-header strip" as *u8, a2fire, afire, ctr)
404
  // AND IT MUST REFUSE TO STRIP EVERYTHING. A document that is nothing but a header block has no
  // body, and returning an empty text would make it collide with every other empty text at
  // distance 0 -- mass suppression produced by a normaliser, which is the worst available failure.
  let allhdr: *u8 = sys_mmap(SG_HDRCAP)
  var ah: i64 = 0
  ah = sg_cat(allhdr, ah, "HTTP/1.1 200 OK" as *u8); ah = sg_nl(allhdr, ah)
  ah = sg_cat(allhdr, ah, "Content-Type: text/html" as *u8); ah = sg_nl(allhdr, ah)
  ah = sg_cat(allhdr, ah, "Cache-Control: max-age=0" as *u8); ah = sg_nl(allhdr, ah)
  allhdr[ah] = 0 as u8
  var td_h: i64 = 0
  if tn_body_offset(allhdr, ah) == 0 { td_h = 1 }
  gv_check("TD8 neg-control-the-header-strip-REFUSES-when-it-would-consume-the-whole-document" as *u8, td_h, ctr)
417
  // ---- T14/T15: CHAINED components are ABSTAINED on, not collapsed -----------------------------
  // Near-duplication is not transitive. Three documents spaced k apart in a line have a diameter of
  // 2k, so collapsing them would suppress a document that is NOT a near-duplicate of the survivor.
  // Fingerprints are constructed here by flipping an exact number of bits, because the property
  // under test is about DISTANCES and must be exercised at exact distances.
  let cc: *i64 = sys_mmap(SG_CHAIN_N * SG_WORD) as *i64
  let cf: *i64 = sys_mmap(SG_CHAIN_N * SG_WORD) as *i64
  let cp: *i64 = sys_mmap(SG_CHAIN_N * SG_WORD) as *i64
  let cu: *i64 = sys_mmap(SG_CHAIN_N * SG_WORD) as *i64
  let ce: *i64 = sys_mmap(SG_CHAIN_N * SG_WORD) as *i64
  let ch: *i64 = sys_mmap(SDD_HIST * SG_WORD) as *i64
  var m1: i64 = 0
  var m2: i64 = 0
  var bit: i64 = 0
  while bit < k { m1 = m1 | (1 << bit); bit = bit + 1 }
  while bit < k + k { m2 = m2 | (1 << bit); bit = bit + 1 }
  cc[0] = 100; cf[0] = fps[2]
  cc[1] = 200; cf[1] = fps[2] ^ m1
  cc[2] = 300; cf[2] = fps[2] ^ m1 ^ m2
  let cd02: i64 = nx_simhash_hamming(cf[0], cf[2])
  gv_puts("  chain fixture: d(0,1)=" as *u8); gv_num(nx_simhash_hamming(cf[0], cf[1]))
  gv_puts(" d(1,2)=" as *u8); gv_num(nx_simhash_hamming(cf[1], cf[2]))
  gv_puts(" d(0,2)=" as *u8); gv_num(cd02); gv_puts("\n" as *u8)
  var t14a: i64 = 0
  if cd02 > k { t14a = 1 }
  gv_check("T14 fixture-reached-the-condition: the chain really does exceed the bar end to end" as *u8, t14a, ctr)
  sdd_cluster(cc, cf, SG_CHAIN_N, k, cp, ch, ce)
  let cbad: i64 = sdd_mark_chained(cf, SG_CHAIN_N, k, cp, cu)
  var csupp: i64 = 0
  i = 0
  while i < SG_CHAIN_N { if sdd_is_suppressed(i, cp, cu, ce) == 1 { csupp = csupp + 1 } i = i + 1 }
  gv_puts("  chain: marked=" as *u8); gv_num(cbad); gv_puts(" suppressed=" as *u8); gv_num(csupp); gv_puts("\n" as *u8)
  var t15: i64 = 0
  if csupp == 0 { if cbad == SG_CHAIN_N { t15 = 1 } }
  gv_check("T15 a CHAINED component with no exact twins in it is abstained on entirely -- no member suppressed" as *u8, t15, ctr)
453
  // ---- T15b/T15c: AN EXACT TWIN INSIDE A CHAINED COMPONENT IS STILL COLLAPSED -------------------
  // MEASURED LIVE 2026-08-25 and this is the tooth that exists because of it: the first commit
  // suppressed 25 documents and the SERP did not change, because all four visible duplicate pairs
  // were byte-identical documents sitting inside WIDE components, and whole-component abstention
  // sheltered them. Abstaining on a component abstains on its tightest pairs too.
  // The fixture is a chain (diameter 2k, so genuinely unsafe as a whole) that ALSO contains one
  // exact fingerprint twin. The twin must collapse -- equality is transitive, so its survivor is at
  // distance 0 from it -- and the three non-identical members must all survive.
  let xc: *i64 = sys_mmap(SG_CHAIN2_N * SG_WORD) as *i64
  let xf: *i64 = sys_mmap(SG_CHAIN2_N * SG_WORD) as *i64
  let xp: *i64 = sys_mmap(SG_CHAIN2_N * SG_WORD) as *i64
  let xu: *i64 = sys_mmap(SG_CHAIN2_N * SG_WORD) as *i64
  let xe: *i64 = sys_mmap(SG_CHAIN2_N * SG_WORD) as *i64
  let xh: *i64 = sys_mmap(SDD_HIST * SG_WORD) as *i64
  xc[0] = 100; xf[0] = fps[2]
  xc[1] = 400; xf[1] = fps[2]
  xc[2] = 200; xf[2] = fps[2] ^ m1
  xc[3] = 300; xf[3] = fps[2] ^ m1 ^ m2
  sdd_cluster(xc, xf, SG_CHAIN2_N, k, xp, xh, xe)
  let xbad: i64 = sdd_mark_chained(xf, SG_CHAIN2_N, k, xp, xu)
  var xsupp: i64 = 0
  i = 0
  while i < SG_CHAIN2_N { if sdd_is_suppressed(i, xp, xu, xe) == 1 { xsupp = xsupp + 1 } i = i + 1 }
  gv_puts("  chain-with-twin fixture: marked_chained=" as *u8); gv_num(xbad)
  gv_puts(" suppressed=" as *u8); gv_num(xsupp); gv_puts("\n" as *u8)
  var t15b: i64 = 0
  if xbad == SG_CHAIN2_N { t15b = 1 }
  gv_check("T15b fixture-reached-the-condition: the whole component really IS chained, so this exercises the chained path and not the safe one" as *u8, t15b, ctr)
  var t15c: i64 = 0
  if sdd_is_suppressed(1, xp, xu, xe) == 1 {
      if sdd_is_suppressed(0, xp, xu, xe) == 0 {
          if sdd_is_suppressed(2, xp, xu, xe) == 0 {
              if sdd_is_suppressed(3, xp, xu, xe) == 0 { if xsupp == 1 { t15c = 1 } }
          }
      }
  }
  gv_check("T15c the EXACT twin inside the chained component collapses and the three non-identical members all SURVIVE" as *u8, t15c, ctr)
  var t15d: i64 = 0
  if sdd_survivor_of(1, xp, xu, xe) == 0 { t15d = 1 }
  gv_check("T15d the twin is reported against the document that actually replaces it (the lowest cid of its EXACT class, not the near-dup component root)" as *u8, t15d, ctr)
494
  // ---- T16/T17/T18: the consent bit is edited surgically ---------------------------------------
  // DP_USE_PUB_SEARCH is bit 0. AI-blog(2) / AI-summary(4) / share-external(8) are the owner's
  // and must survive untouched -- suppressing a duplicate from search is not consent to change
  // what an owner allowed elsewhere.
  let before: i64 = 1 + 2 + 8
  let after: i64 = sdd_clear_search_bit(before)
  gv_puts("  policy flags " as *u8); gv_num(before); gv_puts(" -> " as *u8); gv_num(after); gv_puts("\n" as *u8)
  var t16: i64 = 0
  if sdd_search_allowed(after) == 0 { t16 = 1 }
  gv_check("T16 clearing the search bit actually clears it" as *u8, t16, ctr)
  var t17: i64 = 0
  if after == 2 + 8 { t17 = 1 }
  gv_check("T17 every OTHER owner consent flag is preserved exactly" as *u8, t17, ctr)
  var t18: i64 = 0
  if sdd_search_allowed(sdd_clear_search_bit(after)) == 0 { if sdd_clear_search_bit(after) == after { t18 = 1 } }
  gv_check("T18 clearing an already-cleared bit is idempotent (re-runs cannot drift the flags)" as *u8, t18, ctr)
511
  // ---- T19: the distinct-cid set really does collapse repeats ----------------------------------
  let tsz: i64 = 64
  let tab: *i64 = sys_mmap(tsz * SG_WORD) as *i64
  let tmask: i64 = tsz - 1
  var adds: i64 = 0
  adds = adds + sdd_set_add(tab, tmask, SG_CID_A)
  adds = adds + sdd_set_add(tab, tmask, SG_CID_B)
  adds = adds + sdd_set_add(tab, tmask, SG_CID_A)
  adds = adds + sdd_set_add(tab, tmask, SG_CID_A)
  gv_puts("  set: 4 insertions of 2 distinct cids -> new=" as *u8); gv_num(adds); gv_puts("\n" as *u8)
  var t19: i64 = 0
  if adds == 2 { t19 = 1 }
  gv_check("T19 repeated cids collapse to distinct documents (a doc cannot duplicate ITSELF)" as *u8, t19, ctr)
525
  // ---- T20: the rate is bound to its denominator -----------------------------------------------
  var t20: i64 = 0
  if sdd_permil(1, SG_N) == 250 { if sdd_permil(0, 0) == 0 { t20 = 1 } }
  gv_check("T20 duplicate rate is permil of the measured population, and an empty population is not a divide" as *u8, t20, ctr)
530
  // ---- T21: the persisted fingerprint plane is a SEPARATE keyspace ------------------------------
  // The defect this whole lane repairs is two different hash functions writing one keyspace.
  // A normalised fingerprint stored under the raw fp: prefix would commit it a second time.
  let kbuf: *u8 = sys_mmap(64)
  let klen: i64 = tn_mkfpnkey(SG_CID_A, kbuf)
  gv_puts("  normalised fingerprint key for cid 4200 = " as *u8); gv_puts(kbuf); gv_puts("\n" as *u8)
  var t21: i64 = 0
  if klen > 4 {
      if (kbuf[0] as i64) == 102 { if (kbuf[1] as i64) == 112 { if (kbuf[2] as i64) == 110 { if (kbuf[3] as i64) == 58 { t21 = 1 } } } }
  }
  gv_check("T21 normalised fingerprints persist to their OWN keyspace, not into the raw fp: plane that a different ruler writes" as *u8, t21, ctr)
542
  return gv_verdict("SITEDEDUP-GATE" as *u8, ctr, "a real near-duplicate collapses; two real distinct documents survive; chains abstain; digit normalisation is proven to be the step that makes the collapse possible" as *u8)
544}
nx_sitededup_gate.nx source

`nx_sitededup_gate.nx` source
