code wiki / _hdl_build / nx_dup_detect.nx
nx_dup_detect.nx source
↩ module page · 158 lines · 9200 B
1// nx_dup_detect.nx -- FUSED duplicate detection (operator 2026-07-16: "we dont know what's in what bucket ...
2// fuzzy vs exact"). Exact-match (imports) sees 0 dup; crude NAME-fuzzy over-merges verbs (add x999). This
3// FUSES two axes from the family graph: (1) canonical NAME (strip nx_/.nx + version/redundancy qualifiers)
4// AND (2) IMPORT-SIGNATURE (the order-independent set of files an organ imports = its structural fingerprint,
5// = sum of FNV(neighbor-name) over out-edges). Two organs are DUPLICATES only if BOTH match -> nx_bignum_add
6// and nx_p256_add share canon "add" but have DIFFERENT import signatures -> correctly SPLIT (the verb
7// false-merge dies). Supersedes nx_fuzzy_dup (name-only). Read-only over the eco_graph store.
8// nx_dup_detect <store-prefix> license_tier: ORIGINAL module: nishi-core.architect.dup_detect
9import "nx_syscalls.nx"
10import "nx_eco_graph.nx"
11const DD_MAGIC_2166136261: i64 = 2166136261
12const DD_MAGIC_16777619: i64 = 16777619
13const DD_MAGIC_4294967295: i64 = 4294967295
14const DD_MAGIC_4611686018427387903: i64 = 4611686018427387903
15const DD_MAGIC_1000003: i64 = 1000003
16
17const DD_SLOTS: i64 = 131072
18const DD_ARENA: i64 = 4194304
19
20func dd_p(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
21func dd_pn(v: i64) -> i64 { let b: *u8 = sys_mmap(24); var m: i64 = v; var k: i64 = 0; let t: *u8 = sys_mmap(24); if m == 0 { t[0] = 48 as u8; k = 1 } while m > 0 { t[k] = (48+(m%10)) as u8; m = m/10; k = k+1 } var i: i64 = 0; while i < k { b[i] = t[k-1-i]; i = i+1 } sys_write(1, b, k); return 0 }
22func dd_streq_ao(a: *u8, ao: i64, al: i64, b: *u8) -> i64 {
23 var bl: i64 = 0; while b[bl] != (0 as u8) { bl = bl + 1 }
24 if bl != al { return 0 }
25 var i: i64 = 0; while i < al { if a[ao+i] != b[i] { return 0 } i = i + 1 }
26 if a[ao+al] != (0 as u8) { return 0 } // stored canon must ALSO terminate at al -- else it is a longer string that merely shares a prefix ("add" != stored "adder"). Without this the compare is a PREFIX match and every add*/sub* canon falsely collapses.
27 return 1
28}
29func dd_is_stop(buf: *u8, to: i64, tl: i64) -> i64 {
30 var alldig: i64 = 1; var i: i64 = 0
31 while i < tl { let c: i64 = buf[to+i] as i64; if c < 48 { alldig = 0; i = tl } else { if c > 57 { alldig = 0; i = tl } } i = i + 1 }
32 if tl > 0 { if alldig == 1 { return 1 } }
33 if tl >= 2 { if buf[to] == (118 as u8) { var vd: i64 = 1; var j: i64 = 1; while j < tl { let c2: i64 = buf[to+j] as i64; if c2 < 48 { vd = 0; j = tl } else { if c2 > 57 { vd = 0; j = tl } } j = j + 1 } if vd == 1 { return 1 } } }
34 if dd_streq_ao(buf, to, tl, "new" as *u8) == 1 { return 1 }
35 if dd_streq_ao(buf, to, tl, "old" as *u8) == 1 { return 1 }
36 if dd_streq_ao(buf, to, tl, "impl" as *u8) == 1 { return 1 }
37 if dd_streq_ao(buf, to, tl, "alt" as *u8) == 1 { return 1 }
38 if dd_streq_ao(buf, to, tl, "copy" as *u8) == 1 { return 1 }
39 if dd_streq_ao(buf, to, tl, "tmp" as *u8) == 1 { return 1 }
40 if dd_streq_ao(buf, to, tl, "backup" as *u8) == 1 { return 1 }
41 if dd_streq_ao(buf, to, tl, "legacy" as *u8) == 1 { return 1 }
42 if dd_streq_ao(buf, to, tl, "real" as *u8) == 1 { return 1 }
43 if dd_streq_ao(buf, to, tl, "final" as *u8) == 1 { return 1 }
44 if dd_streq_ao(buf, to, tl, "fix" as *u8) == 1 { return 1 }
45 if dd_streq_ao(buf, to, tl, "fixed" as *u8) == 1 { return 1 }
46 if dd_streq_ao(buf, to, tl, "lazy" as *u8) == 1 { return 1 }
47 return 0
48}
49// canon of node idx's name into out; return len. strip nx_ / .nx / trailing stop-tokens.
50func dd_canon(g: *EcoGraph, idx: i64, out: *u8) -> i64 {
51 let off: i64 = g.node_off[idx]
52 var e: i64 = 0
53 while g.arena[off+e] != (0 as u8) { e = e + 1 }
54 var s: i64 = 0
55 if e - s > 3 { if g.arena[off+e-3]==(46 as u8) { if g.arena[off+e-2]==(110 as u8) { if g.arena[off+e-1]==(120 as u8) { e = e - 3 } } } }
56 if e - s > 3 { if g.arena[off+s]==(110 as u8) { if g.arena[off+s+1]==(120 as u8) { if g.arena[off+s+2]==(95 as u8) { s = s + 3 } } } }
57 var go: i64 = 1
58 while go == 1 {
59 var lu: i64 = 0 - 1
60 var i: i64 = s
61 while i < e { if g.arena[off+i] == (95 as u8) { lu = i } i = i + 1 }
62 if lu < 0 { go = 0 } else { let tl: i64 = e - (lu + 1); if dd_is_stop(g.arena, off+lu+1, tl) == 1 { e = lu } else { go = 0 } }
63 }
64 var o: i64 = 0; var k: i64 = s
65 while k < e { out[o] = g.arena[off+k]; o = o + 1; k = k + 1 }
66 out[o] = 0 as u8
67 return o
68}
69func dd_hash(s: *u8, l: i64) -> i64 { var h: i64 = DD_MAGIC_2166136261; var i: i64 = 0; while i < l { h = h ^ (s[i] as i64); h = (h * DD_MAGIC_16777619) & DD_MAGIC_4294967295; i = i + 1 } return h }
70// FNV of node idx's raw name (for the import-signature sum)
71func dd_name_fnv(g: *EcoGraph, idx: i64) -> i64 {
72 let off: i64 = g.node_off[idx]
73 var h: i64 = DD_MAGIC_2166136261; var i: i64 = 0
74 while g.arena[off+i] != (0 as u8) { h = h ^ (g.arena[off+i] as i64); h = (h * DD_MAGIC_16777619) & DD_MAGIC_4294967295; i = i + 1 }
75 return h
76}
77
78func main(argc: i64, argv: *i64) -> i64 {
79 if argc < 2 { dd_p("usage: nx_dup_detect <store-prefix>\n" as *u8); return 2 }
80 let g: *EcoGraph = eg_load(argv[1] as *u8)
81 if (g as i64) == 0 { dd_p("dup_detect: store load failed\n" as *u8); return 1 }
82 let n: i64 = g.node_count
83 // DIAGNOSTIC: `nx_dup_detect <store> why <canon>` -> print every raw node name whose canon == <canon>
84 if argc >= 4 {
85 if dd_streq_ao("why" as *u8, 0, 3, argv[2] as *u8) == 1 {
86 let tgt: *u8 = argv[3] as *u8
87 let dk: *u8 = sys_mmap(128)
88 var di: i64 = 0; var shown: i64 = 0
89 while di < n {
90 let dl: i64 = dd_canon(g, di, dk)
91 if dd_streq_ao(dk, 0, dl, tgt) == 1 {
92 if shown < 30 { let doff: i64 = g.node_off[di]; var dln: i64 = 0; while g.arena[doff+dln] != (0 as u8) { dln = dln + 1 } dd_p(" raw=" as *u8); sys_write(1, ((g.arena as i64)+doff) as *u8, dln); dd_p("\n" as *u8) }
93 shown = shown + 1
94 }
95 di = di + 1
96 }
97 dd_p("total nodes canon-> '" as *u8); dd_p(tgt); dd_p("' = " as *u8); dd_pn(shown); dd_p(" (showing first 30)\n" as *u8)
98 return 0
99 }
100 }
101 // precompute per-node raw-name FNV (for import-signature sums)
102 let nfnv: *i64 = sys_mmap(n * 8) as *i64
103 var i: i64 = 0
104 while i < n { nfnv[i] = dd_name_fnv(g, i); i = i + 1 }
105 // fused hash table: slot -> canon-offset, sig, count
106 let hco: *i64 = sys_mmap(DD_SLOTS * 8) as *i64
107 let hsig: *i64 = sys_mmap(DD_SLOTS * 8) as *i64
108 let hcnt: *i64 = sys_mmap(DD_SLOTS * 8) as *i64
109 var s: i64 = 0
110 while s < DD_SLOTS { hco[s] = 0 - 1; s = s + 1 }
111 let carena: *u8 = sys_mmap(DD_ARENA)
112 var cused: i64 = 0
113 let ckey: *u8 = sys_mmap(128)
114 // also track: for the "add" verb-split proof, count distinct sigs per canon "add"
115 i = 0
116 while i < n {
117 let cl: i64 = dd_canon(g, i, ckey)
118 // import-signature = order-independent sum of FNV(out-neighbor raw name)
119 var sig: i64 = 0
120 var p: i64 = g.out_head[i]
121 while p < g.out_head[i+1] { sig = (sig + nfnv[g.out_list[p]]) & DD_MAGIC_4611686018427387903; p = p + 1 }
122 // fused slot: hash(canon) mixed with sig
123 let fused: i64 = ((dd_hash(ckey, cl) * DD_MAGIC_1000003) + sig) & DD_MAGIC_4611686018427387903
124 var slot: i64 = fused % DD_SLOTS
125 if slot < 0 { slot = 0 - slot }
126 var probe: i64 = 1
127 while probe == 1 {
128 if hco[slot] < 0 {
129 var k: i64 = 0; while k < cl { carena[cused+k] = ckey[k]; k = k + 1 } carena[cused+cl] = 0 as u8
130 hco[slot] = cused; hsig[slot] = sig; hcnt[slot] = 1; cused = cused + cl + 1; probe = 0
131 } else {
132 if hsig[slot] == sig { if dd_streq_ao(carena, hco[slot], cl, ckey) == 1 { hcnt[slot] = hcnt[slot] + 1; probe = 0 } else { slot = slot + 1; if slot >= DD_SLOTS { slot = 0 } } }
133 else { slot = slot + 1; if slot >= DD_SLOTS { slot = 0 } }
134 }
135 }
136 i = i + 1
137 }
138 dd_p("=== FUSED DUP DETECT (name canon AND import-signature) -- CONFIRMED duplicates ===\n" as *u8)
139 var uniq: i64 = 0; var d2: i64 = 0; var d3: i64 = 0; var maxc: i64 = 0
140 s = 0
141 while s < DD_SLOTS {
142 if hco[s] >= 0 {
143 uniq = uniq + 1
144 let c: i64 = hcnt[s]
145 if c >= 2 { d2 = d2 + 1 }
146 if c >= 3 { d3 = d3 + 1; dd_p(" x" as *u8); dd_pn(c); dd_p(" " as *u8); dd_p((carena as i64 + hco[s]) as *u8); dd_p(" (same name-root AND same imports)\n" as *u8) }
147 if c > maxc { maxc = c }
148 }
149 s = s + 1
150 }
151 dd_p("--- organs=" as *u8); dd_pn(n)
152 dd_p(" unique-by-name+sig=" as *u8); dd_pn(uniq)
153 dd_p(" CONFIRMED-dup-clusters>=2=" as *u8); dd_pn(d2)
154 dd_p(" >=3=" as *u8); dd_pn(d3)
155 dd_p(" max=" as *u8); dd_pn(maxc); dd_p("\n" as *u8)
156 dd_p("READ: fusing name+import-signature SPLITS the verb false-merges (add x999 name-only -> many distinct sigs). CONFIRMED dups = built the same thing (same root) with the same deps = the REAL retire/merge worklist.\n" as *u8)
157 return 0
158}