nx_asset_autotag.nx source
↩ module page · 73 lines · 3707 B
1// nx_asset_autotag.nx -- PERCEPTUAL NEAR-DUPLICATE detection (R3 of the universal org-tooling arc).
2//
3// Makes the catalog know which media are NEAR-duplicates -- the ones exact-CID dedup CANNOT catch.
4// Content-addressing (ar_cid) gives EXACT dedup: identical bytes collapse to one CID, but ONE changed
5// pixel / a recompression / a resize yields a DIFFERENT CID while the image is visually the same. A
6// PERCEPTUAL hash fingerprints what the media LOOKS like, so near-identical media land a tiny Hamming
7// distance apart and cluster together (org_research.tsv CONFIRMED perceptual-autotag: perceptual
8// hashing groups near-dups that exact-hash/CID misses; machines pre-tag, humans curate).
9//
10// ZERO NEW ALGORITHMS -- pure COMPOSITION over the media-studio kernels:
11// * perceptual fingerprint (dHash, brightness/recompression-robust) -> nx_phash (nx_phash_dhash)
12// * bit-distance + near-dup clustering -> nx_simhash (nx_simhash_hamming)
13// dHash takes a grayscale (w x h, row-major u8) buffer and is FORMAT-AGNOSTIC by design, so at_phash
14// fingerprints any decoded grayscale media plane (the decode step is the caller's; this layer is the
15// fingerprint + grouping, kept pure/deterministic).
16//
17// No hardware/persistent-firmware writes (Rule 26). license_tier: ORIGINAL
18import "nx_syscalls.nx"
19import "nx_phash.nx"
20import "nx_simhash.nx"
21
22// at_phash: perceptual fingerprint of a decoded grayscale media buffer (w x h, row-major u8) -> a
23// 64-bit dHash. REUSES nx_phash_dhash verbatim (Krawetz 2011: encodes the RELATIVE order of adjacent
24// pixels -> invariant to uniform brightness/contrast, robust to recompression/resize). Deterministic:
25// same pixels -> same fingerprint, everywhere.
26func at_phash(gray: *u8, w: i64, h: i64) -> i64 {
27 return nx_phash_dhash(gray, w, h)
28}
29
30// at_hamming: bit distance between two perceptual hashes (0 = visually identical fingerprint, 64 =
31// maximally different). REUSES the SimHash SWAR-popcount Hamming kernel.
32func at_hamming(h1: i64, h2: i64) -> i64 {
33 return nx_simhash_hamming(h1, h2)
34}
35
36// at_neardup: group near-duplicate assets by perceptual fingerprint. hashes[0..n) are the per-asset
37// dHashes; two assets are near-dups iff at_hamming < threshold (STRICT: < threshold, so threshold=1
38// means "Hamming 0 only" = perceptually identical). out_groups[i] = the group id of asset i = the
39// index of the EARLIEST asset within `threshold` of it (its canonical representative), or i itself if
40// it starts a new group. Assets sharing a group id are the near-dup cluster (modeled on
41// nx_simhash_cluster's earliest-canonical assignment, exposed as explicit group ids). Returns the
42// number of DISTINCT groups (= unique-after-near-dup count). Pure/deterministic; O(n^2) over n hashes.
43func at_neardup(hashes: *i64, n: i64, threshold: i64, out_groups: *i64) -> i64 {
44 var ngroups: i64 = 0
45 var i: i64 = 0
46 while i < n {
47 var assigned: i64 = 0 - 1
48 var j: i64 = 0
49 while j < i {
50 if assigned < 0 {
51 if at_hamming(hashes[i], hashes[j]) < threshold {
52 assigned = out_groups[j]
53 }
54 }
55 j = j + 1
56 }
57 if assigned < 0 {
58 out_groups[i] = i
59 ngroups = ngroups + 1
60 } else {
61 out_groups[i] = assigned
62 }
63 i = i + 1
64 }
65 return ngroups
66}
67
68// at_in_same_group: convenience predicate -- are assets a and b in the same near-dup group? (after
69// at_neardup populated out_groups). 1 if same cluster, else 0.
70func at_in_same_group(out_groups: *i64, a: i64, b: i64) -> i64 {
71 if out_groups[a] == out_groups[b] { return 1 }
72 return 0
73}