code wiki / (root) / nx_imgsearch_engine_gate.nx

nx_imgsearch_engine_gate.nx source

↩ module page · 379 lines · 23042 B

1// nx_imgsearch_engine_gate.nx -- referee for the multi-tier reverse-image engine. 2// 3// The ruler (nx_imgbench) says WHERE we are weak. This gate proves the fixes are REAL, and it does so 4// adversarially: the deformations are applied by nx_imgxform (the attacker) while the invariance is 5// supplied by nx_imgorient (the defender), which implements its geometry from scratch. A shared 6// implementation would let one sign error cancel itself out and score as a pass. 7// 8// row1 D4 CANONICAL INVARIANCE -- canonical(g.A) == canonical(A) for all 8 group elements, all 9// images. This is the mathematical claim the orientation tier rests on; if it is false the 10// tier is a coincidence generator. Also the only row that can catch the SIGNED-COMPARE trap 11// (a fingerprint with bit 63 set reads negative; a signed min picks a different orbit 12// representative per variant and invariance silently breaks). 13// row2 ORBIT NON-DEGENERACY -- the 8 orientations of an asymmetric image really are different 14// images. Without this, row1 could pass trivially by transforming nothing. 15// row3 HEAP EXACTNESS -- the O(N log K) bounded-heap top-K returns the SAME distance sequence as an 16// O(N^2) selection-sort reference. Speed that changes the answer is not speed. 17// row4 MIRROR RECOVERY -- measured against the ruler's baseline of 0 permille. 18// row5 CROP RECOVERY -- measured against the ruler's baseline of 31 permille at crop-20. 19// row6 FUSION ORDERING -- fused results come back in non-increasing score order. 20// row7 HONEST ABSENT -- an image that is not in the corpus returns -1, not the nearest thing anyway. 21// row8 DETERMINISM -- same query twice, identical payloads and scores, bit for bit. 22// 23// Also DERIVES (does not assume) the similarity tier's match threshold by measuring the separation 24// between true-source L1 and unrelated-image L1, so the constant is data-driven per the no-magic- 25// numbers law rather than chosen by taste. 26// GREEN iff 8/8. Durable verdict -> knowledge/status/imgsearch_engine_gate.log. license_tier: ORIGINAL 27import "nx_imgsearch_engine.nx" 28import "nx_imgxform.nx" 29import "nx_imgcorpus.nx" 30import "nx_gate_verdict.nx" 31 32const EG_N: i64 = 128 // corpus size 33const EG_W: i64 = 64 34const EG_H: i64 = 64 35const EG_M: i64 = 24 // images used for the invariance sweep 36const EG_Q: i64 = 24 // queries used for the recovery rows 37const EG_K: i64 = 8 // top-K 38 39func g_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 } 40func g_num(v: i64) -> i64 { let bb: *u8 = sys_mmap(28); var m: i64 = v; if m < 0 { m = 0 - m; sys_write(1, "-" as *u8, 1) }; let t: *u8 = sys_mmap(28); var k: i64 = 0; if m == 0 { t[0] = (48 as u8); k = 1 }; while m > 0 { t[k] = ((48 + (m % 10)) as u8); m = m / 10; k = k + 1 }; var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 }; sys_write(1, bb, k); return 0 } 41func g_w(fd: i64, s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(fd, s, n); return 0 } 42func g_wn(fd: i64, v: i64) -> i64 { let bb: *u8 = sys_mmap(28); var m: i64 = v; if m < 0 { m = 0 - m }; let t: *u8 = sys_mmap(28); var k: i64 = 0; if m == 0 { t[0] = (48 as u8); k = 1 }; while m > 0 { t[k] = ((48 + (m % 10)) as u8); m = m / 10; k = k + 1 }; var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 }; sys_write(fd, bb, k); return 0 } 43func g_permille(a: i64, b: i64) -> i64 { if b <= 0 { return 0 } return a * 1000 / b } 44 45func main() -> i64 { 46 g_puts("=== IMGSEARCH ENGINE GATE (multi-tier fusion, D4 invariance, heap exactness) ===\n" as *u8) 47 var pass: i64 = 0 48 let rows: i64 = 13 49 let wh: *i64 = sys_mmap(16) as *i64 50 51 // ---- corpus ---- 52 let imgs: *i64 = sys_mmap(8 * EG_N) as *i64 53 var i: i64 = 0 54 while i < EG_N { imgs[i] = nx_imgcorpus_new(i, EG_W, EG_H) as i64; i = i + 1 } 55 56 // ---- row1: D4 canonical invariance (the defender's own geometry) ---- 57 var inv_ok: i64 = 1 58 var inv_checked: i64 = 0 59 i = 0 60 while i < EG_M { 61 let src: *u8 = imgs[i] as *u8 62 let base: i64 = nx_orient_canonical(src, EG_W, EG_H) 63 var k: i64 = 0 64 while k < nx_orient_count() { 65 let v: *u8 = nx_orient_apply(k, src, EG_W, EG_H, wh) 66 let c: i64 = nx_orient_canonical(v, wh[0], wh[1]) 67 if c != base { inv_ok = 0 } 68 inv_checked = inv_checked + 1 69 k = k + 1 70 } 71 i = i + 1 72 } 73 g_puts(" row1 D4 canonical invariance (" as *u8); g_num(inv_checked); g_puts(" variants) -> " as *u8) 74 if inv_ok == 1 { pass = pass + 1; g_puts("PASS\n" as *u8) } else { g_puts("FAIL\n" as *u8) } 75 76 // ---- row2: orbit non-degeneracy (the transforms really transform) ---- 77 let h8: *i64 = sys_mmap(8 * 8) as *i64 78 nx_orient_hashes(imgs[3] as *u8, EG_W, EG_H, h8) 79 var distinct: i64 = 0 80 var a: i64 = 0 81 while a < 8 { 82 var seen: i64 = 0 83 var b: i64 = 0 84 while b < a { if h8[b] == h8[a] { seen = 1 } b = b + 1 } 85 if seen == 0 { distinct = distinct + 1 } 86 a = a + 1 87 } 88 g_puts(" row2 orbit non-degeneracy -> " as *u8); g_num(distinct); g_puts(" of 8 distinct -> " as *u8) 89 if distinct >= 6 { pass = pass + 1; g_puts("PASS\n" as *u8) } else { g_puts("FAIL\n" as *u8) } 90 91 // ---- build the engine: four tiers, one base class ---- 92 let eng: *nx_imgengine = nx_imgengine_new(EG_N) 93 nx_imgengine_add_tier(eng, nx_imgtier_new(NX_IT_KIND_COPY, NX_IT_COPY_THRESH, 1000)) 94 nx_imgengine_add_tier(eng, nx_imgtier_new(NX_IT_KIND_ORIENT, NX_IT_COPY_THRESH, 1000)) 95 nx_imgengine_add_tier(eng, nx_imgtier_new(NX_IT_KIND_LOCAL, NX_IT_LOCAL_THRESH, 900)) 96 nx_imgengine_add_tier(eng, nx_imgtier_new(NX_IT_KIND_SIMILAR, NX_IT_SIMILAR_THRESH, 800)) 97 i = 0 98 while i < EG_N { nx_imgengine_add_image(eng, imgs[i] as *u8, 0 as *u8, EG_W, EG_H, 10000 + i); i = i + 1 } 99 g_puts(" engine: tiers=" as *u8); g_num(nx_imgengine_ntiers(eng)) 100 g_puts(" indexed=" as *u8); g_num(nx_imgengine_count(eng)); g_puts("\n" as *u8) 101 102 // ---- row3: BK-ACCELERATED hash tier == exact within-threshold set, examining << N ---- 103 // The copy tier is BK-tree accelerated. It must return the SAME within-threshold candidate set a 104 // linear O(N) scan would (exactness by triangle inequality -- no recall loss), while TOUCHING a 105 // sublinear fraction of the corpus. This is the claim that makes the engine scale past a linear 106 // scan; asserting it against the brute-force set is the liar-killer for "fast but wrong". 107 let ti_item: *i64 = sys_mmap(8 * EG_K) as *i64 108 let ti_dist: *i64 = sys_mmap(8 * EG_K) as *i64 109 let t0: *nx_imgtier = nx_imgengine_tier(eng, 0) 110 let qd0: *i64 = sys_mmap(8 * nx_imgtier_qry_dim(t0)) as *i64 111 let radius0: i64 = nx_imgtier_threshold(t0) 112 var bk_ok: i64 = 1 113 var tot_visited: i64 = 0 114 var qi: i64 = 0 115 while qi < 6 { 116 let q: *u8 = xf_apply(XF_NOISE, imgs[qi * 7] as *u8, EG_W, EG_H, wh) 117 nx_imgtier_describe_query(t0, q, 0 as *u8, wh[0], wh[1], qd0) 118 let nh: i64 = nx_imgengine_tier_topk(eng, 0, qd0, EG_K, ti_item, ti_dist) 119 tot_visited = tot_visited + nx_imgengine_last_visited(eng) 120 // brute-force within-threshold count -- the exact set the BK path must reproduce 121 var bf: i64 = 0 122 i = 0 123 while i < EG_N { if nx_imgtier_distance(t0, qd0, ie_desc_at(eng, 0, i)) <= radius0 { bf = bf + 1 } i = i + 1 } 124 var kk_exp: i64 = bf 125 if kk_exp > EG_K { kk_exp = EG_K } // heap keeps at most K; the true match dominates 126 if nh != kk_exp { bk_ok = 0 } 127 // every returned item is genuinely within threshold and its reported distance is its true one 128 var r: i64 = 0 129 while r < nh { 130 if ti_dist[r] > radius0 { bk_ok = 0 } 131 if nx_imgtier_distance(t0, qd0, ie_desc_at(eng, 0, ti_item[r])) != ti_dist[r] { bk_ok = 0 } 132 r = r + 1 133 } 134 // the true source (query qi*7 with noise) must be found 135 if nh > 0 { if ti_item[0] != qi * 7 { bk_ok = 0 } } else { bk_ok = 0 } 136 qi = qi + 1 137 } 138 let avg_vis: i64 = tot_visited / 6 139 g_puts(" row3 BK-accel == exact within-threshold set, examining " as *u8); g_num(avg_vis) 140 g_puts(" of N=" as *u8); g_num(EG_N); g_puts(" -> " as *u8) 141 if bk_ok == 1 { if avg_vis < EG_N { pass = pass + 1; g_puts("PASS\n" as *u8) } else { g_puts("FAIL (not sublinear)\n" as *u8) } } else { g_puts("FAIL (candidate set != linear)\n" as *u8) } 142 g_puts(" sublinear candidate generation: ~" as *u8) 143 if avg_vis > 0 { g_num(EG_N / avg_vis) } else { g_num(0) } 144 g_puts("x fewer comparisons than the linear scan (N=" as *u8); g_num(EG_N) 145 g_puts(")\n" as *u8) 146 147 let op: *i64 = sys_mmap(8 * EG_K) as *i64 148 let os: *i64 = sys_mmap(8 * EG_K) as *i64 149 let ot: *i64 = sys_mmap(8 * EG_K) as *i64 150 let od: *i64 = sys_mmap(8 * EG_K) as *i64 151 152 // ---- row4: MIRROR RECOVERY (ruler baseline: 0 permille) ---- 153 var mhit: i64 = 0 154 qi = 0 155 while qi < EG_Q { 156 let q: *u8 = xf_apply(XF_FLIPH, imgs[qi] as *u8, EG_W, EG_H, wh) 157 let n: i64 = nx_imgengine_query(eng, q, 0 as *u8, wh[0], wh[1], EG_K, op, os, ot, od) 158 if n > 0 { if op[0] == 10000 + qi { mhit = mhit + 1 } } 159 qi = qi + 1 160 } 161 let mrec: i64 = g_permille(mhit, EG_Q) 162 g_puts(" row4 mirror recovery -> recall@1 " as *u8); g_num(mrec) 163 g_puts(" permille (ruler baseline 0) -> " as *u8) 164 if mrec >= 900 { pass = pass + 1; g_puts("PASS\n" as *u8) } else { g_puts("FAIL\n" as *u8) } 165 166 // ---- row5: CROP RECOVERY (ruler baseline: 31 permille at crop-20) + threshold derivation ---- 167 var chit: i64 = 0 168 var l1_true: i64 = 0 169 var l1_other: i64 = 0 170 let tsim: *nx_imgtier = nx_imgengine_tier(eng, 3) 171 let qds: *i64 = sys_mmap(8 * nx_imgtier_qry_dim(tsim)) as *i64 172 qi = 0 173 while qi < EG_Q { 174 let q: *u8 = xf_apply(XF_CROP20, imgs[qi] as *u8, EG_W, EG_H, wh) 175 let n: i64 = nx_imgengine_query(eng, q, 0 as *u8, wh[0], wh[1], EG_K, op, os, ot, od) 176 if n > 0 { if op[0] == 10000 + qi { chit = chit + 1 } } 177 nx_imgtier_describe_query(tsim, q, 0 as *u8, wh[0], wh[1], qds) 178 l1_true = l1_true + nx_imgtier_distance(tsim, qds, ie_desc_at(eng, 3, qi)) 179 l1_other = l1_other + nx_imgtier_distance(tsim, qds, ie_desc_at(eng, 3, (qi + 61) % EG_N)) 180 qi = qi + 1 181 } 182 let crec: i64 = g_permille(chit, EG_Q) 183 let mt: i64 = l1_true / EG_Q 184 let mo: i64 = l1_other / EG_Q 185 g_puts(" row5 crop-20 recovery -> recall@1 " as *u8); g_num(crec) 186 g_puts(" permille (ruler baseline 31) -> " as *u8) 187 if crec >= 900 { pass = pass + 1; g_puts("PASS\n" as *u8) } else { g_puts("FAIL\n" as *u8) } 188 g_puts(" DERIVED similarity threshold: mean L1 true-source=" as *u8); g_num(mt) 189 g_puts(" unrelated=" as *u8); g_num(mo) 190 g_puts(" -> midpoint=" as *u8); g_num((mt + mo) / 2); g_puts("\n" as *u8) 191 192 // ---- row9: the shipped threshold constant is still the DERIVED one, not a stale guess ---- 193 // This is the no-magic-numbers law mechanised: the populations the constant separates are 194 // re-measured every run, so a descriptor or corpus change that invalidates it fails loudly. 195 g_puts(" row9 threshold derivation holds (true " as *u8); g_num(mt) 196 g_puts(" < NX_IT_SIMILAR_THRESH " as *u8); g_num(NX_IT_SIMILAR_THRESH) 197 g_puts(" < unrelated " as *u8); g_num(mo); g_puts(") -> " as *u8) 198 if mt < NX_IT_SIMILAR_THRESH { if NX_IT_SIMILAR_THRESH < mo { pass = pass + 1; g_puts("PASS\n" as *u8) } else { g_puts("FAIL\n" as *u8) } } else { g_puts("FAIL\n" as *u8) } 199 200 // ---- row6: fusion ordering (non-increasing fused score) ---- 201 let q6: *u8 = xf_apply(XF_BLOCKQ, imgs[11] as *u8, EG_W, EG_H, wh) 202 let n6: i64 = nx_imgengine_query(eng, q6, 0 as *u8, wh[0], wh[1], EG_K, op, os, ot, od) 203 var ord_ok: i64 = 1 204 i = 1 205 while i < n6 { if os[i] > os[i - 1] { ord_ok = 0 } i = i + 1 } 206 g_puts(" row6 fusion ordering (" as *u8); g_num(n6); g_puts(" results, non-increasing) -> " as *u8) 207 if ord_ok == 1 { if n6 > 1 { pass = pass + 1; g_puts("PASS\n" as *u8) } else { g_puts("FAIL (too few)\n" as *u8) } } else { g_puts("FAIL\n" as *u8) } 208 209 // ---- row7: honest ABSENT for an image that is not in the corpus ---- 210 let stranger: *u8 = nx_imgcorpus_new(90210, EG_W, EG_H) 211 let bt: *i64 = sys_mmap(8) as *i64 212 let bd: *i64 = sys_mmap(8) as *i64 213 let bm: i64 = nx_imgengine_best_match(eng, stranger, 0 as *u8, EG_W, EG_H, bt, bd) 214 // Two properties at once, because fixing the first by deleting the second would be a regression: 215 // (a) the IDENTITY question honestly answers ABSENT -- this image is not in the corpus; 216 // (b) the SIMILARITY question still returns its nearest look-alikes -- the capability is 217 // scoped, not stripped. An engine that answers "no match" by going silent is less useful 218 // than one that answers "no match, but these are the closest things I have". 219 let ns: i64 = nx_imgengine_query(eng, stranger, 0 as *u8, EG_W, EG_H, EG_K, op, os, ot, od) 220 g_puts(" row7 out-of-corpus image -> identity best_match=" as *u8); g_num(bm) 221 g_puts(" (nearest identity dist=" as *u8); g_num(bd[0]) 222 g_puts("), similarity results=" as *u8); g_num(ns); g_puts(" -> " as *u8) 223 if bm < 0 { if ns > 0 { pass = pass + 1; g_puts("PASS (honest ABSENT, look-alikes retained)\n" as *u8) } else { g_puts("FAIL (similarity capability lost)\n" as *u8) } } else { g_puts("FAIL (claimed an identity match)\n" as *u8) } 224 225 // ---- row8: determinism ---- 226 let p2: *i64 = sys_mmap(8 * EG_K) as *i64 227 let s2: *i64 = sys_mmap(8 * EG_K) as *i64 228 let t2: *i64 = sys_mmap(8 * EG_K) as *i64 229 let d2: *i64 = sys_mmap(8 * EG_K) as *i64 230 let qq: *u8 = xf_apply(XF_CROP10, imgs[5] as *u8, EG_W, EG_H, wh) 231 let na: i64 = nx_imgengine_query(eng, qq, 0 as *u8, wh[0], wh[1], EG_K, op, os, ot, od) 232 let nb: i64 = nx_imgengine_query(eng, qq, 0 as *u8, wh[0], wh[1], EG_K, p2, s2, t2, d2) 233 var det_ok: i64 = 1 234 if na != nb { det_ok = 0 } 235 i = 0 236 while i < na { if op[i] != p2[i] { det_ok = 0 } if os[i] != s2[i] { det_ok = 0 } i = i + 1 } 237 g_puts(" row8 determinism (same query twice, " as *u8); g_num(na); g_puts(" results) -> " as *u8) 238 if det_ok == 1 { pass = pass + 1; g_puts("PASS\n" as *u8) } else { g_puts("FAIL\n" as *u8) } 239 240 // ---- row10: structureless images are REFUSED, not confidently mismatched ---- 241 // Regression guard for a defect found on real data: eleven real PNGs indexed, a held-out query 242 // returned THREE different images at dHash distance 0 as confident matches, because flat and 243 // smooth-gradient images all hash to zero and collide. Two halves, because refusing everything 244 // would also "fix" it: a flat image must be refused AND a structured one must still match. 245 let flat: *u8 = sys_mmap(EG_W * EG_H) 246 i = 0 247 while i < EG_W * EG_H { flat[i] = 128 as u8; i = i + 1 } 248 let ramp: *u8 = sys_mmap(EG_W * EG_H) 249 var ry: i64 = 0 250 while ry < EG_H { 251 var rx: i64 = 0 252 while rx < EG_W { ramp[ry * EG_W + rx] = ((rx * 255) / EG_W) as u8; rx = rx + 1 } 253 ry = ry + 1 254 } 255 let rr: *i64 = sys_mmap(8) as *i64 256 let fm: i64 = nx_imgengine_best_match_r(eng, flat, 0 as *u8, EG_W, EG_H, bt, bd, rr) 257 let flat_reason: i64 = rr[0] 258 let rm: i64 = nx_imgengine_best_match_r(eng, ramp, 0 as *u8, EG_W, EG_H, bt, bd, rr) 259 let ramp_reason: i64 = rr[0] 260 let gm: i64 = nx_imgengine_best_match_r(eng, imgs[9] as *u8, 0 as *u8, EG_W, EG_H, bt, bd, rr) 261 g_puts(" row10 structureless refused / structured still matched -> flat=" as *u8); g_num(fm) 262 g_puts("(reason " as *u8); g_num(flat_reason); g_puts(") ramp=" as *u8); g_num(rm) 263 g_puts("(reason " as *u8); g_num(ramp_reason); g_puts(") real=" as *u8); g_num(gm); g_puts(" -> " as *u8) 264 if fm < 0 { if flat_reason == 2 { if rm < 0 { if ramp_reason == 2 { if gm == 10009 { pass = pass + 1; g_puts("PASS\n" as *u8) } else { g_puts("FAIL (structured image no longer matches)\n" as *u8) } } else { g_puts("FAIL (gradient not refused as uninformative)\n" as *u8) } } else { g_puts("FAIL (gradient claimed a match)\n" as *u8) } } else { g_puts("FAIL (flat refused for the wrong reason)\n" as *u8) } } else { g_puts("FAIL (flat image claimed a match)\n" as *u8) } 265 266 // ---- row11: LOCAL geometric verification is REAL, not descriptor coincidence (the liar-killer) 267 // A crop-30 of the true source must produce MANY MORE RANSAC inliers than the same query against 268 // an unrelated image. If they were comparable, the tier would be counting chance patch matches and 269 // its recall would be luck. This also DERIVES the confidence threshold rather than trusting it. 270 let tloc: *nx_imgtier = nx_imgengine_tier(eng, 2) 271 let qdl: *i64 = sys_mmap(8 * nx_imgtier_qry_dim(tloc)) as *i64 272 var inl_true: i64 = 0 273 var inl_decoy: i64 = 0 274 var loc_hit: i64 = 0 275 qi = 0 276 while qi < EG_Q { 277 let q: *u8 = xf_apply(XF_CROP30, imgs[qi] as *u8, EG_W, EG_H, wh) 278 nx_imgtier_describe_query(tloc, q, 0 as *u8, wh[0], wh[1], qdl) 279 // distance = KP_MAX - inliers, so inliers = KP_MAX - distance 280 let dt: i64 = nx_imgtier_distance(tloc, qdl, ie_desc_at(eng, 2, qi)) 281 let de: i64 = nx_imgtier_distance(tloc, qdl, ie_desc_at(eng, 2, (qi + 61) % EG_N)) 282 inl_true = inl_true + (NX_IT_LOCAL_MAX - dt) 283 inl_decoy = inl_decoy + (NX_IT_LOCAL_MAX - de) 284 if dt <= nx_imgtier_threshold(tloc) { loc_hit = loc_hit + 1 } 285 qi = qi + 1 286 } 287 let it2: i64 = inl_true / EG_Q 288 let id2: i64 = inl_decoy / EG_Q 289 g_puts(" row11 LOCAL inlier separation: crop-30 true-source=" as *u8); g_num(it2) 290 g_puts(" vs decoy=" as *u8); g_num(id2) 291 g_puts(" (thresh inliers=" as *u8); g_num(NX_IT_LOCAL_MININLIERS); g_puts(") -> " as *u8) 292 if id2 < NX_IT_LOCAL_MININLIERS { if NX_IT_LOCAL_MININLIERS <= it2 { pass = pass + 1; g_puts("PASS\n" as *u8) } else { g_puts("FAIL (true source below threshold)\n" as *u8) } } else { g_puts("FAIL (decoy reaches threshold -- matching noise)\n" as *u8) } 293 294 // ---- row12: crop-30 ENGINE recovery -- the ruler's last gap class. Measures the same thing the 295 // ruler does (full-engine recall@1) so the two numbers are comparable, and asserts the LOCAL tier 296 // lifts it clear of the global-only ceiling the ruler measured (visdesc-alone 593 permille). 297 var chit30: i64 = 0 298 qi = 0 299 while qi < EG_Q { 300 let q: *u8 = xf_apply(XF_CROP30, imgs[qi] as *u8, EG_W, EG_H, wh) 301 let n: i64 = nx_imgengine_query(eng, q, 0 as *u8, wh[0], wh[1], EG_K, op, os, ot, od) 302 if n > 0 { if op[0] == 10000 + qi { chit30 = chit30 + 1 } } 303 qi = qi + 1 304 } 305 let crec30: i64 = g_permille(chit30, EG_Q) 306 g_puts(" row12 crop-30 ENGINE recall@1 -> " as *u8); g_num(crec30) 307 g_puts(" permille (global-only ceiling 593) -> " as *u8) 308 if crec30 > 593 { pass = pass + 1; g_puts("PASS\n" as *u8) } else { g_puts("FAIL\n" as *u8) } 309 310 // ---- row13: BK pruning on a CLUSTERED corpus -- the scale win where it matters ---- 311 // row3 proved exactness on a DIVERSE corpus (the BK-tree's worst case, where it degrades toward 312 // linear -- honest). But a real reverse-image index is CLUSTERED: near-duplicate reposts, 313 // thumbnails, re-encodes. On clustered data the triangle inequality prunes the vast majority of 314 // nodes. Here: C base images x M near-duplicate noise variants; a query near one cluster must 315 // examine FAR fewer than N and STILL return its true cluster-mate. Proves the acceleration 316 // delivers the sublinearity on the data distribution reverse-image actually has. 317 let CN: i64 = 16 318 let MN: i64 = 8 319 let clus: *nx_imgengine = nx_imgengine_new(CN * MN) 320 nx_imgengine_add_tier(clus, nx_imgtier_new(NX_IT_KIND_COPY, NX_IT_COPY_THRESH, 1000)) 321 var cb: i64 = 0 322 while cb < CN { 323 var mv: i64 = 0 324 while mv < MN { 325 // variant = base image + light noise (a near-duplicate, dHash within a few Hamming) 326 let v: *u8 = xf_apply(XF_NOISE, imgs[cb] as *u8, EG_W, EG_H, wh) 327 nx_imgengine_add_image(clus, v, 0 as *u8, EG_W, EG_H, cb * 1000 + mv) 328 mv = mv + 1 329 } 330 cb = cb + 1 331 } 332 let ci: *i64 = sys_mmap(8 * EG_K) as *i64 333 let cd: *i64 = sys_mmap(8 * EG_K) as *i64 334 let tc0: *nx_imgtier = nx_imgengine_tier(clus, 0) 335 let cqd: *i64 = sys_mmap(8 * nx_imgtier_qry_dim(tc0)) as *i64 336 var clus_vis: i64 = 0 337 var clus_ok: i64 = 1 338 var cq: i64 = 0 339 while cq < CN { 340 let q: *u8 = xf_apply(XF_NOISE, imgs[cq] as *u8, EG_W, EG_H, wh) 341 nx_imgtier_describe_query(tc0, q, 0 as *u8, wh[0], wh[1], cqd) 342 let nn: i64 = nx_imgengine_tier_topk(clus, 0, cqd, EG_K, ci, cd) 343 clus_vis = clus_vis + nx_imgengine_last_visited(clus) 344 // tier_topk returns IMAGE INDICES; images were inserted cluster-major (cb*MN+mv), so the 345 // nearest's cluster is index/MN and must equal the queried cluster cq. 346 if nn > 0 { if ci[0] / MN != cq { clus_ok = 0 } } else { clus_ok = 0 } 347 cq = cq + 1 348 } 349 let clus_avg: i64 = clus_vis / CN 350 let clus_n: i64 = CN * MN 351 g_puts(" row13 BK pruning on CLUSTERED corpus: examined " as *u8); g_num(clus_avg) 352 g_puts(" of N=" as *u8); g_num(clus_n); g_puts(" (~" as *u8) 353 if clus_avg > 0 { g_num(clus_n / clus_avg) } else { g_num(0) } 354 g_puts("x fewer) correct-cluster -> " as *u8) 355 if clus_ok == 1 { if clus_avg * 2 < clus_n { pass = pass + 1; g_puts("PASS\n" as *u8) } else { g_puts("FAIL (no real pruning)\n" as *u8) } } else { g_puts("FAIL (wrong cluster)\n" as *u8) } 356 357 g_puts("----\nIMGSEARCH-ENGINE-GATE rows=" as *u8); g_num(rows) 358 g_puts(" pass=" as *u8); g_num(pass); g_puts("\n" as *u8) 359 let lg: i64 = sys_openat_append("knowledge/status/imgsearch_engine_gate.log" as *u8, 0x1a4) 360 if lg >= 0 { 361 g_w(lg, "IMGSEARCH-ENGINE-GATE rows=" as *u8); g_wn(lg, rows) 362 g_w(lg, " pass=" as *u8); g_wn(lg, pass) 363 g_w(lg, " mirror_recall=" as *u8); g_wn(lg, mrec) 364 g_w(lg, " crop20_recall=" as *u8); g_wn(lg, crec) 365 g_w(lg, " sim_l1_true=" as *u8); g_wn(lg, mt) 366 g_w(lg, " sim_l1_unrelated=" as *u8); g_wn(lg, mo) 367 if pass == rows { g_w(lg, " verdict=GREEN\n" as *u8) } else { g_w(lg, " verdict=RED\n" as *u8) } 368 sys_close(lg) 369 } 370 // MIGRATED onto nx_gate_verdict by nx_gate_dry_apply (D001, minimal form): every check 371 // row above is untouched, so the PASS/FAIL vector cannot change; only the hand-rolled 372 // verdict emission is replaced by the ONE shared base class. Proven by nx_gate_migrate verify. 373 let ctr__dry: *i64 = gv_ctr() 374 ctr__dry[0] = pass 375 ctr__dry[1] = rows 376 let rc__dry: i64 = gv_verdict("IMGSEARCH-ENGINE-GATE" as *u8, ctr__dry, "teeth unchanged; verdict emission migrated onto the shared base class" as *u8) 377 sys_exit(rc__dry) 378 return rc__dry 379}