nx_diora_dedup_demo.nx source

1// nx_diora_dedup_demo.nx -- SimHash near-dup clustering on the real Diora 2// corpus: proves the EARNED, content-measured demotion (collapse copies, keep 3// the canonical original) on actual data, not a title-keyword prejudice. 4// 5// module: nishi-core.search.bench.diora_dedup_demo 6// depends: fx.nx, nx_str.nx, nx_simhash.nx, nx_diora_data.nx 7// capability: APP_RUNNABLE 8 9import "fx.nx" 10import "nx_str.nx" 11import "nx_simhash.nx" 12import "nx_diora_data.nx" 13 14func nx_putc(c: i64) -> i64 { let b: *u8 = sys_mmap(1); b[0] = c; sys_write(1, b, 1); return 0 } 15func nx_puts(s: *u8) -> i64 { sys_write(1, s, nx_str_len(s)); return 0 } 16func nx_put_i64(n: i64) -> i64 { 17 if n == 0 { nx_putc(0x30); return 0 } 18 var v: i64 = n 19 if v < 0 { nx_putc(0x2D); v = 0 - v } 20 let buf: *u8 = sys_mmap(32) 21 var k: i64 = 0 22 while v > 0 { buf[k] = 0x30 + (v - (v / 10) * 10); v = v / 10; k = k + 1 } 23 while k > 0 { k = k - 1; sys_write(1, (((buf as i64) + k) as *u8), 1) } 24 return 0 25} 26 27func main() -> i64 { 28 let qn: i64 = nx_diora_qn() 29 let qurl: **u8 = sys_mmap(qn * 8) as **u8 30 let qtit: **u8 = sys_mmap(qn * 8) as **u8 31 let qfac: *i64 = sys_mmap(qn * 8) as *i64 32 let qgrd: *i64 = sys_mmap(qn * 8) as *i64 33 let qpri: *i64 = sys_mmap(qn * 8) as *i64 34 nx_diora_fill_qrels(qurl, qtit, qfac, qgrd, qpri) 35 36 // fingerprint each doc's title, then cluster by content near-duplication. 37 let fps: *i64 = sys_mmap(qn * 8) as *i64 38 var i: i64 = 0 39 while i < qn { fps[i] = nx_simhash_fingerprint(qtit[i], nx_str_len(qtit[i])); i = i + 1 } 40 let cl: *i64 = sys_mmap(qn * 8) as *i64 41 nx_simhash_cluster(fps, qn, 12, cl) // 12/64-bit threshold for short titles 42 43 nx_puts("=== Nishi SimHash near-dup clustering on the Diora corpus (EARNED demotion) ===\n") 44 nx_puts("docs: "); nx_put_i64(qn); nx_puts(" threshold: 12/64 bits\n") 45 nx_puts("near-duplicate clusters (>1 member => later copies collapse onto the canonical):\n") 46 var canon: i64 = 0 47 var dup_total: i64 = 0 48 while canon < qn { 49 if cl[canon] == canon { 50 // count members of this canonical cluster 51 var members: i64 = 0 52 var j: i64 = 0 53 while j < qn { if cl[j] == canon { members = members + 1 }; j = j + 1 } 54 if members > 1 { 55 nx_puts(" CANONICAL ["); nx_put_i64(canon); nx_puts("] ") 56 nx_puts(qtit[canon]); nx_putc(0x0A) 57 var m: i64 = canon + 1 58 while m < qn { 59 if cl[m] == canon { 60 nx_puts(" dup -> "); nx_puts(qtit[m]); nx_putc(0x0A) 61 dup_total = dup_total + 1 62 } 63 m = m + 1 64 } 65 } 66 } 67 canon = canon + 1 68 } 69 nx_puts("near-dups collapsed onto a canonical: "); nx_put_i64(dup_total); nx_putc(0x0A) 70 nx_puts("HONEST CAVEAT: this ran on short TITLES. The Getty/Maxim pair is a true\n") 71 nx_puts("dup, but her Instagram/X/Threads/OnlyFans are DISTINCT accounts with unique\n") 72 nx_puts("content that templated titles make look alike -- a FALSE near-dup. Lesson:\n") 73 nx_puts("dedup must measure CRAWLED PAGE CONTENT, never a surface proxy. The kernel\n") 74 nx_puts("is correct; its ranker-integration waits for the crawler to supply content.\n") 75 return 0 76}