nx_curate.nx source

1// nx_curate.nx -- MLIB-005 (CALLOUT-012). The CURATION / DEDUP / QUALITY-JUDGE engine 2// ("detectors in reverse", COMP-003 pattern): mechanical keep/remove decisions over a media 3// corpus. Powers find-fast + remove-fast. Composes the PROVEN nx_referee_v2 (rf2_tpr/tnr/ 4// testset_trustworthy) for precision/recall -- the engine does NOT self-grade. 5// - EXACT dedup: same content fingerprint (MLIB-003 nx_media_record CID) -> duplicate. 6// - NEAR dedup: perceptual-hash Hamming distance <= NEAR_THRESH -> near-duplicate. 7// - QUALITY: keep-score < QUAL_THRESH -> low-quality remove candidate (thresholds = config, 8// rule 11; here as named consts, store-backed later). 9// - CORRUPTION: corrupt flag -> remove candidate. 10// REMOVE decisions are emitted as SOFT-DELETE PROPOSALS (is_current=0, rule 13 -- ADDITIVE, 11// NEVER hard delete; the operator/store keeps history). Self-validating gate (CURATEGATE): 12// precision/recall == 1000permil on a seeded ground-truth corpus + near/far/exact controls. 13// license_tier: ORIGINAL 14import "nx_syscalls.nx" 15import "nx_referee_v2.nx" 16 17const CU_N: i64 = 7 18const NEAR_THRESH: i64 = 4 // <=4 differing bits in the phash = near-duplicate 19const QUAL_THRESH: i64 = 500 // keep-score below this = low-quality remove candidate 20 21func cu_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 } 22func cu_putn(v: i64) -> i64 { let bb: *u8 = sys_mmap(28); var m: i64=v; if m<0 {m=0-m}; let t: *u8 = sys_mmap(28); var k: i64=0; if m==0 {t[0]=48;k=1}; while m>0 {t[k]=48+(m%10); m=m/10; k=k+1}; var i: i64=0; while i<k {bb[i]=t[k-1-i]; i=i+1}; sys_write(1, bb, k); return 0 } 23 24func cu_streq(a: *u8, b: *u8) -> i64 { var i: i64 = 0; var s: i64 = 1; while s == 1 { if a[i] != b[i] { s = 0 } else { if a[i] == (0 as u8) { return 1 } else { i = i + 1 } } } return 0 } 25func cu_hexval(c: i64) -> i64 { if c >= 97 { return (c - 97) + 10 } if c >= 65 { return (c - 65) + 10 } return c - 48 } 26func cu_pop4(x: i64) -> i64 { var c: i64 = 0; var v: i64 = x; var i: i64 = 0; while i < 4 { c = c + (v & 1); v = v >> 1; i = i + 1 } return c } 27 28// Hamming distance (in bits) between two equal-length hex phash strings. 29func cu_hamming(a: *u8, b: *u8) -> i64 { 30 var d: i64 = 0 31 var i: i64 = 0 32 while a[i] != (0 as u8) { if b[i] == (0 as u8) { return d } d = d + cu_pop4(cu_hexval(a[i] as i64) ^ cu_hexval(b[i] as i64)); i = i + 1 } 33 return d 34} 35 36func cu_isneardup(a: *u8, b: *u8) -> i64 { if cu_hamming(a, b) <= NEAR_THRESH { return 1 } return 0 } 37 38func main() -> i64 { 39 let cid: *i64 = sys_mmap(8 * CU_N) as *i64 40 let ph: *i64 = sys_mmap(8 * CU_N) as *i64 41 let qual: *i64 = sys_mmap(8 * CU_N) as *i64 42 let corr: *i64 = sys_mmap(8 * CU_N) as *i64 43 let gt: *i64 = sys_mmap(8 * CU_N) as *i64 // ground-truth: 1 = should be REMOVED 44 let dec: *i64 = sys_mmap(8 * CU_N) as *i64 45 let rsn: *i64 = sys_mmap(8 * CU_N) as *i64 // reason string ptr 46 47 // seeded corpus (ground truth): 0 keep, 1 exact-dup, 2 near-dup, 3 keep, 4 low-qual, 5 corrupt, 6 keep 48 cid[0]=("nxc1-aaa" as *u8) as i64; ph[0]=("ab12cd34" as *u8) as i64; qual[0]=900; corr[0]=0; gt[0]=0 49 cid[1]=("nxc1-aaa" as *u8) as i64; ph[1]=("ab12cd34" as *u8) as i64; qual[1]=900; corr[1]=0; gt[1]=1 50 cid[2]=("nxc1-bbb" as *u8) as i64; ph[2]=("ab12cd35" as *u8) as i64; qual[2]=900; corr[2]=0; gt[2]=1 51 cid[3]=("nxc1-ccc" as *u8) as i64; ph[3]=("00000000" as *u8) as i64; qual[3]=900; corr[3]=0; gt[3]=0 52 cid[4]=("nxc1-ddd" as *u8) as i64; ph[4]=("ff00ff00" as *u8) as i64; qual[4]=100; corr[4]=0; gt[4]=1 53 cid[5]=("nxc1-eee" as *u8) as i64; ph[5]=("12345678" as *u8) as i64; qual[5]=900; corr[5]=1; gt[5]=1 54 cid[6]=("nxc1-fff" as *u8) as i64; ph[6]=("abcdef01" as *u8) as i64; qual[6]=900; corr[6]=0; gt[6]=0 55 56 var i: i64 = 0 57 while i < CU_N { 58 var rm: i64 = 0 59 rsn[i] = ("keep" as *u8) as i64 60 var j: i64 = 0 61 while j < i { 62 if cu_streq((cid[i]) as *u8, (cid[j]) as *u8) == 1 { rm = 1; rsn[i] = ("exact-dup" as *u8) as i64 } 63 else { if cu_isneardup((ph[i]) as *u8, (ph[j]) as *u8) == 1 { rm = 1; rsn[i] = ("near-dup" as *u8) as i64 } } 64 j = j + 1 65 } 66 if corr[i] == 1 { rm = 1; rsn[i] = ("corrupt" as *u8) as i64 } 67 if qual[i] < QUAL_THRESH { rm = 1; rsn[i] = ("low-quality" as *u8) as i64 } 68 dec[i] = rm 69 i = i + 1 70 } 71 72 // emit SOFT-DELETE PROPOSALS (additive; is_current=0; never hard delete -- rule 13) 73 i = 0 74 while i < CU_N { 75 if dec[i] == 1 { cu_puts("REMOVE-PROPOSAL item=" as *u8); cu_putn(i); cu_puts(" reason=" as *u8); cu_puts((rsn[i]) as *u8); cu_puts(" action=soft-delete is_current=0 (additive, operator-confirms)\n" as *u8) } 76 i = i + 1 77 } 78 79 var kept_ok: i64 = 0; var keeps: i64 = 0 80 var rem_ok: i64 = 0; var rems: i64 = 0 81 i = 0 82 while i < CU_N { 83 if gt[i] == 0 { keeps = keeps + 1; if dec[i] == 0 { kept_ok = kept_ok + 1 } } 84 else { rems = rems + 1; if dec[i] == 1 { rem_ok = rem_ok + 1 } } 85 i = i + 1 86 } 87 let tpr: i64 = rf2_tpr(kept_ok, keeps) // keeps correctly kept / total keeps 88 let tnr: i64 = rf2_tnr(rem_ok, rems) // removes correctly caught / total removes 89 let trust: i64 = rf2_testset_trustworthy(tpr, tnr, 900) 90 91 let ctrl_near: i64 = cu_isneardup((ph[0]) as *u8, (ph[2]) as *u8) // item2 = 1 bit off item0 -> near 92 let ctrl_far: i64 = cu_isneardup((ph[0]) as *u8, (ph[3]) as *u8) // item3 far -> NOT near 93 let ctrl_exact: i64 = cu_streq((cid[0]) as *u8, (cid[1]) as *u8) // item1 exact dup of item0 94 95 cu_puts("CURATEGATE keeps_correct=" as *u8); cu_putn(kept_ok); cu_puts("/" as *u8); cu_putn(keeps) 96 cu_puts(" removes_correct=" as *u8); cu_putn(rem_ok); cu_puts("/" as *u8); cu_putn(rems) 97 cu_puts(" tpr=" as *u8); cu_putn(tpr); cu_puts(" tnr=" as *u8); cu_putn(tnr) 98 cu_puts(" near_ctrl=" as *u8); cu_putn(ctrl_near); cu_puts(" far_ctrl=" as *u8); cu_putn(ctrl_far); cu_puts(" exact_ctrl=" as *u8); cu_putn(ctrl_exact) 99 100 var ok: i64 = 1 101 if trust != 1 { ok = 0 } 102 if tpr != 1000 { ok = 0 } 103 if tnr != 1000 { ok = 0 } 104 if ctrl_near != 1 { ok = 0 } 105 if ctrl_far != 0 { ok = 0 } 106 if ctrl_exact != 1 { ok = 0 } 107 if ok == 1 { cu_puts(" verdict=GREEN\n" as *u8); return 0 } 108 cu_puts(" verdict=RED\n" as *u8) 109 return 1 110}