nx_ppmi_svd_test.nx source

1// nx_ppmi_svd_test.nx -- PPMI fixes raw co-occurrence's frequent-term pollution. Corpus where "legal" 2// is in EVERY doc (frequent, uninformative) and co-occurs with "settlement" only at chance: 3// doc0 "settlement compensation legal" doc1 "settlement compensation legal" 4// doc2 "claim legal" doc3 "hearing legal" 5// Query "settlement": RAW co-occurrence falsely credits doc2/doc3 (they share the frequent "legal"); 6// PPMI scores them 0 (legal co-occurs at chance -> PMI 0) while keeping doc0 high (compensation is a 7// genuine association). Triangulated: PPMI(settlement,compensation) ~ ln(2) = 693147 micro. Exit 0 on 6/6. 8// license_tier: ORIGINAL 9 10import "nx_ppmi_svd.nx" 11import "nx_research_extract.nx" // re_strlen 12import "nx_syscalls.nx" 13 14func qt_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 } 15func qt_num(v: i64) -> i64 { let bb: *u8 = sys_mmap(28); var m: i64=v; if m<0 {m=0-m; sys_write(1,"-" as *u8,1)}; let t: *u8 = sys_mmap(28); var k: i64=0; if m==0 {t[0]=48;k=1}; while m>0 {t[k]=48+(m%10); m=m/10; k=k+1}; var i: i64=0; while i<k {bb[i]=t[k-1-i]; i=i+1}; sys_write(1, bb, k); return 0 } 16 17func main() -> i64 { 18 qt_puts("=== PPMI semantic weighting -- kills frequent-term pollution (bits-up, triangulated) ===\n" as *u8) 19 let N: i64 = 4 20 let d0: *u8 = "settlement compensation legal" as *u8 21 let d1: *u8 = "settlement compensation legal" as *u8 22 let d2: *u8 = "claim legal" as *u8 23 let d3: *u8 = "hearing legal" as *u8 24 let ptrs: *i64 = sys_mmap(8 * 8) as *i64; let lens: *i64 = sys_mmap(8 * 8) as *i64 25 ptrs[0]=d0 as i64; ptrs[1]=d1 as i64; ptrs[2]=d2 as i64; ptrs[3]=d3 as i64 26 var i: i64 = 0; while i < N { lens[i] = re_strlen(ptrs[i] as *u8); i = i + 1 } 27 28 let vocab: *i64 = sys_mmap(8 * 8) as *i64 29 vocab[0]="compensation" as *u8 as i64; vocab[1]="claim" as *u8 as i64; vocab[2]="legal" as *u8 as i64; vocab[3]="hearing" as *u8 as i64 30 let V: i64 = 4 31 let qterm: *u8 = "settlement" as *u8 32 33 let pq: *i64 = sys_mmap(8 * 8) as *i64; ps_build_qvec(ptrs, lens, N, qterm, vocab, V, N, pq) 34 let rq: *i64 = sys_mmap(8 * 8) as *i64; ps_build_raw_qvec(ptrs, lens, N, qterm, vocab, V, rq) 35 36 let ppmi_d0: i64 = ps_score(ptrs, lens, 0, pq, vocab, V) 37 let ppmi_d2: i64 = ps_score(ptrs, lens, 2, pq, vocab, V) 38 let ppmi_d3: i64 = ps_score(ptrs, lens, 3, pq, vocab, V) 39 let raw_d2: i64 = ps_score(ptrs, lens, 2, rq, vocab, V) 40 let raw_d3: i64 = ps_score(ptrs, lens, 3, rq, vocab, V) 41 let pbest: i64 = ps_best(ptrs, lens, N, pq, vocab, V) 42 43 qt_puts(" PPMI qvec[compensation,claim,legal,hearing] = [" as *u8); i=0; while i<V { qt_num(pq[i]); if i<V-1 { qt_puts("," as *u8) } i=i+1 } qt_puts("]\n" as *u8) 44 qt_puts(" doc2 'claim legal' : raw=" as *u8); qt_num(raw_d2); qt_puts(" (FALSE positive via frequent 'legal') PPMI=" as *u8); qt_num(ppmi_d2); qt_puts(" (correctly 0)\n" as *u8) 45 qt_puts(" doc3 'hearing legal': raw=" as *u8); qt_num(raw_d3); qt_puts(" (FALSE positive) PPMI=" as *u8); qt_num(ppmi_d3); qt_puts(" (correctly 0)\n" as *u8) 46 qt_puts(" doc0 (compensation) : PPMI=" as *u8); qt_num(ppmi_d0); qt_puts(" best-doc=" as *u8); qt_num(pbest); qt_puts("\n" as *u8) 47 qt_puts("PPMI_COMP " as *u8); qt_num(pq[0]); qt_puts("\n" as *u8) // <- triangulated vs Python ~693147 48 49 let r: *i64 = sys_mmap(8 * 8) as *i64 50 r[0] = 0; if raw_d2 > 0 { if ppmi_d2 == 0 { r[0] = 1 } } // raw false-positive killed by PPMI 51 r[1] = 0; if raw_d3 > 0 { if ppmi_d3 == 0 { r[1] = 1 } } // ...for both frequent-term docs 52 r[2] = 0; if ppmi_d0 > 0 { r[2] = 1 } // the genuine association kept 53 r[3] = 0; if pbest == 0 { r[3] = 1 } // PPMI ranks the real doc top 54 r[4] = 0; if pq[2] == 0 { if pq[0] > 0 { r[4] = 1 } } // frequent 'legal'->0, informative 'compensation'->>0 55 r[5] = 0; if pq[0] >= 690000 { if pq[0] <= 696000 { r[5] = 1 } } // PPMI ~ ln(2) (triangulated) 56 var pass: i64 = 0; i = 0 57 while i < 6 { pass = pass + r[i]; i = i + 1 } 58 qt_puts("----\n passed " as *u8); qt_num(pass); qt_puts("/6\n" as *u8) 59 if pass == 6 { qt_puts(" PPMI PROVEN: frequent-term false positives removed, genuine associations kept, value triangulated -- the sovereign semantic upgrade over raw co-occurrence. SVD reduction = next sub-rung.\n" as *u8); sys_exit(0); return 0 } 60 qt_puts(" FAIL\n" as *u8); sys_exit(1); return 1 61}