`nx_ppmi_svd.nx` source

1// nx_ppmi_svd.nx -- the sovereign semantic upgrade (Researcher-specced, built bits-up): PPMI weighting
2// over the team's co-occurrence counts. Raw co-occurrence over-credits FREQUENT terms (a term that is
3// in every doc co-occurs with everything, so it inflates unrelated docs). Pointwise Mutual Information
4// fixes this: PMI(a,b) = log( P(a,b) / (P(a)P(b)) ) = log( cooc(a,b)*D / (df(a)*df(b)) ); PPMI keeps the
5// POSITIVE part, so a term that co-occurs only AT CHANCE scores 0 and stops polluting retrieval. This is
6// the count-based dense-embedding rung (Levy & Goldberg 2014: PPMI is competitive with word2vec; the
7// further SVD dimensionality reduction is the next sub-rung, flagged not faked). Integer-only: PMI uses
8// the fixed-point ln from nx_bm25. license_tier: ORIGINAL
9
10import "nx_research_extract.nx"   // re_has
11import "nx_bm25.nx"               // bm_ln_micro (fixed-point ln)
12import "nx_syscalls.nx"
13
14func ps_doc(ptrs: *i64, k: i64) -> *u8 { return ptrs[k] as *u8 }
15
16// document frequency / co-occurrence over the corpus.
17func ps_df(ptrs: *i64, lens: *i64, N: i64, term: *u8) -> i64 {
18    var c: i64 = 0; var i: i64 = 0
19    while i < N { if re_has(ps_doc(ptrs, i), lens[i], term) == 1 { c = c + 1 } i = i + 1 }
20    return c
21}
22func ps_cooc(ptrs: *i64, lens: *i64, N: i64, a: *u8, b: *u8) -> i64 {
23    var c: i64 = 0; var i: i64 = 0
24    while i < N { if re_has(ps_doc(ptrs, i), lens[i], a) == 1 { if re_has(ps_doc(ptrs, i), lens[i], b) == 1 { c = c + 1 } } i = i + 1 }
25    return c
26}
27
28// PPMI in micro-units: max(0, ln( cooc*D / (df_a*df_b) )). 0 if they never co-occur OR only at chance.
29func ps_ppmi_micro(cooc: i64, df_a: i64, df_b: i64, D: i64) -> i64 {
30    if cooc <= 0 { return 0 }
31    if df_a <= 0 { return 0 }
32    if df_b <= 0 { return 0 }
33    let pmi: i64 = bm_ln_micro(cooc * D, df_a * df_b)
34    if pmi < 0 { return 0 }
35    return pmi
36}
37
38// the query's PPMI vector over the vocabulary.
39func ps_build_qvec(ptrs: *i64, lens: *i64, N: i64, qterm: *u8, vocab: *i64, V: i64, D: i64, qvec: *i64) -> i64 {
40    let dfq: i64 = ps_df(ptrs, lens, N, qterm)
41    var t: i64 = 0
42    while t < V {
43        let term: *u8 = vocab[t] as *u8
44        let cooc: i64 = ps_cooc(ptrs, lens, N, qterm, term)
45        let dft: i64 = ps_df(ptrs, lens, N, term)
46        qvec[t] = ps_ppmi_micro(cooc, dfq, dft, D)
47        t = t + 1
48    }
49    return 0
50}
51
52// PPMI-weighted semantic score: sum of qvec[t] for vocab terms present in the doc.
53func ps_score(ptrs: *i64, lens: *i64, k: i64, qvec: *i64, vocab: *i64, V: i64) -> i64 {
54    var s: i64 = 0; var t: i64 = 0
55    while t < V { if re_has(ps_doc(ptrs, k), lens[k], vocab[t] as *u8) == 1 { s = s + qvec[t] } t = t + 1 }
56    return s
57}
58
59func ps_best(ptrs: *i64, lens: *i64, N: i64, qvec: *i64, vocab: *i64, V: i64) -> i64 {
60    var best: i64 = 0 - 1; var bestscore: i64 = 0; var i: i64 = 0
61    while i < N { let sc: i64 = ps_score(ptrs, lens, i, qvec, vocab, V); if sc > bestscore { bestscore = sc; best = i } i = i + 1 }
62    return best
63}
64
65// the RAW co-occurrence vector + score (no PPMI) -- kept to PROVE PPMI removes frequent-term pollution.
66func ps_build_raw_qvec(ptrs: *i64, lens: *i64, N: i64, qterm: *u8, vocab: *i64, V: i64, qvec: *i64) -> i64 {
67    var t: i64 = 0
68    while t < V { qvec[t] = ps_cooc(ptrs, lens, N, qterm, vocab[t] as *u8); t = t + 1 }
69    return 0
70}
nx_ppmi_svd.nx source

`nx_ppmi_svd.nx` source
