nx_dr_densejudge.nx source

1// nx_dr_densejudge.nx -- TRAINED DENSE representation for the judge (DR-13). 2// Three measured rejections (§10 vocabulary, §12 term weighting, §14 learned fusion) localised 3// the ceiling to the FEATURES themselves: sparse PPMI co-occurrence carries only ~65% pairwise 4// signal on insight-vs-distractor, so no amount of reweighting or fusing fixes it. The 5// remaining move is a DIFFERENT REPRESENTATION. 6// 7// This uses the ecosystem's TRAINED dense embeddings (knowledge/index/embed_v1.bin, Q10, 8// nv x DIM), produced by nx_embed_train factorising the PPMI matrix (Levy & Goldberg 2014: 9// factorising PPMI == word2vec/SGNS). On the known-hard pair the trained dense vectors ordered 10// far more confidently than raw counts (607 vs 559, where sparse counts gave 86 vs 25), which 11// is exactly the kind of representational change the diagnosis calls for. 12// 13// Vocabulary: embed_v1.bin rows are indexed by the SAME ids as semppmi_v1.bin (the trainer read 14// that vocab), so word resolution still goes through nx_ppmi_lib -- BOTH models are loaded. 15// on-disk: 24B header [magic "NXEMB1", i64 nv, i64 DIM] then nv*DIM Q10 i64 (block-padded tail 16// is ignored -- we read exactly nv*DIM). 17// Every func <=6 params (NAS nx_cc >6-arg skew, seq239). No hardware writes (Rule 26). 18// 19// module: nishi-core.research.dr_densejudge 20// depends: nx_ppmi_lib.nx 21// genealogy_id: levy_goldberg_2014_sgns + colbert_late_interaction 22import "nx_ppmi_lib.nx" 23const K_MAGIC_536870912: i64 = 536870912 24const K_MAGIC_4096: i64 = 4096 25const K_MAGIC_262144: i64 = 262144 26 27// embed slots (disjoint from the PPMI slots 70..78): 80 base, 81 nv, 82 dim, 83 loaded 28func dj_load_embed(g: *i64, path: *u8) -> i64 { 29 g[83] = 0 30 let fd: i64 = sys_openat_rd(path) 31 if fd < 0 { return 0 } 32 let hb: *u8 = sys_mmap(64) 33 var got: i64 = 0 34 var r: i64 = 1 35 while r > 0 { if got >= 24 { r = 0 } else { r = sys_read(fd, (hb as i64 + got) as *u8, 24 - got); if r > 0 { got = got + r } } } 36 if got < 24 { sys_close(fd); return 0 } 37 if hb[0] != (78 as u8) { sys_close(fd); return 0 } 38 if hb[5] != (49 as u8) { sys_close(fd); return 0 } 39 let hi: *i64 = (hb as i64 + 8) as *i64 40 let nv: i64 = hi[0] 41 let dim: i64 = hi[1] 42 if nv < 1 { sys_close(fd); return 0 } 43 if dim < 1 { sys_close(fd); return 0 } 44 let need: i64 = nv * dim * 8 45 if need > K_MAGIC_536870912 { sys_close(fd); return 0 } 46 let blob: *u8 = sys_mmap(need + K_MAGIC_4096) 47 var total: i64 = 0 48 r = 1 49 while r > 0 { 50 let left: i64 = need - total 51 if left <= 0 { r = 0 } else { 52 var want: i64 = K_MAGIC_262144 53 if want > left { want = left } 54 r = sys_read(fd, (blob as i64 + total) as *u8, want) 55 if r > 0 { total = total + r } 56 } 57 } 58 sys_close(fd) 59 if total < need { return 0 } 60 g[80] = blob as i64 61 g[81] = nv 62 g[82] = dim 63 g[83] = 1 64 return 1 65} 66 67// dense cosine between two vocab rows, permille. Negative similarity is clamped to 0 (for 68// coverage purposes "anti-related" and "unrelated" are both simply not evidence). 69func dj_cos(g: *i64, a: i64, b: i64) -> i64 { 70 if a < 0 { return 0 } 71 if b < 0 { return 0 } 72 if g[83] != 1 { return 0 } 73 let nv: i64 = g[81] 74 if a >= nv { return 0 } 75 if b >= nv { return 0 } 76 if a == b { return 1000 } 77 let dim: i64 = g[82] 78 let e: *i64 = g[80] as *i64 79 let ba: i64 = a * dim 80 let bb: i64 = b * dim 81 var dot: i64 = 0 82 var na: i64 = 0 83 var nb: i64 = 0 84 var d: i64 = 0 85 while d < dim { 86 let x: i64 = e[ba + d] 87 let y: i64 = e[bb + d] 88 dot = dot + x * y 89 na = na + x * x 90 nb = nb + y * y 91 d = d + 1 92 } 93 if dot <= 0 { return 0 } 94 let la: i64 = ppl_isqrt(na) 95 let lb: i64 = ppl_isqrt(nb) 96 if la == 0 { return 0 } 97 if lb == 0 { return 0 } 98 var cv: i64 = (dot * 1000) / (la * lb) 99 if cv > 1000 { cv = 1000 } 100 return cv 101} 102 103// Build the mean vector over the WHOLE vocabulary into slot 84. Trained embeddings acquire a 104// dominant common direction that inflates every cosine (the recorded symptom on embed_v1: 105// "cosines compressed by a common component"). Removing it is the standard fix -- Mu & 106// Viswanath 2018, "All-but-the-Top". Measured, not assumed: dj_cos_centered is compared against 107// dj_cos on the same benchmark. 108func dj_build_mean(g: *i64) -> i64 { 109 if g[83] != 1 { return 0 } 110 let nv: i64 = g[81] 111 let dim: i64 = g[82] 112 let e: *i64 = g[80] as *i64 113 let m: *i64 = sys_mmap(dim * 8) as *i64 114 var d: i64 = 0 115 while d < dim { m[d] = 0; d = d + 1 } 116 var i: i64 = 0 117 while i < nv { 118 let b: i64 = i * dim 119 d = 0 120 while d < dim { m[d] = m[d] + e[b + d]; d = d + 1 } 121 i = i + 1 122 } 123 d = 0 124 while d < dim { m[d] = m[d] / nv; d = d + 1 } 125 g[84] = m as i64 126 return 1 127} 128 129// dense cosine with the common component removed (requires dj_build_mean first) 130func dj_cos_centered(g: *i64, a: i64, b: i64) -> i64 { 131 if a < 0 { return 0 } 132 if b < 0 { return 0 } 133 if g[83] != 1 { return 0 } 134 if a == b { return 1000 } 135 let nv: i64 = g[81] 136 if a >= nv { return 0 } 137 if b >= nv { return 0 } 138 let dim: i64 = g[82] 139 let e: *i64 = g[80] as *i64 140 let m: *i64 = g[84] as *i64 141 let ba: i64 = a * dim 142 let bb: i64 = b * dim 143 var dot: i64 = 0 144 var na: i64 = 0 145 var nb: i64 = 0 146 var d: i64 = 0 147 while d < dim { 148 let x: i64 = e[ba + d] - m[d] 149 let y: i64 = e[bb + d] - m[d] 150 dot = dot + x * y 151 na = na + x * x 152 nb = nb + y * y 153 d = d + 1 154 } 155 if dot <= 0 { return 0 } 156 let la: i64 = ppl_isqrt(na) 157 let lb: i64 = ppl_isqrt(nb) 158 if la == 0 { return 0 } 159 if lb == 0 { return 0 } 160 var cv: i64 = (dot * 1000) / (la * lb) 161 if cv > 1000 { cv = 1000 } 162 return cv 163} 164 165// late-interaction coverage under the COMMON-COMPONENT-REMOVED dense representation 166func dj_maxsim_centered(g: *i64, a: *i64, na: i64, b: *i64, nb: i64) -> i64 { 167 if na < 1 { return 0 } 168 var total: i64 = 0 169 var i: i64 = 0 170 while i < na { 171 var best: i64 = 0 172 var j: i64 = 0 173 while j < nb { 174 let c: i64 = dj_cos_centered(g, a[i], b[j]) 175 if c > best { best = c } 176 j = j + 1 177 } 178 total = total + best 179 i = i + 1 180 } 181 return total / na 182} 183 184// Late-interaction coverage of `a` by `b` under the DENSE representation, permille. 185func dj_maxsim(g: *i64, a: *i64, na: i64, b: *i64, nb: i64) -> i64 { 186 if na < 1 { return 0 } 187 var total: i64 = 0 188 var i: i64 = 0 189 while i < na { 190 var best: i64 = 0 191 var j: i64 = 0 192 while j < nb { 193 let c: i64 = dj_cos(g, a[i], b[j]) 194 if c > best { best = c } 195 j = j + 1 196 } 197 total = total + best 198 i = i + 1 199 } 200 return total / na 201}