nx_ppmi_lib.nx source

1// nx_ppmi_lib.nx -- THE canonical PPMI model library (eats debt seq283). 2// The loader + sparse-merge cosine over knowledge/index/semppmi_v1.bin had been hand-rolled 3// twice: pb_* inside nx_recall_dense.nx (the recall lane's reranker) and sj_* inside 4// nx_dr_semjudge.nx (the deep-research judge). Two copies of the same numeric code silently 5// diverging would change similarity scores without any gate noticing -- exactly the drift 6// class D001 was raised for. This is the ONE copy both should import. 7// 8// MIGRATION STATE (honest): nx_dr_semjudge migrates onto this lib now. nx_recall_dense is a 9// LIVE organ owned by the recall lane and is NOT edited here -- it migrates on its owner's 10// next touch (D001 migrate-on-touch), at which point seq283 fully closes. 11// 12// Model format (NXPPMI1): 32B header [magic, nv, nt] then vocab-hash[nv], row-index[nv+1], 13// norm^2[nv], ctx-id[nt], value[nt]. Vocabulary is keyed by db_semhash -- resolving a word 14// with any other hash silently yields -1 and zeroes every similarity. 15// Every func <=6 params (NAS nx_cc >6-arg skew, seq239). No hardware writes (Rule 26). 16// 17// module: nishi-core.index.ppmi_lib 18// depends: nx_qabench_engine.nx (db_semhash/db_bsearch_i64), nx_syscalls.nx 19// genealogy_id: levy_goldberg_2014_ppmi 20import "nx_qabench_engine.nx" 21import "nx_syscalls.nx" 22 23// g-slots: 70 blob, 71 nv, 72 nt, 73 vh, 74 ridx, 75 nrm2, 76 tctx, 77 tval, 78 loaded 24func ppl_isqrt(v: i64) -> i64 { 25 if v <= 0 { return 0 } 26 var x: i64 = v 27 var y: i64 = (x + 1) / 2 28 while y < x { x = y; let q: i64 = v / x; y = (x + q) / 2 } 29 return x 30} 31 32// Load a PPMI model from `path` into g. Returns 1 on success, 0 on any failure (fail-closed). 33// Path is a PARAMETER so callers can A/B different models (e.g. the general QA-prose model vs 34// a domain-augmented one) with identical code -- a hardcoded path made that impossible. 35func ppl_load(g: *i64, path: *u8) -> i64 { 36 g[78] = 0 37 let fd: i64 = sys_openat_rd(path) 38 if fd < 0 { return 0 } 39 let hdrb: *u8 = sys_mmap(4096) 40 var hgot: i64 = 0 41 var hr: i64 = 1 42 while hr > 0 { if hgot >= 32 { hr = 0 } else { hr = sys_read(fd, (hdrb as i64 + hgot) as *u8, 32 - hgot); if hr > 0 { hgot = hgot + hr } } } 43 if hgot < 32 { sys_close(fd); return 0 } 44 let hh: *i64 = (hdrb as i64 + 8) as *i64 45 let hnv: i64 = hh[0] 46 let hnt: i64 = hh[1] 47 let need0: i64 = 32 + (hnv*8) + ((hnv+1)*8) + (hnv*8) + (hnt*8) + (hnt*8) 48 if need0 <= 32 { sys_close(fd); return 0 } 49 if need0 > 536870912 { sys_close(fd); return 0 } 50 let blob: *u8 = sys_mmap(need0 + 4096) 51 var i0: i64 = 0 52 while i0 < 32 { blob[i0] = hdrb[i0]; i0 = i0 + 1 } 53 var total: i64 = 32 54 var r: i64 = 1 55 while r > 0 { 56 let left: i64 = need0 - total 57 if left <= 0 { r = 0 } else { 58 var want: i64 = 262144 59 if want > left { want = left } 60 r = sys_read(fd, (blob as i64 + total) as *u8, want) 61 if r > 0 { total = total + r } 62 } 63 } 64 sys_close(fd) 65 if total < 64 { return 0 } 66 if blob[0] != (78 as u8) { return 0 } 67 if blob[6] != (49 as u8) { return 0 } 68 let hi: *i64 = (blob as i64 + 8) as *i64 69 let nv: i64 = hi[0] 70 let nt: i64 = hi[1] 71 let need: i64 = 32 + (nv*8) + ((nv+1)*8) + (nv*8) + (nt*8) + (nt*8) 72 if total < need { return 0 } 73 g[70] = blob as i64 74 g[71] = nv 75 g[72] = nt 76 var off: i64 = 32 77 g[73] = (blob as i64) + off; off = off + nv*8 78 g[74] = (blob as i64) + off; off = off + (nv+1)*8 79 g[75] = (blob as i64) + off; off = off + nv*8 80 g[76] = (blob as i64) + off; off = off + nt*8 81 g[77] = (blob as i64) + off 82 g[78] = 1 83 return 1 84} 85 86// PPMI cosine between two vocab rows (sorted sparse merge), permille. 87func ppl_cos(g: *i64, a: i64, b: i64) -> i64 { 88 let ridx: *i64 = g[74] as *i64 89 let nrm2: *i64 = g[75] as *i64 90 let tctx: *i64 = g[76] as *i64 91 let tval: *i64 = g[77] as *i64 92 var ia: i64 = ridx[a] 93 var ib: i64 = ridx[b] 94 let ea: i64 = ridx[a+1] 95 let eb: i64 = ridx[b+1] 96 var dot: i64 = 0 97 while ia < ea { 98 if ib >= eb { ia = ea } else { 99 if tctx[ia] == tctx[ib] { dot = dot + tval[ia]*tval[ib]; ia = ia + 1; ib = ib + 1 } 100 else { if tctx[ia] < tctx[ib] { ia = ia + 1 } else { ib = ib + 1 } } 101 } 102 } 103 if dot <= 0 { return 0 } 104 let d1: i64 = ppl_isqrt(nrm2[a]) 105 let d2: i64 = ppl_isqrt(nrm2[b]) 106 if d1 == 0 { return 0 } 107 if d2 == 0 { return 0 } 108 var cv: i64 = (dot*1000)/(d1*d2) 109 if cv > 1000 { cv = 1000 } 110 return cv 111} 112 113// cosine with identity + out-of-vocabulary guard (id -1 contributes nothing). 114func ppl_dcos(g: *i64, a: i64, b: i64) -> i64 { 115 if a < 0 { return 0 } 116 if b < 0 { return 0 } 117 if a == b { return 1000 } 118 return ppl_cos(g, a, b) 119} 120 121// Resolve buf[start,end) to a PPMI vocab id, or -1 if OOV. Lowercases into scratch first so 122// the hash matches the vocabulary as built (db_semhash is the ONLY correct hash here). 123func ppl_wid_range(g: *i64, buf: *u8, start: i64, end: i64) -> i64 { 124 let n: i64 = end - start 125 if n <= 0 { return 0 - 1 } 126 if n > 64 { return 0 - 1 } 127 let t: *u8 = sys_mmap(80) 128 var i: i64 = 0 129 while i < n { 130 var c: i64 = buf[start + i] as i64 131 if c >= 65 { if c <= 90 { c = c + 32 } } 132 t[i] = c as u8 133 i = i + 1 134 } 135 return db_bsearch_i64(g[73] as *i64, g[71], db_semhash(t, 0, n)) 136} 137 138// Tokenize text into PPMI vocab ids (-1 = OOV). Returns the token count. 139func ppl_tokenize_ids(g: *i64, buf: *u8, len: i64, out: *i64, maxn: i64) -> i64 { 140 var cnt: i64 = 0 141 var i: i64 = 0 142 var start: i64 = 0 - 1 143 while i < len { 144 let c: i64 = buf[i] as i64 145 var alnum: i64 = 0 146 if c >= 48 { if c <= 57 { alnum = 1 } } 147 if c >= 65 { if c <= 90 { alnum = 1 } } 148 if c >= 97 { if c <= 122 { alnum = 1 } } 149 if alnum == 1 { 150 if start < 0 { start = i } 151 } else { 152 if start >= 0 { 153 if cnt < maxn { out[cnt] = ppl_wid_range(g, buf, start, i); cnt = cnt + 1 } 154 start = 0 - 1 155 } 156 } 157 i = i + 1 158 } 159 if start >= 0 { if cnt < maxn { out[cnt] = ppl_wid_range(g, buf, start, len); cnt = cnt + 1 } } 160 return cnt 161} 162 163// Informativeness weight of a vocab row (an IDF proxy read straight off the model, so it needs 164// no document collection): ubiquitous function words occupy MANY contexts, discriminative words 165// few. Weight therefore DECREASES with row length. Out-of-vocabulary ids weigh 0 -- which also 166// removes them from the denominator instead of letting them drag a mean toward zero. 167func ppl_idf_w(g: *i64, a: i64) -> i64 { 168 if a < 0 { return 0 } 169 let ridx: *i64 = g[74] as *i64 170 let n: i64 = ridx[a+1] - ridx[a] 171 if n < 1 { return 0 } 172 return 1000000 / (100 + n) 173} 174 175// IDF-WEIGHTED late-interaction coverage. The flat mean below treats "the" exactly like 176// "personalization", so terms present in every candidate add a constant baseline that 177// COMPRESSES the gap the judge is measured on. This weights each term by informativeness, so 178// discriminative terms drive the score. Permille, same scale as ppl_maxsim. 179func ppl_maxsim_idf(g: *i64, a: *i64, na: i64, b: *i64, nb: i64) -> i64 { 180 if na < 1 { return 0 } 181 var num: i64 = 0 182 var den: i64 = 0 183 var i: i64 = 0 184 while i < na { 185 let w: i64 = ppl_idf_w(g, a[i]) 186 if w > 0 { 187 var best: i64 = 0 188 var j: i64 = 0 189 while j < nb { 190 let c: i64 = ppl_dcos(g, a[i], b[j]) 191 if c > best { best = c } 192 j = j + 1 193 } 194 num = num + best * w 195 den = den + w 196 } 197 i = i + 1 198 } 199 if den < 1 { return 0 } 200 return num / den 201} 202 203// Late-interaction coverage of `a` by `b`, permille: mean over a-terms of the MAX cosine 204// against any b-term. Exact match scores 1000. 205func ppl_maxsim(g: *i64, a: *i64, na: i64, b: *i64, nb: i64) -> i64 { 206 if na < 1 { return 0 } 207 var total: i64 = 0 208 var i: i64 = 0 209 while i < na { 210 var best: i64 = 0 211 var j: i64 = 0 212 while j < nb { 213 let c: i64 = ppl_dcos(g, a[i], b[j]) 214 if c > best { best = c } 215 j = j + 1 216 } 217 total = total + best 218 i = i + 1 219 } 220 return total / na 221}