`nx_ppmi_lib.nx` source

↩ module page · 220 lines · 8573 B
1// nx_ppmi_lib.nx -- THE canonical PPMI model library (eats debt seq283).
2// The loader + sparse-merge cosine over knowledge/index/semppmi_v1.bin had been hand-rolled
3// twice: pb_* inside nx_recall_dense.nx (the recall lane's reranker) and sj_* inside
4// nx_dr_semjudge.nx (the deep-research judge). Two copies of the same numeric code silently
5// diverging would change similarity scores without any gate noticing -- exactly the drift
6// class D001 was raised for. This is the ONE copy both should import.
7//
8// MIGRATION STATE (honest): nx_dr_semjudge migrates onto this lib now. nx_recall_dense is a
9// LIVE organ owned by the recall lane and is NOT edited here -- it migrates on its owner's
10// next touch (D001 migrate-on-touch), at which point seq283 fully closes.
11//
12// Model format (NXPPMI1): 32B header [magic, nv, nt] then vocab-hash[nv], row-index[nv+1],
13// norm^2[nv], ctx-id[nt], value[nt]. Vocabulary is keyed by db_semhash -- resolving a word
14// with any other hash silently yields -1 and zeroes every similarity.
15// Every func <=6 params (NAS nx_cc >6-arg skew, seq239). No hardware writes (Rule 26).
16//
17// module: nishi-core.index.ppmi_lib
18// depends: nx_qabench_engine.nx (db_semhash/db_bsearch_i64), nx_syscalls.nx
19// genealogy_id: levy_goldberg_2014_ppmi
20import "nx_qabench_engine.nx"
21import "nx_syscalls.nx"
22import "nx_vecmath.nx"
23
24// g-slots: 69 wid-scratch (lazily allocated ONCE -- a per-token sys_mmap in the maxsim hot loop
25// cost ~4M allocations per BEIR rerank run), 70 blob, 71 nv, 72 nt, 73 vh, 74 ridx, 75 nrm2,
26// 76 tctx, 77 tval, 78 loaded. CALLER CONTRACT: g must be a zeroed block of >= 632 bytes
27// (sys_mmap(1024)); a 64-byte g overruns the arena by 560 bytes and crashes when armed.
28func ppl_isqrt(v: i64) -> i64 { return vm_isqrt(v) }
29
30// Load a PPMI model from `path` into g. Returns 1 on success, 0 on any failure (fail-closed).
31// Path is a PARAMETER so callers can A/B different models (e.g. the general QA-prose model vs
32// a domain-augmented one) with identical code -- a hardcoded path made that impossible.
33func ppl_load(g: *i64, path: *u8) -> i64 {
34    g[78] = 0
35    let fd: i64 = sys_openat_rd(path)
36    if fd < 0 { return 0 }
37    let hdrb: *u8 = sys_mmap(4096)
38    var hgot: i64 = 0
39    var hr: i64 = 1
40    while hr > 0 { if hgot >= 32 { hr = 0 } else { hr = sys_read(fd, (hdrb as i64 + hgot) as *u8, 32 - hgot); if hr > 0 { hgot = hgot + hr } } }
41    if hgot < 32 { sys_close(fd); return 0 }
42    let hh: *i64 = (hdrb as i64 + 8) as *i64
43    let hnv: i64 = hh[0]
44    let hnt: i64 = hh[1]
45    let need0: i64 = 32 + (hnv*8) + ((hnv+1)*8) + (hnv*8) + (hnt*8) + (hnt*8)
46    if need0 <= 32 { sys_close(fd); return 0 }
47    if need0 > 536870912 { sys_close(fd); return 0 }
48    let blob: *u8 = sys_mmap(need0 + 4096)
49    var i0: i64 = 0
50    while i0 < 32 { blob[i0] = hdrb[i0]; i0 = i0 + 1 }
51    var total: i64 = 32
52    var r: i64 = 1
53    while r > 0 {
54        let left: i64 = need0 - total
55        if left <= 0 { r = 0 } else {
56            var want: i64 = 262144
57            if want > left { want = left }
58            r = sys_read(fd, (blob as i64 + total) as *u8, want)
59            if r > 0 { total = total + r }
60        }
61    }
62    sys_close(fd)
63    if total < 64 { return 0 }
64    if blob[0] != (78 as u8) { return 0 }
65    if blob[6] != (49 as u8) { return 0 }
66    let hi: *i64 = (blob as i64 + 8) as *i64
67    let nv: i64 = hi[0]
68    let nt: i64 = hi[1]
69    let need: i64 = 32 + (nv*8) + ((nv+1)*8) + (nv*8) + (nt*8) + (nt*8)
70    if total < need { return 0 }
71    g[70] = blob as i64
72    g[71] = nv
73    g[72] = nt
74    var off: i64 = 32
75    g[73] = (blob as i64) + off; off = off + nv*8
76    g[74] = (blob as i64) + off; off = off + (nv+1)*8
77    g[75] = (blob as i64) + off; off = off + nv*8
78    g[76] = (blob as i64) + off; off = off + nt*8
79    g[77] = (blob as i64) + off
80    g[78] = 1
81    return 1
82}
83
84// PPMI cosine between two vocab rows (sorted sparse merge), permille.
85func ppl_cos(g: *i64, a: i64, b: i64) -> i64 {
86    let ridx: *i64 = g[74] as *i64
87    let nrm2: *i64 = g[75] as *i64
88    let tctx: *i64 = g[76] as *i64
89    let tval: *i64 = g[77] as *i64
90    var ia: i64 = ridx[a]
91    var ib: i64 = ridx[b]
92    let ea: i64 = ridx[a+1]
93    let eb: i64 = ridx[b+1]
94    var dot: i64 = 0
95    while ia < ea {
96        if ib >= eb { ia = ea } else {
97            if tctx[ia] == tctx[ib] { dot = dot + tval[ia]*tval[ib]; ia = ia + 1; ib = ib + 1 }
98            else { if tctx[ia] < tctx[ib] { ia = ia + 1 } else { ib = ib + 1 } }
99        }
100    }
101    if dot <= 0 { return 0 }
102    let d1: i64 = ppl_isqrt(nrm2[a])
103    let d2: i64 = ppl_isqrt(nrm2[b])
104    if d1 == 0 { return 0 }
105    if d2 == 0 { return 0 }
106    var cv: i64 = (dot*1000)/(d1*d2)
107    if cv > 1000 { cv = 1000 }
108    return cv
109}
110
111// cosine with identity + out-of-vocabulary guard (id -1 contributes nothing).
112func ppl_dcos(g: *i64, a: i64, b: i64) -> i64 {
113    if a < 0 { return 0 }
114    if b < 0 { return 0 }
115    if a == b { return 1000 }
116    return ppl_cos(g, a, b)
117}
118
119// Resolve buf[start,end) to a PPMI vocab id, or -1 if OOV. Lowercases into scratch first so
120// the hash matches the vocabulary as built (db_semhash is the ONLY correct hash here).
121func ppl_wid_range(g: *i64, buf: *u8, start: i64, end: i64) -> i64 {
122    let n: i64 = end - start
123    if n <= 0 { return 0 - 1 }
124    if n > 64 { return 0 - 1 }
125    if g[69] == 0 { g[69] = sys_mmap(80) as i64 }
126    let t: *u8 = g[69] as *u8
127    var i: i64 = 0
128    while i < n {
129        var c: i64 = buf[start + i] as i64
130        if c >= 65 { if c <= 90 { c = c + 32 } }
131        t[i] = c as u8
132        i = i + 1
133    }
134    return db_bsearch_i64(g[73] as *i64, g[71], db_semhash(t, 0, n))
135}
136
137// Tokenize text into PPMI vocab ids (-1 = OOV). Returns the token count.
138func ppl_tokenize_ids(g: *i64, buf: *u8, len: i64, out: *i64, maxn: i64) -> i64 {
139    var cnt: i64 = 0
140    var i: i64 = 0
141    var start: i64 = 0 - 1
142    while i < len {
143        let c: i64 = buf[i] as i64
144        var alnum: i64 = 0
145        if c >= 48 { if c <= 57 { alnum = 1 } }
146        if c >= 65 { if c <= 90 { alnum = 1 } }
147        if c >= 97 { if c <= 122 { alnum = 1 } }
148        if alnum == 1 {
149            if start < 0 { start = i }
150        } else {
151            if start >= 0 {
152                if cnt < maxn { out[cnt] = ppl_wid_range(g, buf, start, i); cnt = cnt + 1 }
153                start = 0 - 1
154            }
155        }
156        i = i + 1
157    }
158    if start >= 0 { if cnt < maxn { out[cnt] = ppl_wid_range(g, buf, start, len); cnt = cnt + 1 } }
159    return cnt
160}
161
162// Informativeness weight of a vocab row (an IDF proxy read straight off the model, so it needs
163// no document collection): ubiquitous function words occupy MANY contexts, discriminative words
164// few. Weight therefore DECREASES with row length. Out-of-vocabulary ids weigh 0 -- which also
165// removes them from the denominator instead of letting them drag a mean toward zero.
166func ppl_idf_w(g: *i64, a: i64) -> i64 {
167    if a < 0 { return 0 }
168    let ridx: *i64 = g[74] as *i64
169    let n: i64 = ridx[a+1] - ridx[a]
170    if n < 1 { return 0 }
171    return 1000000 / (100 + n)
172}
173
174// IDF-WEIGHTED late-interaction coverage. The flat mean below treats "the" exactly like
175// "personalization", so terms present in every candidate add a constant baseline that
176// COMPRESSES the gap the judge is measured on. This weights each term by informativeness, so
177// discriminative terms drive the score. Permille, same scale as ppl_maxsim.
178func ppl_maxsim_idf(g: *i64, a: *i64, na: i64, b: *i64, nb: i64) -> i64 {
179    if na < 1 { return 0 }
180    var num: i64 = 0
181    var den: i64 = 0
182    var i: i64 = 0
183    while i < na {
184        let w: i64 = ppl_idf_w(g, a[i])
185        if w > 0 {
186            var best: i64 = 0
187            var j: i64 = 0
188            while j < nb {
189                let c: i64 = ppl_dcos(g, a[i], b[j])
190                if c > best { best = c }
191                j = j + 1
192            }
193            num = num + best * w
194            den = den + w
195        }
196        i = i + 1
197    }
198    if den < 1 { return 0 }
199    return num / den
200}
201
202// Late-interaction coverage of `a` by `b`, permille: mean over a-terms of the MAX cosine
203// against any b-term. Exact match scores 1000.
204func ppl_maxsim(g: *i64, a: *i64, na: i64, b: *i64, nb: i64) -> i64 {
205    if na < 1 { return 0 }
206    var total: i64 = 0
207    var i: i64 = 0
208    while i < na {
209        var best: i64 = 0
210        var j: i64 = 0
211        while j < nb {
212            let c: i64 = ppl_dcos(g, a[i], b[j])
213            if c > best { best = c }
214            j = j + 1
215        }
216        total = total + best
217        i = i + 1
218    }
219    return total / na
220}
nx_ppmi_lib.nx source

`nx_ppmi_lib.nx` source
