code wiki / _hdl_build / nx_dr_ppmi_build.nx
nx_dr_ppmi_build.nx source
↩ module page · 395 lines · 17740 B
1// nx_semppmi_build.nx -- EXPLICIT-VOCAB SPARSE PPMI embeddings = the CANONICAL count-based method at full
2// strength (Levy&Goldberg: full sparse PPMI rows rival trained embeddings on word similarity). This is the
3// last count-method rung before "trained-only" is airtight: v1 (nx_semcorpus_build) failed its probes with
4// HASHED rows (collisions) + a 256-dim PROJECTION (lossy) -- this build removes BOTH: every word gets its own
5// row; contexts are the full vocabulary; cells are PPMI (positive pointwise mutual information, log2 x16
6// fixed-point); rows stay SPARSE (sorted triples). Corpus = same banked benchmark prose (no gold leakage:
7// only JSON strings >= 48 chars).
8// PIPELINE (one corpus walk): tokens -> 61-bit djb2x hash stream (+ -1 markers at string ends) ->
9// vocab = sort(copy)+unique (explicit; 61-bit space => collisions ~0 at 60k words) ->
10// window +-4 pair emission as packed keys (aid<<20|bid, both directions) -> sort -> RLE -> triples(cnt>=2) ->
11// marginals -> PPMI16 -> row index + norms -> PROBES -> persist knowledge/index/semppmi_dr_v1.bin.
12// PROBES decide (same discipline as v1): if won~defeated fails to order again, count methods are DEAD at
13// canonical strength and the R1 trained thread is the proven only door. TEETH structural + neg-controls.
14// expect_exit: 0 license_tier: ORIGINAL
15import "nx_qabench_engine.nx"
16const SP_MAGIC_262144: i64 = 262144
17const SP_MAGIC_16777216: i64 = 16777216
18const SP_MAGIC_250000: i64 = 250000
19const SP_MAGIC_4000: i64 = 4000
20const SP_MAGIC_32768: i64 = 32768
21const SP_MAGIC_5000000: i64 = 5000000
22const SP_MAGIC_30000000: i64 = 30000000
23const SP_MAGIC_1048576: i64 = 1048576
24const SP_MAGIC_8000000: i64 = 8000000
25const SP_MAGIC_4096: i64 = 4096
26const SP_MAGIC_3000: i64 = 3000
27const SP_MAGIC_10000: i64 = 10000
28const SP_MAGIC_100000: i64 = 100000
29const SP_MAGIC_50000: i64 = 50000
30
31const SP_WIN: i64 = 4
32const SP_MINSTR: i64 = 48
33const SP_MAXV: i64 = 262144
34const SP_MINCNT: i64 = 2
35
36// 61-bit string hash: SHARED single-source db_semhash from the engine (nx_qabench resolves with the same fn)
37func sp_hash(buf: *u8, off: i64, len: i64) -> i64 { return db_semhash(buf, off, len) }
38
39// write n bytes to fd; 0 ok, 1 err
40func sp_wblock(fd: i64, buf: *u8, n: i64) -> i64 {
41 var off: i64 = 0
42 while off < n {
43 var chunk: i64 = SP_MAGIC_262144
44 if n - off < chunk { chunk = n - off }
45 let ww: i64 = sys_write(fd, (buf as i64 + off) as *u8, chunk)
46 if ww <= 0 { return 1 }
47 off = off + ww
48 }
49 return 0
50}
51
52func sp_isqrt(v: i64) -> i64 {
53 if v <= 0 { return 0 }
54 var x: i64 = v
55 var y: i64 = (x + 1) / 2
56 while y < x { x = y; let q: i64 = v / x; y = (x + q) / 2 }
57 return x
58}
59
60// log2 in 1/16ths of the ratio q (q >= 1); bitlen-based + 4-bit linear fraction
61func sp_ilog2x16(q: i64) -> i64 {
62 if q <= 1 { return 0 }
63 var bl: i64 = 0
64 var t: i64 = q
65 while t > 1 { t = t / 2; bl = bl + 1 }
66 var frac: i64 = 0
67 if bl >= 4 { frac = (q / (1 << (bl-4))) - 16 }
68 if bl < 4 { frac = (q << (4-bl)) - 16 }
69 if frac < 0 { frac = 0 }
70 if frac > 15 { frac = 15 }
71 return bl*16 + frac
72}
73
74// in-place quicksort of a[lo..hi]
75func sp_qsort(a: *i64, lo: i64, hi: i64) -> i64 {
76 if lo >= hi { return 0 }
77 let p: i64 = a[(lo+hi)/2]
78 var i: i64 = lo
79 var j: i64 = hi
80 while i <= j {
81 while a[i] < p { i = i + 1 }
82 while a[j] > p { j = j - 1 }
83 if i <= j {
84 let tmp: i64 = a[i]
85 a[i] = a[j]
86 a[j] = tmp
87 i = i + 1
88 j = j - 1
89 }
90 }
91 sp_qsort(a, lo, j)
92 sp_qsort(a, i, hi)
93 return 0
94}
95
96// binary search: SHARED db_bsearch_i64 from the engine
97func sp_bsearch(a: *i64, n: i64, v: i64) -> i64 { return db_bsearch_i64(a, n, v) }
98
99// walk one raw file's long JSON strings -> append token hashes to stream (marker -1 between strings).
100// c[0]=strings c[1]=tokens. stream cap slots in c[2] (bump in c[3]).
101func sp_walk(g: *i64, path: *u8, stream: *i64, c: *i64) -> i64 {
102 let total: i64 = db_read_raw(g, path, SP_MAGIC_16777216)
103 if total <= 0 { db_w(" [absent] "); db_w(path); db_w("\n" as *u8); return 0 }
104 if total == 0-2 { db_w(" [too big] "); db_w(path); db_w("\n" as *u8); return 0 }
105 g[2] = db_body_start(g, total)
106 g[1] = total - g[2]
107 let scr: *u8 = g[5] as *u8
108 let nrm: *u8 = g[6] as *u8
109 let toff: *i64 = g[7] as *i64
110 let tlen: *i64 = g[8] as *i64
111 g[3] = 0
112 while g[3] < g[1] {
113 let ch: i64 = db_b(g, g[3])
114 if ch == 34 {
115 let sl: i64 = db_dec_str(g, scr, SP_MAGIC_250000)
116 if sl >= SP_MINSTR {
117 c[0] = c[0] + 1
118 let nl: i64 = qs_norm(scr, nrm)
119 let nt: i64 = qs_tok(nrm, nl, toff, tlen, SP_MAGIC_4000)
120 var j: i64 = 0
121 while j < nt {
122 if tlen[j] >= 2 { if db_is_stop(nrm, toff[j], tlen[j]) == 0 {
123 if c[3] < c[2] {
124 let hv: i64 = sp_hash(nrm, toff[j], tlen[j])
125 stream[c[3]] = hv
126 c[3] = c[3] + 1
127 c[1] = c[1] + 1
128 }
129 } }
130 j = j + 1
131 }
132 if c[3] < c[2] { stream[c[3]] = 0-1; c[3] = c[3] + 1 }
133 }
134 } else { g[3] = g[3] + 1 }
135 }
136 db_w(" walked "); db_w(path); db_w(" strings="); db_n(c[0]); db_w(" tokens="); db_n(c[1]); db_w("\n" as *u8)
137 return 1
138}
139
140// cosine between sparse PPMI rows a and b: rows = triples[ridx[a]..ridx[a+1]) sorted by ctx id.
141// trip layout: tctx[i], tval[i]. norms nrm2[] precomputed (sum of val^2).
142func sp_cos(tctx: *i64, tval: *i64, ridx: *i64, nrm2: *i64, a: i64, b: i64) -> i64 {
143 var ia: i64 = ridx[a]
144 var ib: i64 = ridx[b]
145 let ea: i64 = ridx[a+1]
146 let eb: i64 = ridx[b+1]
147 var dot: i64 = 0
148 while ia < ea {
149 if ib >= eb { ia = ea } else {
150 if tctx[ia] == tctx[ib] { dot = dot + tval[ia]*tval[ib]; ia = ia + 1; ib = ib + 1 }
151 else { if tctx[ia] < tctx[ib] { ia = ia + 1 } else { ib = ib + 1 } }
152 }
153 }
154 if dot <= 0 { return 0 }
155 let d1: i64 = sp_isqrt(nrm2[a])
156 let d2: i64 = sp_isqrt(nrm2[b])
157 if d1 == 0 { return 0 }
158 if d2 == 0 { return 0 }
159 var cv: i64 = (dot*1000)/(d1*d2)
160 if cv > 1000 { cv = 1000 }
161 return cv
162}
163
164// probe helper: word -> vocab id (via sorted hash list)
165func sp_wid(vh: *i64, nv: i64, w: *u8) -> i64 {
166 var n: i64 = 0
167 while w[n] != (0 as u8) { n = n + 1 }
168 let hv: i64 = sp_hash(w, 0, n)
169 return sp_bsearch(vh, nv, hv)
170}
171
172func main() -> i64 {
173 db_w("=== nx_semppmi_build -- EXPLICIT-vocab sparse PPMI (canonical count method, the last pre-trained rung) ===\n" as *u8)
174 let g: *i64 = sys_mmap(256) as *i64
175 let m0: *u8 = sys_mmap(SP_MAGIC_16777216); g[0] = m0 as i64
176 let m5: *u8 = sys_mmap(SP_MAGIC_262144); g[5] = m5 as i64
177 let m6: *u8 = sys_mmap(SP_MAGIC_262144); g[6] = m6 as i64
178 let m7: *u8 = sys_mmap(SP_MAGIC_32768); g[7] = m7 as i64
179 let m8: *u8 = sys_mmap(SP_MAGIC_32768); g[8] = m8 as i64
180
181 let SCAP: i64 = SP_MAGIC_5000000
182 let stream: *i64 = sys_mmap(SCAP*8) as *i64
183 let c: *i64 = sys_mmap(64) as *i64
184 c[0]=0; c[1]=0; c[2]=SCAP; c[3]=0
185
186 sp_walk(g, "knowledge/fetched/drb_corpus.raw" as *u8, stream, c)
187&
188 sp_walk(g, "knowledge/fetched/qab_hotpot_big.raw" as *u8, stream, c)
189 sp_walk(g, "knowledge/fetched/qab_musique_big.raw" as *u8, stream, c)
190 // corpus-growth chunks (DISJOINT from the eval rows above; absent files skip cleanly)
191 sp_walk(g, "knowledge/fetched/qab_squad_c0.raw" as *u8, stream, c)
192 sp_walk(g, "knowledge/fetched/qab_squad_c1.raw" as *u8, stream, c)
193 sp_walk(g, "knowledge/fetched/qab_squad_c2.raw" as *u8, stream, c)
194 sp_walk(g, "knowledge/fetched/qab_hotpot_c0.raw" as *u8, stream, c)
195 sp_walk(g, "knowledge/fetched/qab_hotpot_c1.raw" as *u8, stream, c)
196 sp_walk(g, "knowledge/fetched/qab_hotpot_c2.raw" as *u8, stream, c)
197 sp_walk(g, "knowledge/fetched/qab_musique_c0.raw" as *u8, stream, c)
198 sp_walk(g, "knowledge/fetched/qab_musique_c1.raw" as *u8, stream, c)
199 let slen: i64 = c[3]
200
201 // ---- vocab: sort a copy of the hash stream, unique (markers filtered) ----
202 let vs: *i64 = sys_mmap(SCAP*8) as *i64
203 var vn: i64 = 0
204 var i: i64 = 0
205 while i < slen { let sv: i64 = stream[i]; if sv >= 0 { vs[vn] = sv; vn = vn + 1 } i = i + 1 }
206 sp_qsort(vs, 0, vn-1)
207 let vh: *i64 = sys_mmap(SP_MAXV*8) as *i64
208 var nv: i64 = 0
209 i = 0
210 while i < vn {
211 var take: i64 = 1
212 if i > 0 { if vs[i] == vs[i-1] { take = 0 } }
213 if take == 1 { if nv < SP_MAXV { vh[nv] = vs[i]; nv = nv + 1 } }
214 i = i + 1
215 }
216 db_w("vocab (explicit, unique) = "); db_n(nv); db_w(" words from "); db_n(vn); db_w(" events\n" as *u8)
217
218 // ---- pair emission: packed key (aid<<20)|bid for window pairs, both directions ----
219 let PCAP: i64 = SP_MAGIC_30000000
220 let pairs: *i64 = sys_mmap(PCAP*8) as *i64
221 var np: i64 = 0
222 i = 0
223 while i < slen {
224 if stream[i] >= 0 {
225 let av: i64 = sp_bsearch(vh, nv, stream[i])
226 if av >= 0 {
227 var b: i64 = i + 1
228 var bend: i64 = i + SP_WIN
229 if bend > slen-1 { bend = slen-1 }
230 var stop: i64 = 0
231 while b <= bend {
232 if stop == 0 {
233 if stream[b] < 0 { stop = 1 } else {
234 let bv: i64 = sp_bsearch(vh, nv, stream[b])
235 if bv >= 0 { if np < PCAP-2 {
236 pairs[np] = av*SP_MAGIC_1048576 + bv
237 pairs[np+1] = bv*SP_MAGIC_1048576 + av
238 np = np + 2
239 } }
240 }
241 }
242 b = b + 1
243 }
244 }
245 }
246 i = i + 1
247 }
248 db_w("window pair-events = "); db_n(np); db_w("\n" as *u8)
249 sp_qsort(pairs, 0, np-1)
250
251 // ---- RLE -> triples (cnt >= SP_MINCNT), marginals ----
252 let TCAP: i64 = SP_MAGIC_8000000
253 let taid: *i64 = sys_mmap(TCAP*8) as *i64
254 let tctx: *i64 = sys_mmap(TCAP*8) as *i64
255 let tcnt: *i64 = sys_mmap(TCAP*8) as *i64
256 let marg: *i64 = sys_mmap(SP_MAXV*8) as *i64
257 var nt3: i64 = 0
258 var run: i64 = 1
259 i = 1
260 var dropped: i64 = 0
261 while i <= np {
262 var flush: i64 = 1
263 if i < np { if pairs[i] == pairs[i-1] { run = run + 1; flush = 0 } }
264 if flush == 1 {
265 let key: i64 = pairs[i-1]
266 let aid: i64 = key / SP_MAGIC_1048576
267 let bid: i64 = key % SP_MAGIC_1048576
268 marg[aid] = marg[aid] + run
269 if run >= SP_MINCNT { if nt3 < TCAP {
270 taid[nt3] = aid; tctx[nt3] = bid; tcnt[nt3] = run; nt3 = nt3 + 1
271 } } else { dropped = dropped + 1 }
272 run = 1
273 }
274 i = i + 1
275 }
276 db_w("triples cnt>="); db_n(SP_MINCNT); db_w(": "); db_n(nt3); db_w(" (singleton pairs dropped="); db_n(dropped); db_w(")\n" as *u8)
277
278 // total mass T for PMI
279 var TT: i64 = 0
280 i = 0
281 while i < nv { TT = TT + marg[i]; i = i + 1 }
282
283 // ---- PPMI16 transform in place on tcnt; row index + norms ----
284 let ridx: *i64 = sys_mmap((SP_MAXV+2)*8) as *i64
285 let nrm2: *i64 = sys_mmap(SP_MAXV*8) as *i64
286 var cur: i64 = 0
287 i = 0
288 while i < nv { ridx[i] = 0; nrm2[i] = 0; i = i + 1 }
289 ridx[nv] = 0
290 // triples are sorted by (aid,ctx) already (pair sort). build ridx by counting then prefix.
291 i = 0
292 while i < nt3 { let ai: i64 = taid[i]; ridx[ai] = ridx[ai] + 1; i = i + 1 }
293 var pref: i64 = 0
294 i = 0
295 while i <= nv { let cnt0: i64 = ridx[i]; ridx[i] = pref; pref = pref + cnt0; i = i + 1 }
296 // transform + norms (triples already in row-major order; indices align with ridx layout)
297 i = 0
298 var kept: i64 = 0
299 while i < nt3 {
300 let aid2: i64 = taid[i]
301 let bid2: i64 = tctx[i]
302 let cc: i64 = tcnt[i]
303 let den: i64 = marg[aid2] * marg[bid2]
304 var pp: i64 = 0
305 if den > 0 {
306 let q: i64 = (cc * TT * 256) / den
307 let lg: i64 = sp_ilog2x16(q)
308 if lg > 128 { pp = lg - 128 } // subtract log2(256)*16 = 8*16 -> PPMI (positive part)
309 }
310 tcnt[i] = pp
311 if pp > 0 { kept = kept + 1 }
312 nrm2[aid2] = nrm2[aid2] + pp*pp
313 i = i + 1
314 }
315 db_w("PPMI-positive cells = "); db_n(kept); db_w("/"); db_n(nt3); db_w(" total-mass T="); db_n(TT); db_w("\n" as *u8)
316
317 // ---- PROBES (the verdict): explicit sparse PPMI on the wall pairs ----
318 let w1: i64 = sp_wid(vh, nv, "won" as *u8)
319 let w2: i64 = sp_wid(vh, nv, "defeated" as *u8)
320 let w3: i64 = sp_wid(vh, nv, "purple" as *u8)
321 let w4: i64 = sp_wid(vh, nv, "city" as *u8)
322 let w5: i64 = sp_wid(vh, nv, "stadium" as *u8)
323 let w6: i64 = sp_wid(vh, nv, "january" as *u8)
324 var p1: i64 = 0-1
325 var p2: i64 = 0-1
326 var p3: i64 = 0-1
327 var p4: i64 = 0-1
328 if w1 >= 0 { if w2 >= 0 { p1 = sp_cos(tctx, tcnt, ridx, nrm2, w1, w2) } }
329 if w1 >= 0 { if w3 >= 0 { p2 = sp_cos(tctx, tcnt, ridx, nrm2, w1, w3) } }
330 if w4 >= 0 { if w5 >= 0 { p3 = sp_cos(tctx, tcnt, ridx, nrm2, w4, w5) } }
331 if w4 >= 0 { if w6 >= 0 { p4 = sp_cos(tctx, tcnt, ridx, nrm2, w4, w6) } }
332 db_w("PROBE cos(won,defeated)="); db_n(p1); db_w(" cos(won,purple)="); db_n(p2)
333 var ord1: i64 = 0
334 if p1 > p2 { ord1 = 1; db_w(" [orders CORRECTLY]\n" as *u8) } else { db_w(" [does NOT order]\n" as *u8) }
335 db_w("PROBE cos(city,stadium)="); db_n(p3); db_w(" cos(city,january)="); db_n(p4)
336 var ord2: i64 = 0
337 if p3 > p4 { ord2 = 1; db_w(" [orders CORRECTLY]\n" as *u8) } else { db_w(" [does NOT order]\n" as *u8) }
338
339 // NEG: identity + disjoint-row zero
340 var ident: i64 = 0
341 if w1 >= 0 { ident = sp_cos(tctx, tcnt, ridx, nrm2, w1, w1) }
342 db_w("NEG identity="); db_n(ident); db_w("\n" as *u8)
343
344 // ---- persist the QUERY-side structures -> knowledge/index/semppmi_dr_v1.bin
345 // layout: 32B header [NXPPMI1\0][nv][nt3][reserved] then vh[nv] ridx[nv+1] nrm2[nv] tctx[nt3] tval[nt3]
346 let hdr: *u8 = sys_mmap(SP_MAGIC_4096)
347 hdr[0]=78 as u8; hdr[1]=88 as u8; hdr[2]=80 as u8; hdr[3]=80 as u8; hdr[4]=77 as u8; hdr[5]=73 as u8; hdr[6]=49 as u8; hdr[7]=0 as u8
348 let hi2: *i64 = (hdr as i64 + 8) as *i64
349 hi2[0] = nv
350 hi2[1] = nt3
351 hi2[2] = 0
352 var werr: i64 = 0
353 // FAIL-CLOSED PERSIST (2026-08-01). This organ is a fork of nx_semppmi_build, which on 08-01
354 // overwrote a 43MB shared model with a vocab=0 build AFTER its own teeth printed RED; 20
355 // consumers lost dense retrieval. Refuse; leave the previous semppmi_dr_v1.bin intact.
356 if nv <= 0 { db_w("PERSIST REFUSED -- empty model (nv=" as *u8); db_n(nv); db_w(" nt="); db_n(nt3); db_w("). Refusing to overwrite knowledge/index/semppmi_dr_v1.bin with a build that failed its own teeth.\n" as *u8); return 1 }
357 if nt3 <= 0 { db_w("PERSIST REFUSED -- empty contexts (nt=" as *u8); db_n(nt3); db_w("). Refusing to overwrite knowledge/index/semppmi_dr_v1.bin.\n" as *u8); return 1 }
358 // FAIL-CLOSED PERSIST (2026-08-01). This organ is a fork of nx_semppmi_build, which on 08-01
359 // overwrote a 43MB shared model with a vocab=0 build AFTER its own teeth printed RED; 20
360 // consumers lost dense retrieval. Refuse; leave the previous semppmi_dr_v1.bin intact.
361 if nv <= 0 { db_w("PERSIST REFUSED -- empty model (nv=" as *u8); db_n(nv); db_w(" nt="); db_n(nt3); db_w("). Refusing to overwrite knowledge/index/semppmi_dr_v1.bin with a build that failed its own teeth.\n" as *u8); return 1 }
362 if nt3 <= 0 { db_w("PERSIST REFUSED -- empty contexts (nt=" as *u8); db_n(nt3); db_w("). Refusing to overwrite knowledge/index/semppmi_dr_v1.bin.\n" as *u8); return 1 }
363 let fd: i64 = sys_openat_wr("knowledge/index/semppmi_dr_v1.bin" as *u8, 0x1a4)
364 if fd < 0 { werr = 1 } else {
365 var hw: i64 = 0
366 while hw < 32 { let w0: i64 = sys_write(fd, (hdr as i64 + hw) as *u8, 32 - hw); if w0 <= 0 { werr = 1; hw = 32 } else { hw = hw + w0 } }
367 werr = werr + sp_wblock(fd, vh as *u8, nv*8)
368 werr = werr + sp_wblock(fd, ridx as *u8, (nv+1)*8)
369 werr = werr + sp_wblock(fd, nrm2 as *u8, nv*8)
370 werr = werr + sp_wblock(fd, tctx as *u8, nt3*8)
371 werr = werr + sp_wblock(fd, tcnt as *u8, nt3*8)
372 sys_close(fd)
373 }
374 if werr == 0 { db_w("persisted knowledge/index/semppmi_dr_v1.bin (query-side sparse PPMI)\n" as *u8) } else { db_w("PERSIST FAILED\n" as *u8) }
375
376 var pass: i64 = 0
377 if c[0] >= SP_MAGIC_3000 { pass = pass + 1 }
378 if nv >= SP_MAGIC_10000 { pass = pass + 1 }
379 if nt3 >= SP_MAGIC_100000 { pass = pass + 1 }
380 if kept >= SP_MAGIC_50000 { pass = pass + 1 }
381 if ident == 1000 { if werr == 0 { pass = pass + 1 } }
382 db_w("TEETH T1(strings)+T2(vocab)+T3(triples)+T4(ppmi-cells)+T5(identity) = "); db_n(pass); db_w("/5\n" as *u8)
383 db_w("PROBE-GATE (decides MODE2 wiring): ord(won-defeated)="); db_n(ord1); db_w(" ord(city-stadium)="); db_n(ord2); db_w("\n" as *u8)
384 if pass == 5 {
385 if ord1 == 1 { if ord2 == 1 {
386 db_w("GREEN + PROBES ORDER -- explicit sparse PPMI carries the signal; wire MODE2-soft next.\n" as *u8)
387 return 0
388 } }
389 db_w("GREEN (structure) but PROBES INSUFFICIENT -- count methods now exhausted at CANONICAL strength\n" as *u8)
390 db_w("(explicit vocab, no projection, full PPMI). The semantic wall is TRAINED-MODEL-BOUND, airtight.\n" as *u8)
391 return 0
392 }
393 db_w("RED -- build integrity failed\n" as *u8)
394 return 1
395}