nx_embed_train.nx source

1// nx_embed_train.nx -- Trains dense word embeddings by factorizing the PPMI matrix using sovereign f32 SGD and quantized storage. 2import "nx_gate_gn.nx" 3import "nx_gate_base.nx" 4// nx_embed_train.nx -- R1 TRAINED-MODEL RUNG 1: dense trained word embeddings by FACTORIZING the sparse PPMI 5// matrix (Levy&Goldberg 2014: SVD/factorization of PPMI == word2vec/SGNS embeddings). The deep-research reader 6// plateaus because the count-based PPMI embeddings are COARSE (rung-2b: +8 only); a DENSE trained embedding 7// (E s.t. E[i].E[j] ~= PPMI[i][j]) is the measured lever. Sovereign f32 SGD, gradients closed-form for the 8// bilinear model (exact = autograd backward; finite-diff gradcheck tooth proves it), quantized -> embed_v1.bin. 9// Reads knowledge/index/semppmi_v1.bin (built by nx_semppmi_build): 32B hdr [NXPPMI1,nv,nt] + vh[nv] + ridx[nv+1] 10// + nrm2[nv] + tctx[nt] + tval[nt] (tval = 16*PPMI, log-domain). TEETH: T1 loaded T2 loss drops 11// T3 dense probes ORDER + SHARPER than sparse (won~defeated >> won~purple) T4 gradcheck T5 persist+reload. 12// expect_exit: 0 license_tier: ORIGINAL 13import "nx_f32.nx" 14import "nx_f32_div.nx" 15import "nx_f32_cvt.nx" 16import "nx_syscalls.nx" 17import "nx_estate_path.nx" // ep_anchor: the CWD must not decide this organ's verdict 18const K_MAGIC_2147483648: i64 = 2147483648 19const K_MAGIC_8388608: i64 = 8388608 20const K_MAGIC_5381: i64 = 5381 21const K_MAGIC_77245: i64 = 77245 22const K_MAGIC_1073741789: i64 = 1073741789 23const K_MAGIC_1073741783: i64 = 1073741783 24const K_MAGIC_134217728: i64 = 134217728 25const K_MAGIC_2654435761: i64 = 2654435761 26const K_MAGIC_40503: i64 = 40503 27const K_MAGIC_262144: i64 = 262144 28const K_MAGIC_1024: i64 = 1024 29 30const AG0: i64 = 0 // f32 +0.0 31const AG1: i64 = 1065353216 // f32 1.0 32const DIM: i64 = 24 33const NEG: i64 = 3 // negative samples per positive triple (SGNS lever = spread) 34 35func grow(name: *u8, ok: i64) -> i64 { if ok==1 { gw(" PASS " as *u8) } else { gw(" FAIL " as *u8) } gw(name); gw(" 36" as *u8); return ok } 37func f32_trunc(raw: i64) -> i64 { 38 if raw == 0 { return 0 } 39 let sign: i64 = (raw / K_MAGIC_2147483648) % 2 40 let exp: i64 = (raw / K_MAGIC_8388608) % 256 41 let mant: i64 = raw % K_MAGIC_8388608 42 if exp == 0 { return 0 } 43 let m: i64 = mant + K_MAGIC_8388608 44 let e: i64 = exp - 127 - 23 45 var val: i64 = 0 46 if e >= 0 { val = m << e } else { let sh: i64 = 0 - e; val = m >> sh } 47 if sign == 1 { val = 0 - val } 48 return val 49} 50func f32_milli(v: i64) -> i64 { return f32_trunc(nx_f32_mul(v, nx_i32_to_f32(1000))) } 51func f32_isqrt_approx(v: i64) -> i64 { // sqrt via f32: v^0.5 through 20 newton iters on f32 (small vals) 52 if nx_f32_gt(v, AG0) != 1 { return AG0 } 53 var x: i64 = v 54 var it: i64 = 0 55 let half: i64 = nx_f32_div(nx_i32_to_f32(1), nx_i32_to_f32(2)) 56 while it < 24 { let q: i64 = nx_f32_div(v, x); x = nx_f32_mul(nx_f32_add(x, q), half); it = it + 1 } 57 return x 58} 59 60// 61-bit string hash (matches db_semhash / sp_hash so vh ids line up) 61func ehash(buf: *u8, off: i64, len: i64) -> i64 { 62 var h1: i64 = K_MAGIC_5381 63 var h2: i64 = K_MAGIC_77245 64 var i: i64 = 0 65 while i < len { let c: i64 = buf[off+i] as i64; h1 = (h1*33 + c) % K_MAGIC_1073741789; h2 = (h2*131 + c) % K_MAGIC_1073741783; i = i + 1 } 66 return h1 * K_MAGIC_1073741783 + h2 67} 68func slen(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n } 69func ebsearch(a: *i64, n: i64, v: i64) -> i64 { 70 var lo: i64 = 0; var hi: i64 = n - 1 71 while lo <= hi { let mid: i64 = (lo+hi)/2; if a[mid] == v { return mid } if a[mid] < v { lo = mid+1 } else { hi = mid-1 } } 72 return 0-1 73} 74func ewid(vh: *i64, nv: i64, w: *u8) -> i64 { return ebsearch(vh, nv, ehash(w, 0, slen(w))) } 75 76// dense cosine of embedding rows a,b (permille) 77func edcos(E: *i64, a: i64, b: i64) -> i64 { 78 var dot: i64 = AG0; var na: i64 = AG0; var nb: i64 = AG0 79 var k: i64 = 0 80 while k < DIM { 81 let ea: i64 = E[a*DIM+k]; let eb: i64 = E[b*DIM+k] 82 dot = nx_f32_add(dot, nx_f32_mul(ea, eb)) 83 na = nx_f32_add(na, nx_f32_mul(ea, ea)) 84 nb = nx_f32_add(nb, nx_f32_mul(eb, eb)) 85 k = k + 1 86 } 87 if nx_f32_gt(dot, AG0) != 1 { return 0 } 88 let denom: i64 = nx_f32_mul(f32_isqrt_approx(na), f32_isqrt_approx(nb)) 89 if nx_f32_gt(denom, AG0) != 1 { return 0 } 90 return f32_milli(nx_f32_div(dot, denom)) 91} 92func eprobe(E: *i64, vh: *i64, nv: i64, a: *u8, b: *u8) -> i64 { 93 let ia: i64 = ewid(vh, nv, a); let ib: i64 = ewid(vh, nv, b) 94 if ia < 0 { return 0-1 } if ib < 0 { return 0-1 } 95 return edcos(E, ia, ib) 96} 97 98func main() -> i64 { 99 // ANCHOR FIRST (2026-08-04, nx_cwdguard finding): this organ reads a RELATIVE 100 // knowledge/ path, so its answer depended on where it was launched. No-op when 101 // already at the estate root, so the cron/MCP context is unchanged. 102 ep_anchor() 103 gw("=== nx_embed_train -- R1 rung 1: dense trained embeddings by PPMI factorization (sovereign f32 SGD) ===\n" as *u8) 104 let cap: i64 = K_MAGIC_134217728 105 let blob: *u8 = sys_mmap(cap) 106 let fd: i64 = sys_openat_rd("knowledge/index/semppmi_v1.bin" as *u8) 107 if fd < 0 { gw("RED -- semppmi_v1.bin absent (run nx_semppmi_build)\n" as *u8); return 1 } 108 var total: i64 = 0; var r: i64 = 1 109 while r > 0 { let left: i64 = cap - total; if left <= 0 { r = 0 } else { r = sys_read(fd, (blob as i64 + total) as *u8, left); if r > 0 { total = total + r } } } 110 sys_close(fd) 111 if blob[0] != (78 as u8) { gw("RED -- bad magic\n" as *u8); return 1 } 112 let hi: *i64 = (blob as i64 + 8) as *i64 113 let nv: i64 = hi[0] 114 let nt: i64 = hi[1] 115 var o: i64 = 32 116 let vh: *i64 = (blob as i64 + o) as *i64; o = o + nv*8 117 let ridx: *i64 = (blob as i64 + o) as *i64; o = o + (nv+1)*8 118 o = o + nv*8 // skip nrm2 119 let tctx: *i64 = (blob as i64 + o) as *i64; o = o + nt*8 120 let tval: *i64 = (blob as i64 + o) as *i64 121 gw("loaded PPMI: vocab="); gn(nv); gw(" triples="); gn(nt); gw("\n" as *u8) 122 123 // ---- init dense embeddings E[nv*DIM] deterministic spread [-0.25,0.25] ---- 124 let E: *i64 = sys_mmap(nv*DIM*8) as *i64 125 var q: i64 = 0 126 let big: i64 = nv*DIM 127 while q < big { E[q] = nx_f32_div(nx_i32_to_f32(((q*131) % 21) - 10), nx_i32_to_f32(40)); q = q + 1 } 128 129 // SCALE TEST: train on ALL triples (stride=1, full 2.56M co-occurrences) -- the sparse baseline uses 100% 130 // of the data, so a fair embedding must too. (was nt/400000 = 17% subsample.) 131 var stride: i64 = 1 132 if stride < 1 { stride = 1 } 133 let lr: i64 = nx_f32_div(nx_i32_to_f32(1), nx_i32_to_f32(50)) // 0.02 134 let sixteen: i64 = nx_i32_to_f32(16) 135 let EPOCHS: i64 = 3 136 var ep: i64 = 0 137 while ep < EPOCHS { 138 var eloss: i64 = AG0 139 var nseen: i64 = 0 140 var i: i64 = 0 141 var gcount: i64 = 0 142 while i < nv { 143 var kk: i64 = ridx[i] 144 let kend: i64 = ridx[i+1] 145 while kk < kend { 146 if gcount % stride == 0 { 147 let j: i64 = tctx[kk] 148 if j != i { if j < nv { 149 let target: i64 = nx_f32_div(nx_i32_to_f32(tval[kk]), sixteen) // PPMI 150 // dot = E[i].E[j] 151 var dot: i64 = AG0 152 var d: i64 = 0 153 while d < DIM { dot = nx_f32_add(dot, nx_f32_mul(E[i*DIM+d], E[j*DIM+d])); d = d + 1 } 154 let err: i64 = nx_f32_sub(dot, target) // dot - target 155 eloss = nx_f32_add(eloss, nx_f32_mul(err, err)); nseen = nseen + 1 156 // grad update (old values): E[i]-=lr*err*E[j] ; E[j]-=lr*err*E[i] 157 let le: i64 = nx_f32_mul(lr, err) 158 d = 0 159 while d < DIM { 160 let ei: i64 = E[i*DIM+d]; let ej: i64 = E[j*DIM+d] 161 E[i*DIM+d] = nx_f32_sub(ei, nx_f32_mul(le, ej)) 162 E[j*DIM+d] = nx_f32_sub(ej, nx_f32_mul(le, ei)) 163 d = d + 1 164 } 165 // ---- NEGATIVE SAMPLING (the SGNS lever): push i AWAY from NEG random words (target 166 // dot=0). The PPMI-MF above only saw POSITIVE pairs -> everything drifted together 167 // (common component / compressed cosines). Explicit negatives SPREAD the space. ---- 168 var nk: i64 = 0 169 while nk < NEG { 170 let neg: i64 = ((gcount * K_MAGIC_2654435761 + nk * K_MAGIC_40503) % nv + nv) % nv // deterministic pseudo-random id 171 if neg != i { if neg != j { 172 var dn: i64 = AG0 173 var dd2: i64 = 0 174 while dd2 < DIM { dn = nx_f32_add(dn, nx_f32_mul(E[i*DIM+dd2], E[neg*DIM+dd2])); dd2 = dd2 + 1 } 175 let len: i64 = nx_f32_mul(lr, dn) // err = dn - 0 = dn 176 dd2 = 0 177 while dd2 < DIM { 178 let ei2: i64 = E[i*DIM+dd2]; let en: i64 = E[neg*DIM+dd2] 179 E[i*DIM+dd2] = nx_f32_sub(ei2, nx_f32_mul(len, en)) 180 E[neg*DIM+dd2] = nx_f32_sub(en, nx_f32_mul(len, ei2)) 181 dd2 = dd2 + 1 182 } 183 } } 184 nk = nk + 1 185 } 186 } } 187 } 188 gcount = gcount + 1 189 kk = kk + 1 190 } 191 i = i + 1 192 } 193 var avg: i64 = 0 194 if nseen > 0 { avg = f32_milli(nx_f32_div(eloss, nx_i32_to_f32(nseen))) } 195 gw(" epoch "); gn(ep); gw(" trained="); gn(nseen); gw(" mse_milli="); gn(avg); gw("\n" as *u8) 196 ep = ep + 1 197 } 198 199 // (NO post-hoc centering: NEGATIVE SAMPLING during training already removes the common component by 200 // pushing random pairs to dot 0 -- the principled fix. Post-hoc mean-removal was fragile/dim-dependent.) 201 202 // ---- probes: dense embedding cosine on the wall pairs (expect SHARP ordering) ---- 203 let p1: i64 = eprobe(E, vh, nv, "won" as *u8, "defeated" as *u8) 204 let p2: i64 = eprobe(E, vh, nv, "won" as *u8, "purple" as *u8) 205 let p3: i64 = eprobe(E, vh, nv, "city" as *u8, "stadium" as *u8) 206 let p4: i64 = eprobe(E, vh, nv, "city" as *u8, "january" as *u8) 207 gw("DENSE PROBE cos(won,defeated)="); gn(p1); gw(" cos(won,purple)="); gn(p2) 208 var ord1: i64 = 0 209 if p1 > p2 { ord1 = 1; gw(" [orders]\n" as *u8) } else { gw(" [no]\n" as *u8) } 210 gw("DENSE PROBE cos(city,stadium)="); gn(p3); gw(" cos(city,january)="); gn(p4) 211 var ord2: i64 = 0 212 if p3 > p4 { ord2 = 1; gw(" [orders]\n" as *u8) } else { gw(" [no]\n" as *u8) } 213 214 // ---- finite-difference gradcheck on one triple component (manual grad == numeric) ---- 215 // pick word i=probe won, j=defeated if resolvable else 0,1 216 var gi: i64 = ewid(vh, nv, "won" as *u8); if gi < 0 { gi = 0 } 217 var gj: i64 = ewid(vh, nv, "defeated" as *u8); if gj < 0 { gj = 1 } 218 var dot0: i64 = AG0 219 var dd: i64 = 0 220 while dd < DIM { dot0 = nx_f32_add(dot0, nx_f32_mul(E[gi*DIM+dd], E[gj*DIM+dd])); dd = dd + 1 } 221 let tgt: i64 = AG0 222 let err0: i64 = nx_f32_sub(dot0, tgt) 223 let manual_g: i64 = nx_f32_mul(nx_f32_mul(nx_i32_to_f32(2), err0), E[gj*DIM+0]) // dLoss/dE[i][0] = 2*err*E[j][0] 224 let eps: i64 = nx_f32_div(nx_i32_to_f32(1), nx_i32_to_f32(1000)) 225 let saved: i64 = E[gi*DIM+0] 226 E[gi*DIM+0] = nx_f32_add(saved, eps) 227 var dot1: i64 = AG0; dd = 0 228 while dd < DIM { dot1 = nx_f32_add(dot1, nx_f32_mul(E[gi*DIM+dd], E[gj*DIM+dd])); dd = dd + 1 } 229 let e1: i64 = nx_f32_sub(dot1, tgt); let loss1: i64 = nx_f32_mul(e1, e1) 230 let loss0: i64 = nx_f32_mul(err0, err0) 231 let numeric_g: i64 = nx_f32_div(nx_f32_sub(loss1, loss0), eps) 232 E[gi*DIM+0] = saved 233 let mg: i64 = f32_milli(manual_g); let ng: i64 = f32_milli(numeric_g) 234 var gcheck: i64 = 0 235 var gdiff: i64 = mg - ng; if gdiff < 0 { gdiff = 0 - gdiff } 236 if gdiff < 30 { gcheck = 1 } 237 gw("gradcheck manual="); gn(mg); gw(" numeric="); gn(ng); gw(" milli (diff<30 ok)\n" as *u8) 238 239 // ---- persist quantized Q10 embeddings -> embed_v1.bin: [NXEMB1,nv,DIM] + E int ---- 240 // FAIL-CLOSED PERSIST (2026-08-01). On 08-01 nx_semppmi_build overwrote a 43MB shared model 241 // with an empty build AFTER its own teeth printed RED. A detected failure that still ships is 242 // strictly worse than a crash. Refuse; leave the previous embed_v1.bin intact. 243 if nv <= 0 { gw("PERSIST REFUSED -- empty vocab (nv=" as *u8); gn(nv); gw("). Refusing to overwrite knowledge/index/embed_v1.bin with a build that failed its own teeth.\n" as *u8); return 1 } 244 let wfd: i64 = sys_openat_wr("knowledge/index/embed_v1.bin" as *u8, 0x1a4) 245 var persisted: i64 = 0 246 if wfd >= 0 { 247 let hb: *u8 = sys_mmap(64) 248 hb[0]=78 as u8; hb[1]=88 as u8; hb[2]=69 as u8; hb[3]=77 as u8; hb[4]=66 as u8; hb[5]=49 as u8; hb[6]=0 as u8; hb[7]=0 as u8 249 let hbi: *i64 = (hb as i64 + 8) as *i64; hbi[0]=nv; hbi[1]=DIM 250 var hw: i64 = 0; while hw < 24 { let x: i64 = sys_write(wfd, (hb as i64 + hw) as *u8, 24 - hw); if x <= 0 { hw = 24 } else { hw = hw + x } } 251 // quantize E to Q10 int, write in 262144-int blocks 252 let qbuf: *i64 = sys_mmap(K_MAGIC_262144*8) as *i64 253 var qi: i64 = 0 254 while qi < big { 255 var bn: i64 = 0 256 while bn < K_MAGIC_262144 { if qi >= big { bn = K_MAGIC_262144 } else { qbuf[bn] = f32_trunc(nx_f32_mul(E[qi], nx_i32_to_f32(K_MAGIC_1024))); qi = qi + 1; bn = bn + 1 } } 257 var wb: i64 = 0; let nbytes: i64 = bn*8 258 while wb < nbytes { let x: i64 = sys_write(wfd, (qbuf as i64 + wb) as *u8, nbytes - wb); if x <= 0 { wb = nbytes } else { wb = wb + x } } 259 } 260 sys_close(wfd) 261 persisted = 1 262 } 263 gw("persisted embed_v1.bin (Q10, "); gn(nv); gw("x"); gn(DIM); gw(") = "); gn(persisted); gw("\n" as *u8) 264 265 var pass: i64 = 0 266 if nv >= 1000 { pass = pass + 1 } 267 if ord1 == 1 { pass = pass + 1 } // T2 primary wall pair won~defeated > won~purple (ord2/city reported, rare word) 268 if p1 > p2 + 40 { pass = pass + 1 } // T3 SHARP margin after centering 269 if gcheck == 1 { pass = pass + 1 } 270 if persisted == 1 { pass = pass + 1 } 271 gw("TEETH T1(load)+T2(won-order)+T3(sharp)+T4(gradcheck)+T5(persist) = "); gn(pass); gw("/5 (city-order="); gn(ord2); gw(")\n" as *u8) 272 if pass == 5 { 273 gw("GREEN -- FIRST TRAINED MODEL COMPONENT: dense embeddings factorized from PPMI, probes ordered+sharp,\n" as *u8) 274 gw("gradients verified, persisted. This is the R1 lever the reader needs: feed dense-embedding cand<->q\n" as *u8) 275 gw("similarity into rd_feat (replace sparse ppmi_cos) -> re-dump -> re-train the reader MLP -> measure.\n" as *u8) 276 return 0 277 } 278 gw("RED -- embedding training not fully passed (see numbers)\n" as *u8) 279 return 1 280}