nx_semcorpus_build.nx source

1// nx_semcorpus_build.nx -- R3b-scale CORPUS-LEVEL distributional embeddings (the rung after 3 measured-negative 2// per-question climbs; see nx_qabench MODE2 header). Builds ONE persisted hashed count-embedding matrix from 3// ALL banked benchmark context prose (~4.5MB Wikipedia-register: qab_squad_big + qab_hotpot_big + 4// qab_musique_big), then nx_qabench's MODE2 loads it and selects sentences by corpus-informed cosine. 5// METHOD (sovereign, integer, no training loop -- Random-Indexing/feature-hashing lineage): text = every JSON 6// string value >= 48 chars (paragraphs+questions; SHORT strings -- keys, titles, GOLD ANSWERS -- are 7// structurally excluded = no label leakage; corpus is the inference-visible context pool, unsupervised). 8// token row = djb2(token) % 16384; for each content token, each stop-filtered context token within +-4 hashes 9// to one of 256 dims: M[row*256 + dim] += 1. Persisted -> knowledge/index/semcorpus_v1.bin (NXSEMC1 header). 10// HONEST CAVEATS baked in: hash collisions (~2-4 words/row) + raw counts (no PPMI) = v1; probes below REPORT 11// whether corpus stats order won~defeated > won~purple (the exact wall) -- reported, not asserted. 12// TEETH: T1 strings>=3000 T2 token-events>=200000 T3 nonzero cells>=50000 T4 write+reload spot-check 13// T5 NEG: identity cos=1000 and a gibberish token's empty row scores 0 vs everything. 14// expect_exit: 0 license_tier: ORIGINAL 15import "nx_qabench_engine.nx" 16const SC_MAGIC_5381: i64 = 5381 17const SC_MAGIC_1073741789: i64 = 1073741789 18const SC_MAGIC_16777216: i64 = 16777216 19const SC_MAGIC_250000: i64 = 250000 20const SC_MAGIC_4000: i64 = 4000 21const SC_MAGIC_2654435761: i64 = 2654435761 22const SC_MAGIC_262144: i64 = 262144 23const SC_MAGIC_32768: i64 = 32768 24const SC_MAGIC_4096: i64 = 4096 25const SC_MAGIC_65536: i64 = 65536 26const SC_MAGIC_3000: i64 = 3000 27const SC_MAGIC_200000: i64 = 200000 28const SC_MAGIC_50000: i64 = 50000 29 30const SC_ROWS: i64 = 16384 31const SC_DIMS: i64 = 256 32const SC_WIN: i64 = 4 33const SC_MINSTR: i64 = 48 34 35func sc_hash(buf: *u8, off: i64, len: i64) -> i64 { 36 var h: i64 = SC_MAGIC_5381 37 var i: i64 = 0 38 while i < len { h = (h*33 + (buf[off+i] as i64)) % SC_MAGIC_1073741789; i = i + 1 } 39 return h 40} 41 42func sc_isqrt(v: i64) -> i64 { 43 if v <= 0 { return 0 } 44 var x: i64 = v 45 var y: i64 = (x + 1) / 2 46 while y < x { x = y; let q: i64 = v / x; y = (x + q) / 2 } 47 return x 48} 49 50// cosine between two matrix ROWS (256 dims), permille 51func sc_cos_rows(M: *i64, ra: i64, rb: i64) -> i64 { 52 var dot: i64 = 0 53 var na: i64 = 0 54 var nb: i64 = 0 55 var k: i64 = 0 56 let ba: i64 = ra * SC_DIMS 57 let bb: i64 = rb * SC_DIMS 58 while k < SC_DIMS { 59 dot = dot + M[ba+k]*M[bb+k] 60 na = na + M[ba+k]*M[ba+k] 61 nb = nb + M[bb+k]*M[bb+k] 62 k = k + 1 63 } 64 if dot <= 0 { return 0 } 65 let d1: i64 = sc_isqrt(na) 66 let d2: i64 = sc_isqrt(nb) 67 if d1 == 0 { return 0 } 68 if d2 == 0 { return 0 } 69 var cv: i64 = (dot*1000)/(d1*d2) // dot<=4e9, *1000 fits i64; isqrt truncation can push >1000 -> cap 70 if cv > 1000 { cv = 1000 } 71 return cv 72} 73 74func sc_probe(M: *i64, a: *u8, b: *u8) -> i64 { 75 let ha: i64 = sc_hash(a, 0, sc_len(a)) 76 let hb: i64 = sc_hash(b, 0, sc_len(b)) 77 return sc_cos_rows(M, ha % SC_ROWS, hb % SC_ROWS) 78} 79func sc_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n } 80 81// ingest one raw file: walk JSON strings; long ones -> norm -> tok -> windowed hashed co-occurrence. 82// counters c[0]=strings c[1]=token-events 83func sc_ingest(g: *i64, path: *u8, M: *i64, c: *i64) -> i64 { 84 let total: i64 = db_read_raw(g, path, SC_MAGIC_16777216) 85 if total <= 0 { db_w(" [absent] "); db_w(path); db_w("\n" as *u8); return 0 } 86 if total == 0-2 { db_w(" [too big] "); db_w(path); db_w("\n" as *u8); return 0 } 87 g[2] = db_body_start(g, total) 88 g[1] = total - g[2] 89 let scr: *u8 = g[5] as *u8 // decoded string scratch (256KB) 90 let nrm: *u8 = g[6] as *u8 // normalized scratch (256KB) 91 let toff: *i64 = g[7] as *i64 92 let tlen: *i64 = g[8] as *i64 93 let hrow: *i64 = g[9] as *i64 // per-token row hash (content) or -1 94 g[3] = 0 95 while g[3] < g[1] { 96 let ch: i64 = db_b(g, g[3]) 97 if ch == 34 { 98 let sl: i64 = db_dec_str(g, scr, SC_MAGIC_250000) 99 if sl >= SC_MINSTR { 100 c[0] = c[0] + 1 101 let nl: i64 = qs_norm(scr, nrm) 102 let nt: i64 = qs_tok(nrm, nl, toff, tlen, SC_MAGIC_4000) 103 var j: i64 = 0 104 while j < nt { 105 var r: i64 = 0-1 106 if tlen[j] >= 2 { if db_is_stop(nrm, toff[j], tlen[j]) == 0 { let hh: i64 = sc_hash(nrm, toff[j], tlen[j]); r = hh } } 107 hrow[j] = r 108 j = j + 1 109 } 110 var a: i64 = 0 111 while a < nt { 112 if hrow[a] >= 0 { 113 c[1] = c[1] + 1 114 let arow: i64 = (hrow[a] % SC_ROWS) * SC_DIMS 115 var b: i64 = a - SC_WIN 116 if b < 0 { b = 0 } 117 var bend: i64 = a + SC_WIN 118 if bend > nt-1 { bend = nt-1 } 119 while b <= bend { 120 if b != a { if hrow[b] >= 0 { 121 let dim: i64 = (hrow[b] * SC_MAGIC_2654435761) % SC_DIMS 122 let cell: i64 = arow + dim 123 M[cell] = M[cell] + 1 124 } } 125 b = b + 1 126 } 127 } 128 a = a + 1 129 } 130 } 131 } else { g[3] = g[3] + 1 } 132 } 133 db_w(" ingested "); db_w(path); db_w(" strings-so-far="); db_n(c[0]); db_w(" token-events="); db_n(c[1]); db_w("\n" as *u8) 134 return 1 135} 136 137func main() -> i64 { 138 db_w("=== nx_semcorpus_build -- corpus-level hashed count-embeddings from banked benchmark prose (R3b-scale v1) ===\n" as *u8) 139 let g: *i64 = sys_mmap(256) as *i64 140 let m0: *u8 = sys_mmap(SC_MAGIC_16777216); g[0] = m0 as i64 141 let m5: *u8 = sys_mmap(SC_MAGIC_262144); g[5] = m5 as i64 142 let m6: *u8 = sys_mmap(SC_MAGIC_262144); g[6] = m6 as i64 143 let m7: *u8 = sys_mmap(SC_MAGIC_32768); g[7] = m7 as i64 144 let m8: *u8 = sys_mmap(SC_MAGIC_32768); g[8] = m8 as i64 145 let m9: *u8 = sys_mmap(SC_MAGIC_32768); g[9] = m9 as i64 146 let msz: i64 = SC_ROWS * SC_DIMS * 8 147 let M: *i64 = sys_mmap(msz + SC_MAGIC_4096) as *i64 148 let c: *i64 = sys_mmap(64) as *i64 149 c[0]=0; c[1]=0 150 151 sc_ingest(g, "knowledge/fetched/qab_squad_big.raw" as *u8, M, c) 152 sc_ingest(g, "knowledge/fetched/qab_hotpot_big.raw" as *u8, M, c) 153 sc_ingest(g, "knowledge/fetched/qab_musique_big.raw" as *u8, M, c) 154 155 var nz: i64 = 0 156 var z: i64 = 0 157 let zn: i64 = SC_ROWS * SC_DIMS 158 while z < zn { if M[z] != 0 { nz = nz + 1 } z = z + 1 } 159 db_w("strings="); db_n(c[0]); db_w(" token-events="); db_n(c[1]); db_w(" nonzero-cells="); db_n(nz); db_w("/"); db_n(zn); db_w("\n" as *u8) 160 161 // ---- MARGINAL DISCOUNT (PPMI-lineage, Levy&Goldberg-lite): raw counts let hub words dominate every 162 // vector (measured: probes failed to order on raw counts). Standard count-embedding normalization: 163 // cell = C*1000 / (sqrt(RowSum)*sqrt(ColSum)+1) -- discounts frequent rows/dims, keeps integers. ---- 164 let rowm: *i64 = sys_mmap(SC_ROWS * 8) as *i64 165 let colm: *i64 = sys_mmap(SC_DIMS * 8) as *i64 166 var r0: i64 = 0 167 while r0 < SC_ROWS { 168 var s0: i64 = 0 169 var d0: i64 = 0 170 let b0: i64 = r0 * SC_DIMS 171 while d0 < SC_DIMS { s0 = s0 + M[b0+d0]; d0 = d0 + 1 } 172 rowm[r0] = s0 173 r0 = r0 + 1 174 } 175 var d1c: i64 = 0 176 while d1c < SC_DIMS { 177 var s1: i64 = 0 178 var r1: i64 = 0 179 while r1 < SC_ROWS { s1 = s1 + M[r1*SC_DIMS + d1c]; r1 = r1 + 1 } 180 colm[d1c] = s1 181 d1c = d1c + 1 182 } 183 var r2: i64 = 0 184 while r2 < SC_ROWS { 185 let rs: i64 = sc_isqrt(rowm[r2]) 186 let b2: i64 = r2 * SC_DIMS 187 var d2c: i64 = 0 188 while d2c < SC_DIMS { 189 let cellv: i64 = M[b2+d2c] 190 if cellv != 0 { 191 let cs: i64 = sc_isqrt(colm[d2c]) 192 M[b2+d2c] = (cellv*1000) / (rs*cs + 1) 193 } 194 d2c = d2c + 1 195 } 196 r2 = r2 + 1 197 } 198 db_w("marginal-discount applied (PPMI-lite: c*1000/(sqrt(row)*sqrt(col)+1))\n" as *u8) 199 200 // ---- semantic PROBES (REPORTED -- the wall test: corpus should pull won~defeated together) ---- 201 let p1: i64 = sc_probe(M, "won" as *u8, "defeated" as *u8) 202 let p2: i64 = sc_probe(M, "won" as *u8, "purple" as *u8) 203 let p3: i64 = sc_probe(M, "city" as *u8, "stadium" as *u8) 204 let p4: i64 = sc_probe(M, "city" as *u8, "january" as *u8) 205 db_w("PROBE cos(won,defeated)="); db_n(p1); db_w(" cos(won,purple)="); db_n(p2) 206 if p1 > p2 { db_w(" [orders CORRECTLY]\n" as *u8) } else { db_w(" [does NOT order]\n" as *u8) } 207 db_w("PROBE cos(city,stadium)="); db_n(p3); db_w(" cos(city,january)="); db_n(p4) 208 if p3 > p4 { db_w(" [orders CORRECTLY]\n" as *u8) } else { db_w(" [does NOT order]\n" as *u8) } 209 210 // ---- T5 NEG-CONTROLS ---- 211 let hw: i64 = sc_hash("won" as *u8, 0, 3) % SC_ROWS 212 let ident: i64 = sc_cos_rows(M, hw, hw) 213 let pg: i64 = sc_probe(M, "zqxjvkq" as *u8, "won" as *u8) 214 db_w("NEG identity cos(won,won)="); db_n(ident); db_w(" gibberish-row cos="); db_n(pg); db_w("\n" as *u8) 215 216 // ---- persist: header NXSEMC1 + rows + dims, then the matrix ---- 217 let hdr: *u8 = sys_mmap(SC_MAGIC_4096) 218 hdr[0]=78 as u8; hdr[1]=88 as u8; hdr[2]=83 as u8; hdr[3]=69 as u8; hdr[4]=77 as u8; hdr[5]=67 as u8; hdr[6]=49 as u8; hdr[7]=0 as u8 219 let hi: *i64 = (hdr as i64 + 8) as *i64 220 hi[0] = SC_ROWS 221 hi[1] = SC_DIMS 222 hi[2] = c[1] 223 // FAIL-CLOSED PERSIST (2026-08-01). On 08-01 nx_semppmi_build overwrote a 43MB shared model 224 // with a vocab=0 build AFTER its own teeth printed RED; 20 consumers lost dense retrieval. 225 // A detected failure that still ships is strictly worse than a crash. Refuse; leave live intact. 226 if c[1] <= 0 { db_w("PERSIST REFUSED -- empty corpus (events=" as *u8); db_n(c[1]); db_w("). Refusing to overwrite knowledge/index/semcorpus_v1.bin with a build that failed its own teeth.\n" as *u8); return 1 } 227 if msz <= 0 { db_w("PERSIST REFUSED -- empty matrix (msz=" as *u8); db_n(msz); db_w("). Refusing to overwrite knowledge/index/semcorpus_v1.bin.\n" as *u8); return 1 } 228 let fd: i64 = sys_openat_wr("knowledge/index/semcorpus_v1.bin" as *u8, 0x1a4) 229 if fd < 0 { db_w("RED -- cannot open semcorpus_v1.bin for write\n" as *u8); return 1 } 230 var w: i64 = 0 231 while w < 32 { let ww0: i64 = sys_write(fd, (hdr as i64 + w) as *u8, 32 - w); if ww0 <= 0 { w = 32 } else { w = w + ww0 } } 232 var off: i64 = 0 233 var werr: i64 = 0 234 while off < msz { 235 var chunk: i64 = SC_MAGIC_262144 236 if msz - off < chunk { chunk = msz - off } 237 let ww: i64 = sys_write(fd, (M as i64 + off) as *u8, chunk) 238 if ww <= 0 { werr = 1; off = msz } else { off = off + ww } 239 } 240 sys_close(fd) 241 242 // ---- T4 reload spot-check ---- 243 var t4: i64 = 0 244 let lb: *i64 = sys_mmap(SC_MAGIC_65536) as *i64 245 let rfd: i64 = sys_openat_rd("knowledge/index/semcorpus_v1.bin" as *u8) 246 if rfd >= 0 { 247 var got: i64 = 0 248 while got < SC_MAGIC_65536 { let rr: i64 = sys_read(rfd, (lb as i64 + got) as *u8, SC_MAGIC_65536 - got); if rr <= 0 { got = SC_MAGIC_65536 } else { got = got + rr } } 249 sys_close(rfd) 250 let magic_ok: i64 = 0 251 let lbb: *u8 = lb as *u8 252 var mg: i64 = 1 253 if lbb[0] != (78 as u8) { mg = 0 } 254 if lbb[6] != (49 as u8) { mg = 0 } 255 let lrows: i64 = lb[1] 256 let ldims: i64 = lb[2] 257 let cell0: i64 = lb[4] // first matrix cell (after 32B header = i64 idx 4) 258 if mg == 1 { if lrows == SC_ROWS { if ldims == SC_DIMS { if cell0 == M[0] { t4 = 1 } } } } 259 if magic_ok == 0 { } // (magic folded into mg) 260 } 261 262 var pass: i64 = 0 263 if c[0] >= SC_MAGIC_3000 { pass = pass + 1 } 264 if c[1] >= SC_MAGIC_200000 { pass = pass + 1 } 265 if nz >= SC_MAGIC_50000 { pass = pass + 1 } 266 if t4 == 1 { pass = pass + 1 } 267 // T5 for a HASHED space: identity must be exactly 1000; a gibberish token's row is NOT empty by design 268 // (30k+ words hash into 16k rows -> every row occupied) so the honest bound is the collision noise FLOOR. 269 var t5: i64 = 0 270 if ident == 1000 { if pg < 200 { t5 = 1 } } 271 if t5 == 1 { pass = pass + 1 } 272 db_w("TEETH T1(strings)+T2(events)+T3(nonzero)+T4(reload)+T5(neg) = "); db_n(pass); db_w("/5\n" as *u8) 273 if pass == 5 { 274 db_w("GREEN -- artifact integrity OK; matrix persisted (knowledge/index/semcorpus_v1.bin, 33.5MB).\n" as *u8) 275 db_w("VERDICT ON SEMANTIC QUALITY (from the probes above): if the probe pairs do NOT order, this matrix\n" as *u8) 276 db_w("is INSUFFICIENT for MODE2 -- do NOT wire it expecting gains. MEASURED 2026-07-08: raw counts AND\n" as *u8) 277 db_w("PPMI-lite BOTH fail the won~defeated ordering at 16k-rows x 256-dims x 400k events => hashed count\n" as *u8) 278 db_w("embeddings at this scale do not carry word similarity. The semantic wall is TRAINED-MODEL-BOUND\n" as *u8) 279 db_w("(explicit vocab + PPMI + SVD/training + orders more corpus = the R1 no-float-model thread). This\n" as *u8) 280 db_w("builder + .bin format + the MODE2 socket remain the plug-in scaffold for that trained embedder.\n" as *u8) 281 if werr == 1 { db_w("(write short -- rerun)\n" as *u8); return 1 } 282 return 0 283 } 284 db_w("RED -- corpus build integrity failed\n" as *u8) 285 return 1 286}