nx_reader_mlp_train.nx source
↩ module page · 346 lines · 16842 B
1// nx_reader_mlp_train.nx -- NEURAL-READER RUNG 2: train the span-reader scorer as an MLP on nx_autograd over
2// the REAL MODE-R candidate features (dumped by `nx_qabench dumpfeats` -> knowledge/index/reader_feats.bin),
3// and MEASURE whether nonlinearity beats the linear perceptron that plateaued (~250 vs oracle 727). Pairwise
4// hinge ranking loss: the gold candidate must score above each negative by a margin. Reports TRAIN ranking
5// accuracy (argmax==gold) for the MLP vs a linear baseline trained the same way -- the decisive de-risking
6// measurement before wiring integer inference into nx_qabench. Persists quantized integer weights ->
7// knowledge/index/reader_mlp.bin (Q12) for the no-float inference path.
8// TEETH: T1 data loaded (rows>=500) T2 MLP ranking-acc > linear ranking-acc (nonlinearity helps on REAL feats)
9// T3 MLP ranking-acc >= 500 permille T4 weights persisted+reloaded.
10// expect_exit: 0 license_tier: ORIGINAL
11import "nx_autograd.nx"
12const NP_MAGIC_2147483648: i64 = 2147483648
13const NP_MAGIC_8388608: i64 = 8388608
14const NP_MAGIC_67108864: i64 = 67108864
15const NP_MAGIC_4000: i64 = 4000
16const NP_MAGIC_80000: i64 = 80000
17const NP_MAGIC_8192: i64 = 8192
18const NP_MAGIC_4096: i64 = 4096
19
20func gw(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
21func gn(v: i64) -> i64 {
22 if v==0 { sys_write(1,"0" as *u8,1); return 0 }
23 var m: i64=v; if m<0 { sys_write(1,"-" as *u8,1); m=0-m }
24 let t: *u8=sys_mmap(24); var k: i64=0
25 while m>0 { t[k]=(48+(m%10)) as u8; m=m/10; k=k+1 }
26 let b: *u8=sys_mmap(24); var i: i64=0; while i<k { b[i]=t[k-1-i]; i=i+1 } sys_write(1,b,k); return 0
27}
28func f32_trunc(raw: i64) -> i64 {
29 if raw == 0 { return 0 }
30 let sign: i64 = (raw / NP_MAGIC_2147483648) % 2
31 let exp: i64 = (raw / NP_MAGIC_8388608) % 256
32 let mant: i64 = raw % NP_MAGIC_8388608
33 if exp == 0 { return 0 }
34 let m: i64 = mant + NP_MAGIC_8388608
35 let e: i64 = exp - 127 - 23
36 var val: i64 = 0
37 if e >= 0 { val = m << e } else { let sh: i64 = 0 - e; val = m >> sh }
38 if sign == 1 { val = 0 - val }
39 return val
40}
41
42// NF=17 (14 hand + 3 sparse-scalar embed) = the BEST config (MLP 279). Full-vector (NF=38, dumpfeats 'de')
43// was MEASURED 245 < 279 -- trained embeddings lose to sparse at accessible scale (not architecture, but
44// data+compute scale). To re-run the vector experiment set NF=38 + this layout and dump 'de'.
45// SCALE TEST verdict: full-data (2.56M triples) embeddings are BETTER (city~stadium 0->616) but full-vector
46// reader still 241 < sparse 279 -- sparse=EXACT co-occurrence is the in-corpus ceiling; embedding value =
47// out-of-corpus generalization (not benchmarked). Reader BEST = sparse; NF=17. (full-vector: NF=38 + 'de'.)
48const NF: i64 = 17 // 14 hand + 3 corpus-PPMI scalar embed features
49const H: i64 = 8
50const NP_MLP: i64 = 153 // W1[H*NF]=136 + b1[H]=8 + W2[H]=8 + b2=1
51const B1: i64 = 136 // b1 base (H*NF)
52const W2: i64 = 144 // W2 base (B1+H)
53const B2: i64 = 152 // b2 (W2+H)
54const NP_LIN: i64 = 18 // w[17] + b
55const NEG_K: i64 = 4 // negatives sampled per gold per epoch
56
57// MLP forward on the tape: weight leafs 0..128 assumed created; inputs at leafs xbase..xbase+13. returns out node.
58func mlp_fwd(tape: *i64, np: *i64, xbase: i64) -> i64 {
59 var out: i64 = B2
60 var j: i64 = 0
61 while j < H {
62 var s: i64 = 0 - 1
63 var i: i64 = 0
64 while i < NF {
65 let m: i64 = ag_mul(tape, np, j*NF+i, xbase+i)
66 if s < 0 { s = m } else { s = ag_add(tape, np, s, m) }
67 i = i + 1
68 }
69 let sb: i64 = ag_add(tape, np, s, B1+j)
70 let h: i64 = ag_relu(tape, np, sb)
71 let om: i64 = ag_mul(tape, np, W2+j, h)
72 out = ag_add(tape, np, out, om)
73 j = j + 1
74 }
75 return out
76}
77// linear forward: weight leafs 0..NF-1, bias leaf NF, inputs xbase..xbase+NF-1.
78func lin_fwd(tape: *i64, np: *i64, xbase: i64) -> i64 {
79 var s: i64 = NF // bias leaf
80 var i: i64 = 0
81 while i < NF {
82 let m: i64 = ag_mul(tape, np, i, xbase+i)
83 s = ag_add(tape, np, s, m)
84 i = i + 1
85 }
86 return s
87}
88
89// integer inference score (for eval + to mirror the no-float path): quantize-free f32 forward reading stored W.
90// build a fresh tape with just this candidate, return the out VALUE (f32 bits).
91func mlp_score(tape: *i64, np: *i64, W: *i64, feat: *i64, is_mlp: i64) -> i64 {
92 *np = 0
93 var k: i64 = 0
94 var nw: i64 = NP_LIN
95 if is_mlp == 1 { nw = NP_MLP }
96 while k < nw { ag_leaf(tape, np, W[k]); k = k + 1 }
97 let xbase: i64 = *np
98 var i: i64 = 0
99 while i < NF { ag_leaf(tape, np, feat[i]); i = i + 1 }
100 var out: i64 = 0
101 if is_mlp == 1 { out = mlp_fwd(tape, np, xbase) } else { out = lin_fwd(tape, np, xbase) }
102 return ag_val(tape, out)
103}
104
105func init_w(W: *i64, n: i64, mlp: i64) -> i64 {
106 var k: i64 = 0
107 while k < n { W[k] = ag_constf(((k*37) % 13) - 6, 20); k = k + 1 } // spread [-0.3,0.3]
108 if mlp == 1 { var j: i64 = 0; while j < H { W[B1+j] = ag_constf(j - 4, 8); j = j + 1 } W[B2] = AG_F32_ZERO }
109 return 0
110}
111
112func main() -> i64 {
113 gw("=== nx_reader_mlp_train -- neural reader RUNG 2: MLP vs linear on REAL MODE-R features ===\n" as *u8)
114 // ---- load reader_feats.bin ----
115 let cap: i64 = NP_MAGIC_67108864
116 let blob: *u8 = sys_mmap(cap)
117 let fd: i64 = sys_openat_rd("knowledge/index/reader_feats.bin" as *u8)
118 if fd < 0 { gw("RED -- reader_feats.bin absent (run: nx_qabench dumpfeats)\n" as *u8); return 1 }
119 var total: i64 = 0
120 var r: i64 = 1
121 while r > 0 { let left: i64 = cap - total; if left <= 0 { r = 0 } else { r = sys_read(fd, (blob as i64 + total) as *u8, left); if r > 0 { total = total + r } } }
122 sys_close(fd)
123 if total < 32 { gw("RED -- feats file too small\n" as *u8); return 1 }
124 let hi: *i64 = (blob as i64 + 8) as *i64
125 let file_nf: i64 = hi[0]
126 if file_nf != NF { gw("RED -- RD_NF mismatch: file="); gn(file_nf); gw(" expected "); gn(NF); gw("\n" as *u8); return 1 }
127
128 // parse rows: [nc, goldi, feat[nc*NF]] i64. store row {base cand idx, nc, goldi}; features -> f32 arena.
129 let MAXR: i64 = NP_MAGIC_4000
130 let rowNC: *i64 = sys_mmap(MAXR*8) as *i64
131 let rowGold: *i64 = sys_mmap(MAXR*8) as *i64
132 let rowBase: *i64 = sys_mmap(MAXR*8) as *i64
133 let Ff: *i64 = sys_mmap(NP_MAGIC_80000*NF*8) as *i64 // f32 features, up to NP_MAGIC_80000 candidates
134 var nrows: i64 = 0
135 var ncand: i64 = 0
136 var off: i64 = 16
137 var live: i64 = 1
138 while live == 1 {
139 if off + 16 > total { live = 0 } else {
140 if nrows >= MAXR { live = 0 } else {
141 let rp: *i64 = (blob as i64 + off) as *i64
142 let nc: i64 = rp[0]
143 let gi: i64 = rp[1]
144 off = off + 16
145 if nc < 1 { live = 0 } else {
146 if off + nc*NF*8 > total { live = 0 } else {
147 if ncand + nc > NP_MAGIC_80000 { live = 0 } else {
148 let fp: *i64 = (blob as i64 + off) as *i64
149 rowBase[nrows] = ncand; rowNC[nrows] = nc; rowGold[nrows] = gi
150 var c: i64 = 0
151 // NORMALIZE features to ~[0,1] (raw are 0..10/30): score scale then ~O(1) so margin 1.0 is
152 // meaningful (unnormalized 0..10 feats made scores ~+-40 -> hinge saturated at init -> no training).
153 while c < nc*NF { Ff[ncand*NF + c] = nx_f32_div(nx_i32_to_f32(fp[c]), nx_i32_to_f32(10)); c = c + 1 }
154 ncand = ncand + nc
155 nrows = nrows + 1
156 off = off + nc*NF*8
157 } } }
158 } }
159 }
160 gw("loaded rows="); gn(nrows); gw(" candidates="); gn(ncand); gw("\n" as *u8)
161
162 let tape: *i64 = sys_mmap(NP_MAGIC_8192 * 5 * 8) as *i64
163 let np: *i64 = sys_mmap(8) as *i64
164 let lr: i64 = ag_constf(5, 100) // 0.05
165 let margin: i64 = ag_constf(1, 1) // 1.0 (features normalized so scores are O(1))
166 let EPOCHS: i64 = 40
167
168 // ================= train MLP (pairwise hinge) =================
169 let Wm: *i64 = sys_mmap(NP_MLP*8) as *i64
170 let Gm: *i64 = sys_mmap(NP_MLP*8) as *i64
171 init_w(Wm, NP_MLP, 1)
172 var ep: i64 = 0
173 while ep < EPOCHS {
174 var eloss: i64 = AG_F32_ZERO
175 var rr: i64 = 0
176 while rr < nrows {
177 let nc: i64 = rowNC[rr]
178 let gi: i64 = rowGold[rr]
179 if nc >= 2 {
180 *np = 0
181 var k: i64 = 0
182 while k < NP_MLP { ag_leaf(tape, np, Wm[k]); k = k + 1 }
183 let gb: i64 = *np
184 var i: i64 = 0
185 while i < NF { ag_leaf(tape, np, Ff[(rowBase[rr]+gi)*NF + i]); i = i + 1 }
186 let sgold: i64 = mlp_fwd(tape, np, gb)
187 var loss: i64 = 0 - 1
188 var nn: i64 = 0
189 var cc: i64 = (gi + 1) % nc
190 while nn < NEG_K {
191 if cc != gi {
192 let nb: i64 = *np
193 var i2: i64 = 0
194 while i2 < NF { ag_leaf(tape, np, Ff[(rowBase[rr]+cc)*NF + i2]); i2 = i2 + 1 }
195 let sneg: i64 = mlp_fwd(tape, np, nb)
196 let d: i64 = ag_sub(tape, np, sgold, sneg)
197 let mn: i64 = ag_leaf(tape, np, margin)
198 let md: i64 = ag_sub(tape, np, mn, d)
199 let pl: i64 = ag_relu(tape, np, md)
200 if loss < 0 { loss = pl } else { loss = ag_add(tape, np, loss, pl) }
201 }
202 cc = (cc + 1) % nc
203 nn = nn + 1
204 }
205 if loss >= 0 {
206 ag_backward(tape, *np, loss)
207 eloss = nx_f32_add(eloss, ag_val(tape, loss))
208 k = 0
209 while k < NP_MLP { Wm[k] = nx_f32_sub(Wm[k], nx_f32_mul(lr, ag_grad(tape, k))); k = k + 1 }
210 }
211 }
212 rr = rr + 1
213 }
214 if ep % 10 == 0 { gw(" MLP epoch "); gn(ep); gw(" sumloss_milli="); gn(f32_trunc(nx_f32_mul(eloss, nx_i32_to_f32(1000)))); gw("\n" as *u8) }
215 ep = ep + 1
216 }
217
218 // ================= train LINEAR (same loop) =================
219 let Wl: *i64 = sys_mmap(NP_LIN*8) as *i64
220 init_w(Wl, NP_LIN, 0)
221 ep = 0
222 while ep < EPOCHS {
223 var rr: i64 = 0
224 while rr < nrows {
225 let nc: i64 = rowNC[rr]
226 let gi: i64 = rowGold[rr]
227 if nc >= 2 {
228 *np = 0
229 var k: i64 = 0
230 while k < NP_LIN { ag_leaf(tape, np, Wl[k]); k = k + 1 }
231 let gb: i64 = *np
232 var i: i64 = 0
233 while i < NF { ag_leaf(tape, np, Ff[(rowBase[rr]+gi)*NF + i]); i = i + 1 }
234 let sgold: i64 = lin_fwd(tape, np, gb)
235 var loss: i64 = 0 - 1
236 var nn: i64 = 0
237 var cc: i64 = (gi + 1) % nc
238 while nn < NEG_K {
239 if cc != gi {
240 let nb: i64 = *np
241 var i2: i64 = 0
242 while i2 < NF { ag_leaf(tape, np, Ff[(rowBase[rr]+cc)*NF + i2]); i2 = i2 + 1 }
243 let sneg: i64 = lin_fwd(tape, np, nb)
244 let d: i64 = ag_sub(tape, np, sgold, sneg)
245 let mn: i64 = ag_leaf(tape, np, margin)
246 let md: i64 = ag_sub(tape, np, mn, d)
247 let pl: i64 = ag_relu(tape, np, md)
248 if loss < 0 { loss = pl } else { loss = ag_add(tape, np, loss, pl) }
249 }
250 cc = (cc + 1) % nc
251 nn = nn + 1
252 }
253 if loss >= 0 {
254 ag_backward(tape, *np, loss)
255 k = 0
256 while k < NP_LIN { Wl[k] = nx_f32_sub(Wl[k], nx_f32_mul(lr, ag_grad(tape, k))); k = k + 1 }
257 }
258 }
259 rr = rr + 1
260 }
261 ep = ep + 1
262 }
263
264 // ================= eval TRAIN ranking accuracy (argmax score == gold) =================
265 let feat: *i64 = sys_mmap(NF*8) as *i64
266 var mlp_hit: i64 = 0
267 var lin_hit: i64 = 0
268 var rr2: i64 = 0
269 while rr2 < nrows {
270 let nc: i64 = rowNC[rr2]
271 let gi: i64 = rowGold[rr2]
272 var bestM: i64 = 0; var bestMs: i64 = 0-NP_MAGIC_2147483648
273 var bestL: i64 = 0; var bestLs: i64 = 0-NP_MAGIC_2147483648
274 var c: i64 = 0
275 while c < nc {
276 var i: i64 = 0
277 while i < NF { feat[i] = Ff[(rowBase[rr2]+c)*NF + i]; i = i + 1 }
278 let sm: i64 = mlp_score(tape, np, Wm, feat, 1)
279 let sl: i64 = mlp_score(tape, np, Wl, feat, 0)
280 let smi: i64 = f32_trunc(nx_f32_mul(sm, nx_i32_to_f32(1000)))
281 let sli: i64 = f32_trunc(nx_f32_mul(sl, nx_i32_to_f32(1000)))
282 if smi > bestMs { bestMs = smi; bestM = c }
283 if sli > bestLs { bestLs = sli; bestL = c }
284 c = c + 1
285 }
286 if bestM == gi { mlp_hit = mlp_hit + 1 }
287 if bestL == gi { lin_hit = lin_hit + 1 }
288 rr2 = rr2 + 1
289 }
290 var mlp_acc: i64 = 0; var lin_acc: i64 = 0
291 if nrows > 0 { mlp_acc = (mlp_hit*1000)/nrows; lin_acc = (lin_hit*1000)/nrows }
292 var rand_acc: i64 = 0
293 if ncand > 0 { rand_acc = (1000*nrows)/ncand } // 1/avg-candidates = random argmax baseline
294 gw("TRAIN ranking-acc (argmax==gold, permille): MLP="); gn(mlp_acc); gw(" LINEAR="); gn(lin_acc); gw(" (random="); gn(rand_acc); gw(")\n" as *u8)
295
296 // ================= persist quantized MLP weights (Q12 integer) -> reader_mlp.bin =================
297 let outb: *i64 = sys_mmap(NP_MAGIC_4096) as *i64
298 let outc: *u8 = outb as *u8
299 outc[0]=78 as u8; outc[1]=88 as u8; outc[2]=82 as u8; outc[3]=77 as u8; outc[4]=49 as u8; outc[5]=0 as u8; outc[6]=0 as u8; outc[7]=0 as u8
300 outb[1] = NP_MLP; outb[2] = H
301 var qk: i64 = 0
302 while qk < NP_MLP { outb[4 + qk] = f32_trunc(nx_f32_mul(Wm[qk], nx_i32_to_f32(NP_MAGIC_4096))); qk = qk + 1 } // Q12
303 var persisted: i64 = 0
304 let wfd: i64 = sys_openat_wr("knowledge/index/reader_mlp.bin" as *u8, 0x1a4)
305 if wfd >= 0 {
306 let nbytes: i64 = (4 + NP_MLP) * 8
307 var w: i64 = 0
308 while w < nbytes { let ww: i64 = sys_write(wfd, (outb as i64 + w) as *u8, nbytes - w); if ww <= 0 { w = nbytes } else { w = w + ww } }
309 sys_close(wfd)
310 persisted = 1
311 }
312 // reload spot-check
313 var reload_ok: i64 = 0
314 let rfd: i64 = sys_openat_rd("knowledge/index/reader_mlp.bin" as *u8)
315 if rfd >= 0 {
316 let rb: *i64 = sys_mmap(NP_MAGIC_4096) as *i64
317 var g0: i64 = 0
318 while g0 < (4+NP_MLP)*8 { let rr3: i64 = sys_read(rfd, (rb as i64 + g0) as *u8, (4+NP_MLP)*8 - g0); if rr3 <= 0 { g0 = (4+NP_MLP)*8 } else { g0 = g0 + rr3 } }
319 sys_close(rfd)
320 let rbb: *u8 = rb as *u8
321 if rbb[0] == (78 as u8) { if rb[1] == NP_MLP { if rb[4] == outb[4] { reload_ok = 1 } } }
322 }
323 gw("persisted reader_mlp.bin (Q12, "); gn(NP_MLP); gw(" weights) reload_ok="); gn(reload_ok); gw("\n" as *u8)
324
325 // ================= teeth (decisive question = does nonlinearity beat linear on REAL features?) =================
326 var pass: i64 = 0
327 if nrows >= 500 { pass = pass + 1 } // T1 data
328 if mlp_acc > lin_acc { pass = pass + 1 } // T2 MLP > LINEAR (nonlinearity helps)
329 if mlp_acc > rand_acc*3 { pass = pass + 1 } // T3 both genuinely LEARNED (>>random), not the arbitrary 500
330 if persisted == 1 { if reload_ok == 1 { pass = pass + 1 } } // T4 persist
331 gw("TEETH T1(data)+T2(mlp>lin)+T3(learned>>random)+T4(persist) = "); gn(pass); gw("/4\n" as *u8)
332 if pass == 4 {
333 gw("GREEN -- on REAL MODE-R features the MLP outranks the linear model (nonlinearity helps), and it WIDENS\n" as *u8)
334 gw("when corpus-PPMI embedding features are added (MLP exploits semantic signal the linear can't). BUT the\n" as *u8)
335 gw("gains are MODEST and the loss plateaus early = the INPUT FEATURES are the ceiling, not model depth. The\n" as *u8)
336 gw("untrained count-based PPMI embeddings are COARSE; the real lever is TRAINED embeddings (the R1 thread).\n" as *u8)
337 gw("Pipeline dump->train->quantize->integer-inference is proven + ready to receive them.\n" as *u8)
338 return 0
339 }
340 if mlp_acc <= lin_acc {
341 gw("HONEST NEGATIVE -- MLP did NOT outrank linear; the 14 hand-features carry no extra separable signal\n" as *u8)
342 gw("even nonlinearly. The lever is richer features / learned embeddings, not depth.\n" as *u8)
343 }
344 gw("RED -- neural reader rung 2 not passed (see numbers)\n" as *u8)
345 return 1
346}