code wiki / _hdl_build / _t2_bigram_lm_authored.nx
_t2_bigram_lm_authored.nx source
↩ module page · 429 lines · 18130 B
1// _t2_bigram_lm_authored.nx -- T2 rung 2a: the team's FIRST LANGUAGE MODEL. A next-token
2// model over the TRAINED BPE vocab (reloaded from knowledge/store/bpe_vocab_v1.txt -- this
3// rung is also the proof the persisted vocab round-trips into a consumer) on the team corpus.
4// Smallest honest LM: logits = W[cur] (vocab x vocab table; multiclass logistic regression =
5// CONVEX, so zero-init is exact and no symmetry trap exists), verified-CE loss, full-batch
6// accumulation + gated AdamW, deterministic early stop.
7// THE ORACLE IS ANALYTIC: from the integer bigram/unigram counts of the SAME token stream the
8// optimal achievable cross-entropies are computed in closed form --
9// H_uni = -(1/N) sum_b c(b) ln(c(b)/N) (context-free baseline)
10// H_bi = -(1/N) sum_ab c(ab) ln(c(ab)/c(a)) (the bigram optimum)
11// GATES: V vocab reloads (64 BPEMERGE rows parse) | O structure exists (H_bi < H_uni) |
12// L LEARNED: final CE < H_uni AND CE <= midpoint (H_uni+H_bi)/2 -- the model provably
13// uses context; distance-to-optimum reported in permil, never asserted |
14// B bit-exact two-run (the full 320x320 table identical).
15// Emits the TRIAGE training report (train_report_t2lm.log; joins train_reports.conf for the
16// pulse sweep) + MODELWRIGHT T2LM evidence rows + a deterministic greedy sample as evidence.
17// LAWS: struct-free, flat ifs, no &&/||, measured never asserted. license_tier: ORIGINAL
18import "nx_tgrad_core.nx"
19
20const LM_VCAP: i64 = 320
21const LM_MAXTOK: i64 = 1200
22const LM_EPOCHS: i64 = 200
23const LM_LOGEVERY: i64 = 10
24
25func lm_fp(fd: i64, s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(fd,s,n); return 0 }
26func lm_fn(fd: i64, v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(fd,"-" as *u8,1)}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48;k=1}; while m>0{t[k]=48+(m%10);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(fd,bb,k); return 0 }
27func lm_bp(fd: i64, s: *u8) -> i64 { _tg_puts(s); if fd >= 0 { lm_fp(fd, s) } return 0 }
28func lm_bn(fd: i64, v: i64) -> i64 { _tg_num(v); if fd >= 0 { lm_fn(fd, v) } return 0 }
29func lm_match(b: *u8, i: i64, n: i64, lit: *u8) -> i64 {
30 var k: i64 = 0
31 while lit[k] != (0 as u8) {
32 if i + k >= n { return 0 }
33 if b[i+k] != lit[k] { return 0 }
34 k = k + 1
35 }
36 return 1
37}
38func lm_keynum(b: *u8, ls: i64, le: i64, key: *u8) -> i64 {
39 var i: i64 = ls
40 while i < le {
41 if lm_match(b, i, le, key) == 1 {
42 var k: i64 = 0
43 while key[k] != (0 as u8) { k = k + 1 }
44 var j: i64 = i + k
45 var v: i64 = 0
46 var got: i64 = 0
47 while j < le {
48 let c: i64 = b[j] as i64
49 if c < 48 { j = le } else { if c > 57 { j = le } else { v = v*10 + (c - 48); got = 1; j = j + 1 } }
50 }
51 if got == 1 { return v }
52 }
53 i = i + 1
54 }
55 return 0 - 1
56}
57
58// reload the persisted vocab: BPEMERGE rows -> merges table; returns merge count
59func lm_load_vocab(path: *u8, merges: *i64) -> i64 {
60 let lenp: *i64 = sys_mmap(16) as *i64
61 let b: *u8 = sys_read_file(path, lenp)
62 let n: i64 = lenp[0]
63 if n <= 0 { return 0 - 1 }
64 var cnt: i64 = 0
65 var ls: i64 = 0
66 while ls < n {
67 var le: i64 = ls
68 var stop: i64 = 0
69 while stop == 0 {
70 if le >= n { stop = 1 } else { if b[le] == (10 as u8) { stop = 1 } else { le = le + 1 } }
71 }
72 if lm_match(b, ls, le, "BPEMERGE " as *u8) == 1 {
73 let idx: i64 = lm_keynum(b, ls, le, "idx=" as *u8)
74 let l: i64 = lm_keynum(b, ls, le, "left=" as *u8)
75 let r: i64 = lm_keynum(b, ls, le, "right=" as *u8)
76 if idx == 256 + cnt { if l >= 0 { if r >= 0 {
77 merges[cnt*2+0] = l
78 merges[cnt*2+1] = r
79 cnt = cnt + 1
80 } } }
81 }
82 ls = le + 1
83 }
84 return cnt
85}
86// corpus loader (conf rows "C <path>") -- same contract as the vocab trainer's
87func lm_load_corpus(conf: *u8, buf: *u8, cap: i64) -> i64 {
88 let lenp: *i64 = sys_mmap(16) as *i64
89 let b: *u8 = sys_read_file(conf, lenp)
90 let n: i64 = lenp[0]
91 if n <= 0 { return 0 - 1 }
92 var tot: i64 = 0
93 var ls: i64 = 0
94 while ls < n {
95 var le: i64 = ls
96 var stop: i64 = 0
97 while stop == 0 {
98 if le >= n { stop = 1 } else { if b[le] == (10 as u8) { stop = 1 } else { le = le + 1 } }
99 }
100 if lm_match(b, ls, le, "C " as *u8) == 1 {
101 let path: *u8 = sys_mmap(256)
102 var k: i64 = 0
103 var p: i64 = ls + 2
104 var stop2: i64 = 0
105 while stop2 == 0 {
106 if p >= le { stop2 = 1 } else { if b[p] == (32 as u8) { stop2 = 1 } else {
107 if k < 200 { path[k] = b[p]; k = k + 1 }
108 p = p + 1
109 } }
110 }
111 path[k] = 0 as u8
112 let l2: *i64 = sys_mmap(16) as *i64
113 let cb: *u8 = sys_read_file(path, l2)
114 var cn: i64 = l2[0]
115 if cn > 0 {
116 if tot + cn > cap { cn = cap - tot }
117 var i: i64 = 0
118 while i < cn { buf[tot+i] = cb[i]; i = i + 1 }
119 tot = tot + cn
120 }
121 }
122 ls = le + 1
123 }
124 return tot
125}
126// BPE encode by applying merges in training order (the trained-vocab contract)
127func lm_encode(bytes: *u8, n: i64, merges: *i64, mcount: i64, seq: *i64) -> i64 {
128 var i: i64 = 0
129 while i < n { seq[i] = bytes[i] as i64; i = i + 1 }
130 var slen: i64 = n
131 var m: i64 = 0
132 while m < mcount {
133 let a: i64 = merges[m*2+0]
134 let b: i64 = merges[m*2+1]
135 let nid: i64 = 256 + m
136 var wpos: i64 = 0
137 i = 0
138 while i < slen {
139 var took2: i64 = 0
140 if i + 1 < slen {
141 if seq[i] == a { if seq[i+1] == b {
142 seq[wpos] = nid
143 wpos = wpos + 1
144 i = i + 2
145 took2 = 1
146 } }
147 }
148 if took2 == 0 {
149 seq[wpos] = seq[i]
150 wpos = wpos + 1
151 i = i + 1
152 }
153 }
154 slen = wpos
155 m = m + 1
156 }
157 return slen
158}
159func lm_decode_tok(tok: i64, merges: *i64, out: *u8, opos: i64) -> i64 {
160 let stk: *i64 = sys_mmap(8192) as *i64
161 var sp: i64 = 0
162 stk[sp] = tok
163 sp = sp + 1
164 var w: i64 = opos
165 while sp > 0 {
166 sp = sp - 1
167 let t: i64 = stk[sp]
168 if t < 256 {
169 out[w] = t
170 w = w + 1
171 } else {
172 let m: i64 = t - 256
173 stk[sp] = merges[m*2+1]
174 sp = sp + 1
175 stk[sp] = merges[m*2+0]
176 sp = sp + 1
177 }
178 }
179 return w - opos
180}
181
182// train: full-batch accumulation over all transitions + one gated ad_step per epoch;
183// returns nothing -- out2 = {first, final} epoch-avg CE, curve sampled
184func lm_train(tape: *i64, nb: *i64, arena: *i64, ab: *i64, toks: *i64, ntok: i64, w: *i64, stopthr: i64, logfd: i64, out2: *i64, curve: *i64, curven: *i64) -> i64 {
185 var z: i64 = 0
186 while z < LM_VCAP * LM_VCAP { w[z] = 0; z = z + 1 }
187 let gt: *i64 = sys_mmap(1048576) as *i64
188 let mw: *i64 = sys_mmap(1048576) as *i64
189 let vw: *i64 = sys_mmap(1048576) as *i64
190 let onehot: *i64 = sys_mmap(4096) as *i64
191 let lr: i64 = tg_q(1, 10)
192 let b1c: i64 = tg_q(9, 10)
193 let b2c: i64 = tg_q(999, 1000)
194 let eps: i64 = tg_q(1, 100000)
195 let ntrans: i64 = ntok - 1
196 let invn: i64 = nx_f32_div(nx_i32_to_f32(1), nx_i32_to_f32(ntrans))
197 curven[0] = 0
198 curven[1] = 0 - 1
199 var ep: i64 = 0
200 while ep < LM_EPOCHS {
201 var z2: i64 = 0
202 while z2 < LM_VCAP * LM_VCAP { gt[z2] = 0; z2 = z2 + 1 }
203 var epsum: i64 = 0
204 var i: i64 = 0
205 while i < ntrans {
206 let cur: i64 = toks[i]
207 let nxt: i64 = toks[i+1]
208 var c: i64 = 0
209 while c < LM_VCAP { onehot[c] = 0; c = c + 1 }
210 onehot[nxt] = nx_i32_to_f32(1)
211 nb[0] = 0
212 ab[0] = 0
213 let lrow: i64 = tg_leaf(tape, nb, arena, ab, (w as i64 + cur*LM_VCAP*8) as *i64, LM_VCAP, 1)
214 let lt: i64 = tg_leaf(tape, nb, arena, ab, onehot, LM_VCAP, 1)
215 let loss: i64 = tg_celoss(tape, nb, arena, ab, lrow, lt)
216 tg_backward(tape, nb[0], loss)
217 let lv: *i64 = tg_valp(tape, loss)
218 epsum = nx_f32_add(epsum, lv[0])
219 let gr: *i64 = tg_gradp(tape, lrow)
220 c = 0
221 while c < LM_VCAP { gt[cur*LM_VCAP + c] = nx_f32_add(gt[cur*LM_VCAP + c], gr[c]); c = c + 1 }
222 i = i + 1
223 }
224 ad_step(w, gt, mw, vw, LM_VCAP * LM_VCAP, lr, b1c, b2c, eps, 0, ep + 1)
225 let avg: i64 = nx_f32_mul(epsum, invn)
226 if ep == 0 { out2[0] = avg }
227 out2[1] = avg
228 let milli: i64 = tg_milli(avg)
229 if ep % LM_LOGEVERY == 0 {
230 if curven[0] < 120 {
231 curve[curven[0]*2+0] = ep
232 curve[curven[0]*2+1] = milli
233 curven[0] = curven[0] + 1
234 }
235 if logfd >= 0 {
236 lm_bp(logfd, "MODELWRIGHT T2LM LOSSCURVE ep=" as *u8); lm_bn(logfd, ep)
237 lm_bp(logfd, " ce_millinat=" as *u8); lm_bn(logfd, milli); lm_bp(logfd, "\n" as *u8)
238 }
239 }
240 if nx_f32_lt(avg, stopthr) == 1 {
241 if curven[0] < 120 {
242 curve[curven[0]*2+0] = ep
243 curve[curven[0]*2+1] = milli
244 curven[0] = curven[0] + 1
245 }
246 curven[1] = ep
247 if logfd >= 0 { lm_bp(logfd, "MODELWRIGHT T2LM CONVERGED ep=" as *u8); lm_bn(logfd, ep); lm_bp(logfd, "\n" as *u8) }
248 ep = LM_EPOCHS
249 } else { ep = ep + 1 }
250 }
251 return 0
252}
253
254func main(argc: i64, argv: *i64) -> i64 {
255 _tg_puts("=== T2 RUNG 2a: FIRST LANGUAGE MODEL (next-token on the TRAINED BPE vocab; analytic CE oracle) ===\n" as *u8)
256 var conf: *u8 = "knowledge/status/bpe_corpus.conf" as *u8
257 if argc >= 2 { conf = argv[1] as *u8 }
258 let lfd: i64 = sys_openat_append("knowledge/status/modelwright.log" as *u8, 0x1a4)
259 if lfd < 0 { _tg_puts(" modelwright log open FAILED\n" as *u8); sys_exit(1); return 1 }
260 lm_bp(lfd, "MODELWRIGHT T2LM RUN epoch_unix=" as *u8); lm_bn(lfd, sys_now_realtime_sec()); lm_bp(lfd, "\n" as *u8)
261
262 // ---- GATE V: the persisted vocab reloads into a consumer ----
263 let merges: *i64 = sys_mmap(2048) as *i64
264 let mcount: i64 = lm_load_vocab("knowledge/store/bpe_vocab_v1.txt" as *u8, merges)
265 var pv: i64 = 1
266 if mcount < 32 { pv = 0 }
267 lm_bp(lfd, "MODELWRIGHT T2LM VOCAB merges=" as *u8); lm_bn(lfd, mcount); lm_bp(lfd, "\n" as *u8)
268 if pv == 1 { _tg_puts(" GATE V trained vocab reloaded from store: PASS\n" as *u8) } else { _tg_puts(" GATE V vocab reload: FAIL\n" as *u8) }
269
270 // ---- tokenize the corpus with the RELOADED vocab ----
271 let corpus: *u8 = sys_mmap(65536)
272 let total: i64 = lm_load_corpus(conf, corpus, 65536)
273 if total <= 0 { _tg_puts(" corpus conf missing -- LOUD FAIL\n" as *u8); sys_close(lfd); sys_exit(1); return 1 }
274 let seq: *i64 = sys_mmap(524288) as *i64
275 let alltok: i64 = lm_encode(corpus, total, merges, mcount, seq)
276 var ntok: i64 = alltok
277 if ntok > LM_MAXTOK { ntok = LM_MAXTOK }
278 lm_bp(lfd, "MODELWRIGHT T2LM DATA corpus_bytes=" as *u8); lm_bn(lfd, total)
279 lm_bp(lfd, " tokens_total=" as *u8); lm_bn(lfd, alltok)
280 lm_bp(lfd, " tokens_trained=" as *u8); lm_bn(lfd, ntok)
281 lm_bp(lfd, " (cap printed, never silent)\n" as *u8)
282
283 // ---- the ANALYTIC ORACLE: unigram + bigram optimal CE from integer counts ----
284 let uc: *i64 = sys_mmap(4096) as *i64
285 let bc: *i64 = sys_mmap(1048576) as *i64
286 let ntrans: i64 = ntok - 1
287 var i: i64 = 0
288 while i < ntrans {
289 uc[seq[i+1]] = uc[seq[i+1]] + 1
290 bc[seq[i]*LM_VCAP + seq[i+1]] = bc[seq[i]*LM_VCAP + seq[i+1]] + 1
291 i = i + 1
292 }
293 let cnt_a: *i64 = sys_mmap(4096) as *i64
294 i = 0
295 while i < ntrans { cnt_a[seq[i]] = cnt_a[seq[i]] + 1; i = i + 1 }
296 var hu: i64 = 0
297 var c: i64 = 0
298 while c < LM_VCAP {
299 if uc[c] > 0 {
300 let pr: i64 = nx_f32_div(nx_i32_to_f32(uc[c]), nx_i32_to_f32(ntrans))
301 hu = nx_f32_sub(hu, nx_f32_mul(nx_i32_to_f32(uc[c]), nx_f32_log(pr)))
302 }
303 c = c + 1
304 }
305 hu = nx_f32_div(hu, nx_i32_to_f32(ntrans))
306 var hb: i64 = 0
307 var a: i64 = 0
308 while a < LM_VCAP {
309 if cnt_a[a] > 0 {
310 c = 0
311 while c < LM_VCAP {
312 if bc[a*LM_VCAP + c] > 0 {
313 let pr2: i64 = nx_f32_div(nx_i32_to_f32(bc[a*LM_VCAP + c]), nx_i32_to_f32(cnt_a[a]))
314 hb = nx_f32_sub(hb, nx_f32_mul(nx_i32_to_f32(bc[a*LM_VCAP + c]), nx_f32_log(pr2)))
315 }
316 c = c + 1
317 }
318 }
319 a = a + 1
320 }
321 hb = nx_f32_div(hb, nx_i32_to_f32(ntrans))
322 var po: i64 = 1
323 if nx_f32_lt(hb, hu) == 0 { po = 0 }
324 let mid: i64 = nx_f32_mul(nx_f32_add(hu, hb), tg_q(1, 2))
325 lm_bp(lfd, "MODELWRIGHT T2LM ORACLE H_uni_millinat=" as *u8); lm_bn(lfd, tg_milli(hu))
326 lm_bp(lfd, " H_bigram_millinat=" as *u8); lm_bn(lfd, tg_milli(hb))
327 lm_bp(lfd, " midpoint_millinat=" as *u8); lm_bn(lfd, tg_milli(mid)); lm_bp(lfd, "\n" as *u8)
328 if po == 1 { _tg_puts(" GATE O sequential structure exists (H_bigram < H_unigram, analytic): PASS\n" as *u8) } else { _tg_puts(" GATE O: FAIL\n" as *u8) }
329
330 // ---- GATE L: train and provably USE context ----
331 let tape: *i64 = sys_mmap(32768) as *i64
332 let nb: *i64 = sys_mmap(16) as *i64
333 let arena: *i64 = sys_mmap(131072) as *i64
334 let ab: *i64 = sys_mmap(16) as *i64
335 let w1: *i64 = sys_mmap(1048576) as *i64
336 let o1: *i64 = sys_mmap(64) as *i64
337 let curve: *i64 = sys_mmap(4096) as *i64
338 let curven: *i64 = sys_mmap(32) as *i64
339 let stopthr: i64 = nx_f32_mul(mid, tg_q(49, 50))
340 lm_train(tape, nb, arena, ab, seq, ntok, w1, stopthr, lfd, o1, curve, curven)
341 var pl: i64 = 1
342 if nx_f32_lt(o1[1], hu) == 0 { pl = 0 }
343 if nx_f32_lt(o1[1], mid) == 0 { if o1[1] != mid { pl = 0 } }
344 var gap_permil: i64 = 0
345 let denom: i64 = nx_f32_sub(hu, hb)
346 if nx_f32_lt(0, denom) == 1 {
347 gap_permil = tg_milli(nx_f32_div(nx_f32_sub(o1[1], hb), denom))
348 }
349 lm_bp(lfd, "MODELWRIGHT T2LM LEARNED first_ce_millinat=" as *u8); lm_bn(lfd, tg_milli(o1[0]))
350 lm_bp(lfd, " final_ce_millinat=" as *u8); lm_bn(lfd, tg_milli(o1[1]))
351 lm_bp(lfd, " dist_to_optimum_permil_of_gap=" as *u8); lm_bn(lfd, gap_permil); lm_bp(lfd, "\n" as *u8)
352 if pl == 1 { _tg_puts(" GATE L model provably uses context (CE < H_uni and CE <= midpoint): PASS\n" as *u8) } else { _tg_puts(" GATE L: FAIL\n" as *u8) }
353
354 // ---- GATE B: bit-exact two-run over the full table ----
355 let w2: *i64 = sys_mmap(1048576) as *i64
356 let o2: *i64 = sys_mmap(64) as *i64
357 let curve2: *i64 = sys_mmap(4096) as *i64
358 let curven2: *i64 = sys_mmap(32) as *i64
359 lm_train(tape, nb, arena, ab, seq, ntok, w2, stopthr, 0 - 1, o2, curve2, curven2)
360 var pb: i64 = 1
361 i = 0
362 while i < LM_VCAP * LM_VCAP { if w1[i] != w2[i] { pb = 0 } i = i + 1 }
363 lm_bp(lfd, "MODELWRIGHT T2LM BITEXACT cells=" as *u8); lm_bn(lfd, LM_VCAP * LM_VCAP)
364 lm_bp(lfd, " identical=" as *u8); lm_bn(lfd, pb); lm_bp(lfd, "\n" as *u8)
365 if pb == 1 { _tg_puts(" GATE B bit-exact two-run (102400 cells): PASS\n" as *u8) } else { _tg_puts(" GATE B: FAIL\n" as *u8) }
366
367 // ---- evidence: deterministic greedy sample from the trained model ----
368 _tg_puts(" greedy sample: " as *u8)
369 let dec: *u8 = sys_mmap(4096)
370 var cur: i64 = seq[0]
371 var steps: i64 = 0
372 var dw: i64 = 0
373 while steps < 24 {
374 dw = dw + lm_decode_tok(cur, merges, dec, dw)
375 var best: i64 = 0
376 c = 1
377 while c < LM_VCAP {
378 if nx_f32_gt(w1[cur*LM_VCAP + c], w1[cur*LM_VCAP + best]) == 1 { best = c }
379 c = c + 1
380 }
381 cur = best
382 steps = steps + 1
383 }
384 dec[dw] = 0 as u8
385 var q: i64 = 0
386 while q < dw { if dec[q] == (10 as u8) { dec[q] = 32 } q = q + 1 }
387 _tg_puts(dec)
388 _tg_puts("\n" as *u8)
389
390 // ---- TRIAGE report (loss-curve lane; CLASS/PRED degenerate to the 1-class LM view) ----
391 var rpath: *u8 = "knowledge/status/train_report_t2lm.log" as *u8
392 if argc >= 2 { rpath = "/tmp/_t2lm_report_override.log" as *u8 }
393 let rfd: i64 = sys_openat_wr(rpath, 0x1a4)
394 if rfd >= 0 {
395 lm_fp(rfd, "TRIAGE-CLASS id=0 count=" as *u8); lm_fn(rfd, ntrans); lm_fp(rfd, "\n" as *u8)
396 lm_fp(rfd, "TRIAGE-PRED id=0 count=" as *u8); lm_fn(rfd, ntrans); lm_fp(rfd, "\n" as *u8)
397 // ACC for an LM = did training END at/below the gate line (right==total iff GATE L holds)
398 var rok: i64 = 0
399 if pl == 1 { rok = ntrans }
400 lm_fp(rfd, "TRIAGE-ACC right=" as *u8); lm_fn(rfd, rok)
401 lm_fp(rfd, " total=" as *u8); lm_fn(rfd, ntrans); lm_fp(rfd, "\n" as *u8)
402 var ci: i64 = 0
403 while ci < curven[0] {
404 lm_fp(rfd, "TRIAGE-LOSS ep=" as *u8); lm_fn(rfd, curve[ci*2+0])
405 lm_fp(rfd, " micro=" as *u8); lm_fn(rfd, curve[ci*2+1]); lm_fp(rfd, "\n" as *u8)
406 ci = ci + 1
407 }
408 sys_close(rfd)
409 _tg_puts(" training report written: " as *u8); _tg_puts(rpath); _tg_puts("\n" as *u8)
410 }
411
412 var gates: i64 = 0
413 if pv == 1 { gates = gates + 1 }
414 if po == 1 { gates = gates + 1 }
415 if pl == 1 { gates = gates + 1 }
416 if pb == 1 { gates = gates + 1 }
417 lm_bp(lfd, "MODELWRIGHT T2LM VERDICT gates=" as *u8); lm_bn(lfd, gates)
418 lm_bp(lfd, "/4" as *u8)
419 if gates == 4 { lm_bp(lfd, " pass=1\n" as *u8) } else { lm_bp(lfd, " pass=0\n" as *u8) }
420 sys_close(lfd)
421 if gates == 4 {
422 _tg_puts(" T2-LM GATE: PASS (the team's first language model -- trained vocab consumed, context provably used)\n" as *u8)
423 sys_exit(0)
424 return 0
425 }
426 _tg_puts(" T2-LM GATE: FAIL\n" as *u8)
427 sys_exit(1)
428 return 1
429}