code wiki / _hdl_build / _bpe_vocab_authored.nx
_bpe_vocab_authored.nx source
↩ module page · 322 lines · 12900 B
1// _bpe_vocab_authored.nx -- T2 rung 1 of the training-substrate ladder: TRAIN a BPE vocab
2// on the TEAM CORPUS (the read side existed: nx_f32_bpe_load; the TRAINING side was the
3// research-flagged gap -- this rung lands it at the smallest honest scale: greedy byte-pair,
4// deterministic tie-break = lowest-left-then-lowest-right, no regex pretokenization;
5// honest scope noted in the registry).
6// Corpus is conf-driven (law 11): knowledge/status/bpe_corpus.conf rows "C <path>".
7// GATES: K hand-computed merge oracle on the classic aaabdaaabac corpus (merges MUST come out
8// (97,97),(97,98),(256,257) -- construction-known, computed by hand in this header:
9// aa x4 wins; then tie Za=2 vs ab=2 -> ab by lowest-left; then Z,257 x2)
10// | D determinism (two trains -> identical merge tables)
11// | R LOSSLESS roundtrip (encode->decode == original bytes for EVERY corpus line)
12// | C compression MEASURED (tokens < bytes; permil reported, never asserted).
13// Vocab persisted durable + reloadable: knowledge/store/bpe_vocab_v1.txt BPEMERGE rows.
14// Evidence: MODELWRIGHT T2 rows -> modelwright.log. Tamper hook: argv[1] conf override.
15// LAWS: struct-free, flat ifs, no &&/||. license_tier: ORIGINAL
16import "nx_syscalls.nx"
17
18const BP_MERGES: i64 = 64
19const BP_VCAP: i64 = 320
20const BP_MAXB: i64 = 65536
21
22func bp_w(fd: i64, s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(fd,s,n); return 0 }
23func bp_wn(fd: i64, v: i64) -> i64 { let bb: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(fd,"-" as *u8,1)}; let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48;k=1}; while m>0{t[k]=48+(m%10);m=m/10;k=k+1}; var i: i64=0; while i<k{bb[i]=t[k-1-i];i=i+1}; sys_write(fd,bb,k); return 0 }
24func bp_bp(fd: i64, s: *u8) -> i64 { bp_w(1, s); if fd >= 0 { bp_w(fd, s) } return 0 }
25func bp_bn(fd: i64, v: i64) -> i64 { bp_wn(1, v); if fd >= 0 { bp_wn(fd, v) } return 0 }
26func bp_match(b: *u8, i: i64, n: i64, lit: *u8) -> i64 {
27 var k: i64 = 0
28 while lit[k] != (0 as u8) {
29 if i + k >= n { return 0 }
30 if b[i+k] != lit[k] { return 0 }
31 k = k + 1
32 }
33 return 1
34}
35
36// load corpus bytes from conf rows "C <path>" into buf; returns total bytes
37func bp_load(conf: *u8, buf: *u8) -> i64 {
38 let lenp: *i64 = sys_mmap(16) as *i64
39 let b: *u8 = sys_read_file(conf, lenp)
40 let n: i64 = lenp[0]
41 if n <= 0 { return 0 - 1 }
42 var tot: i64 = 0
43 var ls: i64 = 0
44 while ls < n {
45 var le: i64 = ls
46 var stop: i64 = 0
47 while stop == 0 {
48 if le >= n { stop = 1 } else { if b[le] == (10 as u8) { stop = 1 } else { le = le + 1 } }
49 }
50 if bp_match(b, ls, le, "C " as *u8) == 1 {
51 let path: *u8 = sys_mmap(256)
52 var k: i64 = 0
53 var p: i64 = ls + 2
54 var stop2: i64 = 0
55 while stop2 == 0 {
56 if p >= le { stop2 = 1 } else { if b[p] == (32 as u8) { stop2 = 1 } else {
57 if k < 200 { path[k] = b[p]; k = k + 1 }
58 p = p + 1
59 } }
60 }
61 path[k] = 0 as u8
62 let l2: *i64 = sys_mmap(16) as *i64
63 let cb: *u8 = sys_read_file(path, l2)
64 var cn: i64 = l2[0]
65 if cn > 0 {
66 if tot + cn > BP_MAXB { cn = BP_MAXB - tot }
67 var i: i64 = 0
68 while i < cn { buf[tot+i] = cb[i]; i = i + 1 }
69 tot = tot + cn
70 }
71 }
72 ls = le + 1
73 }
74 return tot
75}
76
77// train merges over the byte stream; merges = pairs table (m*2 = left, m*2+1 = right);
78// returns the number of merges actually trained (stops early when no pair repeats)
79func bp_train(bytes: *u8, n: i64, seq: *i64, counts: *i64, merges: *i64) -> i64 {
80 var i: i64 = 0
81 while i < n { seq[i] = bytes[i] as i64; i = i + 1 }
82 var slen: i64 = n
83 var m: i64 = 0
84 while m < BP_MERGES {
85 var z: i64 = 0
86 while z < BP_VCAP * BP_VCAP { counts[z] = 0; z = z + 1 }
87 i = 0
88 while i + 1 < slen {
89 counts[seq[i] * BP_VCAP + seq[i+1]] = counts[seq[i] * BP_VCAP + seq[i+1]] + 1
90 i = i + 1
91 }
92 // deterministic argmax: strict > over ascending pair index = lowest-left-then-lowest-right tie-break
93 var best: i64 = 0 - 1
94 var bestc: i64 = 1
95 z = 0
96 while z < BP_VCAP * BP_VCAP {
97 if counts[z] > bestc { bestc = counts[z]; best = z }
98 z = z + 1
99 }
100 if best < 0 { return m }
101 let a: i64 = best / BP_VCAP
102 let b: i64 = best % BP_VCAP
103 merges[m*2+0] = a
104 merges[m*2+1] = b
105 let nid: i64 = 256 + m
106 var wpos: i64 = 0
107 i = 0
108 while i < slen {
109 var took2: i64 = 0
110 if i + 1 < slen {
111 if seq[i] == a { if seq[i+1] == b {
112 seq[wpos] = nid
113 wpos = wpos + 1
114 i = i + 2
115 took2 = 1
116 } }
117 }
118 if took2 == 0 {
119 seq[wpos] = seq[i]
120 wpos = wpos + 1
121 i = i + 1
122 }
123 }
124 slen = wpos
125 m = m + 1
126 }
127 return m
128}
129
130// encode a byte span into tokens by applying the merges IN TRAINING ORDER; returns token count
131func bp_encode(bytes: *u8, off: i64, n: i64, merges: *i64, mcount: i64, seq: *i64) -> i64 {
132 var i: i64 = 0
133 while i < n { seq[i] = bytes[off+i] as i64; i = i + 1 }
134 var slen: i64 = n
135 var m: i64 = 0
136 while m < mcount {
137 let a: i64 = merges[m*2+0]
138 let b: i64 = merges[m*2+1]
139 let nid: i64 = 256 + m
140 var wpos: i64 = 0
141 i = 0
142 while i < slen {
143 var took2: i64 = 0
144 if i + 1 < slen {
145 if seq[i] == a { if seq[i+1] == b {
146 seq[wpos] = nid
147 wpos = wpos + 1
148 i = i + 2
149 took2 = 1
150 } }
151 }
152 if took2 == 0 {
153 seq[wpos] = seq[i]
154 wpos = wpos + 1
155 i = i + 1
156 }
157 }
158 slen = wpos
159 m = m + 1
160 }
161 return slen
162}
163
164// decode one token into out (iterative stack expansion); returns bytes written
165func bp_decode_tok(tok: i64, merges: *i64, out: *u8, opos: i64) -> i64 {
166 let stk: *i64 = sys_mmap(8192) as *i64
167 var sp: i64 = 0
168 stk[sp] = tok
169 sp = sp + 1
170 var w: i64 = opos
171 while sp > 0 {
172 sp = sp - 1
173 let t: i64 = stk[sp]
174 if t < 256 {
175 out[w] = t
176 w = w + 1
177 } else {
178 let m: i64 = t - 256
179 stk[sp] = merges[m*2+1]
180 sp = sp + 1
181 stk[sp] = merges[m*2+0]
182 sp = sp + 1
183 }
184 }
185 return w - opos
186}
187
188func main(argc: i64, argv: *i64) -> i64 {
189 bp_w(1, "=== T2 RUNG 1: BPE VOCAB TRAINING on the team corpus (oracle KAT + determinism + lossless roundtrip + measured compression) ===\n" as *u8)
190 var conf: *u8 = "knowledge/status/bpe_corpus.conf" as *u8
191 if argc >= 2 { conf = argv[1] as *u8 }
192 let lfd: i64 = sys_openat_append("knowledge/status/modelwright.log" as *u8, 0x1a4)
193 if lfd < 0 { bp_w(1, " modelwright log open FAILED\n" as *u8); sys_exit(1); return 1 }
194 bp_bp(lfd, "MODELWRIGHT T2 RUN epoch_unix=" as *u8); bp_bn(lfd, sys_now_realtime_sec()); bp_bp(lfd, "\n" as *u8)
195
196 let counts: *i64 = sys_mmap(1048576) as *i64
197 let seq: *i64 = sys_mmap(524288) as *i64
198 let seq2: *i64 = sys_mmap(524288) as *i64
199
200 // ---- GATE K: hand-computed oracle on the classic corpus ----
201 let kat: *u8 = "aaabdaaabac" as *u8
202 let kbuf: *u8 = sys_mmap(32)
203 var i: i64 = 0
204 while i < 11 { kbuf[i] = kat[i]; i = i + 1 }
205 let km: *i64 = sys_mmap(1024) as *i64
206 let kn: i64 = bp_train(kbuf, 11, seq, counts, km)
207 var pk: i64 = 1
208 if kn < 3 { pk = 0 }
209 if km[0] != 97 { pk = 0 }
210 if km[1] != 97 { pk = 0 }
211 if km[2] != 97 { pk = 0 }
212 if km[3] != 98 { pk = 0 }
213 if km[4] != 256 { pk = 0 }
214 if km[5] != 257 { pk = 0 }
215 bp_bp(lfd, "MODELWRIGHT T2 KAT merges=" as *u8); bp_bn(lfd, kn)
216 bp_bp(lfd, " m0=(" as *u8); bp_bn(lfd, km[0]); bp_bp(lfd, "," as *u8); bp_bn(lfd, km[1])
217 bp_bp(lfd, ") m1=(" as *u8); bp_bn(lfd, km[2]); bp_bp(lfd, "," as *u8); bp_bn(lfd, km[3])
218 bp_bp(lfd, ") m2=(" as *u8); bp_bn(lfd, km[4]); bp_bp(lfd, "," as *u8); bp_bn(lfd, km[5]); bp_bp(lfd, ")\n" as *u8)
219 if pk == 1 { bp_w(1, " GATE K hand-computed merge oracle (aa, ab, Z+ab): PASS\n" as *u8) } else { bp_w(1, " GATE K oracle: FAIL\n" as *u8) }
220
221 // ---- load the REAL team corpus ----
222 let corpus: *u8 = sys_mmap(BP_MAXB)
223 let total: i64 = bp_load(conf, corpus)
224 if total <= 0 {
225 bp_w(1, " corpus conf missing/empty -- LOUD FAIL\n" as *u8)
226 sys_close(lfd)
227 sys_exit(1)
228 return 1
229 }
230
231 // ---- GATE D: deterministic training (two runs -> identical merge tables) ----
232 let m1: *i64 = sys_mmap(2048) as *i64
233 let m2: *i64 = sys_mmap(2048) as *i64
234 let n1: i64 = bp_train(corpus, total, seq, counts, m1)
235 let n2: i64 = bp_train(corpus, total, seq, counts, m2)
236 var pd: i64 = 1
237 if n1 != n2 { pd = 0 }
238 i = 0
239 while i < n1 * 2 { if m1[i] != m2[i] { pd = 0 } i = i + 1 }
240 bp_bp(lfd, "MODELWRIGHT T2 TRAIN corpus_bytes=" as *u8); bp_bn(lfd, total)
241 bp_bp(lfd, " merges=" as *u8); bp_bn(lfd, n1)
242 bp_bp(lfd, " deterministic=" as *u8); bp_bn(lfd, pd); bp_bp(lfd, "\n" as *u8)
243 if pd == 1 { bp_w(1, " GATE D deterministic vocab training: PASS\n" as *u8) } else { bp_w(1, " GATE D: FAIL\n" as *u8) }
244
245 // ---- GATE R: LOSSLESS roundtrip on every corpus line + GATE C: measured compression ----
246 let dec: *u8 = sys_mmap(4096)
247 var pr: i64 = 1
248 var lines: i64 = 0
249 var toks: i64 = 0
250 var bytesum: i64 = 0
251 var ls: i64 = 0
252 while ls < total {
253 var le: i64 = ls
254 var stop: i64 = 0
255 while stop == 0 {
256 if le >= total { stop = 1 } else { if corpus[le] == (10 as u8) { stop = 1 } else { le = le + 1 } }
257 }
258 let ll: i64 = le - ls
259 if ll > 0 {
260 let tc: i64 = bp_encode(corpus, ls, ll, m1, n1, seq2)
261 toks = toks + tc
262 bytesum = bytesum + ll
263 var w: i64 = 0
264 var t: i64 = 0
265 while t < tc { w = w + bp_decode_tok(seq2[t], m1, dec, w); t = t + 1 }
266 var same: i64 = 1
267 if w != ll { same = 0 }
268 var q: i64 = 0
269 while q < ll { if q < w { if dec[q] != corpus[ls+q] { same = 0 } } q = q + 1 }
270 if same == 0 {
271 pr = 0
272 bp_w(1, " ROUNDTRIP FAIL line_at=" as *u8); bp_wn(1, ls); bp_w(1, "\n" as *u8)
273 }
274 lines = lines + 1
275 }
276 ls = le + 1
277 }
278 var pc: i64 = 1
279 if toks >= bytesum { pc = 0 }
280 var permil: i64 = 0
281 if bytesum > 0 { permil = (toks * 1000) / bytesum }
282 bp_bp(lfd, "MODELWRIGHT T2 ROUNDTRIP lines=" as *u8); bp_bn(lfd, lines)
283 bp_bp(lfd, " lossless=" as *u8); bp_bn(lfd, pr)
284 bp_bp(lfd, " tokens=" as *u8); bp_bn(lfd, toks)
285 bp_bp(lfd, " bytes=" as *u8); bp_bn(lfd, bytesum)
286 bp_bp(lfd, " ratio_permil=" as *u8); bp_bn(lfd, permil); bp_bp(lfd, "\n" as *u8)
287 if pr == 1 { bp_w(1, " GATE R lossless roundtrip (every corpus line): PASS\n" as *u8) } else { bp_w(1, " GATE R roundtrip: FAIL\n" as *u8) }
288 if pc == 1 { bp_w(1, " GATE C compression measured (tokens < bytes): PASS\n" as *u8) } else { bp_w(1, " GATE C: FAIL\n" as *u8) }
289
290 // ---- persist the vocab (durable, reloadable; derived artifact regenerated per run) ----
291 let vfd: i64 = sys_openat_wr("knowledge/store/bpe_vocab_v1.txt" as *u8, 0x1a4)
292 if vfd >= 0 {
293 bp_w(vfd, "# bpe_vocab_v1 -- trained by _bpe_vocab_authored on the team corpus (bpe_corpus.conf)\n" as *u8)
294 i = 0
295 while i < n1 {
296 bp_w(vfd, "BPEMERGE idx=" as *u8); bp_wn(vfd, 256 + i)
297 bp_w(vfd, " left=" as *u8); bp_wn(vfd, m1[i*2+0])
298 bp_w(vfd, " right=" as *u8); bp_wn(vfd, m1[i*2+1]); bp_w(vfd, "\n" as *u8)
299 i = i + 1
300 }
301 sys_close(vfd)
302 bp_w(1, " vocab persisted: knowledge/store/bpe_vocab_v1.txt\n" as *u8)
303 }
304
305 var gates: i64 = 0
306 if pk == 1 { gates = gates + 1 }
307 if pd == 1 { gates = gates + 1 }
308 if pr == 1 { gates = gates + 1 }
309 if pc == 1 { gates = gates + 1 }
310 bp_bp(lfd, "MODELWRIGHT T2 VERDICT gates=" as *u8); bp_bn(lfd, gates)
311 bp_bp(lfd, "/4" as *u8)
312 if gates == 4 { bp_bp(lfd, " pass=1\n" as *u8) } else { bp_bp(lfd, " pass=0\n" as *u8) }
313 sys_close(lfd)
314 if gates == 4 {
315 bp_w(1, " BPE-VOCAB GATE: PASS (the team now TRAINS its tokenizer, not just loads one)\n" as *u8)
316 sys_exit(0)
317 return 0
318 }
319 bp_w(1, " BPE-VOCAB GATE: FAIL\n" as *u8)
320 sys_exit(1)
321 return 1
322}