code wiki / _hdl_build / nx_bpe_vocab_register.nx
nx_bpe_vocab_register.nx source
↩ module page · 38 lines · 3171 B
1// nx_bpe_vocab_register.nx -- registers T2 rung 1 (BPE vocab TRAINING) via the Librarian.
2// ENGINEER evidence = the 4-gate BPE module re-run RIGHT HERE. license_tier: ORIGINAL
3import "nx_cap_register.nx"
4import "nx_capreg_librarian.nx"
5import "nx_syscalls.nx"
6
7func bpr_run(path: *u8) -> i64 {
8 let pid: i64 = sys_fork()
9 if pid == 0 {
10 let dn: i64 = sys_openat_wr("/dev/null" as *u8, 0x1a4)
11 if dn >= 0 { sys_dup3(dn, 1, 0); sys_dup3(dn, 2, 0) }
12 let argv: *i64 = sys_mmap(32) as *i64; argv[0] = path as i64; argv[1] = 0
13 let envp: *i64 = sys_mmap(16) as *i64; envp[0] = 0
14 sys_execve(path, argv, envp); sys_exit(127)
15 }
16 let st: *i64 = sys_mmap(16) as *i64
17 sys_wait4(pid, st, 0)
18 return st[0]
19}
20
21func main() -> i64 {
22 var allok: i64 = 1
23 if bpr_run("/tmp/_bpe_vocab_authored.sov.elf" as *u8) != 0 { allok = 0 }
24 cr_w(1, "ENGINEER: BPE vocab gate (oracle KAT / determinism / lossless roundtrip / measured compression) pass=" as *u8); cr_wn(1, allok); cr_w(1, "\n" as *u8)
25 let eng: i64 = ig_engineer(1, 1, 1, allok)
26 let dec: i64 = ig_decision(eng, ig_council(eng, 1, 1, 2), 1)
27 if dec != IG_INGEST { cr_w(1, "HELD\n" as *u8); sys_exit(1) }
28 if cr_can_register(5, 2, 6, dec) != 1 { cr_w(1, "REFUSED\n" as *u8); sys_exit(1) }
29 let lp: *u8 = "/tmp/nishi_cap_registry.log" as *u8
30 let jp: *u8 = "knowledge/status/cap_registry_durable.log" as *u8
31 let idx: i64 = cl_next_idx(lp, jp)
32 let ok: i64 = cl_register_dual(lp, jp, idx, 5,
33 2, "GEN T2-RUNG1 BPE-VOCAB-TRAINING (training-substrate ladder T2; MODELWRIGHT). The read side existed (nx_f32_bpe_load); _bpe_vocab_authored lands the TRAINING side at the smallest honest scale: greedy byte-pair merges over the TEAM CORPUS (conf-driven bpe_corpus.conf: synth_corpus.log + elder_census.log = 21270 bytes), deterministic tie-break (lowest-left-then-lowest-right via strict-greater ascending-index argmax), 64 merges. GATES 4/4: K = HAND-COMPUTED merge oracle on the classic aaabdaaabac corpus (merge sequence MUST be (97,97),(97,98),(256,257) -- the aa/ab-tiebreak/Zab derivation is written out in the module header, construction-known); D = deterministic two-run identical merge tables; R = LOSSLESS roundtrip encode->decode == original bytes for EVERY corpus line (480/480); C = compression MEASURED never asserted (9378 tokens / 20790 bytes = 451 permil, ~2.2x). Vocab persisted durable+reloadable knowledge/store/bpe_vocab_v1.txt BPEMERGE rows. HONEST SCOPE: no regex pretokenization, byte-level only, 64-merge vocab -- the research-flagged corroboration of vocab-size/pretokenization choices for the REAL LLM tokenizer remains open (Researcher); this rung proves the team TRAINS tokenizers (not just loads them) with exact oracles. NEXT: T2 tiny decoder-only model consumes this vocab; compile+KAT verdicts = RLVR reward. Evidence: MODELWRIGHT T2 rows in modelwright.log" as *u8)
34 if ok != 1 { cr_w(1, "DUAL-WRITE FAILED\n" as *u8); sys_exit(1) }
35 cr_w(1, "CAPREG idx=" as *u8); cr_wn(1, idx); cr_w(1, " REGISTERED via Librarian alloc+dual-write (T2 rung 1 BPE vocab training)\n" as *u8)
36 sys_exit(0)
37 return 0
38}