nx_mt_r3_gate.nx source

1// nx_mt_r3_gate.nx -- GATE for MT-R3: REAL TEXT via the production BPE tokenizer (nx_bpe). The 2// MT arc moves off hand-coded integer toy indices: input/output are now actual TEXT STRINGS, 3// tokenized to subword IDs by the team's shipped Byte-Pair-Encoding organ and detokenized back. 4// The learned word-map (proven in R0/R1/R2, reused from nx_mt_core) maps EN token IDs -> ES 5// token IDs in between. This is the text-interface bookend the rest of the arc needs; it does 6// NOT yet do open-vocabulary sentence MT (that's R4: real corpus / trained weights). 7// 8// Sovereign self-contained vocab (no external files; nx_bpe builds vocab in-code): 9// EN (with merges, so encoding is real BPE): bytes g o s e a t ; merges (e,a)->ea, (s,ea)->sea, 10// (t,ea)->tea, (g,o)->go. Words: "sea" "tea" "go" each encode to ONE token; "ea" is a SHARED 11// subword of sea/tea. Out-of-vocab "seat" decomposes to known subwords [sea, t]. 12// ES (decode side): whole-word tokens "mar" "te" "ir". 13// translation map (learned via nx_mt_core): sea->mar, tea->te, go->ir. 14// 15// FOUR GATES: 16// A BPE ROUND-TRIP: decode(encode(x)) == x byte-exact for "sea","tea","go". 17// B SUBWORD DECOMPOSITION (the BPE value): encode("seat") == [sea, t] (OOV word -> known 18// subwords) and round-trips byte-exact -- a hand-coded index table cannot do this. 19// C END-TO-END TEXT TRANSLATE: translate("sea")=="mar", translate("tea")=="te", 20// translate("go")=="ir" (real string in, real string out; 3/3). 21// D UNTRAINED FAILS (liar-kill): the zero-epoch map mistranslates (< 3/3) -> the LEARNED map 22// does the translating, not the tokenizer. 23// 24// genealogy_id: sennrich_haddow_birch_2016_bpe (realized_in nx_bpe) + rumelhart_1986_backprop 25// lineage_id: sovereign_neural_mt_r3_bpe_realtext_v1 26// license_tier: ORIGINAL 27import "nx_bpe.nx" // nx_bpe_vocab_new / add_token / add_merge / encode / decode (+ nx_intern/loop/tier) 28import "nx_mt_core.nx" // mt_train / mt_argmax (the learned map) 29import "nx_autograd.nx" // ag_constf + nx_syscalls 30import "nx_syscalls.nx" 31import "nx_gate_verdict.nx" 32 33const M3_LOG: *u8 = "knowledge/status/mt_r3.log" 34 35const C_G: *u8 = "g" as *u8 36const C_O: *u8 = "o" as *u8 37const C_S: *u8 = "s" as *u8 38const C_E: *u8 = "e" as *u8 39const C_A: *u8 = "a" as *u8 40const C_T: *u8 = "t" as *u8 41const C_EA: *u8 = "ea" as *u8 42const C_SEA: *u8 = "sea" as *u8 43const C_TEA: *u8 = "tea" as *u8 44const C_GO: *u8 = "go" as *u8 45const C_SEAT: *u8 = "seat" as *u8 46const C_MAR: *u8 = "mar" as *u8 47const C_TE: *u8 = "te" as *u8 48const C_IR: *u8 = "ir" as *u8 49 50func m3_w(fd: i64, s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(fd, s, n); return 0 } 51func m3_wn(fd: i64, v: i64) -> i64 { 52 let bb: *u8 = sys_mmap(28); var m: i64 = v 53 if m < 0 { m = 0 - m; sys_write(fd, "-" as *u8, 1) } 54 let t: *u8 = sys_mmap(28); var k: i64 = 0 55 if m == 0 { t[0] = 48; k = 1 } 56 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 } 57 var i: i64 = 0 58 while i < k { bb[i] = t[k - 1 - i]; i = i + 1 } 59 sys_write(fd, bb, k); return 0 60} 61func m3_streq(a: *u8, la: i64, b: *u8, lb: i64) -> i64 { 62 if la != lb { return 0 } 63 var i: i64 = 0 64 while i < la { if a[i] != b[i] { return 0 } i = i + 1 } 65 return 1 66} 67 68// translate a text string EN->ES via BPE-encode -> learned map (per token) -> BPE-decode. 69// writes ES bytes into out, returns ES byte length (or -1 on encode error). 70func m3_translate(v_en: *NxBpeVocab, v_es: *NxBpeVocab, W: *i64, S: i64, T: i64, text: *u8, tlen: i64, out: *u8) -> i64 { 71 let en_ids: *i64 = (sys_mmap(64 * 8)) as *i64 72 let n: i64 = nx_bpe_encode(v_en, text, tlen, en_ids) 73 if n < 0 { return 0 - 1 } 74 let es_ids: *i64 = (sys_mmap(64 * 8)) as *i64 75 var k: i64 = 0 76 while k < n { es_ids[k] = mt_argmax(W, S, T, en_ids[k]); k = k + 1 } 77 return nx_bpe_decode(v_es, es_ids, n, out) 78} 79 80// does translate(text) == expect ? 81func m3_check(v_en: *NxBpeVocab, v_es: *NxBpeVocab, W: *i64, S: i64, T: i64, text: *u8, tlen: i64, expect: *u8, elen: i64) -> i64 { 82 let out: *u8 = sys_mmap(64) 83 let m: i64 = m3_translate(v_en, v_es, W, S, T, text, tlen, out) 84 if m < 0 { return 0 } 85 return m3_streq(out, m, expect, elen) 86} 87 88func m3_demo(fd: i64, v_en: *NxBpeVocab, v_es: *NxBpeVocab, W: *i64, S: i64, T: i64, text: *u8, tlen: i64) -> i64 { 89 let out: *u8 = sys_mmap(64) 90 let m: i64 = m3_translate(v_en, v_es, W, S, T, text, tlen, out) 91 m3_w(fd, " " as *u8); sys_write(fd, text, tlen); m3_w(fd, "->" as *u8) 92 if m > 0 { sys_write(fd, out, m) } 93 return 0 94} 95 96func m3_emit(fd: i64, r: *i64) -> i64 { 97 m3_w(fd, "MTR3GATE authored=organ engine=bpe-tokenizer+scalar-tape-map task=realtext-translate" as *u8) 98 m3_w(fd, " | A_roundtrip_pass=" as *u8); m3_wn(fd, r[0]) 99 m3_w(fd, " | B_subword_pass=" as *u8); m3_wn(fd, r[1]); m3_w(fd, " seat_ntokens=" as *u8); m3_wn(fd, r[2]) 100 m3_w(fd, " | C_translate_pass=" as *u8); m3_wn(fd, r[3]); m3_w(fd, " correct=" as *u8); m3_wn(fd, r[4]); m3_w(fd, "/3" as *u8) 101 m3_w(fd, " | D_untrained_fails_pass=" as *u8); m3_wn(fd, r[5]); m3_w(fd, " untrained_correct=" as *u8); m3_wn(fd, r[6]); m3_w(fd, "/3" as *u8) 102 if r[7] == 1 { m3_w(fd, " verdict=GREEN\n" as *u8) } else { m3_w(fd, " verdict=RED\n" as *u8) } 103 return 0 104} 105 106func main() -> i64 { 107 var ok: i64 = 1 108 109 // ---- EN vocab with real BPE merges ---- 110 let v_en: *NxBpeVocab = nx_bpe_vocab_new(512, 64, 16) 111 let id_g: i64 = nx_bpe_add_token(v_en, C_G, 1) 112 let id_o: i64 = nx_bpe_add_token(v_en, C_O, 1) 113 let id_s: i64 = nx_bpe_add_token(v_en, C_S, 1) 114 let id_e: i64 = nx_bpe_add_token(v_en, C_E, 1) 115 let id_a: i64 = nx_bpe_add_token(v_en, C_A, 1) 116 let id_t: i64 = nx_bpe_add_token(v_en, C_T, 1) 117 let id_ea: i64 = nx_bpe_add_token(v_en, C_EA, 2) 118 let id_sea: i64 = nx_bpe_add_token(v_en, C_SEA, 3) 119 let id_tea: i64 = nx_bpe_add_token(v_en, C_TEA, 3) 120 let id_go: i64 = nx_bpe_add_token(v_en, C_GO, 2) 121 nx_bpe_add_merge(v_en, id_e, id_a, id_ea) // (e,a)->ea 122 nx_bpe_add_merge(v_en, id_s, id_ea, id_sea) // (s,ea)->sea 123 nx_bpe_add_merge(v_en, id_t, id_ea, id_tea) // (t,ea)->tea 124 nx_bpe_add_merge(v_en, id_g, id_o, id_go) // (g,o)->go 125 126 // ---- ES vocab (decode side) ---- 127 let v_es: *NxBpeVocab = nx_bpe_vocab_new(256, 16, 4) 128 let id_mar: i64 = nx_bpe_add_token(v_es, C_MAR, 3) 129 let id_te: i64 = nx_bpe_add_token(v_es, C_TE, 2) 130 let id_ir: i64 = nx_bpe_add_token(v_es, C_IR, 2) 131 132 let S: i64 = id_go + 1 // max EN id + 1 133 let T: i64 = id_ir + 1 // max ES id + 1 134 135 // ---- learned map: sea->mar, go->ir, tea->te ---- 136 let srcs: *i64 = (sys_mmap(3 * 8)) as *i64 137 let tgts: *i64 = (sys_mmap(3 * 8)) as *i64 138 srcs[0] = id_sea; tgts[0] = id_mar 139 srcs[1] = id_go; tgts[1] = id_ir 140 srcs[2] = id_tea; tgts[2] = id_te 141 let W: *i64 = (sys_mmap(T * S * 8)) as *i64 142 let lf: *i64 = (sys_mmap(8)) as *i64 143 let ll: *i64 = (sys_mmap(8)) as *i64 144 mt_train(W, S, T, srcs, tgts, 3, 1, 300, ag_constf(1, 4), lf, ll) 145 146 // ---------- Gate A: BPE round-trip ---------- 147 let tok: *i64 = (sys_mmap(64 * 8)) as *i64 148 let buf: *u8 = sys_mmap(64) 149 var roundtrip_pass: i64 = 1 150 var ne: i64 = nx_bpe_encode(v_en, C_SEA, 3, tok) 151 var nd: i64 = nx_bpe_decode(v_en, tok, ne, buf) 152 if m3_streq(buf, nd, C_SEA, 3) != 1 { roundtrip_pass = 0 } 153 ne = nx_bpe_encode(v_en, C_TEA, 3, tok); nd = nx_bpe_decode(v_en, tok, ne, buf) 154 if m3_streq(buf, nd, C_TEA, 3) != 1 { roundtrip_pass = 0 } 155 ne = nx_bpe_encode(v_en, C_GO, 2, tok); nd = nx_bpe_decode(v_en, tok, ne, buf) 156 if m3_streq(buf, nd, C_GO, 2) != 1 { roundtrip_pass = 0 } 157 if roundtrip_pass != 1 { ok = 0 } 158 159 // ---------- Gate B: subword decomposition of OOV "seat" ---------- 160 let seat_n: i64 = nx_bpe_encode(v_en, C_SEAT, 4, tok) 161 var subword_pass: i64 = 1 162 if seat_n != 2 { subword_pass = 0 } 163 if subword_pass == 1 { if tok[0] != id_sea { subword_pass = 0 } } 164 if subword_pass == 1 { if tok[1] != id_t { subword_pass = 0 } } 165 nd = nx_bpe_decode(v_en, tok, seat_n, buf) 166 if m3_streq(buf, nd, C_SEAT, 4) != 1 { subword_pass = 0 } 167 if subword_pass != 1 { ok = 0 } 168 169 // ---------- Gate C: end-to-end text translation ---------- 170 var correct: i64 = 0 171 correct = correct + m3_check(v_en, v_es, W, S, T, C_SEA, 3, C_MAR, 3) 172 correct = correct + m3_check(v_en, v_es, W, S, T, C_TEA, 3, C_TE, 2) 173 correct = correct + m3_check(v_en, v_es, W, S, T, C_GO, 2, C_IR, 2) 174 var translate_pass: i64 = 1 175 if correct != 3 { translate_pass = 0 } 176 if translate_pass != 1 { ok = 0 } 177 178 // ---------- Gate D: untrained map fails (liar-kill) ---------- 179 let W0: *i64 = (sys_mmap(T * S * 8)) as *i64 180 let lf0: *i64 = (sys_mmap(8)) as *i64 181 let ll0: *i64 = (sys_mmap(8)) as *i64 182 mt_train(W0, S, T, srcs, tgts, 3, 1, 0, ag_constf(1, 4), lf0, ll0) // 0 epochs 183 var ucorrect: i64 = 0 184 ucorrect = ucorrect + m3_check(v_en, v_es, W0, S, T, C_SEA, 3, C_MAR, 3) 185 ucorrect = ucorrect + m3_check(v_en, v_es, W0, S, T, C_TEA, 3, C_TE, 2) 186 ucorrect = ucorrect + m3_check(v_en, v_es, W0, S, T, C_GO, 2, C_IR, 2) 187 var untrained_fails_pass: i64 = 1 188 if ucorrect >= 3 { untrained_fails_pass = 0 } 189 if untrained_fails_pass != 1 { ok = 0 } 190 191 // ---------- emit ---------- 192 let r: *i64 = (sys_mmap(8 * 8)) as *i64 193 r[0] = roundtrip_pass; r[1] = subword_pass; r[2] = seat_n 194 r[3] = translate_pass; r[4] = correct 195 r[5] = untrained_fails_pass; r[6] = ucorrect; r[7] = ok 196 m3_emit(1, r) 197 m3_w(1, "MT-R3 real-text:" as *u8) 198 m3_demo(1, v_en, v_es, W, S, T, C_SEA, 3) 199 m3_demo(1, v_en, v_es, W, S, T, C_TEA, 3) 200 m3_demo(1, v_en, v_es, W, S, T, C_GO, 2) 201 m3_w(1, "\n" as *u8) 202 let logf: i64 = sys_openat_append(M3_LOG, 420) 203 if logf >= 0 { m3_emit(logf, r); sys_close(logf) } 204 205 // MIGRATED onto nx_gate_verdict by nx_gate_dry_apply (D001, minimal form): every check 206 // row above is untouched, so the PASS/FAIL vector cannot change; only the hand-rolled 207 // verdict emission is replaced by the ONE shared base class. Proven by nx_gate_migrate verify. 208 let ctr__dry: *i64 = gv_ctr() 209 ctr__dry[0] = ok 210 ctr__dry[1] = 1 211 let rc__dry: i64 = gv_verdict("MT-R3-GATE" as *u8, ctr__dry, "teeth unchanged; verdict emission migrated onto the shared base class" as *u8) 212 sys_exit(rc__dry) 213 return rc__dry 214}