nx_mt_r3_gate.nx source

1// nx_mt_r3_gate.nx -- GATE for MT-R3: REAL TEXT via the production BPE tokenizer (nx_bpe). The 2// MT arc moves off hand-coded integer toy indices: input/output are now actual TEXT STRINGS, 3// tokenized to subword IDs by the team's shipped Byte-Pair-Encoding organ and detokenized back. 4// The learned word-map (proven in R0/R1/R2, reused from nx_mt_core) maps EN token IDs -> ES 5// token IDs in between. This is the text-interface bookend the rest of the arc needs; it does 6// NOT yet do open-vocabulary sentence MT (that's R4: real corpus / trained weights). 7// 8// Sovereign self-contained vocab (no external files; nx_bpe builds vocab in-code): 9// EN (with merges, so encoding is real BPE): bytes g o s e a t ; merges (e,a)->ea, (s,ea)->sea, 10// (t,ea)->tea, (g,o)->go. Words: "sea" "tea" "go" each encode to ONE token; "ea" is a SHARED 11// subword of sea/tea. Out-of-vocab "seat" decomposes to known subwords [sea, t]. 12// ES (decode side): whole-word tokens "mar" "te" "ir". 13// translation map (learned via nx_mt_core): sea->mar, tea->te, go->ir. 14// 15// FOUR GATES: 16// A BPE ROUND-TRIP: decode(encode(x)) == x byte-exact for "sea","tea","go". 17// B SUBWORD DECOMPOSITION (the BPE value): encode("seat") == [sea, t] (OOV word -> known 18// subwords) and round-trips byte-exact -- a hand-coded index table cannot do this. 19// C END-TO-END TEXT TRANSLATE: translate("sea")=="mar", translate("tea")=="te", 20// translate("go")=="ir" (real string in, real string out; 3/3). 21// D UNTRAINED FAILS (liar-kill): the zero-epoch map mistranslates (< 3/3) -> the LEARNED map 22// does the translating, not the tokenizer. 23// 24// genealogy_id: sennrich_haddow_birch_2016_bpe (realized_in nx_bpe) + rumelhart_1986_backprop 25// lineage_id: sovereign_neural_mt_r3_bpe_realtext_v1 26// license_tier: ORIGINAL 27import "nx_bpe.nx" // nx_bpe_vocab_new / add_token / add_merge / encode / decode (+ nx_intern/loop/tier) 28import "nx_mt_core.nx" // mt_train / mt_argmax (the learned map) 29import "nx_autograd.nx" // ag_constf + nx_syscalls 30import "nx_syscalls.nx" 31 32const M3_LOG: *u8 = "knowledge/status/mt_r3.log" 33 34const C_G: *u8 = "g" as *u8 35const C_O: *u8 = "o" as *u8 36const C_S: *u8 = "s" as *u8 37const C_E: *u8 = "e" as *u8 38const C_A: *u8 = "a" as *u8 39const C_T: *u8 = "t" as *u8 40const C_EA: *u8 = "ea" as *u8 41const C_SEA: *u8 = "sea" as *u8 42const C_TEA: *u8 = "tea" as *u8 43const C_GO: *u8 = "go" as *u8 44const C_SEAT: *u8 = "seat" as *u8 45const C_MAR: *u8 = "mar" as *u8 46const C_TE: *u8 = "te" as *u8 47const C_IR: *u8 = "ir" as *u8 48 49func m3_w(fd: i64, s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(fd, s, n); return 0 } 50func m3_wn(fd: i64, v: i64) -> i64 { 51 let bb: *u8 = sys_mmap(28); var m: i64 = v 52 if m < 0 { m = 0 - m; sys_write(fd, "-" as *u8, 1) } 53 let t: *u8 = sys_mmap(28); var k: i64 = 0 54 if m == 0 { t[0] = 48; k = 1 } 55 while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 } 56 var i: i64 = 0 57 while i < k { bb[i] = t[k - 1 - i]; i = i + 1 } 58 sys_write(fd, bb, k); return 0 59} 60func m3_streq(a: *u8, la: i64, b: *u8, lb: i64) -> i64 { 61 if la != lb { return 0 } 62 var i: i64 = 0 63 while i < la { if a[i] != b[i] { return 0 } i = i + 1 } 64 return 1 65} 66 67// translate a text string EN->ES via BPE-encode -> learned map (per token) -> BPE-decode. 68// writes ES bytes into out, returns ES byte length (or -1 on encode error). 69func m3_translate(v_en: *NxBpeVocab, v_es: *NxBpeVocab, W: *i64, S: i64, T: i64, text: *u8, tlen: i64, out: *u8) -> i64 { 70 let en_ids: *i64 = (sys_mmap(64 * 8)) as *i64 71 let n: i64 = nx_bpe_encode(v_en, text, tlen, en_ids) 72 if n < 0 { return 0 - 1 } 73 let es_ids: *i64 = (sys_mmap(64 * 8)) as *i64 74 var k: i64 = 0 75 while k < n { es_ids[k] = mt_argmax(W, S, T, en_ids[k]); k = k + 1 } 76 return nx_bpe_decode(v_es, es_ids, n, out) 77} 78 79// does translate(text) == expect ? 80func m3_check(v_en: *NxBpeVocab, v_es: *NxBpeVocab, W: *i64, S: i64, T: i64, text: *u8, tlen: i64, expect: *u8, elen: i64) -> i64 { 81 let out: *u8 = sys_mmap(64) 82 let m: i64 = m3_translate(v_en, v_es, W, S, T, text, tlen, out) 83 if m < 0 { return 0 } 84 return m3_streq(out, m, expect, elen) 85} 86 87func m3_demo(fd: i64, v_en: *NxBpeVocab, v_es: *NxBpeVocab, W: *i64, S: i64, T: i64, text: *u8, tlen: i64) -> i64 { 88 let out: *u8 = sys_mmap(64) 89 let m: i64 = m3_translate(v_en, v_es, W, S, T, text, tlen, out) 90 m3_w(fd, " " as *u8); sys_write(fd, text, tlen); m3_w(fd, "->" as *u8) 91 if m > 0 { sys_write(fd, out, m) } 92 return 0 93} 94 95func m3_emit(fd: i64, r: *i64) -> i64 { 96 m3_w(fd, "MTR3GATE authored=organ engine=bpe-tokenizer+scalar-tape-map task=realtext-translate" as *u8) 97 m3_w(fd, " | A_roundtrip_pass=" as *u8); m3_wn(fd, r[0]) 98 m3_w(fd, " | B_subword_pass=" as *u8); m3_wn(fd, r[1]); m3_w(fd, " seat_ntokens=" as *u8); m3_wn(fd, r[2]) 99 m3_w(fd, " | C_translate_pass=" as *u8); m3_wn(fd, r[3]); m3_w(fd, " correct=" as *u8); m3_wn(fd, r[4]); m3_w(fd, "/3" as *u8) 100 m3_w(fd, " | D_untrained_fails_pass=" as *u8); m3_wn(fd, r[5]); m3_w(fd, " untrained_correct=" as *u8); m3_wn(fd, r[6]); m3_w(fd, "/3" as *u8) 101 if r[7] == 1 { m3_w(fd, " verdict=GREEN\n" as *u8) } else { m3_w(fd, " verdict=RED\n" as *u8) } 102 return 0 103} 104 105func main() -> i64 { 106 var ok: i64 = 1 107 108 // ---- EN vocab with real BPE merges ---- 109 let v_en: *NxBpeVocab = nx_bpe_vocab_new(512, 64, 16) 110 let id_g: i64 = nx_bpe_add_token(v_en, C_G, 1) 111 let id_o: i64 = nx_bpe_add_token(v_en, C_O, 1) 112 let id_s: i64 = nx_bpe_add_token(v_en, C_S, 1) 113 let id_e: i64 = nx_bpe_add_token(v_en, C_E, 1) 114 let id_a: i64 = nx_bpe_add_token(v_en, C_A, 1) 115 let id_t: i64 = nx_bpe_add_token(v_en, C_T, 1) 116 let id_ea: i64 = nx_bpe_add_token(v_en, C_EA, 2) 117 let id_sea: i64 = nx_bpe_add_token(v_en, C_SEA, 3) 118 let id_tea: i64 = nx_bpe_add_token(v_en, C_TEA, 3) 119 let id_go: i64 = nx_bpe_add_token(v_en, C_GO, 2) 120 nx_bpe_add_merge(v_en, id_e, id_a, id_ea) // (e,a)->ea 121 nx_bpe_add_merge(v_en, id_s, id_ea, id_sea) // (s,ea)->sea 122 nx_bpe_add_merge(v_en, id_t, id_ea, id_tea) // (t,ea)->tea 123 nx_bpe_add_merge(v_en, id_g, id_o, id_go) // (g,o)->go 124 125 // ---- ES vocab (decode side) ---- 126 let v_es: *NxBpeVocab = nx_bpe_vocab_new(256, 16, 4) 127 let id_mar: i64 = nx_bpe_add_token(v_es, C_MAR, 3) 128 let id_te: i64 = nx_bpe_add_token(v_es, C_TE, 2) 129 let id_ir: i64 = nx_bpe_add_token(v_es, C_IR, 2) 130 131 let S: i64 = id_go + 1 // max EN id + 1 132 let T: i64 = id_ir + 1 // max ES id + 1 133 134 // ---- learned map: sea->mar, go->ir, tea->te ---- 135 let srcs: *i64 = (sys_mmap(3 * 8)) as *i64 136 let tgts: *i64 = (sys_mmap(3 * 8)) as *i64 137 srcs[0] = id_sea; tgts[0] = id_mar 138 srcs[1] = id_go; tgts[1] = id_ir 139 srcs[2] = id_tea; tgts[2] = id_te 140 let W: *i64 = (sys_mmap(T * S * 8)) as *i64 141 let lf: *i64 = (sys_mmap(8)) as *i64 142 let ll: *i64 = (sys_mmap(8)) as *i64 143 mt_train(W, S, T, srcs, tgts, 3, 1, 300, ag_constf(1, 4), lf, ll) 144 145 // ---------- Gate A: BPE round-trip ---------- 146 let tok: *i64 = (sys_mmap(64 * 8)) as *i64 147 let buf: *u8 = sys_mmap(64) 148 var roundtrip_pass: i64 = 1 149 var ne: i64 = nx_bpe_encode(v_en, C_SEA, 3, tok) 150 var nd: i64 = nx_bpe_decode(v_en, tok, ne, buf) 151 if m3_streq(buf, nd, C_SEA, 3) != 1 { roundtrip_pass = 0 } 152 ne = nx_bpe_encode(v_en, C_TEA, 3, tok); nd = nx_bpe_decode(v_en, tok, ne, buf) 153 if m3_streq(buf, nd, C_TEA, 3) != 1 { roundtrip_pass = 0 } 154 ne = nx_bpe_encode(v_en, C_GO, 2, tok); nd = nx_bpe_decode(v_en, tok, ne, buf) 155 if m3_streq(buf, nd, C_GO, 2) != 1 { roundtrip_pass = 0 } 156 if roundtrip_pass != 1 { ok = 0 } 157 158 // ---------- Gate B: subword decomposition of OOV "seat" ---------- 159 let seat_n: i64 = nx_bpe_encode(v_en, C_SEAT, 4, tok) 160 var subword_pass: i64 = 1 161 if seat_n != 2 { subword_pass = 0 } 162 if subword_pass == 1 { if tok[0] != id_sea { subword_pass = 0 } } 163 if subword_pass == 1 { if tok[1] != id_t { subword_pass = 0 } } 164 nd = nx_bpe_decode(v_en, tok, seat_n, buf) 165 if m3_streq(buf, nd, C_SEAT, 4) != 1 { subword_pass = 0 } 166 if subword_pass != 1 { ok = 0 } 167 168 // ---------- Gate C: end-to-end text translation ---------- 169 var correct: i64 = 0 170 correct = correct + m3_check(v_en, v_es, W, S, T, C_SEA, 3, C_MAR, 3) 171 correct = correct + m3_check(v_en, v_es, W, S, T, C_TEA, 3, C_TE, 2) 172 correct = correct + m3_check(v_en, v_es, W, S, T, C_GO, 2, C_IR, 2) 173 var translate_pass: i64 = 1 174 if correct != 3 { translate_pass = 0 } 175 if translate_pass != 1 { ok = 0 } 176 177 // ---------- Gate D: untrained map fails (liar-kill) ---------- 178 let W0: *i64 = (sys_mmap(T * S * 8)) as *i64 179 let lf0: *i64 = (sys_mmap(8)) as *i64 180 let ll0: *i64 = (sys_mmap(8)) as *i64 181 mt_train(W0, S, T, srcs, tgts, 3, 1, 0, ag_constf(1, 4), lf0, ll0) // 0 epochs 182 var ucorrect: i64 = 0 183 ucorrect = ucorrect + m3_check(v_en, v_es, W0, S, T, C_SEA, 3, C_MAR, 3) 184 ucorrect = ucorrect + m3_check(v_en, v_es, W0, S, T, C_TEA, 3, C_TE, 2) 185 ucorrect = ucorrect + m3_check(v_en, v_es, W0, S, T, C_GO, 2, C_IR, 2) 186 var untrained_fails_pass: i64 = 1 187 if ucorrect >= 3 { untrained_fails_pass = 0 } 188 if untrained_fails_pass != 1 { ok = 0 } 189 190 // ---------- emit ---------- 191 let r: *i64 = (sys_mmap(8 * 8)) as *i64 192 r[0] = roundtrip_pass; r[1] = subword_pass; r[2] = seat_n 193 r[3] = translate_pass; r[4] = correct 194 r[5] = untrained_fails_pass; r[6] = ucorrect; r[7] = ok 195 m3_emit(1, r) 196 m3_w(1, "MT-R3 real-text:" as *u8) 197 m3_demo(1, v_en, v_es, W, S, T, C_SEA, 3) 198 m3_demo(1, v_en, v_es, W, S, T, C_TEA, 3) 199 m3_demo(1, v_en, v_es, W, S, T, C_GO, 2) 200 m3_w(1, "\n" as *u8) 201 let logf: i64 = sys_openat_append(M3_LOG, 420) 202 if logf >= 0 { m3_emit(logf, r); sys_close(logf) } 203 204 if ok == 1 { return 0 } 205 return 1 206}