nx_mt_ruen.nx source

1// nx_mt_ruen.nx -- sovereign BIDIRECTIONAL RU<->EN phrase-table translator (the Belarus<->Texas pair). Token- 2// level, Cyrillic-aware tokenizer (UTF-8 multibyte bytes >=0x80 are token chars), NO float, deterministic. 3// dir 0 = EN->RU (output Cyrillic), dir 1 = RU->EN. Unknown tokens pass through verbatim (Cardinal 25). Lexicon 4// is edit-here R1 (R2 -> seg_store per language-pair, Cardinal 11); OPEN-DOMAIN neural MT + ASR are the 5// RESEARCHER long-poles (V-NMT-1/V-ASR-1) -- this is the gated, real, bidirectional R1. license_tier: ORIGINAL 6import "nx_syscalls.nx" 7 8func mr_lc(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c } 9// token char = ASCII letter/digit OR any UTF-8 multibyte byte (Cyrillic) 10func mr_is_tok(c: i64) -> i64 { 11 if c >= 48 { if c <= 57 { return 1 } } 12 if c >= 65 { if c <= 90 { return 1 } } 13 if c >= 97 { if c <= 122 { return 1 } } 14 if c >= 128 { return 1 } 15 return 0 16} 17// ASCII case-insensitive token match (key = lowercase ASCII) 18func mr_eq_ci(src: *u8, s: i64, e: i64, key: *u8) -> i64 { 19 var i: i64 = s; var k: i64 = 0 20 while i < e { 21 if key[k] == (0 as u8) { return 0 } 22 if mr_lc(src[i] as i64) != (key[k] as i64) { return 0 } 23 i = i + 1; k = k + 1 24 } 25 if key[k] != (0 as u8) { return 0 } 26 return 1 27} 28// byte-exact token match (for Cyrillic keys) 29func mr_eq_exact(src: *u8, s: i64, e: i64, key: *u8) -> i64 { 30 var i: i64 = s; var k: i64 = 0 31 while i < e { 32 if key[k] == (0 as u8) { return 0 } 33 if (src[i] as i64) != (key[k] as i64) { return 0 } 34 i = i + 1; k = k + 1 35 } 36 if key[k] != (0 as u8) { return 0 } 37 return 1 38} 39// EN token -> RU (Cyrillic), or 0 40func mr_ru_for_en(src: *u8, s: i64, e: i64) -> *u8 { 41 if mr_eq_ci(src,s,e,"hello" as *u8)==1 { return "привет" as *u8 } 42 if mr_eq_ci(src,s,e,"hi" as *u8)==1 { return "привет" as *u8 } 43 if mr_eq_ci(src,s,e,"friend" as *u8)==1 { return "друг" as *u8 } 44 if mr_eq_ci(src,s,e,"thanks" as *u8)==1 { return "спасибо" as *u8 } 45 if mr_eq_ci(src,s,e,"thank" as *u8)==1 { return "спасибо" as *u8 } 46 if mr_eq_ci(src,s,e,"love" as *u8)==1 { return "любовь" as *u8 } 47 if mr_eq_ci(src,s,e,"yes" as *u8)==1 { return "да" as *u8 } 48 if mr_eq_ci(src,s,e,"no" as *u8)==1 { return "нет" as *u8 } 49 if mr_eq_ci(src,s,e,"you" as *u8)==1 { return "ты" as *u8 } 50 if mr_eq_ci(src,s,e,"good" as *u8)==1 { return "хорошо" as *u8 } 51 if mr_eq_ci(src,s,e,"bye" as *u8)==1 { return "пока" as *u8 } 52 if mr_eq_ci(src,s,e,"water" as *u8)==1 { return "вода" as *u8 } 53 if mr_eq_ci(src,s,e,"how" as *u8)==1 { return "как" as *u8 } 54 if mr_eq_ci(src,s,e,"texas" as *u8)==1 { return "техас" as *u8 } 55 if mr_eq_ci(src,s,e,"boy" as *u8)==1 { return "мальчик" as *u8 } 56 if mr_eq_ci(src,s,e,"girl" as *u8)==1 { return "девочка" as *u8 } 57 return 0 as *u8 58} 59// RU token (Cyrillic) -> EN, or 0 60func mr_en_for_ru(src: *u8, s: i64, e: i64) -> *u8 { 61 if mr_eq_exact(src,s,e,"привет" as *u8)==1 { return "hello" as *u8 } 62 if mr_eq_exact(src,s,e,"друг" as *u8)==1 { return "friend" as *u8 } 63 if mr_eq_exact(src,s,e,"спасибо" as *u8)==1 { return "thanks" as *u8 } 64 if mr_eq_exact(src,s,e,"любовь" as *u8)==1 { return "love" as *u8 } 65 if mr_eq_exact(src,s,e,"да" as *u8)==1 { return "yes" as *u8 } 66 if mr_eq_exact(src,s,e,"нет" as *u8)==1 { return "no" as *u8 } 67 if mr_eq_exact(src,s,e,"ты" as *u8)==1 { return "you" as *u8 } 68 if mr_eq_exact(src,s,e,"хорошо" as *u8)==1 { return "good" as *u8 } 69 if mr_eq_exact(src,s,e,"пока" as *u8)==1 { return "bye" as *u8 } 70 if mr_eq_exact(src,s,e,"вода" as *u8)==1 { return "water" as *u8 } 71 if mr_eq_exact(src,s,e,"как" as *u8)==1 { return "how" as *u8 } 72 if mr_eq_exact(src,s,e,"техас" as *u8)==1 { return "texas" as *u8 } 73 if mr_eq_exact(src,s,e,"мальчик" as *u8)==1 { return "boy" as *u8 } 74 if mr_eq_exact(src,s,e,"девочка" as *u8)==1 { return "girl" as *u8 } 75 return 0 as *u8 76} 77func mr_app(dst: *u8, pos: *i64, cap: i64, s: *u8) -> i64 { 78 var i: i64 = 0 79 while s[i] != (0 as u8) { let p: i64 = *pos; if p < cap { dst[p]=s[i]; *pos=p+1 } i=i+1 } 80 return 0 81} 82func mr_appb(dst: *u8, pos: *i64, cap: i64, b: i64) -> i64 { let p: i64 = *pos; if p < cap { dst[p]=b as u8; *pos=p+1 } return 0 } 83 84// dir: 0 = EN->RU, 1 = RU->EN. returns dst length. 85func nx_ruen_translate(src: *u8, n: i64, dir: i64, dst: *u8, cap: i64) -> i64 { 86 let pos: *i64 = sys_mmap(8) as *i64 87 *pos = 0 88 var i: i64 = 0 89 var first: i64 = 1 90 while i < n { 91 while i < n { if mr_is_tok(src[i] as i64) == 1 { break } i = i + 1 } 92 if i >= n { break } 93 let s: i64 = i 94 while i < n { if mr_is_tok(src[i] as i64) == 1 { i = i + 1 } else { break } } 95 let e: i64 = i 96 if first == 0 { mr_appb(dst, pos, cap, 32) } 97 first = 0 98 var tr: *u8 = 0 as *u8 99 if dir == 0 { tr = mr_ru_for_en(src, s, e) } else { tr = mr_en_for_ru(src, s, e) } 100 if (tr as i64) != 0 { mr_app(dst, pos, cap, tr) } 101 else { var k: i64 = s; while k < e { mr_appb(dst, pos, cap, src[k] as i64); k = k + 1 } } 102 } 103 return *pos 104}