nx_mt_phrasetable.nx source

1// nx_mt_phrasetable.nx -- sovereign data-driven phrase-table translator (token-level, EN->ES seed lexicon), 2// the deterministic NO-FLOAT R1 that makes caption translation REAL end-to-end for the live-translation video 3// feature. Unknown tokens PASS THROUGH verbatim (never drop content -- Cardinal 25). The lexicon is edit-here 4// today (R2 -> a seg_store lexicon per language-pair, Cardinal 11); OPEN-DOMAIN neural MT is the RESEARCHER 5// long-pole (CAP-VIDEO-XLATE-NMT), a research arc like dense-retrieval -- NOT faked here. license_tier: ORIGINAL 6import "nx_syscalls.nx" 7 8// lowercase one ASCII byte 9func mt_lc(c: i64) -> i64 { if c >= 65 { if c <= 90 { return c + 32 } } return c } 10 11// case-insensitive equality of token src[s..e) vs a nul-terminated lexicon key. 1 if equal. 12func mt_tok_eq(src: *u8, s: i64, e: i64, key: *u8) -> i64 { 13 var i: i64 = s 14 var k: i64 = 0 15 while i < e { 16 if key[k] == (0 as u8) { return 0 } 17 if mt_lc(src[i] as i64) != (key[k] as i64) { return 0 } 18 i = i + 1 19 k = k + 1 20 } 21 if key[k] != (0 as u8) { return 0 } 22 return 1 23} 24 25// the SEED lexicon (EN token -> ES token). ASCII-only values (the bitmap fonts are ASCII) so accents are 26// folded (manana). Edit-here to grow; R2 = data-driven seg_store. Returns 0 (null) if there is no entry. 27func mt_es_for(src: *u8, s: i64, e: i64) -> *u8 { 28 if mt_tok_eq(src,s,e,"hello" as *u8)==1 { return "hola" as *u8 } 29 if mt_tok_eq(src,s,e,"hi" as *u8)==1 { return "hola" as *u8 } 30 if mt_tok_eq(src,s,e,"world" as *u8)==1 { return "mundo" as *u8 } 31 if mt_tok_eq(src,s,e,"friend" as *u8)==1 { return "amigo" as *u8 } 32 if mt_tok_eq(src,s,e,"good" as *u8)==1 { return "bueno" as *u8 } 33 if mt_tok_eq(src,s,e,"morning" as *u8)==1 { return "manana" as *u8 } 34 if mt_tok_eq(src,s,e,"welcome" as *u8)==1 { return "bienvenido" as *u8 } 35 if mt_tok_eq(src,s,e,"thanks" as *u8)==1 { return "gracias" as *u8 } 36 if mt_tok_eq(src,s,e,"game" as *u8)==1 { return "juego" as *u8 } 37 if mt_tok_eq(src,s,e,"play" as *u8)==1 { return "jugar" as *u8 } 38 if mt_tok_eq(src,s,e,"water" as *u8)==1 { return "agua" as *u8 } 39 if mt_tok_eq(src,s,e,"you" as *u8)==1 { return "tu" as *u8 } 40 return 0 as *u8 41} 42 43// append nul-terminated s to dst at *pos (bounded by cap) 44func mt_app(dst: *u8, pos: *i64, cap: i64, s: *u8) -> i64 { 45 var i: i64 = 0 46 while s[i] != (0 as u8) { 47 let p: i64 = *pos 48 if p < cap { dst[p] = s[i]; *pos = p + 1 } 49 i = i + 1 50 } 51 return 0 52} 53// append one raw byte 54func mt_appb(dst: *u8, pos: *i64, cap: i64, b: i64) -> i64 { 55 let p: i64 = *pos 56 if p < cap { dst[p] = b as u8; *pos = p + 1 } 57 return 0 58} 59 60// translate src (n bytes, space-separated ASCII words) EN->ES into dst (cap bytes). returns dst length. 61// runs of spaces collapse to one; unknown words copied verbatim (content never dropped). 62func nx_mt_translate(src: *u8, n: i64, dst: *u8, cap: i64) -> i64 { 63 let pos: *i64 = sys_mmap(8) as *i64 64 *pos = 0 65 var i: i64 = 0 66 var first: i64 = 1 67 while i < n { 68 while i < n { if src[i] != (32 as u8) { break } i = i + 1 } // skip spaces 69 if i >= n { break } 70 let s: i64 = i 71 while i < n { if src[i] == (32 as u8) { break } i = i + 1 } // to end of word 72 let e: i64 = i 73 if first == 0 { mt_appb(dst, pos, cap, 32) } 74 first = 0 75 let tr: *u8 = mt_es_for(src, s, e) 76 if (tr as i64) != 0 { mt_app(dst, pos, cap, tr) } 77 else { 78 var k: i64 = s 79 while k < e { mt_appb(dst, pos, cap, src[k] as i64); k = k + 1 } 80 } 81 } 82 return *pos 83}