`nx_xlate_mt.nx` source

↩ module page · 383 lines · 15112 B
1// nx_xlate_mt.nx -- sovereign DATA-DRIVEN RU<->EN machine-translation engine (the Belarus<->Texas pair).
2// Replaces the 16-word hardcoded nx_mt_ruen stub. The lexicon + phrase table + morphology are DATA loaded
3// from TSV at build time (NOT hardcoded if-else): greedy LONGEST-match over words AND multi-word phrases,
4// Cyrillic-correct case folding (fixes "Привет"=="привет"), ё->е normalization, and Russian inflectional
5// suffix-stripping so inflected surface forms map to their lemma. NO float, deterministic.
6// dir 0 = EN->RU (emit Cyrillic), dir 1 = RU->EN. Unknown tokens pass through verbatim.
7// This is the R2 data-driven lexicon rung; open-domain neural MT stays the long pole. license_tier: ORIGINAL
8import "nx_syscalls.nx"
9const XL_MAGIC_8192: i64 = 8192
10const XL_MAGIC_16384: i64 = 16384
11const XL_MAGIC_2048: i64 = 2048
12
13const XL_MAXENT: i64 = 8192
14const XL_MAXSUF: i64 = 128
15const XL_NORMCAP: i64 = 524288    // normalized-text arena bytes (holds normalized copies of every lexicon side)
16
17// engine handle = *i64 with >=32 slots (caller: sys_mmap(256))
18const XLE_NENT: i64 = 0
19const XLE_TYPE: i64 = 1
20const XLE_RUOFF: i64 = 2
21const XLE_RULEN: i64 = 3
22const XLE_ENOFF: i64 = 4
23const XLE_ENLEN: i64 = 5
24const XLE_NRUOFF: i64 = 6
25const XLE_NRULEN: i64 = 7
26const XLE_NENOFF: i64 = 8
27const XLE_NENLEN: i64 = 9
28const XLE_NORM: i64 = 10
29const XLE_LEX: i64 = 11
30const XLE_NSUF: i64 = 12
31const XLE_SUFOFF: i64 = 13
32const XLE_SUFLEN: i64 = 14
33const XLE_SUF: i64 = 15
34
35func xl_putb(out: *u8, op: *i64, cap: i64, b: i64) -> i64 {
  let p: i64 = *op
  if p < cap { out[p] = b as u8; *op = p + 1 }
  return 0
39}
40// token char = ASCII letter/digit OR any UTF-8 continuation/lead byte (Cyrillic >= 0x80)
41func xl_is_tok(c: i64) -> i64 {
  if c >= 48 { if c <= 57 { return 1 } }
  if c >= 65 { if c <= 90 { return 1 } }
  if c >= 97 { if c <= 122 { return 1 } }
  if c >= 128 { return 1 }
  return 0
47}
48// normalize+append ONE token [s,e) of src to out (ASCII lowercase; Cyrillic uppercase->lowercase; Ё/ё -> е)
49func xl_norm_tok(src: *u8, s: i64, e: i64, out: *u8, op: *i64, cap: i64) -> i64 {
  var i: i64 = s
  while i < e {
      let c: i64 = src[i] as i64
      if c < 128 {
          var lc: i64 = c
          if c >= 65 { if c <= 90 { lc = c + 32 } }
          xl_putb(out, op, cap, lc)
          i = i + 1
      } else {
          if c == 208 {
              if i + 1 < e {
                  let d: i64 = src[i+1] as i64
                  var o1: i64 = 208
                  var o2: i64 = d
                  if d == 129 { o2 = 181 }                                 // Ё (D0 81) -> е
                  if d >= 144 { if d <= 159 { o2 = d + 32 } }              // А-П (D0 90..9F) -> а-п
                  if d >= 160 { if d <= 175 { o1 = 209; o2 = d - 32 } }    // Р-Я (D0 A0..AF) -> р-я (D1 80..8F)
                  xl_putb(out, op, cap, o1); xl_putb(out, op, cap, o2)
                  i = i + 2
              } else { xl_putb(out, op, cap, c); i = i + 1 }
          } else {
              if c == 209 {
                  if i + 1 < e {
                      let d: i64 = src[i+1] as i64
                      var o1: i64 = 209
                      var o2: i64 = d
                      if d == 145 { o1 = 208; o2 = 181 }                   // ё (D1 91) -> е
                      xl_putb(out, op, cap, o1); xl_putb(out, op, cap, o2)
                      i = i + 2
                  } else { xl_putb(out, op, cap, c); i = i + 1 }
              } else {
                  xl_putb(out, op, cap, c); i = i + 1
              }
          }
      }
  }
  return 0
87}
88// normalize a whole field [s,e) (may hold multiple tokens = a phrase) into out, space-joined
89func xl_norm_phrase(src: *u8, s: i64, e: i64, out: *u8, op: *i64, cap: i64) -> i64 {
  var i: i64 = s
  var first: i64 = 1
  while i < e {
      while i < e { if xl_is_tok(src[i] as i64) == 1 { break } i = i + 1 }
      if i >= e { break }
      let ts: i64 = i
      while i < e { if xl_is_tok(src[i] as i64) == 1 { i = i + 1 } else { break } }
      let te: i64 = i
      if first == 0 { xl_putb(out, op, cap, 32) }
      first = 0
      xl_norm_tok(src, ts, te, out, op, cap)
  }
  return 0
103}
104// normalize full input into nbuf (space-joined), record each token's start offset into tstart[]; -> normalized length
105func xl_norm_str(src: *u8, n: i64, nbuf: *u8, ncap: i64, tstart: *i64, maxtok: i64, ntok_out: *i64) -> i64 {
  let op: *i64 = sys_mmap(8) as *i64
  *op = 0
  var ntok: i64 = 0
  var i: i64 = 0
  var first: i64 = 1
  while i < n {
      while i < n { if xl_is_tok(src[i] as i64) == 1 { break } i = i + 1 }
      if i >= n { break }
      let s: i64 = i
      while i < n { if xl_is_tok(src[i] as i64) == 1 { i = i + 1 } else { break } }
      let e: i64 = i
      if first == 0 { xl_putb(nbuf, op, ncap, 32) }
      first = 0
      if ntok < maxtok { tstart[ntok] = *op }
      ntok = ntok + 1
      xl_norm_tok(src, s, e, nbuf, op, ncap)
  }
  *ntok_out = ntok
  let r: i64 = *op
  sys_munmap(op, 8)
  return r
127}
128// read one TAB/newline field from buf at *ip; set [*fs,*fe); *term = 9|10|-1; advance *ip; return 1 if read
129func xl_field(buf: *u8, n: i64, ip: *i64, fs: *i64, fe: *i64, term: *i64) -> i64 {
  var i: i64 = *ip
  if i >= n { return 0 }
  *fs = i
  while i < n {
      let c: i64 = buf[i] as i64
      if c == 9 { break }
      if c == 10 { break }
      i = i + 1
  }
  *fe = i
  if i < n { *term = buf[i] as i64; i = i + 1 } else { *term = 0 - 1 }
  *ip = i
  return 1
143}
144// nbuf[pos..pos+alen) == arena[aoff..aoff+alen) ?
145func xl_meq(nbuf: *u8, pos: i64, nlen: i64, arena: *u8, aoff: i64, alen: i64) -> i64 {
  if alen <= 0 { return 0 }
  if pos + alen > nlen { return 0 }
  var k: i64 = 0
  while k < alen { if (nbuf[pos+k] as i64) != (arena[aoff+k] as i64) { return 0 } k = k + 1 }
  return 1
151}
152func xl_ntok_of(arena: *u8, aoff: i64, alen: i64) -> i64 {
  if alen <= 0 { return 0 }
  var c: i64 = 1
  var k: i64 = 0
  while k < alen { if (arena[aoff+k] as i64) == 32 { c = c + 1 } k = k + 1 }
  return c
158}
159func xl_tok_end(nlen: i64, tstart: *i64, ntok: i64, t: i64) -> i64 {
  if t + 1 < ntok { return tstart[t+1] - 1 }
  return nlen
162}
163// find a WORD-type entry whose normalized source (per dir) exactly equals nbuf[pos..pos+len)
164func xl_find_word(eng: *i64, dir: i64, nbuf: *u8, pos: i64, len: i64) -> i64 {
  let nent: i64 = eng[XLE_NENT]
  let t_type: *u8 = eng[XLE_TYPE] as *u8
  let nru_off: *i64 = eng[XLE_NRUOFF] as *i64
  let nru_len: *i64 = eng[XLE_NRULEN] as *i64
  let nen_off: *i64 = eng[XLE_NENOFF] as *i64
  let nen_len: *i64 = eng[XLE_NENLEN] as *i64
  let norm: *u8 = eng[XLE_NORM] as *u8
  if len <= 0 { return 0 - 1 }
  var ei: i64 = 0
  while ei < nent {
      if (t_type[ei] as i64) == 119 {
          var soff: i64 = nru_off[ei]
          var slen: i64 = nru_len[ei]
          if dir == 0 { soff = nen_off[ei]; slen = nen_len[ei] }
          if slen == len {
              if xl_meq(nbuf, pos, pos + len, norm, soff, slen) == 1 { return ei }
          }
      }
      ei = ei + 1
  }
  return 0 - 1
186}
187// exact word, else strip one inflectional suffix and retry (RU: loaded suffixes; EN: trailing 's')
188func xl_morph_lookup(eng: *i64, dir: i64, nbuf: *u8, pos: i64, tlen: i64) -> i64 {
  let ex: i64 = xl_find_word(eng, dir, nbuf, pos, tlen)
  if ex >= 0 { return ex }
  if dir == 1 {
      let nsuf: i64 = eng[XLE_NSUF]
      let suf: *u8 = eng[XLE_SUF] as *u8
      let suf_off: *i64 = eng[XLE_SUFOFF] as *i64
      let suf_len: *i64 = eng[XLE_SUFLEN] as *i64
      var si: i64 = 0
      while si < nsuf {
          let sl: i64 = suf_len[si]
          if sl < tlen {
              var m: i64 = 1
              var k: i64 = 0
              while k < sl {
                  if (nbuf[pos + tlen - sl + k] as i64) != (suf[suf_off[si] + k] as i64) { m = 0; break }
                  k = k + 1
              }
              if m == 1 {
                  let w: i64 = xl_find_word(eng, 1, nbuf, pos, tlen - sl)
                  if w >= 0 { return w }
              }
          }
          si = si + 1
      }
  } else {
      if tlen > 2 {
          if (nbuf[pos + tlen - 1] as i64) == 115 {
              let w: i64 = xl_find_word(eng, 0, nbuf, pos, tlen - 1)
              if w >= 0 { return w }
          }
      }
  }
  return 0 - 1
222}
223// parse lexicon TSV (type\tru\ten\tweight) + morphology TSV (suffix\tnote) into the engine arena
224func xl_build(eng: *i64, lex: *u8, lexn: i64, morph: *u8, morphn: i64) -> i64 {
  let maxe: i64 = XL_MAXENT
  let t_type: *u8 = sys_mmap(maxe)
  let ru_off: *i64 = sys_mmap(maxe * 8) as *i64
  let ru_len: *i64 = sys_mmap(maxe * 8) as *i64
  let en_off: *i64 = sys_mmap(maxe * 8) as *i64
  let en_len: *i64 = sys_mmap(maxe * 8) as *i64
  let nru_off: *i64 = sys_mmap(maxe * 8) as *i64
  let nru_len: *i64 = sys_mmap(maxe * 8) as *i64
  let nen_off: *i64 = sys_mmap(maxe * 8) as *i64
  let nen_len: *i64 = sys_mmap(maxe * 8) as *i64
  let norm: *u8 = sys_mmap(XL_NORMCAP)
  let np: *i64 = sys_mmap(8) as *i64
  let fs: *i64 = sys_mmap(8) as *i64
  let fe: *i64 = sys_mmap(8) as *i64
  let term: *i64 = sys_mmap(8) as *i64
  let ip: *i64 = sys_mmap(8) as *i64
  *np = 0
  *ip = 0
  var nent: i64 = 0
  while nent < maxe {
      if xl_field(lex, lexn, ip, fs, fe, term) == 0 { break }
      let tlen0: i64 = *fe - *fs
      var ty: i64 = 119
      if tlen0 > 0 { ty = lex[*fs] as i64 }
      if (*term) != 9 { continue }
      if xl_field(lex, lexn, ip, fs, fe, term) == 0 { break }
      let ru_s: i64 = *fs
      let ru_e: i64 = *fe
      if (*term) != 9 { continue }
      if xl_field(lex, lexn, ip, fs, fe, term) == 0 { break }
      let en_s: i64 = *fs
      let en_e: i64 = *fe
      if (*term) == 9 { xl_field(lex, lexn, ip, fs, fe, term) }
      if en_e <= en_s { continue }
      if ru_e <= ru_s { continue }
      t_type[nent] = ty as u8
      ru_off[nent] = ru_s
      ru_len[nent] = ru_e - ru_s
      en_off[nent] = en_s
      en_len[nent] = en_e - en_s
      nru_off[nent] = *np
      xl_norm_phrase(lex, ru_s, ru_e, norm, np, XL_NORMCAP)
      nru_len[nent] = *np - nru_off[nent]
      nen_off[nent] = *np
      xl_norm_phrase(lex, en_s, en_e, norm, np, XL_NORMCAP)
      nen_len[nent] = *np - nen_off[nent]
      nent = nent + 1
  }
  let suf: *u8 = sys_mmap(XL_MAGIC_8192)
  let suf_off: *i64 = sys_mmap(XL_MAXSUF * 8) as *i64
  let suf_len: *i64 = sys_mmap(XL_MAXSUF * 8) as *i64
  let sp: *i64 = sys_mmap(8) as *i64
  *sp = 0
  *ip = 0
  var nsuf: i64 = 0
  while nsuf < XL_MAXSUF {
      if xl_field(morph, morphn, ip, fs, fe, term) == 0 { break }
      let sl: i64 = *fe - *fs
      if sl > 0 {
          suf_off[nsuf] = *sp
          var k: i64 = 0
          while k < sl { suf[*sp] = morph[*fs + k]; *sp = *sp + 1; k = k + 1 }
          suf_len[nsuf] = sl
          nsuf = nsuf + 1
      }
      if (*term) == 9 { xl_field(morph, morphn, ip, fs, fe, term) }
  }
  eng[XLE_NENT] = nent
  eng[XLE_TYPE] = t_type as i64
  eng[XLE_RUOFF] = ru_off as i64
  eng[XLE_RULEN] = ru_len as i64
  eng[XLE_ENOFF] = en_off as i64
  eng[XLE_ENLEN] = en_len as i64
  eng[XLE_NRUOFF] = nru_off as i64
  eng[XLE_NRULEN] = nru_len as i64
  eng[XLE_NENOFF] = nen_off as i64
  eng[XLE_NENLEN] = nen_len as i64
  eng[XLE_NORM] = norm as i64
  eng[XLE_LEX] = lex as i64
  eng[XLE_NSUF] = nsuf
  eng[XLE_SUFOFF] = suf_off as i64
  eng[XLE_SUFLEN] = suf_len as i64
  eng[XLE_SUF] = suf as i64
  sys_munmap(np, 8); sys_munmap(fs, 8); sys_munmap(fe, 8); sys_munmap(term, 8); sys_munmap(ip, 8); sys_munmap(sp, 8)
  return nent
310}
311// translate src[0..n) dir(0=EN->RU,1=RU->EN) into dst; greedy longest phrase/word match + morphology; -> dst length
312func xl_translate(eng: *i64, src: *u8, n: i64, dir: i64, dst: *u8, cap: i64) -> i64 {
  let nent: i64 = eng[XLE_NENT]
  let ru_off: *i64 = eng[XLE_RUOFF] as *i64
  let ru_len: *i64 = eng[XLE_RULEN] as *i64
  let en_off: *i64 = eng[XLE_ENOFF] as *i64
  let en_len: *i64 = eng[XLE_ENLEN] as *i64
  let nru_off: *i64 = eng[XLE_NRUOFF] as *i64
  let nru_len: *i64 = eng[XLE_NRULEN] as *i64
  let nen_off: *i64 = eng[XLE_NENOFF] as *i64
  let nen_len: *i64 = eng[XLE_NENLEN] as *i64
  let norm: *u8 = eng[XLE_NORM] as *u8
  let lex: *u8 = eng[XLE_LEX] as *u8
  let nbuf: *u8 = sys_mmap(XL_MAGIC_16384)
  let tstart: *i64 = sys_mmap(XL_MAGIC_2048 * 8) as *i64
  let ntokp: *i64 = sys_mmap(8) as *i64
  let op: *i64 = sys_mmap(8) as *i64
  let nlen: i64 = xl_norm_str(src, n, nbuf, XL_MAGIC_16384, tstart, XL_MAGIC_2048, ntokp)
  let ntok: i64 = *ntokp
  *op = 0
  var first: i64 = 1
  var t: i64 = 0
  while t < ntok {
      let pos: i64 = tstart[t]
      var best: i64 = 0 - 1
      var best_bytes: i64 = 0 - 1
      var best_toks: i64 = 0
      var ei: i64 = 0
      while ei < nent {
          var soff: i64 = nru_off[ei]
          var slen: i64 = nru_len[ei]
          if dir == 0 { soff = nen_off[ei]; slen = nen_len[ei] }
          if slen > best_bytes {
              if xl_meq(nbuf, pos, nlen, norm, soff, slen) == 1 {
                  var okb: i64 = 0
                  if pos + slen == nlen { okb = 1 } else { if (nbuf[pos+slen] as i64) == 32 { okb = 1 } }
                  if okb == 1 { best = ei; best_bytes = slen; best_toks = xl_ntok_of(norm, soff, slen) }
              }
          }
          ei = ei + 1
      }
      if best >= 0 {
          var ooff: i64 = en_off[best]
          var olen: i64 = en_len[best]
          if dir == 0 { ooff = ru_off[best]; olen = ru_len[best] }
          if first == 0 { xl_putb(dst, op, cap, 32) }
          first = 0
          var k: i64 = 0
          while k < olen { xl_putb(dst, op, cap, lex[ooff+k] as i64); k = k + 1 }
          t = t + best_toks
      } else {
          let tend: i64 = xl_tok_end(nlen, tstart, ntok, t)
          let tlen: i64 = tend - pos
          let m: i64 = xl_morph_lookup(eng, dir, nbuf, pos, tlen)
          if first == 0 { xl_putb(dst, op, cap, 32) }
          first = 0
          if m >= 0 {
              var ooff2: i64 = en_off[m]
              var olen2: i64 = en_len[m]
              if dir == 0 { ooff2 = ru_off[m]; olen2 = ru_len[m] }
              var k2: i64 = 0
              while k2 < olen2 { xl_putb(dst, op, cap, lex[ooff2+k2] as i64); k2 = k2 + 1 }
          } else {
              var k3: i64 = 0
              while k3 < tlen { xl_putb(dst, op, cap, nbuf[pos+k3] as i64); k3 = k3 + 1 }
          }
          t = t + 1
      }
  }
  let r: i64 = *op
  sys_munmap(nbuf, XL_MAGIC_16384); sys_munmap(tstart, XL_MAGIC_2048 * 8); sys_munmap(ntokp, 8); sys_munmap(op, 8)
  return r
383}
nx_xlate_mt.nx source

`nx_xlate_mt.nx` source
