code wiki / _hdl_build / nx_tok_fold_gate.nx
`nx_tok_fold_gate.nx` source

↩ module page · 182 lines · 9084 B
1// nx_tok_fold_gate.nx -- KATs for the accent-folding UTF-8 tokenizer (the Unicode rung): Latin diacritics
2// fold to ASCII base letters on BOTH tokenize sides, unfoldable codepoints separate cleanly (whole sequence
3// consumed -- continuation bytes never leak into tokens), ASCII behavior is byte-identical to before, and
4// the STORE-LEVEL proof: an indexed doc containing "Kyōka" is found by ss_term("kyoka") and the quoted
5// phrase evaluator (fold-consistent positions). license_tier: ORIGINAL
6import "nx_seg_store.nx"
7
8func gf_puts(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
9func gf_num(v: i64) -> i64 {
  let bb: *u8 = sys_mmap(28); var m: i64 = v
  if m < 0 { sys_write(1, "-" as *u8, 1); m = 0 - m }
  let t: *u8 = sys_mmap(28); var k: i64 = 0
  if m == 0 { t[0] = 48 as u8; k = 1 }
  while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
  var i: i64 = 0; while i < k { bb[i] = t[k - 1 - i]; i = i + 1 }
  sys_write(1, bb, k); return 0
17}
18func gf_len(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } return n }
19func gf_check(name: *u8, cond: i64, pass: *i64, total: *i64) -> i64 {
  total[0] = total[0] + 1
  gf_puts(name)
  if cond == 1 { pass[0] = pass[0] + 1; gf_puts(" PASS\n" as *u8) } else { gf_puts(" FAIL\n" as *u8) }
  return 0
24}
25// tokenize a whole string; join tokens with '|' into out; return token count
26func gf_toks(s: *u8, out: *u8) -> i64 {
  let tbl: *u8 = sys_mmap(272)
  ss_tok_table(tbl)
  let pos: *i64 = sys_mmap(16) as *i64
  pos[0] = 0
  let t: *u8 = sys_mmap(64)
  var n: i64 = 0
  var o: i64 = 0
  let sl: i64 = gf_len(s)
  var go: i64 = 1
  while go == 1 {
      let l: i64 = ss_tok_next2(s, sl, pos, t, tbl)
      if l < 0 { go = 0 } else {
          if n > 0 { out[o] = 124 as u8; o = o + 1 }
          var x: i64 = 0
          while x < l { out[o] = t[x]; o = o + 1; x = x + 1 }
          n = n + 1
      }
  }
  out[o] = 0 as u8
  return n
47}
48func gf_eq(a: *u8, b: *u8) -> i64 {
  var i: i64 = 0
  var go: i64 = 1
  while go == 1 {
      if a[i] != b[i] { return 0 }
      if a[i] == (0 as u8) { return 1 }
      i = i + 1
  }
  return 1
57}
58
59func main() -> i64 {
  gf_puts("=== nx_tok_fold gate (UTF-8 accent folding; ASCII purity; store-level round trip) ===\n" as *u8)
  let pass: *i64 = sys_mmap(16) as *i64; pass[0] = 0
  let total: *i64 = sys_mmap(16) as *i64; total[0] = 0
  let out: *u8 = sys_mmap(512)
64
  // T1 the operator's own case: Kyōka (U+014D)
  let n1: i64 = gf_toks("Ky\xC5\x8Dka" as *u8, out)
  var t1: i64 = 0
  if n1 == 1 { if gf_eq(out, "kyoka" as *u8) == 1 { t1 = 1 } }
  gf_check("T1 Ky\\u014Dka -> kyoka" as *u8, t1, pass, total)
70
  // T2 Latin-1: cafe / naive / Muller
  let n2: i64 = gf_toks("caf\xC3\xA9 na\xC3\xAFve M\xC3\xBCller" as *u8, out)
  var t2: i64 = 0
  if n2 == 3 { if gf_eq(out, "cafe|naive|muller" as *u8) == 1 { t2 = 1 } }
  gf_check("T2 cafe|naive|muller (Latin-1 folds)" as *u8, t2, pass, total)
76
  // T3 Latin-Ext-A: Škoda / žižek
  let n3: i64 = gf_toks("\xC5\xA0koda \xC5\xBEi\xC5\xBEek" as *u8, out)
  var t3: i64 = 0
  if n3 == 2 { if gf_eq(out, "skoda|zizek" as *u8) == 1 { t3 = 1 } }
  gf_check("T3 skoda|zizek (Latin-Ext-A folds)" as *u8, t3, pass, total)
82
  // T4 multiplication sign stays a separator
  let n4: i64 = gf_toks("ab\xC3\x97cd" as *u8, out)
  var t4: i64 = 0
  if n4 == 2 { if gf_eq(out, "ab|cd" as *u8) == 1 { t4 = 1 } }
  gf_check("T4 U+00D7 times-sign separates (ab|cd)" as *u8, t4, pass, total)
88
  // T5 CJK bigram rung BUILT (2026-07-23; replaces the old honest-limitation assertion): a 2-char
  // Han run tokenizes as ONE bigram whose bytes are the two chars verbatim
  let n5: i64 = gf_toks("\xE6\x97\xA5\xE6\x9C\xAC" as *u8, out)
  var t5: i64 = 0
  if n5 == 1 { if gf_eq(out, "\xE6\x97\xA5\xE6\x9C\xAC" as *u8) == 1 { t5 = 1 } }
  gf_check("T5 CJK 2-char run -> ONE bigram token" as *u8, t5, pass, total)
95
  // T6 mixed real-world line
  let n6: i64 = gf_toks("Izumi Ky\xC5\x8Dka 1900" as *u8, out)
  var t6: i64 = 0
  if n6 == 3 { if gf_eq(out, "izumi|kyoka|1900" as *u8) == 1 { t6 = 1 } }
  gf_check("T6 Izumi Ky\\u014Dka 1900 -> izumi|kyoka|1900" as *u8, t6, pass, total)
101
  // T7 stray continuation byte never mangles
  let n7: i64 = gf_toks("\x8Dab cd" as *u8, out)
  var t7: i64 = 0
  if n7 == 2 { if gf_eq(out, "ab|cd" as *u8) == 1 { t7 = 1 } }
  gf_check("T7 stray continuation byte -> ab|cd" as *u8, t7, pass, total)
107
  // T8 4-byte plane (emoji) consumed cleanly
  let n8: i64 = gf_toks("\xF0\x9F\x98\x80ok yes" as *u8, out)
  var t8: i64 = 0
  if n8 == 2 { if gf_eq(out, "ok|yes" as *u8) == 1 { t8 = 1 } }
  gf_check("T8 emoji consumed -> ok|yes" as *u8, t8, pass, total)
113
  // T9 ASCII PURITY: identical to the pre-fold tokenizer on pure-ASCII text
  let n9: i64 = gf_toks("Plain ASCII words 42 x" as *u8, out)
  var t9: i64 = 0
  if n9 == 4 { if gf_eq(out, "plain|ascii|words|42" as *u8) == 1 { t9 = 1 } }
  gf_check("T9 ASCII purity (plain|ascii|words|42; 1-char x dropped)" as *u8, t9, pass, total)
119
  // T10 STORE-LEVEL: index a doc containing the accented name; plain-ascii term + phrase both find it
  let prefix: *u8 = "/tmp/foldgate-" as *u8
  let w: *i64 = ss_begin()
  ss_add(w, 1, "doc:1" as *u8, "Izumi Ky\xC5\x8Dka wrote gothic novels" as *u8, gf_len("Izumi Ky\xC5\x8Dka wrote gothic novels" as *u8))
  ss_add(w, 1, "doc:2" as *u8, "plain control document" as *u8, 22)
  ss_commit(prefix, w, 1)
  let h: *i64 = ss_open(prefix)
  let kp: *i64 = sys_mmap(8 * 16) as *i64
  let kl: *i64 = sys_mmap(8 * 16) as *i64
  let nt10: i64 = ss_term(h, "kyoka" as *u8, kp, kl, 16)
  gf_check("T10 ss_term(kyoka) finds the accented doc" as *u8, (nt10 == 1) as i64, pass, total)
131
  // T11 phrase positions fold-consistent: "izumi kyoka" adjacent
  let terms: *i64 = sys_mmap(8 * 4) as *i64
  terms[0] = "izumi" as *u8 as i64
  terms[1] = "kyoka" as *u8 as i64
  let ebox: *i64 = sys_mmap(16) as *i64
  let np11: i64 = ss_phrase(prefix, h, terms, 2, kp, kl, 16, ebox)
  var t11: i64 = 0
  if np11 == 1 { if ebox[0] == 1 { t11 = 1 } }
  gf_check("T11 phrase [izumi kyoka] exact via folded positions" as *u8, t11, pass, total)
141
  // ==== MULTILINGUAL rung (2026-07-23): Cyrillic words + CJK overlapping bigrams + Hangul ====
  // T12 3-char Han run -> TWO overlapping bigrams (ri-hon | hon-go)
  let n12: i64 = gf_toks("\xE6\x97\xA5\xE6\x9C\xAC\xE8\xAA\x9E" as *u8, out)
  var t12: i64 = 0
  if n12 == 2 { if gf_eq(out, "\xE6\x97\xA5\xE6\x9C\xAC|\xE6\x9C\xAC\xE8\xAA\x9E" as *u8) == 1 { t12 = 1 } }
  gf_check("T12 CJK 3-char run -> overlapping bigrams (AB|BC)" as *u8, t12, pass, total)
148
  // T13 Russian case-fold: Moskva ROSSIYA -> lowercase Cyrillic word tokens
  let n13: i64 = gf_toks("\xD0\x9C\xD0\xBE\xD1\x81\xD0\xBA\xD0\xB2\xD0\xB0 \xD0\xA0\xD0\x9E\xD0\xA1\xD0\xA1\xD0\x98\xD0\xAF" as *u8, out)
  var t13: i64 = 0
  if n13 == 2 { if gf_eq(out, "\xD0\xBC\xD0\xBE\xD1\x81\xD0\xBA\xD0\xB2\xD0\xB0|\xD1\x80\xD0\xBE\xD1\x81\xD1\x81\xD0\xB8\xD1\x8F" as *u8) == 1 { t13 = 1 } }
  gf_check("T13 Cyrillic case-fold (Moskva ROSSIYA -> moskva|rossiya)" as *u8, t13, pass, total)
154
  // T14 Hangul syllables -> bigram
  let n14: i64 = gf_toks("\xED\x95\x9C\xEA\xB5\xAD" as *u8, out)
  var t14: i64 = 0
  if n14 == 1 { if gf_eq(out, "\xED\x95\x9C\xEA\xB5\xAD" as *u8) == 1 { t14 = 1 } }
  gf_check("T14 Hangul 2-syllable run -> ONE bigram token" as *u8, t14, pass, total)
160
  // T15 Latin word flushes cleanly at a CJK boundary (abc then the bigram; no byte bleed)
  let n15: i64 = gf_toks("abc\xE6\x97\xA5\xE6\x9C\xAC" as *u8, out)
  var t15: i64 = 0
  if n15 == 2 { if gf_eq(out, "abc|\xE6\x97\xA5\xE6\x9C\xAC" as *u8) == 1 { t15 = 1 } }
  gf_check("T15 latin|CJK boundary flush (abc|bigram)" as *u8, t15, pass, total)
166
  // T16 STORE-LEVEL: Russian + CJK docs indexed and found via their native-script terms
  let w2: *i64 = ss_begin()
  ss_add(w2, 1, "doc:3" as *u8, "\xD0\xBD\xD0\xBE\xD0\xB2\xD0\xBE\xD1\x81\xD1\x82\xD0\xB8 \xD0\xA0\xD0\xBE\xD1\x81\xD1\x81\xD0\xB8\xD0\xB8 \xD1\x81\xD0\xB5\xD0\xB3\xD0\xBE\xD0\xB4\xD0\xBD\xD1\x8F" as *u8, 40)
  ss_add(w2, 1, "doc:4" as *u8, "\xE6\x9D\xB1\xE4\xBA\xAC\xE5\xA4\xA7\xE5\xAD\xA6" as *u8, 12)
  ss_commit(prefix, w2, 2)
  let h2: *i64 = ss_open(prefix)
  let ntru: i64 = ss_term(h2, "\xD1\x80\xD0\xBE\xD1\x81\xD1\x81\xD0\xB8\xD0\xB8" as *u8, kp, kl, 16)
  let ntcj: i64 = ss_term(h2, "\xE6\x9D\xB1\xE4\xBA\xAC" as *u8, kp, kl, 16)
  var t16: i64 = 0
  if ntru == 1 { if ntcj == 1 { t16 = 1 } }
  gf_check("T16 store-level: ss_term(rossii)=1 + ss_term(tokyo-bigram)=1" as *u8, t16, pass, total)
178
  gf_puts("=== tok-fold gate: " as *u8); gf_num(pass[0]); gf_puts("/" as *u8); gf_num(total[0]); gf_puts(" " as *u8)
  if pass[0] == total[0] { gf_puts("verdict=GREEN\n" as *u8); return 0 }
  gf_puts("verdict=RED\n" as *u8); return 1
182}
nx_tok_fold_gate.nx source

`nx_tok_fold_gate.nx` source
