nx_book_mojibake_gate.nx source
↩ module page · 80 lines · 4941 B
1// nx_book_mojibake_gate.nx -- liar-kill gate for the library mojibake fix (nx_charset_repair_utf8).
2// The operator's report: book text "there's" renders as "there<>s" -- the curly apostrophe (Windows-1252
3// byte 0x92) survives raw into a UTF-8-served page, where a lone 0x92 is an invalid continuation byte ->
4// U+FFFD. This gate proves the repair pass byte-for-byte: it (a) fixes the reported byte, (b) fixes the
5// other smart-punctuation + Latin-1 bytes, and CRITICALLY (c) is IDEMPOTENT on already-valid UTF-8 (it must
6// NOT double-encode a real ' / é -- that would be the cure-becomes-the-disease regression). Teeth: a known
7// double-encoded length is rejected. expect_exit: 0 license_tier: ORIGINAL
8import "nx_syscalls.nx"
9import "nx_charset.nx"
10
11func mg_p(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
12func mg_n(v0: i64) -> i64 { var v: i64=v0; if v<0 { sys_write(1,"-" as *u8,1); v=0-v } let b: *u8=sys_mmap(24); var k: i64=0; if v==0 {b[0]=48 as u8;k=1} while v>0 {b[k]=(48+(v%10)) as u8; v=v/10; k=k+1} let o: *u8=sys_mmap(24); var j: i64=0; while j<k {o[j]=b[k-1-j];j=j+1} sys_write(1,o,k); return 0 }
13
14// byte-exact compare of repair(src[0..sn]) against exp[0..en]; prints PASS/FAIL with the case label.
15func chk(label: *u8, src: *u8, sn: i64, exp: *u8, en: i64, pf: *i64) -> i64 {
16 let out: *u8 = sys_mmap(256)
17 let gn: i64 = nx_charset_repair_utf8(src, sn, out, 256)
18 var ok: i64 = 1
19 if gn != en { ok = 0 } else { var i: i64 = 0; while i < en { if (out[i]&0xff) != (exp[i]&0xff) { ok = 0; i = en } else { i = i + 1 } } }
20 mg_p(" "); mg_p(label)
21 if ok == 1 { mg_p(" PASS (len="); mg_n(gn); mg_p(")\n"); pf[0] = pf[0] + 1 }
22 else { mg_p(" FAIL got_len="); mg_n(gn); mg_p(" want_len="); mg_n(en); mg_p("\n"); pf[1] = pf[1] + 1 }
23 return ok
24}
25
26func main() -> i64 {
27 mg_p("=== nx_book_mojibake_gate: CP1252-stray-byte repair to clean UTF-8 ===\n" as *u8)
28 let pf: *i64 = sys_mmap(16) as *i64; pf[0]=0; pf[1]=0
29
30 // (1) THE REPORTED BUG: "there" + 0x92 + "s" -> "there" + U+2019(E2 80 99) + "s"
31 let s1: *u8 = sys_mmap(16); s1[0]=0x74; s1[1]=0x68; s1[2]=0x65; s1[3]=0x72; s1[4]=0x65; s1[5]=0x92; s1[6]=0x73
32 let e1: *u8 = sys_mmap(16); e1[0]=0x74; e1[1]=0x68; e1[2]=0x65; e1[3]=0x72; e1[4]=0x65; e1[5]=0xe2; e1[6]=0x80; e1[7]=0x99; e1[8]=0x73
33 chk("case1 reported: there<0x92>s -> apostrophe" as *u8, s1, 7, e1, 9, pf)
34
35 // (2) em-dash: 0x97 -> U+2014 (E2 80 94)
36 let s2: *u8 = sys_mmap(8); s2[0]=0x97
37 let e2: *u8 = sys_mmap(8); e2[0]=0xe2; e2[1]=0x80; e2[2]=0x94
38 chk("emdash 0x97 -> U+2014" as *u8, s2, 1, e2, 3, pf)
39
40 // (3) ellipsis: 0x85 -> U+2026 (E2 80 A6)
41 let s3: *u8 = sys_mmap(8); s3[0]=0x85
42 let e3: *u8 = sys_mmap(8); e3[0]=0xe2; e3[1]=0x80; e3[2]=0xa6
43 chk("ellipsis 0x85 -> U+2026" as *u8, s3, 1, e3, 3, pf)
44
45 // (4) whole-CP1252/Latin-1 e-acute: lone 0xE9 (invalid as UTF-8 lead w/o continuations) -> C3 A9
46 let s4: *u8 = sys_mmap(8); s4[0]=0xe9
47 let e4: *u8 = sys_mmap(8); e4[0]=0xc3; e4[1]=0xa9
48 chk("latin1 0xE9 -> U+00E9" as *u8, s4, 1, e4, 2, pf)
49
50 // (5) IDEMPOTENCY (anti-double-encode): a REAL UTF-8 ' (E2 80 99) must pass through UNCHANGED.
51 let s5: *u8 = sys_mmap(8); s5[0]=0xe2; s5[1]=0x80; s5[2]=0x99
52 let e5: *u8 = sys_mmap(8); e5[0]=0xe2; e5[1]=0x80; e5[2]=0x99
53 chk("idempotent UTF-8 U+2019 unchanged" as *u8, s5, 3, e5, 3, pf)
54
55 // (6) IDEMPOTENCY: a REAL UTF-8 é (C3 A9) must pass through UNCHANGED (not -> é).
56 let s6: *u8 = sys_mmap(8); s6[0]=0xc3; s6[1]=0xa9
57 let e6: *u8 = sys_mmap(8); e6[0]=0xc3; e6[1]=0xa9
58 chk("idempotent UTF-8 U+00E9 unchanged" as *u8, s6, 2, e6, 2, pf)
59
60 // (7) MIXED: "a" + 0x92 + (C3 A9 valid) + 0x97 + "b" -> a + E28099 + C3A9 + E28094 + b
61 let s7: *u8 = sys_mmap(16); s7[0]=0x61; s7[1]=0x92; s7[2]=0xc3; s7[3]=0xa9; s7[4]=0x97; s7[5]=0x62
62 let e7: *u8 = sys_mmap(24); e7[0]=0x61; e7[1]=0xe2; e7[2]=0x80; e7[3]=0x99; e7[4]=0xc3; e7[5]=0xa9; e7[6]=0xe2; e7[7]=0x80; e7[8]=0x94; e7[9]=0x62
63 chk("mixed stray+valid+stray" as *u8, s7, 6, e7, 10, pf)
64
65 // (8) ASCII-only is a pure copy
66 let s8: *u8 = sys_mmap(8); s8[0]=0x48; s8[1]=0x69
67 let e8: *u8 = sys_mmap(8); e8[0]=0x48; e8[1]=0x69
68 chk("ascii passthrough" as *u8, s8, 2, e8, 2, pf)
69
70 // (9) TEETH: prove discrimination -- the raw byte 0x92 must NOT survive in the output of case (1).
71 let to: *u8 = sys_mmap(64)
72 let tn: i64 = nx_charset_repair_utf8(s1, 7, to, 64)
73 var raw92: i64 = 0; var i: i64 = 0; while i < tn { if (to[i]&0xff)==0x92 { raw92 = 1 } i = i + 1 }
74 if raw92 == 0 { mg_p(" teeth: raw 0x92 absent from output PASS\n"); pf[0]=pf[0]+1 }
75 else { mg_p(" teeth: raw 0x92 SURVIVED FAIL\n"); pf[1]=pf[1]+1 }
76
77 mg_p("MOJIBAKE-GATE pass="); mg_n(pf[0]); mg_p(" fail="); mg_n(pf[1])
78 if pf[1] == 0 { mg_p(" verdict=GREEN\n"); sys_exit(0); return 0 }
79 mg_p(" verdict=RED\n"); sys_exit(1); return 1
80}