nx_charset.nx source
↩ module page · 233 lines · 9035 B
1// nx_charset.nx -- charset detection + transcode-to-UTF-8 for the sovereign
2// browser. Many live pages still serve ISO-8859-1 (Latin-1) or Windows-1252;
3// their 0x80-0xFF bytes are garbage/invalid UTF-8 unless transcoded. This
4// organ detects the charset from a Content-Type value and transcodes a byte
5// buffer to UTF-8 so the renderer (which then sees UTF-8) shows correct text.
6// Windows-1252 0x80-0x9F table sourced AUTHORITATIVELY from python cp1252.
7// Single-responsibility; nx_html_to_text / nish run this BEFORE rendering.
8// expect_exit: 0
9// license_tier: ORIGINAL
10
11import "nx_syscalls.nx"
12
13const NX_CS_UTF8: i64 = 0
14const NX_CS_LATIN1: i64 = 1
15const NX_CS_WIN1252: i64 = 2
16
17func _cs_lc(b: i64) -> i64 { if b >= 65 { if b <= 90 { return b + 32 } } return b }
18
19// Emit codepoint cp as UTF-8 at out[pos]; return new pos, or -1 on overflow.
20func _cs_emit(out: *u8, pos: i64, cap: i64, cp: i64) -> i64 {
21 if cp < 0x80 {
22 if pos >= cap { return 0 - 1 }
23 out[pos] = cp as u8
24 return pos + 1
25 }
26 if cp < 0x800 {
27 if (pos + 1) >= cap { return 0 - 1 }
28 out[pos] = (0xc0 | (cp >> 6)) as u8
29 out[pos + 1] = (0x80 | (cp & 0x3f)) as u8
30 return pos + 2
31 }
32 if (pos + 2) >= cap { return 0 - 1 }
33 out[pos] = (0xe0 | (cp >> 12)) as u8
34 out[pos + 1] = (0x80 | ((cp >> 6) & 0x3f)) as u8
35 out[pos + 2] = (0x80 | (cp & 0x3f)) as u8
36 return pos + 3
37}
38
39// Fill w[0..32) with the Windows-1252 0x80-0x9F -> Unicode table (authoritative,
40// python cp1252). Shared by nx_charset_to_utf8 + nx_charset_repair_utf8 (rule 15:
41// one home for the table). Undefined CP1252 slots (0x81/0x8d/0x8f/0x90/0x9d) keep
42// their byte value as the codepoint (Latin-1 identity), matching browser fallback.
43func _cs_fill_w1252(w: *i64) -> i64 {
44 w[0] = 0x20ac
45 w[1] = 0x81
46 w[2] = 0x201a
47 w[3] = 0x192
48 w[4] = 0x201e
49 w[5] = 0x2026
50 w[6] = 0x2020
51 w[7] = 0x2021
52 w[8] = 0x2c6
53 w[9] = 0x2030
54 w[10] = 0x160
55 w[11] = 0x2039
56 w[12] = 0x152
57 w[13] = 0x8d
58 w[14] = 0x17d
59 w[15] = 0x8f
60 w[16] = 0x90
61 w[17] = 0x2018
62 w[18] = 0x2019
63 w[19] = 0x201c
64 w[20] = 0x201d
65 w[21] = 0x2022
66 w[22] = 0x2013
67 w[23] = 0x2014
68 w[24] = 0x2dc
69 w[25] = 0x2122
70 w[26] = 0x161
71 w[27] = 0x203a
72 w[28] = 0x153
73 w[29] = 0x9d
74 w[30] = 0x17e
75 w[31] = 0x178
76 return 0
77}
78
79// Transcode src[0..n) in charset cs to UTF-8 in out; return out length or -1.
80func nx_charset_to_utf8(src: *u8, n: i64, cs: i64, out: *u8, cap: i64) -> i64 {
81 if cs == NX_CS_UTF8 {
82 if n > cap { return 0 - 1 }
83 var k: i64 = 0
84 while k < n { out[k] = src[k]; k = k + 1 }
85 return n
86 }
87 // Windows-1252 high table (0x80-0x9F).
88 let w: *i64 = sys_mmap(8 * 32) as *i64
89 _cs_fill_w1252(w)
90 var i: i64 = 0
91 var pos: i64 = 0
92 while i < n {
93 let b: i64 = src[i] & 0xff
94 var cp: i64 = b
95 if cs == NX_CS_WIN1252 { if b >= 0x80 { if b <= 0x9f { cp = w[b - 0x80] } } }
96 pos = _cs_emit(out, pos, cap, cp)
97 if pos < 0 { return 0 - 1 }
98 i = i + 1
99 }
100 return pos
101}
102
103// Case-insensitive substring search (lowercased src vs lowercase literal).
104func _cs_contains(s: *u8, n: i64, pat: *u8, pn: i64) -> i64 {
105 var i: i64 = 0
106 while (i + pn) <= n {
107 var j: i64 = 0
108 var ok: i64 = 1
109 while j < pn {
110 if _cs_lc(s[i + j] & 0xff) != (pat[j] & 0xff) { ok = 0; j = pn } else { j = j + 1 }
111 }
112 if ok == 1 { return 1 }
113 i = i + 1
114 }
115 return 0
116}
117
118// Detect charset from a Content-Type header value. Default UTF-8 (safe).
119func nx_charset_from_content_type(val: *u8, len: i64) -> i64 {
120 let p1252: *u8 = sys_mmap(8); p1252[0]=0x31; p1252[1]=0x32; p1252[2]=0x35; p1252[3]=0x32 // 1252
121 if _cs_contains(val, len, p1252, 4) == 1 { return NX_CS_WIN1252 }
122 let p8859: *u8 = sys_mmap(8); p8859[0]=0x38; p8859[1]=0x38; p8859[2]=0x35; p8859[3]=0x39 // 8859
123 if _cs_contains(val, len, p8859, 4) == 1 { return NX_CS_LATIN1 }
124 let plat: *u8 = sys_mmap(8); plat[0]=0x6c; plat[1]=0x61; plat[2]=0x74; plat[3]=0x69; plat[4]=0x6e // latin
125 if _cs_contains(val, len, plat, 5) == 1 { return NX_CS_LATIN1 }
126 return NX_CS_UTF8
127}
128
129// First case-insensitive index of pat in s[0..n), or -1.
130func _cs_find(s: *u8, n: i64, pat: *u8, pn: i64) -> i64 {
131 var i: i64 = 0
132 while (i + pn) <= n {
133 var j: i64 = 0
134 var ok: i64 = 1
135 while j < pn {
136 if _cs_lc(s[i + j] & 0xff) != (pat[j] & 0xff) { ok = 0; j = pn } else { j = j + 1 }
137 }
138 if ok == 1 { return i }
139 i = i + 1
140 }
141 return 0 - 1
142}
143
144// Sniff a charset DECLARATION ("charset" token) in the first ~2048 bytes of
145// buf -- works on an HTTP Content-Type value OR an HTML <head> carrying a
146// <meta charset=...> / <meta http-equiv=... content="...charset=...">.
147// Returns NX_CS_* when a charset is declared (utf-8/unknown -> NX_CS_UTF8),
148// or -1 when none is present, so callers can fall back / keep a default.
149// Per HTML5 the declaration must appear within the first ~1024 bytes.
150func nx_charset_sniff(buf: *u8, n: i64) -> i64 {
151 var lim: i64 = n
152 if lim > 2048 { lim = 2048 }
153 let cpat: *u8 = sys_mmap(8)
154 cpat[0]=0x63; cpat[1]=0x68; cpat[2]=0x61; cpat[3]=0x72 // char
155 cpat[4]=0x73; cpat[5]=0x65; cpat[6]=0x74 // set
156 let pos: i64 = _cs_find(buf, lim, cpat, 7)
157 if pos < 0 { return 0 - 1 }
158 let wstart: i64 = pos + 7
159 var wlen: i64 = lim - wstart
160 if wlen <= 0 { return 0 - 1 }
161 if wlen > 48 { wlen = 48 }
162 let wp: *u8 = ((buf as i64) + wstart) as *u8
163 return nx_charset_from_content_type(wp, wlen)
164}
165
166// ===== UTF-8 repair (charset UNKNOWN / mixed) =====================================
167// The library-book reality: most files are UTF-8, but many carry a few stray
168// Windows-1252 bytes (a curly apostrophe 0x92, em-dash 0x97, ellipsis 0x85...) left
169// by a Word/HTML conversion -- even when the file DECLARES utf-8. Served as UTF-8 a
170// lone 0x92 is an invalid continuation byte, so the browser shows U+FFFD () -- this
171// is the "there<>s" the operator reported. nx_charset_to_utf8 can't help: it needs a
172// declared charset, and blindly applying CP1252 to real UTF-8 DOUBLE-encodes it.
173//
174// nx_charset_repair_utf8 needs no declaration: it copies every VALID UTF-8 sequence
175// verbatim (so it is idempotent on clean UTF-8 -- never double-encodes) and expands
176// only the bytes that are NOT valid UTF-8, treating each as Windows-1252. The reported
177// byte (0x92) is a lone continuation byte = unambiguously invalid = cleanly repaired.
178
179// Length (1..4) of the valid UTF-8 sequence at src[i] within [0,n); 0 if invalid here.
180// Pragmatic validator: correct lead-byte class + the right number of 0x80-0xBF
181// continuation bytes (rejects 0xC0/0xC1 overlong leads and 0xF5-0xFF).
182func _cs_utf8_seqlen(src: *u8, i: i64, n: i64) -> i64 {
183 let b0: i64 = src[i] & 0xff
184 if b0 < 0x80 { return 1 } // ASCII
185 if b0 < 0xc2 { return 0 } // lone 0x80-0xBF, or 0xC0/0xC1 overlong
186 if b0 < 0xe0 { // 0xC2-0xDF: 2-byte
187 if i + 1 >= n { return 0 }
188 if (src[i+1] & 0xc0) != 0x80 { return 0 }
189 return 2
190 }
191 if b0 < 0xf0 { // 0xE0-0xEF: 3-byte
192 if i + 2 >= n { return 0 }
193 if (src[i+1] & 0xc0) != 0x80 { return 0 }
194 if (src[i+2] & 0xc0) != 0x80 { return 0 }
195 return 3
196 }
197 if b0 < 0xf5 { // 0xF0-0xF4: 4-byte
198 if i + 3 >= n { return 0 }
199 if (src[i+1] & 0xc0) != 0x80 { return 0 }
200 if (src[i+2] & 0xc0) != 0x80 { return 0 }
201 if (src[i+3] & 0xc0) != 0x80 { return 0 }
202 return 4
203 }
204 return 0 // 0xF5-0xFF
205}
206
207// Repair src[0..n) to clean UTF-8 in out (cap bytes). Returns out length. Valid UTF-8
208// runs are copied byte-identically; each invalid byte is expanded as Windows-1252
209// (0x80-0x9F via the table, 0xA0-0xFF as Latin-1 identity). On output overflow it
210// stops at the cap and returns what fit (never fails the caller).
211func nx_charset_repair_utf8(src: *u8, n: i64, out: *u8, cap: i64) -> i64 {
212 let w: *i64 = sys_mmap(8 * 32) as *i64
213 _cs_fill_w1252(w)
214 var i: i64 = 0
215 var pos: i64 = 0
216 while i < n {
217 let k: i64 = _cs_utf8_seqlen(src, i, n)
218 if k > 0 {
219 if pos + k > cap { i = n } else {
220 var j: i64 = 0
221 while j < k { out[pos] = src[i+j]; pos = pos + 1; j = j + 1 }
222 i = i + k
223 }
224 } else {
225 let b: i64 = src[i] & 0xff
226 var cp: i64 = b
227 if b >= 0x80 { if b <= 0x9f { cp = w[b - 0x80] } }
228 let np: i64 = _cs_emit(out, pos, cap, cp)
229 if np < 0 { i = n } else { pos = np; i = i + 1 }
230 }
231 }
232 return pos
233}