nx_utf8.nx source
↩ module page · 237 lines · 8837 B
1// utf8.nx -- UTF-8 validator + codepoint decoder (RFC 3629).
2//
3// Needed by: X.509 UTF8String fields, HTTP headers, JSON strings,
4// NishiLang source code itself. Without a validator we either
5// accept malformed sequences (security problem: overlong forms
6// can hide dangerous characters) or reject valid input (bugs).
7//
8// UTF-8 byte patterns (RFC 3629 §3):
9// 0xxxxxxx 1 byte, U+0000..U+007F
10// 110xxxxx 10xxxxxx 2 bytes, U+0080..U+07FF
11// 1110xxxx 10xxxxxx 10xxxxxx 3 bytes, U+0800..U+FFFF
12// 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx 4 bytes, U+10000..U+10FFFF
13//
14// RFC 3629 §4 additional constraints:
15// - Reject overlong encodings (fewer bytes than minimum for
16// the codepoint). E.g., U+0000 must be `00`, not `C0 80`.
17// - Reject surrogates U+D800..U+DFFF (reserved for UTF-16).
18// - Reject codepoints > U+10FFFF.
19//
20// Invariants:
21// U1 utf8_validate returns exact byte position of first
22// invalid byte, or input length on clean success.
23// U2 utf8_decode_one advances the caller's position by the
24// number of bytes consumed; returns negative codepoint
25// on invalid sequence.
26// U3 No reads beyond the input's declared length; no buffer
27// overruns possible from malformed input.
28
29// nx_safety_envelope:
30// intended_use: AUTO_APPLIED -- primitive-specific tuning queued
31// sil_target: SIL1
32// evidence: [bulk_applied_2026-05-16, see-file-comment-for-detail]
33// verdict: NOT_YET_EVALUATED
34
35import "nx_syscalls.nx"
36
37// Error codes for utf8_decode_one (returned as negative).
38const UTF8_ERR_INVALID_BYTE: i64 = -1
39const UTF8_ERR_TRUNCATED: i64 = -2
40const UTF8_ERR_OVERLONG: i64 = -3
41const UTF8_ERR_SURROGATE: i64 = -4
42const UTF8_ERR_OUT_OF_RANGE: i64 = -5
43
44// Validate a UTF-8 byte sequence. Returns `n` on success (clean
45// through all bytes), or the byte offset where validation first
46// failed. Does NOT decode codepoints -- just validates structure.
47// Faster than decode-then-discard because the branch structure is
48// tighter.
49func utf8_validate(bytes: *u8, n: i64) -> i64 {
50 var i: i64 = 0
51 while i < n {
52 let b: i64 = bytes[i]
53 if b < 0x80 {
54 // 1-byte ASCII. Always valid.
55 i = i + 1
56 } else {
57 if b < 0xC2 {
58 // Either a continuation byte in lead position (0x80..0xBF)
59 // or an overlong 2-byte form (0xC0..0xC1).
60 return i
61 }
62 if b < 0xE0 {
63 // 2-byte sequence. Needs one continuation.
64 if i + 2 > n { return i }
65 let b1: i64 = bytes[i + 1]
66 if (b1 & 0xC0) != 0x80 { return i }
67 i = i + 2
68 } else {
69 if b < 0xF0 {
70 // 3-byte sequence.
71 if i + 3 > n { return i }
72 let b1: i64 = bytes[i + 1]
73 let b2: i64 = bytes[i + 2]
74 if (b1 & 0xC0) != 0x80 { return i }
75 if (b2 & 0xC0) != 0x80 { return i }
76 // Overlong check: for E0, b1 must be >= 0xA0.
77 if b == 0xE0 {
78 if b1 < 0xA0 { return i }
79 }
80 // Surrogate check: for ED, b1 must be < 0xA0
81 // (0xED 0xA0..0xED 0xBF encode U+D800..U+DFFF).
82 if b == 0xED {
83 if b1 >= 0xA0 { return i }
84 }
85 i = i + 3
86 } else {
87 if b < 0xF5 {
88 // 4-byte sequence.
89 if i + 4 > n { return i }
90 let b1: i64 = bytes[i + 1]
91 let b2: i64 = bytes[i + 2]
92 let b3: i64 = bytes[i + 3]
93 if (b1 & 0xC0) != 0x80 { return i }
94 if (b2 & 0xC0) != 0x80 { return i }
95 if (b3 & 0xC0) != 0x80 { return i }
96 // Overlong: F0 requires b1 >= 0x90.
97 if b == 0xF0 {
98 if b1 < 0x90 { return i }
99 }
100 // Out-of-range: F4 requires b1 < 0x90 (F4 90..
101 // would be > U+10FFFF).
102 if b == 0xF4 {
103 if b1 >= 0x90 { return i }
104 }
105 i = i + 4
106 } else {
107 // 0xF5..0xFF: all invalid (codepoints > U+10FFFF
108 // or continuation-only lead bytes).
109 return i
110 }
111 }
112 }
113 }
114 }
115 return n
116}
117
118// Decode one codepoint starting at `bytes[*pos]`. Advances *pos
119// past the consumed bytes on success. Returns the codepoint, or
120// a negative UTF8_ERR_* on malformed input.
121func utf8_decode_one(bytes: *u8, len: i64, pos: *i64) -> i64 {
122 let i: i64 = *pos
123 if i >= len { return UTF8_ERR_TRUNCATED }
124 let b: i64 = bytes[i]
125 if b < 0x80 {
126 *pos = i + 1
127 return b
128 }
129 if b < 0xC2 { return UTF8_ERR_INVALID_BYTE }
130 if b < 0xE0 {
131 if i + 2 > len { return UTF8_ERR_TRUNCATED }
132 let b1: i64 = bytes[i + 1]
133 if (b1 & 0xC0) != 0x80 { return UTF8_ERR_INVALID_BYTE }
134 let cp: i64 = ((b & 0x1F) << 6) | (b1 & 0x3F)
135 if cp < 0x80 { return UTF8_ERR_OVERLONG }
136 *pos = i + 2
137 return cp
138 }
139 if b < 0xF0 {
140 if i + 3 > len { return UTF8_ERR_TRUNCATED }
141 let b1: i64 = bytes[i + 1]
142 let b2: i64 = bytes[i + 2]
143 if (b1 & 0xC0) != 0x80 { return UTF8_ERR_INVALID_BYTE }
144 if (b2 & 0xC0) != 0x80 { return UTF8_ERR_INVALID_BYTE }
145 let cp: i64 = ((b & 0x0F) << 12) | ((b1 & 0x3F) << 6) | (b2 & 0x3F)
146 if cp < 0x800 { return UTF8_ERR_OVERLONG }
147 if cp >= 0xD800 {
148 if cp <= 0xDFFF { return UTF8_ERR_SURROGATE }
149 }
150 *pos = i + 3
151 return cp
152 }
153 if b < 0xF5 {
154 if i + 4 > len { return UTF8_ERR_TRUNCATED }
155 let b1: i64 = bytes[i + 1]
156 let b2: i64 = bytes[i + 2]
157 let b3: i64 = bytes[i + 3]
158 if (b1 & 0xC0) != 0x80 { return UTF8_ERR_INVALID_BYTE }
159 if (b2 & 0xC0) != 0x80 { return UTF8_ERR_INVALID_BYTE }
160 if (b3 & 0xC0) != 0x80 { return UTF8_ERR_INVALID_BYTE }
161 let cp: i64 = ((b & 0x07) << 18) | ((b1 & 0x3F) << 12)
162 | ((b2 & 0x3F) << 6) | (b3 & 0x3F)
163 if cp < 0x10000 { return UTF8_ERR_OVERLONG }
164 if cp > 0x10FFFF { return UTF8_ERR_OUT_OF_RANGE }
165 *pos = i + 4
166 return cp
167 }
168 return UTF8_ERR_INVALID_BYTE
169}
170
171// Encode one codepoint to UTF-8 at `out[*pos]`. Advances *pos by
172// 1-4 bytes. Returns 0 on success, negative on invalid codepoint
173// (surrogate or out-of-range).
174func utf8_encode_one(cp: i64, out: *u8, pos: *i64) -> i64 {
175 let p: i64 = *pos
176 if cp < 0 { return UTF8_ERR_OUT_OF_RANGE }
177 if cp < 0x80 {
178 out[p] = cp & 0xFF
179 *pos = p + 1
180 return 0
181 }
182 if cp < 0x800 {
183 out[p] = 0xC0 | ((cp >> 6) & 0x1F)
184 out[p + 1] = 0x80 | (cp & 0x3F)
185 *pos = p + 2
186 return 0
187 }
188 if cp >= 0xD800 {
189 if cp <= 0xDFFF { return UTF8_ERR_SURROGATE }
190 }
191 if cp < 0x10000 {
192 out[p] = 0xE0 | ((cp >> 12) & 0x0F)
193 out[p + 1] = 0x80 | ((cp >> 6) & 0x3F)
194 out[p + 2] = 0x80 | (cp & 0x3F)
195 *pos = p + 3
196 return 0
197 }
198 if cp > 0x10FFFF { return UTF8_ERR_OUT_OF_RANGE }
199 out[p] = 0xF0 | ((cp >> 18) & 0x07)
200 out[p + 1] = 0x80 | ((cp >> 12) & 0x3F)
201 out[p + 2] = 0x80 | ((cp >> 6) & 0x3F)
202 out[p + 3] = 0x80 | (cp & 0x3F)
203 *pos = p + 4
204 return 0
205}
206
207// Compile-only smoke with the RFC 3629 §5 "canonical forms" idea:
208// round-trip three common codepoints.
209func main() -> i64 {
210 let buf: *u8 = sys_mmap(32)
211 let pos_raw: *u8 = sys_mmap(16)
212 let pos: *i64 = pos_raw as *i64
213
214 // Encode 'A' (U+0041), 'é' (U+00E9), '€' (U+20AC), '𝄞' (U+1D11E).
215 *pos = 0
216 utf8_encode_one(0x41, buf, pos)
217 utf8_encode_one(0xE9, buf, pos)
218 utf8_encode_one(0x20AC, buf, pos)
219 utf8_encode_one(0x1D11E, buf, pos)
220 let total: i64 = *pos
221
222 // Validate; should succeed all the way through.
223 let v: i64 = utf8_validate(buf, total)
224 if v != total { return 1 }
225
226 // Decode back and sum the codepoints -- expect
227 // 0x41 + 0xE9 + 0x20AC + 0x1D11E = 0x1F23D8.
228 *pos = 0
229 var sum: i64 = 0
230 while *pos < total {
231 let cp: i64 = utf8_decode_one(buf, total, pos)
232 if cp < 0 { return 2 }
233 sum = sum + cp
234 }
235 if sum != 0x1F23D8 { return 3 }
236 return 0
237}