nx_nlex.nx source
↩ module page · 368 lines · 12305 B
1// nx_nlex.nx -- NishiLang surface-syntax tokenizer, in pure NishiLang.
2//
3// FIRST CONCRETE STEP of the nxc2 self-hosting trajectory ("get us off
4// C"). This tokenizer recognizes the NishiLang surface syntax that
5// nxc2/lex.c + nxc2/parse.c handle in C today. When the parser + IR
6// builder + backend(s) are also written in NishiLang, the compiler
7// becomes self-hosting.
8//
9// Distinct from runtime/nx_lex.nx (low-level character classifiers
10// like is_ws / is_alpha for generic parsing). This module is the
11// FULL NishiLang tokenizer.
12//
13// Token kinds:
14// - keywords: func, let, var, if, else, while, return, const,
15// struct, import, as
16// - identifiers: [a-zA-Z_][a-zA-Z0-9_]*
17// - integers: decimal + 0x hex
18// - punctuation: ( ) { } [ ] , ; : . -> = == != < <= > >= + - * /
19// % & | ^ ~ << >>
20// - line comments: // to EOL (skipped, not emitted)
21//
22// Output: NxToken array (kind + start + length + int_val).
23
24import "syscalls.nx"
25import "nx_loop.nx"
26
27// === token kinds (sealed enum) =====================================
28const NX_TOK_EOF: i64 = 0
29const NX_TOK_IDENT: i64 = 1
30const NX_TOK_INT: i64 = 2
31const NX_TOK_FUNC: i64 = 10
32const NX_TOK_LET: i64 = 11
33const NX_TOK_VAR: i64 = 12
34const NX_TOK_IF: i64 = 13
35const NX_TOK_ELSE: i64 = 14
36const NX_TOK_WHILE: i64 = 15
37const NX_TOK_RETURN: i64 = 16
38const NX_TOK_CONST: i64 = 17
39const NX_TOK_STRUCT: i64 = 18
40const NX_TOK_IMPORT: i64 = 19
41const NX_TOK_AS: i64 = 20
42const NX_TOK_LPAREN: i64 = 30
43const NX_TOK_RPAREN: i64 = 31
44const NX_TOK_LBRACE: i64 = 32
45const NX_TOK_RBRACE: i64 = 33
46const NX_TOK_LBRACK: i64 = 34
47const NX_TOK_RBRACK: i64 = 35
48const NX_TOK_COMMA: i64 = 36
49const NX_TOK_SEMI: i64 = 37
50const NX_TOK_COLON: i64 = 38
51const NX_TOK_ARROW: i64 = 39 // ->
52const NX_TOK_ASSIGN: i64 = 40
53const NX_TOK_EQ: i64 = 41 // ==
54const NX_TOK_NE: i64 = 42 // !=
55const NX_TOK_LT: i64 = 43
56const NX_TOK_LE: i64 = 44
57const NX_TOK_GT: i64 = 45
58const NX_TOK_GE: i64 = 46
59const NX_TOK_PLUS: i64 = 50
60const NX_TOK_MINUS: i64 = 51
61const NX_TOK_STAR: i64 = 52
62const NX_TOK_SLASH: i64 = 53
63const NX_TOK_PERCENT: i64 = 54
64const NX_TOK_AMP: i64 = 55
65const NX_TOK_PIPE: i64 = 56
66const NX_TOK_CARET: i64 = 57
67const NX_TOK_TILDE: i64 = 58
68const NX_TOK_LSHIFT: i64 = 59
69const NX_TOK_RSHIFT: i64 = 60
70const NX_TOK_DOT: i64 = 61
71
72struct NxToken {
73 kind: i64,
74 start: i64,
75 len: i64,
76 int_val: i64,
77}
78
79struct NxNLexer {
80 src: *u8,
81 src_len: i64,
82 pos: i64,
83 tokens: *NxToken,
84 n_tokens: i64,
85 cap: i64,
86}
87
88func nx_nlex_new(src: *u8, src_len: i64, cap: i64) -> *NxNLexer {
89 let raw: *u8 = sys_mmap(48)
90 let l: *NxNLexer = raw as *NxNLexer
91 l.src = src
92 l.src_len = src_len
93 l.pos = 0
94 l.tokens = sys_mmap(cap * 32) as *NxToken
95 l.n_tokens = 0
96 l.cap = cap
97 return l
98}
99
100func nx_nlex_token_at(l: *NxNLexer, i: i64) -> *NxToken {
101 return (l.tokens as i64 + i * 32) as *NxToken
102}
103
104func nx_nlex_is_digit(c: i64) -> i64 {
105 if c < 48 { return 0 }
106 if c > 57 { return 0 }
107 return 1
108}
109
110func nx_nlex_is_hex(c: i64) -> i64 {
111 if nx_nlex_is_digit(c) == 1 { return 1 }
112 if c >= 97 { if c <= 102 { return 1 } }
113 if c >= 65 { if c <= 70 { return 1 } }
114 return 0
115}
116
117func nx_nlex_is_alpha(c: i64) -> i64 {
118 if c == 95 { return 1 }
119 if c >= 65 { if c <= 90 { return 1 } }
120 if c >= 97 { if c <= 122 { return 1 } }
121 return 0
122}
123
124func nx_nlex_is_alnum(c: i64) -> i64 {
125 if nx_nlex_is_alpha(c) == 1 { return 1 }
126 if nx_nlex_is_digit(c) == 1 { return 1 }
127 return 0
128}
129
130func nx_nlex_is_ws(c: i64) -> i64 {
131 if c == 32 { return 1 }
132 if c == 9 { return 1 }
133 if c == 10 { return 1 }
134 if c == 13 { return 1 }
135 return 0
136}
137
138func nx_nlex_peek(l: *NxNLexer, off: i64) -> i64 {
139 let p: i64 = l.pos + off
140 if p >= l.src_len { return -1 }
141 return l.src[p] as i64
142}
143
144func nx_nlex_advance(l: *NxNLexer) -> i64 {
145 if l.pos >= l.src_len { return -1 }
146 let c: i64 = l.src[l.pos] as i64
147 l.pos = l.pos + 1
148 return c
149}
150
151func nx_nlex_emit(l: *NxNLexer, kind: i64, start: i64, len: i64, int_val: i64) -> i64 {
152 if l.n_tokens >= l.cap { return -1 }
153 let t: *NxToken = nx_nlex_token_at(l, l.n_tokens)
154 t.kind = kind
155 t.start = start
156 t.len = len
157 t.int_val = int_val
158 l.n_tokens = l.n_tokens + 1
159 return 0
160}
161
162// Compare substring against ASCII byte sequence.
163func nx_nlex_streq(l: *NxNLexer, start: i64, len: i64, b0: i64, b1: i64, b2: i64, b3: i64, b4: i64, b5: i64, expected_len: i64) -> i64 {
164 if len != expected_len { return 0 }
165 if len >= 1 { if l.src[start + 0] as i64 != b0 { return 0 } }
166 if len >= 2 { if l.src[start + 1] as i64 != b1 { return 0 } }
167 if len >= 3 { if l.src[start + 2] as i64 != b2 { return 0 } }
168 if len >= 4 { if l.src[start + 3] as i64 != b3 { return 0 } }
169 if len >= 5 { if l.src[start + 4] as i64 != b4 { return 0 } }
170 if len >= 6 { if l.src[start + 5] as i64 != b5 { return 0 } }
171 return 1
172}
173
174func nx_nlex_classify(l: *NxNLexer, start: i64, len: i64) -> i64 {
175 // func (102 117 110 99)
176 if nx_nlex_streq(l, start, len, 102, 117, 110, 99, 0, 0, 4) == 1 { return NX_TOK_FUNC }
177 // let
178 if nx_nlex_streq(l, start, len, 108, 101, 116, 0, 0, 0, 3) == 1 { return NX_TOK_LET }
179 // var
180 if nx_nlex_streq(l, start, len, 118, 97, 114, 0, 0, 0, 3) == 1 { return NX_TOK_VAR }
181 // if
182 if nx_nlex_streq(l, start, len, 105, 102, 0, 0, 0, 0, 2) == 1 { return NX_TOK_IF }
183 // else
184 if nx_nlex_streq(l, start, len, 101, 108, 115, 101, 0, 0, 4) == 1 { return NX_TOK_ELSE }
185 // while
186 if nx_nlex_streq(l, start, len, 119, 104, 105, 108, 101, 0, 5) == 1 { return NX_TOK_WHILE }
187 // return
188 if nx_nlex_streq(l, start, len, 114, 101, 116, 117, 114, 110, 6) == 1 { return NX_TOK_RETURN }
189 // const
190 if nx_nlex_streq(l, start, len, 99, 111, 110, 115, 116, 0, 5) == 1 { return NX_TOK_CONST }
191 // struct
192 if nx_nlex_streq(l, start, len, 115, 116, 114, 117, 99, 116, 6) == 1 { return NX_TOK_STRUCT }
193 // import
194 if nx_nlex_streq(l, start, len, 105, 109, 112, 111, 114, 116, 6) == 1 { return NX_TOK_IMPORT }
195 // as
196 if nx_nlex_streq(l, start, len, 97, 115, 0, 0, 0, 0, 2) == 1 { return NX_TOK_AS }
197 return NX_TOK_IDENT
198}
199
200func nx_nlex_skip_ws_and_comments(l: *NxNLexer) -> i64 {
201 let outer: *NxLoopFrame = nx_loop_begin(l.src_len + 1)
202 while nx_loop_step(outer) == 1 {
203 let c: i64 = nx_nlex_peek(l, 0)
204 if c < 0 { return 0 }
205 if nx_nlex_is_ws(c) == 1 {
206 nx_nlex_advance(l)
207 } else {
208 // // comment
209 if c == 47 {
210 if nx_nlex_peek(l, 1) == 47 {
211 let inner: *NxLoopFrame = nx_loop_begin(l.src_len + 1)
212 while nx_loop_step(inner) == 1 {
213 let d: i64 = nx_nlex_advance(l)
214 if d < 0 { nx_loop_break(inner) }
215 if d == 10 { nx_loop_break(inner) }
216 }
217 }
218 if nx_nlex_peek(l, 0) != 47 { nx_loop_break(outer) }
219 }
220 if c != 47 { nx_loop_break(outer) }
221 }
222 }
223 return 0
224}
225
226func nx_nlex_lex_ident(l: *NxNLexer, tok_start: i64) -> i64 {
227 let lp: *NxLoopFrame = nx_loop_begin(l.src_len + 1)
228 while nx_loop_step(lp) == 1 {
229 let cc: i64 = nx_nlex_peek(l, 0)
230 if cc < 0 {
231 nx_loop_break(lp)
232 } else {
233 if nx_nlex_is_alnum(cc) == 1 {
234 nx_nlex_advance(l)
235 } else {
236 nx_loop_break(lp)
237 }
238 }
239 }
240 let id_len: i64 = l.pos - tok_start
241 let kind: i64 = nx_nlex_classify(l, tok_start, id_len)
242 nx_nlex_emit(l, kind, tok_start, id_len, 0)
243 return 0
244}
245
246func nx_nlex_lex_int(l: *NxNLexer, tok_start: i64) -> i64 {
247 var val: i64 = 0
248 let first: i64 = nx_nlex_peek(l, 0)
249 if first == 48 { // 0
250 if nx_nlex_peek(l, 1) == 120 { // x
251 nx_nlex_advance(l)
252 nx_nlex_advance(l)
253 // Bound: max 16 hex digits in an i64 + slack.
254 let hlp: *NxLoopFrame = nx_loop_begin(32)
255 while nx_loop_step(hlp) == 1 {
256 let cc: i64 = nx_nlex_peek(l, 0)
257 if cc < 0 {
258 nx_loop_break(hlp)
259 } else {
260 if nx_nlex_is_hex(cc) == 1 {
261 var digit: i64 = 0
262 if cc <= 57 { digit = cc - 48 }
263 if cc > 57 {
264 if cc <= 70 { digit = cc - 55 }
265 if cc > 70 { digit = cc - 87 }
266 }
267 val = val * 16 + digit
268 nx_nlex_advance(l)
269 } else {
270 nx_loop_break(hlp)
271 }
272 }
273 }
274 nx_nlex_emit(l, NX_TOK_INT, tok_start, l.pos - tok_start, val)
275 return 0
276 }
277 }
278 // Bound: max 19 decimal digits in i64 + slack.
279 let dlp: *NxLoopFrame = nx_loop_begin(32)
280 while nx_loop_step(dlp) == 1 {
281 let cc: i64 = nx_nlex_peek(l, 0)
282 if cc < 0 {
283 nx_loop_break(dlp)
284 } else {
285 if nx_nlex_is_digit(cc) == 1 {
286 val = val * 10 + (cc - 48)
287 nx_nlex_advance(l)
288 } else {
289 nx_loop_break(dlp)
290 }
291 }
292 }
293 nx_nlex_emit(l, NX_TOK_INT, tok_start, l.pos - tok_start, val)
294 return 0
295}
296
297func nx_nlex_lex_punct(l: *NxNLexer, c0: i64, tok_start: i64) -> i64 {
298 nx_nlex_advance(l)
299 var kind: i64 = -1
300 if c0 == 40 { kind = NX_TOK_LPAREN }
301 if c0 == 41 { kind = NX_TOK_RPAREN }
302 if c0 == 123 { kind = NX_TOK_LBRACE }
303 if c0 == 125 { kind = NX_TOK_RBRACE }
304 if c0 == 91 { kind = NX_TOK_LBRACK }
305 if c0 == 93 { kind = NX_TOK_RBRACK }
306 if c0 == 44 { kind = NX_TOK_COMMA }
307 if c0 == 59 { kind = NX_TOK_SEMI }
308 if c0 == 58 { kind = NX_TOK_COLON }
309 if c0 == 46 { kind = NX_TOK_DOT }
310 if c0 == 43 { kind = NX_TOK_PLUS }
311 if c0 == 42 { kind = NX_TOK_STAR }
312 if c0 == 37 { kind = NX_TOK_PERCENT }
313 if c0 == 38 { kind = NX_TOK_AMP }
314 if c0 == 124 { kind = NX_TOK_PIPE }
315 if c0 == 94 { kind = NX_TOK_CARET }
316 if c0 == 126 { kind = NX_TOK_TILDE }
317 if c0 == 47 { kind = NX_TOK_SLASH }
318 if c0 == 45 {
319 if nx_nlex_peek(l, 0) == 62 { nx_nlex_advance(l); kind = NX_TOK_ARROW }
320 if kind < 0 { kind = NX_TOK_MINUS }
321 }
322 if c0 == 61 {
323 if nx_nlex_peek(l, 0) == 61 { nx_nlex_advance(l); kind = NX_TOK_EQ }
324 if kind < 0 { kind = NX_TOK_ASSIGN }
325 }
326 if c0 == 33 {
327 if nx_nlex_peek(l, 0) == 61 { nx_nlex_advance(l); kind = NX_TOK_NE }
328 }
329 if c0 == 60 {
330 if nx_nlex_peek(l, 0) == 61 { nx_nlex_advance(l); kind = NX_TOK_LE }
331 if nx_nlex_peek(l, 0) == 60 { nx_nlex_advance(l); kind = NX_TOK_LSHIFT }
332 if kind < 0 { kind = NX_TOK_LT }
333 }
334 if c0 == 62 {
335 if nx_nlex_peek(l, 0) == 61 { nx_nlex_advance(l); kind = NX_TOK_GE }
336 if nx_nlex_peek(l, 0) == 62 { nx_nlex_advance(l); kind = NX_TOK_RSHIFT }
337 if kind < 0 { kind = NX_TOK_GT }
338 }
339 if kind >= 0 {
340 nx_nlex_emit(l, kind, tok_start, l.pos - tok_start, 0)
341 }
342 return 0
343}
344
345func nx_nlex_run(l: *NxNLexer) -> i64 {
346 // Bound: cap is the max tokens the lexer can emit.
347 let lp: *NxLoopFrame = nx_loop_begin(l.cap)
348 while nx_loop_step(lp) == 1 {
349 nx_nlex_skip_ws_and_comments(l)
350 let c0: i64 = nx_nlex_peek(l, 0)
351 if c0 < 0 {
352 nx_nlex_emit(l, NX_TOK_EOF, l.pos, 0, 0)
353 nx_loop_break(lp)
354 } else {
355 let tok_start: i64 = l.pos
356 if nx_nlex_is_alpha(c0) == 1 {
357 nx_nlex_lex_ident(l, tok_start)
358 } else {
359 if nx_nlex_is_digit(c0) == 1 {
360 nx_nlex_lex_int(l, tok_start)
361 } else {
362 nx_nlex_lex_punct(l, c0, tok_start)
363 }
364 }
365 }
366 }
367 return l.n_tokens
368}