lex.nx source

1// lex.nx -- NishiLang source tokenizer, ported from lex.c. 2// 3// Accepts a null-terminated source buffer and produces a stream of 4// Tok records. Mirrors lex.c's token layout (kind + line/col + 5// either int_val or text[]). Same keyword set. 6// 7// Single-pass; zero allocations per token beyond the growing token 8// array. Extends at 2x when full. Whitespace + // comments are 9// skipped. Numbers support 0x/0b/0o prefixes with underscore 10// separators. String literals read bytes until closing `"`. 11 12// ---- syscalls ---- 13 14// ---- token kinds (subset of lex.h, integer constants) ---- 15// 16// 0 EOF 17// 1 INT 18// 2 IDENT 19// 3 STRING 20// 10 FUNC 11 LET 12 VAR 13 CONST 21// 14 STATIC 15 IF 16 ELSE 17 WHILE 22// 18 LOOP 19 FOR 20 IN 21 BREAK 23// 22 CONTINUE 23 RETURN 24 TRUE 25 FALSE 24// 26 STRUCT 27 ENUM 28 MATCH 29 COMPTIME 25// 30 EXTERN 31 AS 26// 40 PLUS 41 MINUS 42 STAR 43 SLASH 27// 44 PERCENT 45 ASSIGN 46 EQ 47 NE 28// 48 LT 49 GT 50 LE 51 GE 29// 52 AND_AND 53 OR_OR 54 BANG 30// 55 AMP 56 PIPE 57 CARET 58 TILDE 31// 59 SHL 60 SHR 32// 61 ARROW 62 DOT_DOT 63 DOT 64 COLON 33// 65 SEMI 66 COMMA 34// 67 LPAREN 68 RPAREN 69 LBRACE 70 RBRACE 35// 71 LBRACKET 72 RBRACKET 73 AT 74 HASH 36 37// ---- Tok struct ---- 38// 39// Laid out to match lex.h approximately. text[] is fixed-size 40// 64-byte for identifiers and string literals; that's the same 41// MAX_IDENT the C lexer uses. 42 43import "syscalls.nx" 44import "types.nx" 45import "lex_kinds.nx" 46// Tok struct + TOK_BYTES const live in lex_kinds.nx (canonical 47// home). Removed duplicate definition here 2026-04-26 per 48// nx_type_identity_check.sh -- duplicate Type instances at 49// compile-time create T#selfhost-006-class bugs even when 50// structurally identical. 51 52// ---- character classes ---- 53 54func is_alpha(c: i64) -> i64 { 55 if c >= 0x41 { if c <= 0x5A { return 1 } } 56 if c >= 0x61 { if c <= 0x7A { return 1 } } 57 if c == 0x5F { return 1 } 58 return 0 59} 60func is_digit(c: i64) -> i64 { 61 if c >= 0x30 { if c <= 0x39 { return 1 } } 62 return 0 63} 64func is_hexdigit(c: i64) -> i64 { 65 if is_digit(c) { return 1 } 66 if c >= 0x41 { if c <= 0x46 { return 1 } } 67 if c >= 0x61 { if c <= 0x66 { return 1 } } 68 return 0 69} 70func hex_val(c: i64) -> i64 { 71 if c >= 0x30 { if c <= 0x39 { return c - 0x30 } } 72 if c >= 0x41 { if c <= 0x46 { return c - 0x37 } } 73 if c >= 0x61 { if c <= 0x66 { return c - 0x57 } } 74 return 0 75} 76func is_alnum(c: i64) -> i64 { 77 if is_alpha(c) { return 1 } 78 if is_digit(c) { return 1 } 79 return 0 80} 81 82// ---- keyword table ---- 83// 84// Linear scan; we have ~20 keywords. Returns token kind or -1. 85 86func keyword_lookup(t: *u8, len: i64) -> i64 { 87 if streq_n(t, "func", len) { return 10 } 88 if streq_n(t, "let", len) { return 11 } 89 if streq_n(t, "var", len) { return 12 } 90 if streq_n(t, "const", len) { return 13 } 91 if streq_n(t, "static", len) { return 14 } 92 if streq_n(t, "if", len) { return 15 } 93 if streq_n(t, "else", len) { return 16 } 94 if streq_n(t, "while", len) { return 17 } 95 if streq_n(t, "loop", len) { return 18 } 96 if streq_n(t, "for", len) { return 19 } 97 if streq_n(t, "in", len) { return 20 } 98 if streq_n(t, "break", len) { return 21 } 99 if streq_n(t, "continue", len) { return 22 } 100 if streq_n(t, "return", len) { return 23 } 101 if streq_n(t, "true", len) { return 24 } 102 if streq_n(t, "false", len) { return 25 } 103 if streq_n(t, "struct", len) { return 26 } 104 if streq_n(t, "enum", len) { return 27 } 105 if streq_n(t, "match", len) { return 28 } 106 if streq_n(t, "comptime", len) { return 29 } 107 if streq_n(t, "extern", len) { return 30 } 108 if streq_n(t, "as", len) { return 31 } 109 return 2 // generic IDENT 110} 111 112// ---- Lex state ---- 113 114struct Lex { 115 src: *u8, 116 pos: i64, 117 line: i64, 118 col: i64, 119 tokens: *Tok, 120 n_tokens: i64, 121 cap: i64, 122} 123 124// Push a token into the growing array. Simple, no geometric 125// growth for now: caller pre-allocates enough. 126 127func push_tok(L: *Lex, t: *Tok) -> i64 { 128 let base: i64 = L.tokens as i64 129 let slot: *Tok = (base + L.n_tokens * TOK_BYTES) as *Tok 130 slot.kind = t.kind 131 slot.line = t.line 132 slot.col = t.col 133 slot.int_val = t.int_val 134 slot.text0 = t.text0 135 slot.text1 = t.text1 136 slot.text2 = t.text2 137 slot.text3 = t.text3 138 slot.text4 = t.text4 139 slot.text5 = t.text5 140 slot.text6 = t.text6 141 slot.text7 = t.text7 142 slot.str_data = t.str_data 143 slot.str_len = t.str_len 144 L.n_tokens = L.n_tokens + 1 145 return 0 146} 147 148func advance(L: *Lex) -> i64 { 149 let s: *u8 = L.src 150 let c: i64 = s[L.pos] 151 if c == 0x0A { 152 L.line = L.line + 1 153 L.col = 1 154 } 155 if c != 0x0A { 156 if c != 0 { L.col = L.col + 1 } 157 } 158 L.pos = L.pos + 1 159 return c 160} 161func peek(L: *Lex) -> i64 { 162 let s: *u8 = L.src 163 return s[L.pos] 164} 165func peek2(L: *Lex) -> i64 { 166 let s: *u8 = L.src 167 if s[L.pos] == 0 { return 0 } 168 return s[L.pos + 1] 169} 170 171// ---- sub-lexers ---- 172 173func skip_ws_comments(L: *Lex) -> i64 { 174 var go: i64 = 1 175 while go { 176 let c: i64 = peek(L) 177 if c == 0x20 { advance(L) } 178 if c == 0x09 { advance(L) } 179 if c == 0x0A { advance(L) } 180 if c == 0x0D { advance(L) } 181 if c == 0x2F { 182 if peek2(L) == 0x2F { 183 // Line comment 184 while peek(L) != 0 { 185 if peek(L) == 0x0A { break } 186 advance(L) 187 } 188 continue 189 } 190 // '/' not followed by another '/' -- division operator. 191 // Hand it back to the outer lexer to tokenise. Setting 192 // go=0 directly here avoids an infinite loop through the 193 // cascade below (which treats '/' as whitespace-like and 194 // never sets go=0, looping forever on a bare slash). 195 go = 0 196 continue 197 } 198 if c != 0x20 { 199 if c != 0x09 { 200 if c != 0x0A { 201 if c != 0x0D { 202 go = 0 203 } 204 } 205 } 206 } 207 } 208 return 0 209} 210 211func lex_ident_or_kw(L: *Lex) -> i64 { 212 let tr: *u8 = sys_mmap(TOK_BYTES) 213 let t: *Tok = tr as *Tok 214 t.kind = 2 215 t.line = L.line 216 t.col = L.col 217 t.int_val = 0 218 t.text0 = 0; t.text1 = 0; t.text2 = 0; t.text3 = 0 219 t.text4 = 0; t.text5 = 0; t.text6 = 0; t.text7 = 0 220 let text: *u8 = tok_text_ptr(t) 221 var i: i64 = 0 222 while i < 63 { 223 let c: i64 = peek(L) 224 if is_alnum(c) { 225 text[i] = c & 0xFF 226 advance(L) 227 i = i + 1 228 } 229 if is_alnum(c) == 0 { break } 230 } 231 text[i] = 0 232 t.kind = keyword_lookup(text, i) 233 push_tok(L, t) 234 return 0 235} 236 237func lex_number(L: *Lex) -> i64 { 238 let tr: *u8 = sys_mmap(TOK_BYTES) 239 let t: *Tok = tr as *Tok 240 t.kind = 1 241 t.line = L.line 242 t.col = L.col 243 t.int_val = 0 244 t.text0 = 0; t.text1 = 0; t.text2 = 0; t.text3 = 0 245 t.text4 = 0; t.text5 = 0; t.text6 = 0; t.text7 = 0 246 247 var n: i64 = 0 248 let c0: i64 = peek(L) 249 if c0 == 0x30 { 250 let c1: i64 = peek2(L) 251 if c1 == 0x78 { // 0x hex 252 advance(L); advance(L) 253 var go: i64 = 1 254 while go { 255 let c: i64 = peek(L) 256 if c == 0x5F { advance(L); continue } 257 if is_hexdigit(c) { 258 n = (n << 4) | hex_val(c) 259 advance(L) 260 } 261 if is_hexdigit(c) == 0 { if c != 0x5F { go = 0 } } 262 } 263 t.int_val = n 264 push_tok(L, t) 265 return 0 266 } 267 if c1 == 0x62 { // 0b binary 268 advance(L); advance(L) 269 var go: i64 = 1 270 while go { 271 let c: i64 = peek(L) 272 if c == 0x5F { advance(L); continue } 273 if c == 0x30 { n = (n << 1); advance(L); continue } 274 if c == 0x31 { n = (n << 1) | 1; advance(L); continue } 275 go = 0 276 } 277 t.int_val = n 278 push_tok(L, t) 279 return 0 280 } 281 if c1 == 0x6F { // 0o octal 282 advance(L); advance(L) 283 var go: i64 = 1 284 while go { 285 let c: i64 = peek(L) 286 if c == 0x5F { advance(L); continue } 287 if c >= 0x30 { 288 if c <= 0x37 { 289 n = (n << 3) | (c - 0x30) 290 advance(L) 291 continue 292 } 293 } 294 go = 0 295 } 296 t.int_val = n 297 push_tok(L, t) 298 return 0 299 } 300 } 301 // Decimal. Consume whole-part digits. 302 var go: i64 = 1 303 while go { 304 let c: i64 = peek(L) 305 if c == 0x5F { advance(L); continue } 306 if is_digit(c) { 307 n = n * 10 + (c - 0x30) 308 advance(L) 309 continue 310 } 311 go = 0 312 } 313 314 // Float split: if the current char is '.' AND the next char is a 315 // digit (not a method call like `x.foo` or range like `0..10`), 316 // switch to TK_FLOAT and consume the fractional part. 317 let c_after: i64 = peek(L) 318 if c_after == 0x2E { 319 let c_next: i64 = peek2(L) 320 if is_digit(c_next) { 321 advance(L) // consume '.' 322 var frac: i64 = 0 323 var frac_digits: i64 = 0 324 var fgo: i64 = 1 325 while fgo { 326 let c: i64 = peek(L) 327 if c == 0x5F { advance(L); continue } 328 if is_digit(c) { 329 frac = frac * 10 + (c - 0x30) 330 frac_digits = frac_digits + 1 331 advance(L) 332 continue 333 } 334 fgo = 0 335 } 336 // Optional 'f' / 'F' suffix -- now distinguished: 337 // `1.5f32` / `1.5F32` -> TK_FLOAT_F32 (explicit single) 338 // `1.5f64` / `1.5F64` -> TK_FLOAT (explicit default) 339 // `1.5f` / `1.5` -> TK_FLOAT (default = f64) 340 // Lexer consumes the suffix bytes; parse.nx dispatches on 341 // the resulting kind. 342 var sfx_kind: i64 = 4 // TK_FLOAT (default f64) 343 let c_sfx: i64 = peek(L) 344 if c_sfx == 0x66 { 345 advance(L) 346 let n1: i64 = L.src[L.pos] 347 let n2: i64 = L.src[L.pos + 1] 348 if n1 == 0x33 { 349 if n2 == 0x32 { 350 advance(L); advance(L) 351 sfx_kind = 5 // TK_FLOAT_F32 352 } 353 } 354 if n1 == 0x36 { 355 if n2 == 0x34 { 356 advance(L); advance(L) 357 // sfx_kind stays at TK_FLOAT (default) 358 } 359 } 360 } 361 if c_sfx == 0x46 { 362 advance(L) 363 let n1: i64 = L.src[L.pos] 364 let n2: i64 = L.src[L.pos + 1] 365 if n1 == 0x33 { 366 if n2 == 0x32 { 367 advance(L); advance(L) 368 sfx_kind = 5 369 } 370 } 371 if n1 == 0x36 { 372 if n2 == 0x34 { 373 advance(L); advance(L) 374 } 375 } 376 } 377 t.kind = sfx_kind 378 t.int_val = n 379 t.text0 = frac 380 t.text1 = frac_digits 381 push_tok(L, t) 382 return 0 383 } 384 } 385 386 t.int_val = n 387 push_tok(L, t) 388 return 0 389} 390 391func lex_string(L: *Lex) -> i64 { 392 let tr: *u8 = sys_mmap(TOK_BYTES) 393 let t: *Tok = tr as *Tok 394 t.kind = 3 395 t.line = L.line 396 t.col = L.col 397 t.int_val = 0 398 t.text0 = 0; t.text1 = 0; t.text2 = 0; t.text3 = 0 399 t.text4 = 0; t.text5 = 0; t.text6 = 0; t.text7 = 0 400 t.str_data = 0 401 t.str_len = 0 402 advance(L) // opening " 403 404 // Pre-scan to determine the raw length (upper bound; escapes 405 // shrink it). Self-host pipeline NEEDS escape processing here: 406 // string literals like "nxc: expand start\n" arrive with the 407 // backslash-n two-byte sequence; without processing, sys_write 408 // emits the literal `\` + `n` instead of a newline, so anything 409 // that uses a string literal at runtime miscompiles. 410 let scan_start: i64 = L.pos 411 var scan: i64 = L.pos 412 let src_chk: *u8 = L.src 413 while src_chk[scan] != 0 { 414 if src_chk[scan] == 0x22 { break } 415 scan = scan + 1 416 } 417 let raw_len: i64 = scan - scan_start 418 419 let buf: *u8 = sys_mmap(raw_len + 16) 420 421 // Inline copy + populate text[] prefix for legacy consumers. 422 let text: *u8 = tok_text_ptr(t) 423 var i: i64 = 0 424 while peek(L) != 0 { 425 if peek(L) == 0x22 { break } 426 var c: i64 = peek(L) & 0xFF 427 if c == 0x5C { 428 // Escape sequence: peek next char, advance past both. 429 advance(L) 430 let nx: i64 = peek(L) & 0xFF 431 if nx == 0x6E { c = 0x0A } // \n 432 if nx == 0x74 { c = 0x09 } // \t 433 if nx == 0x72 { c = 0x0D } // \r 434 if nx == 0x5C { c = 0x5C } // \\ 435 if nx == 0x22 { c = 0x22 } // \" 436 if nx == 0x30 { c = 0x00 } // \0 (single, no octal) 437 // Other escapes: drop the backslash + use the literal char 438 // (matches gas-style "unknown escape passes literal"). 439 if nx != 0x6E { if nx != 0x74 { if nx != 0x72 { 440 if nx != 0x5C { if nx != 0x22 { if nx != 0x30 { 441 c = nx 442 } } } } } } 443 } 444 buf[i] = c 445 if i < 63 { text[i] = c } 446 i = i + 1 447 advance(L) 448 } 449 buf[i] = 0 450 if i < 63 { text[i] = 0 } 451 452 t.str_data = buf as i64 453 t.str_len = i 454 455 if peek(L) == 0x22 { advance(L) } 456 push_tok(L, t) 457 return 0 458} 459 460func emit_punct(L: *Lex, kind: i64, len: i64) -> i64 { 461 let tr: *u8 = sys_mmap(TOK_BYTES) 462 let t: *Tok = tr as *Tok 463 t.kind = kind 464 t.line = L.line 465 t.col = L.col 466 t.int_val = 0 467 t.text0 = 0; t.text1 = 0; t.text2 = 0; t.text3 = 0 468 t.text4 = 0; t.text5 = 0; t.text6 = 0; t.text7 = 0 469 var k: i64 = 0 470 while k < len { 471 advance(L) 472 k = k + 1 473 } 474 push_tok(L, t) 475 return 0 476} 477 478// ---- public entry ---- 479// 480// Tokenise the entire source, terminate with an EOF token, return 481// the tokens array. 482 483func lex_source(src: *u8, cap: i64) -> *Tok { 484 let toks_raw: *u8 = sys_mmap(cap * TOK_BYTES + TOK_BYTES) 485 let toks: *Tok = toks_raw as *Tok 486 487 let L_raw: *u8 = sys_mmap(64) 488 let L: *Lex = L_raw as *Lex 489 L.src = src 490 L.pos = 0 491 L.line = 1 492 L.col = 1 493 L.tokens = toks 494 L.n_tokens = 0 495 L.cap = cap 496 497 var go: i64 = 1 498 while go { 499 skip_ws_comments(L) 500 let c: i64 = peek(L) 501 if c == 0 { 502 go = 0 503 continue 504 } 505 if is_alpha(c) { lex_ident_or_kw(L); continue } 506 if c == 0x5F { lex_ident_or_kw(L); continue } 507 if is_digit(c) { lex_number(L); continue } 508 if c == 0x22 { lex_string(L); continue } 509 510 let c2: i64 = peek2(L) 511 // Multi-char first 512 if c == 0x2D { 513 if c2 == 0x3E { emit_punct(L, 61, 2); continue } // -> 514 } 515 if c == 0x3D { 516 if c2 == 0x3D { emit_punct(L, 46, 2); continue } // == 517 } 518 if c == 0x21 { 519 if c2 == 0x3D { emit_punct(L, 47, 2); continue } // != 520 } 521 if c == 0x3C { 522 if c2 == 0x3D { emit_punct(L, 50, 2); continue } // <= 523 if c2 == 0x3C { emit_punct(L, 59, 2); continue } // << 524 } 525 if c == 0x3E { 526 if c2 == 0x3D { emit_punct(L, 51, 2); continue } // >= 527 if c2 == 0x3E { emit_punct(L, 60, 2); continue } // >> 528 } 529 if c == 0x26 { 530 if c2 == 0x26 { emit_punct(L, 52, 2); continue } // && 531 } 532 if c == 0x7C { 533 if c2 == 0x7C { emit_punct(L, 53, 2); continue } // || 534 } 535 if c == 0x2E { 536 if c2 == 0x2E { emit_punct(L, 62, 2); continue } // .. 537 } 538 if c == 0x3A { 539 if c2 == 0x3A { emit_punct(L, 75, 2); continue } // :: 540 } 541 if c == 0x3D { 542 if c2 == 0x3E { emit_punct(L, 76, 2); continue } // => 543 } 544 545 if c == 0x2B { emit_punct(L, 40, 1); continue } 546 if c == 0x2D { emit_punct(L, 41, 1); continue } 547 if c == 0x2A { emit_punct(L, 42, 1); continue } 548 if c == 0x2F { emit_punct(L, 43, 1); continue } 549 if c == 0x25 { emit_punct(L, 44, 1); continue } 550 if c == 0x3D { emit_punct(L, 45, 1); continue } 551 if c == 0x3C { emit_punct(L, 48, 1); continue } 552 if c == 0x3E { emit_punct(L, 49, 1); continue } 553 if c == 0x21 { emit_punct(L, 54, 1); continue } 554 if c == 0x26 { emit_punct(L, 55, 1); continue } 555 if c == 0x7C { emit_punct(L, 56, 1); continue } 556 if c == 0x5E { emit_punct(L, 57, 1); continue } 557 if c == 0x7E { emit_punct(L, 58, 1); continue } 558 if c == 0x2E { emit_punct(L, 63, 1); continue } 559 if c == 0x3A { emit_punct(L, 64, 1); continue } 560 if c == 0x3B { emit_punct(L, 65, 1); continue } 561 if c == 0x2C { emit_punct(L, 66, 1); continue } 562 if c == 0x28 { emit_punct(L, 67, 1); continue } 563 if c == 0x29 { emit_punct(L, 68, 1); continue } 564 if c == 0x7B { emit_punct(L, 69, 1); continue } 565 if c == 0x7D { emit_punct(L, 70, 1); continue } 566 if c == 0x5B { emit_punct(L, 71, 1); continue } 567 if c == 0x5D { emit_punct(L, 72, 1); continue } 568 if c == 0x40 { emit_punct(L, 73, 1); continue } 569 if c == 0x23 { emit_punct(L, 74, 1); continue } 570 if c == 0x3F { emit_punct(L, 77, 1); continue } // ? TK_QUESTION (ternary) 571 572 // Unknown: skip to prevent infinite loop. 573 advance(L) 574 } 575 576 // Terminator EOF 577 let last_raw: *u8 = sys_mmap(TOK_BYTES) 578 let last: *Tok = last_raw as *Tok 579 last.kind = 0 580 last.line = L.line 581 last.col = L.col 582 last.int_val = 0 583 last.text0 = 0; last.text1 = 0; last.text2 = 0; last.text3 = 0 584 last.text4 = 0; last.text5 = 0; last.text6 = 0; last.text7 = 0 585 push_tok(L, last) 586 587 return toks 588} 589 590// Library only; self-test lives in lex_test.nx.