nx_lex_kinds.nx source

1// lex_kinds.nx -- Tok record + named TokenKind constants. 2// 3// Shared by lex.nx (producer) and parse.nx (consumer). Both used 4// to carry their own copy of the `Tok` struct and compare 5// `t.kind == 67` with a comment saying "LPAREN"; with this module 6// imported they use `t.kind == TK_LPAREN` directly. 7// 8// Numbers below must match the comment table in lex.nx and the 9// emission sites in lex.nx's lexer -- do not renumber without 10// auditing both callers and the existing test fixtures. 11 12// ---- Tok record --------------------------------------------------- 13// 14// text[] (64 bytes) lives inline as 8 i64 words starting at offset 15// 32. Access via tok_text_ptr() defined by lex.nx. 16 17struct Tok { 18 kind: i64, 19 line: i64, 20 col: i64, 21 int_val: i64, 22 text0: i64, 23 text1: i64, 24 text2: i64, 25 text3: i64, 26 text4: i64, 27 text5: i64, 28 text6: i64, 29 text7: i64, 30 // Overflow buffer for TK_STRING longer than 63 bytes; mirrors 31 // the str_data/str_len fields added to nxc2/lex.h. 32 str_data: i64, 33 str_len: i64, 34} 35 36const TOK_BYTES: i64 = 112 37 38// ---- literals ----------------------------------------------------- 39 40const TK_EOF: i64 = 0 41const TK_INT: i64 = 1 42const TK_IDENT: i64 = 2 43const TK_STRING: i64 = 3 44// Floating-point literal. Lexer splits the number into whole / 45// fractional parts and stores both in the Tok: 46// t.int_val = whole part 47// t.text0 = fractional digits as i64 (e.g. "14" in "3.14" -> 14) 48// t.text1 = number of fractional digits (2 in the example) 49// Parser converts to IEEE 754 via quant.nx when building the IR. 50// Keeping whole/frac as two ints in the token avoids pulling fp 51// conversion into the lexer and lets parse.nx decide f32 vs f64 52// based on the declared variable type. 53const TK_FLOAT: i64 = 4 54 55// Explicit-f32 float literal -- written `1.5f32` (or `1.5F32`) in 56// source. Same int_val / text0 / text1 packing as TK_FLOAT (whole / 57// frac_num / frac_digits); the kind tells parse.nx to use 58// fp32_from_parts + TY_F32 instead of the default fp64 path. 59// `1.5f64` is allowed as an explicit-default and emits TK_FLOAT. 60const TK_FLOAT_F32: i64 = 5 61 62// ---- keywords ----------------------------------------------------- 63 64const TK_FUNC: i64 = 10 65const TK_LET: i64 = 11 66const TK_VAR: i64 = 12 67const TK_CONST: i64 = 13 68const TK_STATIC: i64 = 14 69const TK_IF: i64 = 15 70const TK_ELSE: i64 = 16 71const TK_WHILE: i64 = 17 72const TK_LOOP: i64 = 18 73const TK_FOR: i64 = 19 74const TK_IN: i64 = 20 75const TK_BREAK: i64 = 21 76const TK_CONTINUE: i64 = 22 77const TK_RETURN: i64 = 23 78const TK_TRUE: i64 = 24 79const TK_FALSE: i64 = 25 80const TK_STRUCT: i64 = 26 81const TK_ENUM: i64 = 27 82// @priv VISIBILITY ATTRIBUTE. Ported from the laptop SSOT 2026-07-31 -- buildroot had NO const at slot 33 83// (verified free before claiming it), so the trees can converge without renumbering anything. 84const TK_PRIV: i64 = 33 85const TK_MATCH: i64 = 28 86const TK_COMPTIME: i64 = 29 87const TK_EXTERN: i64 = 30 88const TK_AS: i64 = 31 89// V-LANGEXT M2: if-then-else as expression keyword. Distinct from 90// statement-form `if c { ... } else { ... }` (which uses braces); the 91// expression form `if c then a else b` matches Haskell/OCaml/Elm 92// convention. Required to disambiguate at parse_primary. 93const TK_THEN: i64 = 32 94 95// ---- arithmetic / assign ----------------------------------------- 96 97const TK_PLUS: i64 = 40 98const TK_MINUS: i64 = 41 99const TK_STAR: i64 = 42 100const TK_SLASH: i64 = 43 101const TK_PERCENT: i64 = 44 102const TK_ASSIGN: i64 = 45 103 104// ---- comparisons -------------------------------------------------- 105 106const TK_EQ: i64 = 46 107const TK_NE: i64 = 47 108const TK_LT: i64 = 48 109const TK_GT: i64 = 49 110const TK_LE: i64 = 50 111const TK_GE: i64 = 51 112 113// ---- logical / bitwise -------------------------------------------- 114 115const TK_AND_AND: i64 = 52 116const TK_OR_OR: i64 = 53 117const TK_BANG: i64 = 54 118const TK_AMP: i64 = 55 119const TK_PIPE: i64 = 56 120const TK_CARET: i64 = 57 121const TK_TILDE: i64 = 58 122const TK_SHL: i64 = 59 123const TK_SHR: i64 = 60 124 125// ---- structural punctuation --------------------------------------- 126 127const TK_ARROW: i64 = 61 // -> 128const TK_DOT_DOT: i64 = 62 // .. 129const TK_DOT: i64 = 63 130const TK_COLON: i64 = 64 131const TK_SEMI: i64 = 65 132const TK_COMMA: i64 = 66 133const TK_LPAREN: i64 = 67 134const TK_RPAREN: i64 = 68 135const TK_LBRACE: i64 = 69 136const TK_RBRACE: i64 = 70 137const TK_LBRACKET: i64 = 71 138const TK_RBRACKET: i64 = 72 139const TK_AT: i64 = 73 140const TK_HASH: i64 = 74 141const TK_COLON_COLON: i64 = 75 // :: 142const TK_FAT_ARROW: i64 = 76 // => (used by `match` arms) 143const TK_QUESTION: i64 = 77 // ? (C-style ternary cond ? a : b) 144 145// ---- shared Tok helpers ---- 146// 147// Kept here because they depend only on the Tok layout. Both lex.nx 148// (token producer) and parse.nx (token consumer) used to carry local 149// copies -- now they share one. 150 151func tok_text_ptr(t: *Tok) -> *u8 { 152 let base: i64 = t as i64 153 return (base + 32) as *u8 154} 155 156// Compare `n` bytes between `a` and `b`. If the shorter side (`b`) 157// isn't null-terminated at position n the match fails -- matches the 158// C strncmp-style "prefix of a equals exactly b" semantics used by 159// keyword lookup. 160func streq_n(a: *u8, b: *u8, n: i64) -> i64 { 161 var i: i64 = 0 162 while i < n { 163 if a[i] != b[i] { return 0 } 164 i = i + 1 165 } 166 if b[n] != 0 { return 0 } 167 return 1 168}