nx_lex_kinds.nx source

1// lex_kinds.nx -- Tok record + named TokenKind constants. 2// 3// Shared by lex.nx (producer) and parse.nx (consumer). Both used 4// to carry their own copy of the `Tok` struct and compare 5// `t.kind == 67` with a comment saying "LPAREN"; with this module 6// imported they use `t.kind == TK_LPAREN` directly. 7// 8// Numbers below must match the comment table in lex.nx and the 9// emission sites in lex.nx's lexer -- do not renumber without 10// auditing both callers and the existing test fixtures. 11 12// ---- Tok record --------------------------------------------------- 13// 14// text[] (64 bytes) lives inline as 8 i64 words starting at offset 15// 32. Access via tok_text_ptr() defined by lex.nx. 16 17struct Tok { 18 kind: i64, 19 line: i64, 20 col: i64, 21 int_val: i64, 22 text0: i64, 23 text1: i64, 24 text2: i64, 25 text3: i64, 26 text4: i64, 27 text5: i64, 28 text6: i64, 29 text7: i64, 30 // Overflow buffer for TK_STRING longer than 63 bytes; mirrors 31 // the str_data/str_len fields added to nxc2/lex.h. 32 str_data: i64, 33 str_len: i64, 34} 35 36const TOK_BYTES: i64 = 112 37 38// ---- literals ----------------------------------------------------- 39 40const TK_EOF: i64 = 0 41const TK_INT: i64 = 1 42const TK_IDENT: i64 = 2 43const TK_STRING: i64 = 3 44// Floating-point literal. Lexer splits the number into whole / 45// fractional parts and stores both in the Tok: 46// t.int_val = whole part 47// t.text0 = fractional digits as i64 (e.g. "14" in "3.14" -> 14) 48// t.text1 = number of fractional digits (2 in the example) 49// Parser converts to IEEE 754 via quant.nx when building the IR. 50// Keeping whole/frac as two ints in the token avoids pulling fp 51// conversion into the lexer and lets parse.nx decide f32 vs f64 52// based on the declared variable type. 53const TK_FLOAT: i64 = 4 54 55// Explicit-f32 float literal -- written `1.5f32` (or `1.5F32`) in 56// source. Same int_val / text0 / text1 packing as TK_FLOAT (whole / 57// frac_num / frac_digits); the kind tells parse.nx to use 58// fp32_from_parts + TY_F32 instead of the default fp64 path. 59// `1.5f64` is allowed as an explicit-default and emits TK_FLOAT. 60const TK_FLOAT_F32: i64 = 5 61 62// ---- keywords ----------------------------------------------------- 63 64const TK_FUNC: i64 = 10 65const TK_LET: i64 = 11 66const TK_VAR: i64 = 12 67const TK_CONST: i64 = 13 68const TK_STATIC: i64 = 14 69const TK_IF: i64 = 15 70const TK_ELSE: i64 = 16 71const TK_WHILE: i64 = 17 72const TK_LOOP: i64 = 18 73const TK_FOR: i64 = 19 74const TK_IN: i64 = 20 75const TK_BREAK: i64 = 21 76const TK_CONTINUE: i64 = 22 77const TK_RETURN: i64 = 23 78const TK_TRUE: i64 = 24 79const TK_FALSE: i64 = 25 80const TK_STRUCT: i64 = 26 81const TK_ENUM: i64 = 27 82// @priv VISIBILITY ATTRIBUTE. Ported from the laptop SSOT 2026-07-31 -- buildroot had NO const at slot 33 83// (verified free before claiming it), so the trees can converge without renumbering anything. 84const TK_PRIV: i64 = 33 85const TK_MATCH: i64 = 28 86const TK_COMPTIME: i64 = 29 87const TK_EXTERN: i64 = 30 88const TK_AS: i64 = 31 89// V-LANGEXT M2: if-then-else as expression keyword. Distinct from 90// statement-form `if c { ... } else { ... }` (which uses braces); the 91// expression form `if c then a else b` matches Haskell/OCaml/Elm 92// convention. Required to disambiguate at parse_primary. 93const TK_THEN: i64 = 32 94 95// ---- arithmetic / assign ----------------------------------------- 96 97const TK_PLUS: i64 = 40 98const TK_MINUS: i64 = 41 99const TK_STAR: i64 = 42 100const TK_SLASH: i64 = 43 101const TK_PERCENT: i64 = 44 102const TK_ASSIGN: i64 = 45 103 104// ---- comparisons -------------------------------------------------- 105 106const TK_EQ: i64 = 46 107const TK_NE: i64 = 47 108const TK_LT: i64 = 48 109const TK_GT: i64 = 49 110const TK_LE: i64 = 50 111const TK_GE: i64 = 51 112 113// ---- logical / bitwise -------------------------------------------- 114 115const TK_AND_AND: i64 = 52 116const TK_OR_OR: i64 = 53 117const TK_BANG: i64 = 54 118const TK_AMP: i64 = 55 119const TK_PIPE: i64 = 56 120const TK_CARET: i64 = 57 121const TK_TILDE: i64 = 58 122const TK_SHL: i64 = 59 123const TK_SHR: i64 = 60 124 125// ---- structural punctuation --------------------------------------- 126 127const TK_ARROW: i64 = 61 // -> 128const TK_DOT_DOT: i64 = 62 // .. 129const TK_DOT: i64 = 63 130const TK_COLON: i64 = 64 131const TK_SEMI: i64 = 65 132const TK_COMMA: i64 = 66 133const TK_LPAREN: i64 = 67 134const TK_RPAREN: i64 = 68 135const TK_LBRACE: i64 = 69 136const TK_RBRACE: i64 = 70 137const TK_LBRACKET: i64 = 71 138const TK_RBRACKET: i64 = 72 139const TK_AT: i64 = 73 140const TK_HASH: i64 = 74 141const TK_COLON_COLON: i64 = 75 // :: 142const TK_FAT_ARROW: i64 = 76 // => (used by `match` arms) 143const TK_QUESTION: i64 = 77 // ? (C-style ternary cond ? a : b) 144 145// ---- shared Tok helpers ---- 146// 147// Kept here because they depend only on the Tok layout. Both lex.nx 148// (token producer) and parse.nx (token consumer) used to carry local 149// copies -- now they share one. 150 151func tok_text_ptr(t: *Tok) -> *u8 { 152 let base: i64 = t as i64 153 return (base + 32) as *u8 154} 155 156// Compare `n` bytes between `a` and `b`. If the shorter side (`b`) 157// isn't null-terminated at position n the match fails -- matches the 158// C strncmp-style "prefix of a equals exactly b" semantics used by 159// keyword lookup. 160// ---- LN15: FREE MULTI-LINE EXPRESSIONS ---------------------------- 161// 162// ONE question, asked by parse.nx's at_stmt_boundary: when a token is 163// the FIRST on a new source line, does it CONTINUE the expression on 164// the line above, or can it legitimately START a fresh statement? 165// 166// It lives here, beside the kinds, for the same reason tok_text_ptr and 167// streq_n do: it is a pure function of a token kind, and lex.nx (the 168// producer) and parse.nx (the consumer) both already import this module. 169// So the shared ruler costs no new import edge and there is only ever 170// ONE of it -- a second copy in the parser is how the two sides drift. 171// 172// WHY THE AMBIGUOUS PREFIXES ARE NOT AMBIGUOUS AT STATEMENT LEVEL. 173// NishiLang has exactly ONE operator-led statement form: parse_stmt_star, 174// the pointer store `*p = v`. No statement form begins with `+`, `-` or 175// `&`. A line `- b` can therefore only reach parse_stmt_expr_fallback, 176// where it computes a value and throws it away -- and since 2026-08-13 177// that fallback REFUSES exactly that as a discarded pure expression. 178// 179// So the programs this predicate reclassifies are precisely the programs 180// that DO NOT BUILD TODAY, and joining them cannot change the meaning of 181// any program that does. That is the safety argument, and it is checkable 182// rather than asserted: the corpus A/B must come back byte-identical. 183// 184// `~` and the logical-not stay boundaries: they are unary-ONLY, so a line 185// opening with one can never have been a continuation of anything. 186// 187// HISTORY. The det2x2 defect (2026-07-09, runtime-wide sweep) was this 188// class before that refusal existed: `let d = a*d` / `- b*c` silently 189// kept only the first term. The refusal made the wrong value LOUD. 190// This predicate makes it CORRECT. 191 192const TLC_BOUNDARY: i64 = 0 // may start a statement -- keep the boundary 193const TLC_CONTINUATION: i64 = 1 // no prefix form exists -- join the line above 194const TLC_STORE_OR_JOIN: i64 = 2 // TK_STAR only: `*p = v` stores, `* b` joins 195 196func tok_line_continuation(k: i64) -> i64 { 197 // Always-binary (established 2026-05-16): no prefix form exists. 198 if k == TK_PIPE { return TLC_CONTINUATION } 199 if k == TK_CARET { return TLC_CONTINUATION } 200 if k == TK_SLASH { return TLC_CONTINUATION } 201 if k == TK_PERCENT { return TLC_CONTINUATION } 202 if k == TK_EQ { return TLC_CONTINUATION } 203 if k == TK_NE { return TLC_CONTINUATION } 204 if k == TK_LT { return TLC_CONTINUATION } 205 if k == TK_GT { return TLC_CONTINUATION } 206 if k == TK_LE { return TLC_CONTINUATION } 207 if k == TK_GE { return TLC_CONTINUATION } 208 if k == TK_AND_AND { return TLC_CONTINUATION } 209 if k == TK_OR_OR { return TLC_CONTINUATION } 210 if k == TK_SHL { return TLC_CONTINUATION } 211 if k == TK_SHR { return TLC_CONTINUATION } 212 if k == TK_ASSIGN { return TLC_CONTINUATION } 213 // LN15 adds these three: legal unary PREFIXES, but no statement FORM 214 // begins with them, so at statement level they can only have been a 215 // continuation of the line above. 216 if k == TK_PLUS { return TLC_CONTINUATION } 217 if k == TK_MINUS { return TLC_CONTINUATION } 218 if k == TK_AMP { return TLC_CONTINUATION } 219 // The single genuine ambiguity, handed back to the caller because 220 // deciding it needs the token stream, not just the kind. 221 if k == TK_STAR { return TLC_STORE_OR_JOIN } 222 return TLC_BOUNDARY 223} 224 225func streq_n(a: *u8, b: *u8, n: i64) -> i64 { 226 var i: i64 = 0 227 while i < n { 228 if a[i] != b[i] { return 0 } 229 i = i + 1 230 } 231 if b[n] != 0 { return 0 } 232 return 1 233}