nx_tokenizer.nx source
↩ module page · 2619 lines · 111969 B
1// lex.nx -- NishiLang source tokenizer, ported from lex.c.
2//
3// Accepts a null-terminated source buffer and produces a stream of
4// Tok records. Mirrors lex.c's token layout (kind + line/col +
5// either int_val or text[]). Same keyword set.
6//
7// Single-pass; zero allocations per token beyond the growing token
8// array. Extends at 2x when full. Whitespace + // comments are
9// skipped. Numbers support 0x/0b/0o prefixes with underscore
10// separators. String literals read bytes until closing `"`.
11
12// ---- syscalls ----
13
14// ---- token kinds (subset of lex.h, integer constants) ----
15//
16// 0 EOF
17// 1 INT
18// 2 IDENT
19// 3 STRING
20// 10 FUNC 11 LET 12 VAR 13 CONST
21// 14 STATIC 15 IF 16 ELSE 17 WHILE
22// 18 LOOP 19 FOR 20 IN 21 BREAK
23// 22 CONTINUE 23 RETURN 24 TRUE 25 FALSE
24// 26 STRUCT 27 ENUM 28 MATCH 29 COMPTIME
25// 30 EXTERN 31 AS
26// 40 PLUS 41 MINUS 42 STAR 43 SLASH
27// 44 PERCENT 45 ASSIGN 46 EQ 47 NE
28// 48 LT 49 GT 50 LE 51 GE
29// 52 AND_AND 53 OR_OR 54 BANG
30// 55 AMP 56 PIPE 57 CARET 58 TILDE
31// 59 SHL 60 SHR
32// 61 ARROW 62 DOT_DOT 63 DOT 64 COLON
33// 65 SEMI 66 COMMA
34// 67 LPAREN 68 RPAREN 69 LBRACE 70 RBRACE
35// 71 LBRACKET 72 RBRACKET 73 AT 74 HASH
36
37// ---- Tok struct ----
38//
39// Laid out to match lex.h approximately. text[] is fixed-size
40// 64-byte for identifiers and string literals; that's the same
41// MAX_IDENT the C lexer uses.
42
43// nx_safety_envelope:
44// intended_use: AUTO_APPLIED -- primitive-specific tuning queued
45// sil_target: SIL1
46// evidence: [bulk_applied_2026-05-16, see-file-comment-for-detail]
47// verdict: NOT_YET_EVALUATED
48
49import "nx_syscalls.nx"
50import "nx_types.nx"
51import "nx_lex_kinds.nx"
52
53// LN24 (2026-09-02): a lexer DESYNC is recorded HERE, at the byte that broke, and reported by the caller
54// (nx_compile_x86) through the diagnostic layer with the file:line map -- the tokenizer cannot import that
55// layer (nx_parse imports the tokenizer). Before this, a raw NUL or an unclosed literal in the LAST user
56// file surfaced as twenty unknown-name errors attributed to the auto-appended nx_crash.nx (measured).
57// kind: 0 none, 1 control byte outside a literal, 2 string literal never closed (end of unit, or a NUL),
58// 3 integer literal exceeds i64 (LN40), 4 malformed exponent -- e/E and a sign with no digit (LN36).
59// Decimal literal machine bounds (LN36/LN40): 18 significant digits fit i64 with room (10^18 < 2^63);
60// A decimal integer literal may carry any 64-bit pattern: values in [2^63, 2^64) are the unsigned pattern
61// (FNV offset bases and all-ones masks are written that way across the estate), so the bound is u64 max
62// 18446744073709551615 = LEX_U64_MAX_DIV10 * 10 + LEX_U64_MAX_LAST; only a literal that needs a 65th bit is
63// refused. An exponent past the cap is +inf / 0 regardless of its digits, so the cap only stops the
64// accumulator from wrapping.
65const LEX_DEC_SIG_DIGITS: i64 = 18
66const LEX_U64_MAX_DIV10: i64 = 1844674407370955161
67const LEX_U64_MAX_LAST: i64 = 5
68const LEX_EXP_CAP: i64 = 99999
69static lex_err_kind: i64
70static lex_err_line: i64
71static lex_err_col: i64
72static lex_err_byte: i64
73func lex_err_get_kind() -> i64 { return lex_err_kind }
74func lex_err_get_line() -> i64 { return lex_err_line }
75func lex_err_get_col() -> i64 { return lex_err_col }
76func lex_err_get_byte() -> i64 { return lex_err_byte }
77// LN24 + LN27 (2026-09-02): THE ONE RECORDER for a lexer desync -- the symbol the lang.matrix watch names
78// (diag_desync_origin). FIRST BREAK WINS: every later break is a consequence of this one, so recording a
79// later site would move the blame away from the cause. The caller (nx_compile_x86) reports it in the
80// teaching voice (where with a caret, why the build stopped, fix) through the file:line map it owns.
81func diag_desync_origin(kind: i64, line: i64, col: i64, byte: i64) -> i64 {
82 if lex_err_kind != 0 { return 0 }
83 lex_err_kind = kind
84 lex_err_line = line
85 lex_err_col = col
86 lex_err_byte = byte
87 return 1
88}
89const LEXM_MAGIC_65536: i64 = 65536
90// Tok struct + TOK_BYTES const live in lex_kinds.nx (canonical
91// home). Removed duplicate definition here 2026-04-26 per
92// nx_type_identity_check.sh -- duplicate Type instances at
93// compile-time create T#selfhost-006-class bugs even when
94// structurally identical.
95
96// ---- character classes ----
97
98func is_alpha(c: i64) -> i64 {
99 if c >= 0x41 { if c <= 0x5A { return 1 } }
100 if c >= 0x61 { if c <= 0x7A { return 1 } }
101 if c == 0x5F { return 1 }
102 return 0
103}
104func is_digit(c: i64) -> i64 {
105 if c >= 0x30 { if c <= 0x39 { return 1 } }
106 return 0
107}
108func is_hexdigit(c: i64) -> i64 {
109 if is_digit(c) { return 1 }
110 if c >= 0x41 { if c <= 0x46 { return 1 } }
111 if c >= 0x61 { if c <= 0x66 { return 1 } }
112 return 0
113}
114func hex_val(c: i64) -> i64 {
115 if c >= 0x30 { if c <= 0x39 { return c - 0x30 } }
116 if c >= 0x41 { if c <= 0x46 { return c - 0x37 } }
117 if c >= 0x61 { if c <= 0x66 { return c - 0x57 } }
118 return 0
119}
120func is_alnum(c: i64) -> i64 {
121 if is_alpha(c) { return 1 }
122 if is_digit(c) { return 1 }
123 return 0
124}
125
126// ---- keyword table ----
127//
128// Linear scan; we have ~20 keywords. Returns token kind or -1.
129
130func keyword_lookup(t: *u8, len: i64) -> i64 {
131 if streq_n(t, "func", len) { return 10 }
132 if streq_n(t, "let", len) { return 11 }
133 if streq_n(t, "var", len) { return 12 }
134 if streq_n(t, "const", len) { return 13 }
135 if streq_n(t, "static", len) { return 14 }
136 if streq_n(t, "if", len) { return 15 }
137 if streq_n(t, "then", len) { return 32 } // V-LANGEXT M2 if-expr
138 if streq_n(t, "else", len) { return 16 }
139 if streq_n(t, "while", len) { return 17 }
140 if streq_n(t, "loop", len) { return 18 }
141 if streq_n(t, "for", len) { return 19 }
142 if streq_n(t, "in", len) { return 20 }
143 if streq_n(t, "break", len) { return 21 }
144 if streq_n(t, "continue", len) { return 22 }
145 if streq_n(t, "return", len) { return 23 }
146 if streq_n(t, "true", len) { return 24 }
147 if streq_n(t, "false", len) { return 25 }
148 if streq_n(t, "struct", len) { return 26 }
149 if streq_n(t, "enum", len) { return 27 }
150 if streq_n(t, "match", len) { return 28 }
151 if streq_n(t, "comptime", len) { return 29 }
152 if streq_n(t, "extern", len) { return 30 }
153 if streq_n(t, "as", len) { return 31 }
154 return 2 // generic IDENT
155}
156
157// ---- Lex state ----
158
159struct Lex {
160 src: *u8,
161 pos: i64,
162 line: i64,
163 col: i64,
164 tokens: *Tok,
165 n_tokens: i64,
166 cap: i64,
167}
168
169// Push a token into the token pool. The pool is sized BY DERIVATION in lex_source
170// (tokens <= post-expansion source bytes), so this guard cannot fire on that path;
171// it is the fail-closed floor for any other caller: an exhausted pool REFUSES
172// loudly instead of spraying TOK_BYTES records over the neighbouring allocation
173// (the silent position-dependent parse corruption class; sibling of the
174// ir_new_function pool guard added 2026-08-16).
175
176func push_tok(L: *Lex, t: *Tok) -> i64 {
177 if L.n_tokens >= L.cap {
178 sys_write(2, "nx_lex: token pool FULL -- this unit has more tokens than the pool lex_source sized; refusing to overrun (the overflow would silently corrupt the neighbouring allocation and surface as parser desync far away). This cannot fire on the derived-size path; if you see it, a caller bypassed the derivation.
179" as *u8, 302)
180 sys_exit(2)
181 }
182 let base: i64 = L.tokens as i64
183 let slot: *Tok = (base + L.n_tokens * TOK_BYTES) as *Tok
184 slot.kind = t.kind
185 slot.line = t.line
186 slot.col = t.col
187 slot.int_val = t.int_val
188 slot.text0 = t.text0
189 slot.text1 = t.text1
190 slot.text2 = t.text2
191 slot.text3 = t.text3
192 slot.text4 = t.text4
193 slot.text5 = t.text5
194 slot.text6 = t.text6
195 slot.text7 = t.text7
196 slot.str_data = t.str_data
197 slot.str_len = t.str_len
198 L.n_tokens = L.n_tokens + 1
199 return 0
200}
201
202func advance(L: *Lex) -> i64 {
203 let s: *u8 = L.src
204 let c: i64 = s[L.pos]
205 if c == 0x0A {
206 L.line = L.line + 1
207 L.col = 1
208 }
209 if c != 0x0A {
210 if c != 0 { L.col = L.col + 1 }
211 }
212 L.pos = L.pos + 1
213 return c
214}
215func peek(L: *Lex) -> i64 {
216 let s: *u8 = L.src
217 return s[L.pos]
218}
219func peek2(L: *Lex) -> i64 {
220 let s: *u8 = L.src
221 if s[L.pos] == 0 { return 0 }
222 return s[L.pos + 1]
223}
224
225// Bytes-equal: compare L.src[L.pos..L.pos+n] to needle[0..n].
226// Used by the preprocessor for directive name matching.
227// ===== @macro TABLE -- self-host parity with the C bootstrap ========================================
228// main.c implements @macro and pre-defines TARGET_X86_64 as one; the self-host preprocessor below
229// implemented ONLY the conditional directives, so `@macro NAME body` was never expanded and
230// nx_macro_test .. nx_macro_v8_test could never compile.
231//
232// STATIC POINTERS TO LAZILY-MMAPPED BUFFERS, NEVER BSS ARRAYS: a static [N]i64 in a module crashes the
233// process silently at startup (hit twice in this corpus -- see the nx_mgmt_api rate/idem tables). mmap
234// zeroes, so an untouched slot reads empty with no init loop.
235const LEXM_MAX: i64 = 256
236const LEXM_NAMELEN: i64 = 64
237// 1024, not 192: a `{ ... }` body holds a whole function definition. A body that overflows is REFUSED
238// by lexm_define rather than truncated -- a half-stored macro body expands into syntactically valid
239// garbage, which is a silent miscompile.
240const LEXM_BODYLEN: i64 = 1024
241
242static lexm_names: *u8
243static lexm_bodies: *u8
244static lexm_count: *i64
245static lexm_gensym: *i64
246
247func lexm_init() -> i64 {
248 if (lexm_count as i64) != 0 { return 0 }
249 lexm_names = sys_mmap(LEXM_MAX * LEXM_NAMELEN)
250 lexm_bodies = sys_mmap(LEXM_MAX * LEXM_BODYLEN)
251 lexm_count = sys_mmap(16) as *i64
252 lexm_gensym = sys_mmap(16) as *i64
253 return 0
254}
255
256// Emit a NUL-terminated literal / a decimal integer into out. Used to build gensym identifiers.
257func lexm_emit_str(out: *u8, o0: i64, s: *u8) -> i64 {
258 var o: i64 = o0
259 var i: i64 = 0
260 while s[i] != (0 as u8) { out[o] = s[i]; o = o + 1; i = i + 1 }
261 return o
262}
263
264func lexm_emit_dec(out: *u8, o0: i64, v: i64) -> i64 {
265 if v <= 0 { out[o0] = 48 as u8; return o0 + 1 }
266 let tmp: *u8 = sys_mmap(32)
267 var m: i64 = 0
268 var x: i64 = v
269 while x > 0 { tmp[m] = ((x % 10) + 48) as u8; m = m + 1; x = x / 10 }
270 var o: i64 = o0
271 var j: i64 = m - 1
272 while j >= 0 { out[o] = tmp[j]; o = o + 1; j = j - 1 }
273 return o
274}
275
276func lexm_strlen(s: *u8) -> i64 {
277 var i: i64 = 0
278 while s[i] != (0 as u8) { i = i + 1 }
279 return i
280}
281
282// End offset of a macro body starting at bp. A `{` body runs to its MATCHING close brace and may span
283// many lines; any other body ends at the newline. Depth-aware, and it steps over string literals and
284// `//` comments so a brace inside either cannot close the body early. Returns the offset ONE PAST the
285// closing brace for a brace body, or the offset of the terminating newline otherwise.
286func lexm_body_end(src: *u8, n: i64, bp: i64) -> i64 {
287 if bp >= n { return bp }
288 // Step over a leading parameter list FIRST, so `@macro F(a, b) { ... }` is recognised as a BRACE
289 // body. Without this the body would look like it starts at `(`, which is not `{`, and the whole
290 // multi-line block would be cut off at the first newline.
291 var q: i64 = bp
292 if src[q] == (40 as u8) {
293 var d0: i64 = 0
294 var sp: i64 = 0
295 while sp == 0 {
296 if q >= n { sp = 1 } else {
297 let c0: i64 = src[q] as i64
298 if c0 == 0 { sp = 1 } else {
299 if c0 == 10 { sp = 1 } else {
300 if c0 == 40 { d0 = d0 + 1; q = q + 1 } else {
301 if c0 == 41 { d0 = d0 - 1; q = q + 1; if d0 == 0 { sp = 1 } } else { q = q + 1 } } } }
302 }
303 }
304 var sw: i64 = 0
305 while sw == 0 {
306 if q < n { if src[q] == (32 as u8) { q = q + 1 } else { sw = 1 } } else { sw = 1 }
307 }
308 }
309 if q >= n { return q }
310 if src[q] != (123 as u8) {
311 var e: i64 = q
312 var st: i64 = 0
313 while st == 0 {
314 if e >= n { st = 1 } else {
315 let c: i64 = src[e] as i64
316 if c == 10 { st = 1 } else {
317 if c == 13 { st = 1 } else {
318 if c == 0 { st = 1 } else { e = e + 1 } } }
319 }
320 }
321 return e
322 }
323 var i: i64 = q
324 var depth: i64 = 0
325 var st2: i64 = 0
326 while st2 == 0 {
327 if i >= n { st2 = 1 } else {
328 let c: i64 = src[i] as i64
329 if c == 0 { st2 = 1 } else {
330 if c == 34 {
331 i = i + 1
332 var sq: i64 = 0
333 while sq == 0 {
334 if i >= n { sq = 1 } else {
335 if src[i] == (0 as u8) { sq = 1 } else {
336 if src[i] == (92 as u8) { i = i + 2 } else {
337 if src[i] == (34 as u8) { i = i + 1; sq = 1 } else { i = i + 1 } } }
338 }
339 }
340 } else {
341 if c == 47 {
342 var isc: i64 = 0
343 if i + 1 < n { if src[i+1] == (47 as u8) { isc = 1 } }
344 if isc == 1 {
345 var cq: i64 = 0
346 while cq == 0 {
347 if i >= n { cq = 1 } else {
348 if src[i] == (10 as u8) { cq = 1 } else {
349 if src[i] == (0 as u8) { cq = 1 } else { i = i + 1 } }
350 }
351 }
352 } else { i = i + 1 }
353 } else {
354 if c == 123 { depth = depth + 1; i = i + 1 } else {
355 if c == 125 { depth = depth - 1; i = i + 1; if depth == 0 { st2 = 1 } } else {
356 i = i + 1
357 } } } } }
358 }
359 }
360 return i
361}
362
363func lexm_name_at(i: i64) -> *u8 { return ((lexm_names as i64) + i * LEXM_NAMELEN) as *u8 }
364func lexm_body_at(i: i64) -> *u8 { return ((lexm_bodies as i64) + i * LEXM_BODYLEN) as *u8 }
365
366// ===== LEGIBLE REFUSAL ==============================================================================
367// Refusing an oversized `{ }` macro body is CORRECT -- a truncated body loses its closing brace and
368// expands into text that still parses, which is a silent miscompile. But the refusal was SILENT: the
369// macro never registered and the author saw `undefined function: MYMACRO` at the USE site, with nothing
370// pointing at the definition or the body length.
371// The tokenizer had NO error channel before this; every diagnostic came from nx_parse downstream.
372// A WARNING, not an abort: the existing undefined-function error already stops the build; what was
373// missing was the REASON, not the stop.
374// ★ REFUSING SAFELY AND REFUSING LEGIBLY ARE TWO SEPARATE JOBS.
375static lexm_diagbuf: *u8
376
377func lexm_diag_cell() -> *u8 {
378 if (lexm_diagbuf as i64) == 0 { lexm_diagbuf = sys_mmap(512) }
379 return lexm_diagbuf
380}
381
382// Bounded by construction: the name is clamped so a pathological identifier cannot run past the buffer.
383// A diagnostic that overruns while reporting an overrun would be a poor joke.
384func lexm_warn_body_too_long(name: *u8, nlen: i64, blen: i64) -> i64 {
385 let buf: *u8 = lexm_diag_cell()
386 var o: i64 = 0
387 o = lexm_emit_str(buf, o, "nx_tokenizer: @macro '" as *u8)
388 var i: i64 = 0
389 while i < nlen {
390 if o < 300 { buf[o] = name[i]; o = o + 1 }
391 i = i + 1
392 }
393 o = lexm_emit_str(buf, o, "' body is " as *u8)
394 o = lexm_emit_dec(buf, o, blen)
395 o = lexm_emit_str(buf, o, " bytes, over the " as *u8)
396 o = lexm_emit_dec(buf, o, LEXM_BODYLEN)
397 o = lexm_emit_str(buf, o, " byte limit -- NOT DEFINED. Uses will report 'undefined function'. Split the macro or raise LEXM_BODYLEN.\n" as *u8)
398 sys_write(2, buf, o)
399 return 0
400}
401
402// Define NAME -> BODY. A later definition of the same name OVERWRITES, matching the C bootstrap, rather
403// than silently keeping the first -- a redefinition that is ignored makes the build differ from its
404// source with no diagnostic.
405func lexm_define(name: *u8, nlen: i64, body: *u8, blen: i64) -> i64 {
406 lexm_init()
407 if nlen <= 0 { return 0 }
408 if nlen >= LEXM_NAMELEN { return 0 }
409 // REFUSE an oversized body instead of truncating it. A truncated `{ ... }` body loses its closing
410 // brace and expands into text that can still parse, which is a silent miscompile; refusing leaves the
411 // macro name unexpanded and surfaces as a loud `undefined function` at the use site.
412 if blen >= LEXM_BODYLEN { lexm_warn_body_too_long(name, nlen, blen); return 0 }
413 var bl: i64 = blen
414 if bl < 0 { bl = 0 }
415 var slot: i64 = 0 - 1
416 var i: i64 = 0
417 while i < lexm_count[0] {
418 let p: *u8 = lexm_name_at(i)
419 var eq: i64 = 1
420 var k: i64 = 0
421 while k < nlen { if p[k] != name[k] { eq = 0; k = nlen } else { k = k + 1 } }
422 if eq == 1 { if p[nlen] == (0 as u8) { slot = i } }
423 i = i + 1
424 }
425 if slot < 0 {
426 if lexm_count[0] >= LEXM_MAX { return 0 }
427 slot = lexm_count[0]
428 lexm_count[0] = lexm_count[0] + 1
429 }
430 let np: *u8 = lexm_name_at(slot)
431 var a: i64 = 0
432 while a < nlen { np[a] = name[a]; a = a + 1 }
433 np[nlen] = 0 as u8
434 let bp: *u8 = lexm_body_at(slot)
435 var b: i64 = 0
436 while b < bl { bp[b] = body[b]; b = b + 1 }
437 bp[bl] = 0 as u8
438 return 1
439}
440
441func lexm_lookup(name: *u8, nlen: i64) -> *u8 {
442 if (lexm_count as i64) == 0 { return 0 as *u8 }
443 if nlen <= 0 { return 0 as *u8 }
444 if nlen >= LEXM_NAMELEN { return 0 as *u8 }
445 var i: i64 = 0
446 while i < lexm_count[0] {
447 let p: *u8 = lexm_name_at(i)
448 var eq: i64 = 1
449 var k: i64 = 0
450 while k < nlen { if p[k] != name[k] { eq = 0; k = nlen } else { k = k + 1 } }
451 if eq == 1 { if p[nlen] == (0 as u8) { return lexm_body_at(i) } }
452 i = i + 1
453 }
454 return 0 as *u8
455}
456
457// Remove NAME from the table. `@undef` exists so a conditional can be RETRACTED (nx_macro_v4_test
458// defines EXPERIMENT, checks @ifdef, retracts, then checks @ifndef). Removal swaps the LAST entry into
459// the freed slot -- lookup is order-independent, so compaction costs nothing and leaves no tombstone a
460// later scan could mistake for a live entry.
461func lexm_undef(name: *u8, nlen: i64) -> i64 {
462 if (lexm_count as i64) == 0 { return 0 }
463 if nlen <= 0 { return 0 }
464 if nlen >= LEXM_NAMELEN { return 0 }
465 var slot: i64 = 0 - 1
466 var i: i64 = 0
467 while i < lexm_count[0] {
468 let p: *u8 = lexm_name_at(i)
469 var eq: i64 = 1
470 var k: i64 = 0
471 while k < nlen { if p[k] != name[k] { eq = 0; k = nlen } else { k = k + 1 } }
472 if eq == 1 { if p[nlen] == (0 as u8) { slot = i } }
473 i = i + 1
474 }
475 if slot < 0 { return 0 }
476 let last: i64 = lexm_count[0] - 1
477 if slot != last {
478 let dn: *u8 = lexm_name_at(slot)
479 let sn: *u8 = lexm_name_at(last)
480 var a: i64 = 0
481 while a < LEXM_NAMELEN { dn[a] = sn[a]; a = a + 1 }
482 let db: *u8 = lexm_body_at(slot)
483 let sb: *u8 = lexm_body_at(last)
484 var b: i64 = 0
485 while b < LEXM_BODYLEN { db[b] = sb[b]; b = b + 1 }
486 }
487 lexm_count[0] = last
488 return 1
489}
490
491// Drop every definition. Called BETWEEN the arg-expansion pre-pass and lexing. lexm_prescan populates
492// the table from the WHOLE file up front -- it must, to expand a use that appears before its textual
493// definition -- which would otherwise make `@ifdef FOO` at line 10 see a `@macro FOO` at line 100 as
494// already defined. Conditionals are POSITION-DEPENDENT, so the sequential directive handler has to
495// repopulate the table in source order or `@ifdef` silently answers about the whole file.
496// BACKEND-OWNED PREDEFINES, re-seeded on every reset so they survive the pre-pass clear.
497//
498// TARGET_RV64 is DEFINED, and that is the exact complement of TARGET_X86_64 being hard-pinned
499// UNDEFINED (see _lex_pp_defined): source is WRITTEN against RV64 syscall numbering, and the x86
500// self-host translates at emit time. So "which target is this source written for" answers RV64, while
501// "am I emitting raw x86" answers no. Both halves describe the same one pipeline.
502//
503// Seeded via lexm_define rather than special-cased in _lex_pp_defined so they are ordinary table
504// entries -- `@undef`-able and usable as value macros. TARGET_X86_64 is the ONLY name that gets a hard
505// pin, because that is the one a source file could use to make every syscall translate twice.
506func lexm_predefine() -> i64 {
507 lexm_define("TARGET_RV64" as *u8, 11, "1" as *u8, 1)
508 // Compiler identity. Bump when the language surface changes in a way source may need to branch on:
509 // v2 = @macro/@undef/@ifdef-table, v3 = multi-line `{ }` bodies + `$$ident` gensym (2026-07-31).
510 lexm_define("NXC_VERSION" as *u8, 11, "3" as *u8, 1)
511 return 0
512}
513
514func lexm_reset() -> i64 {
515 lexm_init()
516 lexm_count[0] = 0
517 lexm_predefine()
518 return 0
519}
520
521// Is the body a plain non-negative integer? V1 SCOPE IS DELIBERATE: zero-arg NUMERIC macros only
522// (@macro UNIVERSAL_ANSWER 42 = nx_macro_test T1, and the FEATURE_* flags in v3/v6). Arg substitution and
523// @concat pasting are the next rungs and are NOT claimed here. A body this cannot parse simply does not
524// expand, leaving today's behaviour unchanged rather than half-applied -- a partially substituted macro
525// is a SILENT MISCOMPILE, which is strictly worse than no macro at all.
526func lexm_body_int(body: *u8, out: *i64) -> i64 {
527 if (body as i64) == 0 { return 0 }
528 var i: i64 = 0
529 var sk: i64 = 0
530 while sk == 0 {
531 if body[i] == (32 as u8) { i = i + 1 } else { sk = 1 }
532 }
533 var v: i64 = 0
534 var n: i64 = 0
535 var go: i64 = 1
536 while go == 1 {
537 let c: i64 = body[i] as i64
538 var isd: i64 = 0
539 if c >= 48 { if c <= 57 { isd = 1 } }
540 if isd == 1 { v = v * 10 + (c - 48); n = n + 1; i = i + 1 } else { go = 0 }
541 }
542 if n == 0 { return 0 }
543 var st: i64 = 0
544 while st == 0 {
545 if body[i] == (32 as u8) { i = i + 1 } else { st = 1 }
546 }
547 if body[i] != (0 as u8) { return 0 }
548 out[0] = v
549 return 1
550}
551
552// ===== ARG-MACRO EXPANSION (source-level pre-pass) ==================================================
553// WHY A PRE-PASS AND NOT A TOKEN-STREAM REWRITE: expanding at the token level needs a re-entrant lexer
554// (push a synthetic source, lex it, pop) which touches peek/advance/L.pos -- the core of a compiler that
555// builds 10,822 targets. A textual pre-pass is ADDITIVE: if no arg-macro is used it returns 0 and the
556// caller lexes the ORIGINAL buffer, so the existing path is byte-for-byte unchanged.
557//
558// The stored body ALREADY carries the params: `@macro SQ(x) (x * x)` stores name="SQ",
559// body="(x) (x * x)". So an arg macro is exactly "a macro whose body starts with '('" -- no extra table
560// column, and the zero-arg numeric path is untouched.
561
562// ONE named bound for BOTH the parameter cap and the argument cap. They used to be two independent
563// bare `8` literals in different functions -- nothing tied them together, so they could drift apart and
564// mis-substitute in a THIRD way.
565const LEXM_MAXARGS: i64 = 8
566
567func lexm_warn_too_many_args(got: i64) -> i64 {
568 let buf: *u8 = lexm_diag_cell()
569 var o: i64 = 0
570 o = lexm_emit_str(buf, o, "nx_tokenizer: @macro invocation uses " as *u8)
571 o = lexm_emit_dec(buf, o, got)
572 o = lexm_emit_str(buf, o, " arguments, over the limit of " as *u8)
573 o = lexm_emit_dec(buf, o, LEXM_MAXARGS)
574 o = lexm_emit_str(buf, o, " -- REFUSED, not truncated. Previously the extra arguments were dropped and their parameter names were emitted LITERALLY, which could compile and run with wrong values.\n" as *u8)
575 sys_write(2, buf, o)
576 return 0
577}
578
579// The identifier storage bound. Named, not a bare 63, because nx_parse.nx depends on the SAME contract
580// (a NUL inside 64 bytes) at ~14 sites.
581const LEX_IDENT_MAX: i64 = 63
582
583func lex_warn_ident_too_long(text: *u8, kept: i64, actual: i64) -> i64 {
584 let buf: *u8 = lexm_diag_cell()
585 var o: i64 = 0
586 o = lexm_emit_str(buf, o, "nx_tokenizer: identifier is " as *u8)
587 o = lexm_emit_dec(buf, o, actual)
588 o = lexm_emit_str(buf, o, " chars, over the limit of " as *u8)
589 o = lexm_emit_dec(buf, o, LEX_IDENT_MAX)
590 o = lexm_emit_str(buf, o, " -- kept '" as *u8)
591 var i: i64 = 0
592 while i < kept {
593 if o < 300 { buf[o] = text[i]; o = o + 1 }
594 i = i + 1
595 }
596 o = lexm_emit_str(buf, o, "'. Two identifiers sharing these leading chars would COLLIDE. Shorten the name; if it is macro-generated, shorten the prefix or the loop variable.\n" as *u8)
597 sys_write(2, buf, o)
598 return 0
599}
600
601func lexm_isid(c: i64) -> i64 { return is_alnum(c) }
602
603// Copy body-after-params into out, replacing each param NAME with the matching actual-arg text.
604// argp/argl carry up to 8 actual args as (offset,len) into asrc.
605func lexm_subst(body: *u8, asrc: *u8, argp: *i64, argl: *i64, nargs: i64, out: *u8, o0: i64) -> i64 {
606 var o: i64 = o0
607 var pn: i64 = 0
608 let pofs: *i64 = sys_mmap(128) as *i64
609 let plen: *i64 = sys_mmap(128) as *i64
610 var i: i64 = 1
611 var s1: i64 = 0
612 while s1 == 0 {
613 if body[i] == (0 as u8) { s1 = 1 } else {
614 if body[i] == (41 as u8) { s1 = 1 } else {
615 if lexm_isid(body[i] as i64) == 1 {
616 let ps: i64 = i
617 var s2: i64 = 0
618 while s2 == 0 { if lexm_isid(body[i] as i64) == 1 { i = i + 1 } else { s2 = 1 } }
619 // REFUSE, never truncate: a 9th parameter that is silently not registered is never
620 // recognised in the body, falls through to copy-verbatim, and is emitted as a LITERAL
621 // identifier -- which can compile and run with the wrong value.
622 if pn < LEXM_MAXARGS { pofs[pn] = ps; plen[pn] = i - ps; pn = pn + 1 } else { pn = pn + 1 }
623 } else { i = i + 1 }
624 } }
625 }
626 // REFUSE the whole expansion if the macro declares more parameters than we can hold. Emitting a
627 // partially-substituted body is a SILENT MISCOMPILE; returning o0 unchanged emits nothing, so the
628 // macro name survives to the parser and fails loudly as `undefined function`.
629 // Also protects the matching loop below, which would otherwise index pofs/plen past their mapping.
630 if pn > LEXM_MAXARGS { lexm_warn_too_many_args(pn); return o0 }
631
632 var b: i64 = i
633 if body[b] == (41 as u8) { b = b + 1 }
634 var s3: i64 = 0
635 while s3 == 0 { if body[b] == (32 as u8) { b = b + 1 } else { s3 = 1 } }
636
637 // A `{ ... }` body emits its INTERIOR: the braces delimit the macro, they are not part of the
638 // expansion. Any other body runs to the end of the stored text.
639 let blen: i64 = lexm_strlen(body)
640 var k: i64 = b
641 var kend: i64 = blen
642 if body[b] == (123 as u8) {
643 k = b + 1
644 kend = lexm_body_end(body, blen, b) - 1
645 }
646
647 // ONE gensym serial per INVOCATION. Every `$$name` in this expansion shares it, so `let $$got` and a
648 // later `$$got` in the same body resolve to the SAME identifier -- while a second invocation takes
649 // the next serial, so two expansions in one scope cannot collide. Both halves are load-bearing:
650 // sharing within a body is what makes the macro compile, and differing across bodies is what makes
651 // two uses in one function legal.
652 lexm_gensym[0] = lexm_gensym[0] + 1
653 let serial: i64 = lexm_gensym[0]
654
655 while k < kend {
656 if body[k] == (36 as u8) {
657 // ${param} -- BRACED parameter reference inside a @macro body. Needed so a substitution can
658 // ABUT following identifier characters: `nx_${name}_apply` must paste to `nx_foo_apply`,
659 // which a bare `$name` cannot express because `name_apply` reads as one identifier run.
660 // Checked BEFORE the $$ gensym test: `${` and `$$` are distinct two-char openers.
661 var brc: i64 = 0
662 if k + 1 < kend { if body[k+1] == (123 as u8) { brc = 1 } }
663 if brc == 1 {
664 // ${$$name} -- a BRACED GENSYM reference. Braces only DELIMIT, so this must mean exactly
665 // what bare `$$name` means; the braces exist so the gensym can ABUT following identifier
666 // characters (`describe_${$$counter}()`), which bare `$$counter` cannot express because
667 // `counter` would swallow the following text as one identifier run.
668 // Same rule as ${param}: same answer, explicit boundary.
669 var gbr: i64 = 0
670 if k + 3 < kend { if body[k+2] == (36 as u8) { if body[k+3] == (36 as u8) { gbr = 1 } } }
671 if gbr == 1 {
672 var gk: i64 = k + 4
673 let ggs: i64 = gk
674 var gs1: i64 = 0
675 while gs1 == 0 {
676 if gk < kend { if lexm_isid(body[gk] as i64) == 1 { gk = gk + 1 } else { gs1 = 1 } } else { gs1 = 1 }
677 }
678 var gclosed: i64 = 0
679 if gk < kend { if body[gk] == (125 as u8) { gclosed = 1 } }
680 if gclosed == 1 {
681 o = lexm_emit_str(out, o, "__nx_g_" as *u8)
682 var gg: i64 = ggs
683 while gg < gk { out[o] = body[gg]; o = o + 1; gg = gg + 1 }
684 out[o] = 95 as u8
685 o = o + 1
686 o = lexm_emit_dec(out, o, serial)
687 k = gk + 1
688 gbr = 2
689 }
690 }
691 if gbr == 2 { } else {
692 let bs2: i64 = k + 2
693 var be2: i64 = bs2
694 var sb: i64 = 0
695 while sb == 0 {
696 if be2 < kend { if lexm_isid(body[be2] as i64) == 1 { be2 = be2 + 1 } else { sb = 1 } } else { sb = 1 }
697 }
698 var closed: i64 = 0
699 if be2 < kend { if body[be2] == (125 as u8) { closed = 1 } }
700 if closed == 1 {
701 let bl2: i64 = be2 - bs2
702 var bh: i64 = 0 - 1
703 var bp2: i64 = 0
704 while bp2 < pn {
705 if plen[bp2] == bl2 {
706 var beq: i64 = 1
707 var bz: i64 = 0
708 while bz < bl2 { if body[pofs[bp2] + bz] != body[bs2 + bz] { beq = 0; bz = bl2 } else { bz = bz + 1 } }
709 if beq == 1 { bh = bp2 }
710 }
711 bp2 = bp2 + 1
712 }
713 if bh >= 0 {
714 if bh < nargs {
715 var ba: i64 = 0
716 while ba < argl[bh] { out[o] = asrc[argp[bh] + ba]; o = o + 1; ba = ba + 1 }
717 }
718 k = be2 + 1
719 } else {
720 // Not one of THIS macro's parameters -- copy verbatim so a @for's ${var} inside a
721 // macro body survives to the @for pass instead of being silently deleted.
722 while k <= be2 { out[o] = body[k]; o = o + 1; k = k + 1 }
723 }
724 } else { out[o] = body[k]; o = o + 1; k = k + 1 }
725 }
726 } else {
727 var dbl: i64 = 0
728 if k + 1 < kend { if body[k+1] == (36 as u8) { dbl = 1 } }
729 if dbl == 1 {
730 k = k + 2
731 let gs: i64 = k
732 var s6: i64 = 0
733 while s6 == 0 {
734 if k < kend { if lexm_isid(body[k] as i64) == 1 { k = k + 1 } else { s6 = 1 } } else { s6 = 1 }
735 }
736 o = lexm_emit_str(out, o, "__nx_g_" as *u8)
737 var g: i64 = gs
738 while g < k { out[o] = body[g]; o = o + 1; g = g + 1 }
739 out[o] = 95 as u8
740 o = o + 1
741 o = lexm_emit_dec(out, o, serial)
742 } else { out[o] = body[k]; o = o + 1; k = k + 1 }
743 }
744 } else {
745 if lexm_isid(body[k] as i64) == 1 {
746 let ts: i64 = k
747 var s5: i64 = 0
748 while s5 == 0 {
749 if k < kend { if lexm_isid(body[k] as i64) == 1 { k = k + 1 } else { s5 = 1 } } else { s5 = 1 }
750 }
751 let tl: i64 = k - ts
752 var hit: i64 = 0 - 1
753 var p: i64 = 0
754 while p < pn {
755 if plen[p] == tl {
756 var eq: i64 = 1
757 var z: i64 = 0
758 while z < tl { if body[pofs[p] + z] != body[ts + z] { eq = 0; z = tl } else { z = z + 1 } }
759 if eq == 1 { hit = p }
760 }
761 p = p + 1
762 }
763 if hit >= 0 {
764 if hit < nargs {
765 var a: i64 = 0
766 while a < argl[hit] { out[o] = asrc[argp[hit] + a]; o = o + 1; a = a + 1 }
767 }
768 } else {
769 var y: i64 = ts
770 while y < k { out[o] = body[y]; o = o + 1; y = y + 1 }
771 }
772 } else { out[o] = body[k]; o = o + 1; k = k + 1 } }
773 }
774 return o
775}
776
777// Scan src for `@macro` definitions so they are known BEFORE any use is expanded. The tokenizer's own
778// directive handler re-defines them later; lexm_define OVERWRITES, so that is a harmless no-op.
779func lexm_prescan(src: *u8, n: i64) -> i64 {
780 var i: i64 = 0
781 while i + 7 < n {
782 var hit: i64 = 0
783 if src[i] == (64 as u8) {
784 if src[i+1] == (109 as u8) { if src[i+2] == (97 as u8) { if src[i+3] == (99 as u8) {
785 if src[i+4] == (114 as u8) { if src[i+5] == (111 as u8) { if src[i+6] == (32 as u8) { hit = 1 } } } } } }
786 }
787 if hit == 1 {
788 var np: i64 = i + 7
789 var k1: i64 = 0
790 while k1 == 0 { if src[np] == (32 as u8) { np = np + 1 } else { k1 = 1 } }
791 var ne: i64 = np
792 var k2: i64 = 0
793 while k2 == 0 { if lexm_isid(src[ne] as i64) == 1 { ne = ne + 1 } else { k2 = 1 } }
794 var bp: i64 = ne
795 var k3: i64 = 0
796 while k3 == 0 { if src[bp] == (32 as u8) { bp = bp + 1 } else { k3 = 1 } }
797 let be: i64 = lexm_body_end(src, n, bp)
798 lexm_define(((src as i64) + np) as *u8, ne - np, ((src as i64) + bp) as *u8, be - bp)
799 i = be
800 }
801 i = i + 1
802 }
803 return 0
804}
805
806// ===== @for GENERATIVE REPETITION ===================================================================
807// @for(i in 0..5) { func atom_$i() -> nx_int { return $i * 100 } }
808// @for(z in 1..ATOM_COUNT) { func mass_$z() -> nx_int { return $z * 10 } }
809// @for(op in {add, sub, mul}) { func calc_${op}(a: nx_int, b: nx_int) -> nx_int { ... } }
810//
811// RANGE IS HALF-OPEN (0..5 yields 0,1,2,3,4) -- fixed by the tests, not by taste: nx_macro_v5_test
812// checks atom_0..atom_4 and never atom_5. Half-open also makes 0..N mean N iterations, the only
813// convention that composes.
814//
815// The braced form ${op} exists so a substitution can ABUT following identifier characters:
816// calc_${op}(...) must paste to calc_add, whereas a bare $op cannot tell where the name ends when the
817// next character is itself an identifier character. Bare $i stays legal where the boundary is clear.
818
819// Resolve a @for bound: a decimal literal, or a macro name whose body is a plain integer.
820// Trims its own leading/trailing spaces. An earlier draft trimmed in the CALLER with a loop whose
821// else-branch was a no-op assignment -- any non-space character spun FOREVER. An infinite loop in the
822// tokenizer hangs every build in the ecosystem and reports nothing, so trimming lives HERE, once,
823// where it terminates by construction.
824func lexm_for_bound(src: *u8, s0: i64, e0: i64, out: *i64) -> i64 {
825 var s: i64 = s0
826 var e: i64 = e0
827 var t1: i64 = 0
828 while t1 == 0 { if s < e { if src[s] == (32 as u8) { s = s + 1 } else { t1 = 1 } } else { t1 = 1 } }
829 var t2: i64 = 0
830 while t2 == 0 { if e > s { if src[e-1] == (32 as u8) { e = e - 1 } else { t2 = 1 } } else { t2 = 1 } }
831 if e <= s { return 0 }
832 var alldig: i64 = 1
833 var k: i64 = s
834 while k < e {
835 let c: i64 = src[k] as i64
836 var isd: i64 = 0
837 if c >= 48 { if c <= 57 { isd = 1 } }
838 if isd == 0 { alldig = 0; k = e } else { k = k + 1 }
839 }
840 if alldig == 1 {
841 var v: i64 = 0
842 var j: i64 = s
843 while j < e { v = v * 10 + ((src[j] as i64) - 48); j = j + 1 }
844 out[0] = v
845 return 1
846 }
847 let mb: *u8 = lexm_lookup(((src as i64) + s) as *u8, e - s)
848 if (mb as i64) == 0 { return 0 }
849 return lexm_body_int(mb, out)
850}
851
852// Emit body [bs,be) once, replacing $VAR and ${VAR} with vtxt[0..vtl).
853// A `$` not followed by THIS loop's variable is copied through untouched, so $$ident gensym inside a
854// @for body still reaches lexm_subst intact.
855// `ocap` is load-bearing: a @for emits BODY x ITERATIONS, far exceeding the single-body headroom the
856// caller checked before entering. Returns 0-1 on overflow so the whole @for is REFUSED and the text
857// reaches the parser unexpanded -- a loud failure instead of a write past the buffer.
858func lexm_for_emit(src: *u8, bs: i64, be: i64, vname: *u8, vlen: i64, vtxt: *u8, vtl: i64, out: *u8, o0: i64, ocap: i64) -> i64 {
859 var o: i64 = o0
860 var k: i64 = bs
861 while k < be {
862 // Reserve vtl too, not a bare constant: each pass can emit the substituted TOKEN, so a long
863 // value would step past a fixed cushion and overrun anyway.
864 if o >= ocap - vtl - 64 { return 0 - 1 }
865 if src[k] == (36 as u8) {
866 var p: i64 = k + 1
867 var braced: i64 = 0
868 if p < be { if src[p] == (123 as u8) { braced = 1; p = p + 1 } }
869 let ns: i64 = p
870 var st: i64 = 0
871 while st == 0 {
872 if p < be { if lexm_isid(src[p] as i64) == 1 { p = p + 1 } else { st = 1 } } else { st = 1 }
873 }
874 var nl: i64 = p - ns
875 var hit: i64 = 0
876 // EXACT run match first: `$i` followed by a non-identifier char.
877 if nl == vlen {
878 var eq: i64 = 1
879 var z: i64 = 0
880 while z < nl { if src[ns + z] != vname[z] { eq = 0; z = nl } else { z = z + 1 } }
881 hit = eq
882 }
883 // PREFIX match second, and it is REQUIRED for nested @for: `pair_$i_$j` reads the run after
884 // `$` as `i_` (underscore IS an identifier char), which never equals `i`, so an exact-only
885 // matcher silently leaves `$i_$j` unexpanded and nested loops cannot work at all.
886 // Consuming just the variable name and leaving the rest is what makes `$i_$j` paste.
887 // `${i}` remains the way to disambiguate when the following text could itself be the name.
888 if hit == 0 {
889 if braced == 0 {
890 if nl > vlen {
891 var eq2: i64 = 1
892 var z2: i64 = 0
893 while z2 < vlen { if src[ns + z2] != vname[z2] { eq2 = 0; z2 = vlen } else { z2 = z2 + 1 } }
894 if eq2 == 1 { hit = 1; p = ns + vlen; nl = vlen }
895 }
896 }
897 }
898 if braced == 1 {
899 if p < be {
900 if src[p] == (125 as u8) { p = p + 1 } else { hit = 0 }
901 } else { hit = 0 }
902 }
903 if hit == 1 {
904 var a: i64 = 0
905 while a < vtl { out[o] = vtxt[a]; o = o + 1; a = a + 1 }
906 k = p
907 } else { out[o] = src[k]; o = o + 1; k = k + 1 }
908 } else { out[o] = src[k]; o = o + 1; k = k + 1 }
909 }
910 return o
911}
912
913// ===== @concat / @stringify -- TOKEN PASTING AND TEXT CAPTURE =======================================
914// @macro DEFINE_DOUBLE(prefix) { func @concat(prefix, _double_x)(x: nx_int) -> nx_int { ... } }
915// DEFINE_DOUBLE(foo) => func foo_double_x(x: nx_int) -> nx_int { ... }
916// @stringify(name) => "name"
917//
918// THE ORDERING IS THE WHOLE DESIGN, AND IT IS ALREADY SOLVED. @concat(prefix, _double_x) must see
919// `prefix` ALREADY replaced by `foo`, or it pastes the PARAMETER NAME and silently emits
920// `prefix_double_x` -- a function nobody called, plus an undefined-function error at every call site
921// naming the right symbol for the wrong reason. No new machinery is needed: lexm_subst substitutes and
922// emits text, and lex_source ALREADY iterates lexm_expand to a fixed point, so pass N emits
923// `@concat(foo, _double_x)` and pass N+1 splices it. This is why the fixed-point loop was worth
924// building even when one pass looked sufficient.
925
926// Trim spaces/tabs/newlines from both ends of [s,e); returns new start, writes new end to eout.
927func lexm_trim(src: *u8, s: i64, e: i64, eout: *i64) -> i64 {
928 var a: i64 = s
929 var b: i64 = e
930 var t1: i64 = 0
931 while t1 == 0 {
932 if a < b {
933 let c: i64 = src[a] as i64
934 if c == 32 { a = a + 1 } else { if c == 9 { a = a + 1 } else {
935 if c == 10 { a = a + 1 } else { if c == 13 { a = a + 1 } else { t1 = 1 } } } }
936 } else { t1 = 1 }
937 }
938 var t2: i64 = 0
939 while t2 == 0 {
940 if b > a {
941 let c: i64 = src[b-1] as i64
942 if c == 32 { b = b - 1 } else { if c == 9 { b = b - 1 } else {
943 if c == 10 { b = b - 1 } else { if c == 13 { b = b - 1 } else { t2 = 1 } } } }
944 } else { t2 = 1 }
945 }
946 eout[0] = b
947 return a
948}
949
950func lexm_is_ident_text(p: *u8, s: i64, e: i64) -> i64 {
951 if e <= s { return 0 }
952 let c0: i64 = p[s] as i64
953 var okstart: i64 = 0
954 if c0 == 95 { okstart = 1 }
955 if is_alpha(c0) == 1 { okstart = 1 }
956 if okstart == 0 { return 0 }
957 var i: i64 = s
958 while i < e {
959 if lexm_isid(p[i] as i64) == 0 { return 0 }
960 i = i + 1
961 }
962 return 1
963}
964
965// A pasted result must be a legal identifier, or the failure surfaces far away as
966// `reserved keyword '.' cannot be used as an identifier name` naming a line that is not the cause.
967// REFUSE LEGIBLY: name the result.
968func lexm_warn_bad_paste(out: *u8, s: i64, e: i64) -> i64 {
969 let buf: *u8 = lexm_diag_cell()
970 var o: i64 = 0
971 o = lexm_emit_str(buf, o, "nx_tokenizer: @concat produced '" as *u8)
972 var i: i64 = s
973 while i < e {
974 if o < 300 { buf[o] = out[i]; o = o + 1 }
975 i = i + 1
976 }
977 o = lexm_emit_str(buf, o, "', which is not a legal identifier (must start with a letter or _ and contain only letters, digits, _). Check that every @concat argument is a bare token.\n" as *u8)
978 sys_write(2, buf, o)
979 return 0
980}
981
982// Shared parser for @concat(...) / @stringify(...).
983// mode 0 = concat (paste args, no separator), mode 1 = stringify (quote the arg text).
984func lexm_paste_expand(src: *u8, n: i64, istart: i64, kwlen: i64, mode: i64, out: *u8, o0: i64, ocap: i64, iout: *i64) -> i64 {
985 var p: i64 = istart + kwlen
986 var sk: i64 = 0
987 while sk == 0 { if p < n { if src[p] == (32 as u8) { p = p + 1 } else { sk = 1 } } else { sk = 1 } }
988 if p >= n { return 0 - 1 }
989 if src[p] != (40 as u8) { return 0 - 1 }
990
991 let argstart: i64 = p + 1
992 var d: i64 = 1
993 var q: i64 = argstart
994 var st: i64 = 0
995 while st == 0 {
996 if q >= n { st = 1 } else {
997 let c: i64 = src[q] as i64
998 if c == 0 { st = 1 } else {
999 if c == 40 { d = d + 1; q = q + 1 } else {
1000 if c == 41 { d = d - 1; if d == 0 { st = 1 } else { q = q + 1 } } else { q = q + 1 } } }
1001 }
1002 }
1003 if q >= n { return 0 - 1 }
1004 let argend: i64 = q
1005
1006 var o: i64 = o0
1007 if o >= ocap - (argend - argstart) - 64 { return 0 - 1 }
1008
1009 let eout: *i64 = sys_mmap(16) as *i64
1010 let rstart: i64 = o
1011
1012 if mode == 1 { out[o] = 34 as u8; o = o + 1 }
1013
1014 var cur: i64 = argstart
1015 var k: i64 = argstart
1016 var d2: i64 = 0
1017 var go: i64 = 1
1018 while go == 1 {
1019 var flush: i64 = 0
1020 if k >= argend { flush = 1; go = 0 } else {
1021 let c: i64 = src[k] as i64
1022 if c == 40 { d2 = d2 + 1 } else { if c == 41 { d2 = d2 - 1 } else {
1023 if c == 44 { if d2 == 0 { flush = 1 } } } }
1024 }
1025 if flush == 1 {
1026 let ts: i64 = lexm_trim(src, cur, k, eout)
1027 let te: i64 = eout[0]
1028 var z: i64 = ts
1029 while z < te { out[o] = src[z]; o = o + 1; z = z + 1 }
1030 cur = k + 1
1031 }
1032 if go == 1 { k = k + 1 }
1033 }
1034
1035 if mode == 1 { out[o] = 34 as u8; o = o + 1 }
1036
1037 // Validate only the CONCAT result -- a stringify body is arbitrary text by definition.
1038 if mode == 0 {
1039 if lexm_is_ident_text(out, rstart, o) == 0 { lexm_warn_bad_paste(out, rstart, o) }
1040 }
1041
1042 iout[0] = argend + 1
1043 return o
1044}
1045
1046func lexm_kw_is(src: *u8, n: i64, at: i64, kw: *u8, kwlen: i64) -> i64 {
1047 if at + kwlen > n { return 0 }
1048 var i: i64 = 0
1049 while i < kwlen {
1050 if src[at + i] != kw[i] { return 0 }
1051 i = i + 1
1052 }
1053 return 1
1054}
1055
1056// Expand one `@for(...) { ... }` starting at istart (which points at '@').
1057// Returns the new output offset and writes the new input offset to iout[0]; 0-1 if not well-formed.
1058// REFUSING is the honest failure here: a @for that silently expands to nothing DELETES code.
1059func lexm_for_expand(src: *u8, n: i64, istart: i64, out: *u8, o0: i64, ocap: i64, iout: *i64) -> i64 {
1060 var p: i64 = istart + 4
1061 if p >= n { return 0 - 1 }
1062 var sk: i64 = 0
1063 while sk == 0 { if p < n { if src[p] == (32 as u8) { p = p + 1 } else { sk = 1 } } else { sk = 1 } }
1064 if p >= n { return 0 - 1 }
1065 if src[p] != (40 as u8) { return 0 - 1 }
1066 p = p + 1
1067 var sk2: i64 = 0
1068 while sk2 == 0 { if p < n { if src[p] == (32 as u8) { p = p + 1 } else { sk2 = 1 } } else { sk2 = 1 } }
1069
1070 let vs: i64 = p
1071 var sk3: i64 = 0
1072 while sk3 == 0 { if p < n { if lexm_isid(src[p] as i64) == 1 { p = p + 1 } else { sk3 = 1 } } else { sk3 = 1 } }
1073 let vlen: i64 = p - vs
1074 if vlen <= 0 { return 0 - 1 }
1075
1076 var sk4: i64 = 0
1077 while sk4 == 0 { if p < n { if src[p] == (32 as u8) { p = p + 1 } else { sk4 = 1 } } else { sk4 = 1 } }
1078 if p + 2 >= n { return 0 - 1 }
1079 if src[p] != (105 as u8) { return 0 - 1 }
1080 if src[p+1] != (110 as u8) { return 0 - 1 }
1081 p = p + 2
1082 var sk5: i64 = 0
1083 while sk5 == 0 { if p < n { if src[p] == (32 as u8) { p = p + 1 } else { sk5 = 1 } } else { sk5 = 1 } }
1084 if p >= n { return 0 - 1 }
1085
1086 var listmode: i64 = 0
1087 if src[p] == (123 as u8) { listmode = 1 }
1088
1089 // Find the matching ')' by DEPTH -- the value-list form uses braces inside the parens, so a
1090 // first-')' scan would cut the sequence short.
1091 let seqs: i64 = p
1092 var d: i64 = 1
1093 var q: i64 = p
1094 var sk6: i64 = 0
1095 while sk6 == 0 {
1096 if q >= n { sk6 = 1 } else {
1097 let c: i64 = src[q] as i64
1098 if c == 0 { sk6 = 1 } else {
1099 if c == 40 { d = d + 1; q = q + 1 } else {
1100 if c == 41 { d = d - 1; if d == 0 { sk6 = 1 } else { q = q + 1 } } else { q = q + 1 } } }
1101 }
1102 }
1103 if q >= n { return 0 - 1 }
1104 let seqe: i64 = q
1105 p = q + 1
1106
1107 var sk7: i64 = 0
1108 while sk7 == 0 {
1109 if p < n {
1110 let c: i64 = src[p] as i64
1111 if c == 32 { p = p + 1 } else { if c == 10 { p = p + 1 } else { if c == 13 { p = p + 1 } else { if c == 9 { p = p + 1 } else { sk7 = 1 } } } }
1112 } else { sk7 = 1 }
1113 }
1114 if p >= n { return 0 - 1 }
1115 if src[p] != (123 as u8) { return 0 - 1 }
1116 let bodyclose: i64 = lexm_body_end(src, n, p)
1117 if bodyclose <= p { return 0 - 1 }
1118 let bs: i64 = p + 1
1119 let be: i64 = bodyclose - 1
1120
1121 let vname: *u8 = ((src as i64) + vs) as *u8
1122 var o: i64 = o0
1123
1124 if listmode == 1 {
1125 var t: i64 = seqs + 1
1126 var stop: i64 = 0
1127 while stop == 0 {
1128 if t >= seqe { stop = 1 } else {
1129 var sk8: i64 = 0
1130 while sk8 == 0 {
1131 if t < seqe {
1132 let c: i64 = src[t] as i64
1133 if c == 32 { t = t + 1 } else { if c == 44 { t = t + 1 } else { if c == 10 { t = t + 1 } else { if c == 125 { t = t + 1 } else { sk8 = 1 } } } }
1134 } else { sk8 = 1 }
1135 }
1136 if t >= seqe { stop = 1 } else {
1137 let ts: i64 = t
1138 var sk9: i64 = 0
1139 while sk9 == 0 {
1140 if t < seqe { if lexm_isid(src[t] as i64) == 1 { t = t + 1 } else { sk9 = 1 } } else { sk9 = 1 }
1141 }
1142 let tl: i64 = t - ts
1143 if tl <= 0 { stop = 1 } else {
1144 let eo: i64 = lexm_for_emit(src, bs, be, vname, vlen, ((src as i64) + ts) as *u8, tl, out, o, ocap)
1145 if eo < 0 { return 0 - 1 }
1146 o = eo
1147 out[o] = 10 as u8
1148 o = o + 1
1149 }
1150 }
1151 }
1152 }
1153 } else {
1154 var dots: i64 = 0 - 1
1155 var r: i64 = seqs
1156 while r + 1 < seqe {
1157 if src[r] == (46 as u8) { if src[r+1] == (46 as u8) { dots = r; r = seqe } else { r = r + 1 } } else { r = r + 1 }
1158 }
1159 if dots < 0 { return 0 - 1 }
1160 let lo_out: *i64 = sys_mmap(16) as *i64
1161 let hi_out: *i64 = sys_mmap(16) as *i64
1162 if lexm_for_bound(src, seqs, dots, lo_out) == 0 { return 0 - 1 }
1163 if lexm_for_bound(src, dots + 2, seqe, hi_out) == 0 { return 0 - 1 }
1164 let numbuf: *u8 = sys_mmap(64)
1165 var v: i64 = lo_out[0]
1166 while v < hi_out[0] {
1167 // lexm_emit_dec guards on v <= 0, so v == 0 correctly yields the single digit "0".
1168 let nl2: i64 = lexm_emit_dec(numbuf, 0, v)
1169 let eo2: i64 = lexm_for_emit(src, bs, be, vname, vlen, numbuf, nl2, out, o, ocap)
1170 if eo2 < 0 { return 0 - 1 }
1171 o = eo2
1172 out[o] = 10 as u8
1173 o = o + 1
1174 v = v + 1
1175 }
1176 }
1177
1178 iout[0] = bodyclose
1179 return o
1180}
1181
1182// Expand every ARG-macro use into out. Returns the expanded length, or 0 if nothing was expanded (in
1183// which case the caller MUST keep using the original buffer -- returning a copy would be a silent
1184// behaviour change on every file that uses no macros).
1185func lexm_expand(src: *u8, n: i64, out: *u8, outcap: i64) -> i64 {
1186 lexm_prescan(src, n)
1187 let argp: *i64 = sys_mmap(128) as *i64
1188 let argl: *i64 = sys_mmap(128) as *i64
1189 var did: i64 = 0
1190 var o: i64 = 0
1191 var i: i64 = 0
1192 while i < n {
1193 // Headroom for ONE emission: a whole `{ ... }` body plus the actual-arg text substituted into it.
1194 // Checked before each iteration so a single expansion can never run past the buffer. Bailing with
1195 // 0 makes the caller lex the ORIGINAL source, so the macro name reaches the parser unexpanded and
1196 // fails loudly rather than emitting a half-written body.
1197 if o >= outcap - LEXM_BODYLEN * 8 { return 0 }
1198 // COPY AN @macro DEFINITION LINE VERBATIM. `@macro AXIOM_CONST(ch, sym_id) <body>` contains the
1199 // text `AXIOM_CONST(` -- so without this the expander expands a macro INSIDE ITS OWN DEFINITION,
1200 // the definition line becomes garbage, and the NEXT iteration's prescan stores that garbage as the
1201 // body. The corruption then cascades and surfaces far away as `reserved keyword '.' cannot be used
1202 // as an identifier name`, pointing at a line that is not the cause.
1203 // STRING LITERALS ARE NOT EXPANDABLE TEXT. Copy them through verbatim, escapes included.
1204 // Without this, ANY macro name or @directive appearing inside a string literal gets expanded:
1205 // nx_macro_v7_test's own PASS message contains the text `@stringify(name)`, which was spliced
1206 // into a nested quote, broke the literal, and surfaced as `UNRESOLVED identifier 'name' in fn
1207 // main` -- an error naming a line that merely DESCRIBES the feature.
1208 // Same class as the @macro self-corruption: text that must not be expanded, being expanded.
1209 var handled: i64 = 0
1210 if src[i] == (34 as u8) {
1211 out[o] = src[i]; o = o + 1; i = i + 1
1212 var sq: i64 = 0
1213 while sq == 0 {
1214 if i >= n { sq = 1 } else {
1215 if src[i] == (0 as u8) { sq = 1 } else {
1216 if src[i] == (92 as u8) {
1217 out[o] = src[i]; o = o + 1; i = i + 1
1218 if i < n { out[o] = src[i]; o = o + 1; i = i + 1 }
1219 } else {
1220 if src[i] == (34 as u8) { out[o] = src[i]; o = o + 1; i = i + 1; sq = 1 } else {
1221 out[o] = src[i]; o = o + 1; i = i + 1
1222 } } }
1223 }
1224 }
1225 handled = 1
1226 }
1227 if handled == 0 {
1228 if src[i] == (64 as u8) {
1229 var isdef: i64 = 0
1230 if i + 7 < n {
1231 if src[i+1] == (109 as u8) { if src[i+2] == (97 as u8) { if src[i+3] == (99 as u8) {
1232 if src[i+4] == (114 as u8) { if src[i+5] == (111 as u8) { if src[i+6] == (32 as u8) { isdef = 1 } } } } } }
1233 }
1234 if isdef == 1 {
1235 // Copy through the END OF THE BODY. For a `{ ... }` macro that is the matching close
1236 // brace many lines below; copying only to end-of-line would leave the body itself exposed
1237 // to expansion and the definition would corrupt ITSELF -- the bug that previously
1238 // surfaced ~1900 lines away as `reserved keyword '.'`, pointing at an innocent line.
1239 var mp: i64 = i + 7
1240 var m1: i64 = 0
1241 while m1 == 0 { if src[mp] == (32 as u8) { mp = mp + 1 } else { m1 = 1 } }
1242 var mn: i64 = mp
1243 var m2: i64 = 0
1244 while m2 == 0 { if lexm_isid(src[mn] as i64) == 1 { mn = mn + 1 } else { m2 = 1 } }
1245 var mb: i64 = mn
1246 var m3: i64 = 0
1247 while m3 == 0 { if src[mb] == (32 as u8) { mb = mb + 1 } else { m3 = 1 } }
1248 let de: i64 = lexm_body_end(src, n, mb)
1249 while i < de { out[o] = src[i]; o = o + 1; i = i + 1 }
1250 handled = 1
1251 }
1252 }
1253 // @for(...) { ... } -- generative repetition. Placed AFTER the @macro-verbatim branch so a
1254 // `@for` written INSIDE a @macro body is copied whole rather than expanded within its own
1255 // definition (the self-corruption that once surfaced ~1900 lines away as `reserved keyword '.'`).
1256 // `@format(` cannot false-match: lexm_for_expand skips spaces after "@for" then REQUIRES '(',
1257 // so "mat(" fails, returns 0-1, and the text is copied verbatim.
1258 if handled == 0 {
1259 if src[i] == (64 as u8) {
1260 var isfor: i64 = 0
1261 if i + 4 < n {
1262 if src[i+1] == (102 as u8) { if src[i+2] == (111 as u8) { if src[i+3] == (114 as u8) { isfor = 1 } } }
1263 }
1264 if isfor == 1 {
1265 let iout: *i64 = sys_mmap(16) as *i64
1266 let no: i64 = lexm_for_expand(src, n, i, out, o, outcap, iout)
1267 if no >= 0 { o = no; i = iout[0]; did = 1; handled = 1 }
1268 }
1269 // @concat / @stringify -- same placement rationale as @for: AFTER the @macro-verbatim
1270 // branch, so a @concat inside a macro BODY is copied whole and only spliced on the NEXT
1271 // fixed-point pass, once its parameters have been substituted. Splicing before
1272 // substitution would paste the PARAMETER NAME (`prefix_double_x`), a function nobody
1273 // called.
1274 if handled == 0 {
1275 let iout2: *i64 = sys_mmap(16) as *i64
1276 var no2: i64 = 0 - 1
1277 if lexm_kw_is(src, n, i, "@concat" as *u8, 7) == 1 {
1278 no2 = lexm_paste_expand(src, n, i, 7, 0, out, o, outcap, iout2)
1279 } else {
1280 if lexm_kw_is(src, n, i, "@stringify" as *u8, 10) == 1 {
1281 no2 = lexm_paste_expand(src, n, i, 10, 1, out, o, outcap, iout2)
1282 } }
1283 if no2 >= 0 { o = no2; i = iout2[0]; did = 1; handled = 1 }
1284 }
1285 }
1286 }
1287 }
1288 if handled == 1 { } else {
1289 if lexm_isid(src[i] as i64) == 1 {
1290 let ns: i64 = i
1291 var s1: i64 = 0
1292 while s1 == 0 { if i < n { if lexm_isid(src[i] as i64) == 1 { i = i + 1 } else { s1 = 1 } } else { s1 = 1 } }
1293 let nl: i64 = i - ns
1294 var used: i64 = 0
1295 if i < n {
1296 if src[i] == (40 as u8) {
1297 let mb: *u8 = lexm_lookup(((src as i64) + ns) as *u8, nl)
1298 if (mb as i64) != 0 {
1299 if mb[0] == (40 as u8) {
1300 var nargs: i64 = 0
1301 var d: i64 = 0
1302 var a0: i64 = i + 1
1303 var p: i64 = i
1304 var s2: i64 = 0
1305 while s2 == 0 {
1306 if p >= n { s2 = 1 } else {
1307 let c: i64 = src[p] as i64
1308 if c == 40 { d = d + 1 } else {
1309 if c == 41 { d = d - 1
1310 if d == 0 {
1311 if nargs < LEXM_MAXARGS { argp[nargs] = a0; argl[nargs] = p - a0; nargs = nargs + 1 } else { nargs = nargs + 1 }
1312 s2 = 1
1313 }
1314 } else {
1315 if c == 44 { if d == 1 {
1316 if nargs < LEXM_MAXARGS { argp[nargs] = a0; argl[nargs] = p - a0; nargs = nargs + 1 } else { nargs = nargs + 1 }
1317 a0 = p + 1
1318 } } } }
1319 if s2 == 0 { p = p + 1 }
1320 }
1321 }
1322 // REFUSE an over-wide invocation rather than substituting the first 8 and emitting
1323 // the remaining parameter NAMES literally. used stays 0, so the call text is copied
1324 // verbatim and fails loudly at the parser instead of running with wrong values.
1325 if nargs > LEXM_MAXARGS { lexm_warn_too_many_args(nargs) } else {
1326 if p < n {
1327 o = lexm_subst(mb, src, argp, argl, nargs, out, o)
1328 i = p + 1
1329 used = 1
1330 did = 1
1331 } }
1332 } }
1333 } }
1334 if used == 0 {
1335 var y: i64 = ns
1336 while y < i { out[o] = src[y]; o = o + 1; y = y + 1 }
1337 }
1338 } else {
1339 out[o] = src[i]
1340 o = o + 1
1341 i = i + 1
1342 } }
1343 }
1344 out[o] = 0 as u8
1345 if did == 0 { return 0 }
1346 return o
1347}
1348
1349// ===== @if(expr) CONSTANT-EXPRESSION EVALUATOR ======================================================
1350// `@ifdef` asks does this NAME exist. `@if` asks what does this EXPRESSION evaluate to -- a strictly
1351// larger question needing a real parser.
1352//
1353// UNDEFINED-IS-ZERO is deliberate and matches every C preprocessor: it makes @if(FEATURE_X) work as a
1354// feature test without forcing every flag to be pre-declared. It is also the only SAFE default here --
1355// erroring on unknown names makes @if unusable for capability probing, and defaulting to TRUE would
1356// repeat exactly the defect this arc fixed (a guard that waves through what it cannot measure).
1357// Zero is the value that makes an unknown feature ABSENT.
1358//
1359// Precedence, lowest to highest -- standard C, because surprising a reader with novel precedence in a
1360// preprocessor is how a wrong branch compiles silently:
1361// || < && < == != >= <= > < < + - < * / % < unary ! - < ( ) literal ident
1362
1363func ppe_or(src: *u8, n: i64, pos: *i64) -> i64;
1364
1365// Scratch cell for macro-value decoding. STATIC, not a per-identifier sys_mmap: an earlier draft called
1366// sys_mmap(16) for EVERY identifier evaluated and never released it, so a file with many @if arms would
1367// burn a fresh page per name. The value is consumed immediately, so one cell suffices and the
1368// allocation is bounded by construction.
1369static ppe_scratch: *i64
1370
1371func ppe_scratch_cell() -> *i64 {
1372 if (ppe_scratch as i64) == 0 { ppe_scratch = sys_mmap(16) as *i64 }
1373 return ppe_scratch
1374}
1375
1376func ppe_skipws(src: *u8, n: i64, pos: *i64) -> i64 {
1377 var p: i64 = pos[0]
1378 var st: i64 = 0
1379 while st == 0 {
1380 if p < n {
1381 let c: i64 = src[p] as i64
1382 if c == 32 { p = p + 1 } else {
1383 if c == 9 { p = p + 1 } else { st = 1 } }
1384 } else { st = 1 }
1385 }
1386 pos[0] = p
1387 return 0
1388}
1389
1390func ppe_primary(src: *u8, n: i64, pos: *i64) -> i64 {
1391 ppe_skipws(src, n, pos)
1392 var p: i64 = pos[0]
1393 if p >= n { pos[0] = p; return 0 }
1394
1395 if src[p] == (40 as u8) {
1396 pos[0] = p + 1
1397 let v: i64 = ppe_or(src, n, pos)
1398 ppe_skipws(src, n, pos)
1399 p = pos[0]
1400 if p < n { if src[p] == (41 as u8) { p = p + 1 } }
1401 pos[0] = p
1402 return v
1403 }
1404
1405 let c0: i64 = src[p] as i64
1406 var isd: i64 = 0
1407 if c0 >= 48 { if c0 <= 57 { isd = 1 } }
1408 if isd == 1 {
1409 var v: i64 = 0
1410 var go: i64 = 1
1411 while go == 1 {
1412 if p < n {
1413 let c: i64 = src[p] as i64
1414 var d: i64 = 0
1415 if c >= 48 { if c <= 57 { d = 1 } }
1416 if d == 1 { v = v * 10 + (c - 48); p = p + 1 } else { go = 0 }
1417 } else { go = 0 }
1418 }
1419 pos[0] = p
1420 return v
1421 }
1422
1423 if lexm_isid(c0) == 1 {
1424 let s: i64 = p
1425 var go2: i64 = 1
1426 while go2 == 1 {
1427 if p < n { if lexm_isid(src[p] as i64) == 1 { p = p + 1 } else { go2 = 0 } } else { go2 = 0 }
1428 }
1429 // defined(NAME) BUILTIN. Must use the SAME definedness rule as @ifdef -- including the
1430 // TARGET_X86_64 hard pin -- or `@if(defined(TARGET_X86_64))` and `@ifdef TARGET_X86_64` would
1431 // disagree, which is a contradiction the corpus would eventually compile through.
1432 // ★ TWO SPELLINGS OF ONE QUESTION MUST SHARE ONE ANSWER.
1433 if p - s == 7 {
1434 var isdef: i64 = 1
1435 if src[s] != (100 as u8) { isdef = 0 }
1436 if src[s+1] != (101 as u8) { isdef = 0 }
1437 if src[s+2] != (102 as u8) { isdef = 0 }
1438 if src[s+3] != (105 as u8) { isdef = 0 }
1439 if src[s+4] != (110 as u8) { isdef = 0 }
1440 if src[s+5] != (101 as u8) { isdef = 0 }
1441 if src[s+6] != (100 as u8) { isdef = 0 }
1442 if isdef == 1 {
1443 var q: i64 = p
1444 var sw: i64 = 0
1445 while sw == 0 { if q < n { if src[q] == (32 as u8) { q = q + 1 } else { sw = 1 } } else { sw = 1 } }
1446 if q < n {
1447 if src[q] == (40 as u8) {
1448 q = q + 1
1449 var sw2: i64 = 0
1450 while sw2 == 0 { if q < n { if src[q] == (32 as u8) { q = q + 1 } else { sw2 = 1 } } else { sw2 = 1 } }
1451 let ns2: i64 = q
1452 var sw3: i64 = 0
1453 while sw3 == 0 { if q < n { if lexm_isid(src[q] as i64) == 1 { q = q + 1 } else { sw3 = 1 } } else { sw3 = 1 } }
1454 let nl2: i64 = q - ns2
1455 var sw4: i64 = 0
1456 while sw4 == 0 { if q < n { if src[q] == (32 as u8) { q = q + 1 } else { sw4 = 1 } } else { sw4 = 1 } }
1457 if q < n { if src[q] == (41 as u8) { q = q + 1 } }
1458 pos[0] = q
1459 if nl2 <= 0 { return 0 }
1460 // TARGET_X86_64 is BACKEND-OWNED and hard-pinned UNDEFINED (see _lex_pp_defined).
1461 if nl2 == 13 {
1462 var istx: i64 = 1
1463 var tz: i64 = 0
1464 let txn: *u8 = "TARGET_X86_64" as *u8
1465 while tz < 13 { if src[ns2 + tz] != txn[tz] { istx = 0; tz = 13 } else { tz = tz + 1 } }
1466 if istx == 1 { return 0 }
1467 }
1468 if (lexm_lookup(((src as i64) + ns2) as *u8, nl2) as i64) != 0 { return 1 }
1469 return 0
1470 } }
1471 }
1472 }
1473 pos[0] = p
1474 // An UNDEFINED name is 0. A defined name whose body is not a plain integer is also 0:
1475 // @if(SOME_TEXT_MACRO) has no numeric meaning, and 0 keeps it ABSENT rather than truthy.
1476 let mb: *u8 = lexm_lookup(((src as i64) + s) as *u8, p - s)
1477 if (mb as i64) == 0 { return 0 }
1478 let vo: *i64 = ppe_scratch_cell()
1479 if lexm_body_int(mb, vo) == 0 { return 0 }
1480 return vo[0]
1481 }
1482
1483 pos[0] = p + 1
1484 return 0
1485}
1486
1487func ppe_unary(src: *u8, n: i64, pos: *i64) -> i64 {
1488 ppe_skipws(src, n, pos)
1489 let p: i64 = pos[0]
1490 if p < n {
1491 if src[p] == (33 as u8) {
1492 // `!` only -- `!=` is a comparison and must not be eaten here.
1493 var isne: i64 = 0
1494 if p + 1 < n { if src[p+1] == (61 as u8) { isne = 1 } }
1495 if isne == 0 {
1496 pos[0] = p + 1
1497 let v: i64 = ppe_unary(src, n, pos)
1498 if v == 0 { return 1 }
1499 return 0
1500 }
1501 }
1502 if src[p] == (45 as u8) {
1503 pos[0] = p + 1
1504 let v2: i64 = ppe_unary(src, n, pos)
1505 return 0 - v2
1506 }
1507 }
1508 return ppe_primary(src, n, pos)
1509}
1510
1511func ppe_mul(src: *u8, n: i64, pos: *i64) -> i64 {
1512 var v: i64 = ppe_unary(src, n, pos)
1513 var go: i64 = 1
1514 while go == 1 {
1515 ppe_skipws(src, n, pos)
1516 let p: i64 = pos[0]
1517 if p >= n { go = 0 } else {
1518 let c: i64 = src[p] as i64
1519 if c == 42 {
1520 pos[0] = p + 1
1521 v = v * ppe_unary(src, n, pos)
1522 } else {
1523 if c == 47 {
1524 pos[0] = p + 1
1525 let d: i64 = ppe_unary(src, n, pos)
1526 // Division by zero must not trap the COMPILER. Yield 0 and continue: a wrong branch is a
1527 // visible bug, a crashed build with no line number is not.
1528 if d == 0 { v = 0 } else { v = v / d }
1529 } else {
1530 if c == 37 {
1531 pos[0] = p + 1
1532 let m: i64 = ppe_unary(src, n, pos)
1533 if m == 0 { v = 0 } else { v = v % m }
1534 } else { go = 0 } } }
1535 }
1536 }
1537 return v
1538}
1539
1540func ppe_add(src: *u8, n: i64, pos: *i64) -> i64 {
1541 var v: i64 = ppe_mul(src, n, pos)
1542 var go: i64 = 1
1543 while go == 1 {
1544 ppe_skipws(src, n, pos)
1545 let p: i64 = pos[0]
1546 if p >= n { go = 0 } else {
1547 let c: i64 = src[p] as i64
1548 if c == 43 { pos[0] = p + 1; v = v + ppe_mul(src, n, pos) } else {
1549 if c == 45 { pos[0] = p + 1; v = v - ppe_mul(src, n, pos) } else { go = 0 } }
1550 }
1551 }
1552 return v
1553}
1554
1555func ppe_cmp(src: *u8, n: i64, pos: *i64) -> i64 {
1556 var v: i64 = ppe_add(src, n, pos)
1557 var go: i64 = 1
1558 while go == 1 {
1559 ppe_skipws(src, n, pos)
1560 let p: i64 = pos[0]
1561 if p + 1 >= n { go = 0 } else {
1562 let a: i64 = src[p] as i64
1563 let b: i64 = src[p+1] as i64
1564 // TWO-CHARACTER OPERATORS FIRST. Testing `>` before `>=` would consume the `>` and leave a
1565 // stray `=`, silently turning `>=` into `>` -- an off-by-one in a CONDITIONAL, which selects
1566 // the wrong block with no diagnostic at all.
1567 if a == 61 { if b == 61 {
1568 pos[0] = p + 2
1569 let r: i64 = ppe_add(src, n, pos)
1570 if v == r { v = 1 } else { v = 0 }
1571 } else { go = 0 } } else {
1572 if a == 33 { if b == 61 {
1573 pos[0] = p + 2
1574 let r2: i64 = ppe_add(src, n, pos)
1575 if v != r2 { v = 1 } else { v = 0 }
1576 } else { go = 0 } } else {
1577 if a == 62 { if b == 61 {
1578 pos[0] = p + 2
1579 let r3: i64 = ppe_add(src, n, pos)
1580 if v >= r3 { v = 1 } else { v = 0 }
1581 } else {
1582 pos[0] = p + 1
1583 let r4: i64 = ppe_add(src, n, pos)
1584 if v > r4 { v = 1 } else { v = 0 }
1585 } } else {
1586 if a == 60 { if b == 61 {
1587 pos[0] = p + 2
1588 let r5: i64 = ppe_add(src, n, pos)
1589 if v <= r5 { v = 1 } else { v = 0 }
1590 } else {
1591 pos[0] = p + 1
1592 let r6: i64 = ppe_add(src, n, pos)
1593 if v < r6 { v = 1 } else { v = 0 }
1594 } } else { go = 0 } } } }
1595 }
1596 }
1597 return v
1598}
1599
1600func ppe_and(src: *u8, n: i64, pos: *i64) -> i64 {
1601 var v: i64 = ppe_cmp(src, n, pos)
1602 var go: i64 = 1
1603 while go == 1 {
1604 ppe_skipws(src, n, pos)
1605 let p: i64 = pos[0]
1606 if p + 1 >= n { go = 0 } else {
1607 if src[p] == (38 as u8) { if src[p+1] == (38 as u8) {
1608 pos[0] = p + 2
1609 // NO SHORT-CIRCUIT: the right side is ALWAYS parsed, because skipping it would leave the
1610 // cursor mid-expression and corrupt everything after. Constant folding has no side
1611 // effects, so evaluating both sides is free and keeps the parse position honest.
1612 let r: i64 = ppe_cmp(src, n, pos)
1613 var t: i64 = 0
1614 if v != 0 { if r != 0 { t = 1 } }
1615 v = t
1616 } else { go = 0 } } else { go = 0 }
1617 }
1618 }
1619 return v
1620}
1621
1622func ppe_or(src: *u8, n: i64, pos: *i64) -> i64 {
1623 var v: i64 = ppe_and(src, n, pos)
1624 var go: i64 = 1
1625 while go == 1 {
1626 ppe_skipws(src, n, pos)
1627 let p: i64 = pos[0]
1628 if p + 1 >= n { go = 0 } else {
1629 if src[p] == (124 as u8) { if src[p+1] == (124 as u8) {
1630 pos[0] = p + 2
1631 let r: i64 = ppe_and(src, n, pos)
1632 var t: i64 = 0
1633 if v != 0 { t = 1 }
1634 if r != 0 { t = 1 }
1635 v = t
1636 } else { go = 0 } } else { go = 0 }
1637 }
1638 }
1639 return v
1640}
1641
1642// The cursor is a STATIC shared cell: only one @if is evaluated at a time, and the recursive `( ... )`
1643// case in ppe_primary deliberately SHARES this cursor with its caller -- that sharing is what advances
1644// the parse position through a nested expression.
1645static ppe_pos: *i64
1646
1647func ppe_eval_at(src: *u8, n: i64, openp: i64) -> i64 {
1648 if (ppe_pos as i64) == 0 { ppe_pos = sys_mmap(16) as *i64 }
1649 let pos: *i64 = ppe_pos
1650 pos[0] = openp + 1
1651 let v: i64 = ppe_or(src, n, pos)
1652 if v != 0 { return 1 }
1653 return 0
1654}
1655
1656func _lex_src_eq(L: *Lex, needle: *u8, n: i64) -> i64 {
1657 let s: *u8 = L.src
1658 var i: i64 = 0
1659 while i < n {
1660 if (s[L.pos + i] & 0xff) != (needle[i] & 0xff) { return 0 }
1661 i = i + 1
1662 }
1663 return 1
1664}
1665
1666// Bytes-equal at an arbitrary src position (not L.pos).
1667func _lex_src_eq_at(L: *Lex, at: i64, needle: *u8, n: i64) -> i64 {
1668 let s: *u8 = L.src
1669 var i: i64 = 0
1670 while i < n {
1671 if (s[at + i] & 0xff) != (needle[i] & 0xff) { return 0 }
1672 i = i + 1
1673 }
1674 return 1
1675}
1676
1677// Skip from current pos to past the next newline (consume EOL).
1678func _lex_skip_to_eol(L: *Lex) -> i64 {
1679 while peek(L) != 0 {
1680 let c: i64 = advance(L)
1681 if c == 0x0A { return 0 }
1682 }
1683 return 0
1684}
1685
1686// Skip src until we find a matching @endif (depth-tracked for nested).
1687func _lex_skip_to_matching_endif(L: *Lex) -> i64 {
1688 var depth: i64 = 1
1689 while depth > 0 {
1690 if peek(L) == 0 { return 0 } // EOF
1691 if peek(L) == 0x40 {
1692 let at1: i64 = L.pos + 1
1693 if _lex_src_eq_at(L, at1, "ifdef " as *u8, 6) == 1 { depth = depth + 1; L.pos = at1 + 6; _lex_skip_to_eol(L); continue }
1694 if _lex_src_eq_at(L, at1, "ifndef " as *u8, 7) == 1 { depth = depth + 1; L.pos = at1 + 7; _lex_skip_to_eol(L); continue }
1695 if _lex_src_eq_at(L, at1, "endif" as *u8, 5) == 1 { depth = depth - 1; L.pos = at1 + 5; _lex_skip_to_eol(L); continue }
1696 }
1697 advance(L)
1698 }
1699 return 0
1700}
1701
1702// Preprocessor directive handler (2026-05-20, four-pillar response to
1703// the silent-divergence bug where the self-host parser ignored
1704// @ifdef/@ifndef while the C bootstrap honored them, causing
1705// nx_syscalls.nx const SYS_SOCKET to resolve differently across
1706// build paths). Substrate now OWNS the preprocessor; both build
1707// paths agree on conditional inclusion.
1708//
1709// Polarity: TARGET_X86_64 is treated as NOT defined here because the
1710// self-host driver's codegen (x86ctx_rv64_to_x86_64_syscall) expects
1711// source to carry RV64 syscall numbers and translates them at emit
1712// time. Therefore the @ifndef TARGET_X86_64 branch (RV64 numbers)
1713// is what should reach the parser. The C bootstrap's --target x86_64
1714// path does the inverse and pre-defines TARGET_X86_64; same source
1715// resolves to different bytes per backend, but each backend gets the
1716// numbers it expects.
1717// Is NAME defined, for `@ifdef` / `@ifndef`? THIS REPLACED A SINGLE HARDCODED STRING COMPARE against
1718// "TARGET_X86_64", under which every OTHER name resolved as DEFINED -- so `@ifdef ANYTHING` always kept
1719// its block and `@ifndef ANYTHING` always dropped one, with no diagnostic. Same shape as a guard that
1720// waves through whatever it cannot measure.
1721//
1722// TARGET_X86_64 stays HARD-PINNED UNDEFINED and is deliberately NOT answered from the macro table. It is
1723// a BACKEND-OWNED axis, not a user macro: per the polarity note above, the x86 self-host emits through
1724// x86ctx_rv64_to_x86_64_syscall, which expects RV64-numbered syscalls in the source. A file that could
1725// `@macro TARGET_X86_64 1` would flip on the raw-x86 branch of nx_syscalls / nx_hal / nx_dirent /
1726// nx_poll / nx_probe / nx_fcntl / nx_self_build, and every syscall const would then be translated TWICE.
1727// Source must not be able to lie about its own target.
1728func _lex_pp_defined(L: *Lex, name_pos: i64) -> i64 {
1729 let s: *u8 = L.src
1730 var e: i64 = name_pos
1731 var st: i64 = 0
1732 while st == 0 { if is_alnum(s[e] as i64) == 1 { e = e + 1 } else { st = 1 } }
1733 let nl: i64 = e - name_pos
1734 if nl <= 0 { return 0 }
1735 if nl == 13 { if _lex_src_eq_at(L, name_pos, "TARGET_X86_64" as *u8, 13) == 1 { return 0 } }
1736 if (lexm_lookup(((s as i64) + name_pos) as *u8, nl) as i64) != 0 { return 1 }
1737 return 0
1738}
1739
1740// ===== @if/@elif/@else BRANCH STATE =================================================================
1741// The evaluator answers is this arm's condition true. THIS answers the harder question: has an earlier
1742// arm of THIS chain already run, and which chain is this one. Get it wrong and the failure mode is
1743// emitting EXTRA arms -- which link, run, and pass every runtime test.
1744const PPB_MAX: i64 = 32
1745
1746static ppb_taken: *i64
1747static ppb_depth: *i64
1748
1749func ppb_init() -> i64 {
1750 if (ppb_depth as i64) != 0 { return 0 }
1751 ppb_taken = sys_mmap(PPB_MAX * 8) as *i64
1752 ppb_depth = sys_mmap(16) as *i64
1753 return 0
1754}
1755
1756// REFUSE past the cap rather than wrapping. A wrapped index silently corrupts an OUTER chain's state,
1757// so the arm selection goes wrong far from the deep nesting that caused it.
1758// The index is hoisted to a local: this corpus has a recorded CONST[i] index-expression miscompile, and
1759// a nested subscript whose index is itself a load is exactly that shape.
1760func ppb_push(v: i64) -> i64 {
1761 ppb_init()
1762 let d: i64 = ppb_depth[0]
1763 if d >= PPB_MAX { return 0 }
1764 ppb_taken[d] = v
1765 ppb_depth[0] = d + 1
1766 return 1
1767}
1768
1769func ppb_pop() -> i64 {
1770 ppb_init()
1771 let d: i64 = ppb_depth[0]
1772 if d > 0 { ppb_depth[0] = d - 1 }
1773 return 0
1774}
1775
1776// Defaults to 1 (=already taken) on an empty stack, so a stray @elif/@else with no opener SKIPS its
1777// body instead of compiling it. Unbalanced directives should drop code, never inject it.
1778func ppb_top_taken() -> i64 {
1779 ppb_init()
1780 let d: i64 = ppb_depth[0]
1781 if d <= 0 { return 1 }
1782 let i: i64 = d - 1
1783 return ppb_taken[i]
1784}
1785
1786func ppb_set_top(v: i64) -> i64 {
1787 ppb_init()
1788 let d: i64 = ppb_depth[0]
1789 if d <= 0 { return 0 }
1790 let i: i64 = d - 1
1791 ppb_taken[i] = v
1792 return 0
1793}
1794
1795func ppb_reset() -> i64 {
1796 ppb_init()
1797 ppb_depth[0] = 0
1798 return 0
1799}
1800
1801// Which directive is at `at`? 1=@if 2=@ifdef 3=@ifndef 4=@elif 5=@else 6=@endif 0=none.
1802// LONGER FORMS FIRST: "@if" is a prefix of "@ifdef"/"@ifndef", so testing "@if" first would classify
1803// every @ifdef in nx_syscalls as an @if and push a bogus chain onto the stack.
1804func _lex_pp_kw_at(L: *Lex, at: i64) -> i64 {
1805 if _lex_src_eq_at(L, at, "@ifdef " as *u8, 7) == 1 { return 2 }
1806 if _lex_src_eq_at(L, at, "@ifndef " as *u8, 8) == 1 { return 3 }
1807 if _lex_src_eq_at(L, at, "@endif" as *u8, 6) == 1 { return 6 }
1808 if _lex_src_eq_at(L, at, "@elif" as *u8, 5) == 1 { return 4 }
1809 if _lex_src_eq_at(L, at, "@else" as *u8, 5) == 1 { return 5 }
1810 if _lex_src_eq_at(L, at, "@if(" as *u8, 4) == 1 { return 1 }
1811 return 0
1812}
1813
1814// Skip to the next @elif / @else / @endif AT DEPTH 0, leaving L.pos on its '@'.
1815// ALL THREE OPENERS COUNT (@if, @ifdef, @ifndef): a nested @ifdef inside an @if arm would otherwise let
1816// its @endif close the OUTER chain, after which the outer @else is treated as top-level and its dead
1817// body compiles.
1818func _lex_skip_to_next_branch(L: *Lex) -> i64 {
1819 let s: *u8 = L.src
1820 var p: i64 = L.pos
1821 var depth: i64 = 0
1822 var stop: i64 = 0
1823 while stop == 0 {
1824 if s[p] == (0 as u8) { stop = 1 } else {
1825 if s[p] == (64 as u8) {
1826 let k: i64 = _lex_pp_kw_at(L, p)
1827 if k == 1 { depth = depth + 1 } else {
1828 if k == 2 { depth = depth + 1 } else {
1829 if k == 3 { depth = depth + 1 } else {
1830 if k == 6 {
1831 if depth == 0 { L.pos = p; stop = 1 } else { depth = depth - 1 }
1832 } else {
1833 if k == 4 { if depth == 0 { L.pos = p; stop = 1 } } else {
1834 if k == 5 { if depth == 0 { L.pos = p; stop = 1 } } else { } } } } } }
1835 }
1836 if stop == 0 {
1837 // Count newlines we pass or every diagnostic after a skipped arm drifts.
1838 if s[p] == (10 as u8) { L.line = L.line + 1 }
1839 p = p + 1
1840 }
1841 }
1842 }
1843 return 0
1844}
1845
1846func _lex_handle_preprocessor(L: *Lex) -> i64 {
1847 if peek(L) != 0x40 { return 0 } // not '@'
1848 let after_at: i64 = L.pos + 1
1849 // @macro NAME BODY -- zero-arg form. Parsed and stored, then the line is consumed exactly like the
1850 // other directives. The ARG form (@macro NAME(a,b) ...) is stored too, but its body will not parse as
1851 // an integer so it stays INERT rather than half-expanded -- a partially substituted macro is a silent
1852 // miscompile, and inert is the honest state until arg substitution ships.
1853 if _lex_src_eq_at(L, after_at, "macro " as *u8, 6) == 1 {
1854 let s: *u8 = L.src
1855 var np: i64 = after_at + 6
1856 var sk1: i64 = 0
1857 while sk1 == 0 { if s[np] == (32 as u8) { np = np + 1 } else { sk1 = 1 } }
1858 var ne: i64 = np
1859 var sk2: i64 = 0
1860 while sk2 == 0 { if is_alnum(s[ne] as i64) == 1 { ne = ne + 1 } else { sk2 = 1 } }
1861 var bp: i64 = ne
1862 var sk3: i64 = 0
1863 while sk3 == 0 { if s[bp] == (32 as u8) { bp = bp + 1 } else { sk3 = 1 } }
1864 let slen: i64 = lexm_strlen(s)
1865 let be: i64 = lexm_body_end(s, slen, bp)
1866 lexm_define(((s as i64) + np) as *u8, ne - np, ((s as i64) + bp) as *u8, be - bp)
1867 // Skip past the WHOLE body -- for a `{ ... }` macro that spans many lines. Leaving L.pos on the
1868 // first line would feed the body text to the lexer as if it were ordinary code. Count the
1869 // newlines we jump over, or every diagnostic after the first multi-line macro reports a line
1870 // number that drifts further off with each definition.
1871 var nlp: i64 = L.pos
1872 while nlp < be {
1873 if s[nlp] == (10 as u8) { L.line = L.line + 1 }
1874 nlp = nlp + 1
1875 }
1876 L.pos = be
1877 _lex_skip_to_eol(L)
1878 return 1
1879 }
1880 if _lex_src_eq_at(L, after_at, "endif" as *u8, 5) == 1 {
1881 ppb_pop()
1882 L.pos = after_at + 5
1883 _lex_skip_to_eol(L)
1884 return 1
1885 }
1886 // @undef NAME -- retract a definition so a later @ifdef/@ifndef sees it as gone.
1887 if _lex_src_eq_at(L, after_at, "undef " as *u8, 6) == 1 {
1888 let s2: *u8 = L.src
1889 var up: i64 = after_at + 6
1890 var uk: i64 = 0
1891 while uk == 0 { if s2[up] == (32 as u8) { up = up + 1 } else { uk = 1 } }
1892 var ue: i64 = up
1893 var uk2: i64 = 0
1894 while uk2 == 0 { if is_alnum(s2[ue] as i64) == 1 { ue = ue + 1 } else { uk2 = 1 } }
1895 lexm_undef(((s2 as i64) + up) as *u8, ue - up)
1896 L.pos = up
1897 _lex_skip_to_eol(L)
1898 return 1
1899 }
1900 // @if(expr) -- open a chain. Cannot collide with @ifdef /@ifndef : char 3 is '(' not 'd'/'n'.
1901 if _lex_src_eq_at(L, after_at, "if(" as *u8, 3) == 1 {
1902 let openp: i64 = after_at + 2
1903 let v: i64 = ppe_eval_at(L.src, lexm_strlen(L.src), openp)
1904 ppb_push(v)
1905 L.pos = openp
1906 _lex_skip_to_eol(L)
1907 if v == 0 { _lex_skip_to_next_branch(L) }
1908 return 1
1909 }
1910 // @elif(expr) -- evaluated ONLY if no earlier arm ran.
1911 if _lex_src_eq_at(L, after_at, "elif(" as *u8, 5) == 1 {
1912 let openp: i64 = after_at + 4
1913 L.pos = openp
1914 _lex_skip_to_eol(L)
1915 if ppb_top_taken() == 1 { _lex_skip_to_next_branch(L) } else {
1916 let v: i64 = ppe_eval_at(L.src, lexm_strlen(L.src), openp)
1917 if v == 1 { ppb_set_top(1) } else { _lex_skip_to_next_branch(L) }
1918 }
1919 return 1
1920 }
1921 // @else -- runs only if no earlier arm ran. Uses skip_to_matching_endif, not next_branch: nothing
1922 // can follow @else in a chain, so the only legal next stop is the @endif.
1923 if _lex_src_eq_at(L, after_at, "else" as *u8, 4) == 1 {
1924 L.pos = after_at + 4
1925 _lex_skip_to_eol(L)
1926 if ppb_top_taken() == 1 { _lex_skip_to_matching_endif(L) } else { ppb_set_top(1) }
1927 return 1
1928 }
1929 if _lex_src_eq_at(L, after_at, "ifdef " as *u8, 6) == 1 {
1930 let name_pos: i64 = after_at + 6
1931 let defd: i64 = _lex_pp_defined(L, name_pos)
1932 // Push a CONSTANT so @endif's pop stays balanced. Deliberately NOT the real truth value with a
1933 // skip-to-next-branch: that would give `@ifdef X ... @else ... @endif` for free, but EVERY
1934 // syscall number in the ecosystem flows through @ifdef/@ifndef TARGET_X86_64 pairs that contain
1935 // NO @else, so the two behaviours are identical there -- untestable by the corpus and pure
1936 // downside risk. @else-after-@ifdef is its own rung with its own gate.
1937 ppb_push(1)
1938 L.pos = name_pos
1939 _lex_skip_to_eol(L)
1940 // Keep the block only when NAME is defined. For TARGET_X86_64 _lex_pp_defined returns 0, so this
1941 // skips exactly as the old hardcoded compare did -- byte-identical for the whole production corpus.
1942 if defd == 0 { _lex_skip_to_matching_endif(L) }
1943 return 1
1944 }
1945 if _lex_src_eq_at(L, after_at, "ifndef " as *u8, 7) == 1 {
1946 let name_pos: i64 = after_at + 7
1947 let defd: i64 = _lex_pp_defined(L, name_pos)
1948 ppb_push(1)
1949 L.pos = name_pos
1950 _lex_skip_to_eol(L)
1951 // Inverse. TARGET_X86_64 -> defd 0 -> block KEPT (the RV64-numbered branch), matching the old
1952 // behaviour that the syscall translator depends on.
1953 if defd == 1 { _lex_skip_to_matching_endif(L) }
1954 return 1
1955 }
1956 return 0
1957}
1958
1959// ---- PER-TOKEN SCRATCH ARENA (2026-08-06, debt 1785516350) ----------------------------------------
1960// lex_ident_or_kw / lex_number / lex_string / emit_punct each did sys_mmap(TOK_BYTES) for a Tok that
1961// push_tok immediately COPIES into the pre-allocated token array -- so the allocation is dead the
1962// instant push_tok returns. sys_mmap here is page-granular and this runtime never munmaps, so that
1963// was A FULL PAGE LEAKED PER TOKEN, in every nx_cc invocation, for every file in the import closure.
1964// The compiler was the ecosystem's worst leaker and it compounds into every build on the host.
1965//
1966// Reuse is safe because the scratch Tok's LIFETIME ENDS AT push_tok and the four emitters are never
1967// reentrant -- none of them calls another. The lazy null-checked init is the SAME idiom lexm_init and
1968// ppb_init already use in this file; copied deliberately rather than invented.
1969//
1970// WARNING -- REUSE MAKES ZERO-INIT THE CALLER'S JOB. A fresh mmap arrives kernel-zeroed; a reused
1971// buffer carries the PREVIOUS token's bytes, and push_tok copies str_data/str_len too. lex_string is
1972// the only emitter that sets those, so the other three MUST clear them or an IDENT lexed straight
1973// after a STRING inherits a live str_data pointer and a nonzero str_len -- a silent miscompile, not a
1974// crash. Each of the three does that explicitly below; do not remove those two lines.
1975static lex_scr_tok: *u8
1976static lex_scr_mv: *i64
1977
1978func lex_scr() -> *u8 {
1979 if (lex_scr_tok as i64) != 0 { return lex_scr_tok }
1980 lex_scr_tok = sys_mmap(TOK_BYTES)
1981 return lex_scr_tok
1982}
1983
1984func lex_scr_macroval() -> *i64 {
1985 if (lex_scr_mv as i64) != 0 { return lex_scr_mv }
1986 lex_scr_mv = sys_mmap(16) as *i64
1987 return lex_scr_mv
1988}
1989
1990// ---- sub-lexers ----
1991
1992func skip_ws_comments(L: *Lex) -> i64 {
1993 var go: i64 = 1
1994 while go {
1995 let c: i64 = peek(L)
1996 if c == 0x20 { advance(L) }
1997 if c == 0x09 { advance(L) }
1998 if c == 0x0A { advance(L) }
1999 if c == 0x0D { advance(L) }
2000 if c == 0x2F {
2001 if peek2(L) == 0x2F {
2002 // Line comment
2003 while peek(L) != 0 {
2004 if peek(L) == 0x0A { break }
2005 advance(L)
2006 }
2007 continue
2008 }
2009 // '/' not followed by another '/' -- division operator.
2010 // Hand it back to the outer lexer to tokenise. Setting
2011 // go=0 directly here avoids an infinite loop through the
2012 // cascade below (which treats '/' as whitespace-like and
2013 // never sets go=0, looping forever on a bare slash).
2014 go = 0
2015 continue
2016 }
2017 if c != 0x20 {
2018 if c != 0x09 {
2019 if c != 0x0A {
2020 if c != 0x0D {
2021 go = 0
2022 }
2023 }
2024 }
2025 }
2026 }
2027 return 0
2028}
2029
2030func lex_ident_or_kw(L: *Lex) -> i64 {
2031 let tr: *u8 = lex_scr()
2032 let t: *Tok = tr as *Tok
2033 t.kind = 2
2034 t.line = L.line
2035 t.col = L.col
2036 t.int_val = 0
2037 t.text0 = 0; t.text1 = 0; t.text2 = 0; t.text3 = 0
2038 t.text4 = 0; t.text5 = 0; t.text6 = 0; t.text7 = 0
2039 // REUSED SCRATCH: push_tok copies str_data/str_len, so a STRING lexed just before this IDENT
2040 // would otherwise ride its live buffer pointer in. The old fresh mmap zeroed these for us.
2041 t.str_data = 0
2042 t.str_len = 0
2043 let text: *u8 = tok_text_ptr(t)
2044 // ALWAYS CONSUME THE WHOLE IDENTIFIER. The old loop was `while i < 63 { ... advance(L) ... }`, so on
2045 // reaching 63 it stopped WITHOUT consuming the tail -- a 70-char name emitted a 63-char IDENT and the
2046 // lexer then resumed at char 64 and lexed the remainder as a SECOND identifier token. That is a token
2047 // SPLIT, and the resulting parse error names a FABRICATED identifier that appears nowhere in the
2048 // source. Storing stops at LEX_IDENT_MAX; consuming does not.
2049 //
2050 // We store 63 rather than silently consuming-without-warning because two identifiers sharing 63
2051 // leading chars would then collapse to ONE token -- a silent miscompile, strictly worse than a loud
2052 // parse error. The warning ANNOUNCES that risk instead of hiding it.
2053 //
2054 // The 63/64 contract is LOAD-BEARING beyond this function: nx_parse.nx copy_name_into_local and
2055 // local_name_eq (and ~12 more loops) are bounded at 64 and are correct ONLY because a NUL is
2056 // guaranteed inside 64 bytes. Do not remove this cap without migrating those.
2057 var i: i64 = 0
2058 var over: i64 = 0
2059 var go: i64 = 1
2060 while go == 1 {
2061 let c: i64 = peek(L)
2062 if is_alnum(c) == 0 { go = 0 } else {
2063 if i < LEX_IDENT_MAX { text[i] = c & 0xFF; i = i + 1 } else { over = over + 1 }
2064 advance(L)
2065 }
2066 }
2067 text[i] = 0
2068 if over > 0 { lex_warn_ident_too_long(text, i, i + over) }
2069 t.kind = keyword_lookup(text, i)
2070 // @macro EXPANSION -- zero-arg NUMERIC form (self-host parity with the C bootstrap).
2071 // Guarded to kind 2 (generic IDENT) ON PURPOSE: keyword_lookup returns a keyword kind for `func`,
2072 // `while`, ... and a macro must NEVER shadow a keyword, or `@macro func 3` would silently break the
2073 // language rather than erroring. Kind 1 is INT (see lex_number), so a hit becomes a literal exactly
2074 // as if the number had been typed -- which is what the C bootstrap does.
2075 // A body that is not a plain integer does NOT expand: the identifier passes through unchanged, so an
2076 // arg-form or token-pasting macro is INERT rather than half-applied.
2077 if t.kind == 2 {
2078 let mb: *u8 = lexm_lookup(text, i)
2079 if (mb as i64) != 0 {
2080 let mv: *i64 = lex_scr_macroval()
2081 if lexm_body_int(mb, mv) == 1 {
2082 t.kind = 1
2083 t.int_val = mv[0]
2084 }
2085 }
2086 }
2087 push_tok(L, t)
2088 return 0
2089}
2090
2091func lex_number(L: *Lex) -> i64 {
2092 let tr: *u8 = lex_scr()
2093 let t: *Tok = tr as *Tok
2094 t.kind = 1
2095 t.line = L.line
2096 t.col = L.col
2097 t.int_val = 0
2098 t.text0 = 0; t.text1 = 0; t.text2 = 0; t.text3 = 0
2099 t.text4 = 0; t.text5 = 0; t.text6 = 0; t.text7 = 0
2100 // REUSED SCRATCH: clear the string fields push_tok copies -- see lex_scr.
2101 t.str_data = 0
2102 t.str_len = 0
2103
2104 var n: i64 = 0
2105 let c0: i64 = peek(L)
2106 if c0 == 0x30 {
2107 let c1: i64 = peek2(L)
2108 if c1 == 0x78 { // 0x hex
2109 advance(L); advance(L)
2110 var go: i64 = 1
2111 while go {
2112 let c: i64 = peek(L)
2113 if c == 0x5F { advance(L); continue }
2114 if is_hexdigit(c) {
2115 n = (n << 4) | hex_val(c)
2116 advance(L)
2117 }
2118 if is_hexdigit(c) == 0 { if c != 0x5F { go = 0 } }
2119 }
2120 t.int_val = n
2121 push_tok(L, t)
2122 return 0
2123 }
2124 if c1 == 0x62 { // 0b binary
2125 advance(L); advance(L)
2126 var go: i64 = 1
2127 while go {
2128 let c: i64 = peek(L)
2129 if c == 0x5F { advance(L); continue }
2130 if c == 0x30 { n = (n << 1); advance(L); continue }
2131 if c == 0x31 { n = (n << 1) | 1; advance(L); continue }
2132 go = 0
2133 }
2134 t.int_val = n
2135 push_tok(L, t)
2136 return 0
2137 }
2138 if c1 == 0x6F { // 0o octal
2139 advance(L); advance(L)
2140 var go: i64 = 1
2141 while go {
2142 let c: i64 = peek(L)
2143 if c == 0x5F { advance(L); continue }
2144 if c >= 0x30 {
2145 if c <= 0x37 {
2146 n = (n << 3) | (c - 0x30)
2147 advance(L)
2148 continue
2149 }
2150 }
2151 go = 0
2152 }
2153 t.int_val = n
2154 push_tok(L, t)
2155 return 0
2156 }
2157 }
2158 // Decimal. LN36 + LN40 (2026-09-03): ONE digit machine feeds both the integer token and the float
2159 // token. value = m x 10^e10, m = the first LEX_DEC_SIG_DIGITS significant digits (leading zeros never
2160 // enter m), e10 = fraction digits consumed minus whole digits dropped past the cap plus the explicit
2161 // exponent, sticky = 1 iff a dropped digit was nonzero. 18 digits fit i64 with room, so the machine
2162 // cannot overflow: a 20-digit fraction used to overflow the packer denominator and LOOP forever; here
2163 // it costs one sticky bit. An INTEGER token whose digits exceed 64 bits (u64 max, so the unsigned
2164 // patterns the estate writes in decimal keep compiling) is REFUSED by name (kind 3), never wrapped --
2165 // 99999999999999999999 silently became a different number before this.
2166 var go: i64 = 1
2167 var m: i64 = 0
2168 var mdig: i64 = 0
2169 var e10: i64 = 0
2170 var sticky: i64 = 0
2171 var iover: i64 = 0
2172 while go {
2173 let c: i64 = peek(L)
2174 if c == 0x5F { advance(L); continue }
2175 if is_digit(c) {
2176 let d: i64 = c - 0x30
2177 // n is read as UNSIGNED here: once it passed 2^63 it reads negative as i64, and any further
2178 // digit would need a 65th bit.
2179 if n < 0 { iover = 1 }
2180 if n > LEX_U64_MAX_DIV10 { iover = 1 }
2181 if n == LEX_U64_MAX_DIV10 { if d > LEX_U64_MAX_LAST { iover = 1 } }
2182 if iover == 0 { n = n * 10 + d }
2183 var took: i64 = 0
2184 if m == 0 { if d == 0 { took = 1 } }
2185 if took == 0 { if mdig < LEX_DEC_SIG_DIGITS { m = m * 10 + d; mdig = mdig + 1; took = 1 } }
2186 if took == 0 { e10 = e10 + 1; if d != 0 { sticky = 1 } }
2187 advance(L)
2188 continue
2189 }
2190 go = 0
2191 }
2192
2193 // Float split: `.` followed by a digit (not a method call `x.foo`, not a range `0..10`).
2194 var isfloat: i64 = 0
2195 var frac: i64 = 0
2196 var frac_digits: i64 = 0
2197 let c_after: i64 = peek(L)
2198 if c_after == 0x2E {
2199 let c_next: i64 = peek2(L)
2200 if is_digit(c_next) {
2201 advance(L) // consume the point
2202 isfloat = 1
2203 var fgo: i64 = 1
2204 while fgo {
2205 let c: i64 = peek(L)
2206 if c == 0x5F { advance(L); continue }
2207 if is_digit(c) {
2208 let d: i64 = c - 0x30
2209 // legacy (whole, frac, digits) triple for the fp32 path, capped so it cannot overflow
2210 if frac_digits < LEX_DEC_SIG_DIGITS { frac = frac * 10 + d; frac_digits = frac_digits + 1 }
2211 var took: i64 = 0
2212 if m == 0 { if d == 0 { e10 = e10 - 1; took = 1 } }
2213 if took == 0 { if mdig < LEX_DEC_SIG_DIGITS { m = m * 10 + d; mdig = mdig + 1; e10 = e10 - 1; took = 1 } }
2214 if took == 0 { if d != 0 { sticky = 1 } }
2215 advance(L)
2216 continue
2217 }
2218 fgo = 0
2219 }
2220 }
2221 }
2222
2223 // LN36: decimal exponent -- `1e9`, `2.5E-3`, `6.02e+23`. Taken only when e/E is followed by a digit,
2224 // or by a sign and a digit; `e` followed by anything else is not this number. A sign with no digit
2225 // after it is a malformed exponent and is refused by name (kind 4).
2226 var hasexp: i64 = 0
2227 let c_e: i64 = peek(L)
2228 var eok: i64 = 0
2229 if c_e == 0x65 { eok = 1 }
2230 if c_e == 0x45 { eok = 1 }
2231 if eok == 1 {
2232 let c_1: i64 = peek2(L)
2233 var esign: i64 = 1
2234 var estart: i64 = 0
2235 if is_digit(c_1) { estart = 1 }
2236 if c_1 == 0x2B { estart = 2 }
2237 if c_1 == 0x2D { estart = 2; esign = 0 - 1 }
2238 if estart == 2 { if is_digit(L.src[L.pos + 2]) == 0 { estart = 3 } }
2239 if estart == 3 { diag_desync_origin(4, L.line, L.col, c_e) }
2240 if estart == 1 { advance(L); hasexp = 1 }
2241 if estart == 2 { advance(L); advance(L); hasexp = 1 }
2242 if hasexp == 1 {
2243 var ev: i64 = 0
2244 var ego: i64 = 1
2245 while ego {
2246 let c: i64 = peek(L)
2247 if c == 0x5F { advance(L); continue }
2248 if is_digit(c) {
2249 if ev < LEX_EXP_CAP { ev = ev * 10 + (c - 0x30) }
2250 advance(L)
2251 continue
2252 }
2253 ego = 0
2254 }
2255 e10 = e10 + esign * ev
2256 isfloat = 1
2257 }
2258 }
2259
2260 if isfloat == 1 {
2261 // Optional suffix: `1.5f32` -> TK_FLOAT_F32; `1.5f64` / `1.5f` / bare -> TK_FLOAT (default f64).
2262 var sfx_kind: i64 = 4
2263 let c_sfx: i64 = peek(L)
2264 var sfx: i64 = 0
2265 if c_sfx == 0x66 { sfx = 1 }
2266 if c_sfx == 0x46 { sfx = 1 }
2267 if sfx == 1 {
2268 advance(L)
2269 let n1: i64 = L.src[L.pos]
2270 let n2: i64 = L.src[L.pos + 1]
2271 if n1 == 0x33 { if n2 == 0x32 { advance(L); advance(L); sfx_kind = 5 } }
2272 if n1 == 0x36 { if n2 == 0x34 { advance(L); advance(L) } }
2273 }
2274 t.kind = sfx_kind
2275 t.int_val = n // legacy whole part (fp32 path); text2..text5 carry the machine
2276 t.text0 = frac
2277 t.text1 = frac_digits
2278 t.text2 = m
2279 t.text3 = e10
2280 t.text4 = sticky
2281 t.text5 = hasexp
2282 push_tok(L, t)
2283 return 0
2284 }
2285
2286 if iover == 1 { diag_desync_origin(3, t.line, t.col, 0) }
2287 t.int_val = n
2288 push_tok(L, t)
2289 return 0
2290}
2291
2292func lex_string(L: *Lex) -> i64 {
2293 // Scratch Tok is reused; the STRING BUFFER below is NOT -- it escapes through t.str_data and must
2294 // outlive this call, which is why only one of this function's two allocations changed.
2295 let tr: *u8 = lex_scr()
2296 let t: *Tok = tr as *Tok
2297 t.kind = 3
2298 t.line = L.line
2299 t.col = L.col
2300 t.int_val = 0
2301 t.text0 = 0; t.text1 = 0; t.text2 = 0; t.text3 = 0
2302 t.text4 = 0; t.text5 = 0; t.text6 = 0; t.text7 = 0
2303 t.str_data = 0
2304 t.str_len = 0
2305 advance(L) // opening "
2306
2307 // Pre-scan to determine the raw length (upper bound; escapes
2308 // shrink it). Self-host pipeline NEEDS escape processing here:
2309 // string literals like "nxc: expand start\n" arrive with the
2310 // backslash-n two-byte sequence; without processing, sys_write
2311 // emits the literal `\` + `n` instead of a newline, so anything
2312 // that uses a string literal at runtime miscompiles.
2313 let scan_start: i64 = L.pos
2314 var scan: i64 = L.pos
2315 let src_chk: *u8 = L.src
2316 // ESCAPE-AWARE pre-scan (root-cause fix 2026-07-29): the old scan stopped at the FIRST
2317 // 0x22 byte INCLUDING an escaped \" -- but the copy loop below correctly continues past
2318 // escaped quotes, so any literal containing \" allocated a TOO-SMALL buffer and the copy
2319 // trampled the heap. Page rounding hid it for small literals; large escape-heavy literals
2320 // corrupted neighboring allocations or SIGSEGVd -- the ecosystem's recurring "~4.5KB
2321 // literal ceiling" was THIS bug wearing a size costume. Witness: runtime/nx_litprobe2.nx.
2322 while src_chk[scan] != 0 {
2323 var stepped: i64 = 0
2324 if src_chk[scan] == 0x5C {
2325 if src_chk[scan + 1] == 0 { break }
2326 scan = scan + 2
2327 stepped = 1
2328 }
2329 if stepped == 0 {
2330 if src_chk[scan] == 0x22 { break }
2331 scan = scan + 1
2332 }
2333 }
2334 let raw_len: i64 = scan - scan_start
2335
2336 let buf: *u8 = sys_mmap(raw_len + 16)
2337
2338 // Inline copy + populate text[] prefix for legacy consumers.
2339 let text: *u8 = tok_text_ptr(t)
2340 var i: i64 = 0
2341 while peek(L) != 0 {
2342 if peek(L) == 0x22 { break }
2343 var c: i64 = peek(L) & 0xFF
2344 if c == 0x5C {
2345 // Escape sequence: peek next char, advance past both.
2346 advance(L)
2347 let nx: i64 = peek(L) & 0xFF
2348 if nx == 0x6E { c = 0x0A } // \n
2349 if nx == 0x74 { c = 0x09 } // \t
2350 if nx == 0x72 { c = 0x0D } // \r
2351 if nx == 0x5C { c = 0x5C } // \\
2352 if nx == 0x22 { c = 0x22 } // \"
2353 if nx == 0x30 { c = 0x00 } // \0 (single, no octal)
2354 // \xNN hex escape -- two hex digits encode one byte.
2355 // Added 2026-05-20 per cardinal
2356 // feedback-lints-are-nishi-bits-up-not-bash: substrate-
2357 // honesty requires the self-host parser to match the C
2358 // bootstrap on string literals. Without this branch the
2359 // self-host emitted literal `\`,`x`,`0`,`0` for `\x00`
2360 // while the C bootstrap emitted a single NUL byte --
2361 // diverged behavior on the same source.
2362 if nx == 0x78 { // 'x'
2363 advance(L) // past 'x'
2364 let h1: i64 = peek(L) & 0xFF
2365 advance(L) // past first hex digit
2366 let h2: i64 = peek(L) & 0xFF
2367 // Don't advance past h2 here -- the outer loop's
2368 // trailing advance(L) handles it.
2369 var v1: i64 = 0
2370 if h1 >= 0x30 { if h1 <= 0x39 { v1 = h1 - 0x30 } }
2371 if h1 >= 0x41 { if h1 <= 0x46 { v1 = h1 - 0x37 } } // A-F
2372 if h1 >= 0x61 { if h1 <= 0x66 { v1 = h1 - 0x57 } } // a-f
2373 var v2: i64 = 0
2374 if h2 >= 0x30 { if h2 <= 0x39 { v2 = h2 - 0x30 } }
2375 if h2 >= 0x41 { if h2 <= 0x46 { v2 = h2 - 0x37 } }
2376 if h2 >= 0x61 { if h2 <= 0x66 { v2 = h2 - 0x57 } }
2377 c = (v1 << 4) | v2
2378 }
2379 // Other escapes: drop the backslash + use the literal char
2380 // (matches gas-style "unknown escape passes literal").
2381 if nx != 0x6E { if nx != 0x74 { if nx != 0x72 {
2382 if nx != 0x5C { if nx != 0x22 { if nx != 0x30 {
2383 if nx != 0x78 {
2384 c = nx
2385 }
2386 } } } } } }
2387 }
2388 buf[i] = c
2389 if i < 63 { text[i] = c }
2390 i = i + 1
2391 advance(L)
2392 }
2393 buf[i] = 0
2394 if i < 63 { text[i] = 0 }
2395
2396 t.str_data = buf as i64
2397 t.str_len = i
2398
2399 // LN24: the copy loop stops at the closing quote OR at a NUL (end of unit, or a raw NUL inside the
2400 // literal). Anything but the quote means the literal never closed -- record it at the OPENING line.
2401 if peek(L) != 0x22 { diag_desync_origin(2, t.line, t.col, peek(L) & 0xFF) }
2402 if peek(L) == 0x22 { advance(L) }
2403 push_tok(L, t)
2404 return 0
2405}
2406
2407func emit_punct(L: *Lex, kind: i64, len: i64) -> i64 {
2408 let tr: *u8 = lex_scr()
2409 let t: *Tok = tr as *Tok
2410 t.kind = kind
2411 t.line = L.line
2412 t.col = L.col
2413 t.int_val = 0
2414 t.text0 = 0; t.text1 = 0; t.text2 = 0; t.text3 = 0
2415 t.text4 = 0; t.text5 = 0; t.text6 = 0; t.text7 = 0
2416 // REUSED SCRATCH: clear the string fields push_tok copies -- see lex_scr. This is the hottest
2417 // emitter in the lexer (every operator and every bracket), so it was also the biggest leaker.
2418 t.str_data = 0
2419 t.str_len = 0
2420 var k: i64 = 0
2421 while k < len {
2422 advance(L)
2423 k = k + 1
2424 }
2425 push_tok(L, t)
2426 return 0
2427}
2428
2429// ---- public entry ----
2430//
2431// Tokenise the entire source, terminate with an EOF token, return
2432// the tokens array.
2433
2434func lex_source(src: *u8, cap: i64) -> *Tok {
2435 // TOKEN POOL SIZED BY DERIVATION, NOT BY THE CALLER'S GUESS (2026-08-18).
2436 // Every token consumes at least one source byte, so tokens <= bytes + 1 (EOF) --
2437 // an invariant, not a tuning knob. The pool is allocated AFTER the macro
2438 // pre-expansion below, from the post-expansion byte count that loop already
2439 // computes; `cap` survives only as a floor for callers that reserve more.
2440 // WHY: the previous caller-guessed fixed pool (the driver passed 262,144 tokens)
2441 // plus the then-unguarded push_tok silently overran into the neighbouring
2442 // allocation on any unit past that token count -- the browser closure (~64,600
2443 // expanded lines) was the estate's largest unit and its position-dependent
2444 // parser desync (2026-08-16..18) was exactly this overflow, not a construct.
2445 // mmap is lazy, so the derived reservation costs address space, not resident set.
2446
2447 // ARG-MACRO PRE-EXPANSION. Returns 0 when nothing expanded, and in that case we keep the ORIGINAL
2448 // buffer -- so a file with no arg macros takes byte-for-byte the same path as before this existed.
2449 // ARG-MACRO PRE-EXPANSION, ITERATED TO A FIXED POINT.
2450 // A SINGLE pass is not enough: a macro body may itself invoke another macro (nx_macro_test defines
2451 // MAX(a,b) whose body calls SQ(a - b)), and a macro emitted BY an expansion is never seen by the pass
2452 // that emitted it. So re-run until a pass reports nothing expanded -- that is the fixed point.
2453 // BOUNDED AT 8: a self-referential macro (`@macro A A`) would otherwise expand forever and HANG THE
2454 // COMPILER, which is far worse than failing. At the cap we stop and let the unexpanded name reach the
2455 // parser, where it surfaces as a loud `undefined function` rather than a silent wrong program.
2456 // 8 levels is far beyond any real nesting depth.
2457 var srcn: i64 = 0
2458 while src[srcn] != (0 as u8) { srcn = srcn + 1 }
2459 var cur: *u8 = src
2460 var curn: i64 = srcn
2461 var expanded: i64 = 0
2462 var iter: i64 = 0
2463 var more: i64 = 1
2464 while more == 1 {
2465 if iter >= 8 { more = 0 } else {
2466 let cap2: i64 = curn * 4 + LEXM_MAGIC_65536
2467 let bufN: *u8 = sys_mmap(cap2)
2468 let rn: i64 = lexm_expand(cur, curn, bufN, cap2)
2469 if rn <= 0 { more = 0 } else {
2470 cur = bufN
2471 curn = rn
2472 expanded = 1
2473 iter = iter + 1
2474 }
2475 }
2476 }
2477
2478 // The pre-pass above defined every macro in the file (lexm_prescan must scan the whole buffer). Drop
2479 // them so the sequential directive handler repopulates in SOURCE ORDER -- otherwise `@ifdef FOO`
2480 // would see a `@macro FOO` written 200 lines LATER as already defined, and `@undef` could never
2481 // retract anything. Textual arg expansion has already happened, so nothing is lost by clearing.
2482 lexm_reset()
2483 // Branch state is PER-FILE: a chain left open by one file must not leak its already-taken flag
2484 // into the next.
2485 ppb_reset()
2486
2487 var pool_toks: i64 = curn + 2
2488 if cap > pool_toks { pool_toks = cap }
2489 let toks_raw: *u8 = sys_mmap(pool_toks * TOK_BYTES + TOK_BYTES)
2490 let toks: *Tok = toks_raw as *Tok
2491
2492 let L_raw: *u8 = sys_mmap(64)
2493 let L: *Lex = L_raw as *Lex
2494 L.src = src
2495 if expanded == 1 { L.src = cur }
2496 L.pos = 0
2497 L.line = 1
2498 L.col = 1
2499 L.tokens = toks
2500 L.n_tokens = 0
2501 L.cap = pool_toks
2502
2503 var go: i64 = 1
2504 while go {
2505 skip_ws_comments(L)
2506 let c: i64 = peek(L)
2507 if c == 0 {
2508 go = 0
2509 continue
2510 }
2511 // Preprocessor directives: @ifdef / @ifndef / @endif. Must
2512 // run BEFORE the @ punct emission so directive lines are
2513 // consumed at lex time and don't reach the parser.
2514 if c == 0x40 {
2515 // `@priv` VISIBILITY ATTRIBUTE -> a REAL token (TK_PRIV), not a silently skipped
2516 // punct+ident pair. Emitting it means the parser ACKNOWLEDGES the marker instead of
2517 // merely tolerating it, which is the difference between syntax and a comment
2518 // convention. The trailing boundary check is what stops `@privv` (or `@priv_x`)
2519 // from matching the first four characters and silently authorising nothing.
2520 // PORTED FROM THE LAPTOP SSOT 2026-07-31: this feature was SSOT-only, and building the
2521 // compiler from buildroot would have silently dropped it (the 731-byte tokenizer delta).
2522 if _lex_src_eq_at(L, L.pos + 1, "priv" as *u8, 4) == 1 {
2523 let nb: i64 = L.src[L.pos + 5] as i64
2524 if is_alnum(nb) == 0 {
2525 if nb != 0x5F { emit_punct(L, TK_PRIV, 5); continue }
2526 }
2527 }
2528 if _lex_handle_preprocessor(L) == 1 { continue }
2529 }
2530 // LN24: skip_ws_comments consumed tab/LF/CR/space, so any remaining byte below 32 (or DEL) is a
2531 // control byte that no token can start with -- record it and stop; the caller reports it.
2532 if c < 32 { diag_desync_origin(1, L.line, L.col, c); go = 0; continue }
2533 if c == 127 { diag_desync_origin(1, L.line, L.col, c); go = 0; continue }
2534 if is_alpha(c) { lex_ident_or_kw(L); continue }
2535 if c == 0x5F { lex_ident_or_kw(L); continue }
2536 if is_digit(c) { lex_number(L); continue }
2537 if c == 0x22 { lex_string(L); continue }
2538
2539 let c2: i64 = peek2(L)
2540 // Multi-char first
2541 if c == 0x2D {
2542 if c2 == 0x3E { emit_punct(L, 61, 2); continue } // ->
2543 }
2544 if c == 0x3D {
2545 if c2 == 0x3D { emit_punct(L, 46, 2); continue } // ==
2546 }
2547 if c == 0x21 {
2548 if c2 == 0x3D { emit_punct(L, 47, 2); continue } // !=
2549 }
2550 if c == 0x3C {
2551 if c2 == 0x3D { emit_punct(L, 50, 2); continue } // <=
2552 if c2 == 0x3C { emit_punct(L, 59, 2); continue } // <<
2553 }
2554 if c == 0x3E {
2555 if c2 == 0x3D { emit_punct(L, 51, 2); continue } // >=
2556 if c2 == 0x3E { emit_punct(L, 60, 2); continue } // >>
2557 }
2558 if c == 0x26 {
2559 if c2 == 0x26 { emit_punct(L, 52, 2); continue } // &&
2560 }
2561 if c == 0x7C {
2562 if c2 == 0x7C { emit_punct(L, 53, 2); continue } // ||
2563 }
2564 if c == 0x2E {
2565 if c2 == 0x2E { emit_punct(L, 62, 2); continue } // ..
2566 }
2567 if c == 0x3A {
2568 if c2 == 0x3A { emit_punct(L, 75, 2); continue } // ::
2569 }
2570 if c == 0x3D {
2571 if c2 == 0x3E { emit_punct(L, 76, 2); continue } // =>
2572 }
2573
2574 if c == 0x2B { emit_punct(L, 40, 1); continue }
2575 if c == 0x2D { emit_punct(L, 41, 1); continue }
2576 if c == 0x2A { emit_punct(L, 42, 1); continue }
2577 if c == 0x2F { emit_punct(L, 43, 1); continue }
2578 if c == 0x25 { emit_punct(L, 44, 1); continue }
2579 if c == 0x3D { emit_punct(L, 45, 1); continue }
2580 if c == 0x3C { emit_punct(L, 48, 1); continue }
2581 if c == 0x3E { emit_punct(L, 49, 1); continue }
2582 if c == 0x21 { emit_punct(L, 54, 1); continue }
2583 if c == 0x26 { emit_punct(L, 55, 1); continue }
2584 if c == 0x7C { emit_punct(L, 56, 1); continue }
2585 if c == 0x5E { emit_punct(L, 57, 1); continue }
2586 if c == 0x7E { emit_punct(L, 58, 1); continue }
2587 if c == 0x2E { emit_punct(L, 63, 1); continue }
2588 if c == 0x3A { emit_punct(L, 64, 1); continue }
2589 if c == 0x3B { emit_punct(L, 65, 1); continue }
2590 if c == 0x2C { emit_punct(L, 66, 1); continue }
2591 if c == 0x28 { emit_punct(L, 67, 1); continue }
2592 if c == 0x29 { emit_punct(L, 68, 1); continue }
2593 if c == 0x7B { emit_punct(L, 69, 1); continue }
2594 if c == 0x7D { emit_punct(L, 70, 1); continue }
2595 if c == 0x5B { emit_punct(L, 71, 1); continue }
2596 if c == 0x5D { emit_punct(L, 72, 1); continue }
2597 if c == 0x40 { emit_punct(L, 73, 1); continue }
2598 if c == 0x23 { emit_punct(L, 74, 1); continue }
2599 if c == 0x3F { emit_punct(L, 77, 1); continue } // ? TK_QUESTION (ternary)
2600
2601 // Unknown: skip to prevent infinite loop.
2602 advance(L)
2603 }
2604
2605 // Terminator EOF
2606 let last_raw: *u8 = sys_mmap(TOK_BYTES)
2607 let last: *Tok = last_raw as *Tok
2608 last.kind = 0
2609 last.line = L.line
2610 last.col = L.col
2611 last.int_val = 0
2612 last.text0 = 0; last.text1 = 0; last.text2 = 0; last.text3 = 0
2613 last.text4 = 0; last.text5 = 0; last.text6 = 0; last.text7 = 0
2614 push_tok(L, last)
2615
2616 return toks
2617}
2618
2619// Library only; self-test lives in lex_test.nx.