nx_tokenizer.nx source
↩ module page · 2464 lines · 102335 B
1// lex.nx -- NishiLang source tokenizer, ported from lex.c.
2//
3// Accepts a null-terminated source buffer and produces a stream of
4// Tok records. Mirrors lex.c's token layout (kind + line/col +
5// either int_val or text[]). Same keyword set.
6//
7// Single-pass; zero allocations per token beyond the growing token
8// array. Extends at 2x when full. Whitespace + // comments are
9// skipped. Numbers support 0x/0b/0o prefixes with underscore
10// separators. String literals read bytes until closing `"`.
11
12// ---- syscalls ----
13
14// ---- token kinds (subset of lex.h, integer constants) ----
15//
16// 0 EOF
17// 1 INT
18// 2 IDENT
19// 3 STRING
20// 10 FUNC 11 LET 12 VAR 13 CONST
21// 14 STATIC 15 IF 16 ELSE 17 WHILE
22// 18 LOOP 19 FOR 20 IN 21 BREAK
23// 22 CONTINUE 23 RETURN 24 TRUE 25 FALSE
24// 26 STRUCT 27 ENUM 28 MATCH 29 COMPTIME
25// 30 EXTERN 31 AS
26// 40 PLUS 41 MINUS 42 STAR 43 SLASH
27// 44 PERCENT 45 ASSIGN 46 EQ 47 NE
28// 48 LT 49 GT 50 LE 51 GE
29// 52 AND_AND 53 OR_OR 54 BANG
30// 55 AMP 56 PIPE 57 CARET 58 TILDE
31// 59 SHL 60 SHR
32// 61 ARROW 62 DOT_DOT 63 DOT 64 COLON
33// 65 SEMI 66 COMMA
34// 67 LPAREN 68 RPAREN 69 LBRACE 70 RBRACE
35// 71 LBRACKET 72 RBRACKET 73 AT 74 HASH
36
37// ---- Tok struct ----
38//
39// Laid out to match lex.h approximately. text[] is fixed-size
40// 64-byte for identifiers and string literals; that's the same
41// MAX_IDENT the C lexer uses.
42
43// nx_safety_envelope:
44// intended_use: AUTO_APPLIED -- primitive-specific tuning queued
45// sil_target: SIL1
46// evidence: [bulk_applied_2026-05-16, see-file-comment-for-detail]
47// verdict: NOT_YET_EVALUATED
48
49import "nx_syscalls.nx"
50import "nx_types.nx"
51import "nx_lex_kinds.nx"
52const LEXM_MAGIC_65536: i64 = 65536
53// Tok struct + TOK_BYTES const live in lex_kinds.nx (canonical
54// home). Removed duplicate definition here 2026-04-26 per
55// nx_type_identity_check.sh -- duplicate Type instances at
56// compile-time create T#selfhost-006-class bugs even when
57// structurally identical.
58
59// ---- character classes ----
60
61func is_alpha(c: i64) -> i64 {
62 if c >= 0x41 { if c <= 0x5A { return 1 } }
63 if c >= 0x61 { if c <= 0x7A { return 1 } }
64 if c == 0x5F { return 1 }
65 return 0
66}
67func is_digit(c: i64) -> i64 {
68 if c >= 0x30 { if c <= 0x39 { return 1 } }
69 return 0
70}
71func is_hexdigit(c: i64) -> i64 {
72 if is_digit(c) { return 1 }
73 if c >= 0x41 { if c <= 0x46 { return 1 } }
74 if c >= 0x61 { if c <= 0x66 { return 1 } }
75 return 0
76}
77func hex_val(c: i64) -> i64 {
78 if c >= 0x30 { if c <= 0x39 { return c - 0x30 } }
79 if c >= 0x41 { if c <= 0x46 { return c - 0x37 } }
80 if c >= 0x61 { if c <= 0x66 { return c - 0x57 } }
81 return 0
82}
83func is_alnum(c: i64) -> i64 {
84 if is_alpha(c) { return 1 }
85 if is_digit(c) { return 1 }
86 return 0
87}
88
89// ---- keyword table ----
90//
91// Linear scan; we have ~20 keywords. Returns token kind or -1.
92
93func keyword_lookup(t: *u8, len: i64) -> i64 {
94 if streq_n(t, "func", len) { return 10 }
95 if streq_n(t, "let", len) { return 11 }
96 if streq_n(t, "var", len) { return 12 }
97 if streq_n(t, "const", len) { return 13 }
98 if streq_n(t, "static", len) { return 14 }
99 if streq_n(t, "if", len) { return 15 }
100 if streq_n(t, "then", len) { return 32 } // V-LANGEXT M2 if-expr
101 if streq_n(t, "else", len) { return 16 }
102 if streq_n(t, "while", len) { return 17 }
103 if streq_n(t, "loop", len) { return 18 }
104 if streq_n(t, "for", len) { return 19 }
105 if streq_n(t, "in", len) { return 20 }
106 if streq_n(t, "break", len) { return 21 }
107 if streq_n(t, "continue", len) { return 22 }
108 if streq_n(t, "return", len) { return 23 }
109 if streq_n(t, "true", len) { return 24 }
110 if streq_n(t, "false", len) { return 25 }
111 if streq_n(t, "struct", len) { return 26 }
112 if streq_n(t, "enum", len) { return 27 }
113 if streq_n(t, "match", len) { return 28 }
114 if streq_n(t, "comptime", len) { return 29 }
115 if streq_n(t, "extern", len) { return 30 }
116 if streq_n(t, "as", len) { return 31 }
117 return 2 // generic IDENT
118}
119
120// ---- Lex state ----
121
122struct Lex {
123 src: *u8,
124 pos: i64,
125 line: i64,
126 col: i64,
127 tokens: *Tok,
128 n_tokens: i64,
129 cap: i64,
130}
131
132// Push a token into the growing array. Simple, no geometric
133// growth for now: caller pre-allocates enough.
134
135func push_tok(L: *Lex, t: *Tok) -> i64 {
136 let base: i64 = L.tokens as i64
137 let slot: *Tok = (base + L.n_tokens * TOK_BYTES) as *Tok
138 slot.kind = t.kind
139 slot.line = t.line
140 slot.col = t.col
141 slot.int_val = t.int_val
142 slot.text0 = t.text0
143 slot.text1 = t.text1
144 slot.text2 = t.text2
145 slot.text3 = t.text3
146 slot.text4 = t.text4
147 slot.text5 = t.text5
148 slot.text6 = t.text6
149 slot.text7 = t.text7
150 slot.str_data = t.str_data
151 slot.str_len = t.str_len
152 L.n_tokens = L.n_tokens + 1
153 return 0
154}
155
156func advance(L: *Lex) -> i64 {
157 let s: *u8 = L.src
158 let c: i64 = s[L.pos]
159 if c == 0x0A {
160 L.line = L.line + 1
161 L.col = 1
162 }
163 if c != 0x0A {
164 if c != 0 { L.col = L.col + 1 }
165 }
166 L.pos = L.pos + 1
167 return c
168}
169func peek(L: *Lex) -> i64 {
170 let s: *u8 = L.src
171 return s[L.pos]
172}
173func peek2(L: *Lex) -> i64 {
174 let s: *u8 = L.src
175 if s[L.pos] == 0 { return 0 }
176 return s[L.pos + 1]
177}
178
179// Bytes-equal: compare L.src[L.pos..L.pos+n] to needle[0..n].
180// Used by the preprocessor for directive name matching.
181// ===== @macro TABLE -- self-host parity with the C bootstrap ========================================
182// main.c implements @macro and pre-defines TARGET_X86_64 as one; the self-host preprocessor below
183// implemented ONLY the conditional directives, so `@macro NAME body` was never expanded and
184// nx_macro_test .. nx_macro_v8_test could never compile.
185//
186// STATIC POINTERS TO LAZILY-MMAPPED BUFFERS, NEVER BSS ARRAYS: a static [N]i64 in a module crashes the
187// process silently at startup (hit twice in this corpus -- see the nx_mgmt_api rate/idem tables). mmap
188// zeroes, so an untouched slot reads empty with no init loop.
189const LEXM_MAX: i64 = 256
190const LEXM_NAMELEN: i64 = 64
191// 1024, not 192: a `{ ... }` body holds a whole function definition. A body that overflows is REFUSED
192// by lexm_define rather than truncated -- a half-stored macro body expands into syntactically valid
193// garbage, which is a silent miscompile.
194const LEXM_BODYLEN: i64 = 1024
195
196static lexm_names: *u8
197static lexm_bodies: *u8
198static lexm_count: *i64
199static lexm_gensym: *i64
200
201func lexm_init() -> i64 {
202 if (lexm_count as i64) != 0 { return 0 }
203 lexm_names = sys_mmap(LEXM_MAX * LEXM_NAMELEN)
204 lexm_bodies = sys_mmap(LEXM_MAX * LEXM_BODYLEN)
205 lexm_count = sys_mmap(16) as *i64
206 lexm_gensym = sys_mmap(16) as *i64
207 return 0
208}
209
210// Emit a NUL-terminated literal / a decimal integer into out. Used to build gensym identifiers.
211func lexm_emit_str(out: *u8, o0: i64, s: *u8) -> i64 {
212 var o: i64 = o0
213 var i: i64 = 0
214 while s[i] != (0 as u8) { out[o] = s[i]; o = o + 1; i = i + 1 }
215 return o
216}
217
218func lexm_emit_dec(out: *u8, o0: i64, v: i64) -> i64 {
219 if v <= 0 { out[o0] = 48 as u8; return o0 + 1 }
220 let tmp: *u8 = sys_mmap(32)
221 var m: i64 = 0
222 var x: i64 = v
223 while x > 0 { tmp[m] = ((x % 10) + 48) as u8; m = m + 1; x = x / 10 }
224 var o: i64 = o0
225 var j: i64 = m - 1
226 while j >= 0 { out[o] = tmp[j]; o = o + 1; j = j - 1 }
227 return o
228}
229
230func lexm_strlen(s: *u8) -> i64 {
231 var i: i64 = 0
232 while s[i] != (0 as u8) { i = i + 1 }
233 return i
234}
235
236// End offset of a macro body starting at bp. A `{` body runs to its MATCHING close brace and may span
237// many lines; any other body ends at the newline. Depth-aware, and it steps over string literals and
238// `//` comments so a brace inside either cannot close the body early. Returns the offset ONE PAST the
239// closing brace for a brace body, or the offset of the terminating newline otherwise.
240func lexm_body_end(src: *u8, n: i64, bp: i64) -> i64 {
241 if bp >= n { return bp }
242 // Step over a leading parameter list FIRST, so `@macro F(a, b) { ... }` is recognised as a BRACE
243 // body. Without this the body would look like it starts at `(`, which is not `{`, and the whole
244 // multi-line block would be cut off at the first newline.
245 var q: i64 = bp
246 if src[q] == (40 as u8) {
247 var d0: i64 = 0
248 var sp: i64 = 0
249 while sp == 0 {
250 if q >= n { sp = 1 } else {
251 let c0: i64 = src[q] as i64
252 if c0 == 0 { sp = 1 } else {
253 if c0 == 10 { sp = 1 } else {
254 if c0 == 40 { d0 = d0 + 1; q = q + 1 } else {
255 if c0 == 41 { d0 = d0 - 1; q = q + 1; if d0 == 0 { sp = 1 } } else { q = q + 1 } } } }
256 }
257 }
258 var sw: i64 = 0
259 while sw == 0 {
260 if q < n { if src[q] == (32 as u8) { q = q + 1 } else { sw = 1 } } else { sw = 1 }
261 }
262 }
263 if q >= n { return q }
264 if src[q] != (123 as u8) {
265 var e: i64 = q
266 var st: i64 = 0
267 while st == 0 {
268 if e >= n { st = 1 } else {
269 let c: i64 = src[e] as i64
270 if c == 10 { st = 1 } else {
271 if c == 13 { st = 1 } else {
272 if c == 0 { st = 1 } else { e = e + 1 } } }
273 }
274 }
275 return e
276 }
277 var i: i64 = q
278 var depth: i64 = 0
279 var st2: i64 = 0
280 while st2 == 0 {
281 if i >= n { st2 = 1 } else {
282 let c: i64 = src[i] as i64
283 if c == 0 { st2 = 1 } else {
284 if c == 34 {
285 i = i + 1
286 var sq: i64 = 0
287 while sq == 0 {
288 if i >= n { sq = 1 } else {
289 if src[i] == (0 as u8) { sq = 1 } else {
290 if src[i] == (92 as u8) { i = i + 2 } else {
291 if src[i] == (34 as u8) { i = i + 1; sq = 1 } else { i = i + 1 } } }
292 }
293 }
294 } else {
295 if c == 47 {
296 var isc: i64 = 0
297 if i + 1 < n { if src[i+1] == (47 as u8) { isc = 1 } }
298 if isc == 1 {
299 var cq: i64 = 0
300 while cq == 0 {
301 if i >= n { cq = 1 } else {
302 if src[i] == (10 as u8) { cq = 1 } else {
303 if src[i] == (0 as u8) { cq = 1 } else { i = i + 1 } }
304 }
305 }
306 } else { i = i + 1 }
307 } else {
308 if c == 123 { depth = depth + 1; i = i + 1 } else {
309 if c == 125 { depth = depth - 1; i = i + 1; if depth == 0 { st2 = 1 } } else {
310 i = i + 1
311 } } } } }
312 }
313 }
314 return i
315}
316
317func lexm_name_at(i: i64) -> *u8 { return ((lexm_names as i64) + i * LEXM_NAMELEN) as *u8 }
318func lexm_body_at(i: i64) -> *u8 { return ((lexm_bodies as i64) + i * LEXM_BODYLEN) as *u8 }
319
320// ===== LEGIBLE REFUSAL ==============================================================================
321// Refusing an oversized `{ }` macro body is CORRECT -- a truncated body loses its closing brace and
322// expands into text that still parses, which is a silent miscompile. But the refusal was SILENT: the
323// macro never registered and the author saw `undefined function: MYMACRO` at the USE site, with nothing
324// pointing at the definition or the body length.
325// The tokenizer had NO error channel before this; every diagnostic came from nx_parse downstream.
326// A WARNING, not an abort: the existing undefined-function error already stops the build; what was
327// missing was the REASON, not the stop.
328// ★ REFUSING SAFELY AND REFUSING LEGIBLY ARE TWO SEPARATE JOBS.
329static lexm_diagbuf: *u8
330
331func lexm_diag_cell() -> *u8 {
332 if (lexm_diagbuf as i64) == 0 { lexm_diagbuf = sys_mmap(512) }
333 return lexm_diagbuf
334}
335
336// Bounded by construction: the name is clamped so a pathological identifier cannot run past the buffer.
337// A diagnostic that overruns while reporting an overrun would be a poor joke.
338func lexm_warn_body_too_long(name: *u8, nlen: i64, blen: i64) -> i64 {
339 let buf: *u8 = lexm_diag_cell()
340 var o: i64 = 0
341 o = lexm_emit_str(buf, o, "nx_tokenizer: @macro '" as *u8)
342 var i: i64 = 0
343 while i < nlen {
344 if o < 300 { buf[o] = name[i]; o = o + 1 }
345 i = i + 1
346 }
347 o = lexm_emit_str(buf, o, "' body is " as *u8)
348 o = lexm_emit_dec(buf, o, blen)
349 o = lexm_emit_str(buf, o, " bytes, over the " as *u8)
350 o = lexm_emit_dec(buf, o, LEXM_BODYLEN)
351 o = lexm_emit_str(buf, o, " byte limit -- NOT DEFINED. Uses will report 'undefined function'. Split the macro or raise LEXM_BODYLEN.\n" as *u8)
352 sys_write(2, buf, o)
353 return 0
354}
355
356// Define NAME -> BODY. A later definition of the same name OVERWRITES, matching the C bootstrap, rather
357// than silently keeping the first -- a redefinition that is ignored makes the build differ from its
358// source with no diagnostic.
359func lexm_define(name: *u8, nlen: i64, body: *u8, blen: i64) -> i64 {
360 lexm_init()
361 if nlen <= 0 { return 0 }
362 if nlen >= LEXM_NAMELEN { return 0 }
363 // REFUSE an oversized body instead of truncating it. A truncated `{ ... }` body loses its closing
364 // brace and expands into text that can still parse, which is a silent miscompile; refusing leaves the
365 // macro name unexpanded and surfaces as a loud `undefined function` at the use site.
366 if blen >= LEXM_BODYLEN { lexm_warn_body_too_long(name, nlen, blen); return 0 }
367 var bl: i64 = blen
368 if bl < 0 { bl = 0 }
369 var slot: i64 = 0 - 1
370 var i: i64 = 0
371 while i < lexm_count[0] {
372 let p: *u8 = lexm_name_at(i)
373 var eq: i64 = 1
374 var k: i64 = 0
375 while k < nlen { if p[k] != name[k] { eq = 0; k = nlen } else { k = k + 1 } }
376 if eq == 1 { if p[nlen] == (0 as u8) { slot = i } }
377 i = i + 1
378 }
379 if slot < 0 {
380 if lexm_count[0] >= LEXM_MAX { return 0 }
381 slot = lexm_count[0]
382 lexm_count[0] = lexm_count[0] + 1
383 }
384 let np: *u8 = lexm_name_at(slot)
385 var a: i64 = 0
386 while a < nlen { np[a] = name[a]; a = a + 1 }
387 np[nlen] = 0 as u8
388 let bp: *u8 = lexm_body_at(slot)
389 var b: i64 = 0
390 while b < bl { bp[b] = body[b]; b = b + 1 }
391 bp[bl] = 0 as u8
392 return 1
393}
394
395func lexm_lookup(name: *u8, nlen: i64) -> *u8 {
396 if (lexm_count as i64) == 0 { return 0 as *u8 }
397 if nlen <= 0 { return 0 as *u8 }
398 if nlen >= LEXM_NAMELEN { return 0 as *u8 }
399 var i: i64 = 0
400 while i < lexm_count[0] {
401 let p: *u8 = lexm_name_at(i)
402 var eq: i64 = 1
403 var k: i64 = 0
404 while k < nlen { if p[k] != name[k] { eq = 0; k = nlen } else { k = k + 1 } }
405 if eq == 1 { if p[nlen] == (0 as u8) { return lexm_body_at(i) } }
406 i = i + 1
407 }
408 return 0 as *u8
409}
410
411// Remove NAME from the table. `@undef` exists so a conditional can be RETRACTED (nx_macro_v4_test
412// defines EXPERIMENT, checks @ifdef, retracts, then checks @ifndef). Removal swaps the LAST entry into
413// the freed slot -- lookup is order-independent, so compaction costs nothing and leaves no tombstone a
414// later scan could mistake for a live entry.
415func lexm_undef(name: *u8, nlen: i64) -> i64 {
416 if (lexm_count as i64) == 0 { return 0 }
417 if nlen <= 0 { return 0 }
418 if nlen >= LEXM_NAMELEN { return 0 }
419 var slot: i64 = 0 - 1
420 var i: i64 = 0
421 while i < lexm_count[0] {
422 let p: *u8 = lexm_name_at(i)
423 var eq: i64 = 1
424 var k: i64 = 0
425 while k < nlen { if p[k] != name[k] { eq = 0; k = nlen } else { k = k + 1 } }
426 if eq == 1 { if p[nlen] == (0 as u8) { slot = i } }
427 i = i + 1
428 }
429 if slot < 0 { return 0 }
430 let last: i64 = lexm_count[0] - 1
431 if slot != last {
432 let dn: *u8 = lexm_name_at(slot)
433 let sn: *u8 = lexm_name_at(last)
434 var a: i64 = 0
435 while a < LEXM_NAMELEN { dn[a] = sn[a]; a = a + 1 }
436 let db: *u8 = lexm_body_at(slot)
437 let sb: *u8 = lexm_body_at(last)
438 var b: i64 = 0
439 while b < LEXM_BODYLEN { db[b] = sb[b]; b = b + 1 }
440 }
441 lexm_count[0] = last
442 return 1
443}
444
445// Drop every definition. Called BETWEEN the arg-expansion pre-pass and lexing. lexm_prescan populates
446// the table from the WHOLE file up front -- it must, to expand a use that appears before its textual
447// definition -- which would otherwise make `@ifdef FOO` at line 10 see a `@macro FOO` at line 100 as
448// already defined. Conditionals are POSITION-DEPENDENT, so the sequential directive handler has to
449// repopulate the table in source order or `@ifdef` silently answers about the whole file.
450// BACKEND-OWNED PREDEFINES, re-seeded on every reset so they survive the pre-pass clear.
451//
452// TARGET_RV64 is DEFINED, and that is the exact complement of TARGET_X86_64 being hard-pinned
453// UNDEFINED (see _lex_pp_defined): source is WRITTEN against RV64 syscall numbering, and the x86
454// self-host translates at emit time. So "which target is this source written for" answers RV64, while
455// "am I emitting raw x86" answers no. Both halves describe the same one pipeline.
456//
457// Seeded via lexm_define rather than special-cased in _lex_pp_defined so they are ordinary table
458// entries -- `@undef`-able and usable as value macros. TARGET_X86_64 is the ONLY name that gets a hard
459// pin, because that is the one a source file could use to make every syscall translate twice.
460func lexm_predefine() -> i64 {
461 lexm_define("TARGET_RV64" as *u8, 11, "1" as *u8, 1)
462 // Compiler identity. Bump when the language surface changes in a way source may need to branch on:
463 // v2 = @macro/@undef/@ifdef-table, v3 = multi-line `{ }` bodies + `$$ident` gensym (2026-07-31).
464 lexm_define("NXC_VERSION" as *u8, 11, "3" as *u8, 1)
465 return 0
466}
467
468func lexm_reset() -> i64 {
469 lexm_init()
470 lexm_count[0] = 0
471 lexm_predefine()
472 return 0
473}
474
475// Is the body a plain non-negative integer? V1 SCOPE IS DELIBERATE: zero-arg NUMERIC macros only
476// (@macro UNIVERSAL_ANSWER 42 = nx_macro_test T1, and the FEATURE_* flags in v3/v6). Arg substitution and
477// @concat pasting are the next rungs and are NOT claimed here. A body this cannot parse simply does not
478// expand, leaving today's behaviour unchanged rather than half-applied -- a partially substituted macro
479// is a SILENT MISCOMPILE, which is strictly worse than no macro at all.
480func lexm_body_int(body: *u8, out: *i64) -> i64 {
481 if (body as i64) == 0 { return 0 }
482 var i: i64 = 0
483 var sk: i64 = 0
484 while sk == 0 {
485 if body[i] == (32 as u8) { i = i + 1 } else { sk = 1 }
486 }
487 var v: i64 = 0
488 var n: i64 = 0
489 var go: i64 = 1
490 while go == 1 {
491 let c: i64 = body[i] as i64
492 var isd: i64 = 0
493 if c >= 48 { if c <= 57 { isd = 1 } }
494 if isd == 1 { v = v * 10 + (c - 48); n = n + 1; i = i + 1 } else { go = 0 }
495 }
496 if n == 0 { return 0 }
497 var st: i64 = 0
498 while st == 0 {
499 if body[i] == (32 as u8) { i = i + 1 } else { st = 1 }
500 }
501 if body[i] != (0 as u8) { return 0 }
502 out[0] = v
503 return 1
504}
505
506// ===== ARG-MACRO EXPANSION (source-level pre-pass) ==================================================
507// WHY A PRE-PASS AND NOT A TOKEN-STREAM REWRITE: expanding at the token level needs a re-entrant lexer
508// (push a synthetic source, lex it, pop) which touches peek/advance/L.pos -- the core of a compiler that
509// builds 10,822 targets. A textual pre-pass is ADDITIVE: if no arg-macro is used it returns 0 and the
510// caller lexes the ORIGINAL buffer, so the existing path is byte-for-byte unchanged.
511//
512// The stored body ALREADY carries the params: `@macro SQ(x) (x * x)` stores name="SQ",
513// body="(x) (x * x)". So an arg macro is exactly "a macro whose body starts with '('" -- no extra table
514// column, and the zero-arg numeric path is untouched.
515
516// ONE named bound for BOTH the parameter cap and the argument cap. They used to be two independent
517// bare `8` literals in different functions -- nothing tied them together, so they could drift apart and
518// mis-substitute in a THIRD way.
519const LEXM_MAXARGS: i64 = 8
520
521func lexm_warn_too_many_args(got: i64) -> i64 {
522 let buf: *u8 = lexm_diag_cell()
523 var o: i64 = 0
524 o = lexm_emit_str(buf, o, "nx_tokenizer: @macro invocation uses " as *u8)
525 o = lexm_emit_dec(buf, o, got)
526 o = lexm_emit_str(buf, o, " arguments, over the limit of " as *u8)
527 o = lexm_emit_dec(buf, o, LEXM_MAXARGS)
528 o = lexm_emit_str(buf, o, " -- REFUSED, not truncated. Previously the extra arguments were dropped and their parameter names were emitted LITERALLY, which could compile and run with wrong values.\n" as *u8)
529 sys_write(2, buf, o)
530 return 0
531}
532
533// The identifier storage bound. Named, not a bare 63, because nx_parse.nx depends on the SAME contract
534// (a NUL inside 64 bytes) at ~14 sites.
535const LEX_IDENT_MAX: i64 = 63
536
537func lex_warn_ident_too_long(text: *u8, kept: i64, actual: i64) -> i64 {
538 let buf: *u8 = lexm_diag_cell()
539 var o: i64 = 0
540 o = lexm_emit_str(buf, o, "nx_tokenizer: identifier is " as *u8)
541 o = lexm_emit_dec(buf, o, actual)
542 o = lexm_emit_str(buf, o, " chars, over the limit of " as *u8)
543 o = lexm_emit_dec(buf, o, LEX_IDENT_MAX)
544 o = lexm_emit_str(buf, o, " -- kept '" as *u8)
545 var i: i64 = 0
546 while i < kept {
547 if o < 300 { buf[o] = text[i]; o = o + 1 }
548 i = i + 1
549 }
550 o = lexm_emit_str(buf, o, "'. Two identifiers sharing these leading chars would COLLIDE. Shorten the name; if it is macro-generated, shorten the prefix or the loop variable.\n" as *u8)
551 sys_write(2, buf, o)
552 return 0
553}
554
555func lexm_isid(c: i64) -> i64 { return is_alnum(c) }
556
557// Copy body-after-params into out, replacing each param NAME with the matching actual-arg text.
558// argp/argl carry up to 8 actual args as (offset,len) into asrc.
559func lexm_subst(body: *u8, asrc: *u8, argp: *i64, argl: *i64, nargs: i64, out: *u8, o0: i64) -> i64 {
560 var o: i64 = o0
561 var pn: i64 = 0
562 let pofs: *i64 = sys_mmap(128) as *i64
563 let plen: *i64 = sys_mmap(128) as *i64
564 var i: i64 = 1
565 var s1: i64 = 0
566 while s1 == 0 {
567 if body[i] == (0 as u8) { s1 = 1 } else {
568 if body[i] == (41 as u8) { s1 = 1 } else {
569 if lexm_isid(body[i] as i64) == 1 {
570 let ps: i64 = i
571 var s2: i64 = 0
572 while s2 == 0 { if lexm_isid(body[i] as i64) == 1 { i = i + 1 } else { s2 = 1 } }
573 // REFUSE, never truncate: a 9th parameter that is silently not registered is never
574 // recognised in the body, falls through to copy-verbatim, and is emitted as a LITERAL
575 // identifier -- which can compile and run with the wrong value.
576 if pn < LEXM_MAXARGS { pofs[pn] = ps; plen[pn] = i - ps; pn = pn + 1 } else { pn = pn + 1 }
577 } else { i = i + 1 }
578 } }
579 }
580 // REFUSE the whole expansion if the macro declares more parameters than we can hold. Emitting a
581 // partially-substituted body is a SILENT MISCOMPILE; returning o0 unchanged emits nothing, so the
582 // macro name survives to the parser and fails loudly as `undefined function`.
583 // Also protects the matching loop below, which would otherwise index pofs/plen past their mapping.
584 if pn > LEXM_MAXARGS { lexm_warn_too_many_args(pn); return o0 }
585
586 var b: i64 = i
587 if body[b] == (41 as u8) { b = b + 1 }
588 var s3: i64 = 0
589 while s3 == 0 { if body[b] == (32 as u8) { b = b + 1 } else { s3 = 1 } }
590
591 // A `{ ... }` body emits its INTERIOR: the braces delimit the macro, they are not part of the
592 // expansion. Any other body runs to the end of the stored text.
593 let blen: i64 = lexm_strlen(body)
594 var k: i64 = b
595 var kend: i64 = blen
596 if body[b] == (123 as u8) {
597 k = b + 1
598 kend = lexm_body_end(body, blen, b) - 1
599 }
600
601 // ONE gensym serial per INVOCATION. Every `$$name` in this expansion shares it, so `let $$got` and a
602 // later `$$got` in the same body resolve to the SAME identifier -- while a second invocation takes
603 // the next serial, so two expansions in one scope cannot collide. Both halves are load-bearing:
604 // sharing within a body is what makes the macro compile, and differing across bodies is what makes
605 // two uses in one function legal.
606 lexm_gensym[0] = lexm_gensym[0] + 1
607 let serial: i64 = lexm_gensym[0]
608
609 while k < kend {
610 if body[k] == (36 as u8) {
611 // ${param} -- BRACED parameter reference inside a @macro body. Needed so a substitution can
612 // ABUT following identifier characters: `nx_${name}_apply` must paste to `nx_foo_apply`,
613 // which a bare `$name` cannot express because `name_apply` reads as one identifier run.
614 // Checked BEFORE the $$ gensym test: `${` and `$$` are distinct two-char openers.
615 var brc: i64 = 0
616 if k + 1 < kend { if body[k+1] == (123 as u8) { brc = 1 } }
617 if brc == 1 {
618 // ${$$name} -- a BRACED GENSYM reference. Braces only DELIMIT, so this must mean exactly
619 // what bare `$$name` means; the braces exist so the gensym can ABUT following identifier
620 // characters (`describe_${$$counter}()`), which bare `$$counter` cannot express because
621 // `counter` would swallow the following text as one identifier run.
622 // Same rule as ${param}: same answer, explicit boundary.
623 var gbr: i64 = 0
624 if k + 3 < kend { if body[k+2] == (36 as u8) { if body[k+3] == (36 as u8) { gbr = 1 } } }
625 if gbr == 1 {
626 var gk: i64 = k + 4
627 let ggs: i64 = gk
628 var gs1: i64 = 0
629 while gs1 == 0 {
630 if gk < kend { if lexm_isid(body[gk] as i64) == 1 { gk = gk + 1 } else { gs1 = 1 } } else { gs1 = 1 }
631 }
632 var gclosed: i64 = 0
633 if gk < kend { if body[gk] == (125 as u8) { gclosed = 1 } }
634 if gclosed == 1 {
635 o = lexm_emit_str(out, o, "__nx_g_" as *u8)
636 var gg: i64 = ggs
637 while gg < gk { out[o] = body[gg]; o = o + 1; gg = gg + 1 }
638 out[o] = 95 as u8
639 o = o + 1
640 o = lexm_emit_dec(out, o, serial)
641 k = gk + 1
642 gbr = 2
643 }
644 }
645 if gbr == 2 { } else {
646 let bs2: i64 = k + 2
647 var be2: i64 = bs2
648 var sb: i64 = 0
649 while sb == 0 {
650 if be2 < kend { if lexm_isid(body[be2] as i64) == 1 { be2 = be2 + 1 } else { sb = 1 } } else { sb = 1 }
651 }
652 var closed: i64 = 0
653 if be2 < kend { if body[be2] == (125 as u8) { closed = 1 } }
654 if closed == 1 {
655 let bl2: i64 = be2 - bs2
656 var bh: i64 = 0 - 1
657 var bp2: i64 = 0
658 while bp2 < pn {
659 if plen[bp2] == bl2 {
660 var beq: i64 = 1
661 var bz: i64 = 0
662 while bz < bl2 { if body[pofs[bp2] + bz] != body[bs2 + bz] { beq = 0; bz = bl2 } else { bz = bz + 1 } }
663 if beq == 1 { bh = bp2 }
664 }
665 bp2 = bp2 + 1
666 }
667 if bh >= 0 {
668 if bh < nargs {
669 var ba: i64 = 0
670 while ba < argl[bh] { out[o] = asrc[argp[bh] + ba]; o = o + 1; ba = ba + 1 }
671 }
672 k = be2 + 1
673 } else {
674 // Not one of THIS macro's parameters -- copy verbatim so a @for's ${var} inside a
675 // macro body survives to the @for pass instead of being silently deleted.
676 while k <= be2 { out[o] = body[k]; o = o + 1; k = k + 1 }
677 }
678 } else { out[o] = body[k]; o = o + 1; k = k + 1 }
679 }
680 } else {
681 var dbl: i64 = 0
682 if k + 1 < kend { if body[k+1] == (36 as u8) { dbl = 1 } }
683 if dbl == 1 {
684 k = k + 2
685 let gs: i64 = k
686 var s6: i64 = 0
687 while s6 == 0 {
688 if k < kend { if lexm_isid(body[k] as i64) == 1 { k = k + 1 } else { s6 = 1 } } else { s6 = 1 }
689 }
690 o = lexm_emit_str(out, o, "__nx_g_" as *u8)
691 var g: i64 = gs
692 while g < k { out[o] = body[g]; o = o + 1; g = g + 1 }
693 out[o] = 95 as u8
694 o = o + 1
695 o = lexm_emit_dec(out, o, serial)
696 } else { out[o] = body[k]; o = o + 1; k = k + 1 }
697 }
698 } else {
699 if lexm_isid(body[k] as i64) == 1 {
700 let ts: i64 = k
701 var s5: i64 = 0
702 while s5 == 0 {
703 if k < kend { if lexm_isid(body[k] as i64) == 1 { k = k + 1 } else { s5 = 1 } } else { s5 = 1 }
704 }
705 let tl: i64 = k - ts
706 var hit: i64 = 0 - 1
707 var p: i64 = 0
708 while p < pn {
709 if plen[p] == tl {
710 var eq: i64 = 1
711 var z: i64 = 0
712 while z < tl { if body[pofs[p] + z] != body[ts + z] { eq = 0; z = tl } else { z = z + 1 } }
713 if eq == 1 { hit = p }
714 }
715 p = p + 1
716 }
717 if hit >= 0 {
718 if hit < nargs {
719 var a: i64 = 0
720 while a < argl[hit] { out[o] = asrc[argp[hit] + a]; o = o + 1; a = a + 1 }
721 }
722 } else {
723 var y: i64 = ts
724 while y < k { out[o] = body[y]; o = o + 1; y = y + 1 }
725 }
726 } else { out[o] = body[k]; o = o + 1; k = k + 1 } }
727 }
728 return o
729}
730
731// Scan src for `@macro` definitions so they are known BEFORE any use is expanded. The tokenizer's own
732// directive handler re-defines them later; lexm_define OVERWRITES, so that is a harmless no-op.
733func lexm_prescan(src: *u8, n: i64) -> i64 {
734 var i: i64 = 0
735 while i + 7 < n {
736 var hit: i64 = 0
737 if src[i] == (64 as u8) {
738 if src[i+1] == (109 as u8) { if src[i+2] == (97 as u8) { if src[i+3] == (99 as u8) {
739 if src[i+4] == (114 as u8) { if src[i+5] == (111 as u8) { if src[i+6] == (32 as u8) { hit = 1 } } } } } }
740 }
741 if hit == 1 {
742 var np: i64 = i + 7
743 var k1: i64 = 0
744 while k1 == 0 { if src[np] == (32 as u8) { np = np + 1 } else { k1 = 1 } }
745 var ne: i64 = np
746 var k2: i64 = 0
747 while k2 == 0 { if lexm_isid(src[ne] as i64) == 1 { ne = ne + 1 } else { k2 = 1 } }
748 var bp: i64 = ne
749 var k3: i64 = 0
750 while k3 == 0 { if src[bp] == (32 as u8) { bp = bp + 1 } else { k3 = 1 } }
751 let be: i64 = lexm_body_end(src, n, bp)
752 lexm_define(((src as i64) + np) as *u8, ne - np, ((src as i64) + bp) as *u8, be - bp)
753 i = be
754 }
755 i = i + 1
756 }
757 return 0
758}
759
760// ===== @for GENERATIVE REPETITION ===================================================================
761// @for(i in 0..5) { func atom_$i() -> nx_int { return $i * 100 } }
762// @for(z in 1..ATOM_COUNT) { func mass_$z() -> nx_int { return $z * 10 } }
763// @for(op in {add, sub, mul}) { func calc_${op}(a: nx_int, b: nx_int) -> nx_int { ... } }
764//
765// RANGE IS HALF-OPEN (0..5 yields 0,1,2,3,4) -- fixed by the tests, not by taste: nx_macro_v5_test
766// checks atom_0..atom_4 and never atom_5. Half-open also makes 0..N mean N iterations, the only
767// convention that composes.
768//
769// The braced form ${op} exists so a substitution can ABUT following identifier characters:
770// calc_${op}(...) must paste to calc_add, whereas a bare $op cannot tell where the name ends when the
771// next character is itself an identifier character. Bare $i stays legal where the boundary is clear.
772
773// Resolve a @for bound: a decimal literal, or a macro name whose body is a plain integer.
774// Trims its own leading/trailing spaces. An earlier draft trimmed in the CALLER with a loop whose
775// else-branch was a no-op assignment -- any non-space character spun FOREVER. An infinite loop in the
776// tokenizer hangs every build in the ecosystem and reports nothing, so trimming lives HERE, once,
777// where it terminates by construction.
778func lexm_for_bound(src: *u8, s0: i64, e0: i64, out: *i64) -> i64 {
779 var s: i64 = s0
780 var e: i64 = e0
781 var t1: i64 = 0
782 while t1 == 0 { if s < e { if src[s] == (32 as u8) { s = s + 1 } else { t1 = 1 } } else { t1 = 1 } }
783 var t2: i64 = 0
784 while t2 == 0 { if e > s { if src[e-1] == (32 as u8) { e = e - 1 } else { t2 = 1 } } else { t2 = 1 } }
785 if e <= s { return 0 }
786 var alldig: i64 = 1
787 var k: i64 = s
788 while k < e {
789 let c: i64 = src[k] as i64
790 var isd: i64 = 0
791 if c >= 48 { if c <= 57 { isd = 1 } }
792 if isd == 0 { alldig = 0; k = e } else { k = k + 1 }
793 }
794 if alldig == 1 {
795 var v: i64 = 0
796 var j: i64 = s
797 while j < e { v = v * 10 + ((src[j] as i64) - 48); j = j + 1 }
798 out[0] = v
799 return 1
800 }
801 let mb: *u8 = lexm_lookup(((src as i64) + s) as *u8, e - s)
802 if (mb as i64) == 0 { return 0 }
803 return lexm_body_int(mb, out)
804}
805
806// Emit body [bs,be) once, replacing $VAR and ${VAR} with vtxt[0..vtl).
807// A `$` not followed by THIS loop's variable is copied through untouched, so $$ident gensym inside a
808// @for body still reaches lexm_subst intact.
809// `ocap` is load-bearing: a @for emits BODY x ITERATIONS, far exceeding the single-body headroom the
810// caller checked before entering. Returns 0-1 on overflow so the whole @for is REFUSED and the text
811// reaches the parser unexpanded -- a loud failure instead of a write past the buffer.
812func lexm_for_emit(src: *u8, bs: i64, be: i64, vname: *u8, vlen: i64, vtxt: *u8, vtl: i64, out: *u8, o0: i64, ocap: i64) -> i64 {
813 var o: i64 = o0
814 var k: i64 = bs
815 while k < be {
816 // Reserve vtl too, not a bare constant: each pass can emit the substituted TOKEN, so a long
817 // value would step past a fixed cushion and overrun anyway.
818 if o >= ocap - vtl - 64 { return 0 - 1 }
819 if src[k] == (36 as u8) {
820 var p: i64 = k + 1
821 var braced: i64 = 0
822 if p < be { if src[p] == (123 as u8) { braced = 1; p = p + 1 } }
823 let ns: i64 = p
824 var st: i64 = 0
825 while st == 0 {
826 if p < be { if lexm_isid(src[p] as i64) == 1 { p = p + 1 } else { st = 1 } } else { st = 1 }
827 }
828 var nl: i64 = p - ns
829 var hit: i64 = 0
830 // EXACT run match first: `$i` followed by a non-identifier char.
831 if nl == vlen {
832 var eq: i64 = 1
833 var z: i64 = 0
834 while z < nl { if src[ns + z] != vname[z] { eq = 0; z = nl } else { z = z + 1 } }
835 hit = eq
836 }
837 // PREFIX match second, and it is REQUIRED for nested @for: `pair_$i_$j` reads the run after
838 // `$` as `i_` (underscore IS an identifier char), which never equals `i`, so an exact-only
839 // matcher silently leaves `$i_$j` unexpanded and nested loops cannot work at all.
840 // Consuming just the variable name and leaving the rest is what makes `$i_$j` paste.
841 // `${i}` remains the way to disambiguate when the following text could itself be the name.
842 if hit == 0 {
843 if braced == 0 {
844 if nl > vlen {
845 var eq2: i64 = 1
846 var z2: i64 = 0
847 while z2 < vlen { if src[ns + z2] != vname[z2] { eq2 = 0; z2 = vlen } else { z2 = z2 + 1 } }
848 if eq2 == 1 { hit = 1; p = ns + vlen; nl = vlen }
849 }
850 }
851 }
852 if braced == 1 {
853 if p < be {
854 if src[p] == (125 as u8) { p = p + 1 } else { hit = 0 }
855 } else { hit = 0 }
856 }
857 if hit == 1 {
858 var a: i64 = 0
859 while a < vtl { out[o] = vtxt[a]; o = o + 1; a = a + 1 }
860 k = p
861 } else { out[o] = src[k]; o = o + 1; k = k + 1 }
862 } else { out[o] = src[k]; o = o + 1; k = k + 1 }
863 }
864 return o
865}
866
867// ===== @concat / @stringify -- TOKEN PASTING AND TEXT CAPTURE =======================================
868// @macro DEFINE_DOUBLE(prefix) { func @concat(prefix, _double_x)(x: nx_int) -> nx_int { ... } }
869// DEFINE_DOUBLE(foo) => func foo_double_x(x: nx_int) -> nx_int { ... }
870// @stringify(name) => "name"
871//
872// THE ORDERING IS THE WHOLE DESIGN, AND IT IS ALREADY SOLVED. @concat(prefix, _double_x) must see
873// `prefix` ALREADY replaced by `foo`, or it pastes the PARAMETER NAME and silently emits
874// `prefix_double_x` -- a function nobody called, plus an undefined-function error at every call site
875// naming the right symbol for the wrong reason. No new machinery is needed: lexm_subst substitutes and
876// emits text, and lex_source ALREADY iterates lexm_expand to a fixed point, so pass N emits
877// `@concat(foo, _double_x)` and pass N+1 splices it. This is why the fixed-point loop was worth
878// building even when one pass looked sufficient.
879
880// Trim spaces/tabs/newlines from both ends of [s,e); returns new start, writes new end to eout.
881func lexm_trim(src: *u8, s: i64, e: i64, eout: *i64) -> i64 {
882 var a: i64 = s
883 var b: i64 = e
884 var t1: i64 = 0
885 while t1 == 0 {
886 if a < b {
887 let c: i64 = src[a] as i64
888 if c == 32 { a = a + 1 } else { if c == 9 { a = a + 1 } else {
889 if c == 10 { a = a + 1 } else { if c == 13 { a = a + 1 } else { t1 = 1 } } } }
890 } else { t1 = 1 }
891 }
892 var t2: i64 = 0
893 while t2 == 0 {
894 if b > a {
895 let c: i64 = src[b-1] as i64
896 if c == 32 { b = b - 1 } else { if c == 9 { b = b - 1 } else {
897 if c == 10 { b = b - 1 } else { if c == 13 { b = b - 1 } else { t2 = 1 } } } }
898 } else { t2 = 1 }
899 }
900 eout[0] = b
901 return a
902}
903
904func lexm_is_ident_text(p: *u8, s: i64, e: i64) -> i64 {
905 if e <= s { return 0 }
906 let c0: i64 = p[s] as i64
907 var okstart: i64 = 0
908 if c0 == 95 { okstart = 1 }
909 if is_alpha(c0) == 1 { okstart = 1 }
910 if okstart == 0 { return 0 }
911 var i: i64 = s
912 while i < e {
913 if lexm_isid(p[i] as i64) == 0 { return 0 }
914 i = i + 1
915 }
916 return 1
917}
918
919// A pasted result must be a legal identifier, or the failure surfaces far away as
920// `reserved keyword '.' cannot be used as an identifier name` naming a line that is not the cause.
921// REFUSE LEGIBLY: name the result.
922func lexm_warn_bad_paste(out: *u8, s: i64, e: i64) -> i64 {
923 let buf: *u8 = lexm_diag_cell()
924 var o: i64 = 0
925 o = lexm_emit_str(buf, o, "nx_tokenizer: @concat produced '" as *u8)
926 var i: i64 = s
927 while i < e {
928 if o < 300 { buf[o] = out[i]; o = o + 1 }
929 i = i + 1
930 }
931 o = lexm_emit_str(buf, o, "', which is not a legal identifier (must start with a letter or _ and contain only letters, digits, _). Check that every @concat argument is a bare token.\n" as *u8)
932 sys_write(2, buf, o)
933 return 0
934}
935
936// Shared parser for @concat(...) / @stringify(...).
937// mode 0 = concat (paste args, no separator), mode 1 = stringify (quote the arg text).
938func lexm_paste_expand(src: *u8, n: i64, istart: i64, kwlen: i64, mode: i64, out: *u8, o0: i64, ocap: i64, iout: *i64) -> i64 {
939 var p: i64 = istart + kwlen
940 var sk: i64 = 0
941 while sk == 0 { if p < n { if src[p] == (32 as u8) { p = p + 1 } else { sk = 1 } } else { sk = 1 } }
942 if p >= n { return 0 - 1 }
943 if src[p] != (40 as u8) { return 0 - 1 }
944
945 let argstart: i64 = p + 1
946 var d: i64 = 1
947 var q: i64 = argstart
948 var st: i64 = 0
949 while st == 0 {
950 if q >= n { st = 1 } else {
951 let c: i64 = src[q] as i64
952 if c == 0 { st = 1 } else {
953 if c == 40 { d = d + 1; q = q + 1 } else {
954 if c == 41 { d = d - 1; if d == 0 { st = 1 } else { q = q + 1 } } else { q = q + 1 } } }
955 }
956 }
957 if q >= n { return 0 - 1 }
958 let argend: i64 = q
959
960 var o: i64 = o0
961 if o >= ocap - (argend - argstart) - 64 { return 0 - 1 }
962
963 let eout: *i64 = sys_mmap(16) as *i64
964 let rstart: i64 = o
965
966 if mode == 1 { out[o] = 34 as u8; o = o + 1 }
967
968 var cur: i64 = argstart
969 var k: i64 = argstart
970 var d2: i64 = 0
971 var go: i64 = 1
972 while go == 1 {
973 var flush: i64 = 0
974 if k >= argend { flush = 1; go = 0 } else {
975 let c: i64 = src[k] as i64
976 if c == 40 { d2 = d2 + 1 } else { if c == 41 { d2 = d2 - 1 } else {
977 if c == 44 { if d2 == 0 { flush = 1 } } } }
978 }
979 if flush == 1 {
980 let ts: i64 = lexm_trim(src, cur, k, eout)
981 let te: i64 = eout[0]
982 var z: i64 = ts
983 while z < te { out[o] = src[z]; o = o + 1; z = z + 1 }
984 cur = k + 1
985 }
986 if go == 1 { k = k + 1 }
987 }
988
989 if mode == 1 { out[o] = 34 as u8; o = o + 1 }
990
991 // Validate only the CONCAT result -- a stringify body is arbitrary text by definition.
992 if mode == 0 {
993 if lexm_is_ident_text(out, rstart, o) == 0 { lexm_warn_bad_paste(out, rstart, o) }
994 }
995
996 iout[0] = argend + 1
997 return o
998}
999
1000func lexm_kw_is(src: *u8, n: i64, at: i64, kw: *u8, kwlen: i64) -> i64 {
1001 if at + kwlen > n { return 0 }
1002 var i: i64 = 0
1003 while i < kwlen {
1004 if src[at + i] != kw[i] { return 0 }
1005 i = i + 1
1006 }
1007 return 1
1008}
1009
1010// Expand one `@for(...) { ... }` starting at istart (which points at '@').
1011// Returns the new output offset and writes the new input offset to iout[0]; 0-1 if not well-formed.
1012// REFUSING is the honest failure here: a @for that silently expands to nothing DELETES code.
1013func lexm_for_expand(src: *u8, n: i64, istart: i64, out: *u8, o0: i64, ocap: i64, iout: *i64) -> i64 {
1014 var p: i64 = istart + 4
1015 if p >= n { return 0 - 1 }
1016 var sk: i64 = 0
1017 while sk == 0 { if p < n { if src[p] == (32 as u8) { p = p + 1 } else { sk = 1 } } else { sk = 1 } }
1018 if p >= n { return 0 - 1 }
1019 if src[p] != (40 as u8) { return 0 - 1 }
1020 p = p + 1
1021 var sk2: i64 = 0
1022 while sk2 == 0 { if p < n { if src[p] == (32 as u8) { p = p + 1 } else { sk2 = 1 } } else { sk2 = 1 } }
1023
1024 let vs: i64 = p
1025 var sk3: i64 = 0
1026 while sk3 == 0 { if p < n { if lexm_isid(src[p] as i64) == 1 { p = p + 1 } else { sk3 = 1 } } else { sk3 = 1 } }
1027 let vlen: i64 = p - vs
1028 if vlen <= 0 { return 0 - 1 }
1029
1030 var sk4: i64 = 0
1031 while sk4 == 0 { if p < n { if src[p] == (32 as u8) { p = p + 1 } else { sk4 = 1 } } else { sk4 = 1 } }
1032 if p + 2 >= n { return 0 - 1 }
1033 if src[p] != (105 as u8) { return 0 - 1 }
1034 if src[p+1] != (110 as u8) { return 0 - 1 }
1035 p = p + 2
1036 var sk5: i64 = 0
1037 while sk5 == 0 { if p < n { if src[p] == (32 as u8) { p = p + 1 } else { sk5 = 1 } } else { sk5 = 1 } }
1038 if p >= n { return 0 - 1 }
1039
1040 var listmode: i64 = 0
1041 if src[p] == (123 as u8) { listmode = 1 }
1042
1043 // Find the matching ')' by DEPTH -- the value-list form uses braces inside the parens, so a
1044 // first-')' scan would cut the sequence short.
1045 let seqs: i64 = p
1046 var d: i64 = 1
1047 var q: i64 = p
1048 var sk6: i64 = 0
1049 while sk6 == 0 {
1050 if q >= n { sk6 = 1 } else {
1051 let c: i64 = src[q] as i64
1052 if c == 0 { sk6 = 1 } else {
1053 if c == 40 { d = d + 1; q = q + 1 } else {
1054 if c == 41 { d = d - 1; if d == 0 { sk6 = 1 } else { q = q + 1 } } else { q = q + 1 } } }
1055 }
1056 }
1057 if q >= n { return 0 - 1 }
1058 let seqe: i64 = q
1059 p = q + 1
1060
1061 var sk7: i64 = 0
1062 while sk7 == 0 {
1063 if p < n {
1064 let c: i64 = src[p] as i64
1065 if c == 32 { p = p + 1 } else { if c == 10 { p = p + 1 } else { if c == 13 { p = p + 1 } else { if c == 9 { p = p + 1 } else { sk7 = 1 } } } }
1066 } else { sk7 = 1 }
1067 }
1068 if p >= n { return 0 - 1 }
1069 if src[p] != (123 as u8) { return 0 - 1 }
1070 let bodyclose: i64 = lexm_body_end(src, n, p)
1071 if bodyclose <= p { return 0 - 1 }
1072 let bs: i64 = p + 1
1073 let be: i64 = bodyclose - 1
1074
1075 let vname: *u8 = ((src as i64) + vs) as *u8
1076 var o: i64 = o0
1077
1078 if listmode == 1 {
1079 var t: i64 = seqs + 1
1080 var stop: i64 = 0
1081 while stop == 0 {
1082 if t >= seqe { stop = 1 } else {
1083 var sk8: i64 = 0
1084 while sk8 == 0 {
1085 if t < seqe {
1086 let c: i64 = src[t] as i64
1087 if c == 32 { t = t + 1 } else { if c == 44 { t = t + 1 } else { if c == 10 { t = t + 1 } else { if c == 125 { t = t + 1 } else { sk8 = 1 } } } }
1088 } else { sk8 = 1 }
1089 }
1090 if t >= seqe { stop = 1 } else {
1091 let ts: i64 = t
1092 var sk9: i64 = 0
1093 while sk9 == 0 {
1094 if t < seqe { if lexm_isid(src[t] as i64) == 1 { t = t + 1 } else { sk9 = 1 } } else { sk9 = 1 }
1095 }
1096 let tl: i64 = t - ts
1097 if tl <= 0 { stop = 1 } else {
1098 let eo: i64 = lexm_for_emit(src, bs, be, vname, vlen, ((src as i64) + ts) as *u8, tl, out, o, ocap)
1099 if eo < 0 { return 0 - 1 }
1100 o = eo
1101 out[o] = 10 as u8
1102 o = o + 1
1103 }
1104 }
1105 }
1106 }
1107 } else {
1108 var dots: i64 = 0 - 1
1109 var r: i64 = seqs
1110 while r + 1 < seqe {
1111 if src[r] == (46 as u8) { if src[r+1] == (46 as u8) { dots = r; r = seqe } else { r = r + 1 } } else { r = r + 1 }
1112 }
1113 if dots < 0 { return 0 - 1 }
1114 let lo_out: *i64 = sys_mmap(16) as *i64
1115 let hi_out: *i64 = sys_mmap(16) as *i64
1116 if lexm_for_bound(src, seqs, dots, lo_out) == 0 { return 0 - 1 }
1117 if lexm_for_bound(src, dots + 2, seqe, hi_out) == 0 { return 0 - 1 }
1118 let numbuf: *u8 = sys_mmap(64)
1119 var v: i64 = lo_out[0]
1120 while v < hi_out[0] {
1121 // lexm_emit_dec guards on v <= 0, so v == 0 correctly yields the single digit "0".
1122 let nl2: i64 = lexm_emit_dec(numbuf, 0, v)
1123 let eo2: i64 = lexm_for_emit(src, bs, be, vname, vlen, numbuf, nl2, out, o, ocap)
1124 if eo2 < 0 { return 0 - 1 }
1125 o = eo2
1126 out[o] = 10 as u8
1127 o = o + 1
1128 v = v + 1
1129 }
1130 }
1131
1132 iout[0] = bodyclose
1133 return o
1134}
1135
1136// Expand every ARG-macro use into out. Returns the expanded length, or 0 if nothing was expanded (in
1137// which case the caller MUST keep using the original buffer -- returning a copy would be a silent
1138// behaviour change on every file that uses no macros).
1139func lexm_expand(src: *u8, n: i64, out: *u8, outcap: i64) -> i64 {
1140 lexm_prescan(src, n)
1141 let argp: *i64 = sys_mmap(128) as *i64
1142 let argl: *i64 = sys_mmap(128) as *i64
1143 var did: i64 = 0
1144 var o: i64 = 0
1145 var i: i64 = 0
1146 while i < n {
1147 // Headroom for ONE emission: a whole `{ ... }` body plus the actual-arg text substituted into it.
1148 // Checked before each iteration so a single expansion can never run past the buffer. Bailing with
1149 // 0 makes the caller lex the ORIGINAL source, so the macro name reaches the parser unexpanded and
1150 // fails loudly rather than emitting a half-written body.
1151 if o >= outcap - LEXM_BODYLEN * 8 { return 0 }
1152 // COPY AN @macro DEFINITION LINE VERBATIM. `@macro AXIOM_CONST(ch, sym_id) <body>` contains the
1153 // text `AXIOM_CONST(` -- so without this the expander expands a macro INSIDE ITS OWN DEFINITION,
1154 // the definition line becomes garbage, and the NEXT iteration's prescan stores that garbage as the
1155 // body. The corruption then cascades and surfaces far away as `reserved keyword '.' cannot be used
1156 // as an identifier name`, pointing at a line that is not the cause.
1157 // STRING LITERALS ARE NOT EXPANDABLE TEXT. Copy them through verbatim, escapes included.
1158 // Without this, ANY macro name or @directive appearing inside a string literal gets expanded:
1159 // nx_macro_v7_test's own PASS message contains the text `@stringify(name)`, which was spliced
1160 // into a nested quote, broke the literal, and surfaced as `UNRESOLVED identifier 'name' in fn
1161 // main` -- an error naming a line that merely DESCRIBES the feature.
1162 // Same class as the @macro self-corruption: text that must not be expanded, being expanded.
1163 var handled: i64 = 0
1164 if src[i] == (34 as u8) {
1165 out[o] = src[i]; o = o + 1; i = i + 1
1166 var sq: i64 = 0
1167 while sq == 0 {
1168 if i >= n { sq = 1 } else {
1169 if src[i] == (0 as u8) { sq = 1 } else {
1170 if src[i] == (92 as u8) {
1171 out[o] = src[i]; o = o + 1; i = i + 1
1172 if i < n { out[o] = src[i]; o = o + 1; i = i + 1 }
1173 } else {
1174 if src[i] == (34 as u8) { out[o] = src[i]; o = o + 1; i = i + 1; sq = 1 } else {
1175 out[o] = src[i]; o = o + 1; i = i + 1
1176 } } }
1177 }
1178 }
1179 handled = 1
1180 }
1181 if handled == 0 {
1182 if src[i] == (64 as u8) {
1183 var isdef: i64 = 0
1184 if i + 7 < n {
1185 if src[i+1] == (109 as u8) { if src[i+2] == (97 as u8) { if src[i+3] == (99 as u8) {
1186 if src[i+4] == (114 as u8) { if src[i+5] == (111 as u8) { if src[i+6] == (32 as u8) { isdef = 1 } } } } } }
1187 }
1188 if isdef == 1 {
1189 // Copy through the END OF THE BODY. For a `{ ... }` macro that is the matching close
1190 // brace many lines below; copying only to end-of-line would leave the body itself exposed
1191 // to expansion and the definition would corrupt ITSELF -- the bug that previously
1192 // surfaced ~1900 lines away as `reserved keyword '.'`, pointing at an innocent line.
1193 var mp: i64 = i + 7
1194 var m1: i64 = 0
1195 while m1 == 0 { if src[mp] == (32 as u8) { mp = mp + 1 } else { m1 = 1 } }
1196 var mn: i64 = mp
1197 var m2: i64 = 0
1198 while m2 == 0 { if lexm_isid(src[mn] as i64) == 1 { mn = mn + 1 } else { m2 = 1 } }
1199 var mb: i64 = mn
1200 var m3: i64 = 0
1201 while m3 == 0 { if src[mb] == (32 as u8) { mb = mb + 1 } else { m3 = 1 } }
1202 let de: i64 = lexm_body_end(src, n, mb)
1203 while i < de { out[o] = src[i]; o = o + 1; i = i + 1 }
1204 handled = 1
1205 }
1206 }
1207 // @for(...) { ... } -- generative repetition. Placed AFTER the @macro-verbatim branch so a
1208 // `@for` written INSIDE a @macro body is copied whole rather than expanded within its own
1209 // definition (the self-corruption that once surfaced ~1900 lines away as `reserved keyword '.'`).
1210 // `@format(` cannot false-match: lexm_for_expand skips spaces after "@for" then REQUIRES '(',
1211 // so "mat(" fails, returns 0-1, and the text is copied verbatim.
1212 if handled == 0 {
1213 if src[i] == (64 as u8) {
1214 var isfor: i64 = 0
1215 if i + 4 < n {
1216 if src[i+1] == (102 as u8) { if src[i+2] == (111 as u8) { if src[i+3] == (114 as u8) { isfor = 1 } } }
1217 }
1218 if isfor == 1 {
1219 let iout: *i64 = sys_mmap(16) as *i64
1220 let no: i64 = lexm_for_expand(src, n, i, out, o, outcap, iout)
1221 if no >= 0 { o = no; i = iout[0]; did = 1; handled = 1 }
1222 }
1223 // @concat / @stringify -- same placement rationale as @for: AFTER the @macro-verbatim
1224 // branch, so a @concat inside a macro BODY is copied whole and only spliced on the NEXT
1225 // fixed-point pass, once its parameters have been substituted. Splicing before
1226 // substitution would paste the PARAMETER NAME (`prefix_double_x`), a function nobody
1227 // called.
1228 if handled == 0 {
1229 let iout2: *i64 = sys_mmap(16) as *i64
1230 var no2: i64 = 0 - 1
1231 if lexm_kw_is(src, n, i, "@concat" as *u8, 7) == 1 {
1232 no2 = lexm_paste_expand(src, n, i, 7, 0, out, o, outcap, iout2)
1233 } else {
1234 if lexm_kw_is(src, n, i, "@stringify" as *u8, 10) == 1 {
1235 no2 = lexm_paste_expand(src, n, i, 10, 1, out, o, outcap, iout2)
1236 } }
1237 if no2 >= 0 { o = no2; i = iout2[0]; did = 1; handled = 1 }
1238 }
1239 }
1240 }
1241 }
1242 if handled == 1 { } else {
1243 if lexm_isid(src[i] as i64) == 1 {
1244 let ns: i64 = i
1245 var s1: i64 = 0
1246 while s1 == 0 { if i < n { if lexm_isid(src[i] as i64) == 1 { i = i + 1 } else { s1 = 1 } } else { s1 = 1 } }
1247 let nl: i64 = i - ns
1248 var used: i64 = 0
1249 if i < n {
1250 if src[i] == (40 as u8) {
1251 let mb: *u8 = lexm_lookup(((src as i64) + ns) as *u8, nl)
1252 if (mb as i64) != 0 {
1253 if mb[0] == (40 as u8) {
1254 var nargs: i64 = 0
1255 var d: i64 = 0
1256 var a0: i64 = i + 1
1257 var p: i64 = i
1258 var s2: i64 = 0
1259 while s2 == 0 {
1260 if p >= n { s2 = 1 } else {
1261 let c: i64 = src[p] as i64
1262 if c == 40 { d = d + 1 } else {
1263 if c == 41 { d = d - 1
1264 if d == 0 {
1265 if nargs < LEXM_MAXARGS { argp[nargs] = a0; argl[nargs] = p - a0; nargs = nargs + 1 } else { nargs = nargs + 1 }
1266 s2 = 1
1267 }
1268 } else {
1269 if c == 44 { if d == 1 {
1270 if nargs < LEXM_MAXARGS { argp[nargs] = a0; argl[nargs] = p - a0; nargs = nargs + 1 } else { nargs = nargs + 1 }
1271 a0 = p + 1
1272 } } } }
1273 if s2 == 0 { p = p + 1 }
1274 }
1275 }
1276 // REFUSE an over-wide invocation rather than substituting the first 8 and emitting
1277 // the remaining parameter NAMES literally. used stays 0, so the call text is copied
1278 // verbatim and fails loudly at the parser instead of running with wrong values.
1279 if nargs > LEXM_MAXARGS { lexm_warn_too_many_args(nargs) } else {
1280 if p < n {
1281 o = lexm_subst(mb, src, argp, argl, nargs, out, o)
1282 i = p + 1
1283 used = 1
1284 did = 1
1285 } }
1286 } }
1287 } }
1288 if used == 0 {
1289 var y: i64 = ns
1290 while y < i { out[o] = src[y]; o = o + 1; y = y + 1 }
1291 }
1292 } else {
1293 out[o] = src[i]
1294 o = o + 1
1295 i = i + 1
1296 } }
1297 }
1298 out[o] = 0 as u8
1299 if did == 0 { return 0 }
1300 return o
1301}
1302
1303// ===== @if(expr) CONSTANT-EXPRESSION EVALUATOR ======================================================
1304// `@ifdef` asks does this NAME exist. `@if` asks what does this EXPRESSION evaluate to -- a strictly
1305// larger question needing a real parser.
1306//
1307// UNDEFINED-IS-ZERO is deliberate and matches every C preprocessor: it makes @if(FEATURE_X) work as a
1308// feature test without forcing every flag to be pre-declared. It is also the only SAFE default here --
1309// erroring on unknown names makes @if unusable for capability probing, and defaulting to TRUE would
1310// repeat exactly the defect this arc fixed (a guard that waves through what it cannot measure).
1311// Zero is the value that makes an unknown feature ABSENT.
1312//
1313// Precedence, lowest to highest -- standard C, because surprising a reader with novel precedence in a
1314// preprocessor is how a wrong branch compiles silently:
1315// || < && < == != >= <= > < < + - < * / % < unary ! - < ( ) literal ident
1316
1317func ppe_or(src: *u8, n: i64, pos: *i64) -> i64;
1318
1319// Scratch cell for macro-value decoding. STATIC, not a per-identifier sys_mmap: an earlier draft called
1320// sys_mmap(16) for EVERY identifier evaluated and never released it, so a file with many @if arms would
1321// burn a fresh page per name. The value is consumed immediately, so one cell suffices and the
1322// allocation is bounded by construction.
1323static ppe_scratch: *i64
1324
1325func ppe_scratch_cell() -> *i64 {
1326 if (ppe_scratch as i64) == 0 { ppe_scratch = sys_mmap(16) as *i64 }
1327 return ppe_scratch
1328}
1329
1330func ppe_skipws(src: *u8, n: i64, pos: *i64) -> i64 {
1331 var p: i64 = pos[0]
1332 var st: i64 = 0
1333 while st == 0 {
1334 if p < n {
1335 let c: i64 = src[p] as i64
1336 if c == 32 { p = p + 1 } else {
1337 if c == 9 { p = p + 1 } else { st = 1 } }
1338 } else { st = 1 }
1339 }
1340 pos[0] = p
1341 return 0
1342}
1343
1344func ppe_primary(src: *u8, n: i64, pos: *i64) -> i64 {
1345 ppe_skipws(src, n, pos)
1346 var p: i64 = pos[0]
1347 if p >= n { pos[0] = p; return 0 }
1348
1349 if src[p] == (40 as u8) {
1350 pos[0] = p + 1
1351 let v: i64 = ppe_or(src, n, pos)
1352 ppe_skipws(src, n, pos)
1353 p = pos[0]
1354 if p < n { if src[p] == (41 as u8) { p = p + 1 } }
1355 pos[0] = p
1356 return v
1357 }
1358
1359 let c0: i64 = src[p] as i64
1360 var isd: i64 = 0
1361 if c0 >= 48 { if c0 <= 57 { isd = 1 } }
1362 if isd == 1 {
1363 var v: i64 = 0
1364 var go: i64 = 1
1365 while go == 1 {
1366 if p < n {
1367 let c: i64 = src[p] as i64
1368 var d: i64 = 0
1369 if c >= 48 { if c <= 57 { d = 1 } }
1370 if d == 1 { v = v * 10 + (c - 48); p = p + 1 } else { go = 0 }
1371 } else { go = 0 }
1372 }
1373 pos[0] = p
1374 return v
1375 }
1376
1377 if lexm_isid(c0) == 1 {
1378 let s: i64 = p
1379 var go2: i64 = 1
1380 while go2 == 1 {
1381 if p < n { if lexm_isid(src[p] as i64) == 1 { p = p + 1 } else { go2 = 0 } } else { go2 = 0 }
1382 }
1383 // defined(NAME) BUILTIN. Must use the SAME definedness rule as @ifdef -- including the
1384 // TARGET_X86_64 hard pin -- or `@if(defined(TARGET_X86_64))` and `@ifdef TARGET_X86_64` would
1385 // disagree, which is a contradiction the corpus would eventually compile through.
1386 // ★ TWO SPELLINGS OF ONE QUESTION MUST SHARE ONE ANSWER.
1387 if p - s == 7 {
1388 var isdef: i64 = 1
1389 if src[s] != (100 as u8) { isdef = 0 }
1390 if src[s+1] != (101 as u8) { isdef = 0 }
1391 if src[s+2] != (102 as u8) { isdef = 0 }
1392 if src[s+3] != (105 as u8) { isdef = 0 }
1393 if src[s+4] != (110 as u8) { isdef = 0 }
1394 if src[s+5] != (101 as u8) { isdef = 0 }
1395 if src[s+6] != (100 as u8) { isdef = 0 }
1396 if isdef == 1 {
1397 var q: i64 = p
1398 var sw: i64 = 0
1399 while sw == 0 { if q < n { if src[q] == (32 as u8) { q = q + 1 } else { sw = 1 } } else { sw = 1 } }
1400 if q < n {
1401 if src[q] == (40 as u8) {
1402 q = q + 1
1403 var sw2: i64 = 0
1404 while sw2 == 0 { if q < n { if src[q] == (32 as u8) { q = q + 1 } else { sw2 = 1 } } else { sw2 = 1 } }
1405 let ns2: i64 = q
1406 var sw3: i64 = 0
1407 while sw3 == 0 { if q < n { if lexm_isid(src[q] as i64) == 1 { q = q + 1 } else { sw3 = 1 } } else { sw3 = 1 } }
1408 let nl2: i64 = q - ns2
1409 var sw4: i64 = 0
1410 while sw4 == 0 { if q < n { if src[q] == (32 as u8) { q = q + 1 } else { sw4 = 1 } } else { sw4 = 1 } }
1411 if q < n { if src[q] == (41 as u8) { q = q + 1 } }
1412 pos[0] = q
1413 if nl2 <= 0 { return 0 }
1414 // TARGET_X86_64 is BACKEND-OWNED and hard-pinned UNDEFINED (see _lex_pp_defined).
1415 if nl2 == 13 {
1416 var istx: i64 = 1
1417 var tz: i64 = 0
1418 let txn: *u8 = "TARGET_X86_64" as *u8
1419 while tz < 13 { if src[ns2 + tz] != txn[tz] { istx = 0; tz = 13 } else { tz = tz + 1 } }
1420 if istx == 1 { return 0 }
1421 }
1422 if (lexm_lookup(((src as i64) + ns2) as *u8, nl2) as i64) != 0 { return 1 }
1423 return 0
1424 } }
1425 }
1426 }
1427 pos[0] = p
1428 // An UNDEFINED name is 0. A defined name whose body is not a plain integer is also 0:
1429 // @if(SOME_TEXT_MACRO) has no numeric meaning, and 0 keeps it ABSENT rather than truthy.
1430 let mb: *u8 = lexm_lookup(((src as i64) + s) as *u8, p - s)
1431 if (mb as i64) == 0 { return 0 }
1432 let vo: *i64 = ppe_scratch_cell()
1433 if lexm_body_int(mb, vo) == 0 { return 0 }
1434 return vo[0]
1435 }
1436
1437 pos[0] = p + 1
1438 return 0
1439}
1440
1441func ppe_unary(src: *u8, n: i64, pos: *i64) -> i64 {
1442 ppe_skipws(src, n, pos)
1443 let p: i64 = pos[0]
1444 if p < n {
1445 if src[p] == (33 as u8) {
1446 // `!` only -- `!=` is a comparison and must not be eaten here.
1447 var isne: i64 = 0
1448 if p + 1 < n { if src[p+1] == (61 as u8) { isne = 1 } }
1449 if isne == 0 {
1450 pos[0] = p + 1
1451 let v: i64 = ppe_unary(src, n, pos)
1452 if v == 0 { return 1 }
1453 return 0
1454 }
1455 }
1456 if src[p] == (45 as u8) {
1457 pos[0] = p + 1
1458 let v2: i64 = ppe_unary(src, n, pos)
1459 return 0 - v2
1460 }
1461 }
1462 return ppe_primary(src, n, pos)
1463}
1464
1465func ppe_mul(src: *u8, n: i64, pos: *i64) -> i64 {
1466 var v: i64 = ppe_unary(src, n, pos)
1467 var go: i64 = 1
1468 while go == 1 {
1469 ppe_skipws(src, n, pos)
1470 let p: i64 = pos[0]
1471 if p >= n { go = 0 } else {
1472 let c: i64 = src[p] as i64
1473 if c == 42 {
1474 pos[0] = p + 1
1475 v = v * ppe_unary(src, n, pos)
1476 } else {
1477 if c == 47 {
1478 pos[0] = p + 1
1479 let d: i64 = ppe_unary(src, n, pos)
1480 // Division by zero must not trap the COMPILER. Yield 0 and continue: a wrong branch is a
1481 // visible bug, a crashed build with no line number is not.
1482 if d == 0 { v = 0 } else { v = v / d }
1483 } else {
1484 if c == 37 {
1485 pos[0] = p + 1
1486 let m: i64 = ppe_unary(src, n, pos)
1487 if m == 0 { v = 0 } else { v = v % m }
1488 } else { go = 0 } } }
1489 }
1490 }
1491 return v
1492}
1493
1494func ppe_add(src: *u8, n: i64, pos: *i64) -> i64 {
1495 var v: i64 = ppe_mul(src, n, pos)
1496 var go: i64 = 1
1497 while go == 1 {
1498 ppe_skipws(src, n, pos)
1499 let p: i64 = pos[0]
1500 if p >= n { go = 0 } else {
1501 let c: i64 = src[p] as i64
1502 if c == 43 { pos[0] = p + 1; v = v + ppe_mul(src, n, pos) } else {
1503 if c == 45 { pos[0] = p + 1; v = v - ppe_mul(src, n, pos) } else { go = 0 } }
1504 }
1505 }
1506 return v
1507}
1508
1509func ppe_cmp(src: *u8, n: i64, pos: *i64) -> i64 {
1510 var v: i64 = ppe_add(src, n, pos)
1511 var go: i64 = 1
1512 while go == 1 {
1513 ppe_skipws(src, n, pos)
1514 let p: i64 = pos[0]
1515 if p + 1 >= n { go = 0 } else {
1516 let a: i64 = src[p] as i64
1517 let b: i64 = src[p+1] as i64
1518 // TWO-CHARACTER OPERATORS FIRST. Testing `>` before `>=` would consume the `>` and leave a
1519 // stray `=`, silently turning `>=` into `>` -- an off-by-one in a CONDITIONAL, which selects
1520 // the wrong block with no diagnostic at all.
1521 if a == 61 { if b == 61 {
1522 pos[0] = p + 2
1523 let r: i64 = ppe_add(src, n, pos)
1524 if v == r { v = 1 } else { v = 0 }
1525 } else { go = 0 } } else {
1526 if a == 33 { if b == 61 {
1527 pos[0] = p + 2
1528 let r2: i64 = ppe_add(src, n, pos)
1529 if v != r2 { v = 1 } else { v = 0 }
1530 } else { go = 0 } } else {
1531 if a == 62 { if b == 61 {
1532 pos[0] = p + 2
1533 let r3: i64 = ppe_add(src, n, pos)
1534 if v >= r3 { v = 1 } else { v = 0 }
1535 } else {
1536 pos[0] = p + 1
1537 let r4: i64 = ppe_add(src, n, pos)
1538 if v > r4 { v = 1 } else { v = 0 }
1539 } } else {
1540 if a == 60 { if b == 61 {
1541 pos[0] = p + 2
1542 let r5: i64 = ppe_add(src, n, pos)
1543 if v <= r5 { v = 1 } else { v = 0 }
1544 } else {
1545 pos[0] = p + 1
1546 let r6: i64 = ppe_add(src, n, pos)
1547 if v < r6 { v = 1 } else { v = 0 }
1548 } } else { go = 0 } } } }
1549 }
1550 }
1551 return v
1552}
1553
1554func ppe_and(src: *u8, n: i64, pos: *i64) -> i64 {
1555 var v: i64 = ppe_cmp(src, n, pos)
1556 var go: i64 = 1
1557 while go == 1 {
1558 ppe_skipws(src, n, pos)
1559 let p: i64 = pos[0]
1560 if p + 1 >= n { go = 0 } else {
1561 if src[p] == (38 as u8) { if src[p+1] == (38 as u8) {
1562 pos[0] = p + 2
1563 // NO SHORT-CIRCUIT: the right side is ALWAYS parsed, because skipping it would leave the
1564 // cursor mid-expression and corrupt everything after. Constant folding has no side
1565 // effects, so evaluating both sides is free and keeps the parse position honest.
1566 let r: i64 = ppe_cmp(src, n, pos)
1567 var t: i64 = 0
1568 if v != 0 { if r != 0 { t = 1 } }
1569 v = t
1570 } else { go = 0 } } else { go = 0 }
1571 }
1572 }
1573 return v
1574}
1575
1576func ppe_or(src: *u8, n: i64, pos: *i64) -> i64 {
1577 var v: i64 = ppe_and(src, n, pos)
1578 var go: i64 = 1
1579 while go == 1 {
1580 ppe_skipws(src, n, pos)
1581 let p: i64 = pos[0]
1582 if p + 1 >= n { go = 0 } else {
1583 if src[p] == (124 as u8) { if src[p+1] == (124 as u8) {
1584 pos[0] = p + 2
1585 let r: i64 = ppe_and(src, n, pos)
1586 var t: i64 = 0
1587 if v != 0 { t = 1 }
1588 if r != 0 { t = 1 }
1589 v = t
1590 } else { go = 0 } } else { go = 0 }
1591 }
1592 }
1593 return v
1594}
1595
1596// The cursor is a STATIC shared cell: only one @if is evaluated at a time, and the recursive `( ... )`
1597// case in ppe_primary deliberately SHARES this cursor with its caller -- that sharing is what advances
1598// the parse position through a nested expression.
1599static ppe_pos: *i64
1600
1601func ppe_eval_at(src: *u8, n: i64, openp: i64) -> i64 {
1602 if (ppe_pos as i64) == 0 { ppe_pos = sys_mmap(16) as *i64 }
1603 let pos: *i64 = ppe_pos
1604 pos[0] = openp + 1
1605 let v: i64 = ppe_or(src, n, pos)
1606 if v != 0 { return 1 }
1607 return 0
1608}
1609
1610func _lex_src_eq(L: *Lex, needle: *u8, n: i64) -> i64 {
1611 let s: *u8 = L.src
1612 var i: i64 = 0
1613 while i < n {
1614 if (s[L.pos + i] & 0xff) != (needle[i] & 0xff) { return 0 }
1615 i = i + 1
1616 }
1617 return 1
1618}
1619
1620// Bytes-equal at an arbitrary src position (not L.pos).
1621func _lex_src_eq_at(L: *Lex, at: i64, needle: *u8, n: i64) -> i64 {
1622 let s: *u8 = L.src
1623 var i: i64 = 0
1624 while i < n {
1625 if (s[at + i] & 0xff) != (needle[i] & 0xff) { return 0 }
1626 i = i + 1
1627 }
1628 return 1
1629}
1630
1631// Skip from current pos to past the next newline (consume EOL).
1632func _lex_skip_to_eol(L: *Lex) -> i64 {
1633 while peek(L) != 0 {
1634 let c: i64 = advance(L)
1635 if c == 0x0A { return 0 }
1636 }
1637 return 0
1638}
1639
1640// Skip src until we find a matching @endif (depth-tracked for nested).
1641func _lex_skip_to_matching_endif(L: *Lex) -> i64 {
1642 var depth: i64 = 1
1643 while depth > 0 {
1644 if peek(L) == 0 { return 0 } // EOF
1645 if peek(L) == 0x40 {
1646 let at1: i64 = L.pos + 1
1647 if _lex_src_eq_at(L, at1, "ifdef " as *u8, 6) == 1 { depth = depth + 1; L.pos = at1 + 6; _lex_skip_to_eol(L); continue }
1648 if _lex_src_eq_at(L, at1, "ifndef " as *u8, 7) == 1 { depth = depth + 1; L.pos = at1 + 7; _lex_skip_to_eol(L); continue }
1649 if _lex_src_eq_at(L, at1, "endif" as *u8, 5) == 1 { depth = depth - 1; L.pos = at1 + 5; _lex_skip_to_eol(L); continue }
1650 }
1651 advance(L)
1652 }
1653 return 0
1654}
1655
1656// Preprocessor directive handler (2026-05-20, four-pillar response to
1657// the silent-divergence bug where the self-host parser ignored
1658// @ifdef/@ifndef while the C bootstrap honored them, causing
1659// nx_syscalls.nx const SYS_SOCKET to resolve differently across
1660// build paths). Substrate now OWNS the preprocessor; both build
1661// paths agree on conditional inclusion.
1662//
1663// Polarity: TARGET_X86_64 is treated as NOT defined here because the
1664// self-host driver's codegen (x86ctx_rv64_to_x86_64_syscall) expects
1665// source to carry RV64 syscall numbers and translates them at emit
1666// time. Therefore the @ifndef TARGET_X86_64 branch (RV64 numbers)
1667// is what should reach the parser. The C bootstrap's --target x86_64
1668// path does the inverse and pre-defines TARGET_X86_64; same source
1669// resolves to different bytes per backend, but each backend gets the
1670// numbers it expects.
1671// Is NAME defined, for `@ifdef` / `@ifndef`? THIS REPLACED A SINGLE HARDCODED STRING COMPARE against
1672// "TARGET_X86_64", under which every OTHER name resolved as DEFINED -- so `@ifdef ANYTHING` always kept
1673// its block and `@ifndef ANYTHING` always dropped one, with no diagnostic. Same shape as a guard that
1674// waves through whatever it cannot measure.
1675//
1676// TARGET_X86_64 stays HARD-PINNED UNDEFINED and is deliberately NOT answered from the macro table. It is
1677// a BACKEND-OWNED axis, not a user macro: per the polarity note above, the x86 self-host emits through
1678// x86ctx_rv64_to_x86_64_syscall, which expects RV64-numbered syscalls in the source. A file that could
1679// `@macro TARGET_X86_64 1` would flip on the raw-x86 branch of nx_syscalls / nx_hal / nx_dirent /
1680// nx_poll / nx_probe / nx_fcntl / nx_self_build, and every syscall const would then be translated TWICE.
1681// Source must not be able to lie about its own target.
1682func _lex_pp_defined(L: *Lex, name_pos: i64) -> i64 {
1683 let s: *u8 = L.src
1684 var e: i64 = name_pos
1685 var st: i64 = 0
1686 while st == 0 { if is_alnum(s[e] as i64) == 1 { e = e + 1 } else { st = 1 } }
1687 let nl: i64 = e - name_pos
1688 if nl <= 0 { return 0 }
1689 if nl == 13 { if _lex_src_eq_at(L, name_pos, "TARGET_X86_64" as *u8, 13) == 1 { return 0 } }
1690 if (lexm_lookup(((s as i64) + name_pos) as *u8, nl) as i64) != 0 { return 1 }
1691 return 0
1692}
1693
1694// ===== @if/@elif/@else BRANCH STATE =================================================================
1695// The evaluator answers is this arm's condition true. THIS answers the harder question: has an earlier
1696// arm of THIS chain already run, and which chain is this one. Get it wrong and the failure mode is
1697// emitting EXTRA arms -- which link, run, and pass every runtime test.
1698const PPB_MAX: i64 = 32
1699
1700static ppb_taken: *i64
1701static ppb_depth: *i64
1702
1703func ppb_init() -> i64 {
1704 if (ppb_depth as i64) != 0 { return 0 }
1705 ppb_taken = sys_mmap(PPB_MAX * 8) as *i64
1706 ppb_depth = sys_mmap(16) as *i64
1707 return 0
1708}
1709
1710// REFUSE past the cap rather than wrapping. A wrapped index silently corrupts an OUTER chain's state,
1711// so the arm selection goes wrong far from the deep nesting that caused it.
1712// The index is hoisted to a local: this corpus has a recorded CONST[i] index-expression miscompile, and
1713// a nested subscript whose index is itself a load is exactly that shape.
1714func ppb_push(v: i64) -> i64 {
1715 ppb_init()
1716 let d: i64 = ppb_depth[0]
1717 if d >= PPB_MAX { return 0 }
1718 ppb_taken[d] = v
1719 ppb_depth[0] = d + 1
1720 return 1
1721}
1722
1723func ppb_pop() -> i64 {
1724 ppb_init()
1725 let d: i64 = ppb_depth[0]
1726 if d > 0 { ppb_depth[0] = d - 1 }
1727 return 0
1728}
1729
1730// Defaults to 1 (=already taken) on an empty stack, so a stray @elif/@else with no opener SKIPS its
1731// body instead of compiling it. Unbalanced directives should drop code, never inject it.
1732func ppb_top_taken() -> i64 {
1733 ppb_init()
1734 let d: i64 = ppb_depth[0]
1735 if d <= 0 { return 1 }
1736 let i: i64 = d - 1
1737 return ppb_taken[i]
1738}
1739
1740func ppb_set_top(v: i64) -> i64 {
1741 ppb_init()
1742 let d: i64 = ppb_depth[0]
1743 if d <= 0 { return 0 }
1744 let i: i64 = d - 1
1745 ppb_taken[i] = v
1746 return 0
1747}
1748
1749func ppb_reset() -> i64 {
1750 ppb_init()
1751 ppb_depth[0] = 0
1752 return 0
1753}
1754
1755// Which directive is at `at`? 1=@if 2=@ifdef 3=@ifndef 4=@elif 5=@else 6=@endif 0=none.
1756// LONGER FORMS FIRST: "@if" is a prefix of "@ifdef"/"@ifndef", so testing "@if" first would classify
1757// every @ifdef in nx_syscalls as an @if and push a bogus chain onto the stack.
1758func _lex_pp_kw_at(L: *Lex, at: i64) -> i64 {
1759 if _lex_src_eq_at(L, at, "@ifdef " as *u8, 7) == 1 { return 2 }
1760 if _lex_src_eq_at(L, at, "@ifndef " as *u8, 8) == 1 { return 3 }
1761 if _lex_src_eq_at(L, at, "@endif" as *u8, 6) == 1 { return 6 }
1762 if _lex_src_eq_at(L, at, "@elif" as *u8, 5) == 1 { return 4 }
1763 if _lex_src_eq_at(L, at, "@else" as *u8, 5) == 1 { return 5 }
1764 if _lex_src_eq_at(L, at, "@if(" as *u8, 4) == 1 { return 1 }
1765 return 0
1766}
1767
1768// Skip to the next @elif / @else / @endif AT DEPTH 0, leaving L.pos on its '@'.
1769// ALL THREE OPENERS COUNT (@if, @ifdef, @ifndef): a nested @ifdef inside an @if arm would otherwise let
1770// its @endif close the OUTER chain, after which the outer @else is treated as top-level and its dead
1771// body compiles.
1772func _lex_skip_to_next_branch(L: *Lex) -> i64 {
1773 let s: *u8 = L.src
1774 var p: i64 = L.pos
1775 var depth: i64 = 0
1776 var stop: i64 = 0
1777 while stop == 0 {
1778 if s[p] == (0 as u8) { stop = 1 } else {
1779 if s[p] == (64 as u8) {
1780 let k: i64 = _lex_pp_kw_at(L, p)
1781 if k == 1 { depth = depth + 1 } else {
1782 if k == 2 { depth = depth + 1 } else {
1783 if k == 3 { depth = depth + 1 } else {
1784 if k == 6 {
1785 if depth == 0 { L.pos = p; stop = 1 } else { depth = depth - 1 }
1786 } else {
1787 if k == 4 { if depth == 0 { L.pos = p; stop = 1 } } else {
1788 if k == 5 { if depth == 0 { L.pos = p; stop = 1 } } else { } } } } } }
1789 }
1790 if stop == 0 {
1791 // Count newlines we pass or every diagnostic after a skipped arm drifts.
1792 if s[p] == (10 as u8) { L.line = L.line + 1 }
1793 p = p + 1
1794 }
1795 }
1796 }
1797 return 0
1798}
1799
1800func _lex_handle_preprocessor(L: *Lex) -> i64 {
1801 if peek(L) != 0x40 { return 0 } // not '@'
1802 let after_at: i64 = L.pos + 1
1803 // @macro NAME BODY -- zero-arg form. Parsed and stored, then the line is consumed exactly like the
1804 // other directives. The ARG form (@macro NAME(a,b) ...) is stored too, but its body will not parse as
1805 // an integer so it stays INERT rather than half-expanded -- a partially substituted macro is a silent
1806 // miscompile, and inert is the honest state until arg substitution ships.
1807 if _lex_src_eq_at(L, after_at, "macro " as *u8, 6) == 1 {
1808 let s: *u8 = L.src
1809 var np: i64 = after_at + 6
1810 var sk1: i64 = 0
1811 while sk1 == 0 { if s[np] == (32 as u8) { np = np + 1 } else { sk1 = 1 } }
1812 var ne: i64 = np
1813 var sk2: i64 = 0
1814 while sk2 == 0 { if is_alnum(s[ne] as i64) == 1 { ne = ne + 1 } else { sk2 = 1 } }
1815 var bp: i64 = ne
1816 var sk3: i64 = 0
1817 while sk3 == 0 { if s[bp] == (32 as u8) { bp = bp + 1 } else { sk3 = 1 } }
1818 let slen: i64 = lexm_strlen(s)
1819 let be: i64 = lexm_body_end(s, slen, bp)
1820 lexm_define(((s as i64) + np) as *u8, ne - np, ((s as i64) + bp) as *u8, be - bp)
1821 // Skip past the WHOLE body -- for a `{ ... }` macro that spans many lines. Leaving L.pos on the
1822 // first line would feed the body text to the lexer as if it were ordinary code. Count the
1823 // newlines we jump over, or every diagnostic after the first multi-line macro reports a line
1824 // number that drifts further off with each definition.
1825 var nlp: i64 = L.pos
1826 while nlp < be {
1827 if s[nlp] == (10 as u8) { L.line = L.line + 1 }
1828 nlp = nlp + 1
1829 }
1830 L.pos = be
1831 _lex_skip_to_eol(L)
1832 return 1
1833 }
1834 if _lex_src_eq_at(L, after_at, "endif" as *u8, 5) == 1 {
1835 ppb_pop()
1836 L.pos = after_at + 5
1837 _lex_skip_to_eol(L)
1838 return 1
1839 }
1840 // @undef NAME -- retract a definition so a later @ifdef/@ifndef sees it as gone.
1841 if _lex_src_eq_at(L, after_at, "undef " as *u8, 6) == 1 {
1842 let s2: *u8 = L.src
1843 var up: i64 = after_at + 6
1844 var uk: i64 = 0
1845 while uk == 0 { if s2[up] == (32 as u8) { up = up + 1 } else { uk = 1 } }
1846 var ue: i64 = up
1847 var uk2: i64 = 0
1848 while uk2 == 0 { if is_alnum(s2[ue] as i64) == 1 { ue = ue + 1 } else { uk2 = 1 } }
1849 lexm_undef(((s2 as i64) + up) as *u8, ue - up)
1850 L.pos = up
1851 _lex_skip_to_eol(L)
1852 return 1
1853 }
1854 // @if(expr) -- open a chain. Cannot collide with @ifdef /@ifndef : char 3 is '(' not 'd'/'n'.
1855 if _lex_src_eq_at(L, after_at, "if(" as *u8, 3) == 1 {
1856 let openp: i64 = after_at + 2
1857 let v: i64 = ppe_eval_at(L.src, lexm_strlen(L.src), openp)
1858 ppb_push(v)
1859 L.pos = openp
1860 _lex_skip_to_eol(L)
1861 if v == 0 { _lex_skip_to_next_branch(L) }
1862 return 1
1863 }
1864 // @elif(expr) -- evaluated ONLY if no earlier arm ran.
1865 if _lex_src_eq_at(L, after_at, "elif(" as *u8, 5) == 1 {
1866 let openp: i64 = after_at + 4
1867 L.pos = openp
1868 _lex_skip_to_eol(L)
1869 if ppb_top_taken() == 1 { _lex_skip_to_next_branch(L) } else {
1870 let v: i64 = ppe_eval_at(L.src, lexm_strlen(L.src), openp)
1871 if v == 1 { ppb_set_top(1) } else { _lex_skip_to_next_branch(L) }
1872 }
1873 return 1
1874 }
1875 // @else -- runs only if no earlier arm ran. Uses skip_to_matching_endif, not next_branch: nothing
1876 // can follow @else in a chain, so the only legal next stop is the @endif.
1877 if _lex_src_eq_at(L, after_at, "else" as *u8, 4) == 1 {
1878 L.pos = after_at + 4
1879 _lex_skip_to_eol(L)
1880 if ppb_top_taken() == 1 { _lex_skip_to_matching_endif(L) } else { ppb_set_top(1) }
1881 return 1
1882 }
1883 if _lex_src_eq_at(L, after_at, "ifdef " as *u8, 6) == 1 {
1884 let name_pos: i64 = after_at + 6
1885 let defd: i64 = _lex_pp_defined(L, name_pos)
1886 // Push a CONSTANT so @endif's pop stays balanced. Deliberately NOT the real truth value with a
1887 // skip-to-next-branch: that would give `@ifdef X ... @else ... @endif` for free, but EVERY
1888 // syscall number in the ecosystem flows through @ifdef/@ifndef TARGET_X86_64 pairs that contain
1889 // NO @else, so the two behaviours are identical there -- untestable by the corpus and pure
1890 // downside risk. @else-after-@ifdef is its own rung with its own gate.
1891 ppb_push(1)
1892 L.pos = name_pos
1893 _lex_skip_to_eol(L)
1894 // Keep the block only when NAME is defined. For TARGET_X86_64 _lex_pp_defined returns 0, so this
1895 // skips exactly as the old hardcoded compare did -- byte-identical for the whole production corpus.
1896 if defd == 0 { _lex_skip_to_matching_endif(L) }
1897 return 1
1898 }
1899 if _lex_src_eq_at(L, after_at, "ifndef " as *u8, 7) == 1 {
1900 let name_pos: i64 = after_at + 7
1901 let defd: i64 = _lex_pp_defined(L, name_pos)
1902 ppb_push(1)
1903 L.pos = name_pos
1904 _lex_skip_to_eol(L)
1905 // Inverse. TARGET_X86_64 -> defd 0 -> block KEPT (the RV64-numbered branch), matching the old
1906 // behaviour that the syscall translator depends on.
1907 if defd == 1 { _lex_skip_to_matching_endif(L) }
1908 return 1
1909 }
1910 return 0
1911}
1912
1913// ---- sub-lexers ----
1914
1915func skip_ws_comments(L: *Lex) -> i64 {
1916 var go: i64 = 1
1917 while go {
1918 let c: i64 = peek(L)
1919 if c == 0x20 { advance(L) }
1920 if c == 0x09 { advance(L) }
1921 if c == 0x0A { advance(L) }
1922 if c == 0x0D { advance(L) }
1923 if c == 0x2F {
1924 if peek2(L) == 0x2F {
1925 // Line comment
1926 while peek(L) != 0 {
1927 if peek(L) == 0x0A { break }
1928 advance(L)
1929 }
1930 continue
1931 }
1932 // '/' not followed by another '/' -- division operator.
1933 // Hand it back to the outer lexer to tokenise. Setting
1934 // go=0 directly here avoids an infinite loop through the
1935 // cascade below (which treats '/' as whitespace-like and
1936 // never sets go=0, looping forever on a bare slash).
1937 go = 0
1938 continue
1939 }
1940 if c != 0x20 {
1941 if c != 0x09 {
1942 if c != 0x0A {
1943 if c != 0x0D {
1944 go = 0
1945 }
1946 }
1947 }
1948 }
1949 }
1950 return 0
1951}
1952
1953func lex_ident_or_kw(L: *Lex) -> i64 {
1954 let tr: *u8 = sys_mmap(TOK_BYTES)
1955 let t: *Tok = tr as *Tok
1956 t.kind = 2
1957 t.line = L.line
1958 t.col = L.col
1959 t.int_val = 0
1960 t.text0 = 0; t.text1 = 0; t.text2 = 0; t.text3 = 0
1961 t.text4 = 0; t.text5 = 0; t.text6 = 0; t.text7 = 0
1962 let text: *u8 = tok_text_ptr(t)
1963 // ALWAYS CONSUME THE WHOLE IDENTIFIER. The old loop was `while i < 63 { ... advance(L) ... }`, so on
1964 // reaching 63 it stopped WITHOUT consuming the tail -- a 70-char name emitted a 63-char IDENT and the
1965 // lexer then resumed at char 64 and lexed the remainder as a SECOND identifier token. That is a token
1966 // SPLIT, and the resulting parse error names a FABRICATED identifier that appears nowhere in the
1967 // source. Storing stops at LEX_IDENT_MAX; consuming does not.
1968 //
1969 // We store 63 rather than silently consuming-without-warning because two identifiers sharing 63
1970 // leading chars would then collapse to ONE token -- a silent miscompile, strictly worse than a loud
1971 // parse error. The warning ANNOUNCES that risk instead of hiding it.
1972 //
1973 // The 63/64 contract is LOAD-BEARING beyond this function: nx_parse.nx copy_name_into_local and
1974 // local_name_eq (and ~12 more loops) are bounded at 64 and are correct ONLY because a NUL is
1975 // guaranteed inside 64 bytes. Do not remove this cap without migrating those.
1976 var i: i64 = 0
1977 var over: i64 = 0
1978 var go: i64 = 1
1979 while go == 1 {
1980 let c: i64 = peek(L)
1981 if is_alnum(c) == 0 { go = 0 } else {
1982 if i < LEX_IDENT_MAX { text[i] = c & 0xFF; i = i + 1 } else { over = over + 1 }
1983 advance(L)
1984 }
1985 }
1986 text[i] = 0
1987 if over > 0 { lex_warn_ident_too_long(text, i, i + over) }
1988 t.kind = keyword_lookup(text, i)
1989 // @macro EXPANSION -- zero-arg NUMERIC form (self-host parity with the C bootstrap).
1990 // Guarded to kind 2 (generic IDENT) ON PURPOSE: keyword_lookup returns a keyword kind for `func`,
1991 // `while`, ... and a macro must NEVER shadow a keyword, or `@macro func 3` would silently break the
1992 // language rather than erroring. Kind 1 is INT (see lex_number), so a hit becomes a literal exactly
1993 // as if the number had been typed -- which is what the C bootstrap does.
1994 // A body that is not a plain integer does NOT expand: the identifier passes through unchanged, so an
1995 // arg-form or token-pasting macro is INERT rather than half-applied.
1996 if t.kind == 2 {
1997 let mb: *u8 = lexm_lookup(text, i)
1998 if (mb as i64) != 0 {
1999 let mv: *i64 = sys_mmap(16) as *i64
2000 if lexm_body_int(mb, mv) == 1 {
2001 t.kind = 1
2002 t.int_val = mv[0]
2003 }
2004 }
2005 }
2006 push_tok(L, t)
2007 return 0
2008}
2009
2010func lex_number(L: *Lex) -> i64 {
2011 let tr: *u8 = sys_mmap(TOK_BYTES)
2012 let t: *Tok = tr as *Tok
2013 t.kind = 1
2014 t.line = L.line
2015 t.col = L.col
2016 t.int_val = 0
2017 t.text0 = 0; t.text1 = 0; t.text2 = 0; t.text3 = 0
2018 t.text4 = 0; t.text5 = 0; t.text6 = 0; t.text7 = 0
2019
2020 var n: i64 = 0
2021 let c0: i64 = peek(L)
2022 if c0 == 0x30 {
2023 let c1: i64 = peek2(L)
2024 if c1 == 0x78 { // 0x hex
2025 advance(L); advance(L)
2026 var go: i64 = 1
2027 while go {
2028 let c: i64 = peek(L)
2029 if c == 0x5F { advance(L); continue }
2030 if is_hexdigit(c) {
2031 n = (n << 4) | hex_val(c)
2032 advance(L)
2033 }
2034 if is_hexdigit(c) == 0 { if c != 0x5F { go = 0 } }
2035 }
2036 t.int_val = n
2037 push_tok(L, t)
2038 return 0
2039 }
2040 if c1 == 0x62 { // 0b binary
2041 advance(L); advance(L)
2042 var go: i64 = 1
2043 while go {
2044 let c: i64 = peek(L)
2045 if c == 0x5F { advance(L); continue }
2046 if c == 0x30 { n = (n << 1); advance(L); continue }
2047 if c == 0x31 { n = (n << 1) | 1; advance(L); continue }
2048 go = 0
2049 }
2050 t.int_val = n
2051 push_tok(L, t)
2052 return 0
2053 }
2054 if c1 == 0x6F { // 0o octal
2055 advance(L); advance(L)
2056 var go: i64 = 1
2057 while go {
2058 let c: i64 = peek(L)
2059 if c == 0x5F { advance(L); continue }
2060 if c >= 0x30 {
2061 if c <= 0x37 {
2062 n = (n << 3) | (c - 0x30)
2063 advance(L)
2064 continue
2065 }
2066 }
2067 go = 0
2068 }
2069 t.int_val = n
2070 push_tok(L, t)
2071 return 0
2072 }
2073 }
2074 // Decimal. Consume whole-part digits.
2075 var go: i64 = 1
2076 while go {
2077 let c: i64 = peek(L)
2078 if c == 0x5F { advance(L); continue }
2079 if is_digit(c) {
2080 n = n * 10 + (c - 0x30)
2081 advance(L)
2082 continue
2083 }
2084 go = 0
2085 }
2086
2087 // Float split: if the current char is '.' AND the next char is a
2088 // digit (not a method call like `x.foo` or range like `0..10`),
2089 // switch to TK_FLOAT and consume the fractional part.
2090 let c_after: i64 = peek(L)
2091 if c_after == 0x2E {
2092 let c_next: i64 = peek2(L)
2093 if is_digit(c_next) {
2094 advance(L) // consume '.'
2095 var frac: i64 = 0
2096 var frac_digits: i64 = 0
2097 var fgo: i64 = 1
2098 while fgo {
2099 let c: i64 = peek(L)
2100 if c == 0x5F { advance(L); continue }
2101 if is_digit(c) {
2102 frac = frac * 10 + (c - 0x30)
2103 frac_digits = frac_digits + 1
2104 advance(L)
2105 continue
2106 }
2107 fgo = 0
2108 }
2109 // Optional 'f' / 'F' suffix -- now distinguished:
2110 // `1.5f32` / `1.5F32` -> TK_FLOAT_F32 (explicit single)
2111 // `1.5f64` / `1.5F64` -> TK_FLOAT (explicit default)
2112 // `1.5f` / `1.5` -> TK_FLOAT (default = f64)
2113 // Lexer consumes the suffix bytes; parse.nx dispatches on
2114 // the resulting kind.
2115 var sfx_kind: i64 = 4 // TK_FLOAT (default f64)
2116 let c_sfx: i64 = peek(L)
2117 if c_sfx == 0x66 {
2118 advance(L)
2119 let n1: i64 = L.src[L.pos]
2120 let n2: i64 = L.src[L.pos + 1]
2121 if n1 == 0x33 {
2122 if n2 == 0x32 {
2123 advance(L); advance(L)
2124 sfx_kind = 5 // TK_FLOAT_F32
2125 }
2126 }
2127 if n1 == 0x36 {
2128 if n2 == 0x34 {
2129 advance(L); advance(L)
2130 // sfx_kind stays at TK_FLOAT (default)
2131 }
2132 }
2133 }
2134 if c_sfx == 0x46 {
2135 advance(L)
2136 let n1: i64 = L.src[L.pos]
2137 let n2: i64 = L.src[L.pos + 1]
2138 if n1 == 0x33 {
2139 if n2 == 0x32 {
2140 advance(L); advance(L)
2141 sfx_kind = 5
2142 }
2143 }
2144 if n1 == 0x36 {
2145 if n2 == 0x34 {
2146 advance(L); advance(L)
2147 }
2148 }
2149 }
2150 t.kind = sfx_kind
2151 t.int_val = n
2152 t.text0 = frac
2153 t.text1 = frac_digits
2154 push_tok(L, t)
2155 return 0
2156 }
2157 }
2158
2159 t.int_val = n
2160 push_tok(L, t)
2161 return 0
2162}
2163
2164func lex_string(L: *Lex) -> i64 {
2165 let tr: *u8 = sys_mmap(TOK_BYTES)
2166 let t: *Tok = tr as *Tok
2167 t.kind = 3
2168 t.line = L.line
2169 t.col = L.col
2170 t.int_val = 0
2171 t.text0 = 0; t.text1 = 0; t.text2 = 0; t.text3 = 0
2172 t.text4 = 0; t.text5 = 0; t.text6 = 0; t.text7 = 0
2173 t.str_data = 0
2174 t.str_len = 0
2175 advance(L) // opening "
2176
2177 // Pre-scan to determine the raw length (upper bound; escapes
2178 // shrink it). Self-host pipeline NEEDS escape processing here:
2179 // string literals like "nxc: expand start\n" arrive with the
2180 // backslash-n two-byte sequence; without processing, sys_write
2181 // emits the literal `\` + `n` instead of a newline, so anything
2182 // that uses a string literal at runtime miscompiles.
2183 let scan_start: i64 = L.pos
2184 var scan: i64 = L.pos
2185 let src_chk: *u8 = L.src
2186 // ESCAPE-AWARE pre-scan (root-cause fix 2026-07-29): the old scan stopped at the FIRST
2187 // 0x22 byte INCLUDING an escaped \" -- but the copy loop below correctly continues past
2188 // escaped quotes, so any literal containing \" allocated a TOO-SMALL buffer and the copy
2189 // trampled the heap. Page rounding hid it for small literals; large escape-heavy literals
2190 // corrupted neighboring allocations or SIGSEGVd -- the ecosystem's recurring "~4.5KB
2191 // literal ceiling" was THIS bug wearing a size costume. Witness: runtime/nx_litprobe2.nx.
2192 while src_chk[scan] != 0 {
2193 var stepped: i64 = 0
2194 if src_chk[scan] == 0x5C {
2195 if src_chk[scan + 1] == 0 { break }
2196 scan = scan + 2
2197 stepped = 1
2198 }
2199 if stepped == 0 {
2200 if src_chk[scan] == 0x22 { break }
2201 scan = scan + 1
2202 }
2203 }
2204 let raw_len: i64 = scan - scan_start
2205
2206 let buf: *u8 = sys_mmap(raw_len + 16)
2207
2208 // Inline copy + populate text[] prefix for legacy consumers.
2209 let text: *u8 = tok_text_ptr(t)
2210 var i: i64 = 0
2211 while peek(L) != 0 {
2212 if peek(L) == 0x22 { break }
2213 var c: i64 = peek(L) & 0xFF
2214 if c == 0x5C {
2215 // Escape sequence: peek next char, advance past both.
2216 advance(L)
2217 let nx: i64 = peek(L) & 0xFF
2218 if nx == 0x6E { c = 0x0A } // \n
2219 if nx == 0x74 { c = 0x09 } // \t
2220 if nx == 0x72 { c = 0x0D } // \r
2221 if nx == 0x5C { c = 0x5C } // \\
2222 if nx == 0x22 { c = 0x22 } // \"
2223 if nx == 0x30 { c = 0x00 } // \0 (single, no octal)
2224 // \xNN hex escape -- two hex digits encode one byte.
2225 // Added 2026-05-20 per cardinal
2226 // feedback-lints-are-nishi-bits-up-not-bash: substrate-
2227 // honesty requires the self-host parser to match the C
2228 // bootstrap on string literals. Without this branch the
2229 // self-host emitted literal `\`,`x`,`0`,`0` for `\x00`
2230 // while the C bootstrap emitted a single NUL byte --
2231 // diverged behavior on the same source.
2232 if nx == 0x78 { // 'x'
2233 advance(L) // past 'x'
2234 let h1: i64 = peek(L) & 0xFF
2235 advance(L) // past first hex digit
2236 let h2: i64 = peek(L) & 0xFF
2237 // Don't advance past h2 here -- the outer loop's
2238 // trailing advance(L) handles it.
2239 var v1: i64 = 0
2240 if h1 >= 0x30 { if h1 <= 0x39 { v1 = h1 - 0x30 } }
2241 if h1 >= 0x41 { if h1 <= 0x46 { v1 = h1 - 0x37 } } // A-F
2242 if h1 >= 0x61 { if h1 <= 0x66 { v1 = h1 - 0x57 } } // a-f
2243 var v2: i64 = 0
2244 if h2 >= 0x30 { if h2 <= 0x39 { v2 = h2 - 0x30 } }
2245 if h2 >= 0x41 { if h2 <= 0x46 { v2 = h2 - 0x37 } }
2246 if h2 >= 0x61 { if h2 <= 0x66 { v2 = h2 - 0x57 } }
2247 c = (v1 << 4) | v2
2248 }
2249 // Other escapes: drop the backslash + use the literal char
2250 // (matches gas-style "unknown escape passes literal").
2251 if nx != 0x6E { if nx != 0x74 { if nx != 0x72 {
2252 if nx != 0x5C { if nx != 0x22 { if nx != 0x30 {
2253 if nx != 0x78 {
2254 c = nx
2255 }
2256 } } } } } }
2257 }
2258 buf[i] = c
2259 if i < 63 { text[i] = c }
2260 i = i + 1
2261 advance(L)
2262 }
2263 buf[i] = 0
2264 if i < 63 { text[i] = 0 }
2265
2266 t.str_data = buf as i64
2267 t.str_len = i
2268
2269 if peek(L) == 0x22 { advance(L) }
2270 push_tok(L, t)
2271 return 0
2272}
2273
2274func emit_punct(L: *Lex, kind: i64, len: i64) -> i64 {
2275 let tr: *u8 = sys_mmap(TOK_BYTES)
2276 let t: *Tok = tr as *Tok
2277 t.kind = kind
2278 t.line = L.line
2279 t.col = L.col
2280 t.int_val = 0
2281 t.text0 = 0; t.text1 = 0; t.text2 = 0; t.text3 = 0
2282 t.text4 = 0; t.text5 = 0; t.text6 = 0; t.text7 = 0
2283 var k: i64 = 0
2284 while k < len {
2285 advance(L)
2286 k = k + 1
2287 }
2288 push_tok(L, t)
2289 return 0
2290}
2291
2292// ---- public entry ----
2293//
2294// Tokenise the entire source, terminate with an EOF token, return
2295// the tokens array.
2296
2297func lex_source(src: *u8, cap: i64) -> *Tok {
2298 let toks_raw: *u8 = sys_mmap(cap * TOK_BYTES + TOK_BYTES)
2299 let toks: *Tok = toks_raw as *Tok
2300
2301 // ARG-MACRO PRE-EXPANSION. Returns 0 when nothing expanded, and in that case we keep the ORIGINAL
2302 // buffer -- so a file with no arg macros takes byte-for-byte the same path as before this existed.
2303 // ARG-MACRO PRE-EXPANSION, ITERATED TO A FIXED POINT.
2304 // A SINGLE pass is not enough: a macro body may itself invoke another macro (nx_macro_test defines
2305 // MAX(a,b) whose body calls SQ(a - b)), and a macro emitted BY an expansion is never seen by the pass
2306 // that emitted it. So re-run until a pass reports nothing expanded -- that is the fixed point.
2307 // BOUNDED AT 8: a self-referential macro (`@macro A A`) would otherwise expand forever and HANG THE
2308 // COMPILER, which is far worse than failing. At the cap we stop and let the unexpanded name reach the
2309 // parser, where it surfaces as a loud `undefined function` rather than a silent wrong program.
2310 // 8 levels is far beyond any real nesting depth.
2311 var srcn: i64 = 0
2312 while src[srcn] != (0 as u8) { srcn = srcn + 1 }
2313 var cur: *u8 = src
2314 var curn: i64 = srcn
2315 var expanded: i64 = 0
2316 var iter: i64 = 0
2317 var more: i64 = 1
2318 while more == 1 {
2319 if iter >= 8 { more = 0 } else {
2320 let cap2: i64 = curn * 4 + LEXM_MAGIC_65536
2321 let bufN: *u8 = sys_mmap(cap2)
2322 let rn: i64 = lexm_expand(cur, curn, bufN, cap2)
2323 if rn <= 0 { more = 0 } else {
2324 cur = bufN
2325 curn = rn
2326 expanded = 1
2327 iter = iter + 1
2328 }
2329 }
2330 }
2331
2332 // The pre-pass above defined every macro in the file (lexm_prescan must scan the whole buffer). Drop
2333 // them so the sequential directive handler repopulates in SOURCE ORDER -- otherwise `@ifdef FOO`
2334 // would see a `@macro FOO` written 200 lines LATER as already defined, and `@undef` could never
2335 // retract anything. Textual arg expansion has already happened, so nothing is lost by clearing.
2336 lexm_reset()
2337 // Branch state is PER-FILE: a chain left open by one file must not leak its already-taken flag
2338 // into the next.
2339 ppb_reset()
2340
2341 let L_raw: *u8 = sys_mmap(64)
2342 let L: *Lex = L_raw as *Lex
2343 L.src = src
2344 if expanded == 1 { L.src = cur }
2345 L.pos = 0
2346 L.line = 1
2347 L.col = 1
2348 L.tokens = toks
2349 L.n_tokens = 0
2350 L.cap = cap
2351
2352 var go: i64 = 1
2353 while go {
2354 skip_ws_comments(L)
2355 let c: i64 = peek(L)
2356 if c == 0 {
2357 go = 0
2358 continue
2359 }
2360 // Preprocessor directives: @ifdef / @ifndef / @endif. Must
2361 // run BEFORE the @ punct emission so directive lines are
2362 // consumed at lex time and don't reach the parser.
2363 if c == 0x40 {
2364 // `@priv` VISIBILITY ATTRIBUTE -> a REAL token (TK_PRIV), not a silently skipped
2365 // punct+ident pair. Emitting it means the parser ACKNOWLEDGES the marker instead of
2366 // merely tolerating it, which is the difference between syntax and a comment
2367 // convention. The trailing boundary check is what stops `@privv` (or `@priv_x`)
2368 // from matching the first four characters and silently authorising nothing.
2369 // PORTED FROM THE LAPTOP SSOT 2026-07-31: this feature was SSOT-only, and building the
2370 // compiler from buildroot would have silently dropped it (the 731-byte tokenizer delta).
2371 if _lex_src_eq_at(L, L.pos + 1, "priv" as *u8, 4) == 1 {
2372 let nb: i64 = L.src[L.pos + 5] as i64
2373 if is_alnum(nb) == 0 {
2374 if nb != 0x5F { emit_punct(L, TK_PRIV, 5); continue }
2375 }
2376 }
2377 if _lex_handle_preprocessor(L) == 1 { continue }
2378 }
2379 if is_alpha(c) { lex_ident_or_kw(L); continue }
2380 if c == 0x5F { lex_ident_or_kw(L); continue }
2381 if is_digit(c) { lex_number(L); continue }
2382 if c == 0x22 { lex_string(L); continue }
2383
2384 let c2: i64 = peek2(L)
2385 // Multi-char first
2386 if c == 0x2D {
2387 if c2 == 0x3E { emit_punct(L, 61, 2); continue } // ->
2388 }
2389 if c == 0x3D {
2390 if c2 == 0x3D { emit_punct(L, 46, 2); continue } // ==
2391 }
2392 if c == 0x21 {
2393 if c2 == 0x3D { emit_punct(L, 47, 2); continue } // !=
2394 }
2395 if c == 0x3C {
2396 if c2 == 0x3D { emit_punct(L, 50, 2); continue } // <=
2397 if c2 == 0x3C { emit_punct(L, 59, 2); continue } // <<
2398 }
2399 if c == 0x3E {
2400 if c2 == 0x3D { emit_punct(L, 51, 2); continue } // >=
2401 if c2 == 0x3E { emit_punct(L, 60, 2); continue } // >>
2402 }
2403 if c == 0x26 {
2404 if c2 == 0x26 { emit_punct(L, 52, 2); continue } // &&
2405 }
2406 if c == 0x7C {
2407 if c2 == 0x7C { emit_punct(L, 53, 2); continue } // ||
2408 }
2409 if c == 0x2E {
2410 if c2 == 0x2E { emit_punct(L, 62, 2); continue } // ..
2411 }
2412 if c == 0x3A {
2413 if c2 == 0x3A { emit_punct(L, 75, 2); continue } // ::
2414 }
2415 if c == 0x3D {
2416 if c2 == 0x3E { emit_punct(L, 76, 2); continue } // =>
2417 }
2418
2419 if c == 0x2B { emit_punct(L, 40, 1); continue }
2420 if c == 0x2D { emit_punct(L, 41, 1); continue }
2421 if c == 0x2A { emit_punct(L, 42, 1); continue }
2422 if c == 0x2F { emit_punct(L, 43, 1); continue }
2423 if c == 0x25 { emit_punct(L, 44, 1); continue }
2424 if c == 0x3D { emit_punct(L, 45, 1); continue }
2425 if c == 0x3C { emit_punct(L, 48, 1); continue }
2426 if c == 0x3E { emit_punct(L, 49, 1); continue }
2427 if c == 0x21 { emit_punct(L, 54, 1); continue }
2428 if c == 0x26 { emit_punct(L, 55, 1); continue }
2429 if c == 0x7C { emit_punct(L, 56, 1); continue }
2430 if c == 0x5E { emit_punct(L, 57, 1); continue }
2431 if c == 0x7E { emit_punct(L, 58, 1); continue }
2432 if c == 0x2E { emit_punct(L, 63, 1); continue }
2433 if c == 0x3A { emit_punct(L, 64, 1); continue }
2434 if c == 0x3B { emit_punct(L, 65, 1); continue }
2435 if c == 0x2C { emit_punct(L, 66, 1); continue }
2436 if c == 0x28 { emit_punct(L, 67, 1); continue }
2437 if c == 0x29 { emit_punct(L, 68, 1); continue }
2438 if c == 0x7B { emit_punct(L, 69, 1); continue }
2439 if c == 0x7D { emit_punct(L, 70, 1); continue }
2440 if c == 0x5B { emit_punct(L, 71, 1); continue }
2441 if c == 0x5D { emit_punct(L, 72, 1); continue }
2442 if c == 0x40 { emit_punct(L, 73, 1); continue }
2443 if c == 0x23 { emit_punct(L, 74, 1); continue }
2444 if c == 0x3F { emit_punct(L, 77, 1); continue } // ? TK_QUESTION (ternary)
2445
2446 // Unknown: skip to prevent infinite loop.
2447 advance(L)
2448 }
2449
2450 // Terminator EOF
2451 let last_raw: *u8 = sys_mmap(TOK_BYTES)
2452 let last: *Tok = last_raw as *Tok
2453 last.kind = 0
2454 last.line = L.line
2455 last.col = L.col
2456 last.int_val = 0
2457 last.text0 = 0; last.text1 = 0; last.text2 = 0; last.text3 = 0
2458 last.text4 = 0; last.text5 = 0; last.text6 = 0; last.text7 = 0
2459 push_tok(L, last)
2460
2461 return toks
2462}
2463
2464// Library only; self-test lives in lex_test.nx.