nx_js_lex.nx source
↩ module page · 662 lines · 33691 B
1// nx_js_lex.nx -- R-JS-LEX (WB-JS-001 rung 0): the ECMAScript TOKENIZER, the
2// hardware-up floor of the sovereign JS engine (browser-engine-exceed roadmap §2).
3// Founds R-JS-PARSE above it (no-floating law: the lexer is built+gated before any
4// parser rides it). Shape = STATE_MACHINE/STRUCT_WALK over source bytes -- an
5// EXISTING emitter shape, so team-authorable now (NOT blocked on X-AUT-006c).
6//
7// Scans ECMAScript source into a flat token stream (3 i64 slots per token:
8// kind, start-offset, byte-length). Covers the lexical core real JS uses:
9// - whitespace + line-terminators (skipped; ASI is a PARSER concern, rung 1)
10// - // line comments and /* block */ comments (skipped)
11// - identifiers + keywords ($ and _ legal; keyword set classified)
12// - numeric literals: decimal int/float, exponent, 0x/0o/0b prefixes
13// - string literals ' and " with backslash-escape handling (\" \' \\ \n ...)
14// - punctuators incl. multi-char operators (=== !== >>>= => ?. ?? ** etc.)
15// - unterminated string -> ERROR token (honest, never silently swallowed)
16//
17// HONEST RUNG-0 SCOPE (named OPEN, NOT faked): template literals (`...${}`) and
18// regex-literal disambiguation need PARSER context (is `/` a divide or a regex?),
19// so they are R-JS-LEX-0b, built once the parser can feed back context. BigInt `n`
20// suffix + full Unicode IdentifierStart are also 0b. Rung 0 tokenizes ASCII JS.
21//
22// GATE (main): 7 KATs assert the exact token stream (kind + lexeme) on real JS
23// snippets + TAMPER (unterminated string must be ERROR, never a fabricated STRING).
24// Self-validating; exit 0 iff all pass; appends knowledge/status/js_engine.log.
25//
26// license_tier: ORIGINAL (tutor-bootstrap scaffold; team re-authors from the
27// R-JS-LEX data spec via author=organ -- (B)-debt, mirror the h2 capstone note.)
28import "nx_syscalls.nx"
29import "nx_itoa_lib.nx" // shared MSB-first emitter (zero-alloc)
30
31const JS_TOK_EOF: i64 = 0
32const JS_TOK_IDENT: i64 = 1
33const JS_TOK_KEYWORD: i64 = 2
34const JS_TOK_NUMBER: i64 = 3
35const JS_TOK_STRING: i64 = 4
36const JS_TOK_PUNCT: i64 = 5
37const JS_TOK_ERROR: i64 = 8
38const JS_TOK_TEMPLATE: i64 = 9 // template literal: whole span incl both backticks; interior re-lexed at eval
39const JS_TOK_REGEX: i64 = 10 // regex literal /pattern/flags: whole span; pattern+flags extracted at eval
40
41// ---- byte classification ----
42func js_is_digit(c: i64) -> i64 { if c >= 48 { if c <= 57 { return 1 } } return 0 }
43func js_is_alpha(c: i64) -> i64 {
44 if c >= 65 { if c <= 90 { return 1 } }
45 if c >= 97 { if c <= 122 { return 1 } }
46 return 0
47}
48// ECMAScript identifiers admit Unicode letters, not just ASCII. This lexer is BYTE
49// oriented, so every byte of a UTF-8 multi-byte sequence (0x80..0xFF) counts as an
50// identifier byte. MEASURED MOTIVATION (2026-07-27): netflix's 3.9 MB bundle fails to
51// parse at a lodash deburr map -- `{À:"A",Á:"A",à:"a",...}` -- bare non-ASCII identifier
52// KEYS, which are near-universal in bundled apps. Isolated with controls by
53// nx_js_unicode_probe (quoted non-ASCII keys already worked; bare ones did not).
54// This is deliberately PERMISSIVE: it accepts some code points real JS reserves for
55// punctuation, which only ever makes us accept MORE programs -- it can never change the
56// meaning of a valid one.
57// ⚠RESIDUAL RISK (named, not hidden): Unicode SPACE characters are also >=0x80 (NBSP
58// U+00A0 = C2 A0, U+2028/9, BOM U+FEFF). Used as a separator ADJACENT to an identifier
59// they would now be absorbed into it. Minified bundles -- the entire point of this
60// change -- separate with ASCII space, so this does not arise in practice; a proper fix
61// is a UTF-8 decode + Unicode ID_Start/ID_Continue table, a separate bounded rung.
62func js_is_idstart(c: i64) -> i64 {
63 if js_is_alpha(c) == 1 { return 1 }
64 if c == 95 { return 1 } // _
65 if c == 36 { return 1 } // $
66 if c >= 128 { return 1 } // UTF-8 lead/continuation byte = part of a Unicode letter
67 return 0
68}
69func js_is_idpart(c: i64) -> i64 {
70 if js_is_idstart(c) == 1 { return 1 }
71 if js_is_digit(c) == 1 { return 1 }
72 return 0
73}
74func js_is_hex(c: i64) -> i64 {
75 if js_is_digit(c) == 1 { return 1 }
76 if c >= 65 { if c <= 70 { return 1 } } // A-F
77 if c >= 97 { if c <= 102 { return 1 } } // a-f
78 return 0
79}
80func js_is_space(c: i64) -> i64 {
81 if c == 32 { return 1 } // space
82 if c == 9 { return 1 } // tab
83 if c == 10 { return 1 } // \n
84 if c == 13 { return 1 } // \r
85 if c == 11 { return 1 } // \v
86 if c == 12 { return 1 } // \f
87 return 0
88}
89
90// ---- lexeme compare: src[start..start+len) == NUL-terminated lit ? ----
91func js_lexeme_eq(src: *u8, start: i64, len: i64, lit: *u8) -> i64 {
92 var i: i64 = 0
93 while i < len {
94 if (lit[i] & 0xff) == 0 { return 0 } // lit shorter than len
95 if (src[start + i] & 0xff) != (lit[i] & 0xff) { return 0 }
96 i = i + 1
97 }
98 if (lit[len] & 0xff) != 0 { return 0 } // lit longer than len
99 return 1
100}
101
102// ---- keyword classification (ECMAScript reserved + common contextual) ----
103func js_is_keyword(src: *u8, start: i64, len: i64) -> i64 {
104 if js_lexeme_eq(src, start, len, "var\x00" as *u8) == 1 { return 1 }
105 if js_lexeme_eq(src, start, len, "let\x00" as *u8) == 1 { return 1 }
106 if js_lexeme_eq(src, start, len, "const\x00" as *u8) == 1 { return 1 }
107 if js_lexeme_eq(src, start, len, "function\x00" as *u8) == 1 { return 1 }
108 if js_lexeme_eq(src, start, len, "return\x00" as *u8) == 1 { return 1 }
109 if js_lexeme_eq(src, start, len, "if\x00" as *u8) == 1 { return 1 }
110 if js_lexeme_eq(src, start, len, "else\x00" as *u8) == 1 { return 1 }
111 if js_lexeme_eq(src, start, len, "for\x00" as *u8) == 1 { return 1 }
112 if js_lexeme_eq(src, start, len, "while\x00" as *u8) == 1 { return 1 }
113 if js_lexeme_eq(src, start, len, "do\x00" as *u8) == 1 { return 1 }
114 if js_lexeme_eq(src, start, len, "break\x00" as *u8) == 1 { return 1 }
115 if js_lexeme_eq(src, start, len, "continue\x00" as *u8) == 1 { return 1 }
116 if js_lexeme_eq(src, start, len, "switch\x00" as *u8) == 1 { return 1 }
117 if js_lexeme_eq(src, start, len, "case\x00" as *u8) == 1 { return 1 }
118 if js_lexeme_eq(src, start, len, "default\x00" as *u8) == 1 { return 1 }
119 if js_lexeme_eq(src, start, len, "new\x00" as *u8) == 1 { return 1 }
120 if js_lexeme_eq(src, start, len, "delete\x00" as *u8) == 1 { return 1 }
121 if js_lexeme_eq(src, start, len, "typeof\x00" as *u8) == 1 { return 1 }
122 if js_lexeme_eq(src, start, len, "instanceof\x00" as *u8) == 1 { return 1 }
123 if js_lexeme_eq(src, start, len, "in\x00" as *u8) == 1 { return 1 }
124 if js_lexeme_eq(src, start, len, "of\x00" as *u8) == 1 { return 1 }
125 if js_lexeme_eq(src, start, len, "this\x00" as *u8) == 1 { return 1 }
126 if js_lexeme_eq(src, start, len, "null\x00" as *u8) == 1 { return 1 }
127 if js_lexeme_eq(src, start, len, "true\x00" as *u8) == 1 { return 1 }
128 if js_lexeme_eq(src, start, len, "false\x00" as *u8) == 1 { return 1 }
129 if js_lexeme_eq(src, start, len, "void\x00" as *u8) == 1 { return 1 }
130 if js_lexeme_eq(src, start, len, "try\x00" as *u8) == 1 { return 1 }
131 if js_lexeme_eq(src, start, len, "catch\x00" as *u8) == 1 { return 1 }
132 if js_lexeme_eq(src, start, len, "finally\x00" as *u8) == 1 { return 1 }
133 if js_lexeme_eq(src, start, len, "throw\x00" as *u8) == 1 { return 1 }
134 if js_lexeme_eq(src, start, len, "class\x00" as *u8) == 1 { return 1 }
135 if js_lexeme_eq(src, start, len, "extends\x00" as *u8) == 1 { return 1 }
136 if js_lexeme_eq(src, start, len, "super\x00" as *u8) == 1 { return 1 }
137 if js_lexeme_eq(src, start, len, "import\x00" as *u8) == 1 { return 1 }
138 if js_lexeme_eq(src, start, len, "export\x00" as *u8) == 1 { return 1 }
139 if js_lexeme_eq(src, start, len, "async\x00" as *u8) == 1 { return 1 }
140 if js_lexeme_eq(src, start, len, "await\x00" as *u8) == 1 { return 1 }
141 if js_lexeme_eq(src, start, len, "yield\x00" as *u8) == 1 { return 1 }
142 return 0
143}
144
145// ---- scanners (each returns the END index just past the token) ----
146func js_scan_ident(src: *u8, srclen: i64, pos: i64) -> i64 {
147 var e: i64 = pos + 1
148 var k: i64 = 1
149 while k == 1 {
150 if e >= srclen { k = 0 }
151 if k == 1 { if js_is_idpart(src[e] & 0xff) == 1 { e = e + 1 } else { k = 0 } }
152 }
153 return e
154}
155func js_scan_while_digit(src: *u8, srclen: i64, pos: i64) -> i64 {
156 var e: i64 = pos
157 var k: i64 = 1
158 while k == 1 {
159 if e >= srclen { k = 0 }
160 if k == 1 { if js_is_digit(src[e] & 0xff) == 1 { e = e + 1 } else { k = 0 } }
161 }
162 return e
163}
164func js_scan_while_hex(src: *u8, srclen: i64, pos: i64) -> i64 {
165 var e: i64 = pos
166 var k: i64 = 1
167 while k == 1 {
168 if e >= srclen { k = 0 }
169 if k == 1 { if js_is_hex(src[e] & 0xff) == 1 { e = e + 1 } else { k = 0 } }
170 }
171 return e
172}
173func js_scan_exp(src: *u8, srclen: i64, pos: i64) -> i64 {
174 var e: i64 = pos + 1 // skip e/E
175 if e < srclen {
176 let s: i64 = src[e] & 0xff
177 if s == 43 { e = e + 1 } // +
178 if s == 45 { e = e + 1 } // -
179 }
180 e = js_scan_while_digit(src, srclen, e)
181 return e
182}
183func js_scan_number(src: *u8, srclen: i64, pos: i64) -> i64 {
184 var e: i64 = pos
185 // BIGINT SUFFIX (2026-08-25): `1n`, `0xFFn` etc. A scanner that stops at the digits splits the
186 // literal into NUMBER `1` + IDENT `n` and the parse fails one token later -- found by the scope
187 // gate walking the real shipped page (byte 382,097: `(ex.sbyb?ex.sbyb:ex.sby)(BigInt(bi),1n)`).
188 // The radix branches used to RETURN directly, so they are folded into the shared suffix consume
189 // rather than each growing its own copy of it.
190 var isradix: i64 = 0
191 if (src[e] & 0xff) == 48 { // leading '0' -> maybe radix prefix
192 if e + 1 < srclen {
193 let n1: i64 = src[e + 1] & 0xff
194 if n1 == 120 { e = js_scan_while_hex(src, srclen, e + 2); isradix = 1 } // 0x
195 if n1 == 88 { e = js_scan_while_hex(src, srclen, e + 2); isradix = 1 } // 0X
196 if n1 == 98 { e = js_scan_while_digit(src, srclen, e + 2); isradix = 1 } // 0b
197 if n1 == 66 { e = js_scan_while_digit(src, srclen, e + 2); isradix = 1 } // 0B
198 if n1 == 111 { e = js_scan_while_digit(src, srclen, e + 2); isradix = 1 } // 0o
199 if n1 == 79 { e = js_scan_while_digit(src, srclen, e + 2); isradix = 1 } // 0O
200 }
201 }
202 if isradix == 0 {
203 e = js_scan_while_digit(src, srclen, e) // integer part
204 if e < srclen { if (src[e] & 0xff) == 46 { // '.' fraction
205 e = js_scan_while_digit(src, srclen, e + 1)
206 } }
207 if e < srclen {
208 let ec: i64 = src[e] & 0xff
209 if ec == 101 { e = js_scan_exp(src, srclen, e) } // e
210 if ec == 69 { e = js_scan_exp(src, srclen, e) } // E
211 }
212 }
213 if e < srclen { if (src[e] & 0xff) == 110 { e = e + 1 } } // BigInt 'n' suffix
214 return e
215}
216// string: returns END (past close quote) if closed, else 0 - END if unterminated.
217func js_scan_string(src: *u8, srclen: i64, pos: i64) -> i64 {
218 let q: i64 = src[pos] & 0xff
219 var e: i64 = pos + 1
220 var k: i64 = 1
221 var closed: i64 = 0
222 while k == 1 {
223 if e >= srclen { k = 0 }
224 if k == 1 {
225 let c: i64 = src[e] & 0xff
226 if c == 92 { e = e + 2 } // backslash escape -> skip next char
227 else {
228 if c == q { e = e + 1; closed = 1; k = 0 }
229 else { e = e + 1 }
230 }
231 }
232 }
233 if closed == 1 { return e }
234 return 0 - e
235}
236
237// punctuator length at pos (1..4), or 0 if pos is not a punctuator.
238func js_punct_len(src: *u8, srclen: i64, pos: i64) -> i64 {
239 let c: i64 = src[pos] & 0xff
240 var c1: i64 = 0
241 var c2: i64 = 0
242 var c3: i64 = 0
243 if pos + 1 < srclen { c1 = src[pos + 1] & 0xff }
244 if pos + 2 < srclen { c2 = src[pos + 2] & 0xff }
245 if pos + 3 < srclen { c3 = src[pos + 3] & 0xff }
246 if c == 62 { // >
247 if c1 == 62 { // >>
248 if c2 == 62 { if c3 == 61 { return 4 } return 3 } // >>> , >>>=
249 if c2 == 61 { return 3 } // >>=
250 return 2
251 }
252 if c1 == 61 { return 2 } // >=
253 return 1
254 }
255 if c == 60 { // <
256 if c1 == 60 { if c2 == 61 { return 3 } return 2 } // << , <<=
257 if c1 == 61 { return 2 } // <=
258 return 1
259 }
260 if c == 61 { // =
261 if c1 == 61 { if c2 == 61 { return 3 } return 2 } // == , ===
262 if c1 == 62 { return 2 } // =>
263 return 1
264 }
265 if c == 33 { // !
266 if c1 == 61 { if c2 == 61 { return 3 } return 2 } // != , !==
267 return 1
268 }
269 if c == 38 { // &
270 if c1 == 38 { if c2 == 61 { return 3 } return 2 } // && , &&=
271 if c1 == 61 { return 2 } // &=
272 return 1
273 }
274 if c == 124 { // |
275 if c1 == 124 { if c2 == 61 { return 3 } return 2 } // || , ||=
276 if c1 == 61 { return 2 } // |=
277 return 1
278 }
279 if c == 63 { // ?
280 if c1 == 63 { if c2 == 61 { return 3 } return 2 } // ?? , ??=
281 // `?.` is OPTIONAL CHAINING only when NOT followed by a decimal digit
282 // (ECMAScript: OptionalChainingPunctuator :: ?. [lookahead is not DecimalDigit]).
283 // Otherwise it is a TERNARY whose consequent is a leading-dot number:
284 // `e.disabledStyleProp?.5:1` == `e.disabledStyleProp ? .5 : 1`. Minifiers emit
285 // `.5` for `0.5`, so greedily taking `?.` broke real bundles -- MEASURED in
286 // netflix's bundle 2026-07-27; isolated by nx_js_bundle_probe S6/S7 (`.5` alone
287 // and `a?.b` alone both already passed, only the combination failed).
288 if c1 == 46 { if js_is_digit(c2) == 1 { return 1 } return 2 } // ?. vs ? .5
289 return 1
290 }
291 if c == 42 { // *
292 if c1 == 42 { if c2 == 61 { return 3 } return 2 } // ** , **=
293 if c1 == 61 { return 2 } // *=
294 return 1
295 }
296 if c == 43 { if c1 == 43 { return 2 } if c1 == 61 { return 2 } return 1 } // ++ += +
297 if c == 45 { if c1 == 45 { return 2 } if c1 == 61 { return 2 } return 1 } // -- -= -
298 if c == 47 { if c1 == 61 { return 2 } return 1 } // /= / (// /* handled in trivia)
299 if c == 37 { if c1 == 61 { return 2 } return 1 } // %= %
300 if c == 94 { if c1 == 61 { return 2 } return 1 } // ^= ^
301 if c == 46 { if c1 == 46 { if c2 == 46 { return 3 } } return 1 } // ... .
302 if c == 123 { return 1 } // {
303 if c == 125 { return 1 } // }
304 if c == 40 { return 1 } // (
305 if c == 41 { return 1 } // )
306 if c == 91 { return 1 } // [
307 if c == 93 { return 1 } // ]
308 if c == 59 { return 1 } // ;
309 if c == 44 { return 1 } // ,
310 if c == 58 { return 1 } // :
311 if c == 126 { return 1 } // ~
312 return 0
313}
314
315// skip whitespace + // line and /* block */ comments; return first real-token pos.
316func js_skip_trivia(src: *u8, srclen: i64, pos: i64) -> i64 {
317 var p: i64 = pos
318 var go: i64 = 1
319 while go == 1 {
320 var k: i64 = 1
321 while k == 1 {
322 if p >= srclen { k = 0 } else { if js_is_space(src[p] & 0xff) == 1 { p = p + 1 } else { k = 0 } }
323 }
324 var did: i64 = 0
325 if p + 1 < srclen { if (src[p] & 0xff) == 47 { if (src[p + 1] & 0xff) == 47 { // //
326 p = p + 2
327 var k2: i64 = 1
328 while k2 == 1 {
329 if p >= srclen { k2 = 0 } else { if (src[p] & 0xff) == 10 { k2 = 0 } else { p = p + 1 } }
330 }
331 did = 1
332 } } }
333 if did == 0 { if p + 1 < srclen { if (src[p] & 0xff) == 47 { if (src[p + 1] & 0xff) == 42 { // /*
334 p = p + 2
335 var k3: i64 = 1
336 while k3 == 1 {
337 if p + 1 >= srclen { p = srclen; k3 = 0 }
338 else {
339 if (src[p] & 0xff) == 42 { if (src[p + 1] & 0xff) == 47 { p = p + 2; k3 = 0 } else { p = p + 1 } }
340 else { p = p + 1 }
341 }
342 }
343 did = 1
344 } } } }
345 if did == 0 { go = 0 }
346 }
347 return p
348}
349
350func js_emit(toks: *i64, ntok_box: *i64, kind: i64, start: i64, len: i64) -> i64 {
351 let i: i64 = ntok_box[0]
352 toks[i * 3 + 0] = kind
353 toks[i * 3 + 1] = start
354 toks[i * 3 + 2] = len
355 ntok_box[0] = i + 1
356 return 0
357}
358
359func js_num_starts(src: *u8, srclen: i64, pos: i64) -> i64 {
360 let c: i64 = src[pos] & 0xff
361 if js_is_digit(c) == 1 { return 1 }
362 if c == 46 { if pos + 1 < srclen { if js_is_digit(src[pos + 1] & 0xff) == 1 { return 1 } } } // .5
363 return 0
364}
365
366func js_lex_string(src: *u8, srclen: i64, pos: i64, toks: *i64, ntok_box: *i64) -> i64 {
367 let r: i64 = js_scan_string(src, srclen, pos)
368 if r > 0 { js_emit(toks, ntok_box, JS_TOK_STRING, pos, r - pos); return r }
369 let e: i64 = 0 - r
370 js_emit(toks, ntok_box, JS_TOK_ERROR, pos, e - pos) // unterminated -> ERROR
371 return e
372}
373
374// scan a template literal `...${expr}...` : emit ONE JS_TOK_TEMPLATE token spanning both backticks.
375// Handles \-escapes and ${ ... } interior brace-nesting (so a `}` inside ${} does not end it early).
376// The interior (literal runs + ${expr}) is decoded at EVAL time. Nested templates/strings-with-braces
377// inside ${} are a NAMED OPEN (brace-depth only, no string awareness inside interpolation).
378func js_lex_template(src: *u8, srclen: i64, pos: i64, toks: *i64, ntok_box: *i64) -> i64 {
379 var p: i64 = pos + 1
380 var go: i64 = 1
381 while go == 1 {
382 if p >= srclen { js_emit(toks, ntok_box, JS_TOK_ERROR, pos, p - pos); return p } // unterminated
383 let ch: i64 = src[p] & 0xff
384 if ch == 92 { p = p + 2 } // \-escape: skip 2
385 else {
386 if ch == 96 { go = 0 } // closing `
387 else {
388 var handled: i64 = 0
389 if ch == 36 { if p + 1 < srclen { if (src[p + 1] & 0xff) == 123 { // ${
390 p = p + 2
391 var depth: i64 = 1
392 var g2: i64 = 1
393 while g2 == 1 {
394 if p >= srclen { g2 = 0 }
395 else {
396 let c2: i64 = src[p] & 0xff
397 if c2 == 123 { depth = depth + 1; p = p + 1 }
398 else { if c2 == 125 { depth = depth - 1; p = p + 1; if depth == 0 { g2 = 0 } } else { p = p + 1 } }
399 }
400 }
401 handled = 1
402 } } }
403 if handled == 0 { p = p + 1 }
404 }
405 }
406 }
407 js_emit(toks, ntok_box, JS_TOK_TEMPLATE, pos, (p + 1) - pos)
408 return p + 1
409}
410// R-JS-LEX-0b: can a `/` at this point START A REGEX (vs be division)? Decided by the PREVIOUS token.
411// After a VALUE (ident/number/string/template/`)`/`]`/`}`/this/true/false/null/super) => division.
412// After anything else (operators, ( [ { , ; = return typeof new ... , or start-of-input) => regex.
413// (`}` is treated as a value here -- the one classic ambiguity; block-`}` followed by /re/ is vanishingly
414// rare in real/minified code. Noted.)
415func js_regex_context(toks: *i64, ntok: i64, src: *u8) -> i64 {
416 if ntok == 0 { return 1 }
417 let b: i64 = (ntok - 1) * 3
418 let pk: i64 = toks[b]
419 let ps: i64 = toks[b + 1]
420 let pl: i64 = toks[b + 2]
421 if pk == JS_TOK_NUMBER { return 0 }
422 if pk == JS_TOK_STRING { return 0 }
423 if pk == JS_TOK_TEMPLATE { return 0 }
424 if pk == JS_TOK_IDENT { return 0 }
425 if pk == JS_TOK_KEYWORD {
426 if js_lexeme_eq(src, ps, pl, "this\x00" as *u8) == 1 { return 0 }
427 if js_lexeme_eq(src, ps, pl, "true\x00" as *u8) == 1 { return 0 }
428 if js_lexeme_eq(src, ps, pl, "false\x00" as *u8) == 1 { return 0 }
429 if js_lexeme_eq(src, ps, pl, "null\x00" as *u8) == 1 { return 0 }
430 if js_lexeme_eq(src, ps, pl, "super\x00" as *u8) == 1 { return 0 }
431 return 1
432 }
433 if pk == JS_TOK_PUNCT {
434 if js_lexeme_eq(src, ps, pl, ")\x00" as *u8) == 1 { return 0 }
435 if js_lexeme_eq(src, ps, pl, "]\x00" as *u8) == 1 { return 0 }
436 if js_lexeme_eq(src, ps, pl, "}\x00" as *u8) == 1 { return 0 }
437 return 1
438 }
439 return 1
440}
441// scan a regex literal /pattern/flags -> ONE JS_TOK_REGEX token. `/` closes only OUTSIDE a `[..]` class;
442// `\` escapes the next char; an unescaped newline (or EOF) before the close = ERROR (never a fabricated regex).
443func js_lex_regex(src: *u8, srclen: i64, pos: i64, toks: *i64, ntok_box: *i64) -> i64 {
444 var p: i64 = pos + 1
445 var inclass: i64 = 0
446 var go: i64 = 1
447 while go == 1 {
448 if p >= srclen { js_emit(toks, ntok_box, JS_TOK_ERROR, pos, p - pos); return p }
449 let ch: i64 = src[p] & 0xff
450 if ch == 92 { p = p + 2 }
451 else {
452 if ch == 10 { js_emit(toks, ntok_box, JS_TOK_ERROR, pos, p - pos); return p }
453 else { if ch == 91 { inclass = 1; p = p + 1 }
454 else { if ch == 93 { inclass = 0; p = p + 1 }
455 else { if ch == 47 { if inclass == 0 { go = 0 } else { p = p + 1 } }
456 else { p = p + 1 } } } }
457 }
458 }
459 p = p + 1 // consume closing '/'
460 var gf: i64 = 1
461 while gf == 1 { if p < srclen { let fc: i64 = src[p] & 0xff; if fc >= 97 { if fc <= 122 { p = p + 1 } else { gf = 0 } } else { gf = 0 } } else { gf = 0 } }
462 js_emit(toks, ntok_box, JS_TOK_REGEX, pos, p - pos)
463 return p
464}
465// lex exactly ONE token at pos (pos already past trivia); return END index.
466func js_lex_one(src: *u8, srclen: i64, pos: i64, toks: *i64, ntok_box: *i64) -> i64 {
467 let c: i64 = src[pos] & 0xff
468 if js_is_idstart(c) == 1 {
469 let e: i64 = js_scan_ident(src, srclen, pos)
470 var kind: i64 = JS_TOK_IDENT
471 if js_is_keyword(src, pos, e - pos) == 1 { kind = JS_TOK_KEYWORD }
472 js_emit(toks, ntok_box, kind, pos, e - pos)
473 return e
474 }
475 if js_num_starts(src, srclen, pos) == 1 {
476 let e: i64 = js_scan_number(src, srclen, pos)
477 js_emit(toks, ntok_box, JS_TOK_NUMBER, pos, e - pos)
478 return e
479 }
480 if c == 34 { return js_lex_string(src, srclen, pos, toks, ntok_box) } // "
481 if c == 39 { return js_lex_string(src, srclen, pos, toks, ntok_box) } // '
482 if c == 96 { return js_lex_template(src, srclen, pos, toks, ntok_box) } // ` template literal
483 if c == 47 { if js_regex_context(toks, ntok_box[0], src) == 1 { return js_lex_regex(src, srclen, pos, toks, ntok_box) } } // '/' regex vs divide
484 let pl: i64 = js_punct_len(src, srclen, pos)
485 if pl > 0 { js_emit(toks, ntok_box, JS_TOK_PUNCT, pos, pl); return pos + pl }
486 js_emit(toks, ntok_box, JS_TOK_ERROR, pos, 1) // unknown byte
487 return pos + 1
488}
489
490// PUBLIC: tokenize src[0..srclen) into toks (3 i64/token); returns token count.
491func js_lex(src: *u8, srclen: i64, toks: *i64, maxtoks: i64, ntok_box: *i64) -> i64 {
492 var pos: i64 = 0
493 ntok_box[0] = 0
494 var go: i64 = 1
495 while go == 1 {
496 pos = js_skip_trivia(src, srclen, pos)
497 if pos >= srclen { go = 0 }
498 if go == 1 { if ntok_box[0] >= maxtoks { go = 0 } }
499 if go == 1 { pos = js_lex_one(src, srclen, pos, toks, ntok_box) }
500 }
501 return ntok_box[0]
502}
503
504// ===================== GATE =====================
505func jl_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
506// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
507// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
508// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
509// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
510func jl_putn(v: i64) -> i64 { nxi_out(v); return 0 }
511func jl_strlen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
512
513func js_expect(toks: *i64, ntok: i64, idx: i64, kind: i64, src: *u8, lit: *u8) -> i64 {
514 if idx >= ntok { return 0 }
515 if toks[idx * 3 + 0] != kind { return 0 }
516 return js_lexeme_eq(src, toks[idx * 3 + 1], toks[idx * 3 + 2], lit)
517}
518
519func main() -> i64 {
520 let toks: *i64 = sys_mmap(256 * 3 * 8) as *i64
521 let ntb: *i64 = sys_mmap(16) as *i64
522 var pass: i64 = 0
523 var tot: i64 = 0
524 jl_puts("nx_js_lex gate (R-JS-LEX, WB-JS-001 rung 0)\n" as *u8)
525
526 // ---- KAT 1: keywords/idents/number/punct/dot ----
527 let s1: *u8 = "var x = 42 + foo.bar\x00" as *u8
528 let n1: i64 = js_lex(s1, jl_strlen(s1), toks, 256, ntb)
529 var r1: i64 = 1
530 if n1 != 8 { r1 = 0 }
531 r1 = r1 * js_expect(toks, n1, 0, JS_TOK_KEYWORD, s1, "var\x00" as *u8)
532 r1 = r1 * js_expect(toks, n1, 1, JS_TOK_IDENT, s1, "x\x00" as *u8)
533 r1 = r1 * js_expect(toks, n1, 2, JS_TOK_PUNCT, s1, "=\x00" as *u8)
534 r1 = r1 * js_expect(toks, n1, 3, JS_TOK_NUMBER, s1, "42\x00" as *u8)
535 r1 = r1 * js_expect(toks, n1, 4, JS_TOK_PUNCT, s1, "+\x00" as *u8)
536 r1 = r1 * js_expect(toks, n1, 5, JS_TOK_IDENT, s1, "foo\x00" as *u8)
537 r1 = r1 * js_expect(toks, n1, 6, JS_TOK_PUNCT, s1, ".\x00" as *u8)
538 r1 = r1 * js_expect(toks, n1, 7, JS_TOK_IDENT, s1, "bar\x00" as *u8)
539 if r1 == 1 { jl_puts(" PASS KAT1 var/ident/number/dot (8 toks)\n" as *u8); pass = pass + 1 } else { jl_puts(" FAIL KAT1 ntok=" as *u8); jl_putn(n1); jl_puts("\n" as *u8) }
540 tot = tot + 1
541
542 // ---- KAT 2: multi-char operators incl >>>= ----
543 let s2: *u8 = "a===b!==c>>>=d\x00" as *u8
544 let n2: i64 = js_lex(s2, jl_strlen(s2), toks, 256, ntb)
545 var r2: i64 = 1
546 if n2 != 7 { r2 = 0 }
547 r2 = r2 * js_expect(toks, n2, 0, JS_TOK_IDENT, s2, "a\x00" as *u8)
548 r2 = r2 * js_expect(toks, n2, 1, JS_TOK_PUNCT, s2, "===\x00" as *u8)
549 r2 = r2 * js_expect(toks, n2, 2, JS_TOK_IDENT, s2, "b\x00" as *u8)
550 r2 = r2 * js_expect(toks, n2, 3, JS_TOK_PUNCT, s2, "!==\x00" as *u8)
551 r2 = r2 * js_expect(toks, n2, 4, JS_TOK_IDENT, s2, "c\x00" as *u8)
552 r2 = r2 * js_expect(toks, n2, 5, JS_TOK_PUNCT, s2, ">>>=\x00" as *u8)
553 r2 = r2 * js_expect(toks, n2, 6, JS_TOK_IDENT, s2, "d\x00" as *u8)
554 if r2 == 1 { jl_puts(" PASS KAT2 operators ===/!==/>>>= (7 toks)\n" as *u8); pass = pass + 1 } else { jl_puts(" FAIL KAT2 ntok=" as *u8); jl_putn(n2); jl_puts("\n" as *u8) }
555 tot = tot + 1
556
557 // ---- KAT 3: hex + float-exponent + comments skipped ----
558 let s3: *u8 = "let n = 0xFF; /* c */ z = 3.14e2 // end\x00" as *u8
559 let n3: i64 = js_lex(s3, jl_strlen(s3), toks, 256, ntb)
560 var r3: i64 = 1
561 if n3 != 8 { r3 = 0 }
562 r3 = r3 * js_expect(toks, n3, 0, JS_TOK_KEYWORD, s3, "let\x00" as *u8)
563 r3 = r3 * js_expect(toks, n3, 3, JS_TOK_NUMBER, s3, "0xFF\x00" as *u8)
564 r3 = r3 * js_expect(toks, n3, 4, JS_TOK_PUNCT, s3, ";\x00" as *u8)
565 r3 = r3 * js_expect(toks, n3, 5, JS_TOK_IDENT, s3, "z\x00" as *u8)
566 r3 = r3 * js_expect(toks, n3, 7, JS_TOK_NUMBER, s3, "3.14e2\x00" as *u8)
567 if r3 == 1 { jl_puts(" PASS KAT3 hex/float-exp + comments skipped (8 toks)\n" as *u8); pass = pass + 1 } else { jl_puts(" FAIL KAT3 ntok=" as *u8); jl_putn(n3); jl_puts("\n" as *u8) }
568 tot = tot + 1
569
570 // ---- KAT 4: string literal "hi" (byte-built to avoid source-escape ambiguity) ----
571 let s4: *u8 = sys_mmap(8)
572 s4[0] = 34 as u8; s4[1] = 104 as u8; s4[2] = 105 as u8; s4[3] = 34 as u8 // "hi"
573 let n4: i64 = js_lex(s4, 4, toks, 256, ntb)
574 var r4: i64 = 1
575 if n4 != 1 { r4 = 0 }
576 if toks[0] != JS_TOK_STRING { r4 = 0 }
577 if toks[2] != 4 { r4 = 0 }
578 if r4 == 1 { jl_puts(" PASS KAT4 string \"hi\" (1 STRING, len 4)\n" as *u8); pass = pass + 1 } else { jl_puts(" FAIL KAT4 ntok=" as *u8); jl_putn(n4); jl_puts(" kind=" as *u8); jl_putn(toks[0]); jl_puts("\n" as *u8) }
579 tot = tot + 1
580
581 // ---- KAT 5 (TAMPER): unterminated string -> ERROR, never a fabricated STRING ----
582 let s5: *u8 = sys_mmap(8)
583 s5[0] = 34 as u8; s5[1] = 97 as u8; s5[2] = 98 as u8; s5[3] = 99 as u8 // "abc
584 let n5: i64 = js_lex(s5, 4, toks, 256, ntb)
585 var r5: i64 = 1
586 if n5 != 1 { r5 = 0 }
587 if toks[0] != JS_TOK_ERROR { r5 = 0 }
588 if r5 == 1 { jl_puts(" PASS KAT5 tamper: unterminated string -> ERROR (not STRING)\n" as *u8); pass = pass + 1 } else { jl_puts(" FAIL KAT5 ntok=" as *u8); jl_putn(n5); jl_puts(" kind=" as *u8); jl_putn(toks[0]); jl_puts("\n" as *u8) }
589 tot = tot + 1
590
591 // ---- KAT 6: function decl punctuation ----
592 let s6: *u8 = "function f(){return 1}\x00" as *u8
593 let n6: i64 = js_lex(s6, jl_strlen(s6), toks, 256, ntb)
594 var r6: i64 = 1
595 if n6 != 8 { r6 = 0 }
596 r6 = r6 * js_expect(toks, n6, 0, JS_TOK_KEYWORD, s6, "function\x00" as *u8)
597 r6 = r6 * js_expect(toks, n6, 1, JS_TOK_IDENT, s6, "f\x00" as *u8)
598 r6 = r6 * js_expect(toks, n6, 2, JS_TOK_PUNCT, s6, "(\x00" as *u8)
599 r6 = r6 * js_expect(toks, n6, 5, JS_TOK_KEYWORD, s6, "return\x00" as *u8)
600 r6 = r6 * js_expect(toks, n6, 6, JS_TOK_NUMBER, s6, "1\x00" as *u8)
601 r6 = r6 * js_expect(toks, n6, 7, JS_TOK_PUNCT, s6, "}\x00" as *u8)
602 if r6 == 1 { jl_puts(" PASS KAT6 function decl (8 toks)\n" as *u8); pass = pass + 1 } else { jl_puts(" FAIL KAT6 ntok=" as *u8); jl_putn(n6); jl_puts("\n" as *u8) }
603 tot = tot + 1
604
605 // ---- KAT 7: backslash escape inside string does NOT terminate ('a\nb', literal backslash) ----
606 let s7: *u8 = sys_mmap(8)
607 s7[0] = 39 as u8; s7[1] = 97 as u8; s7[2] = 92 as u8; s7[3] = 110 as u8; s7[4] = 98 as u8; s7[5] = 39 as u8 // 'a\nb'
608 let n7: i64 = js_lex(s7, 6, toks, 256, ntb)
609 var r7: i64 = 1
610 if n7 != 1 { r7 = 0 }
611 if toks[0] != JS_TOK_STRING { r7 = 0 }
612 if toks[2] != 6 { r7 = 0 }
613 if r7 == 1 { jl_puts(" PASS KAT7 escaped quote/backslash handled (1 STRING, len 6)\n" as *u8); pass = pass + 1 } else { jl_puts(" FAIL KAT7 ntok=" as *u8); jl_putn(n7); jl_puts(" len=" as *u8); jl_putn(toks[2]); jl_puts("\n" as *u8) }
614 tot = tot + 1
615
616 // ---- KAT8: regex literal after '=' (expression context) ----
617 let s8: *u8 = "x = /ab+c/gi\x00" as *u8
618 let n8: i64 = js_lex(s8, jl_strlen(s8), toks, 256, ntb)
619 var r8: i64 = 1
620 if n8 != 3 { r8 = 0 }
621 r8 = r8 * js_expect(toks, n8, 0, JS_TOK_IDENT, s8, "x\x00" as *u8)
622 r8 = r8 * js_expect(toks, n8, 1, JS_TOK_PUNCT, s8, "=\x00" as *u8)
623 r8 = r8 * js_expect(toks, n8, 2, JS_TOK_REGEX, s8, "/ab+c/gi\x00" as *u8)
624 if r8 == 1 { jl_puts(" PASS KAT8 regex /ab+c/gi after '='\n" as *u8); pass = pass + 1 } else { jl_puts(" FAIL KAT8 ntok=" as *u8); jl_putn(n8); jl_puts("\n" as *u8) }
625 tot = tot + 1
626 // ---- KAT9: division after ident stays PUNCT '/' (NOT regex) ----
627 let s9: *u8 = "a / b\x00" as *u8
628 let n9: i64 = js_lex(s9, jl_strlen(s9), toks, 256, ntb)
629 var r9: i64 = 1
630 if n9 != 3 { r9 = 0 }
631 r9 = r9 * js_expect(toks, n9, 0, JS_TOK_IDENT, s9, "a\x00" as *u8)
632 r9 = r9 * js_expect(toks, n9, 1, JS_TOK_PUNCT, s9, "/\x00" as *u8)
633 r9 = r9 * js_expect(toks, n9, 2, JS_TOK_IDENT, s9, "b\x00" as *u8)
634 if r9 == 1 { jl_puts(" PASS KAT9 division 'a / b' -> PUNCT '/'\n" as *u8); pass = pass + 1 } else { jl_puts(" FAIL KAT9 ntok=" as *u8); jl_putn(n9); jl_puts("\n" as *u8) }
635 tot = tot + 1
636 // ---- KAT10: regex with '/' inside a [..] class does not close early ----
637 let sA: *u8 = "/[/]x/\x00" as *u8
638 let nA: i64 = js_lex(sA, jl_strlen(sA), toks, 256, ntb)
639 var rA: i64 = 1
640 if nA != 1 { rA = 0 }
641 rA = rA * js_expect(toks, nA, 0, JS_TOK_REGEX, sA, "/[/]x/\x00" as *u8)
642 if rA == 1 { jl_puts(" PASS KAT10 regex '/[/]x/' (/ inside [] does not close)\n" as *u8); pass = pass + 1 } else { jl_puts(" FAIL KAT10 ntok=" as *u8); jl_putn(nA); jl_puts("\n" as *u8) }
643 tot = tot + 1
644 jl_puts("---- nx_js_lex gate: passed " as *u8); jl_putn(pass); jl_puts(" / " as *u8); jl_putn(tot); jl_puts("\n" as *u8)
645 let lfd: i64 = sys_openat_append("knowledge/status/js_engine.log\x00" as *u8, 0x1a4)
646 if lfd >= 0 {
647 sys_write(lfd, "R-JS-LEX organ=nx_js_lex kats=" as *u8, 30)
648 jl_fdn(lfd, pass); sys_write(lfd, "/" as *u8, 1); jl_fdn(lfd, tot)
649 if pass == tot { sys_write(lfd, " tamper=ok verdict=GREEN\n" as *u8, 25) }
650 if pass != tot { sys_write(lfd, " tamper=?? verdict=RED\n" as *u8, 23) }
651 sys_close(lfd)
652 }
653 if pass == tot { sys_exit(0); return 0 }
654 sys_exit(1)
655 return 1
656}
657
658// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
659// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
660// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
661// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
662func jl_fdn(fd: i64, v: i64) -> i64 { nxi_fd(fd, v); return 0 }