`nx_js_lex.nx` source

↩ module page · 653 lines · 33002 B
1// nx_js_lex.nx -- R-JS-LEX (WB-JS-001 rung 0): the ECMAScript TOKENIZER, the
2// hardware-up floor of the sovereign JS engine (browser-engine-exceed roadmap §2).
3// Founds R-JS-PARSE above it (no-floating law: the lexer is built+gated before any
4// parser rides it). Shape = STATE_MACHINE/STRUCT_WALK over source bytes -- an
5// EXISTING emitter shape, so team-authorable now (NOT blocked on X-AUT-006c).
6//
7// Scans ECMAScript source into a flat token stream (3 i64 slots per token:
8// kind, start-offset, byte-length). Covers the lexical core real JS uses:
9//   - whitespace + line-terminators (skipped; ASI is a PARSER concern, rung 1)
10//   - // line comments and /* block */ comments (skipped)
11//   - identifiers + keywords ($ and _ legal; keyword set classified)
12//   - numeric literals: decimal int/float, exponent, 0x/0o/0b prefixes
13//   - string literals ' and " with backslash-escape handling (\" \' \\ \n ...)
14//   - punctuators incl. multi-char operators (=== !== >>>= => ?. ?? ** etc.)
15//   - unterminated string -> ERROR token (honest, never silently swallowed)
16//
17// HONEST RUNG-0 SCOPE (named OPEN, NOT faked): template literals (`...${}`) and
18// regex-literal disambiguation need PARSER context (is `/` a divide or a regex?),
19// so they are R-JS-LEX-0b, built once the parser can feed back context. BigInt `n`
20// suffix + full Unicode IdentifierStart are also 0b. Rung 0 tokenizes ASCII JS.
21//
22// GATE (main): 7 KATs assert the exact token stream (kind + lexeme) on real JS
23// snippets + TAMPER (unterminated string must be ERROR, never a fabricated STRING).
24// Self-validating; exit 0 iff all pass; appends knowledge/status/js_engine.log.
25//
26// license_tier: ORIGINAL  (tutor-bootstrap scaffold; team re-authors from the
27// R-JS-LEX data spec via author=organ -- (B)-debt, mirror the h2 capstone note.)
28import "nx_syscalls.nx"
29import "nx_itoa_lib.nx"   // shared MSB-first emitter (zero-alloc)
30
31const JS_TOK_EOF:     i64 = 0
32const JS_TOK_IDENT:   i64 = 1
33const JS_TOK_KEYWORD: i64 = 2
34const JS_TOK_NUMBER:  i64 = 3
35const JS_TOK_STRING:  i64 = 4
36const JS_TOK_PUNCT:   i64 = 5
37const JS_TOK_ERROR:   i64 = 8
38const JS_TOK_TEMPLATE: i64 = 9   // template literal: whole span incl both backticks; interior re-lexed at eval
39const JS_TOK_REGEX:    i64 = 10  // regex literal /pattern/flags: whole span; pattern+flags extracted at eval
40
41// ---- byte classification ----
42func js_is_digit(c: i64) -> i64 { if c >= 48 { if c <= 57 { return 1 } } return 0 }
43func js_is_alpha(c: i64) -> i64 {
  if c >= 65 { if c <= 90 { return 1 } }
  if c >= 97 { if c <= 122 { return 1 } }
  return 0
47}
48// ECMAScript identifiers admit Unicode letters, not just ASCII.  This lexer is BYTE
49// oriented, so every byte of a UTF-8 multi-byte sequence (0x80..0xFF) counts as an
50// identifier byte.  MEASURED MOTIVATION (2026-07-27): netflix's 3.9 MB bundle fails to
51// parse at a lodash deburr map -- `{À:"A",Á:"A",à:"a",...}` -- bare non-ASCII identifier
52// KEYS, which are near-universal in bundled apps.  Isolated with controls by
53// nx_js_unicode_probe (quoted non-ASCII keys already worked; bare ones did not).
54// This is deliberately PERMISSIVE: it accepts some code points real JS reserves for
55// punctuation, which only ever makes us accept MORE programs -- it can never change the
56// meaning of a valid one.
57// ⚠RESIDUAL RISK (named, not hidden): Unicode SPACE characters are also >=0x80 (NBSP
58// U+00A0 = C2 A0, U+2028/9, BOM U+FEFF).  Used as a separator ADJACENT to an identifier
59// they would now be absorbed into it.  Minified bundles -- the entire point of this
60// change -- separate with ASCII space, so this does not arise in practice; a proper fix
61// is a UTF-8 decode + Unicode ID_Start/ID_Continue table, a separate bounded rung.
62func js_is_idstart(c: i64) -> i64 {
  if js_is_alpha(c) == 1 { return 1 }
  if c == 95 { return 1 }   // _
  if c == 36 { return 1 }   // $
  if c >= 128 { return 1 }  // UTF-8 lead/continuation byte = part of a Unicode letter
  return 0
68}
69func js_is_idpart(c: i64) -> i64 {
  if js_is_idstart(c) == 1 { return 1 }
  if js_is_digit(c) == 1 { return 1 }
  return 0
73}
74func js_is_hex(c: i64) -> i64 {
  if js_is_digit(c) == 1 { return 1 }
  if c >= 65 { if c <= 70 { return 1 } }    // A-F
  if c >= 97 { if c <= 102 { return 1 } }   // a-f
  return 0
79}
80func js_is_space(c: i64) -> i64 {
  if c == 32 { return 1 }   // space
  if c == 9  { return 1 }   // tab
  if c == 10 { return 1 }   // \n
  if c == 13 { return 1 }   // \r
  if c == 11 { return 1 }   // \v
  if c == 12 { return 1 }   // \f
  return 0
88}
89
90// ---- lexeme compare: src[start..start+len) == NUL-terminated lit ? ----
91func js_lexeme_eq(src: *u8, start: i64, len: i64, lit: *u8) -> i64 {
  var i: i64 = 0
  while i < len {
      if (lit[i] & 0xff) == 0 { return 0 }                            // lit shorter than len
      if (src[start + i] & 0xff) != (lit[i] & 0xff) { return 0 }
      i = i + 1
  }
  if (lit[len] & 0xff) != 0 { return 0 }                              // lit longer than len
  return 1
100}
101
102// ---- keyword classification (ECMAScript reserved + common contextual) ----
103func js_is_keyword(src: *u8, start: i64, len: i64) -> i64 {
  if js_lexeme_eq(src, start, len, "var\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "let\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "const\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "function\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "return\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "if\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "else\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "for\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "while\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "do\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "break\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "continue\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "switch\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "case\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "default\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "new\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "delete\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "typeof\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "instanceof\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "in\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "of\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "this\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "null\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "true\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "false\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "void\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "try\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "catch\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "finally\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "throw\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "class\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "extends\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "super\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "import\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "export\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "async\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "await\x00" as *u8) == 1 { return 1 }
  if js_lexeme_eq(src, start, len, "yield\x00" as *u8) == 1 { return 1 }
  return 0
143}
144
145// ---- scanners (each returns the END index just past the token) ----
146func js_scan_ident(src: *u8, srclen: i64, pos: i64) -> i64 {
  var e: i64 = pos + 1
  var k: i64 = 1
  while k == 1 {
      if e >= srclen { k = 0 }
      if k == 1 { if js_is_idpart(src[e] & 0xff) == 1 { e = e + 1 } else { k = 0 } }
  }
  return e
154}
155func js_scan_while_digit(src: *u8, srclen: i64, pos: i64) -> i64 {
  var e: i64 = pos
  var k: i64 = 1
  while k == 1 {
      if e >= srclen { k = 0 }
      if k == 1 { if js_is_digit(src[e] & 0xff) == 1 { e = e + 1 } else { k = 0 } }
  }
  return e
163}
164func js_scan_while_hex(src: *u8, srclen: i64, pos: i64) -> i64 {
  var e: i64 = pos
  var k: i64 = 1
  while k == 1 {
      if e >= srclen { k = 0 }
      if k == 1 { if js_is_hex(src[e] & 0xff) == 1 { e = e + 1 } else { k = 0 } }
  }
  return e
172}
173func js_scan_exp(src: *u8, srclen: i64, pos: i64) -> i64 {
  var e: i64 = pos + 1                                  // skip e/E
  if e < srclen {
      let s: i64 = src[e] & 0xff
      if s == 43 { e = e + 1 }                          // +
      if s == 45 { e = e + 1 }                          // -
  }
  e = js_scan_while_digit(src, srclen, e)
  return e
182}
183func js_scan_number(src: *u8, srclen: i64, pos: i64) -> i64 {
  var e: i64 = pos
  if (src[e] & 0xff) == 48 {                            // leading '0' -> maybe radix prefix
      if e + 1 < srclen {
          let n1: i64 = src[e + 1] & 0xff
          if n1 == 120 { return js_scan_while_hex(src, srclen, e + 2) }    // 0x
          if n1 == 88  { return js_scan_while_hex(src, srclen, e + 2) }    // 0X
          if n1 == 98  { return js_scan_while_digit(src, srclen, e + 2) }  // 0b
          if n1 == 66  { return js_scan_while_digit(src, srclen, e + 2) }  // 0B
          if n1 == 111 { return js_scan_while_digit(src, srclen, e + 2) }  // 0o
          if n1 == 79  { return js_scan_while_digit(src, srclen, e + 2) }  // 0O
      }
  }
  e = js_scan_while_digit(src, srclen, e)               // integer part
  if e < srclen { if (src[e] & 0xff) == 46 {            // '.' fraction
      e = js_scan_while_digit(src, srclen, e + 1)
  } }
  if e < srclen {
      let ec: i64 = src[e] & 0xff
      if ec == 101 { e = js_scan_exp(src, srclen, e) }  // e
      if ec == 69  { e = js_scan_exp(src, srclen, e) }  // E
  }
  return e
206}
207// string: returns END (past close quote) if closed, else 0 - END if unterminated.
208func js_scan_string(src: *u8, srclen: i64, pos: i64) -> i64 {
  let q: i64 = src[pos] & 0xff
  var e: i64 = pos + 1
  var k: i64 = 1
  var closed: i64 = 0
  while k == 1 {
      if e >= srclen { k = 0 }
      if k == 1 {
          let c: i64 = src[e] & 0xff
          if c == 92 { e = e + 2 }                      // backslash escape -> skip next char
          else {
              if c == q { e = e + 1; closed = 1; k = 0 }
              else { e = e + 1 }
          }
      }
  }
  if closed == 1 { return e }
  return 0 - e
226}
227
228// punctuator length at pos (1..4), or 0 if pos is not a punctuator.
229func js_punct_len(src: *u8, srclen: i64, pos: i64) -> i64 {
  let c: i64 = src[pos] & 0xff
  var c1: i64 = 0
  var c2: i64 = 0
  var c3: i64 = 0
  if pos + 1 < srclen { c1 = src[pos + 1] & 0xff }
  if pos + 2 < srclen { c2 = src[pos + 2] & 0xff }
  if pos + 3 < srclen { c3 = src[pos + 3] & 0xff }
  if c == 62 {                                          // >
      if c1 == 62 {                                     // >>
          if c2 == 62 { if c3 == 61 { return 4 } return 3 }   // >>> , >>>=
          if c2 == 61 { return 3 }                      // >>=
          return 2
      }
      if c1 == 61 { return 2 }                          // >=
      return 1
  }
  if c == 60 {                                          // <
      if c1 == 60 { if c2 == 61 { return 3 } return 2 } // << , <<=
      if c1 == 61 { return 2 }                          // <=
      return 1
  }
  if c == 61 {                                          // =
      if c1 == 61 { if c2 == 61 { return 3 } return 2 } // == , ===
      if c1 == 62 { return 2 }                          // =>
      return 1
  }
  if c == 33 {                                          // !
      if c1 == 61 { if c2 == 61 { return 3 } return 2 } // != , !==
      return 1
  }
  if c == 38 {                                          // &
      if c1 == 38 { if c2 == 61 { return 3 } return 2 } // && , &&=
      if c1 == 61 { return 2 }                          // &=
      return 1
  }
  if c == 124 {                                         // |
      if c1 == 124 { if c2 == 61 { return 3 } return 2 } // || , ||=
      if c1 == 61 { return 2 }                          // |=
      return 1
  }
  if c == 63 {                                          // ?
      if c1 == 63 { if c2 == 61 { return 3 } return 2 } // ?? , ??=
      // `?.` is OPTIONAL CHAINING only when NOT followed by a decimal digit
      // (ECMAScript: OptionalChainingPunctuator :: ?. [lookahead is not DecimalDigit]).
      // Otherwise it is a TERNARY whose consequent is a leading-dot number:
      // `e.disabledStyleProp?.5:1` == `e.disabledStyleProp ? .5 : 1`. Minifiers emit
      // `.5` for `0.5`, so greedily taking `?.` broke real bundles -- MEASURED in
      // netflix's bundle 2026-07-27; isolated by nx_js_bundle_probe S6/S7 (`.5` alone
      // and `a?.b` alone both already passed, only the combination failed).
      if c1 == 46 { if js_is_digit(c2) == 1 { return 1 } return 2 }   // ?. vs ? .5
      return 1
  }
  if c == 42 {                                          // *
      if c1 == 42 { if c2 == 61 { return 3 } return 2 } // ** , **=
      if c1 == 61 { return 2 }                          // *=
      return 1
  }
  if c == 43 { if c1 == 43 { return 2 } if c1 == 61 { return 2 } return 1 }   // ++ += +
  if c == 45 { if c1 == 45 { return 2 } if c1 == 61 { return 2 } return 1 }   // -- -= -
  if c == 47 { if c1 == 61 { return 2 } return 1 }      // /= /   (// /* handled in trivia)
  if c == 37 { if c1 == 61 { return 2 } return 1 }      // %= %
  if c == 94 { if c1 == 61 { return 2 } return 1 }      // ^= ^
  if c == 46 { if c1 == 46 { if c2 == 46 { return 3 } } return 1 }  // ... .
  if c == 123 { return 1 }                              // {
  if c == 125 { return 1 }                              // }
  if c == 40  { return 1 }                              // (
  if c == 41  { return 1 }                              // )
  if c == 91  { return 1 }                              // [
  if c == 93  { return 1 }                              // ]
  if c == 59  { return 1 }                              // ;
  if c == 44  { return 1 }                              // ,
  if c == 58  { return 1 }                              // :
  if c == 126 { return 1 }                              // ~
  return 0
304}
305
306// skip whitespace + // line and /* block */ comments; return first real-token pos.
307func js_skip_trivia(src: *u8, srclen: i64, pos: i64) -> i64 {
  var p: i64 = pos
  var go: i64 = 1
  while go == 1 {
      var k: i64 = 1
      while k == 1 {
          if p >= srclen { k = 0 } else { if js_is_space(src[p] & 0xff) == 1 { p = p + 1 } else { k = 0 } }
      }
      var did: i64 = 0
      if p + 1 < srclen { if (src[p] & 0xff) == 47 { if (src[p + 1] & 0xff) == 47 {   // //
          p = p + 2
          var k2: i64 = 1
          while k2 == 1 {
              if p >= srclen { k2 = 0 } else { if (src[p] & 0xff) == 10 { k2 = 0 } else { p = p + 1 } }
          }
          did = 1
      } } }
      if did == 0 { if p + 1 < srclen { if (src[p] & 0xff) == 47 { if (src[p + 1] & 0xff) == 42 {   // /*
          p = p + 2
          var k3: i64 = 1
          while k3 == 1 {
              if p + 1 >= srclen { p = srclen; k3 = 0 }
              else {
                  if (src[p] & 0xff) == 42 { if (src[p + 1] & 0xff) == 47 { p = p + 2; k3 = 0 } else { p = p + 1 } }
                  else { p = p + 1 }
              }
          }
          did = 1
      } } } }
      if did == 0 { go = 0 }
  }
  return p
339}
340
341func js_emit(toks: *i64, ntok_box: *i64, kind: i64, start: i64, len: i64) -> i64 {
  let i: i64 = ntok_box[0]
  toks[i * 3 + 0] = kind
  toks[i * 3 + 1] = start
  toks[i * 3 + 2] = len
  ntok_box[0] = i + 1
  return 0
348}
349
350func js_num_starts(src: *u8, srclen: i64, pos: i64) -> i64 {
  let c: i64 = src[pos] & 0xff
  if js_is_digit(c) == 1 { return 1 }
  if c == 46 { if pos + 1 < srclen { if js_is_digit(src[pos + 1] & 0xff) == 1 { return 1 } } }  // .5
  return 0
355}
356
357func js_lex_string(src: *u8, srclen: i64, pos: i64, toks: *i64, ntok_box: *i64) -> i64 {
  let r: i64 = js_scan_string(src, srclen, pos)
  if r > 0 { js_emit(toks, ntok_box, JS_TOK_STRING, pos, r - pos); return r }
  let e: i64 = 0 - r
  js_emit(toks, ntok_box, JS_TOK_ERROR, pos, e - pos)                 // unterminated -> ERROR
  return e
363}
364
365// scan a template literal `...${expr}...` : emit ONE JS_TOK_TEMPLATE token spanning both backticks.
366// Handles \-escapes and ${ ... } interior brace-nesting (so a `}` inside ${} does not end it early).
367// The interior (literal runs + ${expr}) is decoded at EVAL time. Nested templates/strings-with-braces
368// inside ${} are a NAMED OPEN (brace-depth only, no string awareness inside interpolation).
369func js_lex_template(src: *u8, srclen: i64, pos: i64, toks: *i64, ntok_box: *i64) -> i64 {
  var p: i64 = pos + 1
  var go: i64 = 1
  while go == 1 {
      if p >= srclen { js_emit(toks, ntok_box, JS_TOK_ERROR, pos, p - pos); return p }   // unterminated
      let ch: i64 = src[p] & 0xff
      if ch == 92 { p = p + 2 }                                  // \-escape: skip 2
      else {
          if ch == 96 { go = 0 }                                 // closing `
          else {
              var handled: i64 = 0
              if ch == 36 { if p + 1 < srclen { if (src[p + 1] & 0xff) == 123 {   // ${
                  p = p + 2
                  var depth: i64 = 1
                  var g2: i64 = 1
                  while g2 == 1 {
                      if p >= srclen { g2 = 0 }
                      else {
                          let c2: i64 = src[p] & 0xff
                          if c2 == 123 { depth = depth + 1; p = p + 1 }
                          else { if c2 == 125 { depth = depth - 1; p = p + 1; if depth == 0 { g2 = 0 } } else { p = p + 1 } }
                      }
                  }
                  handled = 1
              } } }
              if handled == 0 { p = p + 1 }
          }
      }
  }
  js_emit(toks, ntok_box, JS_TOK_TEMPLATE, pos, (p + 1) - pos)
  return p + 1
400}
401// R-JS-LEX-0b: can a `/` at this point START A REGEX (vs be division)? Decided by the PREVIOUS token.
402// After a VALUE (ident/number/string/template/`)`/`]`/`}`/this/true/false/null/super) => division.
403// After anything else (operators, ( [ { , ; = return typeof new ... , or start-of-input) => regex.
404// (`}` is treated as a value here -- the one classic ambiguity; block-`}` followed by /re/ is vanishingly
405//  rare in real/minified code. Noted.)
406func js_regex_context(toks: *i64, ntok: i64, src: *u8) -> i64 {
  if ntok == 0 { return 1 }
  let b: i64 = (ntok - 1) * 3
  let pk: i64 = toks[b]
  let ps: i64 = toks[b + 1]
  let pl: i64 = toks[b + 2]
  if pk == JS_TOK_NUMBER { return 0 }
  if pk == JS_TOK_STRING { return 0 }
  if pk == JS_TOK_TEMPLATE { return 0 }
  if pk == JS_TOK_IDENT { return 0 }
  if pk == JS_TOK_KEYWORD {
      if js_lexeme_eq(src, ps, pl, "this\x00" as *u8) == 1 { return 0 }
      if js_lexeme_eq(src, ps, pl, "true\x00" as *u8) == 1 { return 0 }
      if js_lexeme_eq(src, ps, pl, "false\x00" as *u8) == 1 { return 0 }
      if js_lexeme_eq(src, ps, pl, "null\x00" as *u8) == 1 { return 0 }
      if js_lexeme_eq(src, ps, pl, "super\x00" as *u8) == 1 { return 0 }
      return 1
  }
  if pk == JS_TOK_PUNCT {
      if js_lexeme_eq(src, ps, pl, ")\x00" as *u8) == 1 { return 0 }
      if js_lexeme_eq(src, ps, pl, "]\x00" as *u8) == 1 { return 0 }
      if js_lexeme_eq(src, ps, pl, "}\x00" as *u8) == 1 { return 0 }
      return 1
  }
  return 1
431}
432// scan a regex literal /pattern/flags -> ONE JS_TOK_REGEX token. `/` closes only OUTSIDE a `[..]` class;
433// `\` escapes the next char; an unescaped newline (or EOF) before the close = ERROR (never a fabricated regex).
434func js_lex_regex(src: *u8, srclen: i64, pos: i64, toks: *i64, ntok_box: *i64) -> i64 {
  var p: i64 = pos + 1
  var inclass: i64 = 0
  var go: i64 = 1
  while go == 1 {
      if p >= srclen { js_emit(toks, ntok_box, JS_TOK_ERROR, pos, p - pos); return p }
      let ch: i64 = src[p] & 0xff
      if ch == 92 { p = p + 2 }
      else {
          if ch == 10 { js_emit(toks, ntok_box, JS_TOK_ERROR, pos, p - pos); return p }
          else { if ch == 91 { inclass = 1; p = p + 1 }
          else { if ch == 93 { inclass = 0; p = p + 1 }
          else { if ch == 47 { if inclass == 0 { go = 0 } else { p = p + 1 } }
          else { p = p + 1 } } } }
      }
  }
  p = p + 1                                    // consume closing '/'
  var gf: i64 = 1
  while gf == 1 { if p < srclen { let fc: i64 = src[p] & 0xff; if fc >= 97 { if fc <= 122 { p = p + 1 } else { gf = 0 } } else { gf = 0 } } else { gf = 0 } }
  js_emit(toks, ntok_box, JS_TOK_REGEX, pos, p - pos)
  return p
455}
456// lex exactly ONE token at pos (pos already past trivia); return END index.
457func js_lex_one(src: *u8, srclen: i64, pos: i64, toks: *i64, ntok_box: *i64) -> i64 {
  let c: i64 = src[pos] & 0xff
  if js_is_idstart(c) == 1 {
      let e: i64 = js_scan_ident(src, srclen, pos)
      var kind: i64 = JS_TOK_IDENT
      if js_is_keyword(src, pos, e - pos) == 1 { kind = JS_TOK_KEYWORD }
      js_emit(toks, ntok_box, kind, pos, e - pos)
      return e
  }
  if js_num_starts(src, srclen, pos) == 1 {
      let e: i64 = js_scan_number(src, srclen, pos)
      js_emit(toks, ntok_box, JS_TOK_NUMBER, pos, e - pos)
      return e
  }
  if c == 34 { return js_lex_string(src, srclen, pos, toks, ntok_box) }   // "
  if c == 39 { return js_lex_string(src, srclen, pos, toks, ntok_box) }   // '
  if c == 96 { return js_lex_template(src, srclen, pos, toks, ntok_box) } // ` template literal
  if c == 47 { if js_regex_context(toks, ntok_box[0], src) == 1 { return js_lex_regex(src, srclen, pos, toks, ntok_box) } }  // '/' regex vs divide
  let pl: i64 = js_punct_len(src, srclen, pos)
  if pl > 0 { js_emit(toks, ntok_box, JS_TOK_PUNCT, pos, pl); return pos + pl }
  js_emit(toks, ntok_box, JS_TOK_ERROR, pos, 1)                           // unknown byte
  return pos + 1
479}
480
481// PUBLIC: tokenize src[0..srclen) into toks (3 i64/token); returns token count.
482func js_lex(src: *u8, srclen: i64, toks: *i64, maxtoks: i64, ntok_box: *i64) -> i64 {
  var pos: i64 = 0
  ntok_box[0] = 0
  var go: i64 = 1
  while go == 1 {
      pos = js_skip_trivia(src, srclen, pos)
      if pos >= srclen { go = 0 }
      if go == 1 { if ntok_box[0] >= maxtoks { go = 0 } }
      if go == 1 { pos = js_lex_one(src, srclen, pos, toks, ntok_box) }
  }
  return ntok_box[0]
493}
494
495// ===================== GATE =====================
496func jl_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
497// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
498// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
499// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
500// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
501func jl_putn(v: i64) -> i64 { nxi_out(v); return 0 }
502func jl_strlen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
503
504func js_expect(toks: *i64, ntok: i64, idx: i64, kind: i64, src: *u8, lit: *u8) -> i64 {
  if idx >= ntok { return 0 }
  if toks[idx * 3 + 0] != kind { return 0 }
  return js_lexeme_eq(src, toks[idx * 3 + 1], toks[idx * 3 + 2], lit)
508}
509
510func main() -> i64 {
  let toks: *i64 = sys_mmap(256 * 3 * 8) as *i64
  let ntb: *i64 = sys_mmap(16) as *i64
  var pass: i64 = 0
  var tot: i64 = 0
  jl_puts("nx_js_lex gate (R-JS-LEX, WB-JS-001 rung 0)\n" as *u8)
516
  // ---- KAT 1: keywords/idents/number/punct/dot ----
  let s1: *u8 = "var x = 42 + foo.bar\x00" as *u8
  let n1: i64 = js_lex(s1, jl_strlen(s1), toks, 256, ntb)
  var r1: i64 = 1
  if n1 != 8 { r1 = 0 }
  r1 = r1 * js_expect(toks, n1, 0, JS_TOK_KEYWORD, s1, "var\x00" as *u8)
  r1 = r1 * js_expect(toks, n1, 1, JS_TOK_IDENT,   s1, "x\x00" as *u8)
  r1 = r1 * js_expect(toks, n1, 2, JS_TOK_PUNCT,   s1, "=\x00" as *u8)
  r1 = r1 * js_expect(toks, n1, 3, JS_TOK_NUMBER,  s1, "42\x00" as *u8)
  r1 = r1 * js_expect(toks, n1, 4, JS_TOK_PUNCT,   s1, "+\x00" as *u8)
  r1 = r1 * js_expect(toks, n1, 5, JS_TOK_IDENT,   s1, "foo\x00" as *u8)
  r1 = r1 * js_expect(toks, n1, 6, JS_TOK_PUNCT,   s1, ".\x00" as *u8)
  r1 = r1 * js_expect(toks, n1, 7, JS_TOK_IDENT,   s1, "bar\x00" as *u8)
  if r1 == 1 { jl_puts("  PASS KAT1 var/ident/number/dot (8 toks)\n" as *u8); pass = pass + 1 } else { jl_puts("  FAIL KAT1 ntok=" as *u8); jl_putn(n1); jl_puts("\n" as *u8) }
  tot = tot + 1
532
  // ---- KAT 2: multi-char operators incl >>>= ----
  let s2: *u8 = "a===b!==c>>>=d\x00" as *u8
  let n2: i64 = js_lex(s2, jl_strlen(s2), toks, 256, ntb)
  var r2: i64 = 1
  if n2 != 7 { r2 = 0 }
  r2 = r2 * js_expect(toks, n2, 0, JS_TOK_IDENT, s2, "a\x00" as *u8)
  r2 = r2 * js_expect(toks, n2, 1, JS_TOK_PUNCT, s2, "===\x00" as *u8)
  r2 = r2 * js_expect(toks, n2, 2, JS_TOK_IDENT, s2, "b\x00" as *u8)
  r2 = r2 * js_expect(toks, n2, 3, JS_TOK_PUNCT, s2, "!==\x00" as *u8)
  r2 = r2 * js_expect(toks, n2, 4, JS_TOK_IDENT, s2, "c\x00" as *u8)
  r2 = r2 * js_expect(toks, n2, 5, JS_TOK_PUNCT, s2, ">>>=\x00" as *u8)
  r2 = r2 * js_expect(toks, n2, 6, JS_TOK_IDENT, s2, "d\x00" as *u8)
  if r2 == 1 { jl_puts("  PASS KAT2 operators ===/!==/>>>= (7 toks)\n" as *u8); pass = pass + 1 } else { jl_puts("  FAIL KAT2 ntok=" as *u8); jl_putn(n2); jl_puts("\n" as *u8) }
  tot = tot + 1
547
  // ---- KAT 3: hex + float-exponent + comments skipped ----
  let s3: *u8 = "let n = 0xFF; /* c */ z = 3.14e2 // end\x00" as *u8
  let n3: i64 = js_lex(s3, jl_strlen(s3), toks, 256, ntb)
  var r3: i64 = 1
  if n3 != 8 { r3 = 0 }
  r3 = r3 * js_expect(toks, n3, 0, JS_TOK_KEYWORD, s3, "let\x00" as *u8)
  r3 = r3 * js_expect(toks, n3, 3, JS_TOK_NUMBER,  s3, "0xFF\x00" as *u8)
  r3 = r3 * js_expect(toks, n3, 4, JS_TOK_PUNCT,   s3, ";\x00" as *u8)
  r3 = r3 * js_expect(toks, n3, 5, JS_TOK_IDENT,   s3, "z\x00" as *u8)
  r3 = r3 * js_expect(toks, n3, 7, JS_TOK_NUMBER,  s3, "3.14e2\x00" as *u8)
  if r3 == 1 { jl_puts("  PASS KAT3 hex/float-exp + comments skipped (8 toks)\n" as *u8); pass = pass + 1 } else { jl_puts("  FAIL KAT3 ntok=" as *u8); jl_putn(n3); jl_puts("\n" as *u8) }
  tot = tot + 1
560
  // ---- KAT 4: string literal "hi" (byte-built to avoid source-escape ambiguity) ----
  let s4: *u8 = sys_mmap(8)
  s4[0] = 34 as u8; s4[1] = 104 as u8; s4[2] = 105 as u8; s4[3] = 34 as u8   // "hi"
  let n4: i64 = js_lex(s4, 4, toks, 256, ntb)
  var r4: i64 = 1
  if n4 != 1 { r4 = 0 }
  if toks[0] != JS_TOK_STRING { r4 = 0 }
  if toks[2] != 4 { r4 = 0 }
  if r4 == 1 { jl_puts("  PASS KAT4 string \"hi\" (1 STRING, len 4)\n" as *u8); pass = pass + 1 } else { jl_puts("  FAIL KAT4 ntok=" as *u8); jl_putn(n4); jl_puts(" kind=" as *u8); jl_putn(toks[0]); jl_puts("\n" as *u8) }
  tot = tot + 1
571
  // ---- KAT 5 (TAMPER): unterminated string -> ERROR, never a fabricated STRING ----
  let s5: *u8 = sys_mmap(8)
  s5[0] = 34 as u8; s5[1] = 97 as u8; s5[2] = 98 as u8; s5[3] = 99 as u8     // "abc
  let n5: i64 = js_lex(s5, 4, toks, 256, ntb)
  var r5: i64 = 1
  if n5 != 1 { r5 = 0 }
  if toks[0] != JS_TOK_ERROR { r5 = 0 }
  if r5 == 1 { jl_puts("  PASS KAT5 tamper: unterminated string -> ERROR (not STRING)\n" as *u8); pass = pass + 1 } else { jl_puts("  FAIL KAT5 ntok=" as *u8); jl_putn(n5); jl_puts(" kind=" as *u8); jl_putn(toks[0]); jl_puts("\n" as *u8) }
  tot = tot + 1
581
  // ---- KAT 6: function decl punctuation ----
  let s6: *u8 = "function f(){return 1}\x00" as *u8
  let n6: i64 = js_lex(s6, jl_strlen(s6), toks, 256, ntb)
  var r6: i64 = 1
  if n6 != 8 { r6 = 0 }
  r6 = r6 * js_expect(toks, n6, 0, JS_TOK_KEYWORD, s6, "function\x00" as *u8)
  r6 = r6 * js_expect(toks, n6, 1, JS_TOK_IDENT,   s6, "f\x00" as *u8)
  r6 = r6 * js_expect(toks, n6, 2, JS_TOK_PUNCT,   s6, "(\x00" as *u8)
  r6 = r6 * js_expect(toks, n6, 5, JS_TOK_KEYWORD, s6, "return\x00" as *u8)
  r6 = r6 * js_expect(toks, n6, 6, JS_TOK_NUMBER,  s6, "1\x00" as *u8)
  r6 = r6 * js_expect(toks, n6, 7, JS_TOK_PUNCT,   s6, "}\x00" as *u8)
  if r6 == 1 { jl_puts("  PASS KAT6 function decl (8 toks)\n" as *u8); pass = pass + 1 } else { jl_puts("  FAIL KAT6 ntok=" as *u8); jl_putn(n6); jl_puts("\n" as *u8) }
  tot = tot + 1
595
  // ---- KAT 7: backslash escape inside string does NOT terminate ('a\nb', literal backslash) ----
  let s7: *u8 = sys_mmap(8)
  s7[0] = 39 as u8; s7[1] = 97 as u8; s7[2] = 92 as u8; s7[3] = 110 as u8; s7[4] = 98 as u8; s7[5] = 39 as u8  // 'a\nb'
  let n7: i64 = js_lex(s7, 6, toks, 256, ntb)
  var r7: i64 = 1
  if n7 != 1 { r7 = 0 }
  if toks[0] != JS_TOK_STRING { r7 = 0 }
  if toks[2] != 6 { r7 = 0 }
  if r7 == 1 { jl_puts("  PASS KAT7 escaped quote/backslash handled (1 STRING, len 6)\n" as *u8); pass = pass + 1 } else { jl_puts("  FAIL KAT7 ntok=" as *u8); jl_putn(n7); jl_puts(" len=" as *u8); jl_putn(toks[2]); jl_puts("\n" as *u8) }
  tot = tot + 1
606
  // ---- KAT8: regex literal after '=' (expression context) ----
  let s8: *u8 = "x = /ab+c/gi\x00" as *u8
  let n8: i64 = js_lex(s8, jl_strlen(s8), toks, 256, ntb)
  var r8: i64 = 1
  if n8 != 3 { r8 = 0 }
  r8 = r8 * js_expect(toks, n8, 0, JS_TOK_IDENT, s8, "x\x00" as *u8)
  r8 = r8 * js_expect(toks, n8, 1, JS_TOK_PUNCT, s8, "=\x00" as *u8)
  r8 = r8 * js_expect(toks, n8, 2, JS_TOK_REGEX, s8, "/ab+c/gi\x00" as *u8)
  if r8 == 1 { jl_puts("  PASS KAT8 regex /ab+c/gi after '='\n" as *u8); pass = pass + 1 } else { jl_puts("  FAIL KAT8 ntok=" as *u8); jl_putn(n8); jl_puts("\n" as *u8) }
  tot = tot + 1
  // ---- KAT9: division after ident stays PUNCT '/' (NOT regex) ----
  let s9: *u8 = "a / b\x00" as *u8
  let n9: i64 = js_lex(s9, jl_strlen(s9), toks, 256, ntb)
  var r9: i64 = 1
  if n9 != 3 { r9 = 0 }
  r9 = r9 * js_expect(toks, n9, 0, JS_TOK_IDENT, s9, "a\x00" as *u8)
  r9 = r9 * js_expect(toks, n9, 1, JS_TOK_PUNCT, s9, "/\x00" as *u8)
  r9 = r9 * js_expect(toks, n9, 2, JS_TOK_IDENT, s9, "b\x00" as *u8)
  if r9 == 1 { jl_puts("  PASS KAT9 division 'a / b' -> PUNCT '/'\n" as *u8); pass = pass + 1 } else { jl_puts("  FAIL KAT9 ntok=" as *u8); jl_putn(n9); jl_puts("\n" as *u8) }
  tot = tot + 1
  // ---- KAT10: regex with '/' inside a [..] class does not close early ----
  let sA: *u8 = "/[/]x/\x00" as *u8
  let nA: i64 = js_lex(sA, jl_strlen(sA), toks, 256, ntb)
  var rA: i64 = 1
  if nA != 1 { rA = 0 }
  rA = rA * js_expect(toks, nA, 0, JS_TOK_REGEX, sA, "/[/]x/\x00" as *u8)
  if rA == 1 { jl_puts("  PASS KAT10 regex '/[/]x/' (/ inside [] does not close)\n" as *u8); pass = pass + 1 } else { jl_puts("  FAIL KAT10 ntok=" as *u8); jl_putn(nA); jl_puts("\n" as *u8) }
  tot = tot + 1
  jl_puts("---- nx_js_lex gate: passed " as *u8); jl_putn(pass); jl_puts(" / " as *u8); jl_putn(tot); jl_puts("\n" as *u8)
  let lfd: i64 = sys_openat_append("knowledge/status/js_engine.log\x00" as *u8, 0x1a4)
  if lfd >= 0 {
      sys_write(lfd, "R-JS-LEX organ=nx_js_lex kats=" as *u8, 30)
      jl_fdn(lfd, pass); sys_write(lfd, "/" as *u8, 1); jl_fdn(lfd, tot)
      if pass == tot { sys_write(lfd, " tamper=ok verdict=GREEN\n" as *u8, 25) }
      if pass != tot { sys_write(lfd, " tamper=?? verdict=RED\n" as *u8, 23) }
      sys_close(lfd)
  }
  if pass == tot { sys_exit(0); return 0 }
  sys_exit(1)
  return 1
647}
648
649// MIGRATED to the shared emitter (debt 1785563586). The old body mmapped a scratch buffer
650// per call and never freed it. At PAGE granularity that is 4096B leaked PER CALL -- the
651// defect that took 28.5GB of a 36GB host in nx_ts_lumadiff (2MB input, ~3.66M calls).
652// nxi_* is MSB-first, allocates NOTHING, and emits identical bytes including the sign.
653func jl_fdn(fd: i64, v: i64) -> i64 { nxi_fd(fd, v); return 0 }
nx_js_lex.nx source

`nx_js_lex.nx` source
