code wiki / (root) / nx_tokenizer.nx

nx_tokenizer.nx

buildroot/runtime/nx_tokenizer.nx

111969 B2619 linesdepth 2pulls 4 transitivereach 201 importersview sourcekind library
docsdependenciesstructsconstsfunctions

about

lex.nx -- NishiLang source tokenizer, ported from lex.c. Accepts a null-terminated source buffer and produces a stream of Tok records. Mirrors lex.c's token layout (kind + line/col + either int_val or text[]). Same keyword set. Single-pass; zero allocations per token beyond the growing token array. Extends at 2x when full. Whitespace + // comments are skipped. Numbers support 0x/0b/0o prefixes with underscore separators. String literals read bytes until closing `"`.

dependencies 3 imports · 68 importers

nx_syscalls.nx nx_types.nx nx_lex_kinds.nx nx_tokenizer.nx end2end_test.nx nx_cap_detect_ir_gate_t278.nx nx_compile_field_candidate_t280.nx nx_compile_wat.nx nx_compile_wat_call_operands_t143. nx_compile_wat_scalararm.nx nx_compile_x86.nx nx_dom_oracle_gate.nx nx_lang_struct.nx nx_lsp.nx

diagram shows first 10 each side; +0 more imports, +58 more importers in the complete lists below.

imports: nx_syscalls.nxnx_types.nxnx_lex_kinds.nx

imported by: end2end_test.nxnx_cap_detect_ir_gate_t278.nxnx_compile_field_candidate_t280.nxnx_compile_wat.nxnx_compile_wat_call_operands_t143.nxnx_compile_wat_scalararm.nxnx_compile_x86.nxnx_dom_oracle_gate.nxnx_lang_struct.nxnx_lsp.nxnx_main.nxnx_nxc.nxnx_quality_grade_daemon.nxnx_quality_grade_quad.nxnx_quality_grade_self.nxnx_quality_grade_septagon.nxnx_rv64_atomic_gate.nxnx_smoke_lex2.nxnx_smoke_lex3.nxnx_smoke_lex4.nxnx_smoke_lex5.nxnx_smoke_lex6.nxnx_smoke_lex7.nxnx_smoke_lex_min.nxnx_smoke_nxc_pipeline.nxnx_smoke_nxc_pipeline10.nxnx_smoke_nxc_pipeline11.nxnx_smoke_nxc_pipeline12.nxnx_smoke_nxc_pipeline13.nxnx_smoke_nxc_pipeline14.nxnx_smoke_nxc_pipeline15.nxnx_smoke_nxc_pipeline2.nxnx_smoke_nxc_pipeline3.nxnx_smoke_nxc_pipeline4.nxnx_smoke_nxc_pipeline5.nxnx_smoke_nxc_pipeline6.nxnx_smoke_nxc_pipeline7.nxnx_smoke_nxc_pipeline8.nxnx_smoke_nxc_pipeline9.nxnx_smoke_nxc_tilde.nxnx_srcfmt.nxnx_type_layout_gate.nxnx_vcc_color_wat.nxnx_vcc_probe_wat.nxnx_wgsl.nxnx_wgsl_authored_t55.nxnx_wgsl_before_front_t217.nxnx_wgsl_canonical_skin_t140.nxnx_wgsl_cast_control_t181.nxnx_wgsl_cast_lbs_t181.nxnx_wgsl_cast_material_t181.nxnx_wgsl_cloud_transport_t155.nxnx_wgsl_cumulus_release_t161.nxnx_wgsl_cumulus_t156.nxnx_wgsl_depth_candidate_t192.nxnx_wgsl_eye_t138.nxnx_wgsl_face_mask_t163.nxnx_wgsl_front_candidate_t217.nxnx_wgsl_inspection_t165.nxnx_wgsl_sand_baseline_t137.nxnx_wgsl_sand_private_t137.nxnx_wgsl_scene_light_candidate_t364.nxnx_wgsl_skin_t138.nxnx_wgsl_sky_light_t154.nxnx_wgsl_sky_private_20260910.nxnx_wgsl_sky_review_t154.nxnx_wgsl_static_t335.nxnx_wgsl_world_depth_t195.nx

structs

159struct Lex

consts

65const LEX_DEC_SIG_DIGITS: i64 = 18
66const LEX_U64_MAX_DIV10: i64 = 1844674407370955161
67const LEX_U64_MAX_LAST: i64 = 5
68const LEX_EXP_CAP: i64 = 99999
89const LEXM_MAGIC_65536: i64 = 65536
235const LEXM_MAX: i64 = 256
236const LEXM_NAMELEN: i64 = 64
240const LEXM_BODYLEN: i64 = 1024
565const LEXM_MAXARGS: i64 = 8
581const LEX_IDENT_MAX: i64 = 63
1744const PPB_MAX: i64 = 32

functions

73func lex_err_get_kind() -> i64 { return lex_err_kind }
called by 2: mainmain
74func lex_err_get_line() -> i64 { return lex_err_line }
called by 2: mainmain
75func lex_err_get_col() -> i64 { return lex_err_col }
called by 2: mainmain
76func lex_err_get_byte() -> i64 { return lex_err_byte }
called by 2: mainmain
81func diag_desync_origin(kind: i64, line: i64, col: i64, byte: i64) -> i64
called by 3: lex_numberlex_stringlex_source
98func is_alpha(c: i64) -> i64
called by 4: sf_is_idstartis_alnumlexm_is_ident_textlex_source
104func is_digit(c: i64) -> i64
called by 5: sf_runis_hexdigitis_alnumlex_numberlex_source
108func is_hexdigit(c: i64) -> i64
called by 1: lex_number calls 1: is_digit
114func hex_val(c: i64) -> i64
called by 1: lex_number
120func is_alnum(c: i64) -> i64
called by 7: sf_is_idchsf_macro_endlexm_isid_lex_pp_defined_lex_handle_preprocessorlex_ident_or_kw+1 calls 2: is_alphais_digit
130func keyword_lookup(t: *u8, len: i64) -> i64
called by 1: lex_ident_or_kw calls 1: streq_n
176func push_tok(L: *Lex, t: *Tok) -> i64
called by 5: lex_ident_or_kwlex_numberlex_stringemit_punctlex_source calls 2: sys_writesys_exit
202func advance(L: *Lex) -> i64
215func peek(L: *Lex) -> i64
219func peek2(L: *Lex) -> i64
247func lexm_init() -> i64
257func lexm_emit_str(out: *u8, o0: i64, s: *u8) -> i64
264func lexm_emit_dec(out: *u8, o0: i64, v: i64) -> i64
276func lexm_strlen(s: *u8) -> i64
286func lexm_body_end(src: *u8, n: i64, bp: i64) -> i64
363func lexm_name_at(i: i64) -> *u8 { return ((lexm_names as i64) + i * LEXM_NAMELEN) as *u8 }
called by 3: lexm_definelexm_lookuplexm_undef
364func lexm_body_at(i: i64) -> *u8 { return ((lexm_bodies as i64) + i * LEXM_BODYLEN) as *u8 }
377func lexm_diag_cell() -> *u8
384func lexm_warn_body_too_long(name: *u8, nlen: i64, blen: i64) -> i64
405func lexm_define(name: *u8, nlen: i64, body: *u8, blen: i64) -> i64
441func lexm_lookup(name: *u8, nlen: i64) -> *u8
461func lexm_undef(name: *u8, nlen: i64) -> i64
506func lexm_predefine() -> i64
514func lexm_reset() -> i64
526func lexm_body_int(body: *u8, out: *i64) -> i64
567func lexm_warn_too_many_args(got: i64) -> i64
583func lex_warn_ident_too_long(text: *u8, kept: i64, actual: i64) -> i64
601func lexm_isid(c: i64) -> i64 { return is_alnum(c) }
605func lexm_subst(body: *u8, asrc: *u8, argp: *i64, argl: *i64, nargs: i64, out: *u8, o0: i64) -> i64
779func lexm_prescan(src: *u8, n: i64) -> i64
824func lexm_for_bound(src: *u8, s0: i64, e0: i64, out: *i64) -> i64
858func lexm_for_emit(src: *u8, bs: i64, be: i64, vname: *u8, vlen: i64, vtxt: *u8, vtl: i64, out: *u8, o0: i64, ocap: i64) -> i64
927func lexm_trim(src: *u8, s: i64, e: i64, eout: *i64) -> i64
950func lexm_is_ident_text(p: *u8, s: i64, e: i64) -> i64
968func lexm_warn_bad_paste(out: *u8, s: i64, e: i64) -> i64
984func lexm_paste_expand(src: *u8, n: i64, istart: i64, kwlen: i64, mode: i64, out: *u8, o0: i64, ocap: i64, iout: *i64) -> i64
1046func lexm_kw_is(src: *u8, n: i64, at: i64, kw: *u8, kwlen: i64) -> i64
1059func lexm_for_expand(src: *u8, n: i64, istart: i64, out: *u8, o0: i64, ocap: i64, iout: *i64) -> i64
1185func lexm_expand(src: *u8, n: i64, out: *u8, outcap: i64) -> i64
1363func ppe_or(src: *u8, n: i64, pos: *i64) -> i64;
1371func ppe_scratch_cell() -> *i64
1376func ppe_skipws(src: *u8, n: i64, pos: *i64) -> i64
1390func ppe_primary(src: *u8, n: i64, pos: *i64) -> i64
1487func ppe_unary(src: *u8, n: i64, pos: *i64) -> i64
1511func ppe_mul(src: *u8, n: i64, pos: *i64) -> i64
1540func ppe_add(src: *u8, n: i64, pos: *i64) -> i64
1555func ppe_cmp(src: *u8, n: i64, pos: *i64) -> i64
1600func ppe_and(src: *u8, n: i64, pos: *i64) -> i64
1622func ppe_or(src: *u8, n: i64, pos: *i64) -> i64
1647func ppe_eval_at(src: *u8, n: i64, openp: i64) -> i64
1656func _lex_src_eq(L: *Lex, needle: *u8, n: i64) -> i64
1667func _lex_src_eq_at(L: *Lex, at: i64, needle: *u8, n: i64) -> i64
1678func _lex_skip_to_eol(L: *Lex) -> i64
1687func _lex_skip_to_matching_endif(L: *Lex) -> i64
1728func _lex_pp_defined(L: *Lex, name_pos: i64) -> i64
1749func ppb_init() -> i64
1760func ppb_push(v: i64) -> i64
1769func ppb_pop() -> i64
1778func ppb_top_taken() -> i64
1786func ppb_set_top(v: i64) -> i64
1795func ppb_reset() -> i64
1804func _lex_pp_kw_at(L: *Lex, at: i64) -> i64
1818func _lex_skip_to_next_branch(L: *Lex) -> i64
1846func _lex_handle_preprocessor(L: *Lex) -> i64
1978func lex_scr() -> *u8
1984func lex_scr_macroval() -> *i64
1992func skip_ws_comments(L: *Lex) -> i64
2030func lex_ident_or_kw(L: *Lex) -> i64
2091func lex_number(L: *Lex) -> i64
2292func lex_string(L: *Lex) -> i64
2407func emit_punct(L: *Lex, kind: i64, len: i64) -> i64
2434func lex_source(src: *u8, cap: i64) -> *Tok