code wiki / (root) / nx_tokenizer.nx

nx_tokenizer.nx

buildroot/runtime/nx_tokenizer.nx

102335 B2464 linesdepth 2pulls 4 transitivereach 36 importersview sourcekind library
docsdependenciesstructsconstsfunctions

about

lex.nx -- NishiLang source tokenizer, ported from lex.c. Accepts a null-terminated source buffer and produces a stream of Tok records. Mirrors lex.c's token layout (kind + line/col + either int_val or text[]). Same keyword set. Single-pass; zero allocations per token beyond the growing token array. Extends at 2x when full. Whitespace + // comments are skipped. Numbers support 0x/0b/0o prefixes with underscore separators. String literals read bytes until closing `"`.

dependencies 3 imports · 36 importers

nx_syscalls.nx nx_types.nx nx_lex_kinds.nx nx_tokenizer.nx end2end_test.nx nx_compile_wat.nx nx_compile_x86.nx nx_dom_oracle_gate.nx nx_lang_struct.nx nx_main.nx nx_nxc.nx nx_quality_grade_daemon.nx nx_quality_grade_quad.nx nx_quality_grade_self.nx

diagram shows first 10 each side; +0 more imports, +26 more importers in the complete lists below.

imports: nx_syscalls.nxnx_types.nxnx_lex_kinds.nx

imported by: end2end_test.nxnx_compile_wat.nxnx_compile_x86.nxnx_dom_oracle_gate.nxnx_lang_struct.nxnx_main.nxnx_nxc.nxnx_quality_grade_daemon.nxnx_quality_grade_quad.nxnx_quality_grade_self.nxnx_quality_grade_septagon.nxnx_smoke_lex2.nxnx_smoke_lex3.nxnx_smoke_lex4.nxnx_smoke_lex5.nxnx_smoke_lex6.nxnx_smoke_lex7.nxnx_smoke_lex_min.nxnx_smoke_nxc_pipeline.nxnx_smoke_nxc_pipeline10.nxnx_smoke_nxc_pipeline11.nxnx_smoke_nxc_pipeline12.nxnx_smoke_nxc_pipeline13.nxnx_smoke_nxc_pipeline14.nxnx_smoke_nxc_pipeline15.nxnx_smoke_nxc_pipeline2.nxnx_smoke_nxc_pipeline3.nxnx_smoke_nxc_pipeline4.nxnx_smoke_nxc_pipeline5.nxnx_smoke_nxc_pipeline6.nxnx_smoke_nxc_pipeline7.nxnx_smoke_nxc_pipeline8.nxnx_smoke_nxc_pipeline9.nxnx_smoke_nxc_tilde.nxnx_vcc_color_wat.nxnx_vcc_probe_wat.nx

structs

122struct Lex

consts

52const LEXM_MAGIC_65536: i64 = 65536
189const LEXM_MAX: i64 = 256
190const LEXM_NAMELEN: i64 = 64
194const LEXM_BODYLEN: i64 = 1024
519const LEXM_MAXARGS: i64 = 8
535const LEX_IDENT_MAX: i64 = 63
1698const PPB_MAX: i64 = 32

functions

61func is_alpha(c: i64) -> i64
called by 3: is_alnumlexm_is_ident_textlex_source
67func is_digit(c: i64) -> i64
called by 4: is_hexdigitis_alnumlex_numberlex_source
71func is_hexdigit(c: i64) -> i64
called by 1: lex_number calls 1: is_digit
77func hex_val(c: i64) -> i64
called by 1: lex_number
83func is_alnum(c: i64) -> i64
called by 5: lexm_isid_lex_pp_defined_lex_handle_preprocessorlex_ident_or_kwlex_source calls 2: is_alphais_digit
93func keyword_lookup(t: *u8, len: i64) -> i64
called by 1: lex_ident_or_kw calls 1: streq_n
135func push_tok(L: *Lex, t: *Tok) -> i64
called by 5: lex_ident_or_kwlex_numberlex_stringemit_punctlex_source
156func advance(L: *Lex) -> i64
called by 8: _lex_skip_to_eol_lex_skip_to_matching_endifskip_ws_commentslex_ident_or_kwlex_numberlex_string+2
169func peek(L: *Lex) -> i64
called by 8: _lex_skip_to_eol_lex_skip_to_matching_endif_lex_handle_preprocessorskip_ws_commentslex_ident_or_kwlex_number+2
173func peek2(L: *Lex) -> i64
called by 3: skip_ws_commentslex_numberlex_source
201func lexm_init() -> i64
called by 2: lexm_definelexm_reset calls 1: sys_mmap
211func lexm_emit_str(out: *u8, o0: i64, s: *u8) -> i64
called by 5: lexm_warn_body_too_longlexm_warn_too_many_argslex_warn_ident_too_longlexm_substlexm_warn_bad_paste
218func lexm_emit_dec(out: *u8, o0: i64, v: i64) -> i64
called by 5: lexm_warn_body_too_longlexm_warn_too_many_argslex_warn_ident_too_longlexm_substlexm_for_expand calls 1: sys_mmap
230func lexm_strlen(s: *u8) -> i64
called by 2: lexm_subst_lex_handle_preprocessor
240func lexm_body_end(src: *u8, n: i64, bp: i64) -> i64
called by 5: lexm_substlexm_prescanlexm_for_expandlexm_expand_lex_handle_preprocessor
317func lexm_name_at(i: i64) -> *u8 { return ((lexm_names as i64) + i * LEXM_NAMELEN) as *u8 }
called by 3: lexm_definelexm_lookuplexm_undef
318func lexm_body_at(i: i64) -> *u8 { return ((lexm_bodies as i64) + i * LEXM_BODYLEN) as *u8 }
called by 3: lexm_definelexm_lookuplexm_undef
331func lexm_diag_cell() -> *u8
called by 4: lexm_warn_body_too_longlexm_warn_too_many_argslex_warn_ident_too_longlexm_warn_bad_paste calls 1: sys_mmap
338func lexm_warn_body_too_long(name: *u8, nlen: i64, blen: i64) -> i64
called by 1: lexm_define calls 4: lexm_diag_celllexm_emit_strlexm_emit_decsys_write
359func lexm_define(name: *u8, nlen: i64, body: *u8, blen: i64) -> i64
called by 3: lexm_predefinelexm_prescan_lex_handle_preprocessor calls 4: lexm_initlexm_warn_body_too_longlexm_name_atlexm_body_at
395func lexm_lookup(name: *u8, nlen: i64) -> *u8
called by 5: lexm_for_boundlexm_expandppe_primary_lex_pp_definedlex_ident_or_kw calls 2: lexm_name_atlexm_body_at
415func lexm_undef(name: *u8, nlen: i64) -> i64
called by 1: _lex_handle_preprocessor calls 2: lexm_name_atlexm_body_at
460func lexm_predefine() -> i64
called by 1: lexm_reset calls 1: lexm_define
468func lexm_reset() -> i64
called by 1: lex_source calls 2: lexm_initlexm_predefine
480func lexm_body_int(body: *u8, out: *i64) -> i64
called by 3: lexm_for_boundppe_primarylex_ident_or_kw
521func lexm_warn_too_many_args(got: i64) -> i64
called by 2: lexm_substlexm_expand calls 4: lexm_diag_celllexm_emit_strlexm_emit_decsys_write
537func lex_warn_ident_too_long(text: *u8, kept: i64, actual: i64) -> i64
called by 1: lex_ident_or_kw calls 4: lexm_diag_celllexm_emit_strlexm_emit_decsys_write
555func lexm_isid(c: i64) -> i64 { return is_alnum(c) }
called by 7: lexm_substlexm_prescanlexm_for_emitlexm_is_ident_textlexm_for_expandlexm_expand+1 calls 1: is_alnum
559func lexm_subst(body: *u8, asrc: *u8, argp: *i64, argl: *i64, nargs: i64, out: *u8, o0: i64) -> i64
called by 1: lexm_expand calls 7: sys_mmaplexm_isidlexm_warn_too_many_argslexm_strlenlexm_body_endlexm_emit_str+1
733func lexm_prescan(src: *u8, n: i64) -> i64
called by 1: lexm_expand calls 3: lexm_isidlexm_body_endlexm_define
778func lexm_for_bound(src: *u8, s0: i64, e0: i64, out: *i64) -> i64
called by 1: lexm_for_expand calls 2: lexm_lookuplexm_body_int
812func lexm_for_emit(src: *u8, bs: i64, be: i64, vname: *u8, vlen: i64, vtxt: *u8, vtl: i64, out: *u8, o0: i64, ocap: i64) -> i64
called by 1: lexm_for_expand calls 1: lexm_isid
881func lexm_trim(src: *u8, s: i64, e: i64, eout: *i64) -> i64
called by 1: lexm_paste_expand
904func lexm_is_ident_text(p: *u8, s: i64, e: i64) -> i64
called by 1: lexm_paste_expand calls 2: is_alphalexm_isid
922func lexm_warn_bad_paste(out: *u8, s: i64, e: i64) -> i64
called by 1: lexm_paste_expand calls 3: lexm_diag_celllexm_emit_strsys_write
938func lexm_paste_expand(src: *u8, n: i64, istart: i64, kwlen: i64, mode: i64, out: *u8, o0: i64, ocap: i64, iout: *i64) -> i64
called by 1: lexm_expand calls 4: sys_mmaplexm_trimlexm_is_ident_textlexm_warn_bad_paste
1000func lexm_kw_is(src: *u8, n: i64, at: i64, kw: *u8, kwlen: i64) -> i64
called by 1: lexm_expand
1013func lexm_for_expand(src: *u8, n: i64, istart: i64, out: *u8, o0: i64, ocap: i64, iout: *i64) -> i64
called by 1: lexm_expand calls 6: lexm_isidlexm_body_endlexm_for_emitsys_mmaplexm_for_boundlexm_emit_dec
1139func lexm_expand(src: *u8, n: i64, out: *u8, outcap: i64) -> i64
called by 1: lex_source calls 10: lexm_prescansys_mmaplexm_isidlexm_body_endlexm_for_expandlexm_kw_is+4
1317func ppe_or(src: *u8, n: i64, pos: *i64) -> i64;
called by 2: ppe_primaryppe_eval_at
1325func ppe_scratch_cell() -> *i64
called by 1: ppe_primary calls 1: sys_mmap
1330func ppe_skipws(src: *u8, n: i64, pos: *i64) -> i64
called by 7: ppe_primaryppe_unaryppe_mulppe_addppe_cmpppe_and+1
1344func ppe_primary(src: *u8, n: i64, pos: *i64) -> i64
called by 1: ppe_unary calls 6: ppe_skipwsppe_orlexm_isidlexm_lookupppe_scratch_celllexm_body_int
1441func ppe_unary(src: *u8, n: i64, pos: *i64) -> i64
called by 2: ppe_unaryppe_mul calls 3: ppe_skipwsppe_unaryppe_primary
1465func ppe_mul(src: *u8, n: i64, pos: *i64) -> i64
called by 1: ppe_add calls 2: ppe_unaryppe_skipws
1494func ppe_add(src: *u8, n: i64, pos: *i64) -> i64
called by 1: ppe_cmp calls 2: ppe_mulppe_skipws
1509func ppe_cmp(src: *u8, n: i64, pos: *i64) -> i64
called by 1: ppe_and calls 2: ppe_addppe_skipws
1554func ppe_and(src: *u8, n: i64, pos: *i64) -> i64
called by 1: ppe_or calls 2: ppe_cmpppe_skipws
1576func ppe_or(src: *u8, n: i64, pos: *i64) -> i64
calls 2: ppe_andppe_skipws
1601func ppe_eval_at(src: *u8, n: i64, openp: i64) -> i64
called by 1: _lex_handle_preprocessor calls 2: sys_mmapppe_or
1610func _lex_src_eq(L: *Lex, needle: *u8, n: i64) -> i64
1621func _lex_src_eq_at(L: *Lex, at: i64, needle: *u8, n: i64) -> i64
called by 5: _lex_skip_to_matching_endif_lex_pp_defined_lex_pp_kw_at_lex_handle_preprocessorlex_source
1632func _lex_skip_to_eol(L: *Lex) -> i64
called by 2: _lex_skip_to_matching_endif_lex_handle_preprocessor calls 2: peekadvance
1641func _lex_skip_to_matching_endif(L: *Lex) -> i64
called by 1: _lex_handle_preprocessor calls 4: peek_lex_src_eq_at_lex_skip_to_eoladvance
1682func _lex_pp_defined(L: *Lex, name_pos: i64) -> i64
called by 1: _lex_handle_preprocessor calls 3: is_alnum_lex_src_eq_atlexm_lookup
1703func ppb_init() -> i64
called by 5: ppb_pushppb_popppb_top_takenppb_set_topppb_reset calls 1: sys_mmap
1714func ppb_push(v: i64) -> i64
called by 1: _lex_handle_preprocessor calls 1: ppb_init
1723func ppb_pop() -> i64
called by 1: _lex_handle_preprocessor calls 1: ppb_init
1732func ppb_top_taken() -> i64
called by 1: _lex_handle_preprocessor calls 1: ppb_init
1740func ppb_set_top(v: i64) -> i64
called by 1: _lex_handle_preprocessor calls 1: ppb_init
1749func ppb_reset() -> i64
called by 1: lex_source calls 1: ppb_init
1758func _lex_pp_kw_at(L: *Lex, at: i64) -> i64
called by 1: _lex_skip_to_next_branch calls 1: _lex_src_eq_at
1772func _lex_skip_to_next_branch(L: *Lex) -> i64
called by 1: _lex_handle_preprocessor calls 1: _lex_pp_kw_at
1800func _lex_handle_preprocessor(L: *Lex) -> i64
called by 1: lex_source calls 16: peek_lex_src_eq_atis_alnumlexm_strlenlexm_body_endlexm_define+10
1915func skip_ws_comments(L: *Lex) -> i64
called by 1: lex_source calls 3: peekadvancepeek2
1953func lex_ident_or_kw(L: *Lex) -> i64
called by 1: lex_source calls 10: sys_mmaptok_text_ptrpeekis_alnumadvancelex_warn_ident_too_long+4
2010func lex_number(L: *Lex) -> i64
called by 1: lex_source calls 8: sys_mmappeekpeek2advanceis_hexdigithex_val+2
2164func lex_string(L: *Lex) -> i64
called by 1: lex_source calls 5: sys_mmapadvancetok_text_ptrpeekpush_tok
2274func emit_punct(L: *Lex, kind: i64, len: i64) -> i64
called by 1: lex_source calls 3: sys_mmapadvancepush_tok
2297func lex_source(src: *u8, cap: i64) -> *Tok
called by 35: run_srcmainmainmainmaincompile+29 calls 18: sys_mmaplexm_expandlexm_resetppb_resetskip_ws_commentspeek+12