nx_nlex_test.nx source

1// Test the NishiLang-in-NishiLang tokenizer on a small program. 2 3import "syscalls.nx" 4import "nx_nlex.nx" 5 6func write_str(buf: *u8, off: i64, s0: i64, n: i64) -> i64 { 7 buf[off] = s0 as u8 8 return 0 9} 10 11func main() -> i64 { 12 // Source: `func main() -> i64 { return 42 }` 13 let src: *u8 = sys_mmap(64) 14 src[0] = 102 as u8 // f 15 src[1] = 117 as u8 // u 16 src[2] = 110 as u8 // n 17 src[3] = 99 as u8 // c 18 src[4] = 32 as u8 // space 19 src[5] = 109 as u8 // m 20 src[6] = 97 as u8 // a 21 src[7] = 105 as u8 // i 22 src[8] = 110 as u8 // n 23 src[9] = 40 as u8 // ( 24 src[10] = 41 as u8 // ) 25 src[11] = 32 as u8 // space 26 src[12] = 45 as u8 // - 27 src[13] = 62 as u8 // > 28 src[14] = 32 as u8 // space 29 src[15] = 105 as u8 // i 30 src[16] = 54 as u8 // 6 31 src[17] = 52 as u8 // 4 32 src[18] = 32 as u8 // space 33 src[19] = 123 as u8 // { 34 src[20] = 32 as u8 // space 35 src[21] = 114 as u8 // r 36 src[22] = 101 as u8 // e 37 src[23] = 116 as u8 // t 38 src[24] = 117 as u8 // u 39 src[25] = 114 as u8 // r 40 src[26] = 110 as u8 // n 41 src[27] = 32 as u8 // space 42 src[28] = 52 as u8 // 4 43 src[29] = 50 as u8 // 2 44 src[30] = 32 as u8 // space 45 src[31] = 125 as u8 // } 46 47 let l: *NxNLexer = nx_nlex_new(src, 32, 32) 48 let n: i64 = nx_nlex_run(l) 49 50 // Expected tokens: FUNC IDENT(main) LPAREN RPAREN ARROW IDENT(i64) LBRACE RETURN INT(42) RBRACE EOF 51 // That's 11 tokens. 52 if n != 11 { return 10 } 53 54 let t0: *NxToken = nx_nlex_token_at(l, 0) 55 if t0.kind != NX_TOK_FUNC { return 11 } 56 57 let t1: *NxToken = nx_nlex_token_at(l, 1) 58 if t1.kind != NX_TOK_IDENT { return 12 } 59 if t1.len != 4 { return 13 } // "main" 60 61 let t2: *NxToken = nx_nlex_token_at(l, 2) 62 if t2.kind != NX_TOK_LPAREN { return 14 } 63 64 let t3: *NxToken = nx_nlex_token_at(l, 3) 65 if t3.kind != NX_TOK_RPAREN { return 15 } 66 67 let t4: *NxToken = nx_nlex_token_at(l, 4) 68 if t4.kind != NX_TOK_ARROW { return 16 } 69 70 let t5: *NxToken = nx_nlex_token_at(l, 5) 71 if t5.kind != NX_TOK_IDENT { return 17 } 72 if t5.len != 3 { return 18 } // "i64" 73 74 let t6: *NxToken = nx_nlex_token_at(l, 6) 75 if t6.kind != NX_TOK_LBRACE { return 19 } 76 77 let t7: *NxToken = nx_nlex_token_at(l, 7) 78 if t7.kind != NX_TOK_RETURN { return 20 } 79 80 let t8: *NxToken = nx_nlex_token_at(l, 8) 81 if t8.kind != NX_TOK_INT { return 21 } 82 if t8.int_val != 42 { return 22 } 83 84 let t9: *NxToken = nx_nlex_token_at(l, 9) 85 if t9.kind != NX_TOK_RBRACE { return 23 } 86 87 let t10: *NxToken = nx_nlex_token_at(l, 10) 88 if t10.kind != NX_TOK_EOF { return 24 } 89 90 return 0 91}