nx_f32_bpe_load_test.nx source

1// nx_f32_bpe_load_test.nx -- smoke for nx_f32_bpe_load.nx. 2// 3// Builds a synthetic GGUF with tokenizer.ggml.tokens and 4// tokenizer.ggml.merges arrays. Verifies the loader populates 5// the vocab + merge rules. 6 7import "nx_syscalls.nx" 8import "nx_tier.nx" 9import "nx_le.nx" 10import "nx_intern.nx" 11import "nx_bpe.nx" 12import "nx_gguf.nx" 13import "nx_gguf_load.nx" 14import "nx_gguf_meta.nx" 15import "nx_f32_bpe_load.nx" 16 17func _write_str_at(buf: *u8, p_in: i64, s: *u8, s_len: i64) -> i64 { 18 var p: i64 = p_in 19 nx_le_write_u64(buf, p, s_len); p = p + 8 20 var i: i64 = 0 21 while i < s_len { buf[p + i] = s[i]; i = i + 1 } 22 return p + s_len 23} 24 25func main() -> i64 { 26 var vi: nx_int = 0 27 while vi < NX_FBL_N_VERDICTS { 28 if nx_fbl_verdict_is_valid(vi) != 1 { return 5 + vi } 29 vi = vi + 1 30 } 31 32 let buf: *u8 = sys_mmap(2048) 33 buf[0]=0x47 as u8; buf[1]=0x47 as u8; buf[2]=0x55 as u8; buf[3]=0x46 as u8 34 nx_le_write_u32(buf, 4, 3) 35 nx_le_write_u64(buf, 8, 0) // 0 tensors 36 nx_le_write_u64(buf, 16, 2) // 2 metadata entries 37 38 var p: i64 = 24 39 40 // ===== Entry 1: tokenizer.ggml.tokens = ["a", "b", "c", "ab", "cb"] ===== 41 let k_tok: *u8 = sys_mmap(21) 42 k_tok[0]=0x74 as u8; k_tok[1]=0x6f as u8; k_tok[2]=0x6b as u8; k_tok[3]=0x65 as u8 43 k_tok[4]=0x6e as u8; k_tok[5]=0x69 as u8; k_tok[6]=0x7a as u8; k_tok[7]=0x65 as u8 44 k_tok[8]=0x72 as u8; k_tok[9]=0x2e as u8; k_tok[10]=0x67 as u8; k_tok[11]=0x67 as u8 45 k_tok[12]=0x6d as u8; k_tok[13]=0x6c as u8; k_tok[14]=0x2e as u8; k_tok[15]=0x74 as u8 46 k_tok[16]=0x6f as u8; k_tok[17]=0x6b as u8; k_tok[18]=0x65 as u8; k_tok[19]=0x6e as u8 47 k_tok[20]=0x73 as u8 48 49 nx_le_write_u64(buf, p, 21); p = p + 8 50 var ki: i64 = 0 51 while ki < 21 { buf[p + ki] = k_tok[ki]; ki = ki + 1 } 52 p = p + 21 53 nx_le_write_u32(buf, p, NX_GGUF_TYPE_ARRAY); p = p + 4 54 nx_le_write_u32(buf, p, NX_GGUF_TYPE_STRING); p = p + 4 // inner type 55 nx_le_write_u64(buf, p, 5); p = p + 8 // count 56 57 let sa: *u8 = sys_mmap(1); sa[0] = 0x61 as u8 58 let sb: *u8 = sys_mmap(1); sb[0] = 0x62 as u8 59 let sc: *u8 = sys_mmap(1); sc[0] = 0x63 as u8 60 let sab: *u8 = sys_mmap(2); sab[0] = 0x61 as u8; sab[1] = 0x62 as u8 61 let scb: *u8 = sys_mmap(2); scb[0] = 0x63 as u8; scb[1] = 0x62 as u8 62 63 p = _write_str_at(buf, p, sa, 1) 64 p = _write_str_at(buf, p, sb, 1) 65 p = _write_str_at(buf, p, sc, 1) 66 p = _write_str_at(buf, p, sab, 2) 67 p = _write_str_at(buf, p, scb, 2) 68 69 // ===== Entry 2: tokenizer.ggml.merges = ["a b", "c b"] ===== 70 let k_mrg: *u8 = sys_mmap(21) 71 k_mrg[0]=0x74 as u8; k_mrg[1]=0x6f as u8; k_mrg[2]=0x6b as u8; k_mrg[3]=0x65 as u8 72 k_mrg[4]=0x6e as u8; k_mrg[5]=0x69 as u8; k_mrg[6]=0x7a as u8; k_mrg[7]=0x65 as u8 73 k_mrg[8]=0x72 as u8; k_mrg[9]=0x2e as u8; k_mrg[10]=0x67 as u8; k_mrg[11]=0x67 as u8 74 k_mrg[12]=0x6d as u8; k_mrg[13]=0x6c as u8; k_mrg[14]=0x2e as u8; k_mrg[15]=0x6d as u8 75 k_mrg[16]=0x65 as u8; k_mrg[17]=0x72 as u8; k_mrg[18]=0x67 as u8; k_mrg[19]=0x65 as u8 76 k_mrg[20]=0x73 as u8 77 78 nx_le_write_u64(buf, p, 21); p = p + 8 79 var ki2: i64 = 0 80 while ki2 < 21 { buf[p + ki2] = k_mrg[ki2]; ki2 = ki2 + 1 } 81 p = p + 21 82 nx_le_write_u32(buf, p, NX_GGUF_TYPE_ARRAY); p = p + 4 83 nx_le_write_u32(buf, p, NX_GGUF_TYPE_STRING); p = p + 4 84 nx_le_write_u64(buf, p, 2); p = p + 8 85 86 let m_ab: *u8 = sys_mmap(3); m_ab[0]=0x61 as u8; m_ab[1]=0x20 as u8; m_ab[2]=0x62 as u8 87 let m_cb: *u8 = sys_mmap(3); m_cb[0]=0x63 as u8; m_cb[1]=0x20 as u8; m_cb[2]=0x62 as u8 88 p = _write_str_at(buf, p, m_ab, 3) 89 p = _write_str_at(buf, p, m_cb, 3) 90 91 // ===== Parse + load ===== 92 let hdr: *NxGgufHeader = sys_mmap(NX_GGUF_HDR_BYTES) as *NxGgufHeader 93 let v_p: nx_int = nx_gguf_parse(buf, 2048, hdr) 94 if v_p != NX_GGUF_OK { return 20 + v_p } 95 96 let vocab: *NxBpeVocab = nx_bpe_vocab_new(4096, 1024, 16) 97 let n_tok_out: *i64 = sys_mmap(8) as *i64 98 let n_mrg_out: *i64 = sys_mmap(8) as *i64 99 let out_err: *i64 = sys_mmap(8) as *i64 100 101 let v: nx_int = nx_f32_bpe_load_from_gguf(buf, 2048, hdr, vocab, 102 n_tok_out, n_mrg_out, out_err) 103 if v != NX_FBL_OK { return 30 + v } 104 105 if n_tok_out[0] != 5 { return 50 } 106 if n_mrg_out[0] != 2 { return 51 } 107 108 // Verify tokens were interned: lookup should return same ids. 109 let id_a: i64 = nx_bpe_add_token(vocab, sa, 1) 110 let id_b: i64 = nx_bpe_add_token(vocab, sb, 1) 111 let id_c: i64 = nx_bpe_add_token(vocab, sc, 1) 112 let id_ab: i64 = nx_bpe_add_token(vocab, sab, 2) 113 let id_cb: i64 = nx_bpe_add_token(vocab, scb, 2) 114 if id_a != 0 { return 60 } 115 if id_b != 1 { return 61 } 116 if id_c != 2 { return 62 } 117 if id_ab != 3 { return 63 } 118 if id_cb != 4 { return 64 } 119 120 // Verify merge rules registered. 121 if vocab.n_merges != 2 { return 70 } 122 123 return 0 124}