nx_f32_llm_special_tokens_test.nx source
↩ module page · 102 lines · 4511 B
1// nx_f32_llm_special_tokens_test.nx -- smoke for nx_f32_llm_special_tokens.nx.
2//
3// Builds a synthetic GGUF with:
4// tokenizer.ggml.bos_token_id U32 1
5// tokenizer.ggml.eos_token_id U32 2
6// tokenizer.ggml.padding_token_id U32 0
7// Verifies all three readers return the expected ids; missing keys
8// return -1.
9
10import "nx_syscalls.nx"
11import "nx_tier.nx"
12import "nx_le.nx"
13import "nx_gguf.nx"
14import "nx_gguf_load.nx"
15import "nx_gguf_meta.nx"
16import "nx_f32_llm_special_tokens.nx"
17
18func _write_meta_u32(buf: *u8, p_in: i64,
19 key: *u8, key_len: i64, value: i64) -> i64 {
20 var p: i64 = p_in
21 nx_le_write_u64(buf, p, key_len); p = p + 8
22 var i: i64 = 0
23 while i < key_len { buf[p + i] = key[i]; i = i + 1 }
24 p = p + key_len
25 nx_le_write_u32(buf, p, NX_GGUF_TYPE_UINT32); p = p + 4
26 nx_le_write_u32(buf, p, value); p = p + 4
27 return p
28}
29
30func main() -> i64 {
31 let buf: *u8 = sys_mmap(2048)
32 buf[0]=0x47 as u8; buf[1]=0x47 as u8; buf[2]=0x55 as u8; buf[3]=0x46 as u8
33 nx_le_write_u32(buf, 4, 3)
34 nx_le_write_u64(buf, 8, 0) // 0 tensors
35 nx_le_write_u64(buf, 16, 3) // 3 metadata entries
36
37 var p: i64 = 24
38
39 // bos_token_id = 1
40 let k_bos: *u8 = sys_mmap(27)
41 k_bos[0]=0x74 as u8; k_bos[1]=0x6f as u8; k_bos[2]=0x6b as u8; k_bos[3]=0x65 as u8
42 k_bos[4]=0x6e as u8; k_bos[5]=0x69 as u8; k_bos[6]=0x7a as u8; k_bos[7]=0x65 as u8
43 k_bos[8]=0x72 as u8; k_bos[9]=0x2e as u8; k_bos[10]=0x67 as u8; k_bos[11]=0x67 as u8
44 k_bos[12]=0x6d as u8; k_bos[13]=0x6c as u8; k_bos[14]=0x2e as u8; k_bos[15]=0x62 as u8
45 k_bos[16]=0x6f as u8; k_bos[17]=0x73 as u8; k_bos[18]=0x5f as u8; k_bos[19]=0x74 as u8
46 k_bos[20]=0x6f as u8; k_bos[21]=0x6b as u8; k_bos[22]=0x65 as u8; k_bos[23]=0x6e as u8
47 k_bos[24]=0x5f as u8; k_bos[25]=0x69 as u8; k_bos[26]=0x64 as u8
48 p = _write_meta_u32(buf, p, k_bos, 27, 1)
49
50 // eos_token_id = 2
51 let k_eos: *u8 = sys_mmap(27)
52 k_eos[0]=0x74 as u8; k_eos[1]=0x6f as u8; k_eos[2]=0x6b as u8; k_eos[3]=0x65 as u8
53 k_eos[4]=0x6e as u8; k_eos[5]=0x69 as u8; k_eos[6]=0x7a as u8; k_eos[7]=0x65 as u8
54 k_eos[8]=0x72 as u8; k_eos[9]=0x2e as u8; k_eos[10]=0x67 as u8; k_eos[11]=0x67 as u8
55 k_eos[12]=0x6d as u8; k_eos[13]=0x6c as u8; k_eos[14]=0x2e as u8; k_eos[15]=0x65 as u8
56 k_eos[16]=0x6f as u8; k_eos[17]=0x73 as u8; k_eos[18]=0x5f as u8; k_eos[19]=0x74 as u8
57 k_eos[20]=0x6f as u8; k_eos[21]=0x6b as u8; k_eos[22]=0x65 as u8; k_eos[23]=0x6e as u8
58 k_eos[24]=0x5f as u8; k_eos[25]=0x69 as u8; k_eos[26]=0x64 as u8
59 p = _write_meta_u32(buf, p, k_eos, 27, 2)
60
61 // padding_token_id = 0
62 let k_pad: *u8 = sys_mmap(31)
63 k_pad[0]=0x74 as u8; k_pad[1]=0x6f as u8; k_pad[2]=0x6b as u8; k_pad[3]=0x65 as u8
64 k_pad[4]=0x6e as u8; k_pad[5]=0x69 as u8; k_pad[6]=0x7a as u8; k_pad[7]=0x65 as u8
65 k_pad[8]=0x72 as u8; k_pad[9]=0x2e as u8; k_pad[10]=0x67 as u8; k_pad[11]=0x67 as u8
66 k_pad[12]=0x6d as u8; k_pad[13]=0x6c as u8; k_pad[14]=0x2e as u8; k_pad[15]=0x70 as u8
67 k_pad[16]=0x61 as u8; k_pad[17]=0x64 as u8; k_pad[18]=0x64 as u8; k_pad[19]=0x69 as u8
68 k_pad[20]=0x6e as u8; k_pad[21]=0x67 as u8; k_pad[22]=0x5f as u8; k_pad[23]=0x74 as u8
69 k_pad[24]=0x6f as u8; k_pad[25]=0x6b as u8; k_pad[26]=0x65 as u8; k_pad[27]=0x6e as u8
70 k_pad[28]=0x5f as u8; k_pad[29]=0x69 as u8; k_pad[30]=0x64 as u8
71 p = _write_meta_u32(buf, p, k_pad, 31, 0)
72
73 let hdr: *NxGgufHeader = sys_mmap(NX_GGUF_HDR_BYTES) as *NxGgufHeader
74 let v_p: nx_int = nx_gguf_parse(buf, 2048, hdr)
75 if v_p != NX_GGUF_OK { return 10 + v_p }
76
77 let bos: nx_int = nx_f32_llm_read_bos(buf, 2048, hdr)
78 if bos != 1 { return 20 }
79
80 let eos: nx_int = nx_f32_llm_read_eos(buf, 2048, hdr)
81 if eos != 2 { return 21 }
82
83 let pad: nx_int = nx_f32_llm_read_pad(buf, 2048, hdr)
84 if pad != 0 { return 22 }
85
86 // ===== Missing keys -> -1 =====
87 let buf2: *u8 = sys_mmap(64)
88 buf2[0]=0x47 as u8; buf2[1]=0x47 as u8; buf2[2]=0x55 as u8; buf2[3]=0x46 as u8
89 nx_le_write_u32(buf2, 4, 3)
90 nx_le_write_u64(buf2, 8, 0)
91 nx_le_write_u64(buf2, 16, 0) // 0 metadata
92
93 let hdr2: *NxGgufHeader = sys_mmap(NX_GGUF_HDR_BYTES) as *NxGgufHeader
94 let v_p2: nx_int = nx_gguf_parse(buf2, 64, hdr2)
95 if v_p2 != NX_GGUF_OK { return 30 + v_p2 }
96
97 if nx_f32_llm_read_bos(buf2, 64, hdr2) != -1 { return 40 }
98 if nx_f32_llm_read_eos(buf2, 64, hdr2) != -1 { return 41 }
99 if nx_f32_llm_read_pad(buf2, 64, hdr2) != -1 { return 42 }
100
101 return 0
102}