nx_f32_bpe_load.nx source
↩ module page · 182 lines · 6694 B
1// nx_f32_bpe_load.nx -- load BPE tokenizer vocabulary from GGUF.
2//
3// Walks the GGUF metadata section for tokenizer.ggml.tokens (the
4// vocabulary as an array of strings) and tokenizer.ggml.merges (BPE
5// merge rules), populating an NxBpeVocab ready for nx_bpe_encode /
6// nx_bpe_decode.
7//
8// This closes the last 'text-in' gap to running real Qwen2.5 /
9// Llama / etc. inference end-to-end on the f32 substrate.
10//
11// genealogy_id: ggml_tokenizer_metadata_keys + gpt2_byte_bpe_spec
12// lineage_id: substrate_f32_bpe_load_v1
13
14import "nx_syscalls.nx"
15import "nx_tier.nx"
16import "nx_le.nx"
17import "nx_intern.nx"
18import "nx_bpe.nx"
19import "nx_gguf.nx"
20import "nx_gguf_load.nx"
21import "nx_gguf_meta.nx"
22
23const NX_FBL_OK: nx_int = 0
24const NX_FBL_ERR_NULL: nx_int = 1
25const NX_FBL_ERR_NO_TOKENS: nx_int = 2
26const NX_FBL_ERR_NOT_ARRAY: nx_int = 3
27const NX_FBL_ERR_NOT_STRING: nx_int = 4
28const NX_FBL_ERR_OOM: nx_int = 5
29const NX_FBL_N_VERDICTS: nx_int = 6
30
31func nx_fbl_verdict_is_valid(v: nx_int) -> nx_int {
32 if v < 0 { return 0 }
33 if v >= NX_FBL_N_VERDICTS { return 0 }
34 return 1
35}
36
37// Load tokens array. vocab_out must be pre-allocated with adequate caps.
38// Returns OK or error verdict.
39
40func _fbl_load_tokens(buf: *u8, len: i64, hdr: *NxGgufHeader,
41 vocab: *NxBpeVocab,
42 out_count: *i64) -> nx_int {
43 // Key: "tokenizer.ggml.tokens" (21 bytes)
44 let key: *u8 = sys_mmap(21)
45 key[0]=0x74 as u8; key[1]=0x6f as u8; key[2]=0x6b as u8; key[3]=0x65 as u8
46 key[4]=0x6e as u8; key[5]=0x69 as u8; key[6]=0x7a as u8; key[7]=0x65 as u8
47 key[8]=0x72 as u8; key[9]=0x2e as u8; key[10]=0x67 as u8; key[11]=0x67 as u8
48 key[12]=0x6d as u8; key[13]=0x6c as u8; key[14]=0x2e as u8; key[15]=0x74 as u8
49 key[16]=0x6f as u8; key[17]=0x6b as u8; key[18]=0x65 as u8; key[19]=0x6e as u8
50 key[20]=0x73 as u8
51
52 let off_out: *i64 = sys_mmap(8) as *i64
53 let typ_out: *i64 = sys_mmap(8) as *i64
54 let v: nx_int = nx_gguf_meta_find(buf, len, hdr, key, 21, off_out, typ_out)
55 if v != NX_GMETA_OK { return NX_FBL_ERR_NO_TOKENS }
56 if typ_out[0] != NX_GGUF_TYPE_ARRAY { return NX_FBL_ERR_NOT_ARRAY }
57
58 let inner: i64 = nx_gguf_meta_array_inner_type(buf, off_out[0])
59 if inner != NX_GGUF_TYPE_STRING { return NX_FBL_ERR_NOT_STRING }
60
61 let count: i64 = nx_gguf_meta_array_count(buf, off_out[0])
62 out_count[0] = count
63
64 var elt_off: i64 = nx_gguf_meta_array_first_elt_off(buf, off_out[0])
65 var i: i64 = 0
66 while i < count {
67 let s_len: i64 = nx_le_read_u64(buf, elt_off)
68 let s_ptr: *u8 = ((buf as i64) + elt_off + 8) as *u8
69 nx_bpe_add_token(vocab, s_ptr, s_len)
70 elt_off = elt_off + 8 + s_len
71 i = i + 1
72 }
73 return NX_FBL_OK
74}
75
76// Load merges array (optional). Each merge string is "PART_A PART_B"
77// with a single ASCII-space separator. Looks up part_a, part_b, and
78// merged (= concatenation) by intern, then registers via nx_bpe_add_merge.
79
80func _fbl_load_merges(buf: *u8, len: i64, hdr: *NxGgufHeader,
81 vocab: *NxBpeVocab,
82 out_count: *i64) -> nx_int {
83 out_count[0] = 0
84
85 // Key: "tokenizer.ggml.merges" (21 bytes)
86 let key: *u8 = sys_mmap(21)
87 key[0]=0x74 as u8; key[1]=0x6f as u8; key[2]=0x6b as u8; key[3]=0x65 as u8
88 key[4]=0x6e as u8; key[5]=0x69 as u8; key[6]=0x7a as u8; key[7]=0x65 as u8
89 key[8]=0x72 as u8; key[9]=0x2e as u8; key[10]=0x67 as u8; key[11]=0x67 as u8
90 key[12]=0x6d as u8; key[13]=0x6c as u8; key[14]=0x2e as u8; key[15]=0x6d as u8
91 key[16]=0x65 as u8; key[17]=0x72 as u8; key[18]=0x67 as u8; key[19]=0x65 as u8
92 key[20]=0x73 as u8
93
94 let off_out: *i64 = sys_mmap(8) as *i64
95 let typ_out: *i64 = sys_mmap(8) as *i64
96 let v: nx_int = nx_gguf_meta_find(buf, len, hdr, key, 21, off_out, typ_out)
97 if v != NX_GMETA_OK { return NX_FBL_OK } // merges optional
98 if typ_out[0] != NX_GGUF_TYPE_ARRAY { return NX_FBL_OK }
99
100 let inner: i64 = nx_gguf_meta_array_inner_type(buf, off_out[0])
101 if inner != NX_GGUF_TYPE_STRING { return NX_FBL_OK }
102
103 let count: i64 = nx_gguf_meta_array_count(buf, off_out[0])
104
105 let scratch: *u8 = sys_mmap(512) // for concatenated bytes
106
107 var elt_off: i64 = nx_gguf_meta_array_first_elt_off(buf, off_out[0])
108 var i: i64 = 0
109 while i < count {
110 let s_len: i64 = nx_le_read_u64(buf, elt_off)
111 let s_ptr: *u8 = ((buf as i64) + elt_off + 8) as *u8
112
113 // Find the single space separator.
114 var sep: i64 = -1
115 var j: i64 = 0
116 while j < s_len {
117 if s_ptr[j] == (0x20 as u8) {
118 if sep < 0 { sep = j }
119 }
120 j = j + 1
121 }
122 if sep > 0 {
123 let len_a: i64 = sep
124 let len_b: i64 = s_len - sep - 1
125 let ptr_b: *u8 = ((s_ptr as i64) + sep + 1) as *u8
126
127 // Look up part_a, part_b ids by intern.
128 let id_a: i64 = nx_bpe_add_token(vocab, s_ptr, len_a)
129 let id_b: i64 = nx_bpe_add_token(vocab, ptr_b, len_b)
130
131 // Compute merged bytes = a + b.
132 if len_a + len_b <= 512 {
133 var k: i64 = 0
134 while k < len_a { scratch[k] = s_ptr[k]; k = k + 1 }
135 while k < len_a + len_b {
136 scratch[k] = ptr_b[k - len_a]
137 k = k + 1
138 }
139 let id_m: i64 = nx_bpe_add_token(vocab, scratch, len_a + len_b)
140
141 nx_bpe_add_merge(vocab, id_a, id_b, id_m)
142 out_count[0] = out_count[0] + 1
143 }
144 }
145
146 elt_off = elt_off + 8 + s_len
147 i = i + 1
148 }
149 return NX_FBL_OK
150}
151
152// Public: load full BPE vocab from GGUF.
153//
154// Caller pre-allocates the NxBpeVocab with capacities sized for the
155// expected vocab size (Qwen2.5: vocab=151936, merges ~150K, intern
156// bytes ~4 MB are reasonable v1 caps).
157
158func nx_f32_bpe_load_from_gguf(buf: *u8, len: i64, hdr: *NxGgufHeader,
159 vocab: *NxBpeVocab,
160 n_tokens_out: *i64,
161 n_merges_out: *i64,
162 out_err: *i64) -> nx_int {
163 if vocab == (0 as *NxBpeVocab) {
164 out_err[0] = NX_FBL_ERR_NULL
165 return NX_FBL_ERR_NULL
166 }
167
168 let v_tok: nx_int = _fbl_load_tokens(buf, len, hdr, vocab, n_tokens_out)
169 if v_tok != NX_FBL_OK {
170 out_err[0] = v_tok
171 return v_tok
172 }
173
174 let v_mrg: nx_int = _fbl_load_merges(buf, len, hdr, vocab, n_merges_out)
175 if v_mrg != NX_FBL_OK {
176 out_err[0] = v_mrg
177 return v_mrg
178 }
179
180 out_err[0] = NX_FBL_OK
181 return NX_FBL_OK
182}