nx_nofloat_tokdec.nx source
↩ module page · 53 lines · 2538 B
1// nx_nofloat_tokdec.nx -- byte-level-BPE piece decoder (GPT-2/Qwen convention): GGUF vocab pieces are UTF-8
2// strings of REMAPPED codepoints (every raw byte 0..255 maps to a printable codepoint; e.g. space -> U+0120
3// 'G-dot' and newline -> U+010A). This is the INVERSE map, so generated token pieces render as REAL text.
4// Mapping (the standard gpt2 bytes_to_unicode inverse): codepoints 33-126, 161-172, 174-255 are themselves;
5// the 68 excluded bytes (0-32, 127-160, 173) were assigned 256+n in ascending byte order ->
6// cp 256..288 -> byte cp-256 (0..32; cp 288 = U+0120 -> byte 32 = SPACE)
7// cp 289 -> byte 127
8// cp 290..322 -> byte 128+(cp-290) (128..160)
9// cp 323 -> byte 173
10// Pieces therefore only contain codepoints <= 0x143 (1- or 2-byte UTF-8); anything else passes through raw
11// (defensive -- never drop bytes). Pure functions, no main. license_tier: ORIGINAL
12
13// map one remapped codepoint back to its raw byte; -1 = not a remapped byte (pass through).
14func td_cp_to_byte(cp: i64) -> i64 {
15 if cp >= 33 { if cp <= 126 { return cp } }
16 if cp >= 161 { if cp <= 172 { return cp } }
17 if cp >= 174 { if cp <= 255 { return cp } }
18 if cp >= 256 { if cp <= 288 { return cp - 256 } }
19 if cp == 289 { return 127 }
20 if cp >= 290 { if cp <= 322 { return 128 + (cp - 290) } }
21 if cp == 323 { return 173 }
22 return 0 - 1
23}
24
25// decode one vocab piece (UTF-8 remapped) into raw text bytes at out+ooff (capacity ocap total).
26// returns the new output offset. Unknown codepoints copy their original UTF-8 bytes verbatim.
27func td_piece_decode(piece: *u8, plen: i64, out: *u8, ooff: i64, ocap: i64) -> i64 {
28 var i: i64 = 0
29 var o: i64 = ooff
30 while i < plen {
31 let b0: i64 = piece[i] & 0xff
32 var cp: i64 = 0 - 1
33 var adv: i64 = 1
34 if b0 < 0x80 { cp = b0 }
35 if b0 >= 0xC0 { if b0 < 0xE0 { if i + 1 < plen {
36 cp = ((b0 & 0x1f) << 6) | ((piece[i+1] & 0xff) & 0x3f)
37 adv = 2
38 } } }
39 var raw: i64 = 0 - 1
40 if cp >= 0 { raw = td_cp_to_byte(cp) }
41 if raw >= 0 {
42 if o < ocap { out[o] = raw as u8; o = o + 1 }
43 } else {
44 // pass the original bytes through untouched (3/4-byte UTF-8 or unmapped cp)
45 var k: i64 = 0
46 if b0 >= 0xE0 { if b0 < 0xF0 { adv = 3 } }
47 if b0 >= 0xF0 { adv = 4 }
48 while k < adv { if i + k < plen { if o < ocap { out[o] = piece[i+k]; o = o + 1 } } k = k + 1 }
49 }
50 i = i + adv
51 }
52 return o
53}