`nx_tok_probe.nx` source

1// nx_tok_probe.nx -- tokenizer-parity DIAGNOSTIC: dump the exact token ids
2// (and each id's decoded string) that nx_bpe_encode_bytelevel produces for an
3// arithmetic prompt vs a known-good word prompt.  Loads GGUF vocab ONLY (no
4// weights) -- fast.  Answers mechanically whether digits are being merged
5// (Qwen2.5 pre-tokenizer splits every digit into its OWN token; a merged
6// "47" would be a context the model never saw in arithmetic).
7// expect_exit: 0
8
9import "nx_syscalls.nx"
10import "nx_tier.nx"
11import "nx_le.nx"
12import "nx_strconv.nx"
13import "nx_bpe.nx"
14import "nx_gguf.nx"
15import "nx_gguf_load.nx"
16import "nx_f32_bpe_load.nx"
17import "nx_reasoning.nx"
18
19func tp_w(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
20func tp_wn(v: i64) -> i64 {
21    let dec: *u8 = sys_mmap(32)
22    var m: i64 = v
23    if m < 0 { tp_w("-" as *u8); m = 0 - m }
24    let nd: i64 = nx_strconv_format_i64(m, dec)
25    sys_write(1, dec, nd)
26    return 0
27}
28func tp_wtext(b: *u8, n: i64) -> i64 {
29    let t: *u8 = sys_mmap(n + 1)
30    var i: i64 = 0
31    while i < n {
32        var c: i64 = b[i] as i64
33        if c < 32 { c = 46 }
34        t[i] = c as u8
35        i = i + 1
36    }
37    sys_write(1, t, n)
38    return 0
39}
40
41func tp_dump(v: *NxBpeVocab, label: *u8, text: *u8, n: nx_int) -> i64 {
42    tp_w(label); tp_w(" [" as *u8); tp_wtext(text, n as i64); tp_w("] -> " as *u8)
43    let toks: *i64 = sys_mmap((n + 4) * 8) as *i64
44    let nt: nx_int = nx_bpe_encode_bytelevel(v, text, n, toks)
45    tp_w("n_tokens=" as *u8); tp_wn(nt as i64); tp_w("\n" as *u8)
46    var i: nx_int = 0
47    let one: *i64 = sys_mmap(8) as *i64
48    let ob: *u8 = sys_mmap(64)
49    while i < nt {
50        one[0] = toks[i]
51        let nb: nx_int = nx_bpe_decode_bytelevel(v, one, 1, ob)
52        tp_w("  id=" as *u8); tp_wn(toks[i])
53        tp_w(" str=[" as *u8)
54        var sn: i64 = nb as i64
55        if sn < 0 { sn = 0 }
56        tp_wtext(ob, sn)
57        tp_w("]\n" as *u8)
58        i = i + 1
59    }
60    return 0
61}
62
63func main() -> i64 {
64    let path: *u8 = sys_mmap(64)
65    path[0]=0x2F as u8; path[1]=0x74 as u8; path[2]=0x6D as u8; path[3]=0x70 as u8
66    path[4]=0x2F as u8; path[5]=0x6E as u8; path[6]=0x78 as u8; path[7]=0x5F as u8
67    path[8]=0x72 as u8; path[9]=0x65 as u8; path[10]=0x61 as u8; path[11]=0x6C as u8
68    path[12]=0x5F as u8; path[13]=0x6D as u8; path[14]=0x6F as u8; path[15]=0x64 as u8
69    path[16]=0x65 as u8; path[17]=0x6C as u8; path[18]=0x2E as u8
70    path[19]=0x67 as u8; path[20]=0x67 as u8; path[21]=0x75 as u8; path[22]=0x66 as u8
71    path[23]=0 as u8
72    let len_out: *i64 = sys_mmap(8) as *i64
73    let buf: *u8 = sys_read_file(path, len_out)
74    if buf == (0 as *u8) { return 10 }
75    let hdr: *NxGgufHeader = sys_mmap(NX_GGUF_HDR_BYTES) as *NxGgufHeader
76    if nx_gguf_parse(buf, len_out[0], hdr) != NX_GGUF_OK { return 20 }
77    let vocab: *NxBpeVocab = nx_bpe_vocab_new(8388608, 524288, 262144)
78    let nt: *i64 = sys_mmap(8) as *i64
79    let nm: *i64 = sys_mmap(8) as *i64
80    let out_err: *i64 = sys_mmap(8) as *i64
81    if nx_f32_bpe_load_from_gguf(buf, len_out[0], hdr, vocab, nt, nm, out_err) != NX_FBL_OK { return 50 }
82
83    let p1: *u8 = sys_mmap(64)
84    let n1: nx_int = nx_reason_build_arith_prompt(p1, 47, 0, 38)
85    tp_dump(vocab, "ARITH" as *u8, p1, n1)
86
87    let p2: *u8 = sys_mmap(64)
88    p2[0]=0x54 as u8; p2[1]=0x68 as u8; p2[2]=0x65 as u8; p2[3]=0x20 as u8   // "The "
89    p2[4]=0x63 as u8; p2[5]=0x61 as u8; p2[6]=0x70 as u8; p2[7]=0x69 as u8   // "capi"
90    p2[8]=0x74 as u8; p2[9]=0x61 as u8; p2[10]=0x6C as u8                     // "tal"
91    tp_dump(vocab, "WORDS" as *u8, p2, 11)
92
93    return 0
94}
nx_tok_probe.nx source

`nx_tok_probe.nx` source
