nx_nofloat_qwen_tok_gate.nx source
↩ module page · 52 lines · 3689 B
1// nx_nofloat_qwen_tok_gate.nx -- TOKENIZER CHECK (fast, no forward): BPE-encode a multi-word prompt and print the
2// token ids + their decoded pieces. Hypothesis under test: the byte-level space map (0x20 -> U+0120 'Ġ') is missing,
3// so multi-word prompts tokenize wrong -> the forward sees garbage input -> garbage output (NOT a forward bug).
4// Correct Qwen2.5 tokenization of "The capital of France is" = [785, 6722, 315, 9625, 374] = The/Ġcapital/Ġof/ĠFrance/Ġis.
5// No hw writes (Rule 26). expect_exit: 0 license_tier: ORIGINAL
6import "nx_syscalls.nx"
7import "nx_tier.nx"
8import "nx_le.nx"
9import "nx_tensor.nx"
10import "nx_gguf.nx"
11import "nx_gguf_load.nx"
12import "nx_gguf_meta.nx"
13import "nx_nofloat_tok.nx"
14
15func tg_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
16func tg_num(v: i64) -> i64 { let b: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)} let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i: i64=0; while i<k{b[i]=t[k-1-i];i=i+1} sys_write(1,b,k); return 0 }
17func tg_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
18
19func tg_encode_print(buf: *u8, mfirst: i64, nm_c: i64, vfirst: i64, vocab: i64, input: *u8) -> i64 {
20 let ilen: i64=tg_slen(input)
21 let MAXT: i64=32
22 let ids: *i64=sys_mmap(MAXT*8) as *i64; let tokp: *i64=sys_mmap(MAXT*8) as *i64; let tokl: *i64=sys_mmap(MAXT*8) as *i64
23 let ntok: i64=tk_bpe_encode(buf, mfirst, nm_c, vfirst, vocab, input, ilen, tokp, tokl, ids)
24 tg_puts(" in='"); tg_puts(input); tg_puts("' ntok="); tg_num(ntok); tg_puts(" ids: ")
25 var i: i64=0; while i<ntok { tg_num(ids[i]); tg_puts(" "); i=i+1 }
26 tg_puts("\n pieces: ")
27 i=0; while i<ntok { let off: i64=tk_decode_off(buf, vfirst, ids[i]); let pl: i64=nx_gguf_meta_read_string_len(buf, off); tg_puts("["); if pl>0 { sys_write(1, nx_gguf_meta_read_string_ptr(buf, off), pl) } tg_puts("]"); i=i+1 }
28 tg_puts("\n")
29 return 0
30}
31
32func main() -> i64 {
33 tg_puts("TOKENIZER CHECK (no forward)\n")
34 let path: *u8 = "/home/elderwesto/nx_stage/nx_real_model.gguf\x00" as *u8
35 let len_out: *i64 = sys_mmap(8) as *i64; len_out[0]=0
36 let buf: *u8 = sys_read_file(path, len_out)
37 if buf == (0 as *u8) { tg_puts("MODEL ABSENT\n" as *u8); return 1 }
38 let hdr: *NxGgufHeader = sys_mmap(NX_GGUF_HDR_BYTES) as *NxGgufHeader
39 if nx_gguf_parse(buf, len_out[0], hdr) != NX_GGUF_OK { tg_puts("PARSE FAIL\n" as *u8); return 1 }
40 let voff: *i64=sys_mmap(8) as *i64; let vty: *i64=sys_mmap(8) as *i64
41 var mfirst: i64=0; var nm_c: i64=0; var vfirst: i64=0; var vocab: i64=0
42 let km: *u8="tokenizer.ggml.merges\x00" as *u8; let kt: *u8="tokenizer.ggml.tokens\x00" as *u8
43 if nx_gguf_meta_find(buf, len_out[0], hdr, km, tg_slen(km), voff, vty)==NX_GMETA_OK { nm_c=nx_gguf_meta_array_count(buf, voff[0]); mfirst=nx_gguf_meta_array_first_elt_off(buf, voff[0]) }
44 if nx_gguf_meta_find(buf, len_out[0], hdr, kt, tg_slen(kt), voff, vty)==NX_GMETA_OK { vocab=nx_gguf_meta_array_count(buf, voff[0]); vfirst=nx_gguf_meta_array_first_elt_off(buf, voff[0]) }
45 tg_puts("merges="); tg_num(nm_c); tg_puts(" vocab="); tg_num(vocab); tg_puts("\n")
46 tg_encode_print(buf, mfirst, nm_c, vfirst, vocab, "The capital of France is\x00" as *u8)
47 tg_encode_print(buf, mfirst, nm_c, vfirst, vocab, "hello\x00" as *u8)
48 tg_encode_print(buf, mfirst, nm_c, vfirst, vocab, " Paris\x00" as *u8)
49 tg_puts("EXPECT Qwen 'The capital of France is' = ids [785, 6722, 315, 9625, 374] = [The][Ġcapital][Ġof][ĠFrance][Ġis]\n")
50 tg_puts("NX-NOFLOAT-QWEN-TOK done\n")
51 return 0
52}