nx_nofloat_qwen_tok_gate.nx source
↩ module page · 109 lines · 7063 B
1// nx_nofloat_qwen_tok_gate.nx -- TOKENIZER CHECK (fast, no forward): BPE-encode a multi-word prompt and print the
2// token ids + their decoded pieces. Hypothesis under test: the byte-level space map (0x20 -> U+0120 'Ġ') is missing,
3// so multi-word prompts tokenize wrong -> the forward sees garbage input -> garbage output (NOT a forward bug).
4// Correct Qwen2.5 tokenization of "The capital of France is" = [785, 6722, 315, 9625, 374] = The/Ġcapital/Ġof/ĠFrance/Ġis.
5// No hw writes (Rule 26). expect_exit: 0 license_tier: ORIGINAL
6import "nx_syscalls.nx"
7import "nx_tier.nx"
8import "nx_le.nx"
9import "nx_tensor.nx"
10import "nx_gguf.nx"
11import "nx_gguf_load.nx"
12import "nx_gguf_meta.nx"
13import "nx_nofloat_tok.nx"
14// D001 MIGRATION 2026-08-25. Two defects, not one:
15// (1) this file is named _gate but ASSERTED NOTHING -- it printed ids and an EXPECT line for a human to eyeball,
16// so it could never be RED and its GREEN meant only "it ran". The expectation was already written in the
17// header; turning it into a TOOTH is the migration, not bolting a verdict onto a diagnostic.
18// (2) a missing model file returned 1 = FAIL. "I could not look" is not "it is broken": an absent precondition
19// must SKIP, which blocks any claim the tokenizer works while never accusing the tokenizer.
20import "nx_gate_verdict.nx"
21
22// The published Qwen2.5 encoding of "The capital of France is", from this file's own header.
23const TG_KAT_N: i64 = 5
24const TG_KAT_0: i64 = 785
25const TG_KAT_1: i64 = 6722
26const TG_KAT_2: i64 = 315
27const TG_KAT_3: i64 = 9625
28const TG_KAT_4: i64 = 374
29const TG_MAXT: i64 = 32
30const TG_SLOT: i64 = 8
31
32func tg_kat_match(ids: *i64, n: i64) -> i64 {
33 if n != TG_KAT_N { return 0 }
34 if ids[0] != TG_KAT_0 { return 0 }
35 if ids[1] != TG_KAT_1 { return 0 }
36 if ids[2] != TG_KAT_2 { return 0 }
37 if ids[3] != TG_KAT_3 { return 0 }
38 if ids[4] != TG_KAT_4 { return 0 }
39 return 1
40}
41
42func tg_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
43func tg_num(v: i64) -> i64 { let b: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)} let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i: i64=0; while i<k{b[i]=t[k-1-i];i=i+1} sys_write(1,b,k); return 0 }
44func tg_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
45
46func tg_encode_print(buf: *u8, mfirst: i64, nm_c: i64, vfirst: i64, vocab: i64, input: *u8) -> i64 {
47 let ilen: i64=tg_slen(input)
48 let MAXT: i64=32
49 let ids: *i64=sys_mmap(MAXT*8) as *i64; let tokp: *i64=sys_mmap(MAXT*8) as *i64; let tokl: *i64=sys_mmap(MAXT*8) as *i64
50 let ntok: i64=tk_bpe_encode(buf, mfirst, nm_c, vfirst, vocab, input, ilen, tokp, tokl, ids)
51 tg_puts(" in='"); tg_puts(input); tg_puts("' ntok="); tg_num(ntok); tg_puts(" ids: ")
52 var i: i64=0; while i<ntok { tg_num(ids[i]); tg_puts(" "); i=i+1 }
53 tg_puts("\n pieces: ")
54 i=0; while i<ntok { let off: i64=tk_decode_off(buf, vfirst, ids[i]); let pl: i64=nx_gguf_meta_read_string_len(buf, off); tg_puts("["); if pl>0 { sys_write(1, nx_gguf_meta_read_string_ptr(buf, off), pl) } tg_puts("]"); i=i+1 }
55 tg_puts("\n")
56 return 0
57}
58
59func main() -> i64 {
60 let ctr: *i64 = gv_ctr()
61 gv_head("nx_nofloat_qwen_tok -- the Qwen2.5 BPE tokenizer against the PUBLISHED ids" as *u8)
62 let path: *u8 = "/home/elderwesto/nx_stage/nx_real_model.gguf\x00" as *u8
63 let len_out: *i64 = sys_mmap(8) as *i64; len_out[0]=0
64 let buf: *u8 = sys_read_file(path, len_out)
65 var have: i64 = 1
66 if buf == (0 as *u8) { have = 0 }
67 if gv_need("the staged Qwen GGUF model file" as *u8, have, ctr) == 0 {
68 let rcs: i64 = gv_verdict("NOFLOAT-QWEN-TOK-GATE" as *u8, ctr, "the tokenizer KAT could not run because the staged model is absent -- SKIP blocks any claim that the tokenizer is correct and accuses the tokenizer of nothing" as *u8)
69 sys_exit(rcs)
70 return rcs
71 }
72 let hdr: *NxGgufHeader = sys_mmap(NX_GGUF_HDR_BYTES) as *NxGgufHeader
73 var parsed: i64 = 1
74 if nx_gguf_parse(buf, len_out[0], hdr) != NX_GGUF_OK { parsed = 0 }
75 if gv_need("a parseable GGUF header" as *u8, parsed, ctr) == 0 {
76 let rcp: i64 = gv_verdict("NOFLOAT-QWEN-TOK-GATE" as *u8, ctr, "the staged file is present but not parseable as GGUF -- that is a fact about the FILE, so it skips rather than accusing the tokenizer" as *u8)
77 sys_exit(rcp)
78 return rcp
79 }
80 let voff: *i64=sys_mmap(8) as *i64; let vty: *i64=sys_mmap(8) as *i64
81 var mfirst: i64=0; var nm_c: i64=0; var vfirst: i64=0; var vocab: i64=0
82 let km: *u8="tokenizer.ggml.merges\x00" as *u8; let kt: *u8="tokenizer.ggml.tokens\x00" as *u8
83 if nx_gguf_meta_find(buf, len_out[0], hdr, km, tg_slen(km), voff, vty)==NX_GMETA_OK { nm_c=nx_gguf_meta_array_count(buf, voff[0]); mfirst=nx_gguf_meta_array_first_elt_off(buf, voff[0]) }
84 if nx_gguf_meta_find(buf, len_out[0], hdr, kt, tg_slen(kt), voff, vty)==NX_GMETA_OK { vocab=nx_gguf_meta_array_count(buf, voff[0]); vfirst=nx_gguf_meta_array_first_elt_off(buf, voff[0]) }
85 tg_puts("merges="); tg_num(nm_c); tg_puts(" vocab="); tg_num(vocab); tg_puts("\n")
86 var hasvocab: i64 = 0
87 if vocab > 0 { if nm_c > 0 { hasvocab = 1 } }
88 if gv_need("a tokenizer vocab and merge table in the GGUF metadata" as *u8, hasvocab, ctr) == 0 {
89 let rcv: i64 = gv_verdict("NOFLOAT-QWEN-TOK-GATE" as *u8, ctr, "the model carries no tokenizer metadata so there is nothing to encode with -- SKIP, and not a tokenizer failure" as *u8)
90 sys_exit(rcv)
91 return rcv
92 }
93 tg_encode_print(buf, mfirst, nm_c, vfirst, vocab, "The capital of France is\x00" as *u8)
94 tg_encode_print(buf, mfirst, nm_c, vfirst, vocab, "hello\x00" as *u8)
95 tg_encode_print(buf, mfirst, nm_c, vfirst, vocab, " Paris\x00" as *u8)
96 // THE TOOTH THIS FILE'S HEADER ALWAYS DESCRIBED AND NEVER ASSERTED. The hypothesis it was written to test
97 // -- that a missing byte-level space map (0x20 -> U+0120) silently mis-tokenizes multi-word prompts, so the
98 // forward eats garbage and gets blamed for it -- is now DECIDED by an equality instead of by a human reading
99 // three lines of ids. A diagnostic that prints EXPECT is a comment; this is a test.
100 let kids: *i64 = sys_mmap(TG_MAXT * TG_SLOT) as *i64
101 let ktp: *i64 = sys_mmap(TG_MAXT * TG_SLOT) as *i64
102 let ktl: *i64 = sys_mmap(TG_MAXT * TG_SLOT) as *i64
103 let kin: *u8 = "The capital of France is\x00" as *u8
104 let kn: i64 = tk_bpe_encode(buf, mfirst, nm_c, vfirst, vocab, kin, tg_slen(kin), ktp, ktl, kids)
105 gv_check("the PUBLISHED Qwen2.5 ids for a multi-word prompt are reproduced exactly: 785 6722 315 9625 374" as *u8, tg_kat_match(kids, kn), ctr)
106 let rc: i64 = gv_verdict("NOFLOAT-QWEN-TOK-GATE" as *u8, ctr, "the byte-level BPE reproduces the published Qwen2.5 token ids for a multi-word prompt, so the space map is present and the forward is being fed real tokens" as *u8)
107 sys_exit(rc)
108 return rc
109}