nx_nofloat_qwen_decode_gate.nx source
↩ module page · 93 lines · 5556 B
1// nx_nofloat_qwen_decode_gate.nx -- TOKENIZER decode: token id -> text, from the real GGUF vocab.
2// The forward emits token ids; to be human-readable we map them back through tokenizer.ggml.tokens (an ARRAY
3// of STRING in GGUF metadata). Walks the vocab array (each element length-prefixed) to the id and prints its
4// bytes. Pure metadata + string (uses nx_gguf_meta; no math, no copy-paste of the LLM helpers). Decodes the
5// capstone's prompt tokens [9001,137] + its predicted next token 103504. No hw writes (Rule 26).
6// expect_exit: 0 license_tier: ORIGINAL
7import "nx_syscalls.nx"
8import "nx_tier.nx"
9import "nx_le.nx"
10import "nx_tensor.nx"
11import "nx_gguf.nx"
12import "nx_gguf_meta.nx"
13import "nx_gate_verdict.nx"
14import "nx_stage_path.nx"
15
16func dc_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
17func dc_num(v: i64) -> i64 { let b: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)} let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i: i64=0; while i<k{b[i]=t[k-1-i];i=i+1} sys_write(1,b,k); return 0 }
18func dc_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
19
20// byte offset of vocab element `tid` (walk length-prefixed strings from first).
21func tok_eltoff(buf: *u8, first: i64, tid: i64) -> i64 {
22 var off: i64=first; var i: i64=0
23 while i<tid { let sl: i64=nx_gguf_meta_read_string_len(buf, off); off=off+8+sl; i=i+1 }
24 return off
25}
26
27func main() -> i64 {
28 dc_puts("TOKENIZER decode: token id -> text via the real GGUF vocab (tokenizer.ggml.tokens)\n\n" as *u8)
29 let path: *u8 = sp_path("nx_real_model.gguf" as *u8, sys_mmap(SP_PATH_MAX))
30 sp_skip_unless("NOFLOAT-QWEN-DECODE-GATE" as *u8, path)
31 let len_out: *i64 = sys_mmap(8) as *i64
32 len_out[0]=0
33 let buf: *u8 = sys_read_file(path, len_out)
34
35 var found: i64=0
36 var is_str_array: i64=0
37 var vocab: i64=0
38 var first: i64=0
39 var decoded_pred: i64=0
40 var decoded_prompt: i64=0
41 let tids: *i64 = sys_mmap(8*8) as *i64
42 tids[0]=9001; tids[1]=137; tids[2]=103504; tids[3]=0; tids[4]=1
43 let NT: i64=5
44 if buf != (0 as *u8) { if len_out[0] > 1000 {
45 let hdr: *NxGgufHeader = sys_mmap(NX_GGUF_HDR_BYTES) as *NxGgufHeader
46 if nx_gguf_parse(buf, len_out[0], hdr) == NX_GGUF_OK {
47 let key: *u8 = "tokenizer.ggml.tokens\x00" as *u8
48 let voff: *i64 = sys_mmap(8) as *i64
49 let vty: *i64 = sys_mmap(8) as *i64
50 if nx_gguf_meta_find(buf, len_out[0], hdr, key, dc_slen(key), voff, vty) == NX_GMETA_OK {
51 found=1
52 if vty[0]==NX_GGUF_TYPE_ARRAY { if nx_gguf_meta_array_inner_type(buf, voff[0])==NX_GGUF_TYPE_STRING { is_str_array=1 } }
53 vocab = nx_gguf_meta_array_count(buf, voff[0])
54 first = nx_gguf_meta_array_first_elt_off(buf, voff[0])
55 // decode each id -> print "id=<n> '<text>'"
56 var i: i64=0
57 while i<NT {
58 let tid: i64=tids[i]
59 dc_puts(" token "); dc_num(tid); dc_puts(" = '")
60 if tid < vocab {
61 let eo: i64=tok_eltoff(buf, first, tid)
62 let sl: i64=nx_gguf_meta_read_string_len(buf, eo)
63 let sp: *u8=nx_gguf_meta_read_string_ptr(buf, eo)
64 if sl>0 { sys_write(1, sp, sl); if tid==103504 { decoded_pred=1 } if tid==9001 { decoded_prompt=decoded_prompt+1 } if tid==137 { decoded_prompt=decoded_prompt+1 } }
65 }
66 dc_puts("'\n")
67 i=i+1
68 }
69 }
70 }
71 } }
72
73 dc_puts("\n tokens-array found="); dc_num(found); dc_puts(" string-array="); dc_num(is_str_array); dc_puts(" vocab="); dc_num(vocab); dc_puts("\n");
74 dc_puts(" >>> prompt [9001,137] + predicted-next 103504 decoded above (raw BPE token text; Qwen byte-level)\n\n");
75
76 var pass: i64=0
77 var ttl: i64=0
78 ttl=ttl+1; dc_puts(" T1 tokenizer.ggml.tokens found = ARRAY of STRING, vocab=151936: "); if found==1 { if is_str_array==1 { if vocab==151936 { pass=pass+1; dc_puts("PASS\n") } else { dc_puts("FAIL\n") } } else { dc_puts("FAIL\n") } } else { dc_puts("FAIL\n") }
79 ttl=ttl+1; dc_puts(" T2 the forward's predicted token (103504) decoded to non-empty text: "); if decoded_pred==1 { pass=pass+1; dc_puts("PASS\n") } else { dc_puts("FAIL\n") }
80 ttl=ttl+1; dc_puts(" T3 both prompt tokens (9001, 137) decoded to non-empty text: "); if decoded_prompt==2 { pass=pass+1; dc_puts("PASS\n") } else { dc_puts("FAIL\n") }
81 ttl=ttl+1; dc_puts(" T4 vocab size matches the LM head out-dim (151936) -- decode <-> head consistent: "); if vocab==151936 { pass=pass+1; dc_puts("PASS\n") } else { dc_puts("FAIL\n") }
82
83 dc_puts("NX-NOFLOAT-QWEN-DECODE-GATE passed "); dc_num(pass); dc_puts("/"); dc_num(ttl)
84 // MIGRATED onto nx_gate_verdict by nx_gate_dry_apply (D001, minimal form): every check
85 // row above is untouched, so the PASS/FAIL vector cannot change; only the hand-rolled
86 // verdict emission is replaced by the ONE shared base class. Proven by nx_gate_migrate verify.
87 let ctr__dry: *i64 = gv_ctr()
88 ctr__dry[0] = pass
89 ctr__dry[1] = ttl
90 let rc__dry: i64 = gv_verdict("NOFLOAT-QWEN-DECODE-GATE" as *u8, ctr__dry, "sovereign tokenizer decode: token ids -> real text from the GGUF vocab -- the forward's output is now readable)" as *u8)
91 sys_exit(rc__dry)
92 return rc__dry
93}