nx_nofloat_qwen_decode_gate.nx source
↩ module page · 91 lines · 5468 B
1// nx_nofloat_qwen_decode_gate.nx -- TOKENIZER decode: token id -> text, from the real GGUF vocab.
2// The forward emits token ids; to be human-readable we map them back through tokenizer.ggml.tokens (an ARRAY
3// of STRING in GGUF metadata). Walks the vocab array (each element length-prefixed) to the id and prints its
4// bytes. Pure metadata + string (uses nx_gguf_meta; no math, no copy-paste of the LLM helpers). Decodes the
5// capstone's prompt tokens [9001,137] + its predicted next token 103504. No hw writes (Rule 26).
6// expect_exit: 0 license_tier: ORIGINAL
7import "nx_syscalls.nx"
8import "nx_tier.nx"
9import "nx_le.nx"
10import "nx_tensor.nx"
11import "nx_gguf.nx"
12import "nx_gguf_meta.nx"
13import "nx_gate_verdict.nx"
14
15func dc_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
16func dc_num(v: i64) -> i64 { let b: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)} let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i: i64=0; while i<k{b[i]=t[k-1-i];i=i+1} sys_write(1,b,k); return 0 }
17func dc_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
18
19// byte offset of vocab element `tid` (walk length-prefixed strings from first).
20func tok_eltoff(buf: *u8, first: i64, tid: i64) -> i64 {
21 var off: i64=first; var i: i64=0
22 while i<tid { let sl: i64=nx_gguf_meta_read_string_len(buf, off); off=off+8+sl; i=i+1 }
23 return off
24}
25
26func main() -> i64 {
27 dc_puts("TOKENIZER decode: token id -> text via the real GGUF vocab (tokenizer.ggml.tokens)\n\n" as *u8)
28 let path: *u8 = "/home/elderwesto/nx_stage/nx_real_model.gguf\x00" as *u8
29 let len_out: *i64 = sys_mmap(8) as *i64
30 len_out[0]=0
31 let buf: *u8 = sys_read_file(path, len_out)
32
33 var found: i64=0
34 var is_str_array: i64=0
35 var vocab: i64=0
36 var first: i64=0
37 var decoded_pred: i64=0
38 var decoded_prompt: i64=0
39 let tids: *i64 = sys_mmap(8*8) as *i64
40 tids[0]=9001; tids[1]=137; tids[2]=103504; tids[3]=0; tids[4]=1
41 let NT: i64=5
42 if buf != (0 as *u8) { if len_out[0] > 1000 {
43 let hdr: *NxGgufHeader = sys_mmap(NX_GGUF_HDR_BYTES) as *NxGgufHeader
44 if nx_gguf_parse(buf, len_out[0], hdr) == NX_GGUF_OK {
45 let key: *u8 = "tokenizer.ggml.tokens\x00" as *u8
46 let voff: *i64 = sys_mmap(8) as *i64
47 let vty: *i64 = sys_mmap(8) as *i64
48 if nx_gguf_meta_find(buf, len_out[0], hdr, key, dc_slen(key), voff, vty) == NX_GMETA_OK {
49 found=1
50 if vty[0]==NX_GGUF_TYPE_ARRAY { if nx_gguf_meta_array_inner_type(buf, voff[0])==NX_GGUF_TYPE_STRING { is_str_array=1 } }
51 vocab = nx_gguf_meta_array_count(buf, voff[0])
52 first = nx_gguf_meta_array_first_elt_off(buf, voff[0])
53 // decode each id -> print "id=<n> '<text>'"
54 var i: i64=0
55 while i<NT {
56 let tid: i64=tids[i]
57 dc_puts(" token "); dc_num(tid); dc_puts(" = '")
58 if tid < vocab {
59 let eo: i64=tok_eltoff(buf, first, tid)
60 let sl: i64=nx_gguf_meta_read_string_len(buf, eo)
61 let sp: *u8=nx_gguf_meta_read_string_ptr(buf, eo)
62 if sl>0 { sys_write(1, sp, sl); if tid==103504 { decoded_pred=1 } if tid==9001 { decoded_prompt=decoded_prompt+1 } if tid==137 { decoded_prompt=decoded_prompt+1 } }
63 }
64 dc_puts("'\n")
65 i=i+1
66 }
67 }
68 }
69 } }
70
71 dc_puts("\n tokens-array found="); dc_num(found); dc_puts(" string-array="); dc_num(is_str_array); dc_puts(" vocab="); dc_num(vocab); dc_puts("\n");
72 dc_puts(" >>> prompt [9001,137] + predicted-next 103504 decoded above (raw BPE token text; Qwen byte-level)\n\n");
73
74 var pass: i64=0
75 var ttl: i64=0
76 ttl=ttl+1; dc_puts(" T1 tokenizer.ggml.tokens found = ARRAY of STRING, vocab=151936: "); if found==1 { if is_str_array==1 { if vocab==151936 { pass=pass+1; dc_puts("PASS\n") } else { dc_puts("FAIL\n") } } else { dc_puts("FAIL\n") } } else { dc_puts("FAIL\n") }
77 ttl=ttl+1; dc_puts(" T2 the forward's predicted token (103504) decoded to non-empty text: "); if decoded_pred==1 { pass=pass+1; dc_puts("PASS\n") } else { dc_puts("FAIL\n") }
78 ttl=ttl+1; dc_puts(" T3 both prompt tokens (9001, 137) decoded to non-empty text: "); if decoded_prompt==2 { pass=pass+1; dc_puts("PASS\n") } else { dc_puts("FAIL\n") }
79 ttl=ttl+1; dc_puts(" T4 vocab size matches the LM head out-dim (151936) -- decode <-> head consistent: "); if vocab==151936 { pass=pass+1; dc_puts("PASS\n") } else { dc_puts("FAIL\n") }
80
81 dc_puts("NX-NOFLOAT-QWEN-DECODE-GATE passed "); dc_num(pass); dc_puts("/"); dc_num(ttl)
82 // MIGRATED onto nx_gate_verdict by nx_gate_dry_apply (D001, minimal form): every check
83 // row above is untouched, so the PASS/FAIL vector cannot change; only the hand-rolled
84 // verdict emission is replaced by the ONE shared base class. Proven by nx_gate_migrate verify.
85 let ctr__dry: *i64 = gv_ctr()
86 ctr__dry[0] = pass
87 ctr__dry[1] = ttl
88 let rc__dry: i64 = gv_verdict("NOFLOAT-QWEN-DECODE-GATE" as *u8, ctr__dry, "sovereign tokenizer decode: token ids -> real text from the GGUF vocab -- the forward's output is now readable)" as *u8)
89 sys_exit(rc__dry)
90 return rc__dry
91}