nx_nofloat_qwen_encode_gate.nx source
↩ module page · 133 lines · 8234 B
1// nx_nofloat_qwen_encode_gate.nx -- BPE ENCODE: text -> token ids, sovereign, from the real GGUF merges+vocab.
2// Algorithm: start with single-char tokens; repeatedly merge the adjacent pair (X,Y) whose rule "X Y" has the
3// LOWEST rank in tokenizer.ggml.merges; stop when no adjacent pair has a merge; map each final token-string to
4// its id. (ASCII input -> byte-level pre-tokenize is identity, so we encode "hello" directly.) Proven by
5// ROUND-TRIP: the encoded tokens' strings concatenate back to the input, and every token maps to a valid id
6// (so decode(ids) == input). Pure metadata + byte ops (nx_gguf_meta). No hw writes (Rule 26).
7// expect_exit: 0 license_tier: ORIGINAL
8import "nx_syscalls.nx"
9import "nx_tier.nx"
10import "nx_le.nx"
11import "nx_tensor.nx"
12import "nx_gguf.nx"
13import "nx_gguf_meta.nx"
14import "nx_gate_verdict.nx"
15import "nx_stage_path.nx"
16
17func en_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
18func en_num(v: i64) -> i64 { let b: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)} let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i: i64=0; while i<k{b[i]=t[k-1-i];i=i+1} sys_write(1,b,k); return 0 }
19func en_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
20func bytes_eq(a: *u8, b: *u8, n: i64) -> i64 { var i: i64=0; while i<n { if a[i]!=b[i] { return 0 } i=i+1 } return 1 }
21func find_token_id(buf: *u8, first: i64, vocab: i64, target: *u8, tlen: i64) -> i64 {
22 var off: i64=first; var i: i64=0
23 while i<vocab { let sl: i64=nx_gguf_meta_read_string_len(buf, off); if sl==tlen { let sp: *u8=nx_gguf_meta_read_string_ptr(buf, off); if bytes_eq(sp, target, tlen)==1 { return i } } off=off+8+sl; i=i+1 }
24 return 0 - 1
25}
26// rank of merge rule "X Y" (Xbytes + space + Ybytes) in the merges array, or -1.
27func merge_rank(buf: *u8, mfirst: i64, nm: i64, xp: *u8, xl: i64, yp: *u8, yl: i64, tmp: *u8) -> i64 {
28 var p: i64=0; var i: i64=0
29 while i<xl { tmp[p]=xp[i]; p=p+1; i=i+1 }
30 tmp[p]=32 as u8; p=p+1
31 i=0; while i<yl { tmp[p]=yp[i]; p=p+1; i=i+1 }
32 let tl: i64=p
33 var off: i64=mfirst; var j: i64=0
34 while j<nm { let ml: i64=nx_gguf_meta_read_string_len(buf, off); if ml==tl { let mp: *u8=nx_gguf_meta_read_string_ptr(buf, off); if bytes_eq(mp, tmp, tl)==1 { return j } } off=off+8+ml; j=j+1 }
35 return 0 - 1
36}
37
38func main() -> i64 {
39 en_puts("BPE ENCODE: text -> token ids (merge by rank), round-trip vs decode -- sovereign, real GGUF tokenizer\n\n" as *u8)
40 let path: *u8 = sp_path("nx_real_model.gguf" as *u8, sys_mmap(SP_PATH_MAX))
41 sp_skip_unless("NOFLOAT-QWEN-ENCODE-GATE" as *u8, path)
42 let len_out: *i64 = sys_mmap(8) as *i64
43 len_out[0]=0
44 let buf: *u8 = sys_read_file(path, len_out)
45
46 // input "hello" (ASCII -> byte-level identity)
47 let input: *u8 = sys_mmap(16)
48 input[0]=104 as u8; input[1]=101 as u8; input[2]=108 as u8; input[3]=108 as u8; input[4]=111 as u8
49 let ilen: i64=5
50
51 var ran: i64=0
52 var ntok: i64=0
53 var all_ids_valid: i64=0
54 var concat_ok: i64=0
55 let MAXTOK: i64=64
56 let tok_ptr: *i64 = sys_mmap(MAXTOK*8) as *i64
57 let tok_len: *i64 = sys_mmap(MAXTOK*8) as *i64
58 let ids: *i64 = sys_mmap(MAXTOK*8) as *i64
59 let tmp: *u8 = sys_mmap(256)
60 if buf != (0 as *u8) { if len_out[0] > 1000 {
61 let hdr: *NxGgufHeader = sys_mmap(NX_GGUF_HDR_BYTES) as *NxGgufHeader
62 if nx_gguf_parse(buf, len_out[0], hdr) == NX_GGUF_OK {
63 let voff: *i64 = sys_mmap(8) as *i64
64 let vty: *i64 = sys_mmap(8) as *i64
65 let km: *u8 = "tokenizer.ggml.merges\x00" as *u8
66 let kt: *u8 = "tokenizer.ggml.tokens\x00" as *u8
67 var mfirst: i64=0; var nm: i64=0; var vfirst: i64=0; var vocab: i64=0
68 if nx_gguf_meta_find(buf, len_out[0], hdr, km, en_slen(km), voff, vty) == NX_GMETA_OK { nm=nx_gguf_meta_array_count(buf, voff[0]); mfirst=nx_gguf_meta_array_first_elt_off(buf, voff[0]) }
69 if nx_gguf_meta_find(buf, len_out[0], hdr, kt, en_slen(kt), voff, vty) == NX_GMETA_OK { vocab=nx_gguf_meta_array_count(buf, voff[0]); vfirst=nx_gguf_meta_array_first_elt_off(buf, voff[0]) }
70 if nm>0 { if vocab>0 {
71 // init: one token per input byte (pointers into `input`)
72 ntok=ilen
73 var i: i64=0; while i<ilen { tok_ptr[i]=(input as i64)+i; tok_len[i]=1; i=i+1 }
74 // merge loop
75 var go: i64=1
76 while go==1 {
77 var best: i64=0-1; var besti: i64=0-1
78 var k: i64=0
79 while k<ntok-1 {
80 let r: i64=merge_rank(buf, mfirst, nm, tok_ptr[k] as *u8, tok_len[k], tok_ptr[k+1] as *u8, tok_len[k+1], tmp)
81 if r>=0 { if besti<0 { best=r; besti=k } else { if r<best { best=r; besti=k } } }
82 k=k+1
83 }
84 if besti<0 { go=0 } else {
85 // merge besti, besti+1 -> new buffer
86 let xl: i64=tok_len[besti]; let yl: i64=tok_len[besti+1]
87 let nb: *u8=sys_mmap(xl+yl+8)
88 let xp: *u8=tok_ptr[besti] as *u8; let yp: *u8=tok_ptr[besti+1] as *u8
89 var c: i64=0; while c<xl { nb[c]=xp[c]; c=c+1 } var d: i64=0; while d<yl { nb[xl+d]=yp[d]; d=d+1 }
90 tok_ptr[besti]=nb as i64; tok_len[besti]=xl+yl
91 // shift left
92 var s: i64=besti+1; while s<ntok-1 { tok_ptr[s]=tok_ptr[s+1]; tok_len[s]=tok_len[s+1]; s=s+1 }
93 ntok=ntok-1
94 }
95 }
96 ran=1
97 // map tokens -> ids; check validity + concat == input
98 all_ids_valid=1
99 let cc: *u8=sys_mmap(64); var cp: i64=0
100 i=0
101 while i<ntok {
102 ids[i]=find_token_id(buf, vfirst, vocab, tok_ptr[i] as *u8, tok_len[i])
103 if ids[i]<0 { all_ids_valid=0 }
104 let tp: *u8=tok_ptr[i] as *u8; var z: i64=0; while z<tok_len[i] { cc[cp]=tp[z]; cp=cp+1; z=z+1 }
105 i=i+1
106 }
107 if cp==ilen { if bytes_eq(cc, input, ilen)==1 { concat_ok=1 } }
108 } }
109 }
110 } }
111
112 en_puts(" encode('hello') -> "); en_num(ntok); en_puts(" tokens: ids = [")
113 var w: i64=0; while w<ntok { en_num(ids[w]); if w<ntok-1 { en_puts(", ") } w=w+1 } en_puts("] token strings = '")
114 w=0; while w<ntok { let tp: *u8=tok_ptr[w] as *u8; sys_write(1, tp, tok_len[w]); if w<ntok-1 { en_puts("|") } w=w+1 } en_puts("'\n\n")
115
116 var pass: i64=0
117 var ttl: i64=0
118 ttl=ttl+1; en_puts(" T1 encode ran (merge loop applied + produced >=1 token): "); if ran==1 { if ntok>=1 { pass=pass+1; en_puts("PASS\n") } else { en_puts("FAIL\n") } } else { en_puts("FAIL\n") }
119 ttl=ttl+1; en_puts(" T2 every encoded token maps to a VALID vocab id (>=0): "); if all_ids_valid==1 { pass=pass+1; en_puts("PASS\n") } else { en_puts("FAIL\n") }
120 ttl=ttl+1; en_puts(" T3 ROUND-TRIP: the tokens' strings concatenate back to the input 'hello': "); if concat_ok==1 { pass=pass+1; en_puts("PASS\n") } else { en_puts("FAIL\n") }
121 ttl=ttl+1; en_puts(" T4 BPE actually MERGED (fewer tokens than the 5 input chars -- real merges applied): "); if ntok<5 { if ntok>=1 { pass=pass+1; en_puts("PASS\n") } else { en_puts("FAIL\n") } } else { en_puts("FAIL\n") }
122
123 en_puts("NX-NOFLOAT-QWEN-ENCODE-GATE passed "); en_num(pass); en_puts("/"); en_num(ttl)
124 // MIGRATED onto nx_gate_verdict by nx_gate_dry_apply (D001, minimal form): every check
125 // row above is untouched, so the PASS/FAIL vector cannot change; only the hand-rolled
126 // verdict emission is replaced by the ONE shared base class. Proven by nx_gate_migrate verify.
127 let ctr__dry: *i64 = gv_ctr()
128 ctr__dry[0] = pass
129 ctr__dry[1] = ttl
130 let rc__dry: i64 = gv_verdict("NOFLOAT-QWEN-ENCODE-GATE" as *u8, ctr__dry, "sovereign BPE encode: text -> token ids, round-trips decode -- full text<->ids tokenizer)" as *u8)
131 sys_exit(rc__dry)
132 return rc__dry
133}