nx_nofloat_qwen_encode_gate.nx source
↩ module page · 131 lines · 8146 B
1// nx_nofloat_qwen_encode_gate.nx -- BPE ENCODE: text -> token ids, sovereign, from the real GGUF merges+vocab.
2// Algorithm: start with single-char tokens; repeatedly merge the adjacent pair (X,Y) whose rule "X Y" has the
3// LOWEST rank in tokenizer.ggml.merges; stop when no adjacent pair has a merge; map each final token-string to
4// its id. (ASCII input -> byte-level pre-tokenize is identity, so we encode "hello" directly.) Proven by
5// ROUND-TRIP: the encoded tokens' strings concatenate back to the input, and every token maps to a valid id
6// (so decode(ids) == input). Pure metadata + byte ops (nx_gguf_meta). No hw writes (Rule 26).
7// expect_exit: 0 license_tier: ORIGINAL
8import "nx_syscalls.nx"
9import "nx_tier.nx"
10import "nx_le.nx"
11import "nx_tensor.nx"
12import "nx_gguf.nx"
13import "nx_gguf_meta.nx"
14import "nx_gate_verdict.nx"
15
16func en_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
17func en_num(v: i64) -> i64 { let b: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)} let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i: i64=0; while i<k{b[i]=t[k-1-i];i=i+1} sys_write(1,b,k); return 0 }
18func en_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
19func bytes_eq(a: *u8, b: *u8, n: i64) -> i64 { var i: i64=0; while i<n { if a[i]!=b[i] { return 0 } i=i+1 } return 1 }
20func find_token_id(buf: *u8, first: i64, vocab: i64, target: *u8, tlen: i64) -> i64 {
21 var off: i64=first; var i: i64=0
22 while i<vocab { let sl: i64=nx_gguf_meta_read_string_len(buf, off); if sl==tlen { let sp: *u8=nx_gguf_meta_read_string_ptr(buf, off); if bytes_eq(sp, target, tlen)==1 { return i } } off=off+8+sl; i=i+1 }
23 return 0 - 1
24}
25// rank of merge rule "X Y" (Xbytes + space + Ybytes) in the merges array, or -1.
26func merge_rank(buf: *u8, mfirst: i64, nm: i64, xp: *u8, xl: i64, yp: *u8, yl: i64, tmp: *u8) -> i64 {
27 var p: i64=0; var i: i64=0
28 while i<xl { tmp[p]=xp[i]; p=p+1; i=i+1 }
29 tmp[p]=32 as u8; p=p+1
30 i=0; while i<yl { tmp[p]=yp[i]; p=p+1; i=i+1 }
31 let tl: i64=p
32 var off: i64=mfirst; var j: i64=0
33 while j<nm { let ml: i64=nx_gguf_meta_read_string_len(buf, off); if ml==tl { let mp: *u8=nx_gguf_meta_read_string_ptr(buf, off); if bytes_eq(mp, tmp, tl)==1 { return j } } off=off+8+ml; j=j+1 }
34 return 0 - 1
35}
36
37func main() -> i64 {
38 en_puts("BPE ENCODE: text -> token ids (merge by rank), round-trip vs decode -- sovereign, real GGUF tokenizer\n\n" as *u8)
39 let path: *u8 = "/home/elderwesto/nx_stage/nx_real_model.gguf\x00" as *u8
40 let len_out: *i64 = sys_mmap(8) as *i64
41 len_out[0]=0
42 let buf: *u8 = sys_read_file(path, len_out)
43
44 // input "hello" (ASCII -> byte-level identity)
45 let input: *u8 = sys_mmap(16)
46 input[0]=104 as u8; input[1]=101 as u8; input[2]=108 as u8; input[3]=108 as u8; input[4]=111 as u8
47 let ilen: i64=5
48
49 var ran: i64=0
50 var ntok: i64=0
51 var all_ids_valid: i64=0
52 var concat_ok: i64=0
53 let MAXTOK: i64=64
54 let tok_ptr: *i64 = sys_mmap(MAXTOK*8) as *i64
55 let tok_len: *i64 = sys_mmap(MAXTOK*8) as *i64
56 let ids: *i64 = sys_mmap(MAXTOK*8) as *i64
57 let tmp: *u8 = sys_mmap(256)
58 if buf != (0 as *u8) { if len_out[0] > 1000 {
59 let hdr: *NxGgufHeader = sys_mmap(NX_GGUF_HDR_BYTES) as *NxGgufHeader
60 if nx_gguf_parse(buf, len_out[0], hdr) == NX_GGUF_OK {
61 let voff: *i64 = sys_mmap(8) as *i64
62 let vty: *i64 = sys_mmap(8) as *i64
63 let km: *u8 = "tokenizer.ggml.merges\x00" as *u8
64 let kt: *u8 = "tokenizer.ggml.tokens\x00" as *u8
65 var mfirst: i64=0; var nm: i64=0; var vfirst: i64=0; var vocab: i64=0
66 if nx_gguf_meta_find(buf, len_out[0], hdr, km, en_slen(km), voff, vty) == NX_GMETA_OK { nm=nx_gguf_meta_array_count(buf, voff[0]); mfirst=nx_gguf_meta_array_first_elt_off(buf, voff[0]) }
67 if nx_gguf_meta_find(buf, len_out[0], hdr, kt, en_slen(kt), voff, vty) == NX_GMETA_OK { vocab=nx_gguf_meta_array_count(buf, voff[0]); vfirst=nx_gguf_meta_array_first_elt_off(buf, voff[0]) }
68 if nm>0 { if vocab>0 {
69 // init: one token per input byte (pointers into `input`)
70 ntok=ilen
71 var i: i64=0; while i<ilen { tok_ptr[i]=(input as i64)+i; tok_len[i]=1; i=i+1 }
72 // merge loop
73 var go: i64=1
74 while go==1 {
75 var best: i64=0-1; var besti: i64=0-1
76 var k: i64=0
77 while k<ntok-1 {
78 let r: i64=merge_rank(buf, mfirst, nm, tok_ptr[k] as *u8, tok_len[k], tok_ptr[k+1] as *u8, tok_len[k+1], tmp)
79 if r>=0 { if besti<0 { best=r; besti=k } else { if r<best { best=r; besti=k } } }
80 k=k+1
81 }
82 if besti<0 { go=0 } else {
83 // merge besti, besti+1 -> new buffer
84 let xl: i64=tok_len[besti]; let yl: i64=tok_len[besti+1]
85 let nb: *u8=sys_mmap(xl+yl+8)
86 let xp: *u8=tok_ptr[besti] as *u8; let yp: *u8=tok_ptr[besti+1] as *u8
87 var c: i64=0; while c<xl { nb[c]=xp[c]; c=c+1 } var d: i64=0; while d<yl { nb[xl+d]=yp[d]; d=d+1 }
88 tok_ptr[besti]=nb as i64; tok_len[besti]=xl+yl
89 // shift left
90 var s: i64=besti+1; while s<ntok-1 { tok_ptr[s]=tok_ptr[s+1]; tok_len[s]=tok_len[s+1]; s=s+1 }
91 ntok=ntok-1
92 }
93 }
94 ran=1
95 // map tokens -> ids; check validity + concat == input
96 all_ids_valid=1
97 let cc: *u8=sys_mmap(64); var cp: i64=0
98 i=0
99 while i<ntok {
100 ids[i]=find_token_id(buf, vfirst, vocab, tok_ptr[i] as *u8, tok_len[i])
101 if ids[i]<0 { all_ids_valid=0 }
102 let tp: *u8=tok_ptr[i] as *u8; var z: i64=0; while z<tok_len[i] { cc[cp]=tp[z]; cp=cp+1; z=z+1 }
103 i=i+1
104 }
105 if cp==ilen { if bytes_eq(cc, input, ilen)==1 { concat_ok=1 } }
106 } }
107 }
108 } }
109
110 en_puts(" encode('hello') -> "); en_num(ntok); en_puts(" tokens: ids = [")
111 var w: i64=0; while w<ntok { en_num(ids[w]); if w<ntok-1 { en_puts(", ") } w=w+1 } en_puts("] token strings = '")
112 w=0; while w<ntok { let tp: *u8=tok_ptr[w] as *u8; sys_write(1, tp, tok_len[w]); if w<ntok-1 { en_puts("|") } w=w+1 } en_puts("'\n\n")
113
114 var pass: i64=0
115 var ttl: i64=0
116 ttl=ttl+1; en_puts(" T1 encode ran (merge loop applied + produced >=1 token): "); if ran==1 { if ntok>=1 { pass=pass+1; en_puts("PASS\n") } else { en_puts("FAIL\n") } } else { en_puts("FAIL\n") }
117 ttl=ttl+1; en_puts(" T2 every encoded token maps to a VALID vocab id (>=0): "); if all_ids_valid==1 { pass=pass+1; en_puts("PASS\n") } else { en_puts("FAIL\n") }
118 ttl=ttl+1; en_puts(" T3 ROUND-TRIP: the tokens' strings concatenate back to the input 'hello': "); if concat_ok==1 { pass=pass+1; en_puts("PASS\n") } else { en_puts("FAIL\n") }
119 ttl=ttl+1; en_puts(" T4 BPE actually MERGED (fewer tokens than the 5 input chars -- real merges applied): "); if ntok<5 { if ntok>=1 { pass=pass+1; en_puts("PASS\n") } else { en_puts("FAIL\n") } } else { en_puts("FAIL\n") }
120
121 en_puts("NX-NOFLOAT-QWEN-ENCODE-GATE passed "); en_num(pass); en_puts("/"); en_num(ttl)
122 // MIGRATED onto nx_gate_verdict by nx_gate_dry_apply (D001, minimal form): every check
123 // row above is untouched, so the PASS/FAIL vector cannot change; only the hand-rolled
124 // verdict emission is replaced by the ONE shared base class. Proven by nx_gate_migrate verify.
125 let ctr__dry: *i64 = gv_ctr()
126 ctr__dry[0] = pass
127 ctr__dry[1] = ttl
128 let rc__dry: i64 = gv_verdict("NOFLOAT-QWEN-ENCODE-GATE" as *u8, ctr__dry, "sovereign BPE encode: text -> token ids, round-trips decode -- full text<->ids tokenizer)" as *u8)
129 sys_exit(rc__dry)
130 return rc__dry
131}