nx_nofloat_qwen_encode_prep_gate.nx source
↩ module page · 90 lines · 5708 B
1// nx_nofloat_qwen_encode_prep_gate.nx -- ENCODE foundation: the BPE merges + the reverse (string -> id) lookup.
2// BPE encode (text -> ids) needs (a) the merge rules (tokenizer.ggml.merges, an ARRAY of STRING ranked by
3// priority) and (b) a token-string -> id map (the reverse of decode). This gate proves both: merges present +
4// counted, and string->id round-trips decode ("!"->0, double-quote->1) with a negative control. Pure
5// metadata + byte-compare (nx_gguf_meta; no LLM-helper dup). The full merge-apply loop is the next rung.
6// No hw writes (Rule 26). expect_exit: 0 license_tier: ORIGINAL
7import "nx_syscalls.nx"
8import "nx_tier.nx"
9import "nx_le.nx"
10import "nx_tensor.nx"
11import "nx_gguf.nx"
12import "nx_gguf_meta.nx"
13import "nx_gate_verdict.nx"
14import "nx_stage_path.nx"
15
16func ep_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
17func ep_num(v: i64) -> i64 { let b: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)} let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i: i64=0; while i<k{b[i]=t[k-1-i];i=i+1} sys_write(1,b,k); return 0 }
18func ep_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
19func bytes_eq(a: *u8, b: *u8, n: i64) -> i64 { var i: i64=0; while i<n { if a[i]!=b[i] { return 0 } i=i+1 } return 1 }
20// find the vocab id of a token string (search the length-prefixed array), or -1.
21func find_token_id(buf: *u8, first: i64, vocab: i64, target: *u8, tlen: i64) -> i64 {
22 var off: i64=first; var i: i64=0
23 while i<vocab {
24 let sl: i64=nx_gguf_meta_read_string_len(buf, off)
25 if sl==tlen { let sp: *u8=nx_gguf_meta_read_string_ptr(buf, off); if bytes_eq(sp, target, tlen)==1 { return i } }
26 off=off+8+sl; i=i+1
27 }
28 return 0 - 1
29}
30
31func main() -> i64 {
32 ep_puts("ENCODE foundation: BPE merges present + reverse (token-string -> id) lookup, from the real GGUF\n\n" as *u8)
33 let path: *u8 = sp_path("nx_real_model.gguf" as *u8, sys_mmap(SP_PATH_MAX))
34 sp_skip_unless("NOFLOAT-QWEN-ENCODE-PREP-GATE" as *u8, path)
35 let len_out: *i64 = sys_mmap(8) as *i64
36 len_out[0]=0
37 let buf: *u8 = sys_read_file(path, len_out)
38
39 var merges_ok: i64=0
40 var n_merges: i64=0
41 var id_excl: i64=0-9
42 var id_quote: i64=0-9
43 var id_garbage: i64=0-9
44 if buf != (0 as *u8) { if len_out[0] > 1000 {
45 let hdr: *NxGgufHeader = sys_mmap(NX_GGUF_HDR_BYTES) as *NxGgufHeader
46 if nx_gguf_parse(buf, len_out[0], hdr) == NX_GGUF_OK {
47 let voff: *i64 = sys_mmap(8) as *i64
48 let vty: *i64 = sys_mmap(8) as *i64
49 // merges
50 let km: *u8 = "tokenizer.ggml.merges\x00" as *u8
51 if nx_gguf_meta_find(buf, len_out[0], hdr, km, ep_slen(km), voff, vty) == NX_GMETA_OK {
52 if vty[0]==NX_GGUF_TYPE_ARRAY { if nx_gguf_meta_array_inner_type(buf, voff[0])==NX_GGUF_TYPE_STRING { merges_ok=1 } }
53 n_merges = nx_gguf_meta_array_count(buf, voff[0])
54 }
55 // tokens array (for string->id)
56 let kt: *u8 = "tokenizer.ggml.tokens\x00" as *u8
57 if nx_gguf_meta_find(buf, len_out[0], hdr, kt, ep_slen(kt), voff, vty) == NX_GMETA_OK {
58 let vocab: i64 = nx_gguf_meta_array_count(buf, voff[0])
59 let first: i64 = nx_gguf_meta_array_first_elt_off(buf, voff[0])
60 let t1: *u8 = sys_mmap(8); t1[0]=33 as u8 // "!"
61 let t2: *u8 = sys_mmap(8); t2[0]=34 as u8 // double-quote
62 let tg: *u8 = sys_mmap(8); tg[0]=255 as u8; tg[1]=254 as u8; tg[2]=253 as u8; tg[3]=252 as u8 // garbage
63 id_excl = find_token_id(buf, first, vocab, t1, 1)
64 id_quote = find_token_id(buf, first, vocab, t2, 1)
65 id_garbage = find_token_id(buf, first, vocab, tg, 4)
66 }
67 }
68 } }
69
70 ep_puts(" merges: present="); ep_num(merges_ok); ep_puts(" count="); ep_num(n_merges); ep_puts("\n");
71 ep_puts(" string->id: '!'="); ep_num(id_excl); ep_puts(" (want 0) doublequote="); ep_num(id_quote); ep_puts(" (want 1) garbage="); ep_num(id_garbage); ep_puts(" (want -1)\n\n");
72
73 var pass: i64=0
74 var ttl: i64=0
75 ttl=ttl+1; ep_puts(" T1 BPE merges present (tokenizer.ggml.merges = ARRAY of STRING, >100000 rules): "); if merges_ok==1 { if n_merges>100000 { pass=pass+1; ep_puts("PASS\n") } else { ep_puts("FAIL\n") } } else { ep_puts("FAIL\n") }
76 ttl=ttl+1; ep_puts(" T2 string->id round-trips decode ('!' -> 0): "); if id_excl==0 { pass=pass+1; ep_puts("PASS\n") } else { ep_puts("FAIL\n") }
77 ttl=ttl+1; ep_puts(" T3 string->id round-trips decode (double-quote -> 1): "); if id_quote==1 { pass=pass+1; ep_puts("PASS\n") } else { ep_puts("FAIL\n") }
78 ttl=ttl+1; ep_puts(" T4 NEG control: a non-existent token string -> -1 (lookup has teeth): "); if id_garbage==(0-1) { pass=pass+1; ep_puts("PASS\n") } else { ep_puts("FAIL\n") }
79
80 ep_puts("NX-NOFLOAT-QWEN-ENCODE-PREP-GATE passed "); ep_num(pass); ep_puts("/"); ep_num(ttl)
81 // MIGRATED onto nx_gate_verdict by nx_gate_dry_apply (D001, minimal form): every check
82 // row above is untouched, so the PASS/FAIL vector cannot change; only the hand-rolled
83 // verdict emission is replaced by the ONE shared base class. Proven by nx_gate_migrate verify.
84 let ctr__dry: *i64 = gv_ctr()
85 ctr__dry[0] = pass
86 ctr__dry[1] = ttl
87 let rc__dry: i64 = gv_verdict("NOFLOAT-QWEN-ENCODE-PREP-GATE" as *u8, ctr__dry, "BPE merges + reverse string->id ready -- the foundation for text->ids encode)" as *u8)
88 sys_exit(rc__dry)
89 return rc__dry
90}