nx_nofloat_qwen_encode_prep_gate.nx source
↩ module page · 88 lines · 5615 B
1// nx_nofloat_qwen_encode_prep_gate.nx -- ENCODE foundation: the BPE merges + the reverse (string -> id) lookup.
2// BPE encode (text -> ids) needs (a) the merge rules (tokenizer.ggml.merges, an ARRAY of STRING ranked by
3// priority) and (b) a token-string -> id map (the reverse of decode). This gate proves both: merges present +
4// counted, and string->id round-trips decode ("!"->0, double-quote->1) with a negative control. Pure
5// metadata + byte-compare (nx_gguf_meta; no LLM-helper dup). The full merge-apply loop is the next rung.
6// No hw writes (Rule 26). expect_exit: 0 license_tier: ORIGINAL
7import "nx_syscalls.nx"
8import "nx_tier.nx"
9import "nx_le.nx"
10import "nx_tensor.nx"
11import "nx_gguf.nx"
12import "nx_gguf_meta.nx"
13import "nx_gate_verdict.nx"
14
15func ep_puts(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(1,s,n); return 0 }
16func ep_num(v: i64) -> i64 { let b: *u8=sys_mmap(28); var m: i64=v; if m<0{m=0-m;sys_write(1,"-" as *u8,1)} let t: *u8=sys_mmap(28); var k: i64=0; if m==0{t[0]=48 as u8;k=1} while m>0{t[k]=(48+(m%10)) as u8;m=m/10;k=k+1} var i: i64=0; while i<k{b[i]=t[k-1-i];i=i+1} sys_write(1,b,k); return 0 }
17func ep_slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
18func bytes_eq(a: *u8, b: *u8, n: i64) -> i64 { var i: i64=0; while i<n { if a[i]!=b[i] { return 0 } i=i+1 } return 1 }
19// find the vocab id of a token string (search the length-prefixed array), or -1.
20func find_token_id(buf: *u8, first: i64, vocab: i64, target: *u8, tlen: i64) -> i64 {
21 var off: i64=first; var i: i64=0
22 while i<vocab {
23 let sl: i64=nx_gguf_meta_read_string_len(buf, off)
24 if sl==tlen { let sp: *u8=nx_gguf_meta_read_string_ptr(buf, off); if bytes_eq(sp, target, tlen)==1 { return i } }
25 off=off+8+sl; i=i+1
26 }
27 return 0 - 1
28}
29
30func main() -> i64 {
31 ep_puts("ENCODE foundation: BPE merges present + reverse (token-string -> id) lookup, from the real GGUF\n\n" as *u8)
32 let path: *u8 = "/home/elderwesto/nx_stage/nx_real_model.gguf\x00" as *u8
33 let len_out: *i64 = sys_mmap(8) as *i64
34 len_out[0]=0
35 let buf: *u8 = sys_read_file(path, len_out)
36
37 var merges_ok: i64=0
38 var n_merges: i64=0
39 var id_excl: i64=0-9
40 var id_quote: i64=0-9
41 var id_garbage: i64=0-9
42 if buf != (0 as *u8) { if len_out[0] > 1000 {
43 let hdr: *NxGgufHeader = sys_mmap(NX_GGUF_HDR_BYTES) as *NxGgufHeader
44 if nx_gguf_parse(buf, len_out[0], hdr) == NX_GGUF_OK {
45 let voff: *i64 = sys_mmap(8) as *i64
46 let vty: *i64 = sys_mmap(8) as *i64
47 // merges
48 let km: *u8 = "tokenizer.ggml.merges\x00" as *u8
49 if nx_gguf_meta_find(buf, len_out[0], hdr, km, ep_slen(km), voff, vty) == NX_GMETA_OK {
50 if vty[0]==NX_GGUF_TYPE_ARRAY { if nx_gguf_meta_array_inner_type(buf, voff[0])==NX_GGUF_TYPE_STRING { merges_ok=1 } }
51 n_merges = nx_gguf_meta_array_count(buf, voff[0])
52 }
53 // tokens array (for string->id)
54 let kt: *u8 = "tokenizer.ggml.tokens\x00" as *u8
55 if nx_gguf_meta_find(buf, len_out[0], hdr, kt, ep_slen(kt), voff, vty) == NX_GMETA_OK {
56 let vocab: i64 = nx_gguf_meta_array_count(buf, voff[0])
57 let first: i64 = nx_gguf_meta_array_first_elt_off(buf, voff[0])
58 let t1: *u8 = sys_mmap(8); t1[0]=33 as u8 // "!"
59 let t2: *u8 = sys_mmap(8); t2[0]=34 as u8 // double-quote
60 let tg: *u8 = sys_mmap(8); tg[0]=255 as u8; tg[1]=254 as u8; tg[2]=253 as u8; tg[3]=252 as u8 // garbage
61 id_excl = find_token_id(buf, first, vocab, t1, 1)
62 id_quote = find_token_id(buf, first, vocab, t2, 1)
63 id_garbage = find_token_id(buf, first, vocab, tg, 4)
64 }
65 }
66 } }
67
68 ep_puts(" merges: present="); ep_num(merges_ok); ep_puts(" count="); ep_num(n_merges); ep_puts("\n");
69 ep_puts(" string->id: '!'="); ep_num(id_excl); ep_puts(" (want 0) doublequote="); ep_num(id_quote); ep_puts(" (want 1) garbage="); ep_num(id_garbage); ep_puts(" (want -1)\n\n");
70
71 var pass: i64=0
72 var ttl: i64=0
73 ttl=ttl+1; ep_puts(" T1 BPE merges present (tokenizer.ggml.merges = ARRAY of STRING, >100000 rules): "); if merges_ok==1 { if n_merges>100000 { pass=pass+1; ep_puts("PASS\n") } else { ep_puts("FAIL\n") } } else { ep_puts("FAIL\n") }
74 ttl=ttl+1; ep_puts(" T2 string->id round-trips decode ('!' -> 0): "); if id_excl==0 { pass=pass+1; ep_puts("PASS\n") } else { ep_puts("FAIL\n") }
75 ttl=ttl+1; ep_puts(" T3 string->id round-trips decode (double-quote -> 1): "); if id_quote==1 { pass=pass+1; ep_puts("PASS\n") } else { ep_puts("FAIL\n") }
76 ttl=ttl+1; ep_puts(" T4 NEG control: a non-existent token string -> -1 (lookup has teeth): "); if id_garbage==(0-1) { pass=pass+1; ep_puts("PASS\n") } else { ep_puts("FAIL\n") }
77
78 ep_puts("NX-NOFLOAT-QWEN-ENCODE-PREP-GATE passed "); ep_num(pass); ep_puts("/"); ep_num(ttl)
79 // MIGRATED onto nx_gate_verdict by nx_gate_dry_apply (D001, minimal form): every check
80 // row above is untouched, so the PASS/FAIL vector cannot change; only the hand-rolled
81 // verdict emission is replaced by the ONE shared base class. Proven by nx_gate_migrate verify.
82 let ctr__dry: *i64 = gv_ctr()
83 ctr__dry[0] = pass
84 ctr__dry[1] = ttl
85 let rc__dry: i64 = gv_verdict("NOFLOAT-QWEN-ENCODE-PREP-GATE" as *u8, ctr__dry, "BPE merges + reverse string->id ready -- the foundation for text->ids encode)" as *u8)
86 sys_exit(rc__dry)
87 return rc__dry
88}