nx_nofloat_bpe_gate.nx source

1// nx_nofloat_bpe_gate.nx -- R2 (efficiency) from the sovereign-researcher roadmap 2// (knowledge/research/2026-06-23-nofloat-affordable-land-roadmap.md, grounded in nfs_bpe.raw): a LEARNED 3// Byte-Pair-Encoding tokenizer. Greedily merge the most-frequent adjacent token pair K times -> frequent 4// substrings become single sub-word tokens -> FEWER tokens per text -> cheaper LM training + longer effective 5// context for the same compute. Pure integer, self-contained (nx_syscalls only). 6// T1 COMPRESSION: BPE token count << char count (ratio >= 1.4) = the efficiency win. 7// T2 LOSSLESS (teeth): recursively expand the BPE tokens back to bytes == the original corpus, byte-exact. 8// T3 learned K real merges (each a repeated pair). 9// expect_exit: 0 license_tier: ORIGINAL 10import "nx_syscalls.nx" 11import "nx_gate_emit_lib.nx" 12import "nx_gate_verdict.nx" 13const STRIDE: i64 = 320 // pair key = a*STRIDE + b (ids stay < 256+K < 320) 14 15 16func slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n } 17// recursively expand token tok into bytes appended at out[*w]; merges ma/mb indexed by (tok-256). 18func expand(tok: i64, ma: *i64, mb: *i64, out: *u8, w: *i64) -> i64 { 19 if tok < 256 { out[w[0]] = tok as u8; w[0] = w[0] + 1; return 0 } 20 let m: i64 = tok - 256 21 expand(ma[m], ma, mb, out, w) 22 expand(mb[m], ma, mb, out, w) 23 return 0 24} 25 26func main() -> i64 { 27 g_puts("nx_nofloat_bpe gate (learned BPE sub-word compression -- the R2 efficiency win)\n" as *u8) 28 let corpus: *u8 = "the cat sat the cat ran the dog sat the dog ran the cat ran the dog sat" as *u8 29 let L: i64 = slen(corpus) 30 let seq: *i64 = sys_mmap(L*8) as *i64 31 var n: i64=0; while n<L { seq[n]=corpus[n] as i64; n=n+1 } // start: one token per byte 32 let K: i64 = 14 33 let ma: *i64 = sys_mmap(K*8) as *i64; let mb: *i64 = sys_mmap(K*8) as *i64 34 let cnt: *i64 = sys_mmap(STRIDE*STRIDE*8) as *i64 // pair-frequency table 35 let seq2: *i64 = sys_mmap(L*8) as *i64 36 var nextid: i64 = 256 37 var learned: i64 = 0 38 var k: i64=0 39 while k<K { 40 // zero the pair table over the used id range only (ids < nextid) 41 var z: i64=0; let lim: i64 = nextid*STRIDE 42 while z<lim { cnt[z]=0; z=z+1 } 43 // count adjacent pairs in seq[0..n-1] 44 var i: i64=0 45 while i<n-1 { let key: i64=seq[i]*STRIDE+seq[i+1]; cnt[key]=cnt[key]+1; i=i+1 } 46 // find the most frequent pair 47 var bestkey: i64=0-1; var bestc: i64=1 48 i=0 49 while i<n-1 { let key: i64=seq[i]*STRIDE+seq[i+1]; if cnt[key]>bestc { bestc=cnt[key]; bestkey=key } i=i+1 } 50 if bestkey < 0 { k=K } else { 51 let a: i64=bestkey/STRIDE; let b: i64=bestkey-a*STRIDE 52 ma[learned]=a; mb[learned]=b; let newid: i64=nextid; nextid=nextid+1; learned=learned+1 53 // replace every adjacent (a,b) with newid 54 var w: i64=0; i=0 55 while i<n { if i<n-1 { if seq[i]==a { if seq[i+1]==b { seq2[w]=newid; w=w+1; i=i+2 } else { seq2[w]=seq[i]; w=w+1; i=i+1 } } else { seq2[w]=seq[i]; w=w+1; i=i+1 } } else { seq2[w]=seq[i]; w=w+1; i=i+1 } } 56 var c: i64=0; while c<w { seq[c]=seq2[c]; c=c+1 } n=w 57 k=k+1 58 } 59 } 60 let comp_n: i64 = n 61 g_puts(" [measure] chars=" as *u8); g_pn(L); g_puts(" BPE tokens=" as *u8); g_pn(comp_n); g_puts(" merges learned=" as *u8); g_pn(learned); g_puts(" ratio(x100)=" as *u8); g_pn(L*100/comp_n); g_puts("\n" as *u8) 62 63 // ---- T2: lossless reconstruction ---- 64 let recon: *u8 = sys_mmap(L+16); let w: *i64 = sys_mmap(8) as *i64; w[0]=0 65 var t: i64=0; while t<comp_n { expand(seq[t], ma, mb, recon, w); t=t+1 } 66 var lossless: i64=1 67 if w[0] != L { lossless=0 } 68 var ci: i64=0; while ci<L { if recon[ci] != corpus[ci] { lossless=0 } ci=ci+1 } 69 70 var pass: i64=0; var total: i64=0 71 var t1: i64=0; if L*10 >= comp_n*14 { t1=1 } // ratio >= 1.4 72 pass=pass+g_check("T1: COMPRESSION -- BPE token count <= chars/1.4 (fewer tokens/text = cheaper training)" as *u8, t1); total=total+1 73 pass=pass+g_check("T2: LOSSLESS -- BPE tokens expand back to the EXACT original corpus (correct tokenizer)" as *u8, lossless); total=total+1 74 var t3: i64=0; if learned >= 8 { t3=1 } 75 pass=pass+g_check("T3: learned >= 8 real BPE merges (frequent sub-words found)" as *u8, t3); total=total+1 76 77 var okall: i64=0; if pass==total { okall=1 } 78 if okall==1 { 79 let logf: i64 = sys_openat_append("knowledge/status/nofloat_bpe.log" as *u8, 420) 80 if logf >= 0 { let x0: i64=sys_write(logf,"NOFLOATBPE learned-bpe compression+lossless measured\n" as *u8,52); sys_close(logf) } 81 } 82 g_puts("---- bpe gate: passed " as *u8); g_pn(pass); g_puts(" / " as *u8); g_pn(total); g_puts(" ----\n" as *u8) 83 // MIGRATED onto nx_gate_verdict by nx_gate_dry_apply (D001, minimal form): every check 84 // row above is untouched, so the PASS/FAIL vector cannot change; only the hand-rolled 85 // verdict emission is replaced by the ONE shared base class. Proven by nx_gate_migrate verify. 86 let ctr__dry: *i64 = gv_ctr() 87 ctr__dry[0] = pass 88 ctr__dry[1] = total 89 let rc__dry: i64 = gv_verdict("NOFLOAT-BPE-GATE" as *u8, ctr__dry, "learned BPE: lossless sub-word compression = the R2 efficiency win toward landing)" as *u8) 90 sys_exit(rc__dry) 91 return rc__dry 92}