nx_nofloat_bpe_gate.nx source

1// nx_nofloat_bpe_gate.nx -- R2 (efficiency) from the sovereign-researcher roadmap 2// (knowledge/research/2026-06-23-nofloat-affordable-land-roadmap.md, grounded in nfs_bpe.raw): a LEARNED 3// Byte-Pair-Encoding tokenizer. Greedily merge the most-frequent adjacent token pair K times -> frequent 4// substrings become single sub-word tokens -> FEWER tokens per text -> cheaper LM training + longer effective 5// context for the same compute. Pure integer, self-contained (nx_syscalls only). 6// T1 COMPRESSION: BPE token count << char count (ratio >= 1.4) = the efficiency win. 7// T2 LOSSLESS (teeth): recursively expand the BPE tokens back to bytes == the original corpus, byte-exact. 8// T3 learned K real merges (each a repeated pair). 9// expect_exit: 0 license_tier: ORIGINAL 10import "nx_syscalls.nx" 11import "nx_gate_emit_lib.nx" 12const STRIDE: i64 = 320 // pair key = a*STRIDE + b (ids stay < 256+K < 320) 13 14 15func slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n } 16// recursively expand token tok into bytes appended at out[*w]; merges ma/mb indexed by (tok-256). 17func expand(tok: i64, ma: *i64, mb: *i64, out: *u8, w: *i64) -> i64 { 18 if tok < 256 { out[w[0]] = tok as u8; w[0] = w[0] + 1; return 0 } 19 let m: i64 = tok - 256 20 expand(ma[m], ma, mb, out, w) 21 expand(mb[m], ma, mb, out, w) 22 return 0 23} 24 25func main() -> i64 { 26 g_puts("nx_nofloat_bpe gate (learned BPE sub-word compression -- the R2 efficiency win)\n" as *u8) 27 let corpus: *u8 = "the cat sat the cat ran the dog sat the dog ran the cat ran the dog sat" as *u8 28 let L: i64 = slen(corpus) 29 let seq: *i64 = sys_mmap(L*8) as *i64 30 var n: i64=0; while n<L { seq[n]=corpus[n] as i64; n=n+1 } // start: one token per byte 31 let K: i64 = 14 32 let ma: *i64 = sys_mmap(K*8) as *i64; let mb: *i64 = sys_mmap(K*8) as *i64 33 let cnt: *i64 = sys_mmap(STRIDE*STRIDE*8) as *i64 // pair-frequency table 34 let seq2: *i64 = sys_mmap(L*8) as *i64 35 var nextid: i64 = 256 36 var learned: i64 = 0 37 var k: i64=0 38 while k<K { 39 // zero the pair table over the used id range only (ids < nextid) 40 var z: i64=0; let lim: i64 = nextid*STRIDE 41 while z<lim { cnt[z]=0; z=z+1 } 42 // count adjacent pairs in seq[0..n-1] 43 var i: i64=0 44 while i<n-1 { let key: i64=seq[i]*STRIDE+seq[i+1]; cnt[key]=cnt[key]+1; i=i+1 } 45 // find the most frequent pair 46 var bestkey: i64=0-1; var bestc: i64=1 47 i=0 48 while i<n-1 { let key: i64=seq[i]*STRIDE+seq[i+1]; if cnt[key]>bestc { bestc=cnt[key]; bestkey=key } i=i+1 } 49 if bestkey < 0 { k=K } else { 50 let a: i64=bestkey/STRIDE; let b: i64=bestkey-a*STRIDE 51 ma[learned]=a; mb[learned]=b; let newid: i64=nextid; nextid=nextid+1; learned=learned+1 52 // replace every adjacent (a,b) with newid 53 var w: i64=0; i=0 54 while i<n { if i<n-1 { if seq[i]==a { if seq[i+1]==b { seq2[w]=newid; w=w+1; i=i+2 } else { seq2[w]=seq[i]; w=w+1; i=i+1 } } else { seq2[w]=seq[i]; w=w+1; i=i+1 } } else { seq2[w]=seq[i]; w=w+1; i=i+1 } } 55 var c: i64=0; while c<w { seq[c]=seq2[c]; c=c+1 } n=w 56 k=k+1 57 } 58 } 59 let comp_n: i64 = n 60 g_puts(" [measure] chars=" as *u8); g_pn(L); g_puts(" BPE tokens=" as *u8); g_pn(comp_n); g_puts(" merges learned=" as *u8); g_pn(learned); g_puts(" ratio(x100)=" as *u8); g_pn(L*100/comp_n); g_puts("\n" as *u8) 61 62 // ---- T2: lossless reconstruction ---- 63 let recon: *u8 = sys_mmap(L+16); let w: *i64 = sys_mmap(8) as *i64; w[0]=0 64 var t: i64=0; while t<comp_n { expand(seq[t], ma, mb, recon, w); t=t+1 } 65 var lossless: i64=1 66 if w[0] != L { lossless=0 } 67 var ci: i64=0; while ci<L { if recon[ci] != corpus[ci] { lossless=0 } ci=ci+1 } 68 69 var pass: i64=0; var total: i64=0 70 var t1: i64=0; if L*10 >= comp_n*14 { t1=1 } // ratio >= 1.4 71 pass=pass+g_check("T1: COMPRESSION -- BPE token count <= chars/1.4 (fewer tokens/text = cheaper training)" as *u8, t1); total=total+1 72 pass=pass+g_check("T2: LOSSLESS -- BPE tokens expand back to the EXACT original corpus (correct tokenizer)" as *u8, lossless); total=total+1 73 var t3: i64=0; if learned >= 8 { t3=1 } 74 pass=pass+g_check("T3: learned >= 8 real BPE merges (frequent sub-words found)" as *u8, t3); total=total+1 75 76 var okall: i64=0; if pass==total { okall=1 } 77 if okall==1 { 78 let logf: i64 = sys_openat_append("knowledge/status/nofloat_bpe.log" as *u8, 420) 79 if logf >= 0 { let x0: i64=sys_write(logf,"NOFLOATBPE learned-bpe compression+lossless measured\n" as *u8,52); sys_close(logf) } 80 } 81 g_puts("---- bpe gate: passed " as *u8); g_pn(pass); g_puts(" / " as *u8); g_pn(total); g_puts(" ----\n" as *u8) 82 if okall==1 { g_puts("verdict=GREEN (learned BPE: lossless sub-word compression = the R2 efficiency win toward landing)\n" as *u8); sys_exit(0); return 0 } 83 g_puts("verdict=RED\n" as *u8); sys_exit(1); return 1 84}