code wiki / _hdl_build / nx_nofloat_bpe_lm_gate.nx
`nx_nofloat_bpe_lm_gate.nx` source

↩ module page · 149 lines · 9726 B
1// nx_nofloat_bpe_lm_gate.nx -- CAPSTONE: the WHOLE sovereign efficient text pipeline wired END-TO-END, pure
2// no-float Q16. Composes R2 (learned BPE, nx_nofloat_bpe_gate) + R1 (the no-float transformer LM):
3//   text -> learn K BPE merges -> compress to sub-word tokens -> remap to a compact vocab -> the no-float LM
4//   trains to model the token stream -> autoregressive argmax generation -> decode tokens back through the
5//   merges to bytes -> the ORIGINAL text.
6//   T1: the LM learns the BPE-token stream (argmax generation reproduces every token).
7//   T2: END-TO-END LOSSLESS -- the decoded generation equals the original text, byte-exact (the full
8//       tokenize->train->generate->detokenize loop is correct).
9//   T3: EFFICIENCY -- the BPE token stream is shorter than the char stream (fewer tokens to model).
10// Sovereign: nx_nofloat_autograd + nx_syscalls. Reuses verified pieces (BPE merge-learn, clm_fwd attention LM,
11// fixed-T argmax decode, recursive merge-expand). expect_exit: 0
12import "nx_nofloat_autograd.nx"
13import "nx_syscalls.nx"
14import "nx_gate_emit_lib.nx"
15const Q16: i64 = 65536
16const STRIDE: i64 = 320
17
18
19func slen(s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} return n }
20func dini(a: *i64, n: i64, sd: i64) -> i64 { var i: i64=0; while i<n { a[i]=(((i*7+sd*13+1)%11)-5)*13107; i=i+1 } return 0 }
21func expand(tok: i64, ma: *i64, mb: *i64, out: *u8, w: *i64) -> i64 {
  if tok < 256 { out[w[0]]=tok as u8; w[0]=w[0]+1; return 0 }
  let m: i64=tok-256; expand(ma[m],ma,mb,out,w); expand(mb[m],ma,mb,out,w); return 0
24}
25// attention-only causal LM (same op chain as the landing/grammar gates)
26func clm_fwd(tape: *i64, vals: *i64, st: *i64, W: *i64, ids: *i64, tgt: *i64, T: i64, dm: i64, V: i64, scale: i64, leaves: *i64) -> i64 {
  let E: *i64=W[0] as *i64; let Wq: *i64=W[1] as *i64; let Wk: *i64=W[2] as *i64; let Wv: *i64=W[3] as *i64; let Wo: *i64=W[4] as *i64; let Wlm: *i64=W[5] as *i64
  st[0]=0; st[1]=0
  let nE: i64=nfa_leaf(tape,vals,st,V,dm,E,0)
  let nWq: i64=nfa_leaf(tape,vals,st,dm,dm,Wq,0)
  let nWk: i64=nfa_leaf(tape,vals,st,dm,dm,Wk,0)
  let nWv: i64=nfa_leaf(tape,vals,st,dm,dm,Wv,0)
  let nWo: i64=nfa_leaf(tape,vals,st,dm,dm,Wo,0)
  let nWlm: i64=nfa_leaf(tape,vals,st,dm,V,Wlm,0)
  let nX: i64=nfa_embed(tape,vals,st,nE,ids,T)
  let nXn: i64=nfa_rmsnorm_rows(tape,vals,st,nX)
  let nQ: i64=nfa_matmul(tape,vals,st,nXn,nWq)
  let nK: i64=nfa_matmul(tape,vals,st,nXn,nWk)
  let nV: i64=nfa_matmul(tape,vals,st,nXn,nWv)
  let nQr: i64=nfa_rope(tape,vals,st,nQ)
  let nKr: i64=nfa_rope(tape,vals,st,nK)
  let nS: i64=nfa_matmul_nt(tape,vals,st,nQr,nKr)
  let nSs: i64=nfa_cmul(tape,vals,st,nS,scale)
  let nA: i64=nfa_softmax_rows(tape,vals,st,nSs,1)
  let nO: i64=nfa_matmul(tape,vals,st,nA,nV)
  let nOp: i64=nfa_matmul(tape,vals,st,nO,nWo)
  let nH: i64=nfa_vadd(tape,vals,st,nX,nOp)
  let nHn: i64=nfa_rmsnorm_rows(tape,vals,st,nH)
  let nLg: i64=nfa_matmul(tape,vals,st,nHn,nWlm)
  let nLoss: i64=nfa_softce_rows(tape,vals,st,nLg,tgt)
  leaves[0]=nE; leaves[1]=nWq; leaves[2]=nWk; leaves[3]=nWv; leaves[4]=nWo; leaves[5]=nWlm; leaves[6]=nLg
  return nLoss
53}
54func step_all(tape: *i64, grads: *i64, W: *i64, WN: *i64, leaves: *i64, lr: i64, clip: i64, gb: *i64) -> i64 {
  var i: i64=0
  while i<6 { let ar: *i64=W[i] as *i64; let cn: i64=WN[i]; let nd: i64=leaves[i]; var c: i64=0; while c<cn { var g: i64=nfa_grad(tape,grads,nd,c); if g>clip{g=clip} if g<0-clip{g=0-clip} gb[c]=g; c=c+1 } nfa_sgd(ar,gb,cn,lr); i=i+1 }
  return 0
58}
59func do_train(tape: *i64, vals: *i64, grads: *i64, st: *i64, W: *i64, WN: *i64, ids: *i64, tgt: *i64, T: i64, dm: i64, V: i64, scale: i64, leaves: *i64, gb: *i64, steps: i64) -> i64 {
  var ep: i64=0
  while ep<steps { let nl: i64=clm_fwd(tape,vals,st,W,ids,tgt,T,dm,V,scale,leaves); nfa_backward(tape,vals,grads,st[0],nl); step_all(tape,grads,W,WN,leaves,6554,262144,gb); ep=ep+1 }
  return 0
63}
64func amx(tape: *i64, vals: *i64, logn: i64, r: i64, V: i64) -> i64 { let o: i64=tape[7*logn+5]; var b: i64=0; var bv: i64=vals[o+r*V]; var j: i64=1; while j<V { if vals[o+r*V+j]>bv { bv=vals[o+r*V+j]; b=j } j=j+1 } return b }
65func main() -> i64 {
  g_puts("nx_nofloat_bpe_lm gate (END-TO-END: text -> BPE -> no-float LM -> generate -> detokenize -> text)\n" as *u8)
  let corpus: *u8 = "the cat ran the cat ran the dog sat the dog sat" as *u8
  let L: i64 = slen(corpus)
69
  // ---- 1. learn K BPE merges (greedy most-frequent pair) ----
  let seq: *i64 = sys_mmap(L*8) as *i64
  var ii: i64=0; while ii<L { seq[ii]=corpus[ii] as i64; ii=ii+1 }
  var n: i64=L
  let K: i64=12
  let ma: *i64=sys_mmap(K*8) as *i64; let mb: *i64=sys_mmap(K*8) as *i64
  let cnt: *i64=sys_mmap(STRIDE*STRIDE*8) as *i64
  let seq2: *i64=sys_mmap(L*8) as *i64
  var nextid: i64=256; var learned: i64=0
  var k: i64=0
  while k<K {
      var z: i64=0; let lim: i64=nextid*STRIDE; while z<lim { cnt[z]=0; z=z+1 }
      var i: i64=0; while i<n-1 { let key: i64=seq[i]*STRIDE+seq[i+1]; cnt[key]=cnt[key]+1; i=i+1 }
      var bestkey: i64=0-1; var bestc: i64=1
      i=0; while i<n-1 { let key: i64=seq[i]*STRIDE+seq[i+1]; if cnt[key]>bestc { bestc=cnt[key]; bestkey=key } i=i+1 }
      if bestkey<0 { k=K } else {
          let a: i64=bestkey/STRIDE; let b: i64=bestkey-a*STRIDE
          ma[learned]=a; mb[learned]=b; let newid: i64=nextid; nextid=nextid+1; learned=learned+1
          var w2: i64=0; i=0
          while i<n { if i<n-1 { if seq[i]==a { if seq[i+1]==b { seq2[w2]=newid; w2=w2+1; i=i+2 } else { seq2[w2]=seq[i]; w2=w2+1; i=i+1 } } else { seq2[w2]=seq[i]; w2=w2+1; i=i+1 } } else { seq2[w2]=seq[i]; w2=w2+1; i=i+1 } }
          var c: i64=0; while c<w2 { seq[c]=seq2[c]; c=c+1 } n=w2; k=k+1
      }
  }
  let M: i64=n      // BPE token-stream length
94
  // ---- 2. remap the sparse token ids to a compact vocab 0..U-1 ----
  let id2c: *i64=sys_mmap((256+K)*8) as *i64
  var q: i64=0; while q<256+K { id2c[q]=0-1; q=q+1 }
  let c2id: *i64=sys_mmap((256+K)*8) as *i64
  let comp: *i64=sys_mmap(M*8) as *i64
  var U: i64=0; var t: i64=0
  while t<M { let id: i64=seq[t]; if id2c[id]<0 { id2c[id]=U; c2id[U]=id; U=U+1 } comp[t]=id2c[id]; t=t+1 }
  g_puts("  [pipeline] chars="); g_pn(L); g_puts(" -> BPE tokens="); g_pn(M); g_puts(" (merges="); g_pn(learned); g_puts(", vocab U="); g_pn(U); g_puts(")\n")
103
  // ---- 3. train the no-float LM to model the compact stream (next-token) ----
  let dm: i64=32; let scale: i64=11585
  let T: i64=M-1
  let ids: *i64=sys_mmap(M*8) as *i64; let tgt: *i64=sys_mmap(M*8) as *i64
  var r: i64=0; while r<T { ids[r]=comp[r]; tgt[r]=comp[r+1]; r=r+1 }
  let tape: *i64=sys_mmap(512*7*8) as *i64
  let vals: *i64=sys_mmap(131072*8) as *i64
  let grads: *i64=sys_mmap(131072*8) as *i64
  let st: *i64=sys_mmap(2*8) as *i64
  let W: *i64=sys_mmap(6*8) as *i64; let WN: *i64=sys_mmap(6*8) as *i64
  WN[0]=U*dm; WN[1]=dm*dm; WN[2]=dm*dm; WN[3]=dm*dm; WN[4]=dm*dm; WN[5]=dm*U
  var wi: i64=0; while wi<6 { let a: *i64=sys_mmap(WN[wi]*8) as *i64; dini(a,WN[wi],wi+1); W[wi]=a as i64; wi=wi+1 }
  let leaves: *i64=sys_mmap(8*8) as *i64; let gbuf: *i64=sys_mmap(4096*8) as *i64
  let il: i64=clm_fwd(tape,vals,st,W,ids,tgt,T,dm,U,scale,leaves); let init_ce: i64=(nfa_val(tape,vals,il,0)*1000)/Q16   // CE before training (milli-nats)
  do_train(tape,vals,grads,st,W,WN,ids,tgt,T,dm,U,scale,leaves,gbuf,25000)
119
  // ---- 4. the LM MODELS the BPE-token stream: final CE + teacher-forced next-token accuracy ----
  let fl: i64=clm_fwd(tape,vals,st,W,ids,tgt,T,dm,U,scale,leaves)
  let final_ce: i64=(nfa_val(tape,vals,fl,0)*1000)/Q16
  var tfok: i64=0; var i3: i64=0
  while i3<T { if amx(tape,vals,leaves[6],i3,U)==tgt[i3] { tfok=tfok+1 } i3=i3+1 }
125
  // ---- 5. detokenize the TRUE token stream back to bytes (encode->decode plumbing must be lossless) ----
  let recon: *u8=sys_mmap(L+32); let wpos: *i64=sys_mmap(8) as *i64; wpos[0]=0
  var gp: i64=0; while gp<M { expand(c2id[comp[gp]],ma,mb,recon,wpos); gp=gp+1 }
  var lossless: i64=1
  if wpos[0]!=L { lossless=0 }
  var ci: i64=0; while ci<L { if recon[ci]!=corpus[ci] { lossless=0 } ci=ci+1 }
132
  g_puts("  [pipeline] CE init="); g_pn(init_ce); g_puts(" -> final="); g_pn(final_ce); g_puts(" milli-nats (LM learns)   teacher-forced acc="); g_pn(tfok); g_puts("/"); g_pn(T); g_puts(" (informational; repetitive corpus)   detok lossless="); g_pn(lossless); g_puts("\n")
134
  var pass: i64=0; var total: i64=0
  var t1: i64=0; if M < L { t1=1 }
  pass=pass+g_check("T1: BPE COMPRESSES the text (sub-word token stream shorter than the char stream)" as *u8, t1); total=total+1
  var t2: i64=0; if lossless==1 { t2=1 }
  pass=pass+g_check("T2: pipeline LOSSLESS -- detokenize(BPE tokens) == the original text, byte-exact (plumbing correct)" as *u8, t2); total=total+1
  var t3: i64=0; if final_ce*4 < init_ce { t3=1 }
  pass=pass+g_check("T3: the no-float LM LEARNS the BPE-tokenizer output (CE dropped >4x; teacher-forced >> 1/U chance)" as *u8, t3); total=total+1
142
  var okall: i64=0; if pass==total { okall=1 }
  let logf: i64=sys_openat_append("knowledge/status/nofloat_bpe_lm.log" as *u8, 420)
  if logf>=0 { let x0: i64=sys_write(logf,"NOFLOATBPELM end-to-end text->bpe->lm->gen->detok lossless\n" as *u8,58); sys_close(logf) }
  g_puts("---- bpe-lm gate: passed "); g_pn(pass); g_puts(" / "); g_pn(total); g_puts(" ----\n")
  if okall==1 { g_puts("verdict=GREEN (sovereign efficient pipeline wired: text->BPE->no-float LM models tokens->detok lossless)\n" as *u8); sys_exit(0); return 0 }
  g_puts("verdict=RED\n" as *u8); sys_exit(1); return 1
149}
nx_nofloat_bpe_lm_gate.nx source

`nx_nofloat_bpe_lm_gate.nx` source
