code wiki / _hdl_build / nx_nofloat_incontext_gate.nx
`nx_nofloat_incontext_gate.nx` source

↩ module page · 136 lines · 7687 B
1// nx_nofloat_incontext_gate.nx -- IN-CONTEXT COPYING (CAP-NF-INCONTEXT): the basis of in-context learning.
2// Task = ECHO: at each position predict the PREVIOUS token (tgt[i]=S[i-1]). EVERY training sequence is FRESH
3// RANDOM (LCG), so the model CANNOT memorize sequences -- it must learn the copy OPERATION (a "previous-token
4// attention head": attend to relative position -1 and copy its token). Then it echoes NEVER-SEEN random
5// sequences -> generalization to unseen DATA, the foundation of in-context learning.
6//   T1 held-out echo accuracy >> chance (1/V) on fresh random sequences = the model copies arbitrary unseen content.
7//   T2 same on a DIFFERENT test seed = the generalization is robust (not seed-luck).
8// Pure integer Q16, attention-only block (2 clm_fwd call sites). HONEST: copy-back-1 (a previous-token head),
9// the simplest in-context copy -- not full content-based induction. Sovereign: nx_nofloat_autograd + nx_syscalls.
10// expect_exit: 0
11import "nx_nofloat_autograd.nx"
12import "nx_syscalls.nx"
13import "nx_gate_emit_lib.nx"
14const Q16: i64 = 65536
15
16
17func dini(a: *i64, n: i64, sd: i64) -> i64 { var i: i64=0; while i<n { a[i]=(((i*7+sd*13+1)%11)-5)*13107; i=i+1 } return 0 }
18func lcg(st: *i64) -> i64 { st[0]=(st[0]*1103515245 + 12345) & 2147483647; return (st[0] >> 15) }
19// fill S[0..P-1] random in [0,V) and tgt[i]=S[i-1] (echo target; tgt[0]=S[0])
20func make_ex(S: *i64, tgt: *i64, P: i64, V: i64, st: *i64) -> i64 {
  var i: i64=0; while i<P { S[i]=lcg(st)%V; i=i+1 }
  tgt[0]=S[0]; i=1; while i<P { tgt[i]=S[i-1]; i=i+1 }
  return 0
24}
25
26func clm_fwd(tape: *i64, vals: *i64, st: *i64, W: *i64, ids: *i64, tgt: *i64, T: i64, dm: i64, V: i64, scale: i64, leaves: *i64) -> i64 {
  let E: *i64=W[0] as *i64; let Wq: *i64=W[1] as *i64; let Wk: *i64=W[2] as *i64; let Wv: *i64=W[3] as *i64; let Wo: *i64=W[4] as *i64; let Wlm: *i64=W[5] as *i64
  st[0]=0; st[1]=0
  let nE: i64=nfa_leaf(tape,vals,st,V,dm,E,0)
  let nWq: i64=nfa_leaf(tape,vals,st,dm,dm,Wq,0)
  let nWk: i64=nfa_leaf(tape,vals,st,dm,dm,Wk,0)
  let nWv: i64=nfa_leaf(tape,vals,st,dm,dm,Wv,0)
  let nWo: i64=nfa_leaf(tape,vals,st,dm,dm,Wo,0)
  let nWlm: i64=nfa_leaf(tape,vals,st,dm,V,Wlm,0)
  let nX: i64=nfa_embed(tape,vals,st,nE,ids,T)
  let nXn: i64=nfa_rmsnorm_rows(tape,vals,st,nX)
  let nQ: i64=nfa_matmul(tape,vals,st,nXn,nWq)
  let nK: i64=nfa_matmul(tape,vals,st,nXn,nWk)
  let nV: i64=nfa_matmul(tape,vals,st,nXn,nWv)
  let nQr: i64=nfa_rope(tape,vals,st,nQ)
  let nKr: i64=nfa_rope(tape,vals,st,nK)
  let nS: i64=nfa_matmul_nt(tape,vals,st,nQr,nKr)
  let nSs: i64=nfa_cmul(tape,vals,st,nS,scale)
  let nA: i64=nfa_softmax_rows(tape,vals,st,nSs,1)
  let nO: i64=nfa_matmul(tape,vals,st,nA,nV)
  let nOp: i64=nfa_matmul(tape,vals,st,nO,nWo)
  let nH: i64=nfa_vadd(tape,vals,st,nX,nOp)
  let nHn: i64=nfa_rmsnorm_rows(tape,vals,st,nH)
  let nLg: i64=nfa_matmul(tape,vals,st,nHn,nWlm)
  let nLoss: i64=nfa_softce_rows(tape,vals,st,nLg,tgt)
  leaves[0]=nE; leaves[1]=nWq; leaves[2]=nWk; leaves[3]=nWv; leaves[4]=nWo; leaves[5]=nWlm; leaves[6]=nLg
  return nLoss
53}
54func step_all(tape: *i64, grads: *i64, W: *i64, WN: *i64, leaves: *i64, nW: i64, lr: i64, clip: i64, gb: *i64) -> i64 {
  var i: i64=0
  while i<nW { let ar: *i64=W[i] as *i64; let cn: i64=WN[i]; let nd: i64=leaves[i]; var c: i64=0; while c<cn { var g: i64=nfa_grad(tape,grads,nd,c); if g>clip{g=clip} if g<0-clip{g=0-clip} gb[c]=g; c=c+1 } nfa_sgd(ar,gb,cn,lr); i=i+1 }
  return 0
58}
59// call site 1: train on FRESH random echo examples (st_data advances each step -> never repeats)
60func do_train(tape: *i64, vals: *i64, grads: *i64, st: *i64, W: *i64, WN: *i64, S: *i64, tgt: *i64, P: i64, dm: i64, V: i64, scale: i64, leaves: *i64, gb: *i64, steps: i64, sdat: *i64) -> i64 {
  var ep: i64=0
  while ep < steps {
      make_ex(S, tgt, P, V, sdat)
      let nl: i64=clm_fwd(tape,vals,st,W,S,tgt,P,dm,V,scale,leaves)
      nfa_backward(tape,vals,grads,st[0],nl)
      step_all(tape,grads,W,WN,leaves,6,6554,262144,gb)
      ep=ep+1
  }
  return 0
70}
71// call site 2: echo accuracy over N fresh random examples (positions 1..P-1: predict S[i-1]); returns correct count
72func eval_echo(tape: *i64, vals: *i64, st: *i64, W: *i64, S: *i64, tgt: *i64, P: i64, dm: i64, V: i64, scale: i64, leaves: *i64, N: i64, sdat: *i64, totp: *i64) -> i64 {
  var ok: i64=0; var tp: i64=0; var e: i64=0
  while e<N {
      make_ex(S, tgt, P, V, sdat)
      let nl: i64=clm_fwd(tape,vals,st,W,S,tgt,P,dm,V,scale,leaves)
      let off: i64=tape[7*leaves[6]+5]
      var p: i64=1
      while p<P {
          var b: i64=0; var bv: i64=vals[off+p*V]; var j: i64=1
          while j<V { if vals[off+p*V+j]>bv { bv=vals[off+p*V+j]; b=j } j=j+1 }
          if b==S[p-1] { ok=ok+1 }
          tp=tp+1
          p=p+1
      }
      e=e+1
  }
  totp[0]=tp
  return ok
90}
91
92func main() -> i64 {
  g_puts("nx_nofloat_incontext gate (IN-CONTEXT copy/echo head, pure integer Q16)\n" as *u8)
  var pass: i64=0; var total: i64=0
  let V: i64=6; let P: i64=6; let dm: i64=24; let scale: i64=13377
  let tape:  *i64 = sys_mmap(512*7*8) as *i64
  let vals:  *i64 = sys_mmap(65536*8) as *i64
  let grads: *i64 = sys_mmap(65536*8) as *i64
  let st:    *i64 = sys_mmap(2*8) as *i64
  let nW: i64=6
  let W: *i64 = sys_mmap(nW*8) as *i64
  let WN: *i64 = sys_mmap(nW*8) as *i64
  WN[0]=V*dm; WN[1]=dm*dm; WN[2]=dm*dm; WN[3]=dm*dm; WN[4]=dm*dm; WN[5]=dm*V
  var wi: i64=0
  while wi<nW { let a: *i64=sys_mmap(WN[wi]*8) as *i64; dini(a,WN[wi],wi+1); W[wi]=a as i64; wi=wi+1 }
  let leaves: *i64 = sys_mmap(8*8) as *i64
  let gbuf: *i64 = sys_mmap(4096*8) as *i64
  let S: *i64 = sys_mmap(P*8) as *i64; let tgt: *i64 = sys_mmap(P*8) as *i64
  let totp: *i64 = sys_mmap(8) as *i64
  let sdat: *i64 = sys_mmap(8) as *i64
111
  sdat[0]=12345
  do_train(tape,vals,grads,st,W,WN,S,tgt,P,dm,V,scale,leaves,gbuf,30000,sdat)
114
  // held-out test seed A (fresh sequences the model never trained on)
  sdat[0]=987654321
  let okA: i64 = eval_echo(tape,vals,st,W,S,tgt,P,dm,V,scale,leaves,200,sdat,totp)
  let tpA: i64 = totp[0]
  // held-out test seed B (different)
  sdat[0]=192837465
  let okB: i64 = eval_echo(tape,vals,st,W,S,tgt,P,dm,V,scale,leaves,200,sdat,totp)
  let tpB: i64 = totp[0]
  g_puts("  [measure] held-out echo accuracy: seedA=" as *u8); g_pn(okA); g_puts("/" as *u8); g_pn(tpA); g_puts("  seedB=" as *u8); g_pn(okB); g_puts("/" as *u8); g_pn(tpB); g_puts("  (chance~1/" as *u8); g_pn(V); g_puts(")\n" as *u8)
124
  var t1: i64=0; if okA*100 >= tpA*70 { t1=1 }     // >= 70% on never-seen sequences (>> 17% chance)
  pass=pass+g_check("T1: in-context copy -- echoes NEVER-SEEN random sequences >> chance (learned the operation)" as *u8, t1); total=total+1
  var t2: i64=0; if okB*100 >= tpB*70 { t2=1 }     // robust on a different test seed
  pass=pass+g_check("T2: robust -- same high echo accuracy on a DIFFERENT held-out seed (not seed-luck)" as *u8, t2); total=total+1
129
  var okall: i64=0; if pass==total { okall=1 }
  let logf: i64 = sys_openat_append("knowledge/status/nofloat_incontext.log" as *u8, 420)
  if logf >= 0 { let w0: i64=sys_write(logf,"NOFLOATINCONTEXT in-context echo head measured\n" as *u8,46); sys_close(logf) }
  g_puts("---- incontext gate: passed " as *u8); g_pn(pass); g_puts(" / " as *u8); g_pn(total); g_puts(" ----\n" as *u8)
  if okall==1 { g_puts("verdict=GREEN (in-context copy head: echoes arbitrary unseen sequences -- the basis of in-context learning)\n" as *u8); sys_exit(0); return 0 }
  g_puts("verdict=RED\n" as *u8); sys_exit(1); return 1
136}
nx_nofloat_incontext_gate.nx source

`nx_nofloat_incontext_gate.nx` source
