code wiki / _hdl_build / nx_nofloat_xformer_gate.nx
`nx_nofloat_xformer_gate.nx` source

↩ module page · 220 lines · 12226 B
1// nx_nofloat_xformer_gate.nx -- HARD-EVIDENCE gate for the TRANSFORMER-SUBLAYER backward ops added to the
2// no-float autograd (softmax / silu / rmsnorm): the nonlinearities of a Qwen block, now DIFFERENTIABLE in
3// pure integer Q16. This is the CAP-NF-TRAIN-XFORMER rung: backprop through the transformer's own ops.
4//
5//   A  GRADCHECK each op (oracle = mathematics): build loss = mse(OP(x), target), compare the tape's analytic
6//      reverse-sweep grad dL/dx_i against a CENTRAL finite difference (L(x+h)-L(x-h))/2h, all Q16. Done for
7//      softmax (Jacobian-vector product), silu (x*sigmoid(x) derivative), rmsnorm (normalization Jacobian).
8//   D  TEETH (neg-control): negate one analytic grad and assert the SAME check now FAILS -> a wrong gradient
9//      is provably caught (the agreement in A is real, not vacuous).
10//   B  A TRANSFORMER ACTIVATION TRAINS: fit y = silu(w*.x) (realizable, w*=1.5) from w=0 by GD through the
11//      tape+silu backward. Assert loss collapses >=90% and w converges to w*. Proves the FFN nonlinearity is
12//      trainable end-to-end in pure integer.
13//   C  BIT-EXACT: train twice from zero -> identical integer w (determinism is structural for integer).
14//
15// Evidence -> knowledge/status/nofloat_xformer.log. Sovereign: imports nx_nofloat_autograd (pure integer; NO
16// nx_f32 in the import graph) + nx_syscalls. license_tier: ORIGINAL  expect_exit: 0
17import "nx_nofloat_autograd.nx"
18import "nx_g_check_lib.nx"
19import "nx_g_pn_lib.nx"
20import "nx_g_puts_lib.nx"
21import "nx_syscalls.nx"
22
23const XLOG: *u8 = "knowledge/status/nofloat_xformer.log"
24const Q16:  i64 = 65536
25const OP_SOFTMAX: i64 = 5
26const OP_SILU:    i64 = 6
27const OP_RMSNORM: i64 = 7
28
29func g_abs(v: i64) -> i64 { if v < 0 { return 0 - v } return v }
30func q_milli(q: i64) -> i64 { var neg: i64=0; var a: i64=q; if a<0 { neg=1; a=0-a } let m: i64=(a*1000)/Q16; if neg==1 { return 0-m } return m }
31func x_ws(fd: i64, s: *u8) -> i64 { var n: i64=0; while s[n]!=(0 as u8){n=n+1} sys_write(fd,s,n); return 0 }
32func x_wn(fd: i64, v: i64) -> i64 {
  let b: *u8 = sys_mmap(28); var m: i64=v; if m<0 { sys_write(fd,"-" as *u8,1); m=0-m }
  let t: *u8 = sys_mmap(28); var k: i64=0; if m==0 { t[0]=48; k=1 }
  while m>0 { t[k]=(48+(m%10)) as u8; m=m/10; k=k+1 }
  var i: i64=0; while i<k { b[i]=t[k-1-i]; i=i+1 } sys_write(fd,b,k); return 0
37}
38
39// ---- build loss = mse(OP(x), target) on the general tape; leaves[0] = the x leaf. ----
40func xf_loss(tape: *i64, vals: *i64, st: *i64, op_sel: i64, xs: *i64, ts: *i64, n: i64, leaves: *i64) -> i64 {
  st[0]=0; st[1]=0
  let nx: i64 = nfa_leaf(tape,vals,st,n,1,xs,0)
  var nop: i64 = nx
  if op_sel == OP_SOFTMAX { nop = nfa_softmax(tape,vals,st,nx) }
  if op_sel == OP_SILU    { nop = nfa_silu(tape,vals,st,nx) }
  if op_sel == OP_RMSNORM { nop = nfa_rmsnorm(tape,vals,st,nx) }
  let nt: i64 = nfa_leaf(tape,vals,st,n,1,ts,0)
  let loss: i64 = nfa_mse(tape,vals,st,nop,nt)
  leaves[0]=nx
  return loss
51}
52func xf_lossval(tape: *i64, vals: *i64, st: *i64, op_sel: i64, xs: *i64, ts: *i64, n: i64) -> i64 {
  let lv: *i64 = sys_mmap(8) as *i64
  let loss: i64 = xf_loss(tape,vals,st,op_sel,xs,ts,n,lv)
  return nfa_val(tape,vals,loss,0)
56}
57// central finite diff of the loss wrt x[pi], Q16 step h
58func xf_fd(tape: *i64, vals: *i64, st: *i64, op_sel: i64, xs: *i64, ts: *i64, n: i64, pi: i64, h: i64) -> i64 {
  let xp: *i64 = sys_mmap(n*8) as *i64
  let xm: *i64 = sys_mmap(n*8) as *i64
  var i: i64 = 0
  while i < n { xp[i]=xs[i]; xm[i]=xs[i]; i=i+1 }
  xp[pi]=xs[pi]+h; xm[pi]=xs[pi]-h
  let lp: i64 = xf_lossval(tape,vals,st,op_sel,xp,ts,n)
  let lm: i64 = xf_lossval(tape,vals,st,op_sel,xm,ts,n)
  return ((lp - lm) * Q16) / (2 * h)
67}
68// gradcheck one op over all n inputs; sets worst[0]=worst relative-error milli; returns 1 if all within tol.
69func xf_gradcheck(tape: *i64, vals: *i64, grads: *i64, st: *i64, op_sel: i64, xs: *i64, ts: *i64, n: i64, h: i64, tol_q: i64, floor_q: i64, worst: *i64) -> i64 {
  let lv: *i64 = sys_mmap(8) as *i64
  let loss: i64 = xf_loss(tape,vals,st,op_sel,xs,ts,n,lv)
  nfa_backward(tape,vals,grads,st[0],loss)
  let nx: i64 = lv[0]
  var ok: i64 = 1; worst[0]=0
  var i: i64 = 0
  while i < n {
      let ana: i64 = nfa_grad(tape,grads,nx,i)
      let fd: i64 = xf_fd(tape,vals,st,op_sel,xs,ts,n,i,h)
      let num: i64 = g_abs(fd - ana)
      var den: i64 = g_abs(ana); if den < floor_q { den = floor_q }
      let thresh: i64 = (tol_q * den) >> 16
      if num >= thresh { ok = 0 }
      let rel: i64 = (num * 1000) / den
      if rel > worst[0] { worst[0] = rel }
      i = i + 1
  }
  return ok
88}
89
90// ---- Gate B/C: train w to fit y = silu(w*.x) (realizable, w*=1.5 Q16) via tape + silu backward ----
91func xf_silu_build(tape: *i64, vals: *i64, st: *i64, wp: *i64, X: *i64, Y: *i64, T: i64, outW: *i64) -> i64 {
  st[0]=0; st[1]=0
  let nw: i64 = nfa_leaf(tape,vals,st,1,1,wp,0)
  outW[0]=nw
  var root: i64 = 0 - 1
  var t: i64 = 0
  while t < T {
      let nx: i64 = nfa_leaf(tape,vals,st,1,1,X,t)
      let h: i64 = nfa_matvec(tape,vals,st,nw,nx)       // w*x
      let s: i64 = nfa_silu(tape,vals,st,h)             // silu(w*x)
      let ny: i64 = nfa_leaf(tape,vals,st,1,1,Y,t)
      let nm: i64 = nfa_mse(tape,vals,st,s,ny)
      if root < 0 { root = nm } else { root = nfa_vadd(tape,vals,st,root,nm) }
      t = t + 1
  }
  return root
107}
108func xf_silu_train(tape: *i64, vals: *i64, grads: *i64, st: *i64, epochs: i64, lr_q: i64, wout: *i64, lf: *i64, ll: *i64) -> i64 {
  let T: i64 = 4
  let Wt: i64 = 98304                                   // w* = 1.5 Q16
  let X: *i64 = sys_mmap(T*8) as *i64
  X[0]=32768; X[1]=65536; X[2]=98304; X[3]=131072       // x = 0.5,1.0,1.5,2.0
  let Y: *i64 = sys_mmap(T*8) as *i64
  var t: i64 = 0
  while t < T { Y[t] = nfa_siluf((Wt * X[t]) >> 16); t = t + 1 }   // realizable targets via the lib's silu
  let wp: *i64 = sys_mmap(8) as *i64; wp[0]=0           // learn from ZERO
  let outW: *i64 = sys_mmap(8) as *i64
  let g: *i64 = sys_mmap(8) as *i64
  var ep: i64 = 0
  while ep < epochs {
      let root: i64 = xf_silu_build(tape,vals,st,wp,X,Y,T,outW)
      nfa_backward(tape,vals,grads,st[0],root)
      if ep == 0 { *lf = nfa_val(tape,vals,root,0) }
      *ll = nfa_val(tape,vals,root,0)
      g[0] = nfa_grad(tape,grads,outW[0],0)
      nfa_sgd(wp, g, 1, lr_q)
      ep = ep + 1
  }
  wout[0]=wp[0]
  return 0
131}
132
133func main() -> i64 {
  g_puts("nx_nofloat_xformer gate (backprop through softmax/silu/rmsnorm in PURE INTEGER Q16 -- MEASURED)\n" as *u8)
  var pass: i64 = 0; var total: i64 = 0
  let tape:  *i64 = sys_mmap(512*7*8) as *i64
  let vals:  *i64 = sys_mmap(4096*8) as *i64
  let grads: *i64 = sys_mmap(4096*8) as *i64
  let st:    *i64 = sys_mmap(2*8) as *i64
  let n: i64 = 4
  let h: i64 = 512
  let floor_q: i64 = 4096
  let worst: *i64 = sys_mmap(8) as *i64
144
  // ---- A1: softmax gradcheck ----
  let xs1: *i64 = sys_mmap(n*8) as *i64; xs1[0]=32768; xs1[1]=0-16384; xs1[2]=49152; xs1[3]=0
  let ts1: *i64 = sys_mmap(n*8) as *i64; ts1[0]=26214; ts1[1]=6554; ts1[2]=26214; ts1[3]=6554
  let sm_ok: i64 = xf_gradcheck(tape,vals,grads,st,OP_SOFTMAX,xs1,ts1,n,h,4096,floor_q,worst)   // tol 1/16
  g_puts("  [measure] softmax worst rel grad err = " as *u8); g_pn(worst[0]); g_puts(" /1000 (tol=62/1000)\n" as *u8)
  pass = pass + g_check("A1: softmax gradcheck -- tape Jacobian-vector backward == finite differences" as *u8, sm_ok); total=total+1
151
  // ---- A2: silu gradcheck ----
  let xs2: *i64 = sys_mmap(n*8) as *i64; xs2[0]=32768; xs2[1]=0-32768; xs2[2]=65536; xs2[3]=0-65536
  let ts2: *i64 = sys_mmap(n*8) as *i64; ts2[0]=19661; ts2[1]=0-13107; ts2[2]=52429; ts2[3]=0-17695
  let si_ok: i64 = xf_gradcheck(tape,vals,grads,st,OP_SILU,xs2,ts2,n,h,4096,floor_q,worst)      // tol 1/16
  g_puts("  [measure] silu worst rel grad err = " as *u8); g_pn(worst[0]); g_puts(" /1000 (tol=62/1000)\n" as *u8)
  pass = pass + g_check("A2: silu gradcheck -- tape x*sigmoid(x) backward == finite differences" as *u8, si_ok); total=total+1
158
  // ---- A3: rmsnorm gradcheck (normalization Jacobian; fixed-point isqrt/div -> looser honest tol) ----
  let xs3: *i64 = sys_mmap(n*8) as *i64; xs3[0]=32768; xs3[1]=65536; xs3[2]=0-32768; xs3[3]=16384
  let ts3: *i64 = sys_mmap(n*8) as *i64; ts3[0]=13107; ts3[1]=58982; ts3[2]=0-39322; ts3[3]=6554
  let rn_ok: i64 = xf_gradcheck(tape,vals,grads,st,OP_RMSNORM,xs3,ts3,n,h,4096,floor_q,worst)   // tol 1/16 (measured 13/1000)
  g_puts("  [measure] rmsnorm worst rel grad err = " as *u8); g_pn(worst[0]); g_puts(" /1000 (tol=62/1000)\n" as *u8)
  pass = pass + g_check("A3: rmsnorm gradcheck -- tape normalization-Jacobian backward == finite differences" as *u8, rn_ok); total=total+1
165
  // ---- D: neg-control (teeth) on silu ----
  let lv: *i64 = sys_mmap(8) as *i64
  let loss: i64 = xf_loss(tape,vals,st,OP_SILU,xs2,ts2,n,lv)
  nfa_backward(tape,vals,grads,st[0],loss)
  let ana0: i64 = nfa_grad(tape,grads,lv[0],0)
  let fd0: i64 = xf_fd(tape,vals,st,OP_SILU,xs2,ts2,n,0,h)
  let bad: i64 = 0 - ana0
  var den0: i64 = g_abs(ana0); if den0 < floor_q { den0 = floor_q }
  let thr0: i64 = (4096 * den0) >> 16
  var caught: i64 = 1
  if g_abs(fd0 - bad) < thr0 { caught = 0 }
  pass = pass + g_check("D: neg-control -- a deliberately WRONG grad is rejected by the gradcheck (teeth)" as *u8, caught); total=total+1
178
  // ---- B: a transformer activation TRAINS (silu regression, realizable) ----
  let wbox: *i64 = sys_mmap(8) as *i64
  let lf: *i64 = sys_mmap(8) as *i64
  let ll: *i64 = sys_mmap(8) as *i64
  xf_silu_train(tape,vals,grads,st, 6000, 1024, wbox, lf, ll)
  g_puts("  [measure] silu-regression loss: start=" as *u8); g_pn(*lf); g_puts(" end=" as *u8); g_pn(*ll)
  g_puts("  learned w=" as *u8); g_pn(wbox[0]); g_puts(" (milli=" as *u8); g_pn(q_milli(wbox[0])); g_puts(") vs w*=98304 (1500 milli)\n" as *u8)
  var learns: i64 = 1
  if (*ll) * 10 > (*lf) { learns = 0 }                                  // >= 90% loss reduction
  if g_abs(wbox[0] - 98304) > 9830 { learns = 0 }                      // within 15% of w*
  if (*lf) <= 0 { learns = 0 }
  pass = pass + g_check("B: a transformer activation LEARNS in pure integer -- silu regression converges to w*" as *u8, learns); total=total+1
191
  // ---- C: bit-exact reproducible ----
  let wbox2: *i64 = sys_mmap(8) as *i64
  let lf2: *i64 = sys_mmap(8) as *i64
  let ll2: *i64 = sys_mmap(8) as *i64
  xf_silu_train(tape,vals,grads,st, 6000, 1024, wbox2, lf2, ll2)
  var bitexact: i64 = 1
  if wbox2[0] != wbox[0] { bitexact = 0 }
  pass = pass + g_check("C: bit-exact -- training twice gives IDENTICAL integer w (determinism)" as *u8, bitexact); total=total+1
200
  // ---- emit ----
  var okall: i64 = 0
  if pass == total { okall = 1 }
  x_ws(1, "NOFLOATXFORMER authored=organ ops=softmax,silu,rmsnorm" as *u8)
  x_ws(1, " | A1_softmax=" as *u8); x_wn(1, sm_ok); x_ws(1, " A2_silu=" as *u8); x_wn(1, si_ok); x_ws(1, " A3_rmsnorm=" as *u8); x_wn(1, rn_ok)
  x_ws(1, " | D_teeth=" as *u8); x_wn(1, caught); x_ws(1, " | B_learns=" as *u8); x_wn(1, learns); x_ws(1, " w_milli=" as *u8); x_wn(1, q_milli(wbox[0]))
  x_ws(1, " | C_bitexact=" as *u8); x_wn(1, bitexact); x_ws(1, "\n" as *u8)
  let logf: i64 = sys_openat_append(XLOG, 420)
  if logf >= 0 {
      x_ws(logf, "NOFLOATXFORMER ops=softmax,silu,rmsnorm A1=" as *u8); x_wn(logf, sm_ok); x_ws(logf, " A2=" as *u8); x_wn(logf, si_ok)
      x_ws(logf, " A3=" as *u8); x_wn(logf, rn_ok); x_ws(logf, " D=" as *u8); x_wn(logf, caught); x_ws(logf, " B=" as *u8); x_wn(logf, learns)
      x_ws(logf, " C=" as *u8); x_wn(logf, bitexact)
      if okall==1 { x_ws(logf, " verdict=GREEN\n" as *u8) } else { x_ws(logf, " verdict=RED\n" as *u8) }
      sys_close(logf)
  }
216
  g_puts("---- nofloat_xformer gate: passed " as *u8); g_pn(pass); g_puts(" / " as *u8); g_pn(total); g_puts(" ----\n" as *u8)
  if okall == 1 { g_puts("verdict=GREEN\n" as *u8); sys_exit(0); return 0 }
  g_puts("verdict=RED\n" as *u8); sys_exit(1); return 1
220}
nx_nofloat_xformer_gate.nx source

`nx_nofloat_xformer_gate.nx` source
