code wiki / (root) / nx_nofloat_moe_gate.nx
`nx_nofloat_moe_gate.nx` source

↩ module page · 285 lines · 11364 B
1// nx_nofloat_moe_gate.nx -- gates the no-float MoE FFN layer (MoE rung 1, 2026-07-15). The lib does
2// router -> deterministic top-K -> fx_exp softmax -> K SwiGLU experts -> weighted mix, all Q16 integer.
3// Teeth (every one deterministic; an INDEPENDENT serial reference implementation lives in THIS file with
4// REVERSED reduction orders -- exact integer sums are order-free, so lib and reference must agree to the BIT):
5//   T1 EXACT vs the independent reverse-order reference on 3 distinct inputs (0 mismatched cells)
6//   T2 determinism: repeat call byte-identical (and T1's order-reversal already proves order-invariance)
7//   T3 sparsity REAL + router alive: exactly K experts computed per forward (cnt), >=2 distinct experts
8//      selected across the 3 inputs (the router discriminates; selections logged)
9//   T4 NEG-CONTROLS: (a) corrupting one router row CHANGES a selection (routing is weight-driven, not
10//      fabricated); (b) dense K=E output DIFFERS from sparse K=2 (sparsity has measurable effect)
11//   T5 work happened: >= half the output cells nonzero on every input
12// Synthetic Q16 weights/inputs from a fixed integer formula (no randomness, no clock). Runs in ms.
13// expect_exit: 0   license_tier: ORIGINAL   No hw writes (Rule 26).
14import "nx_syscalls.nx"
15import "nx_nofloat_moe.nx"
16import "nx_gate_verdict.nx"
17
18func mg_w(s: *u8) -> i64 { var n: i64 = 0; while s[n] != (0 as u8) { n = n + 1 } sys_write(1, s, n); return 0 }
19func mg_n(v: i64) -> i64 {
  var m: i64 = v
  if m < 0 { mg_w("-" as *u8); m = 0 - m }
  let t: *u8 = sys_mmap(24)
  var k: i64 = 0
  if m == 0 { t[0] = 48 as u8; k = 1 }
  while m > 0 { t[k] = (48 + (m % 10)) as u8; m = m / 10; k = k + 1 }
  let o: *u8 = sys_mmap(24)
  var i: i64 = 0
  while i < k { o[i] = t[k - 1 - i]; i = i + 1 }
  sys_write(1, o, k)
  return 0
31}
32// deterministic synthetic Q16 value in [-4095, 4095] (~ +-0.0625)
33func mg_det(i: i64) -> i64 { return ((i * 2654435761) % 8191) - 4095 }
34
35// ---- INDEPENDENT serial reference (reverse-order reductions everywhere) ----
36func mg_ref_dot_rev(a: *i64, W: *i64, o: i64, ind: i64) -> i64 {
  var s: i64 = 0
  var k: i64 = ind - 1
  while k >= 0 { s = s + (a[k] * W[o*ind + k]); k = k - 1 }
  return s >> 16
41}
42func mg_ref_forward(mp: *i64) -> i64 {
  let x: *i64 = mp[0] as *i64
  let Wr: *i64 = mp[1] as *i64
  let Wg: *i64 = mp[2] as *i64
  let Wu: *i64 = mp[3] as *i64
  let Wd: *i64 = mp[4] as *i64
  let out: *i64 = mp[5] as *i64
  let D: i64 = mp[6]
  let H: i64 = mp[7]
  let E: i64 = mp[8]
  let K: i64 = mp[9]
  let cnt: *i64 = mp[10] as *i64
  let scr: *i64 = mp[11] as *i64
  let r: *i64 = scr
  let sel: *i64 = ((scr as i64) + E*8) as *i64
  let w: *i64 = ((sel as i64) + K*8) as *i64
  let g: *i64 = ((w as i64) + K*8) as *i64
  let u2: *i64 = ((g as i64) + H*8) as *i64
  let h: *i64 = ((u2 as i64) + H*8) as *i64
  let eo: *i64 = ((h as i64) + H*8) as *i64
  var e: i64 = E - 1
  while e >= 0 { r[e] = mg_ref_dot_rev(x, Wr, e, D); e = e - 1 }
  var ki: i64 = 0
  while ki < K {
      var best: i64 = 0 - 1
      var bestv: i64 = 0
      e = 0
      while e < E {
          var taken: i64 = 0
          var t: i64 = 0
          while t < ki { if sel[t] == e { taken = 1 } t = t + 1 }
          if taken == 0 { if best < 0 { best = e; bestv = r[e] } else { if r[e] > bestv { best = e; bestv = r[e] } } }
          e = e + 1
      }
      sel[ki] = best
      ki = ki + 1
  }
  var m: i64 = r[sel[0]]
  var i: i64 = 1
  while i < K { if r[sel[i]] > m { m = r[sel[i]] } i = i + 1 }
  var sum: i64 = 0
  i = 0
  while i < K { let ev: i64 = fx_exp(r[sel[i]] - m); w[i] = ev; sum = sum + ev; i = i + 1 }
  if sum < 1 { sum = 1 }
  i = 0
  while i < K { w[i] = (w[i] << 16) / sum; i = i + 1 }
  var d: i64 = 0
  while d < D { out[d] = 0; d = d + 1 }
  i = K - 1
  while i >= 0 {
      let e2: i64 = sel[i]
      cnt[e2] = cnt[e2] + 1
      var j: i64 = H - 1
      while j >= 0 {
          g[j] = mg_ref_dot_rev(x, ((Wg as i64) + e2*H*D*8) as *i64, j, D)
          u2[j] = mg_ref_dot_rev(x, ((Wu as i64) + e2*H*D*8) as *i64, j, D)
          h[j] = qmul(silu(g[j]), u2[j])
          j = j - 1
      }
      d = D - 1
      while d >= 0 { eo[d] = mg_ref_dot_rev(h, ((Wd as i64) + e2*D*H*8) as *i64, d, H); d = d - 1 }
      d = 0
      while d < D { out[d] = out[d] + qmul(w[i], eo[d]); d = d + 1 }
      i = i - 1
  }
  return 0
108}
109
110func main() -> i64 {
  mg_w("=== NX-NOFLOAT-MOE -- integer Mixture-of-Experts FFN: router -> top-K -> sparse SwiGLU mix ===\n" as *u8)
  let D: i64 = 32
  let H: i64 = 64
  let E: i64 = 4
  let K: i64 = 2
  let Wr: *i64 = sys_mmap(E*D*8) as *i64
  let Wg: *i64 = sys_mmap(E*H*D*8) as *i64
  let Wu: *i64 = sys_mmap(E*H*D*8) as *i64
  let Wd: *i64 = sys_mmap(E*D*H*8) as *i64
  var i: i64 = 0
  while i < E*D { Wr[i] = mg_det(i + 1000); i = i + 1 }
  i = 0
  while i < E*H*D { Wg[i] = mg_det(i + 2000); Wu[i] = mg_det(i + 3000); i = i + 1 }
  i = 0
  while i < E*D*H { Wd[i] = mg_det(i + 4000); i = i + 1 }
  let xA: *i64 = sys_mmap(D*8) as *i64
  let xB: *i64 = sys_mmap(D*8) as *i64
  let xC: *i64 = sys_mmap(D*8) as *i64
  i = 0
  while i < D { xA[i] = mg_det(i + 7); xB[i] = mg_det(i*3 + 77); xC[i] = mg_det(i*5 + 777); i = i + 1 }
  let out1: *i64 = sys_mmap(D*8) as *i64
  let out2: *i64 = sys_mmap(D*8) as *i64
  let scr1: *i64 = sys_mmap((E + 2*K + 3*H + D + 16)*8) as *i64
  let scr2: *i64 = sys_mmap((E + 2*K + 3*H + D + 16)*8) as *i64
  let cnt1: *i64 = sys_mmap(E*8) as *i64
  let cnt2: *i64 = sys_mmap(E*8) as *i64
  let mp: *i64 = sys_mmap(16*8) as *i64
  mp[1] = Wr as i64
  mp[2] = Wg as i64
  mp[3] = Wu as i64
  mp[4] = Wd as i64
  mp[6] = D
  mp[7] = H
  mp[8] = E
  mp[9] = K
146
  // ---- T1: lib vs independent reverse-order reference, 3 inputs, exact ----
  var mism: i64 = 0
  var nz_fail: i64 = 0
  var usedmask: i64 = 0
  var qi: i64 = 0
  while qi < 3 {
      var xp: *i64 = xA
      if qi == 1 { xp = xB }
      if qi == 2 { xp = xC }
      var ci: i64 = 0
      while ci < E { cnt1[ci] = 0; cnt2[ci] = 0; ci = ci + 1 }
      mp[0] = xp as i64
      mp[5] = out1 as i64
      mp[10] = cnt1 as i64
      mp[11] = scr1 as i64
      nmoe_forward(mp)
      mp[5] = out2 as i64
      mp[10] = cnt2 as i64
      mp[11] = scr2 as i64
      mg_ref_forward(mp)
      var d: i64 = 0
      var nz: i64 = 0
      while d < D { if out1[d] != out2[d] { mism = mism + 1 } if out1[d] != 0 { nz = nz + 1 } d = d + 1 }
      if nz < D/2 { nz_fail = nz_fail + 1 }
      // per-call sparsity: exactly K experts computed; accumulate the union mask; log the selection
      var kc: i64 = 0
      ci = 0
      mg_w("  Q" as *u8); mg_n(qi + 1); mg_w(" experts:" as *u8)
      while ci < E { if cnt1[ci] > 0 { kc = kc + 1; usedmask = usedmask | (1 << ci); mg_w(" " as *u8); mg_n(ci) } ci = ci + 1 }
      if kc != K { mism = mism + 1000000 }
      mg_w("  (nz " as *u8); mg_n(nz); mg_w("/" as *u8); mg_n(D); mg_w(")\n" as *u8)
      qi = qi + 1
  }
  // distinct experts used across inputs
  var distinct: i64 = 0
  i = 0
  while i < E { if (usedmask & (1 << i)) != 0 { distinct = distinct + 1 } i = i + 1 }
184
  // ---- T2: repeat call byte-identical ----
  var ci2: i64 = 0
  while ci2 < E { cnt1[ci2] = 0; ci2 = ci2 + 1 }
  mp[0] = xA as i64
  mp[5] = out2 as i64
  mp[10] = cnt1 as i64
  mp[11] = scr1 as i64
  nmoe_forward(mp)
  mp[5] = out1 as i64
  nmoe_forward(mp)
  var rep_mism: i64 = 0
  var d2: i64 = 0
  while d2 < D { if out1[d2] != out2[d2] { rep_mism = rep_mism + 1 } d2 = d2 + 1 }
198
  // ---- T4a: corrupt router row 0 (negate) -> some selection must change ----
  let Wr2: *i64 = sys_mmap(E*D*8) as *i64
  i = 0
  while i < E*D { Wr2[i] = Wr[i]; i = i + 1 }
  i = 0
  while i < D { Wr2[i] = 0 - Wr2[i]; i = i + 1 }
  var sel_changed: i64 = 0
  qi = 0
  while qi < 3 {
      var xp2: *i64 = xA
      if qi == 1 { xp2 = xB }
      if qi == 2 { xp2 = xC }
      var ci3: i64 = 0
      while ci3 < E { cnt1[ci3] = 0; cnt2[ci3] = 0; ci3 = ci3 + 1 }
      mp[0] = xp2 as i64
      mp[5] = out1 as i64
      mp[10] = cnt1 as i64
      mp[11] = scr1 as i64
      mp[1] = Wr as i64
      nmoe_forward(mp)
      mp[1] = Wr2 as i64
      mp[5] = out2 as i64
      mp[10] = cnt2 as i64
      nmoe_forward(mp)
      var ci4: i64 = 0
      while ci4 < E { if cnt1[ci4] != cnt2[ci4] { sel_changed = 1 } ci4 = ci4 + 1 }
      qi = qi + 1
  }
  mp[1] = Wr as i64
228
  // ---- T4b: dense (K=E) differs from sparse (K=2) ----
  var ci5: i64 = 0
  while ci5 < E { cnt1[ci5] = 0; ci5 = ci5 + 1 }
  mp[0] = xA as i64
  mp[5] = out2 as i64
  mp[10] = cnt1 as i64
  mp[11] = scr1 as i64
  mp[9] = E
  nmoe_forward(mp)
  mp[9] = K
  mp[5] = out1 as i64
  var ci6: i64 = 0
  while ci6 < E { cnt2[ci6] = 0; ci6 = ci6 + 1 }
  mp[10] = cnt2 as i64
  nmoe_forward(mp)
  var dense_diff: i64 = 0
  d2 = 0
  while d2 < D { if out1[d2] != out2[d2] { dense_diff = dense_diff + 1 } d2 = d2 + 1 }
247
  // ---- teeth ----
  var pass: i64 = 0
  var ttl: i64 = 0
  ttl = ttl + 1
  let ok1: i64 = (mism == 0) as i64
  mg_w("  T1 lib == independent reverse-order reference, 3 inputs, exact (mism " as *u8); mg_n(mism); mg_w("): " as *u8)
  if ok1 == 1 { pass = pass + 1; mg_w("PASS\n" as *u8) } else { mg_w("FAIL\n" as *u8) }
  ttl = ttl + 1
  let ok2: i64 = (rep_mism == 0) as i64
  mg_w("  T2 repeat call byte-identical: " as *u8)
  if ok2 == 1 { pass = pass + 1; mg_w("PASS\n" as *u8) } else { mg_w("FAIL\n" as *u8) }
  ttl = ttl + 1
  let ok3: i64 = (distinct >= 2) as i64
  mg_w("  T3 sparsity real + router discriminates (distinct experts " as *u8); mg_n(distinct); mg_w("/" as *u8); mg_n(E); mg_w("): " as *u8)
  if ok3 == 1 { pass = pass + 1; mg_w("PASS\n" as *u8) } else { mg_w("FAIL\n" as *u8) }
  ttl = ttl + 1
  let ok4a: i64 = (sel_changed == 1) as i64
  let ok4b: i64 = (dense_diff > 0) as i64
  let ok4: i64 = ok4a & ok4b
  mg_w("  T4 NEG-CONTROLS: corrupted router changes selection=" as *u8); mg_n(sel_changed)
  mg_w(", dense-vs-sparse differing cells=" as *u8); mg_n(dense_diff); mg_w(": " as *u8)
  if ok4 == 1 { pass = pass + 1; mg_w("PASS\n" as *u8) } else { mg_w("FAIL\n" as *u8) }
  ttl = ttl + 1
  let ok5: i64 = (nz_fail == 0) as i64
  mg_w("  T5 output transformed on every input: " as *u8)
  if ok5 == 1 { pass = pass + 1; mg_w("PASS\n" as *u8) } else { mg_w("FAIL\n" as *u8) }
274
  mg_w("NX-NOFLOAT-MOE-GATE passed " as *u8); mg_n(pass); mg_w("/" as *u8); mg_n(ttl)
  // MIGRATED onto nx_gate_verdict by nx_gate_dry_apply (D001, minimal form): every check
  // row above is untouched, so the PASS/FAIL vector cannot change; only the hand-rolled
  // verdict emission is replaced by the ONE shared base class. Proven by nx_gate_migrate verify.
  let ctr__dry: *i64 = gv_ctr()
  ctr__dry[0] = pass
  ctr__dry[1] = ttl
  let rc__dry: i64 = gv_verdict("NOFLOAT-MOE-GATE" as *u8, ctr__dry, "sparse MoE FFN runs in deterministic integer: router routes, K of E compute, mix exact)" as *u8)
  sys_exit(rc__dry)
  return rc__dry
285}
nx_nofloat_moe_gate.nx source

`nx_nofloat_moe_gate.nx` source
