nx_f32_swiglu_gate.nx source

1import "nx_gate_gn.nx" 2import "nx_gate_base.nx" 3// nx_f32_swiglu_gate.nx -- RUNG 3b (Llama-3.2 arch): SwiGLU / SiLU activation backward, gradient-checked. Llama's FFN 4// uses SwiGLU: FFN(x) = (SiLU(x*Wg) (.) (x*Wu)) * Wd, where SiLU(z)=z*sigmoid(z) is the gating activation. The matmuls 5// are covered by the R1 autograd; the NEW op is SiLU (and the elementwise gate). sigmoid built on our f32_exp. 6// SOVEREIGNTY: published research (Shazeer 2020 GLU-variants; SiLU/Swish Elfwing 2017) -- math, not copied code; our 7// implementation on our SSE f32. Same finite-difference proof discipline. 8// T0 sigmoid: s(0)=0.5, s(1)=0.731, s(-1)=0.269. 9// T1 SiLU: silu(1)=0.731, silu(0)=0, silu(-1)=-0.269 (z*sigmoid(z)). 10// T2 SiLU BACKWARD GRADCHECK: silu'(z)=sigmoid(z)*(1+z*(1-sigmoid(z))) == central finite-difference. 11// T3 SwiGLU GATE GRADCHECK: y = SiLU(a)*b ; dL/da and dL/db == finite-difference (the gated FFN element). 12// T4 NEGATIVE CONTROL: a doubled gradient is REJECTED. 13// T5 = SwiGLU backprops in f32 -> with RMSNorm/RoPE (R3) + softmax (R2a), the Llama-arch ops are covered. NEXT: GQA wiring + Adam. 14// license_tier: ORIGINAL 15import "nx_f32_hw.nx" 16import "nx_syscalls.nx" 17 18func grow(name: *u8, ok: i64) -> i64 { if ok==1 { gw(" PASS " as *u8) } else { gw(" FAIL " as *u8) } gw(name); gw(" 19" as *u8); return ok } 20func gm(x: i64) -> i64 { return gn(f32_int(f32_mul(x, f32_of(1000)))) } 21func f32_abs(x: i64) -> i64 { return x & 0x7FFFFFFF } 22func f32_le(x: i64, y: i64) -> i64 { let d: i64=f32_sub(x,y) & 0xFFFFFFFF; if ((d>>31)&1)==1 { return 1 } if (d & 0x7FFFFFFF)==0 { return 1 } return 0 } 23func f32_exp(x: i64) -> i64 { var sum: i64=f32_of(1); var term: i64=f32_of(1); var k: i64=1; while k<=16 { term=f32_div(f32_mul(term,x), f32_of(k)); sum=f32_add(sum,term); k=k+1 } return sum } 24 25// sigmoid(z) = 1/(1+exp(-z)). 26func f32_sigmoid(z: i64) -> i64 { return f32_div(f32_of(1), f32_add(f32_of(1), f32_exp(f32_neg(z)))) } 27// SiLU(z) = z*sigmoid(z). 28func f32_silu(z: i64) -> i64 { return f32_mul(z, f32_sigmoid(z)) } 29// SiLU'(z) = sigmoid(z)*(1 + z*(1 - sigmoid(z))). 30func f32_silu_deriv(z: i64) -> i64 { let s: i64=f32_sigmoid(z); return f32_mul(s, f32_add(f32_of(1), f32_mul(z, f32_sub(f32_of(1), s)))) } 31 32func main() -> i64 { 33 gw("=== nx_f32_swiglu_gate: RUNG 3b -- SwiGLU / SiLU activation backward (gradchecked), Llama-arch FFN ===\n" as *u8) 34 var pass: i64=0; var total: i64=0 35 let h: i64=f32_div(f32_of(1), f32_of(100)); let tol: i64=f32_div(f32_of(2), f32_of(100)); let twoh: i64=f32_mul(f32_of(2), h) 36 let one: i64=f32_of(1); let zero: i64=f32_of(0) 37 38 // T0 sigmoid. 39 let s0: i64=f32_int(f32_mul(f32_sigmoid(zero),f32_of(1000))) 40 let s1: i64=f32_int(f32_mul(f32_sigmoid(one),f32_of(1000))) 41 let sm1: i64=f32_int(f32_mul(f32_sigmoid(f32_neg(one)),f32_of(1000))) 42 total=total+1; if s0==500 { if s1>=729 { if s1<=733 { if sm1>=267 { if sm1<=271 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) } } else { gw(" [FAIL] " as *u8) } } else { gw(" [FAIL] " as *u8) } } else { gw(" [FAIL] " as *u8) } } else { gw(" [FAIL] " as *u8) } 43 gw("T0 sigmoid (milliunits): s(0)=" as *u8); gn(s0); gw(" s(1)=" as *u8); gn(s1); gw(" s(-1)=" as *u8); gn(sm1); gw(" (expect 500/731/269)\n" as *u8) 44 45 // T1 SiLU. 46 let l1: i64=f32_int(f32_mul(f32_silu(one),f32_of(1000))) 47 let l0: i64=f32_int(f32_mul(f32_silu(zero),f32_of(1000))) 48 total=total+1; if l1>=729 { if l1<=733 { if l0==0 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) } } else { gw(" [FAIL] " as *u8) } } else { gw(" [FAIL] " as *u8) } 49 gw("T1 SiLU (milliunits): silu(1)=" as *u8); gn(l1); gw(" silu(0)=" as *u8); gn(l0); gw(" (z*sigmoid(z); expect 731/0)\n" as *u8) 50 51 // T2 SiLU backward gradcheck at z=1. 52 let z: i64=one 53 let ana: i64=f32_silu_deriv(z) 54 let fd: i64=f32_div(f32_sub(f32_silu(f32_add(z,h)), f32_silu(f32_sub(z,h))), twoh) 55 let ok2: i64=f32_le(f32_abs(f32_sub(ana,fd)), tol) 56 total=total+1; if ok2==1 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) } 57 gw("T2 SiLU BACKWARD GRADCHECK: silu'(1) autograd=" as *u8); gm(ana); gw("m finite-diff=" as *u8); gm(fd); gw("m (expect ~928)\n" as *u8) 58 59 // T3 SwiGLU gate gradcheck: y = SiLU(a)*b ; L = y. a=1,b=2. 60 let a: i64=f32_of(1); let b: i64=f32_of(2) 61 let da: i64=f32_mul(f32_silu_deriv(a), b) // dL/da = silu'(a)*b 62 let db: i64=f32_silu(a) // dL/db = silu(a) 63 let fda: i64=f32_div(f32_sub(f32_mul(f32_silu(f32_add(a,h)),b), f32_mul(f32_silu(f32_sub(a,h)),b)), twoh) 64 let fdb: i64=f32_div(f32_sub(f32_mul(f32_silu(a),f32_add(b,h)), f32_mul(f32_silu(a),f32_sub(b,h))), twoh) 65 var ok3: i64=1 66 if f32_le(f32_abs(f32_sub(da,fda)),tol)==0 { ok3=0 } 67 if f32_le(f32_abs(f32_sub(db,fdb)),tol)==0 { ok3=0 } 68 total=total+1; if ok3==1 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) } 69 gw("T3 SwiGLU GATE GRADCHECK: y=SiLU(a)*b -> dL/da=" as *u8); gm(da); gw("m(fd " as *u8); gm(fda); gw("m), dL/db=" as *u8); gm(db); gw("m(fd " as *u8); gm(fdb); gw("m)\n" as *u8) 70 71 // T4 negative control. 72 let wrong: i64=f32_mul(da, f32_of(2)) 73 total=total+1; if f32_le(f32_abs(f32_sub(wrong,fda)),tol)==0 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) } 74 gw("T4 NEGATIVE CONTROL: a doubled dL/da " as *u8); gm(wrong); gw("m is REJECTED vs finite-diff " as *u8); gm(fda); gw("m\n" as *u8) 75 76 // T5. 77 total=total+1; if ok2==1 { if ok3==1 { pass=pass+1; gw(" [PASS] " as *u8) } else { gw(" [FAIL] " as *u8) } } else { gw(" [FAIL] " as *u8) } 78 gw("T5 SwiGLU DONE: SiLU/SwiGLU backprops in f32 -> with RMSNorm/RoPE (R3) + softmax (R2a), the Llama-arch ops backprop correctly\n" as *u8) 79 80 gw("\n RUNG 3b DONE (SOVEREIGN): SwiGLU (SiLU gating) backprops correctly in f32 -- sigmoid on our f32_exp, gradchecked. With\n" as *u8) 81 gw(" softmax + layernorm + RMSNorm + RoPE + SwiGLU, every transformer-block nonlinearity has a verified f32 backward. ALL our\n" as *u8) 82 gw(" implementation of PUBLISHED math/architecture (no copied code, no Llama weights, ORIGINAL license, nx_cc no-gcc). NEXT: GQA\n" as *u8) 83 gw(" wiring (composes the verified attention/softmax) + cross-entropy loss (needs f32_log) + R3 Adam + R4 data + R5 train loop.\n" as *u8) 84 gw("F32-SWIGLU verdict=" as *u8) 85 if pass==total { gw("GREEN passes=" as *u8); gn(pass); gw("/" as *u8); gn(total); gw(" -- SwiGLU/SiLU backward correct (gradchecked), sovereign\n" as *u8); sys_exit(0); return 0 } 86 gw("RED passes=" as *u8); gn(pass); gw("/" as *u8); gn(total); gw("\n" as *u8); sys_exit(1); return 1 87}