nx_silu.nx source

1// nx_silu.nx -- SiLU / Swish activation: x * sigmoid(x). 2// 3// Closes the FFN activation gap. Together with nx_rmsnorm (shipped 4// dfce1e32) this lets NishiLang run a full Llama-class transformer 5// block end-to-end: input -> RMSNorm -> attention -> RMSNorm -> FFN 6// (matmul -> SiLU -> matmul) -> output. 7// 8// SiLU = Sigmoid Linear Unit = x * sigmoid(x). 9// * Ramachandran et al. 2017 _Searching for Activation Functions_ 10// * Elfwing et al. 2018 _Sigmoid-Weighted Linear Units for Neural 11// Network Function Approximation in Reinforcement Learning_ 12// 13// Modern transformer FFN uses SiLU exclusively (or its closely- 14// related variant GeGLU which gates by gelu instead). Used by: 15// 16// Llama 2/3 SwiGLU FFN (uses SiLU as the gate) 17// Mistral, Mixtral, Qwen, Gemma SwiGLU FFN 18// Z-Image / Flux / Stable Diffusion 3 attention FFN 19// 20// ===== Math ======================================================= 21// 22// sigmoid(x) = 1 / (1 + exp(-x)) 23// silu(x) = x * sigmoid(x) 24// 25// Q10 fixed-point: 26// 27// For x >= 0: e_neg = exp(-x) ; sigmoid = Q10 / (Q10 + e_neg) 28// For x < 0: e_neg = exp(x) ; sigmoid = e_neg / (Q10 + e_neg) 29// 30// (Both branches use only the negative-domain exp lookup -- no 31// overflow risk.) Then silu = x * sigmoid / Q10. 32// 33// Per the bits-up cardinal: composes against 34// nx_exp_q10_neg (canonical exp primitive) 35// NxTensor (canonical L1 container) 36// LoopVerdict (bounded-loop discipline) 37// 38// genealogy_id: ramachandran_2017_silu + elfwing_2018_sil + 39// hendrycks_gimpel_2016_gelu (sibling activation) 40// lineage_id: substrate_silu_v1 41 42// nx_safety_envelope: 43// intended_use: AUTO_APPLIED -- primitive-specific tuning queued 44// sil_target: SIL1 45// evidence: [bulk_applied_2026-05-16, see-file-comment-for-detail] 46// verdict: NOT_YET_EVALUATED 47 48import "nx_syscalls.nx" 49import "nx_tier.nx" 50import "nx_loop.nx" 51import "nx_tensor.nx" 52import "nx_exp.nx" 53 54const NX_SILU_Q10: nx_int = 1024 55 56// ===== Sealed-enum: SiluVerdict =================================== 57 58const NX_SILU_OK: nx_int = 0 59const NX_SILU_ERR_BAD_DTYPE: nx_int = 1 60const NX_SILU_ERR_SHAPE_MISMATCH: nx_int = 2 61const NX_SILU_ERR_NOT_CONTIGUOUS: nx_int = 3 62const NX_SILU_N_VERDICTS: nx_int = 4 63 64func nx_silu_verdict_is_valid(v: nx_int) -> nx_int { 65 if v < 0 { return 0 } 66 if v >= NX_SILU_N_VERDICTS { return 0 } 67 return 1 68} 69 70// ===== Sigmoid scalar (Q10) ======================================= 71// 72// Returns Q10 in [0, 1024]. 73 74func nx_sigmoid_q10(x_q10: nx_int) -> nx_int { 75 if x_q10 == 0 { return NX_SILU_Q10 / 2 } 76 if x_q10 > 0 { 77 let e_neg: nx_int = nx_exp_q10_neg(0 - x_q10) 78 let denom: nx_int = NX_SILU_Q10 + e_neg 79 if denom <= 0 { return NX_SILU_Q10 } 80 return (NX_SILU_Q10 * NX_SILU_Q10) / denom 81 } 82 let e_neg: nx_int = nx_exp_q10_neg(x_q10) 83 let denom: nx_int = NX_SILU_Q10 + e_neg 84 if denom <= 0 { return 0 } 85 return (e_neg * NX_SILU_Q10) / denom 86} 87 88// ===== SiLU scalar (Q10) ========================================== 89// 90// silu(x_q10) = x_q10 * sigmoid_q10(x_q10) / Q10. 91 92func nx_silu_q10(x_q10: nx_int) -> nx_int { 93 let s: nx_int = nx_sigmoid_q10(x_q10) 94 return (x_q10 * s) / NX_SILU_Q10 95} 96 97// ===== SiLU forward over a tensor ================================= 98// 99// Applies SiLU elementwise. In-place supported. 100 101func nx_silu_forward(x: *NxTensor, out: *NxTensor) -> nx_int { 102 if x.dtype != NX_DT_I64 { return NX_SILU_ERR_BAD_DTYPE } 103 if out.dtype != NX_DT_I64 { return NX_SILU_ERR_BAD_DTYPE } 104 if x.numel != out.numel { return NX_SILU_ERR_SHAPE_MISMATCH } 105 if nx_t_is_contiguous(x) == 0 { return NX_SILU_ERR_NOT_CONTIGUOUS } 106 if nx_t_is_contiguous(out) == 0 { return NX_SILU_ERR_NOT_CONTIGUOUS } 107 108 let px: *i64 = x.storage as *i64 109 let po: *i64 = out.storage as *i64 110 let n: nx_int = x.numel 111 112 var i: nx_int = 0 113 var iter: nx_int = 0 114 var verdict: nx_int = NX_LOOP_RUNNING 115 let BUDGET: nx_int = n 116 while verdict == NX_LOOP_RUNNING && iter < BUDGET { 117 po[i] = nx_silu_q10(px[i]) 118 i = i + 1 119 iter = iter + 1 120 } 121 return NX_SILU_OK 122} 123 124// ===== Self-test ================================================== 125// 126// Reference values (float) cross-checked: 127// 128// x sigmoid(x) silu(x) 129// -inf 0 0 130// -2.0 0.119 -0.238 131// -1.0 0.269 -0.269 132// -0.5 0.378 -0.189 133// 0.0 0.500 0.000 134// 0.5 0.622 0.311 135// 1.0 0.731 0.731 136// 2.0 0.881 1.762 137// +inf 1 ~x (linear regime) 138// 139// Closed-form invariants: 140// (a) silu(0) == 0 141// (b) sigmoid(0) == Q10 / 2 == 512 142// (c) sigmoid + sigmoid(-x) == Q10 (sigmoid is anti-symmetric 143// around 0.5) 144// (d) silu(x) - silu(-x) == x (silu(x) = silu(-x) + x; 145// identity from x*sigmoid(x) + x*(1 - sigmoid(x)) = x; only 146// approximately holds in Q10 due to exp interpolation rounding) 147// (e) silu(x) for large positive x ≈ x (sigmoid -> 1) 148// (f) silu(x) for large negative x ≈ 0 149 150func main() -> i64 { 151 // --- (a) silu(0) = 0 --- 152 if nx_silu_q10(0) != 0 { return 10 } 153 154 // --- (b) sigmoid(0) = Q10/2 --- 155 if nx_sigmoid_q10(0) != NX_SILU_Q10 / 2 { return 11 } 156 157 // --- (c) sigmoid(x) + sigmoid(-x) = Q10 (within rounding) --- 158 let s_p1: nx_int = nx_sigmoid_q10(1024) // sigmoid(1.0) 159 let s_n1: nx_int = nx_sigmoid_q10(-1024) // sigmoid(-1.0) 160 let sum1: nx_int = s_p1 + s_n1 161 let drift1: nx_int = sum1 - NX_SILU_Q10 162 if drift1 > 8 { return 20 } 163 if drift1 < -8 { return 21 } 164 165 let s_p2: nx_int = nx_sigmoid_q10(2048) 166 let s_n2: nx_int = nx_sigmoid_q10(-2048) 167 let sum2: nx_int = s_p2 + s_n2 168 let drift2: nx_int = sum2 - NX_SILU_Q10 169 if drift2 > 8 { return 22 } 170 if drift2 < -8 { return 23 } 171 172 // --- (d) sigmoid(1.0) ≈ 0.731 -> 748 Q10 --- 173 if s_p1 < 740 { return 30 } 174 if s_p1 > 755 { return 31 } 175 176 // --- (e) silu(1.0) = 1.0 * 0.731 = 0.731 -> 748 Q10 --- 177 let sl_p1: nx_int = nx_silu_q10(1024) 178 if sl_p1 < 740 { return 40 } 179 if sl_p1 > 755 { return 41 } 180 181 // --- (f) silu(-1.0) = -1.0 * 0.269 = -0.269 -> -276 Q10 --- 182 let sl_n1: nx_int = nx_silu_q10(-1024) 183 if sl_n1 < -280 { return 50 } 184 if sl_n1 > -260 { return 51 } 185 186 // --- (g) silu(large positive) approaches x --- 187 let sl_big: nx_int = nx_silu_q10(8192) // x = 8.0 188 let drift_big: nx_int = sl_big - 8192 189 if drift_big > 0 { return 60 } // silu(x) <= x for finite x 190 if drift_big < -50 { return 61 } // close to x 191 192 // --- (h) silu(large negative) approaches 0 --- 193 let sl_nbig: nx_int = nx_silu_q10(-8192) 194 if sl_nbig > 0 { return 70 } 195 if sl_nbig < -100 { return 71 } 196 197 // --- (i) Tensor forward --- 198 let sh: *nx_int = sys_mmap(8) as *nx_int 199 sh[0] = 5 200 let err: *nx_int = sys_mmap(8) as *nx_int 201 err[0] = 0 202 let xt: *NxTensor = nx_t_alloc(NX_DT_I64, sh, 1, err) 203 let yt: *NxTensor = nx_t_alloc(NX_DT_I64, sh, 1, err) 204 if err[0] != 0 { return 80 } 205 206 let px: *i64 = xt.storage as *i64 207 px[0] = -2048 208 px[1] = -1024 209 px[2] = 0 210 px[3] = 1024 211 px[4] = 2048 212 213 let v: nx_int = nx_silu_forward(xt, yt) 214 if v != NX_SILU_OK { return 90 + v } 215 216 let py: *i64 = yt.storage as *i64 217 if py[2] != 0 { return 100 } // silu(0) = 0 218 if py[0] >= 0 { return 101 } // silu(-2) < 0 219 if py[4] <= 0 { return 102 } // silu(2) > 0 220 221 return 0 222}