nx_nofloat_kda_gate.nx source

1// nx_nofloat_kda_gate.nx -- gate: Kimi Delta Attention, the K3-distinctive behaviors (not just "it runs"). 2// T1 DELTA OVERWRITE: writing the SAME key twice (v1 then v2) OVERWRITES to v2 -- vs plain linear-attn 3// which superposes to v1+v2. This is the delta-rule signature (key-value memory, not accumulation). 4// T2 CHANNEL GATING DECAY: a stored value decays monotonically under repeated per-channel gating (alpha<1). 5// T3 GATE=1 PERSISTENCE: with alpha=1 and no write, the memory is bit-identical across steps (stable). 6// T4 DETERMINISTIC: kda_forward twice = bit-identical (the exceed: float delta/gated stacks drift). 7// T5 FINE-GRAINED (per-channel): two key-channels with DIFFERENT alpha decay at different rates -> the 8// cross-channel magnitude RATIO changes (scalar/GLA gating would preserve it). The KDA-vs-GLA distinction. 9// license_tier: ORIGINAL No hw writes (Rule 26). expect_exit: 0 10import "nx_nofloat_kda.nx" 11import "nx_nofloat_linattn.nx" 12import "nx_gate_verdict.nx" 13import "nx_syscalls.nx" 14 15const KG_Q: i64 = 65536 16const KG_D: i64 = 4 17 18func kg_abs(x: i64) -> i64 { if x < 0 { return 0 - x } return x } 19 20func main() -> i64 { 21 let ctr: *i64 = gv_ctr() 22 gv_head("nx_nofloat_kda gate -- Kimi Delta Attention: delta-overwrite + per-channel gating, deterministic" as *u8) 23 let d: i64 = KG_D 24 let Q: i64 = KG_Q 25 26 // one-hot unit key e0 = [1,0,0,0] (k.k = 1.0), and value vectors 27 let e0: *i64 = sys_mmap(16*8) as *i64 28 let ones: *i64 = sys_mmap(16*8) as *i64 29 let v1: *i64 = sys_mmap(16*8) as *i64 30 let v2: *i64 = sys_mmap(16*8) as *i64 31 var i: i64 = 0 32 while i < d { e0[i] = 0; ones[i] = Q; v1[i] = Q/2; v2[i] = (Q*3)/10; i = i + 1 } 33 e0[0] = Q 34 let pred: *i64 = sys_mmap(16*8) as *i64 35 let delta: *i64 = sys_mmap(16*8) as *i64 36 37 // ---- T1: delta OVERWRITE vs plain SUPERPOSE ---- 38 let sk: *i64 = sys_mmap(LA_DMAX*LA_DMAX*8) as *i64 39 let sp: *i64 = sys_mmap(LA_DMAX*LA_DMAX*8) as *i64 40 la_zero(sk, d); la_zero(sp, d) 41 // KDA: two delta-writes of the same key, no gating (alpha=1, beta=1) 42 kda_write(sk, e0, v1, Q, d, pred, delta) 43 kda_write(sk, e0, v2, Q, d, pred, delta) 44 let okda: *i64 = sys_mmap(16*8) as *i64 45 la_readout(sk, e0, okda, d) 46 // plain linear-attn: two accums of the same key 47 la_accum(sp, e0, v1, d) 48 la_accum(sp, e0, v2, d) 49 let oplain: *i64 = sys_mmap(16*8) as *i64 50 la_readout(sp, e0, oplain, d) 51 // assert: kda ~ v2 (overwrite), plain ~ v1+v2 (superpose), and they DIFFER 52 var t1: i64 = 1 53 let eps: i64 = Q/100 54 i = 0 55 while i < d { 56 if kg_abs(okda[i] - v2[i]) > eps { t1 = 0 } 57 if kg_abs(oplain[i] - (v1[i]+v2[i])) > eps { t1 = 0 } 58 if kg_abs(okda[i] - oplain[i]) < eps { t1 = 0 } 59 i = i + 1 60 } 61 gv_check("T1 DELTA-RULE OVERWRITE: same-key rewrite -> v2 (kda) vs v1+v2 (plain linattn)" as *u8, t1, ctr) 62 63 // ---- T2: per-channel gating decays a stored value monotonically ---- 64 let s2: *i64 = sys_mmap(LA_DMAX*LA_DMAX*8) as *i64 65 let ahalf: *i64 = sys_mmap(16*8) as *i64 66 i = 0 67 while i < d { ahalf[i] = Q/2; i = i + 1 } 68 la_zero(s2, d) 69 kda_write(s2, e0, v1, Q, d, pred, delta) // store v1 along e0 70 let o2: *i64 = sys_mmap(16*8) as *i64 71 la_readout(s2, e0, o2, d) 72 let m0: i64 = kg_abs(o2[0]) 73 kda_gate(s2, ahalf, d); la_readout(s2, e0, o2, d); let m1: i64 = kg_abs(o2[0]) 74 kda_gate(s2, ahalf, d); la_readout(s2, e0, o2, d); let m2: i64 = kg_abs(o2[0]) 75 kda_gate(s2, ahalf, d); la_readout(s2, e0, o2, d); let m3: i64 = kg_abs(o2[0]) 76 var t2: i64 = 0 77 if m1 < m0 { if m2 < m1 { if m3 < m2 { if m0 > 0 { t2 = 1 } } } } 78 gv_check("T2 CHANNEL GATING DECAY: stored value shrinks monotonically under alpha<1" as *u8, t2, ctr) 79 80 // ---- T3: alpha=1 + no write -> memory bit-identical across steps ---- 81 let s3: *i64 = sys_mmap(LA_DMAX*LA_DMAX*8) as *i64 82 la_zero(s3, d) 83 kda_write(s3, e0, v1, Q, d, pred, delta) 84 let o3a: *i64 = sys_mmap(16*8) as *i64 85 let o3b: *i64 = sys_mmap(16*8) as *i64 86 la_readout(s3, e0, o3a, d) 87 kda_gate(s3, ones, d); kda_gate(s3, ones, d); kda_gate(s3, ones, d) 88 la_readout(s3, e0, o3b, d) 89 var t3: i64 = 1 90 i = 0 91 while i < d { if o3a[i] != o3b[i] { t3 = 0 } i = i + 1 } 92 gv_check("T3 GATE=1 PERSISTENCE: memory bit-identical across empty gated steps" as *u8, t3, ctr) 93 94 // ---- T4: deterministic forward ---- 95 let t: i64 = 4 96 let qm: *i64 = sys_mmap(64*8) as *i64 97 let km: *i64 = sys_mmap(64*8) as *i64 98 let vm: *i64 = sys_mmap(64*8) as *i64 99 let oA: *i64 = sys_mmap(64*8) as *i64 100 let oB: *i64 = sys_mmap(64*8) as *i64 101 i = 0 102 while i < t*d { qm[i] = ((i%3)+1)*(Q/8); km[i] = ((i%4)+1)*(Q/10); vm[i] = ((i%5)+1)*(Q/12); i = i + 1 } 103 kda_forward(qm, km, vm, oA, ahalf, Q, t, d) 104 kda_forward(qm, km, vm, oB, ahalf, Q, t, d) 105 var t4: i64 = 1 106 i = 0 107 while i < t*d { if oA[i] != oB[i] { t4 = 0 } i = i + 1 } 108 gv_check("T4 DETERMINISTIC bit-identical forward (float delta/gated stacks drift)" as *u8, t4, ctr) 109 110 // ---- T5: fine-grained per-channel gating changes the cross-channel ratio ---- 111 // key with mass in rows 0 AND 1 (0.707,0.707): store v, then gate rows at DIFFERENT rates. 112 let ktwo: *i64 = sys_mmap(16*8) as *i64 113 i = 0 114 while i < d { ktwo[i] = 0; i = i + 1 } 115 ktwo[0] = 46341; ktwo[1] = 46341 // ~1/sqrt2 each (unit key) 116 let adiff: *i64 = sys_mmap(16*8) as *i64 117 i = 0 118 while i < d { adiff[i] = Q; i = i + 1 } 119 adiff[0] = Q/2; adiff[1] = Q/4 // row0 halves, row1 quarters -> differential 120 let s5: *i64 = sys_mmap(LA_DMAX*LA_DMAX*8) as *i64 121 la_zero(s5, d) 122 kda_write(s5, ktwo, v1, Q, d, pred, delta) 123 // pick row0 via e0, row1 via e1 124 let e1: *i64 = sys_mmap(16*8) as *i64 125 i = 0 126 while i < d { e1[i] = 0; i = i + 1 } 127 e1[1] = Q 128 let r0: *i64 = sys_mmap(16*8) as *i64 129 let r1: *i64 = sys_mmap(16*8) as *i64 130 la_readout(s5, e0, r0, d); la_readout(s5, e1, r1, d) 131 let a0: i64 = kg_abs(r0[0]); let b0: i64 = kg_abs(r1[0]) // magnitudes before gating (ratio ~1) 132 kda_gate(s5, adiff, d); kda_gate(s5, adiff, d) // two differential gated steps 133 la_readout(s5, e0, r0, d); la_readout(s5, e1, r1, d) 134 let a1: i64 = kg_abs(r0[0]); let b1: i64 = kg_abs(r1[0]) // after: row1 decayed faster 135 // ratio b/a: before ~1, after should DROP (row1 quartered^2 vs row0 halved^2). scale by Q to compare. 136 var t5: i64 = 0 137 if a0 > 0 { if a1 > 0 { 138 let ratio_before: i64 = (b0 * Q) / a0 139 let ratio_after: i64 = (b1 * Q) / a1 140 if ratio_after < ratio_before { t5 = 1 } 141 } } 142 gv_check("T5 FINE-GRAINED per-channel gating: cross-channel ratio shifts (KDA-vs-GLA distinction)" as *u8, t5, ctr) 143 144 let rc: i64 = gv_verdict("NOFLOAT-KDA-GATE" as *u8, ctr, "Kimi Delta Attention: delta-overwrite memory + per-channel diagonal gating, bit-exact deterministic no-float" as *u8) 145 sys_exit(rc) 146 return rc 147}