nx_nofloat_k3interleave.nx source

1// nx_nofloat_k3interleave.nx -- THE K3 BLOCK STRUCTURE, ASSEMBLED (operator 2026-07-19 "logically integrated, 2// mature, state of the art"). K3 = DeepSeek-V3/Moonlight skeleton with attention layers INTERLEAVED 3:1 -- 3// three Kimi-Delta-Attention (linear, O(1) KV) layers to one Multi-head-Latent-Attention (softmax, LC-latent 4// KV) layer (arXiv 2510.26692). This organ proves that pattern COMPOSES from the already-proven parts: 5// - the "3" KDA layers call the REAL nx_nofloat_kda kda_forward (delta-rule overwrite + per-channel gating) 6// - the "1" MLA layer calls the REAL nx_nofloat_mla nmla_forward_latent (down-proj latent cache -> softmax) 7// - every layer: attention sublayer + residual, then FFN sublayer + residual, then RMSNorm (depth-stable) 8// Schedule: layer l is MLA iff (l % 4 == 3), else KDA -> [KDA,KDA,KDA,MLA] repeating = exactly 3:1. 9// Pure integer Q16, deterministic AT DEPTH (a float interleaved stack drifts + compounds; ours is bit-exact). 10// Weight-tied across same-type layers (the composition proof, not a trained net). license_tier: ORIGINAL 11// No hw writes (Rule 26). Reuses k3_matmul/k3_add/k3_relu/ks_rmsnorm (DRY rule-15). 12import "nx_nofloat_k3stack.nx" 13import "nx_nofloat_kda.nx" 14import "nx_nofloat_mla.nx" 15import "nx_syscalls.nx" 16 17const KIL_Q: i64 = 65536 18const KIL_CELL: i64 = 256 // t*d cap per buffer 19 20// the 3:1 schedule: 1 = MLA layer, 0 = KDA layer. l counts from 0. 21func k3il_layer_type(l: i64) -> i64 { 22 if (l % 4) == 3 { return 1 } 23 return 0 24} 25 26// L-layer K3 interleaved stack. KDA weights wq/wk/wv/wo/w1/w2 (tied across KDA layers); MLA weights 27// Wdkv/Wuk/Wuv/Wqm/Wom (tied across MLA layers). x,out: T x D (Q16). nh*hd = D. lc = latent dim (< D). 28func k3il_stack(x: *i64, wq: *i64, wk: *i64, wv: *i64, wo: *i64, w1: *i64, w2: *i64, 29 Wdkv: *i64, Wuk: *i64, Wuv: *i64, Wqm: *i64, Wom: *i64, 30 layers: i64, t: i64, d: i64, dff: i64, nh: i64, hd: i64, lc: i64, scale: i64, out: *i64) -> i64 { 31 let td: i64 = t * d 32 let hcur: *i64 = sys_mmap(KIL_CELL*8) as *i64 33 let hnew: *i64 = sys_mmap(KIL_CELL*8) as *i64 34 let attn: *i64 = sys_mmap(KIL_CELL*8) as *i64 35 let ao: *i64 = sys_mmap(KIL_CELL*8) as *i64 36 let q: *i64 = sys_mmap(KIL_CELL*8) as *i64 37 let k: *i64 = sys_mmap(KIL_CELL*8) as *i64 38 let v: *i64 = sys_mmap(KIL_CELL*8) as *i64 39 let f1: *i64 = sys_mmap(KIL_CELL*8) as *i64 40 let f2: *i64 = sys_mmap(KIL_CELL*8) as *i64 41 let alpha: *i64 = sys_mmap(LA_DMAX*8) as *i64 42 let scr: *i64 = sys_mmap(1024*8) as *i64 43 let cc: *i64 = sys_mmap(KIL_CELL*8) as *i64 44 let ap: *i64 = sys_mmap(16*8) as *i64 45 var i: i64 = 0 46 while i < d { alpha[i] = KIL_Q; i = i + 1 } // gate=1 -> KDA in pure delta-rule mode 47 i = 0 48 while i < td { hcur[i] = x[i]; i = i + 1 } 49 var l: i64 = 0 50 while l < layers { 51 if k3il_layer_type(l) == 0 { 52 // ---- KDA attention sublayer (the "3") ---- 53 k3_matmul(hcur, wq, q, t, d, d) 54 k3_matmul(hcur, wk, k, t, d, d) 55 k3_matmul(hcur, wv, v, t, d, d) 56 kda_forward(q, k, v, attn, alpha, KIL_Q, t, d) 57 k3_matmul(attn, wo, ao, t, d, d) 58 k3_add(hcur, ao, hnew, t, d) // residual 59 } else { 60 // ---- MLA attention sublayer (the "1") ---- 61 ap[0] = hcur as i64 62 ap[1] = attn as i64 63 ap[2] = t 64 ap[3] = d 65 ap[4] = nh 66 ap[5] = hd 67 ap[6] = lc 68 ap[7] = scale 69 ap[8] = Wdkv as i64 70 ap[9] = Wuk as i64 71 ap[10] = Wuv as i64 72 ap[11] = Wqm as i64 73 ap[12] = Wom as i64 74 ap[13] = scr as i64 75 ap[14] = cc as i64 76 nmla_forward_latent(ap) // attn = MLA(hcur), already through Wo 77 k3_add(hcur, attn, hnew, t, d) // residual 78 } 79 // ---- FFN sublayer (shared) ---- 80 k3_matmul(hnew, w1, f1, t, d, dff) 81 k3_relu(f1, t, dff) 82 k3_matmul(f1, w2, f2, t, dff, d) 83 k3_add(hnew, f2, hcur, t, d) // residual -> hcur is this layer's output 84 ks_rmsnorm(hcur, td) // depth-stable pre-norm 85 l = l + 1 86 } 87 i = 0 88 while i < td { out[i] = hcur[i]; i = i + 1 } 89 return 0 90}