nx_simtcore.nx source

1// nx_simtcore.nx -- the SOVEREIGN SIMT GPGPU EXECUTION CORE. 2// 3// This is the missing organ named by our OWN ruler: nx_sovgpu_census grades the cell 4// "open GPU core on OUR FPGA (Vortex/Nyuzi class)" as GAP. This file closes it. It is a 5// SIMT (single-instruction multiple-thread) execution core built from the first bit in 6// NishiLang on top of the sovereign RV64IM decoder + ALU already in nishi-silicon/hdl. 7// 8// WHAT SIMT ADDS OVER THE SCALAR CORE (rv64im_min_sim): 9// scalar core : 1 PC, 1 register file, 1 instruction -> 1 result 10// SIMT core : NW warps, each with 1 PC and a THREAD MASK; each warp owns NT threads; 11// each thread owns its own 32-entry register file. One fetched instruction 12// executes in lockstep across every ACTIVE thread of the selected warp. 13// A masked-off thread commits NOTHING -- no register write, no memory write. That single 14// rule is what makes control-flow divergence expressible on a machine with one PC. 15// 16// ===== BENCHMARK ORACLE (external, measured, never copied) ===== 17// Vortex (Georgia Tech, MICRO'54 2021, arXiv 2110.10857) is the reference open-source 18// RISC-V GPGPU. We use its PUBLISHED ARCHITECTURE AND MEASUREMENTS as a measuring stick. 19// We copy no Vortex source, RTL, or asset. Verified facts driving this design: 20// - Vortex extends RISC-V with SIX instructions: wspawn, tmc, split, join, bar, tex. 21// "by adding only six new instructions to the standard RISC-V ISA" (MICRO'21 sec 1). 22// - Divergence uses an IPDOM (immediate post-dominator) stack, TWO pushes per split. 23// - Thread mask: "If the bit in the thread mask for a specific thread is zero, no 24// modifications would be made to that thread's register file and no changes to the 25// cache would be made based on that thread." (arXiv 2002.12151). 26// - Scheduler picks ONE wavefront per cycle from a visible mask; stalled/barrier masks 27// remove warps from contention (MICRO'21 sec 4.1.1). 28// - Cache banks are kept equal to the threads-per-warp count (CARRV 2019). 29// - Baseline config 4W-4T; configs evaluated 4W-4T, 2W-8T, 8W-2T, 4W-8T, 8W-4T. 30// 31// ===== SOVEREIGN ENCODING, NOT VORTEX'S ===== 32// RISC-V reserves four opcodes for user extensions: 0x0B, 0x2B, 0x5B, 0x7B. Vortex 33// documents 0x5B for its `tex`. We take custom-0 = 0x0B so there is ZERO encoding 34// collision with Vortex: the CAPABILITY set is benchmarked, the BITS are our own. 35// 36// NEVER-BRICK (cardinal 26): this core is a pure behavioural model over caller-owned 37// memory. It writes no firmware, no device, no persistent hardware state. It is 38// deterministic and bounded by construction -- every run is replayable bit-for-bit. 39// 40// Status: SEED. 2026-07-31. license_tier: ORIGINAL expect_exit: 0 41 42import "nx_syscalls.nx" 43import "nishi_hdl_primitives.nx" 44import "rv64im_min_decoder.nx" 45import "rv64im_min_alu.nx" 46import "rv64im_min_sim.nx" 47 48// ===== Structural limits ================================================= 49// Hard ceilings on allocation. The core's ACTUAL warp/thread counts are runtime 50// configuration (nw/nt) so one binary can run every Vortex comparison point. 51 52const NX_SIMT_MAX_WARPS: i64 = 16 53const NX_SIMT_MAX_THREADS: i64 = 16 54const NX_SIMT_NREGS: i64 = 32 // RISC-V architectural GPRs, per thread 55const NX_SIMT_IPDOM_DEPTH: i64 = 32 // divergence nesting depth, per warp 56const NX_SIMT_NBARRIERS: i64 = 8 // hardware barrier slots 57// "the barrier ID encodes whether it has local scope (intra-core) or global scope 58// (inter-core)... the MSB of the barrier ID indicates global scope" (MICRO'21 sec 4.1.3). 59// We mirror that exactly: bit 7 set = global, and the low bits still select the slot. 60const NX_SIMT_BAR_GLOBAL: i64 = 128 61 62// ===== Timing model constants ================================================= 63// Rule 11 (no magic numbers): every latency is a named knob, and the memory numbers are 64// now SOURCED rather than invented. 65// 66// PROVENANCE: Vortex MICRO'21 Figure 21 ("The effect of memory scaling on performance") 67// carries its parameter legend INSIDE the plot bitmap -- no PDF text layer, so text 68// extraction reports it as UNKNOWN. Read off the rendered figure at 9x: 69// sh: 24-cycle memory latency lg: 200-cycle memory latency 70// 2c: 2-channel memory 8c: 8-channel memory l2: L2 cache enabled 71// The same figure shows sgemm at 103 IPC under `sh` versus 16.5 IPC under `lg` -- a 6.2x 72// swing from memory latency ALONE, which is why a model that assumes short latency will 73// over-predict how much warp/thread partitioning matters. 74// 75// An earlier revision of this file used an invented 20-cycle latency. That silently 76// modelled the OPTIMISTIC end of Vortex's own range and is why the first benchmark could 77// only claim direction, never magnitude. 78 79const NX_SIMT_LAT_ALU: i64 = 1 // issue-to-issue for an ALU op 80 81// ===== MULTIPLY-ACCUMULATE LATENCY -- NOW SOURCED, NOT FITTED ===== 82// ★★★THIS CONSTANT WAS 3 AND UNSOURCED. It is now 8, and the difference matters more than 83// the number: 8 is ADOPTED FROM VORTEX'S OWN SOURCE, not chosen because it fits. 84// 85// PROVENANCE, triple-anchored and independent: 86// 1. `VX_config.toml:116` resolves VX_CFG_FMA_LATENCY to 8 for the default build. 87// 2. Their FPU design doc (docs/designs/floating_point_unit.md:138) independently states 88// "native STD = 8 (F32) / 12 (F64)". 89// 3. `VX_fma_unit_rtl.sv:55-69` DECOMPOSES that 8 into named pipeline stages that sum: 90// INI(1) -> MUL(LATENCY-6) -> ALN(1) -> ACC(1) -> NRM(1) -> RND(2), guarded by a 91// STATIC_ASSERT(MUL_LATENCY >= 1). 92// 93// ★THAT ASSERT RETROSPECTIVELY CONVICTS OUR OLD VALUE: it puts a HARD FLOOR OF 7 on F32 FMA, 94// so the 3 this model used was not merely wrong, it was STRUCTURALLY IMPOSSIBLE in the design 95// we were claiming to model. 96// 97// ⚠WE APPLY AN FMA LATENCY TO AN INTEGER MAC. Our calibration kernel uses the M-extension 98// multiply as a proxy for sgemm, which is floating-point on the real machine. Modelling the 99// proxy at the FMA latency is the faithful choice for calibrating AGAINST sgemm, and it is 100// stated here rather than hidden. 101// ⚠FPGA CAVEAT, DECLARED: Figure 14's IPC was measured on an FPGA, and on that path Vortex 102// selects VENDOR IP whose latency differs (16 on Xilinx; an Altera value of 4 that is 103// CONTESTED inside their own tree by a STATIC_ASSERT expecting 16). 8 is the default 104// soft-logic/SimX figure. If the 1200 baseline is FPGA-derived, the vendor number applies. 105// ★A SWEEP THAT LANDS ON THE SAME VALUE AS A SOURCE IS CORROBORATION, NOT CIRCULARITY -- the 106// sweep is why we looked; the source is why we adopted. Any residual gap after this is to be 107// REPORTED AS UNEXPLAINED, never closed by moving this constant again. 108const NX_SIMT_LAT_MULDIV: i64 = 8 109const NX_SIMT_LAT_MEM_SHORT: i64 = 24 // Vortex "sh" -- cache-resident / short latency 110const NX_SIMT_LAT_MEM_LONG: i64 = 200 // Vortex "lg" -- DRAM / long latency 111const NX_SIMT_CHAN_LOW: i64 = 2 // Vortex "2c" -- 2-channel memory 112const NX_SIMT_CHAN_HIGH: i64 = 8 // Vortex "8c" -- 8-channel memory 113const NX_SIMT_MAX_CHANNELS: i64 = 16 114 115// Cycles a channel stays busy per burst transferred. Without this a request occupied a 116// channel for a single cycle, so the service rate (nchan per cycle) always exceeded the 117// arrival rate (at most one warp issues per cycle) and NO QUEUE COULD EVER FORM -- the 118// channel count was inert and the model was latency-only while claiming to be a memory 119// system. A real DRAM burst holds its channel for several cycles; that is what makes 120// bandwidth a finite resource. Queueing appears once cost*BURST exceeds nchan. 121const NX_SIMT_BURST_CYCLES: i64 = 4 122 123// ===== L1 data cache ================================================= 124// Vortex MICRO'21 Figure 15 puts the D-cache at **35% of core area** -- the single largest 125// block, bigger than the texture unit (20%) and dwarfing the register file (1%). A model 126// without one is missing the most expensive thing on the chip. 127// 128// It is also missing a real BEHAVIOUR, not just a cost: the DATE'25 warp-features study 129// (arXiv 2505.03102) found a kernel, mse_forward, where the SOFTWARE emulation of warp ops 130// BEATS the hardware path, because loop serialisation collapses memory traffic that the 131// hardware path re-issues per lane. That crossover is pure cache reuse. A cacheless model 132// cannot represent it and will over-report the hardware win on EVERY kernel. 133// 134// Geometry from CARRV'19: "a two-way set associative L1 cache with 16 Kbytes". 135// 128 sets x 2 ways x 64B line = 16KB. 136const NX_SIMT_CACHE_LINE_B: i64 = 64 137const NX_SIMT_CACHE_SETS: i64 = 128 138const NX_SIMT_CACHE_WAYS: i64 = 2 139const NX_SIMT_LAT_CACHE_HIT: i64 = 4 // cycles when every touched line is resident 140 141// ===== SIMT opcode: RISC-V custom-0 ================================================= 142// R-type layout, exactly like the base ISA: funct7 | rs2 | rs1 | funct3 | rd | opcode. 143// funct3 selects the SIMT operation. 144 145const NX_SIMT_OPCODE: i64 = 0x0B 146 147const NX_SIMT_F3_TMC: i64 = 0 // tmc rs1 -- thread mask control 148const NX_SIMT_F3_WSPAWN: i64 = 1 // wspawn rs1, rs2 -- activate rs1 warps at PC rs2 149const NX_SIMT_F3_SPLIT: i64 = 2 // split rs1 -- control-flow divergence 150const NX_SIMT_F3_JOIN: i64 = 3 // join -- control-flow reconvergence 151const NX_SIMT_F3_BAR: i64 = 4 // bar rs1, rs2 -- barrier id rs1, warp count rs2 152const NX_SIMT_F3_TID: i64 = 5 // tid rd -- rd = this thread's lane id 153const NX_SIMT_F3_WID: i64 = 6 // wid rd -- rd = this warp's id 154const NX_SIMT_F3_NTID: i64 = 7 // ntid rd -- rd = threads per warp 155 156// ===== Warp-collective extension: RISC-V custom-1 ================================================= 157// Lane-to-lane data exchange and predicate reduction, in one instruction, without a round 158// trip through memory. These are what make a warp a cooperative unit rather than N 159// independent threads that happen to share a PC. 160// 161// ORACLE: Vortex's own DATE'25 study ("Hardware vs. Software Implementation of Warp-Level 162// Features in Vortex RISC-V GPU", arXiv 2505.03102) added hardware vx_shfl / vx_vote and 163// measured 2.42x geomean IPC over the software emulation (the abstract's "up to 4x" is a 164// per-kernel maximum, not the geomean -- cite 2.42x). 165// 166// Reading an INACTIVE lane returns 0 here. CUDA leaves that undefined; we define it, so 167// a divergent shuffle stays deterministic and replayable instead of exposing stale state. 168 169const NX_SIMT_OPCODE_W: i64 = 0x2B 170 171const NX_SIMT_W_SHFL: i64 = 0 // shfl rd, rs1, rs2 -- rd[t] = rs1[ rs2[t] ] 172const NX_SIMT_W_SHFL_UP: i64 = 1 // shfl.up rd, rs1, rs2 -- rd[t] = rs1[t - rs2[t]] 173const NX_SIMT_W_SHFL_DOWN:i64 = 2 // shfl.down rd, rs1, rs2 -- rd[t] = rs1[t + rs2[t]] 174const NX_SIMT_W_SHFL_XOR: i64 = 3 // shfl.xor rd, rs1, rs2 -- rd[t] = rs1[t ^ rs2[t]] 175const NX_SIMT_W_BALLOT: i64 = 4 // ballot rd, rs1 -- rd = mask of lanes with rs1 != 0 176const NX_SIMT_W_ALL: i64 = 5 // all rd, rs1 -- rd = 1 if every active lane nonzero 177const NX_SIMT_W_ANY: i64 = 6 // any rd, rs1 -- rd = 1 if any active lane nonzero 178const NX_SIMT_W_ACTIVEMASK: i64 = 7 // activemask rd -- rd = the current thread mask 179 180// ===== Texture sample unit: RISC-V custom-3, R4-type ================================================= 181// The second-largest block on a real Vortex core: MICRO'21 Figure 15 puts the texture unit at 182// 20 percent of core area, behind only the D-cache at 35 percent. 183// 184// ORACLE, recovered from MICRO'21 Figure 20 (execution time in ms, single core): 185// point SW 128 HW 127 = 1.01x (no benefit) 186// bilinear SW 508 HW 265 = 1.92x ("almost 2x on a single core" in their prose) 187// trilinear SW 560 HW 468 = 1.20x 188// Point sampling shows NO benefit because their source texture is already RGBA, so the 189// software path degenerates to a copy. Trilinear gains less because it doubles memory traffic. 190// 191// TWO DESIGN CHOICES COPIED FROM THEIR PAPER BECAUSE THEY ARE MEASURED, NOT AESTHETIC: 192// 1. "The texel sampler implements only bilinear filtering. Point sampling is executed using 193// bilinear filtering with blend values of 0." They judged the muxing and synchronisation 194// for a variable-latency sampler not worth the single cycle saved. We do the same -- ONE 195// filter path, point is bilinear with zero weights. 196// 2. "The texture memory unit first de-duplicates memory accesses that are repeated across 197// threads." We already have that machinery: nx_simt_coalesce. 198// 199// R4-type is required because a sample needs THREE sources (u, v, lod); rs3 sits in bits 31:27, 200// exactly as the base ISA's FMA encoding does. Vortex documents opcode 0x5B for its tex; we 201// take custom-3 (0x7B) so there is zero encoding collision. 202// 203// Coordinates are 24.8 FIXED POINT texel units -- NishiLang has no float type, and a GPU 204// texture unit is fixed-point in hardware anyway. Texels are packed RGBA8888. 205 206const NX_SIMT_OPCODE_TEX: i64 = 0x7B 207const NX_SIMT_TEX_FRACBITS: i64 = 8 // 24.8 fixed point 208const NX_SIMT_TEX_FRACMASK: i64 = 255 209const NX_SIMT_LAT_TEX_FILTER: i64 = 2 // their sampler's two-cycle bilinear interpolation 210 211// ===== Status codes ================================================= 212 213const NX_SIMT_OK: i64 = 0 214const NX_SIMT_E_ARG: i64 = 1 // bad argument / unconfigured core 215const NX_SIMT_E_IPDOM: i64 = 2 // IPDOM stack overflow or join-without-split 216const NX_SIMT_E_ILLEGAL: i64 = 3 // illegal instruction 217const NX_SIMT_E_BOUNDS: i64 = 4 // memory access out of the modelled aperture 218 219// ===== The core ================================================= 220// 221// All state is flat i64 arrays allocated once at init (rule: never allocate in the hot 222// loop -- sys_mmap is page-granular with no free, so a per-step allocation leaks). 223// 224// Indexing conventions: 225// regs[(w * nt + t) * NX_SIMT_NREGS + r] thread t of warp w, register r 226// ipdom_*[w * NX_SIMT_IPDOM_DEPTH + sp] divergence stack entry sp of warp w 227 228struct NxSimtCore { 229 nw: i64 // warps per core (runtime config) 230 nt: i64 // threads per warp (runtime config) 231 232 regs: *i64 // nw * nt * NX_SIMT_NREGS 233 pc: *i64 // nw -- one program counter per warp 234 tmask: *i64 // nw -- active-thread bitmask per warp 235 wactive: *i64 // nw -- 1 = warp is running 236 wstall_until: *i64 // nw -- cycle at which a memory-stalled warp may issue 237 wbarrier: *i64 // nw -- 1 = parked at a barrier 238 239 ipdom_mask: *i64 // nw * NX_SIMT_IPDOM_DEPTH -- saved thread mask 240 ipdom_pc: *i64 // nw * NX_SIMT_IPDOM_DEPTH -- saved resume PC 241 ipdom_ft: *i64 // nw * NX_SIMT_IPDOM_DEPTH -- 1 = fall-through entry 242 ipdom_sp: *i64 // nw -- divergence stack pointer 243 244 bar_count: *i64 // NX_SIMT_NBARRIERS -- warps still expected at this barrier 245 bar_mask: *i64 // NX_SIMT_NBARRIERS -- bitmask of warps parked here 246 247 // ----- GLOBAL (inter-core) barriers ----- 248 // These three point at CLUSTER-OWNED arrays so every core arrives at the same counter. 249 // Release is performed by the cluster loop, not by a core: a core cannot wake a warp 250 // that belongs to a different core, and pretending otherwise is how you get a barrier 251 // that releases locally and hangs globally. 252 gbar_on: i64 // 1 = wired into a cluster 253 gbar_count: *i64 // NX_SIMT_NBARRIERS, shared -- warps arrived so far 254 gbar_expect: *i64 // NX_SIMT_NBARRIERS, shared -- warps required 255 wbar_id: *i64 // NX_SIMT_MAX_WARPS -- which global barrier a warp waits on, -1 = none 256 257 // Lane-exchange staging for the warp-collective instructions. A shuffle is a 258 // PERMUTATION, so every lane's source value must be snapshotted BEFORE any 259 // destination is written -- writing in place would let an early lane clobber a 260 // source a later lane still needs. Allocated once at init, never in the hot loop. 261 lanebuf: *i64 // NX_SIMT_MAX_THREADS 262 263 // ----- memory system configuration (Vortex Fig 21 parameter space) ----- 264 // Latency alone is not a memory system: with only latency modelled, unlimited requests 265 // proceed in parallel and BANDWIDTH never binds, so adding warps looks free forever. 266 // Channels give the model a throughput ceiling -- each in-flight request occupies a 267 // channel, so once every channel is busy, extra warps queue instead of overlapping. 268 mem_lat: i64 // cycles to first data (NX_SIMT_LAT_MEM_SHORT or _LONG) 269 nchan: i64 // memory channels 270 chan_free: *i64 // NX_SIMT_MAX_CHANNELS -- cycle each channel next goes idle 271 272 // ----- register scoreboard (RAW dependency stalls) ----- 273 // reg_ready[w*NREGS + r] = the cycle at which register r of warp w holds its result. 274 // A warp cannot ISSUE until every source it reads is ready. This is what makes a 275 // multi-cycle unit cost anything: without it, an ALU result feeds the very next 276 // instruction and pipeline depth is invisible. It is also the mechanism by which EXTRA 277 // WARPS EARN THEIR KEEP -- one warp's dependency bubble is another warp's issue slot. 278 scoreboard_on: i64 279 reg_ready: *i64 // NX_SIMT_MAX_WARPS * NX_SIMT_NREGS 280 dep_stall_cycles: i64 // cycles where the picked-over warps were all dependency-blocked 281 282 // ----- L1 data cache ----- 283 cache_on: i64 // 0 = bypass (the old cacheless behaviour, kept for A/B) 284 cache_tag: *i64 // SETS*WAYS -- line address, or -1 for an invalid way 285 cache_age: *i64 // SETS*WAYS -- LRU stamp 286 linebuf: *i64 // NX_SIMT_MAX_THREADS -- distinct line addresses this access 287 cache_hits: i64 288 cache_misses: i64 289 290 // ----- texture unit state (Vortex keeps this in CSRs; we keep it in the core) ----- 291 tex_base: i64 // byte address of texel (0,0) 292 tex_w: i64 // texels 293 tex_h: i64 294 tex_ops: i64 // tex instructions executed 295 tex_texels: i64 // DISTINCT texel fetches after de-duplication 296 297 mem_base: i64 298 mem_buf: *u8 299 mem_size: i64 300 301 rr_next: i64 // round-robin scheduler cursor 302 303 valid: i64 304 halted: i64 305 status: i64 306 307 // ----- instrumentation: the numbers the benchmark reads ----- 308 cycles: i64 // total scheduler cycles elapsed 309 instret: i64 // WARP-instructions retired (Vortex's IPC numerator) 310 lane_instret: i64 // LANE-instructions retired (actual work performed) 311 stall_cycles: i64 // cycles where no warp could issue 312 mem_ops: i64 // warp-level memory instructions executed 313 bank_conflict_cycles: i64 // cycles lost purely to bank serialisation 314 mem_queue_cycles: i64 // cycles spent waiting for a free channel (bandwidth, not latency) 315} 316 317// ===== Mask helpers ================================================= 318// Defined before every user: this toolchain is single-pass, so a call that precedes its 319// definition is a build failure, not a link-time fixup. 320 321// All-ones mask for nt threads. Built by shifting rather than a literal so it stays 322// correct for every configured width. 323func nx_simt_full_mask(nt: i64) -> i64 { 324 var m: i64 = 0 325 var i: i64 = 0 326 while i < nt { m = m | (1 << i); i = i + 1 } 327 return m 328} 329 330func nx_simt_popcount(m: i64) -> i64 { 331 var n: i64 = 0 332 var v: i64 = m 333 while v != 0 { 334 if (v & 1) != 0 { n = n + 1 } 335 v = v >> 1 336 } 337 return n 338} 339 340// Lowest active lane -- the lane that supplies warp-scope operands (thread mask, warp 341// count, barrier id). Returns -1 when the mask is empty. 342func nx_simt_lowest_lane(tmask: i64) -> i64 { 343 var t: i64 = 0 344 while t < NX_SIMT_MAX_THREADS { 345 if (tmask & (1 << t)) != 0 { return t } 346 t = t + 1 347 } 348 return 0 - 1 349} 350 351// ===== Allocation + init ================================================= 352// 353// One allocation pass sized to the STRUCTURAL maximum, so a core can be reconfigured 354// across the Vortex comparison points without reallocating. 355 356func nx_simt_alloc(c: *NxSimtCore) -> i64 { 357 if (c as i64) == 0 { return 0 - NX_SIMT_E_ARG } 358 let nregs: i64 = NX_SIMT_MAX_WARPS * NX_SIMT_MAX_THREADS * NX_SIMT_NREGS 359 let nipd: i64 = NX_SIMT_MAX_WARPS * NX_SIMT_IPDOM_DEPTH 360 c.regs = (sys_mmap(8 * nregs)) as *i64 361 c.pc = (sys_mmap(8 * NX_SIMT_MAX_WARPS)) as *i64 362 c.tmask = (sys_mmap(8 * NX_SIMT_MAX_WARPS)) as *i64 363 c.wactive = (sys_mmap(8 * NX_SIMT_MAX_WARPS)) as *i64 364 c.wstall_until = (sys_mmap(8 * NX_SIMT_MAX_WARPS)) as *i64 365 c.wbarrier = (sys_mmap(8 * NX_SIMT_MAX_WARPS)) as *i64 366 c.ipdom_mask = (sys_mmap(8 * nipd)) as *i64 367 c.ipdom_pc = (sys_mmap(8 * nipd)) as *i64 368 c.ipdom_ft = (sys_mmap(8 * nipd)) as *i64 369 c.ipdom_sp = (sys_mmap(8 * NX_SIMT_MAX_WARPS)) as *i64 370 c.bar_count = (sys_mmap(8 * NX_SIMT_NBARRIERS)) as *i64 371 c.bar_mask = (sys_mmap(8 * NX_SIMT_NBARRIERS)) as *i64 372 c.lanebuf = (sys_mmap(8 * NX_SIMT_MAX_THREADS)) as *i64 373 c.chan_free = (sys_mmap(8 * NX_SIMT_MAX_CHANNELS)) as *i64 374 c.cache_tag = (sys_mmap(8 * NX_SIMT_CACHE_SETS * NX_SIMT_CACHE_WAYS)) as *i64 375 c.cache_age = (sys_mmap(8 * NX_SIMT_CACHE_SETS * NX_SIMT_CACHE_WAYS)) as *i64 376 c.linebuf = (sys_mmap(8 * NX_SIMT_MAX_THREADS)) as *i64 377 c.reg_ready = (sys_mmap(8 * NX_SIMT_MAX_WARPS * NX_SIMT_NREGS)) as *i64 378 c.wbar_id = (sys_mmap(8 * NX_SIMT_MAX_WARPS)) as *i64 379 return NX_SIMT_OK 380} 381 382// Configure and reset. entry_pc is where warp 0 begins; every other warp starts 383// INACTIVE, exactly as a real SIMT core boots -- warp 0 spawns the rest with wspawn. 384 385func nx_simt_init(c: *NxSimtCore, nw: i64, nt: i64, 386 mem_base: i64, mem_buf: *u8, mem_size: i64, 387 entry_pc: i64) -> i64 { 388 if (c as i64) == 0 { return 0 - NX_SIMT_E_ARG } 389 if nw < 1 { return 0 - NX_SIMT_E_ARG } 390 if nt < 1 { return 0 - NX_SIMT_E_ARG } 391 if nw > NX_SIMT_MAX_WARPS { return 0 - NX_SIMT_E_ARG } 392 if nt > NX_SIMT_MAX_THREADS { return 0 - NX_SIMT_E_ARG } 393 if (mem_buf as i64) == 0 { return 0 - NX_SIMT_E_ARG } 394 if mem_size <= 0 { return 0 - NX_SIMT_E_ARG } 395 396 c.nw = nw 397 c.nt = nt 398 c.mem_base = mem_base 399 c.mem_buf = mem_buf 400 c.mem_size = mem_size 401 402 var w: i64 = 0 403 while w < NX_SIMT_MAX_WARPS { 404 c.pc[w] = 0 405 c.tmask[w] = 0 406 c.wactive[w] = 0 407 c.wstall_until[w] = 0 408 c.wbarrier[w] = 0 409 c.ipdom_sp[w] = 0 410 c.wbar_id[w] = 0 - 1 411 w = w + 1 412 } 413 c.gbar_on = 0 414 var i: i64 = 0 415 let nregs: i64 = NX_SIMT_MAX_WARPS * NX_SIMT_MAX_THREADS * NX_SIMT_NREGS 416 while i < nregs { c.regs[i] = 0; i = i + 1 } 417 var b: i64 = 0 418 while b < NX_SIMT_NBARRIERS { c.bar_count[b] = 0; c.bar_mask[b] = 0; b = b + 1 } 419 420 // Warp 0 boots with every thread active; all other warps are dormant. 421 c.pc[0] = entry_pc 422 c.tmask[0] = nx_simt_full_mask(nt) 423 c.wactive[0] = 1 424 425 // Memory system defaults to the OPTIMISTIC corner (short latency, wide channels). 426 // Callers comparing against Vortex must set the corner explicitly with 427 // nx_simt_set_memory -- a benchmark that silently uses the fastest memory in the 428 // parameter space will flatter every configuration equally and hide the effect. 429 c.mem_lat = NX_SIMT_LAT_MEM_SHORT 430 c.nchan = NX_SIMT_CHAN_HIGH 431 var ch: i64 = 0 432 while ch < NX_SIMT_MAX_CHANNELS { c.chan_free[ch] = 0; ch = ch + 1 } 433 434 // Cache starts ON and cold. Every way invalid (-1 is not a reachable line address 435 // because addresses are non-negative), every LRU stamp zero. 436 c.cache_on = 1 437 var cw: i64 = 0 438 let nways: i64 = NX_SIMT_CACHE_SETS * NX_SIMT_CACHE_WAYS 439 while cw < nways { c.cache_tag[cw] = 0 - 1; c.cache_age[cw] = 0; cw = cw + 1 } 440 c.cache_hits = 0 441 c.cache_misses = 0 442 443 // Scoreboard on by default: a core without RAW stalls is not a pipeline. 444 c.tex_base = 0 445 c.tex_w = 0 446 c.tex_h = 0 447 c.tex_ops = 0 448 c.tex_texels = 0 449 450 c.scoreboard_on = 1 451 c.dep_stall_cycles = 0 452 var rr: i64 = 0 453 let nrr: i64 = NX_SIMT_MAX_WARPS * NX_SIMT_NREGS 454 while rr < nrr { c.reg_ready[rr] = 0; rr = rr + 1 } 455 456 c.rr_next = 0 457 c.valid = 1 458 c.halted = 0 459 c.status = NX_SIMT_OK 460 c.cycles = 0 461 c.instret = 0 462 c.lane_instret = 0 463 c.stall_cycles = 0 464 c.mem_ops = 0 465 c.bank_conflict_cycles = 0 466 c.mem_queue_cycles = 0 467 return NX_SIMT_OK 468} 469 470// ===== Per-thread register file ================================================= 471// x0 is hardwired zero, per RISC-V. A write to a masked-off thread never reaches here. 472 473func nx_simt_rf_read(c: *NxSimtCore, w: i64, t: i64, r: i64) -> i64 { 474 if r == 0 { return 0 } 475 if r < 0 { return 0 } 476 if r >= NX_SIMT_NREGS { return 0 } 477 return c.regs[(w * c.nt + t) * NX_SIMT_NREGS + r] 478} 479 480func nx_simt_rf_write(c: *NxSimtCore, w: i64, t: i64, r: i64, v: i64) -> i64 { 481 if r == 0 { return NX_SIMT_OK } 482 if r < 0 { return 0 - NX_SIMT_E_ARG } 483 if r >= NX_SIMT_NREGS { return 0 - NX_SIMT_E_ARG } 484 c.regs[(w * c.nt + t) * NX_SIMT_NREGS + r] = v 485 return NX_SIMT_OK 486} 487 488// ===== Memory ================================================= 489// A flat aperture. Out-of-range accesses are refused, never wrapped -- a silent wrap 490// would manufacture wrong results that look like a working kernel. 491 492func nx_simt_in_range(c: *NxSimtCore, addr: i64, width: i64) -> i64 { 493 let off: i64 = addr - c.mem_base 494 if off < 0 { return 0 } 495 if off + width > c.mem_size { return 0 } 496 return 1 497} 498 499func nx_simt_load32(c: *NxSimtCore, addr: i64) -> i64 { 500 if nx_simt_in_range(c, addr, 4) == 0 { c.status = NX_SIMT_E_BOUNDS; return 0 } 501 let o: i64 = addr - c.mem_base 502 let b0: i64 = c.mem_buf[o] as i64 503 let b1: i64 = c.mem_buf[o + 1] as i64 504 let b2: i64 = c.mem_buf[o + 2] as i64 505 let b3: i64 = c.mem_buf[o + 3] as i64 506 return b0 | (b1 << 8) | (b2 << 16) | (b3 << 24) 507} 508 509func nx_simt_store32(c: *NxSimtCore, addr: i64, v: i64) -> i64 { 510 if nx_simt_in_range(c, addr, 4) == 0 { c.status = NX_SIMT_E_BOUNDS; return 0 - NX_SIMT_E_BOUNDS } 511 let o: i64 = addr - c.mem_base 512 c.mem_buf[o] = (v & 0xff) as u8 513 c.mem_buf[o + 1] = ((v >> 8) & 0xff) as u8 514 c.mem_buf[o + 2] = ((v >> 16) & 0xff) as u8 515 c.mem_buf[o + 3] = ((v >> 24) & 0xff) as u8 516 return NX_SIMT_OK 517} 518 519func nx_simt_load64(c: *NxSimtCore, addr: i64) -> i64 { 520 let lo: i64 = nx_simt_load32(c, addr) & 0xffffffff 521 let hi: i64 = nx_simt_load32(c, addr + 4) & 0xffffffff 522 return lo | (hi << 32) 523} 524 525func nx_simt_store64(c: *NxSimtCore, addr: i64, v: i64) -> i64 { 526 nx_simt_store32(c, addr, v & 0xffffffff) 527 nx_simt_store32(c, addr + 4, (v >> 32) & 0xffffffff) 528 return NX_SIMT_OK 529} 530 531// ===== Bank-conflict model ================================================= 532// 533// Vortex keeps the data-cache bank count equal to the threads-per-warp count (CARRV'19: 534// "The number of banks is always kept the same as the number of threads in a warp"). A 535// warp-wide memory op therefore issues nt addresses into nt word-interleaved banks. 536// 537// Cost = the WORST per-bank load, in accesses. A unit-stride (coalesced) access spreads 538// one address per bank and costs 1. A stride-nt access piles every thread onto the same 539// bank and costs nt. This is the mechanism behind the paper's reported bank utilisation 540// (sgemm 67 percent, vecadd 71 percent at one port). 541// 542// bank_hits is caller-provided scratch of NX_SIMT_MAX_THREADS entries -- never allocated 543// here, because this runs once per memory instruction. 544 545func nx_simt_bank_cost(c: *NxSimtCore, addrs: *i64, active: i64, bank_hits: *i64) -> i64 { 546 var i: i64 = 0 547 while i < c.nt { bank_hits[i] = 0; i = i + 1 } 548 var t: i64 = 0 549 while t < c.nt { 550 if (active & (1 << t)) != 0 { 551 let word: i64 = addrs[t] >> 2 552 var bank: i64 = word % c.nt 553 if bank < 0 { bank = 0 - bank } 554 bank_hits[bank] = bank_hits[bank] + 1 555 } 556 t = t + 1 557 } 558 var worst: i64 = 0 559 var b: i64 = 0 560 while b < c.nt { 561 if bank_hits[b] > worst { worst = bank_hits[b] } 562 b = b + 1 563 } 564 if worst < 1 { worst = 1 } 565 return worst 566} 567 568// ===== Memory system configuration ================================================= 569// Select a corner of Vortex's published parameter space. lat should be 570// NX_SIMT_LAT_MEM_SHORT (their "sh") or NX_SIMT_LAT_MEM_LONG (their "lg"); nchan should be 571// NX_SIMT_CHAN_LOW ("2c") or NX_SIMT_CHAN_HIGH ("8c"). 572 573func nx_simt_set_memory(c: *NxSimtCore, lat: i64, nchan: i64) -> i64 { 574 if c.valid != 1 { return 0 - NX_SIMT_E_ARG } 575 if lat < 1 { return 0 - NX_SIMT_E_ARG } 576 if nchan < 1 { return 0 - NX_SIMT_E_ARG } 577 if nchan > NX_SIMT_MAX_CHANNELS { return 0 - NX_SIMT_E_ARG } 578 c.mem_lat = lat 579 c.nchan = nchan 580 var i: i64 = 0 581 while i < NX_SIMT_MAX_CHANNELS { c.chan_free[i] = 0; i = i + 1 } 582 return NX_SIMT_OK 583} 584 585// Enable or bypass the L1. Bypass exists so the cache's effect can be measured as an A/B 586// on one binary rather than argued for. 587func nx_simt_set_cache(c: *NxSimtCore, on: i64) -> i64 { 588 if c.valid != 1 { return 0 - NX_SIMT_E_ARG } 589 c.cache_on = on 590 var i: i64 = 0 591 let nways: i64 = NX_SIMT_CACHE_SETS * NX_SIMT_CACHE_WAYS 592 while i < nways { c.cache_tag[i] = 0 - 1; c.cache_age[i] = 0; i = i + 1 } 593 c.cache_hits = 0 594 c.cache_misses = 0 595 return NX_SIMT_OK 596} 597 598// Probe the L1 for one line address; install it on a miss. Returns 1 = hit, 0 = miss. 599// Set-associative with LRU victim selection, stamped by cycle. 600 601func nx_simt_cache_probe(c: *NxSimtCore, line: i64) -> i64 { 602 var set: i64 = line % NX_SIMT_CACHE_SETS 603 if set < 0 { set = 0 - set } 604 let base: i64 = set * NX_SIMT_CACHE_WAYS 605 606 var w: i64 = 0 607 while w < NX_SIMT_CACHE_WAYS { 608 if c.cache_tag[base + w] == line { 609 c.cache_age[base + w] = c.cycles 610 c.cache_hits = c.cache_hits + 1 611 return 1 612 } 613 w = w + 1 614 } 615 616 // Miss: prefer an invalid way, else evict the least recently used. 617 var victim: i64 = 0 618 var oldest: i64 = c.cache_age[base] 619 w = 0 620 while w < NX_SIMT_CACHE_WAYS { 621 if c.cache_tag[base + w] < 0 { victim = w; w = NX_SIMT_CACHE_WAYS } else { 622 if c.cache_age[base + w] < oldest { oldest = c.cache_age[base + w]; victim = w } 623 w = w + 1 624 } 625 } 626 c.cache_tag[base + victim] = line 627 c.cache_age[base + victim] = c.cycles 628 c.cache_misses = c.cache_misses + 1 629 return 0 630} 631 632// Collapse a warp's per-lane addresses to DISTINCT cache lines. This is what memory 633// coalescing physically is: lanes touching the same line cost one access, not N. Returns 634// the distinct-line count and fills c.linebuf. 635func nx_simt_coalesce(c: *NxSimtCore, addrs: *i64, active: i64) -> i64 { 636 var n: i64 = 0 637 var t: i64 = 0 638 while t < c.nt { 639 if (active & (1 << t)) != 0 { 640 let line: i64 = addrs[t] / NX_SIMT_CACHE_LINE_B 641 var seen: i64 = 0 642 var i: i64 = 0 643 while i < n { 644 if c.linebuf[i] == line { seen = 1 } 645 i = i + 1 646 } 647 if seen == 0 { c.linebuf[n] = line; n = n + 1 } 648 } 649 t = t + 1 650 } 651 if n < 1 { n = 1 } 652 return n 653} 654 655// Issue one warp-wide memory request through the channel array. 656// 657// The request takes the earliest-available channel and OCCUPIES it for `cost` cycles, 658// where cost is the bank-conflict serialisation factor. The requesting warp resumes 659// `mem_lat` cycles after its request actually starts -- not after it was issued. That 660// distinction is the whole bandwidth model: under contention a request waits for a free 661// channel first, so its total stall grows with load even though latency is constant. 662 663func nx_simt_mem_issue(c: *NxSimtCore, w: i64, cost: i64, nmiss: i64) -> i64 { 664 var best: i64 = 0 665 var best_free: i64 = c.chan_free[0] 666 var i: i64 = 1 667 while i < c.nchan { 668 if c.chan_free[i] < best_free { best_free = c.chan_free[i]; best = i } 669 i = i + 1 670 } 671 var start: i64 = c.cycles 672 if best_free > start { start = best_free } 673 // A channel is held for one burst per MISSING line -- hits never reach the channel. 674 c.chan_free[best] = start + nmiss * NX_SIMT_BURST_CYCLES 675 c.wstall_until[w] = start + c.mem_lat + cost 676 c.mem_ops = c.mem_ops + 1 677 c.bank_conflict_cycles = c.bank_conflict_cycles + (cost - 1) 678 c.mem_queue_cycles = c.mem_queue_cycles + (start - c.cycles) 679 return NX_SIMT_OK 680} 681 682// One warp-wide data access: coalesce to distinct lines, probe the L1, and go to memory 683// only for the lines that miss. A fully-resident access never touches a channel at all -- 684// which is exactly the effect that lets a serialised software path with high reuse beat a 685// per-lane hardware path, the mse_forward crossover a cacheless model cannot show. 686func nx_simt_mem_access(c: *NxSimtCore, w: i64, addrs: *i64, bank_hits: *i64) -> i64 { 687 let cost: i64 = nx_simt_bank_cost(c, addrs, c.tmask[w], bank_hits) 688 if c.cache_on != 1 { 689 return nx_simt_mem_issue(c, w, cost, cost) 690 } 691 let nline: i64 = nx_simt_coalesce(c, addrs, c.tmask[w]) 692 var nmiss: i64 = 0 693 var i: i64 = 0 694 while i < nline { 695 if nx_simt_cache_probe(c, c.linebuf[i]) == 0 { nmiss = nmiss + 1 } 696 i = i + 1 697 } 698 if nmiss == 0 { 699 // Every line resident: L1 hit latency, no channel occupancy, no queueing. 700 c.mem_ops = c.mem_ops + 1 701 c.wstall_until[w] = c.cycles + NX_SIMT_LAT_CACHE_HIT 702 return NX_SIMT_OK 703 } 704 return nx_simt_mem_issue(c, w, cost, nmiss) 705} 706 707// ===== Texture sampler ================================================= 708 709func nx_simt_set_texture(c: *NxSimtCore, base: i64, w: i64, h: i64) -> i64 { 710 if c.valid != 1 { return 0 - NX_SIMT_E_ARG } 711 if w < 1 { return 0 - NX_SIMT_E_ARG } 712 if h < 1 { return 0 - NX_SIMT_E_ARG } 713 c.tex_base = base 714 c.tex_w = w 715 c.tex_h = h 716 return NX_SIMT_OK 717} 718 719// Per-channel lerp over packed RGBA8888. 720// ★Written as (a*(256-f) + b*f) >> 8 rather than a + ((b-a)*f >> 8) DELIBERATELY: the second 721// form needs (b-a) which goes NEGATIVE whenever the texture darkens, and NishiLang's right 722// shift is LOGICAL -- a negative delta would shift in high bits and produce a bright band 723// exactly where the gradient falls. Both terms here are non-negative by construction. 724// Max intermediate is 255*256 = 65280, so no overflow. 725func nx_simt_lerp_rgba(a: i64, b: i64, f: i64) -> i64 { 726 var out: i64 = 0 727 var sh: i64 = 0 728 while sh < 32 { 729 let ca: i64 = (a >> sh) & 0xff 730 let cb: i64 = (b >> sh) & 0xff 731 let cc: i64 = (ca * (256 - f) + cb * f) >> NX_SIMT_TEX_FRACBITS 732 out = out | ((cc & 0xff) << sh) 733 sh = sh + 8 734 } 735 return out 736} 737 738// Fetch one texel with clamp-to-edge addressing. 739func nx_simt_texel(c: *NxSimtCore, ix: i64, iy: i64) -> i64 { 740 var x: i64 = ix 741 var y: i64 = iy 742 if x < 0 { x = 0 } 743 if y < 0 { y = 0 } 744 if x >= c.tex_w { x = c.tex_w - 1 } 745 if y >= c.tex_h { y = c.tex_h - 1 } 746 return nx_simt_load32(c, c.tex_base + (y * c.tex_w + x) * 4) & 0xffffffff 747} 748 749// Bilinear sample at 24.8 fixed-point texel coordinates. 750// Point sampling is this same path with fu = fv = 0, exactly as Vortex does it -- one filter, 751// no variable-latency mux. 752func nx_simt_tex_sample(c: *NxSimtCore, u: i64, v: i64) -> i64 { 753 let iu: i64 = u >> NX_SIMT_TEX_FRACBITS 754 let iv: i64 = v >> NX_SIMT_TEX_FRACBITS 755 let fu: i64 = u & NX_SIMT_TEX_FRACMASK 756 let fv: i64 = v & NX_SIMT_TEX_FRACMASK 757 let t00: i64 = nx_simt_texel(c, iu, iv) 758 let t10: i64 = nx_simt_texel(c, iu + 1, iv) 759 let t01: i64 = nx_simt_texel(c, iu, iv + 1) 760 let t11: i64 = nx_simt_texel(c, iu + 1, iv + 1) 761 let top: i64 = nx_simt_lerp_rgba(t00, t10, fu) 762 let bot: i64 = nx_simt_lerp_rgba(t01, t11, fu) 763 return nx_simt_lerp_rgba(top, bot, fv) 764} 765 766// ===== IPDOM divergence stack ================================================= 767// 768// The exact protocol published for Vortex (arXiv 2002.12151, restated in MICRO'21): 769// 770// SPLIT, when threads DISAGREE on the predicate: 771// 1) push the CURRENT thread mask as a FALL-THROUGH entry 772// 2) push {threads whose predicate is FALSE, PC+4} as a non-fall-through entry 773// 3) continue with the mask set to the threads whose predicate is TRUE 774// SPLIT, when all active threads AGREE or only one is active: acts as a NOP. 775// 776// JOIN pops ONE entry: 777// - non-fall-through -> jump to the stored PC with the stored mask (run the else side) 778// - fall-through -> continue to PC+4 with the stored mask (reconverged) 779// 780// So a fully divergent region executes TWO joins: the first switches to the else path, 781// the second restores the pre-split mask. Getting this wrong is the classic SIMT bug -- 782// it silently drops half the threads' work. 783 784func nx_simt_ipdom_push(c: *NxSimtCore, w: i64, mask: i64, pc: i64, ft: i64) -> i64 { 785 let sp: i64 = c.ipdom_sp[w] 786 if sp >= NX_SIMT_IPDOM_DEPTH { c.status = NX_SIMT_E_IPDOM; return 0 - NX_SIMT_E_IPDOM } 787 let idx: i64 = w * NX_SIMT_IPDOM_DEPTH + sp 788 c.ipdom_mask[idx] = mask 789 c.ipdom_pc[idx] = pc 790 c.ipdom_ft[idx] = ft 791 c.ipdom_sp[w] = sp + 1 792 return NX_SIMT_OK 793} 794 795func nx_simt_do_split(c: *NxSimtCore, w: i64, rs1: i64) -> i64 { 796 let cur: i64 = c.tmask[w] 797 var taken: i64 = 0 798 var t: i64 = 0 799 while t < c.nt { 800 if (cur & (1 << t)) != 0 { 801 if nx_simt_rf_read(c, w, t, rs1) != 0 { taken = taken | (1 << t) } 802 } 803 t = t + 1 804 } 805 let nottaken: i64 = cur & (0 - 1 ^ taken) 806 807 // No divergence: every active thread agrees, or only one is active. NOP. 808 if taken == 0 { return NX_SIMT_OK } 809 if nottaken == 0 { return NX_SIMT_OK } 810 if nx_simt_popcount(cur) < 2 { return NX_SIMT_OK } 811 812 let r1: i64 = nx_simt_ipdom_push(c, w, cur, c.pc[w] + 4, 1) 813 if r1 != NX_SIMT_OK { return r1 } 814 let r2: i64 = nx_simt_ipdom_push(c, w, nottaken, c.pc[w] + 4, 0) 815 if r2 != NX_SIMT_OK { return r2 } 816 c.tmask[w] = taken 817 return NX_SIMT_OK 818} 819 820// Returns 1 if the join redirected the PC (caller must not apply pc+4), else 0. 821// 822// DELIBERATE DEVIATION, documented: CARRV'19 says a join with no matching split raises an 823// exception. We make it a no-op instead, because it is REACHABLE IN CORRECT CODE -- a 824// split whose predicate is uniform across the warp legitimately does not push (arXiv 825// 2002.12151), so the matching join of a correctly-balanced pair finds an empty stack 826// whenever the branch happens to be uniform at run time. Faulting there would turn a 827// data-dependent uniform branch into a spurious exception. 828func nx_simt_do_join(c: *NxSimtCore, w: i64) -> i64 { 829 let sp: i64 = c.ipdom_sp[w] 830 if sp <= 0 { return 0 } 831 let nsp: i64 = sp - 1 832 c.ipdom_sp[w] = nsp 833 let idx: i64 = w * NX_SIMT_IPDOM_DEPTH + nsp 834 c.tmask[w] = c.ipdom_mask[idx] 835 if c.ipdom_ft[idx] == 0 { 836 c.pc[w] = c.ipdom_pc[idx] 837 return 1 838 } 839 return 0 840} 841 842// ===== Barriers ================================================= 843// 844// A barrier releases when the expected number of warps have arrived. Until then each 845// arriving warp parks (removed from scheduler contention). On release every parked warp 846// is woken at once -- that is the whole point of a hardware barrier. 847 848func nx_simt_do_bar(c: *NxSimtCore, w: i64, bar_id: i64, expect: i64) -> i64 { 849 var id: i64 = bar_id & (NX_SIMT_NBARRIERS - 1) 850 if id < 0 { id = 0 } 851 852 // GLOBAL scope: arrive on the shared counter and park. The CLUSTER releases, because a 853 // core has no way to wake a warp on another core. 854 if (bar_id & NX_SIMT_BAR_GLOBAL) != 0 { 855 if c.gbar_on == 1 { 856 c.gbar_count[id] = c.gbar_count[id] + 1 857 c.gbar_expect[id] = expect 858 c.wbarrier[w] = 1 859 c.wbar_id[w] = id 860 return NX_SIMT_OK 861 } 862 // ⚠A global barrier on a core that is NOT in a cluster degrades to a LOCAL one. 863 // Declared here rather than silently: a single-core run of a multi-core kernel then 864 // still completes, but it has NOT tested inter-core synchronisation. 865 } 866 c.bar_mask[id] = c.bar_mask[id] | (1 << w) 867 c.bar_count[id] = c.bar_count[id] + 1 868 if c.bar_count[id] >= expect { 869 // Release: wake every parked warp, reset the slot. 870 var i: i64 = 0 871 while i < c.nw { 872 if (c.bar_mask[id] & (1 << i)) != 0 { c.wbarrier[i] = 0 } 873 i = i + 1 874 } 875 c.bar_mask[id] = 0 876 c.bar_count[id] = 0 877 } else { 878 c.wbarrier[w] = 1 879 } 880 return NX_SIMT_OK 881} 882 883// ===== Warp scheduler ================================================= 884// 885// One warp issues per cycle. A warp is eligible when it is active, not parked at a 886// barrier, and not waiting out a memory stall. Round-robin from a rotating cursor gives 887// every warp equal access -- the property that lets extra warps hide memory latency. 888// Returns the warp id, or -1 when nothing can issue this cycle. 889 890// Stamp a destination register as available `lat` cycles from now. 891// x0 is hardwired zero and can never block, so it is never stamped. 892func nx_simt_mark_dest(c: *NxSimtCore, w: i64, rd: i64, lat: i64) -> i64 { 893 if c.scoreboard_on != 1 { return NX_SIMT_OK } 894 if rd <= 0 { return NX_SIMT_OK } 895 if rd >= NX_SIMT_NREGS { return NX_SIMT_OK } 896 c.reg_ready[w * NX_SIMT_NREGS + rd] = c.cycles + lat 897 return NX_SIMT_OK 898} 899 900// Can warp w ISSUE its next instruction, or is it waiting on a source register? 901// This peeks and decodes at the scheduler, which is exactly what a real scoreboard does: 902// the decision to issue is made from the instruction's source fields, before execute. 903func nx_simt_srcs_ready(c: *NxSimtCore, w: i64) -> i64 { 904 if c.scoreboard_on != 1 { return 1 } 905 let pc: i64 = c.pc[w] 906 if nx_simt_in_range(c, pc, 4) == 0 { return 1 } 907 let o: i64 = pc - c.mem_base 908 let b0: i64 = c.mem_buf[o] as i64 909 let b1: i64 = c.mem_buf[o + 1] as i64 910 let b2: i64 = c.mem_buf[o + 2] as i64 911 let b3: i64 = c.mem_buf[o + 3] as i64 912 let inst: i64 = b0 | (b1 << 8) | (b2 << 16) | (b3 << 24) 913 if inst == 0 { return 1 } 914 915 let op: i64 = inst & 0x7f 916 var use1: i64 = 1 917 var use2: i64 = 1 918 // Forms with no register sources at all. 919 if op == 0x37 { use1 = 0; use2 = 0 } // LUI 920 if op == 0x17 { use1 = 0; use2 = 0 } // AUIPC 921 if op == 0x6f { use1 = 0; use2 = 0 } // JAL 922 // Forms that read rs1 only. 923 if op == 0x13 { use2 = 0 } // OP-IMM 924 if op == 0x1b { use2 = 0 } // OP-IMM-32 925 if op == 0x03 { use2 = 0 } // LOAD 926 if op == 0x67 { use2 = 0 } // JALR 927 928 let base: i64 = w * NX_SIMT_NREGS 929 if use1 == 1 { 930 let rs1: i64 = nx_rv64im_rs1(inst) 931 if rs1 > 0 { if c.reg_ready[base + rs1] > c.cycles { return 0 } } 932 } 933 if use2 == 1 { 934 let rs2: i64 = nx_rv64im_rs2(inst) 935 if rs2 > 0 { if c.reg_ready[base + rs2] > c.cycles { return 0 } } 936 } 937 return 1 938} 939 940func nx_simt_pick_warp(c: *NxSimtCore) -> i64 { 941 var blocked_on_dep: i64 = 0 942 var i: i64 = 0 943 while i < c.nw { 944 var w: i64 = c.rr_next + i 945 while w >= c.nw { w = w - c.nw } 946 if c.wactive[w] == 1 { 947 if c.wbarrier[w] == 0 { 948 if c.wstall_until[w] <= c.cycles { 949 // Memory-ready, but its sources may still be in flight. 950 if nx_simt_srcs_ready(c, w) == 1 { 951 c.rr_next = w + 1 952 while c.rr_next >= c.nw { c.rr_next = c.rr_next - c.nw } 953 return w 954 } 955 blocked_on_dep = 1 956 } 957 } 958 } 959 i = i + 1 960 } 961 // Distinguish "nothing to run" from "everything is waiting on a RAW hazard" -- the 962 // second is the cost more warps are supposed to absorb, so it must be measurable. 963 if blocked_on_dep == 1 { c.dep_stall_cycles = c.dep_stall_cycles + 1 } 964 return 0 - 1 965} 966 967func nx_simt_any_active(c: *NxSimtCore) -> i64 { 968 var w: i64 = 0 969 while w < c.nw { 970 if c.wactive[w] == 1 { return 1 } 971 w = w + 1 972 } 973 return 0 974} 975 976// ===== One SIMT instruction ================================================= 977// 978// Fetch once for the warp, then execute across every active thread. The thread mask is 979// the only thing standing between a lane and a commit. 980 981func nx_simt_exec_warp(c: *NxSimtCore, w: i64, addrs: *i64, bank_hits: *i64) -> i64 { 982 let pc: i64 = c.pc[w] 983 let inst: i64 = nx_simt_load32(c, pc) & 0xffffffff 984 985 // All-zero word is the halt sentinel, matching the scalar core's convention. 986 if inst == 0 { 987 c.wactive[w] = 0 988 return NX_SIMT_OK 989 } 990 991 let opcode: i64 = inst & 0x7f 992 let rd: i64 = nx_rv64im_rd(inst) 993 let rs1: i64 = nx_rv64im_rs1(inst) 994 let rs2: i64 = nx_rv64im_rs2(inst) 995 let funct3: i64 = nx_rv64im_funct3(inst) 996 let funct7: i64 = nx_rv64im_funct7(inst) 997 let tmask: i64 = c.tmask[w] 998 999 c.instret = c.instret + 1 1000 c.lane_instret = c.lane_instret + nx_simt_popcount(tmask) 1001 1002 // ---------- SIMT extension (custom-0) ---------- 1003 if opcode == NX_SIMT_OPCODE { 1004 if funct3 == NX_SIMT_F3_TMC { 1005 // Thread mask comes from lane 0 -- it is a warp-scope control value. 1006 let lane0: i64 = nx_simt_lowest_lane(tmask) 1007 var nm: i64 = 0 1008 if lane0 >= 0 { nm = nx_simt_rf_read(c, w, lane0, rs1) } 1009 nm = nm & nx_simt_full_mask(c.nt) 1010 c.tmask[w] = nm 1011 if nm == 0 { c.wactive[w] = 0 } 1012 c.pc[w] = pc + 4 1013 return NX_SIMT_OK 1014 } 1015 if funct3 == NX_SIMT_F3_WSPAWN { 1016 let lane0: i64 = nx_simt_lowest_lane(tmask) 1017 var n: i64 = 0 1018 var target: i64 = 0 1019 if lane0 >= 0 { 1020 n = nx_simt_rf_read(c, w, lane0, rs1) 1021 target = nx_simt_rf_read(c, w, lane0, rs2) 1022 } 1023 if n > c.nw { n = c.nw } 1024 var i: i64 = 1 1025 while i < n { 1026 c.wactive[i] = 1 1027 c.pc[i] = target 1028 c.tmask[i] = nx_simt_full_mask(c.nt) 1029 c.ipdom_sp[i] = 0 1030 c.wbarrier[i] = 0 1031 c.wstall_until[i] = 0 1032 i = i + 1 1033 } 1034 c.pc[w] = pc + 4 1035 return NX_SIMT_OK 1036 } 1037 if funct3 == NX_SIMT_F3_SPLIT { 1038 let r: i64 = nx_simt_do_split(c, w, rs1) 1039 if r != NX_SIMT_OK { return r } 1040 c.pc[w] = pc + 4 1041 return NX_SIMT_OK 1042 } 1043 if funct3 == NX_SIMT_F3_JOIN { 1044 let redirected: i64 = nx_simt_do_join(c, w) 1045 if redirected == 0 { c.pc[w] = pc + 4 } 1046 return NX_SIMT_OK 1047 } 1048 if funct3 == NX_SIMT_F3_BAR { 1049 let lane0: i64 = nx_simt_lowest_lane(tmask) 1050 var bid: i64 = 0 1051 var expect: i64 = 1 1052 if lane0 >= 0 { 1053 bid = nx_simt_rf_read(c, w, lane0, rs1) 1054 expect = nx_simt_rf_read(c, w, lane0, rs2) 1055 } 1056 c.pc[w] = pc + 4 1057 nx_simt_do_bar(c, w, bid, expect) 1058 return NX_SIMT_OK 1059 } 1060 if funct3 == NX_SIMT_F3_TID { 1061 var t: i64 = 0 1062 while t < c.nt { 1063 if (tmask & (1 << t)) != 0 { nx_simt_rf_write(c, w, t, rd, t) } 1064 t = t + 1 1065 } 1066 nx_simt_mark_dest(c, w, rd, NX_SIMT_LAT_ALU) 1067 c.pc[w] = pc + 4 1068 return NX_SIMT_OK 1069 } 1070 if funct3 == NX_SIMT_F3_WID { 1071 var t: i64 = 0 1072 while t < c.nt { 1073 if (tmask & (1 << t)) != 0 { nx_simt_rf_write(c, w, t, rd, w) } 1074 t = t + 1 1075 } 1076 nx_simt_mark_dest(c, w, rd, NX_SIMT_LAT_ALU) 1077 c.pc[w] = pc + 4 1078 return NX_SIMT_OK 1079 } 1080 if funct3 == NX_SIMT_F3_NTID { 1081 var t: i64 = 0 1082 while t < c.nt { 1083 if (tmask & (1 << t)) != 0 { nx_simt_rf_write(c, w, t, rd, c.nt) } 1084 t = t + 1 1085 } 1086 nx_simt_mark_dest(c, w, rd, NX_SIMT_LAT_ALU) 1087 c.pc[w] = pc + 4 1088 return NX_SIMT_OK 1089 } 1090 c.status = NX_SIMT_E_ILLEGAL 1091 c.wactive[w] = 0 1092 return 0 - NX_SIMT_E_ILLEGAL 1093 } 1094 1095 // ---------- Texture sample (custom-3, R4-type) ---------- 1096 if opcode == NX_SIMT_OPCODE_TEX { 1097 let rs3: i64 = (inst >> 27) & 0x1f 1098 var t: i64 = 0 1099 // Address generation for every lane first -- the paper's stage 1 converts (u,v) to 1100 // texel addresses "for all the threads in parallel", THEN the memory unit 1101 // de-duplicates across threads before issuing. 1102 while t < c.nt { 1103 addrs[t] = 0 1104 if (tmask & (1 << t)) != 0 { 1105 let u: i64 = nx_simt_rf_read(c, w, t, rs1) 1106 let v: i64 = nx_simt_rf_read(c, w, t, rs2) 1107 var iu: i64 = u >> NX_SIMT_TEX_FRACBITS 1108 var iv: i64 = v >> NX_SIMT_TEX_FRACBITS 1109 if iu < 0 { iu = 0 } 1110 if iv < 0 { iv = 0 } 1111 if iu >= c.tex_w { iu = c.tex_w - 1 } 1112 if iv >= c.tex_h { iv = c.tex_h - 1 } 1113 addrs[t] = c.tex_base + (iv * c.tex_w + iu) * 4 1114 } 1115 t = t + 1 1116 } 1117 // Sample. rs3 carries lod; v1 samples a single level, so lod selects nothing yet -- 1118 // ★DECLARED, NOT SILENT: trilinear in Vortex is a PSEUDO-INSTRUCTION issuing two tex 1119 // ops and lerping on frac(lod), so a single-level sampler is the correct primitive to 1120 // build first, but a caller passing lod!=0 today gets level 0 and must know that. 1121 t = 0 1122 while t < c.nt { 1123 if (tmask & (1 << t)) != 0 { 1124 let u: i64 = nx_simt_rf_read(c, w, t, rs1) 1125 let v: i64 = nx_simt_rf_read(c, w, t, rs2) 1126 nx_simt_rf_write(c, w, t, rd, nx_simt_tex_sample(c, u, v)) 1127 } 1128 t = t + 1 1129 } 1130 // One de-duplicated batch through the memory system, then the sampler's filter cost. 1131 let ntex: i64 = nx_simt_coalesce(c, addrs, tmask) 1132 c.tex_ops = c.tex_ops + 1 1133 c.tex_texels = c.tex_texels + ntex 1134 nx_simt_mem_access(c, w, addrs, bank_hits) 1135 c.wstall_until[w] = c.wstall_until[w] + NX_SIMT_LAT_TEX_FILTER 1136 nx_simt_mark_dest(c, w, rd, c.wstall_until[w] - c.cycles) 1137 c.pc[w] = pc + 4 1138 return NX_SIMT_OK 1139 } 1140 1141 // ---------- Warp collectives (custom-1) ---------- 1142 if opcode == NX_SIMT_OPCODE_W { 1143 // Snapshot every lane's source BEFORE writing any destination. A shuffle is a 1144 // permutation: writing in place lets lane 0's result overwrite a value lane 3 1145 // still needs to read. 1146 var t: i64 = 0 1147 while t < c.nt { 1148 c.lanebuf[t] = 0 1149 if (tmask & (1 << t)) != 0 { c.lanebuf[t] = nx_simt_rf_read(c, w, t, rs1) } 1150 t = t + 1 1151 } 1152 1153 if funct3 == NX_SIMT_W_BALLOT { 1154 var ballot: i64 = 0 1155 t = 0 1156 while t < c.nt { 1157 if (tmask & (1 << t)) != 0 { if c.lanebuf[t] != 0 { ballot = ballot | (1 << t) } } 1158 t = t + 1 1159 } 1160 t = 0 1161 while t < c.nt { 1162 if (tmask & (1 << t)) != 0 { nx_simt_rf_write(c, w, t, rd, ballot) } 1163 t = t + 1 1164 } 1165 nx_simt_mark_dest(c, w, rd, NX_SIMT_LAT_ALU) 1166 c.pc[w] = pc + 4 1167 return NX_SIMT_OK 1168 } 1169 if funct3 == NX_SIMT_W_ALL { 1170 var allv: i64 = 1 1171 t = 0 1172 while t < c.nt { 1173 if (tmask & (1 << t)) != 0 { if c.lanebuf[t] == 0 { allv = 0 } } 1174 t = t + 1 1175 } 1176 t = 0 1177 while t < c.nt { 1178 if (tmask & (1 << t)) != 0 { nx_simt_rf_write(c, w, t, rd, allv) } 1179 t = t + 1 1180 } 1181 nx_simt_mark_dest(c, w, rd, NX_SIMT_LAT_ALU) 1182 c.pc[w] = pc + 4 1183 return NX_SIMT_OK 1184 } 1185 if funct3 == NX_SIMT_W_ANY { 1186 var anyv: i64 = 0 1187 t = 0 1188 while t < c.nt { 1189 if (tmask & (1 << t)) != 0 { if c.lanebuf[t] != 0 { anyv = 1 } } 1190 t = t + 1 1191 } 1192 t = 0 1193 while t < c.nt { 1194 if (tmask & (1 << t)) != 0 { nx_simt_rf_write(c, w, t, rd, anyv) } 1195 t = t + 1 1196 } 1197 nx_simt_mark_dest(c, w, rd, NX_SIMT_LAT_ALU) 1198 c.pc[w] = pc + 4 1199 return NX_SIMT_OK 1200 } 1201 if funct3 == NX_SIMT_W_ACTIVEMASK { 1202 t = 0 1203 while t < c.nt { 1204 if (tmask & (1 << t)) != 0 { nx_simt_rf_write(c, w, t, rd, tmask) } 1205 t = t + 1 1206 } 1207 nx_simt_mark_dest(c, w, rd, NX_SIMT_LAT_ALU) 1208 c.pc[w] = pc + 4 1209 return NX_SIMT_OK 1210 } 1211 1212 // Shuffle family: each lane computes its own source lane index. 1213 t = 0 1214 while t < c.nt { 1215 if (tmask & (1 << t)) != 0 { 1216 let delta: i64 = nx_simt_rf_read(c, w, t, rs2) 1217 var src: i64 = t 1218 if funct3 == NX_SIMT_W_SHFL { src = delta } 1219 if funct3 == NX_SIMT_W_SHFL_UP { src = t - delta } 1220 if funct3 == NX_SIMT_W_SHFL_DOWN { src = t + delta } 1221 if funct3 == NX_SIMT_W_SHFL_XOR { src = t ^ delta } 1222 // Out-of-range source clamps to the lane itself, matching the CUDA 1223 // convention that an out-of-bounds shuffle leaves the value unchanged. 1224 if src < 0 { src = t } 1225 if src >= c.nt { src = t } 1226 nx_simt_rf_write(c, w, t, rd, c.lanebuf[src]) 1227 } 1228 t = t + 1 1229 } 1230 nx_simt_mark_dest(c, w, rd, NX_SIMT_LAT_ALU) 1231 c.pc[w] = pc + 4 1232 return NX_SIMT_OK 1233 } 1234 1235 // ---------- Base RV64IM, executed per active lane ---------- 1236 let op_kind: i64 = nx_rv64im_decode_kind(inst) 1237 1238 // Branches and jumps are WARP-SCOPE: one PC per warp. Divergent conditionals are 1239 // expressed with split/join, not with a branch -- that is the SIMT contract. 1240 if op_kind == NX_RV64IM_OP_BRANCH { 1241 let lane0: i64 = nx_simt_lowest_lane(tmask) 1242 var take: i64 = 0 1243 if lane0 >= 0 { 1244 let a: i64 = nx_simt_rf_read(c, w, lane0, rs1) 1245 let b: i64 = nx_simt_rf_read(c, w, lane0, rs2) 1246 if funct3 == 0 { if a == b { take = 1 } } 1247 if funct3 == 1 { if a != b { take = 1 } } 1248 if funct3 == 4 { if a < b { take = 1 } } 1249 if funct3 == 5 { if a >= b { take = 1 } } 1250 if funct3 == 6 { take = nx_rv64im_ltu(a, b) } 1251 if funct3 == 7 { if nx_rv64im_ltu(a, b) == 0 { take = 1 } } 1252 } 1253 if take == 1 { c.pc[w] = pc + nx_rv64im_imm_b(inst) } else { c.pc[w] = pc + 4 } 1254 return NX_SIMT_OK 1255 } 1256 if op_kind == NX_RV64IM_OP_JAL { 1257 var t: i64 = 0 1258 while t < c.nt { 1259 if (tmask & (1 << t)) != 0 { nx_simt_rf_write(c, w, t, rd, pc + 4) } 1260 t = t + 1 1261 } 1262 nx_simt_mark_dest(c, w, rd, NX_SIMT_LAT_ALU) 1263 c.pc[w] = pc + nx_rv64im_imm_j(inst) 1264 return NX_SIMT_OK 1265 } 1266 if op_kind == NX_RV64IM_OP_JALR { 1267 let lane0: i64 = nx_simt_lowest_lane(tmask) 1268 var tgt: i64 = pc + 4 1269 if lane0 >= 0 { tgt = (nx_simt_rf_read(c, w, lane0, rs1) + nx_rv64im_imm_i(inst)) & (0 - 2) } 1270 var t: i64 = 0 1271 while t < c.nt { 1272 if (tmask & (1 << t)) != 0 { nx_simt_rf_write(c, w, t, rd, pc + 4) } 1273 t = t + 1 1274 } 1275 nx_simt_mark_dest(c, w, rd, NX_SIMT_LAT_ALU) 1276 c.pc[w] = tgt 1277 return NX_SIMT_OK 1278 } 1279 1280 // Memory: per-lane addresses, then one bank-conflict cost for the whole warp. 1281 if op_kind == NX_RV64IM_OP_LOAD { 1282 let imm: i64 = nx_rv64im_imm_i(inst) 1283 var t: i64 = 0 1284 while t < c.nt { 1285 addrs[t] = 0 1286 if (tmask & (1 << t)) != 0 { addrs[t] = nx_simt_rf_read(c, w, t, rs1) + imm } 1287 t = t + 1 1288 } 1289 t = 0 1290 while t < c.nt { 1291 if (tmask & (1 << t)) != 0 { 1292 var v: i64 = 0 1293 if funct3 == 2 { v = nx_rv64im_sext32(nx_simt_load32(c, addrs[t])) } 1294 if funct3 == 3 { v = nx_simt_load64(c, addrs[t]) } 1295 if funct3 == 6 { v = nx_simt_load32(c, addrs[t]) & 0xffffffff } 1296 nx_simt_rf_write(c, w, t, rd, v) 1297 } 1298 t = t + 1 1299 } 1300 nx_simt_mem_access(c, w, addrs, bank_hits) 1301 // The loaded value lands when the access completes, not at issue. 1302 nx_simt_mark_dest(c, w, rd, c.wstall_until[w] - c.cycles) 1303 c.pc[w] = pc + 4 1304 return NX_SIMT_OK 1305 } 1306 if op_kind == NX_RV64IM_OP_STORE { 1307 let imm: i64 = nx_rv64im_imm_s(inst) 1308 var t: i64 = 0 1309 while t < c.nt { 1310 addrs[t] = 0 1311 if (tmask & (1 << t)) != 0 { addrs[t] = nx_simt_rf_read(c, w, t, rs1) + imm } 1312 t = t + 1 1313 } 1314 t = 0 1315 while t < c.nt { 1316 if (tmask & (1 << t)) != 0 { 1317 let v: i64 = nx_simt_rf_read(c, w, t, rs2) 1318 if funct3 == 2 { nx_simt_store32(c, addrs[t], v) } 1319 if funct3 == 3 { nx_simt_store64(c, addrs[t], v) } 1320 } 1321 t = t + 1 1322 } 1323 nx_simt_mem_access(c, w, addrs, bank_hits) 1324 c.pc[w] = pc + 4 1325 return NX_SIMT_OK 1326 } 1327 1328 // Upper-immediate forms. 1329 if op_kind == NX_RV64IM_OP_LUI { 1330 let v: i64 = nx_rv64im_imm_u(inst) 1331 var t: i64 = 0 1332 while t < c.nt { 1333 if (tmask & (1 << t)) != 0 { nx_simt_rf_write(c, w, t, rd, v) } 1334 t = t + 1 1335 } 1336 nx_simt_mark_dest(c, w, rd, NX_SIMT_LAT_ALU) 1337 c.pc[w] = pc + 4 1338 return NX_SIMT_OK 1339 } 1340 if op_kind == NX_RV64IM_OP_AUIPC { 1341 let v: i64 = pc + nx_rv64im_imm_u(inst) 1342 var t: i64 = 0 1343 while t < c.nt { 1344 if (tmask & (1 << t)) != 0 { nx_simt_rf_write(c, w, t, rd, v) } 1345 t = t + 1 1346 } 1347 nx_simt_mark_dest(c, w, rd, NX_SIMT_LAT_ALU) 1348 c.pc[w] = pc + 4 1349 return NX_SIMT_OK 1350 } 1351 1352 // Register-register and register-immediate ALU, per lane. 1353 var handled: i64 = 0 1354 var alu_op: i64 = NX_RV64IM_ALU_INVALID 1355 var use_imm: i64 = 0 1356 var imm_val: i64 = 0 1357 1358 if op_kind == NX_RV64IM_OP_OP { 1359 alu_op = nx_rv64im_sim_alu_select(NX_RV64IM_OP_OP, funct3, funct7); handled = 1 1360 } 1361 if op_kind == NX_RV64IM_OP_M_MUL { 1362 alu_op = nx_rv64im_sim_alu_select(NX_RV64IM_OP_OP, funct3, funct7); handled = 1 1363 } 1364 if op_kind == NX_RV64IM_OP_M_DIV { 1365 alu_op = nx_rv64im_sim_alu_select(NX_RV64IM_OP_OP, funct3, funct7); handled = 1 1366 } 1367 if op_kind == NX_RV64IM_OP_OP_32 { 1368 alu_op = nx_rv64im_sim_alu_select(NX_RV64IM_OP_OP_32, funct3, funct7); handled = 1 1369 } 1370 if op_kind == NX_RV64IM_OP_M_MUL_32 { 1371 alu_op = nx_rv64im_sim_alu_select(NX_RV64IM_OP_OP_32, funct3, funct7); handled = 1 1372 } 1373 if op_kind == NX_RV64IM_OP_M_DIV_32 { 1374 alu_op = nx_rv64im_sim_alu_select(NX_RV64IM_OP_OP_32, funct3, funct7); handled = 1 1375 } 1376 if op_kind == NX_RV64IM_OP_OP_IMM { 1377 alu_op = nx_rv64im_sim_alu_select(NX_RV64IM_OP_OP_IMM, funct3, funct7) 1378 use_imm = 1; imm_val = nx_rv64im_imm_i(inst); handled = 1 1379 } 1380 if op_kind == NX_RV64IM_OP_OP_IMM_32 { 1381 alu_op = nx_rv64im_sim_alu_select(NX_RV64IM_OP_OP_IMM_32, funct3, funct7) 1382 use_imm = 1; imm_val = nx_rv64im_imm_i(inst); handled = 1 1383 } 1384 1385 if handled == 1 { 1386 var t: i64 = 0 1387 while t < c.nt { 1388 if (tmask & (1 << t)) != 0 { 1389 let a: i64 = nx_simt_rf_read(c, w, t, rs1) 1390 var b: i64 = imm_val 1391 if use_imm == 0 { b = nx_simt_rf_read(c, w, t, rs2) } 1392 nx_simt_rf_write(c, w, t, rd, nx_rv64im_alu_compute(alu_op, a, b)) 1393 } 1394 t = t + 1 1395 } 1396 // Multiply and divide are multi-cycle. ★They must block their DEPENDENTS, not the 1397 // whole warp: an independent instruction following a multiply can still issue. 1398 // Stalling the warp outright (what this did before the scoreboard) both overstates 1399 // the cost and hides it from the warp-count comparison, because a wholesale warp 1400 // stall is not a bubble another warp can fill in the same way. 1401 var lat: i64 = NX_SIMT_LAT_ALU 1402 if op_kind == NX_RV64IM_OP_M_MUL { lat = NX_SIMT_LAT_MULDIV } 1403 if op_kind == NX_RV64IM_OP_M_DIV { lat = NX_SIMT_LAT_MULDIV } 1404 if op_kind == NX_RV64IM_OP_M_MUL_32 { lat = NX_SIMT_LAT_MULDIV } 1405 if op_kind == NX_RV64IM_OP_M_DIV_32 { lat = NX_SIMT_LAT_MULDIV } 1406 nx_simt_mark_dest(c, w, rd, lat) 1407 c.pc[w] = pc + 4 1408 return NX_SIMT_OK 1409 } 1410 1411 if op_kind == NX_RV64IM_OP_FENCE { 1412 c.pc[w] = pc + 4 1413 return NX_SIMT_OK 1414 } 1415 1416 // Anything else halts this warp loudly rather than silently skipping work. 1417 c.status = NX_SIMT_E_ILLEGAL 1418 c.wactive[w] = 0 1419 return 0 - NX_SIMT_E_ILLEGAL 1420} 1421 1422// ===== Cycle-level run loop ================================================= 1423// 1424// Every iteration is one CYCLE. A cycle in which no warp is eligible is a real stall 1425// cycle and is counted -- that is exactly how extra warps show up as higher IPC, and 1426// dropping it would make every configuration look identical. 1427 1428func nx_simt_run(c: *NxSimtCore, max_cycles: i64, addrs: *i64, bank_hits: *i64) -> i64 { 1429 if c.valid != 1 { return 0 - NX_SIMT_E_ARG } 1430 var n: i64 = 0 1431 while n < max_cycles { 1432 if nx_simt_any_active(c) == 0 { c.halted = 1; return NX_SIMT_OK } 1433 let w: i64 = nx_simt_pick_warp(c) 1434 if w < 0 { 1435 c.stall_cycles = c.stall_cycles + 1 1436 } else { 1437 nx_simt_exec_warp(c, w, addrs, bank_hits) 1438 if c.status == NX_SIMT_E_BOUNDS { return 0 - NX_SIMT_E_BOUNDS } 1439 if c.status == NX_SIMT_E_IPDOM { return 0 - NX_SIMT_E_IPDOM } 1440 } 1441 c.cycles = c.cycles + 1 1442 n = n + 1 1443 } 1444 return 0 - NX_SIMT_E_ARG 1445} 1446 1447// ===== Multi-core cluster ================================================= 1448// 1449// "Cores can be grouped into a cluster that can optionally be attached to a shared L2 cache." 1450// (MICRO'21 sec 4.1.4). We build the cluster; the optional L2 we deliberately do NOT build, 1451// because their own Figure 21 shows L2 barely moves the needle (sgemm 16.5 -> 18.5 IPC at 2 1452// channels, and NO change at 8) while channels move it 2.4x. Build what the data says matters. 1453// 1454// ★THE ONE THING THAT MUST BE SHARED IS BANDWIDTH. Each core keeps its own warps, register 1455// files and L1 -- but every core's `chan_free` is repointed at ONE cluster-owned array, so 1456// memory channels are contended. Without that, adding cores would scale PERFECTLY LINEARLY 1457// and the model would be vacuous: Figure 18 shows sgemm reaching only 53% of linear at 32 1458// cores precisely because memory saturates. 1459// 1460// ORACLE, MICRO'21 Figure 18 (aggregate IPC vs core count, sgemm): 1461// 1 core 1.9 · 2 cores 3.8 · 4 cores 7.8 · 8 cores 15.3 · 16 cores 27.1 · 32 cores 32.5 1462// So sgemm is near-LINEAR to 8 cores (8.05x = 100% of linear), then bends: 89% at 16, 53% at 1463// 32. A model that shows perfect linearity everywhere has not modelled contention at all. 1464 1465const NX_SIMT_MAX_CORES: i64 = 16 1466const NX_SIMT_CORE_STRIDE: i64 = 1024 // bytes reserved per core; struct is well under this 1467 1468struct NxSimtCluster { 1469 ncores: i64 1470 core_mem: *i64 // ncores contiguous NxSimtCore slots, NX_SIMT_CORE_STRIDE apart 1471 chan_free: *i64 // SHARED memory channels -- the contention point 1472 nchan: i64 1473 gbar_count: *i64 // SHARED global-barrier arrival counters 1474 gbar_expect: *i64 // SHARED global-barrier thresholds 1475 gbar_rel: i64 // global barrier releases performed (instrumentation) 1476 cycles: i64 1477 instret: i64 // warp-instructions retired across ALL cores 1478 stall_cycles: i64 // per-core issue slots that went unused 1479 halted: i64 1480 valid: i64 1481} 1482 1483// Cores live in one contiguous allocation indexed by a fixed stride, NOT in an array of 1484// pointers -- an *i64 array holding pointers is exactly the shape that gets read back as 1485// numbers, and this ecosystem has already paid for that once. 1486func nx_simt_cluster_core(cl: *NxSimtCluster, i: i64) -> *NxSimtCore { 1487 return (((cl.core_mem) as i64) + i * NX_SIMT_CORE_STRIDE) as *NxSimtCore 1488} 1489 1490func nx_simt_cluster_init(cl: *NxSimtCluster, ncores: i64, nw: i64, nt: i64, 1491 mem_base: i64, mem_buf: *u8, mem_size: i64, 1492 entry_pc: i64, lat: i64, nchan: i64) -> i64 { 1493 if ncores < 1 { return 0 - NX_SIMT_E_ARG } 1494 if ncores > NX_SIMT_MAX_CORES { return 0 - NX_SIMT_E_ARG } 1495 cl.ncores = ncores 1496 cl.core_mem = (sys_mmap(ncores * NX_SIMT_CORE_STRIDE)) as *i64 1497 cl.chan_free = (sys_mmap(8 * NX_SIMT_MAX_CHANNELS)) as *i64 1498 cl.nchan = nchan 1499 var ch: i64 = 0 1500 while ch < NX_SIMT_MAX_CHANNELS { cl.chan_free[ch] = 0; ch = ch + 1 } 1501 cl.gbar_count = (sys_mmap(8 * NX_SIMT_NBARRIERS)) as *i64 1502 cl.gbar_expect = (sys_mmap(8 * NX_SIMT_NBARRIERS)) as *i64 1503 var gb: i64 = 0 1504 while gb < NX_SIMT_NBARRIERS { cl.gbar_count[gb] = 0; cl.gbar_expect[gb] = 0; gb = gb + 1 } 1505 cl.gbar_rel = 0 1506 1507 var i: i64 = 0 1508 while i < ncores { 1509 let c: *NxSimtCore = nx_simt_cluster_core(cl, i) 1510 nx_simt_alloc(c) 1511 nx_simt_init(c, nw, nt, mem_base, mem_buf, mem_size, entry_pc) 1512 nx_simt_set_memory(c, lat, nchan) 1513 // ★Repoint this core's channel array at the CLUSTER's. Bandwidth is now shared and 1514 // finite across every core, which is what makes scaling sub-linear. 1515 c.chan_free = cl.chan_free 1516 // Wire this core into the cluster's global barrier table. 1517 c.gbar_count = cl.gbar_count 1518 c.gbar_expect = cl.gbar_expect 1519 c.gbar_on = 1 1520 i = i + 1 1521 } 1522 cl.cycles = 0 1523 cl.instret = 0 1524 cl.stall_cycles = 0 1525 cl.halted = 0 1526 cl.valid = 1 1527 return NX_SIMT_OK 1528} 1529 1530// One cluster cycle = one issue opportunity PER CORE. Cores are independent front-ends, so N 1531// cores can retire N warp-instructions in a cycle -- bounded only by the shared channels. 1532func nx_simt_cluster_run(cl: *NxSimtCluster, max_cycles: i64, 1533 addrs: *i64, bank_hits: *i64) -> i64 { 1534 if cl.valid != 1 { return 0 - NX_SIMT_E_ARG } 1535 var n: i64 = 0 1536 while n < max_cycles { 1537 var any_live: i64 = 0 1538 var i: i64 = 0 1539 while i < cl.ncores { 1540 let c: *NxSimtCore = nx_simt_cluster_core(cl, i) 1541 // Drive every core from the CLUSTER clock so channel reservations and stall 1542 // deadlines are compared on one timebase. 1543 c.cycles = cl.cycles 1544 if nx_simt_any_active(c) == 1 { 1545 any_live = 1 1546 let w: i64 = nx_simt_pick_warp(c) 1547 if w < 0 { 1548 cl.stall_cycles = cl.stall_cycles + 1 1549 } else { 1550 nx_simt_exec_warp(c, w, addrs, bank_hits) 1551 cl.instret = cl.instret + 1 1552 } 1553 } 1554 i = i + 1 1555 } 1556 // ----- global barrier release, cluster-scope ----- 1557 // Checked once per cluster cycle, AFTER every core has had its issue opportunity, so 1558 // arrivals within the same cycle all count. A barrier whose threshold is met wakes 1559 // every warp parked on it ACROSS ALL CORES at once -- that simultaneity is the whole 1560 // point of a hardware barrier, and it is unreachable from inside a single core. 1561 var b: i64 = 0 1562 while b < NX_SIMT_NBARRIERS { 1563 if cl.gbar_expect[b] > 0 { 1564 if cl.gbar_count[b] >= cl.gbar_expect[b] { 1565 var ci: i64 = 0 1566 while ci < cl.ncores { 1567 let cc: *NxSimtCore = nx_simt_cluster_core(cl, ci) 1568 var ww: i64 = 0 1569 while ww < cc.nw { 1570 if cc.wbar_id[ww] == b { 1571 cc.wbarrier[ww] = 0 1572 cc.wbar_id[ww] = 0 - 1 1573 } 1574 ww = ww + 1 1575 } 1576 ci = ci + 1 1577 } 1578 cl.gbar_count[b] = 0 1579 cl.gbar_expect[b] = 0 1580 cl.gbar_rel = cl.gbar_rel + 1 1581 } 1582 } 1583 b = b + 1 1584 } 1585 1586 if any_live == 0 { cl.halted = 1; return NX_SIMT_OK } 1587 cl.cycles = cl.cycles + 1 1588 n = n + 1 1589 } 1590 return 0 - NX_SIMT_E_ARG 1591} 1592 1593// Aggregate IPC across the cluster, in per-mille. This is Figure 18's metric. 1594func nx_simt_cluster_ipc_permille(cl: *NxSimtCluster) -> i64 { 1595 if cl.cycles <= 0 { return 0 } 1596 return (cl.instret * 1000) / cl.cycles 1597} 1598 1599// ===== Instrumentation readout ================================================= 1600// IPC in per-mille, matching Vortex's metric: WARP-instructions retired per cycle. 1601// 1000 permille = 1.0 IPC = one warp-instruction issued every cycle. 1602 1603func nx_simt_ipc_permille(c: *NxSimtCore) -> i64 { 1604 if c.cycles <= 0 { return 0 } 1605 return (c.instret * 1000) / c.cycles 1606} 1607 1608// Lane-IPC: the work actually performed per cycle. A wider warp does more per 1609// instruction, which warp-IPC alone cannot see -- reporting only one of these is how a 1610// configuration comparison misleads. 1611 1612func nx_simt_lane_ipc_permille(c: *NxSimtCore) -> i64 { 1613 if c.cycles <= 0 { return 0 } 1614 return (c.lane_instret * 1000) / c.cycles 1615}