nx_rv64_jit.nx source

1// nx_rv64_jit.nx -- a SOVEREIGN RV64 -> x86-64 JIT (dynamic binary translation): the SOTA emulator technique, above 2// the decode-cache interpreter. It TRANSLATES a basic block of predecoded RV64 ops into NATIVE x86-64 machine code 3// (per-instruction, not a shape template): the 32 RV64 registers live in a caller-provided i64 array (base in rdi per 4// SysV); each RV64 instruction emits x86 that loads operands from reg[], computes in rax, stores back; a backward 5// branch emits a NATIVE conditional jump so a loop runs entirely in host code. The block is mmap'd RWX and CALLED 6// (the RWX + cast-fn-ptr primitive is proven by nx_nxe_loader). Equivalence with the golden behavioral sim is proven 7// by nx_rv64_jit_gate; the speedup is measured there. Subset: LUI/ADDI/ADD/SUB/ANDI/ORI/SLLI + BEQ/BNE/BLT/BGE 8// (backward branches). license_tier: ORIGINAL 9import "nx_syscalls.nx" 10import "nx_rv64_fast.nx" // FK_* op kinds (shared decode table) 11 12func j_b(out: *u8, o: i64, v: i64) -> i64 { out[o]=(v&0xff) as u8; return o+1 } 13func j_w32(out: *u8, o: i64, v: i64) -> i64 { out[o]=(v&0xff) as u8; out[o+1]=((v>>8)&0xff) as u8; out[o+2]=((v>>16)&0xff) as u8; out[o+3]=((v>>24)&0xff) as u8; return o+4 } 14// x86-64 emitters (base rdi = &reg[0]; scratch rax). disp32 = reg_index*8. 15func e_load(out: *u8, o: i64, ri: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x48); p=j_b(out,p,0x8B); p=j_b(out,p,0x87); p=j_w32(out,p,ri*8); return p } // mov rax,[rdi+ri*8] 16func e_store(out: *u8, o: i64, ri: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x48); p=j_b(out,p,0x89); p=j_b(out,p,0x87); p=j_w32(out,p,ri*8); return p } // mov [rdi+ri*8],rax 17func e_movimm(out: *u8, o: i64, imm: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x48); p=j_b(out,p,0xC7); p=j_b(out,p,0xC0); p=j_w32(out,p,imm); return p } // mov rax,imm32(sx) 18func e_addimm(out: *u8, o: i64, imm: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x48); p=j_b(out,p,0x05); p=j_w32(out,p,imm); return p } // add rax,imm32 19func e_andimm(out: *u8, o: i64, imm: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x48); p=j_b(out,p,0x25); p=j_w32(out,p,imm); return p } // and rax,imm32 20func e_orimm(out: *u8, o: i64, imm: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x48); p=j_b(out,p,0x0D); p=j_w32(out,p,imm); return p } // or rax,imm32 21func e_opreg(out: *u8, o: i64, opc: i64, ri: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x48); p=j_b(out,p,opc); p=j_b(out,p,0x87); p=j_w32(out,p,ri*8); return p } // <op> rax,[rdi+ri*8] 22func e_imul_mem(out: *u8, o: i64, ri: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x48); p=j_b(out,p,0x0F); p=j_b(out,p,0xAF); p=j_b(out,p,0x87); p=j_w32(out,p,ri*8); return p } // imul rax,[rdi+ri*8] 23func e_mulhi_u(out: *u8, o: i64, ri: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x48); p=j_b(out,p,0xF7); p=j_b(out,p,0xA7); p=j_w32(out,p,ri*8); return p } // mul qword[rdi+ri*8] -> rdx:rax (unsigned) 24func e_mulhi_s(out: *u8, o: i64, ri: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x48); p=j_b(out,p,0xF7); p=j_b(out,p,0xAF); p=j_w32(out,p,ri*8); return p } // imul qword[rdi+ri*8] -> rdx:rax (signed) 25func e_mov_rax_rdx(out: *u8, o: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x48); p=j_b(out,p,0x89); p=j_b(out,p,0xD0); return p } // mov rax,rdx (grab the high half) 26func e_shl(out: *u8, o: i64, sh: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x48); p=j_b(out,p,0xC1); p=j_b(out,p,0xE0); p=j_b(out,p,sh); return p } // shl rax,imm8 27func e_jcc(out: *u8, o: i64, cc: i64, rel: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x0F); p=j_b(out,p,cc); p=j_w32(out,p,rel); return p } // jcc rel32 28// *W (32-bit word) emitters: operate on EAX (32-bit, no REX.W), then MOVSXD rax,eax to sign-extend the low-32 result 29// to 64 (RV64 *W semantics). load32=mov eax,[rdi+ri*8]; the arithmetic mirrors the 64-bit forms without the 0x48 REX. 30func e_load32(out: *u8, o: i64, ri: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x8B); p=j_b(out,p,0x87); p=j_w32(out,p,ri*8); return p } // mov eax,[rdi+ri*8] 31func e_op32_mem(out: *u8, o: i64, opc: i64, ri: i64) -> i64 { var p: i64=o; p=j_b(out,p,opc); p=j_b(out,p,0x87); p=j_w32(out,p,ri*8); return p } // <op> eax,[rdi+ri*8] (03 add / 2B sub) 32func e_imul32_mem(out: *u8, o: i64, ri: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x0F); p=j_b(out,p,0xAF); p=j_b(out,p,0x87); p=j_w32(out,p,ri*8); return p } // imul eax,[rdi+ri*8] 33func e_addimm32(out: *u8, o: i64, imm: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x05); p=j_w32(out,p,imm); return p } // add eax,imm32 34func e_shl32(out: *u8, o: i64, sh: i64) -> i64 { var p: i64=o; p=j_b(out,p,0xC1); p=j_b(out,p,0xE0); p=j_b(out,p,sh); return p } // shl eax,imm8 35func e_shr32(out: *u8, o: i64, sh: i64) -> i64 { var p: i64=o; p=j_b(out,p,0xC1); p=j_b(out,p,0xE8); p=j_b(out,p,sh); return p } // shr eax,imm8 36func e_sar32(out: *u8, o: i64, sh: i64) -> i64 { var p: i64=o; p=j_b(out,p,0xC1); p=j_b(out,p,0xF8); p=j_b(out,p,sh); return p } // sar eax,imm8 37func e_movsxd(out: *u8, o: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x48); p=j_b(out,p,0x63); p=j_b(out,p,0xC0); return p } // movsxd rax,eax (sext32->64) 38func e_ret(out: *u8, o: i64) -> i64 { return j_b(out,o,0xC3) } 39func jit_mmap_rwx(size: i64) -> *u8 { let r: i64 = __syscall(SYS_MMAP, 0, size, 7, 0x22, -1, 0); return r as *u8 } 40func j_w64(out: *u8, o: i64, v: i64) -> i64 { var i: i64=0; while i<8 { out[o+i]=((v>>(i*8))&0xff) as u8; i=i+1 } return o+8 } 41// memory-op emitters. rsi = biased guest-mem base (host_ptr - 0x80000000), baked into the block; rax = computed guest 42// address AND load destination (the CPU reads the [rsi+rax] operand before writing rax, so aliasing is fine); rdx = 43// store value. SIB byte 0x06 = [rsi + rax*1]. 44func e_movimm64_rsi(out: *u8, o: i64, imm: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x48); p=j_b(out,p,0xBE); p=j_w64(out,p,imm); return p } // mov rsi, imm64 45func e_ld_rdx(out: *u8, o: i64, ri: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x48); p=j_b(out,p,0x8B); p=j_b(out,p,0x97); p=j_w32(out,p,ri*8); return p } // mov rdx,[rdi+ri*8] 46func e_lw_mem(out: *u8, o: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x48); p=j_b(out,p,0x63); p=j_b(out,p,0x04); p=j_b(out,p,0x06); return p } // movsxd rax,dword[rsi+rax] 47func e_ld_mem(out: *u8, o: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x48); p=j_b(out,p,0x8B); p=j_b(out,p,0x04); p=j_b(out,p,0x06); return p } // mov rax,[rsi+rax] 48func e_sw_mem(out: *u8, o: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x89); p=j_b(out,p,0x14); p=j_b(out,p,0x06); return p } // mov [rsi+rax],edx (store 32) 49func e_sd_mem(out: *u8, o: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x48); p=j_b(out,p,0x89); p=j_b(out,p,0x14); p=j_b(out,p,0x06); return p } // mov [rsi+rax],rdx (store 64) 50func e_shr(out: *u8, o: i64, sh: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x48); p=j_b(out,p,0xC1); p=j_b(out,p,0xE8); p=j_b(out,p,sh); return p } // shr rax,imm8 (SRLI) 51func e_sar(out: *u8, o: i64, sh: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x48); p=j_b(out,p,0xC1); p=j_b(out,p,0xF8); p=j_b(out,p,sh); return p } // sar rax,imm8 (SRAI) 52// guest-memory base for JIT'd loads/stores (biased: host_ptr - 0x80000000) + the guest memory SIZE (bound). set by 53// the caller before jit_compile so JIT'd loads/stores can bounds-check EXACTLY like the interpreter (fk_memok). 54static jit_membase: i64 = 0 55static jit_membytes: i64 = 0 56func jit_set_membase(v: i64, membytes: i64) -> i64 { jit_membase = v; jit_membytes = membytes; return 0 } 57// emit a guest-memory BOUNDS CHECK before a load/store: offset = rax - 0x80000000 (in rdx); if (unsigned)offset > 58// membytes-width -> jump over the access (a negative offset wraps to huge-unsigned, so one unsigned compare covers 59// both under- and over-run). returns o with a `ja rel8` PLACEHOLDER whose rel8 byte is at o-1 (caller patches it to 60// skip the access) -- so an OOB access is a NO-OP, matching the interpreter's fk_memok (never escapes to host memory). 61func e_bcheck(out: *u8, o: i64, width: i64) -> i64 { 62 var p: i64=o 63 p=j_b(out,p,0x48); p=j_b(out,p,0x8D); p=j_b(out,p,0x90); p=j_w32(out,p,0x80000000) // lea rdx,[rax-2^31] (=guest offset) 64 p=j_b(out,p,0x48); p=j_b(out,p,0x81); p=j_b(out,p,0xFA); p=j_w32(out,p,jit_membytes-width) // cmp rdx, membytes-width 65 p=j_b(out,p,0x77); p=j_b(out,p,0x00) // ja rel8 (placeholder) 66 return p 67} 68 69// TRANSLATE predecoded ops [0, nidx) into native x86-64 in `out`. records each op's x86 offset so backward branches 70// resolve. returns the x86 byte length, or -1 on an unsupported op (fail-loud, no silent miscompile). 71func jit_compile(opk: *i64, rd: *i64, rs1: *i64, rs2: *i64, imm: *i64, nidx: i64, out: *u8, x86off: *i64) -> i64 { 72 var o: i64 = 0; var i: i64 = 0 73 // fixup table for FORWARD branches/jumps (target x86 offset not yet emitted): patched after the emit pass. 74 let fx_loc: *i64 = sys_mmap(nidx*8+64) as *i64; let fx_tgt: *i64 = sys_mmap(nidx*8+64) as *i64; let fx_end: *i64 = sys_mmap(nidx*8+64) as *i64; var nfx: i64 = 0 75 // prologue: save + bake the guest-mem base into rsi (rdx = store scratch). rsi/rdx restored at the epilogue so the 76 // block preserves the caller's ABI regardless of which regs NishiLang treats as callee-saved. 77 o=j_b(out,o,0x56); o=j_b(out,o,0x52); o=e_movimm64_rsi(out,o,jit_membase) // push rsi; push rdx; mov rsi,membase 78 while i < nidx { 79 x86off[i] = o 80 let k: i64=opk[i]; let d: i64=rd[i]; let a: i64=rs1[i]; let b: i64=rs2[i]; let im: i64=imm[i] 81 var ok: i64 = 0 82 if k==FK_ADDI { o=e_load(out,o,a); o=e_addimm(out,o,im); if d!=0 { o=e_store(out,o,d) } ok=1 } 83 if k==FK_LUI { o=e_movimm(out,o,im); if d!=0 { o=e_store(out,o,d) } ok=1 } 84 if k==FK_ADD { o=e_load(out,o,a); o=e_opreg(out,o,0x03,b); if d!=0 { o=e_store(out,o,d) } ok=1 } 85 if k==FK_SUB { o=e_load(out,o,a); o=e_opreg(out,o,0x2B,b); if d!=0 { o=e_store(out,o,d) } ok=1 } 86 if k==FK_AND { o=e_load(out,o,a); o=e_opreg(out,o,0x23,b); if d!=0 { o=e_store(out,o,d) } ok=1 } 87 if k==FK_OR { o=e_load(out,o,a); o=e_opreg(out,o,0x0B,b); if d!=0 { o=e_store(out,o,d) } ok=1 } 88 if k==FK_ANDI { o=e_load(out,o,a); o=e_andimm(out,o,im); if d!=0 { o=e_store(out,o,d) } ok=1 } 89 if k==FK_ORI { o=e_load(out,o,a); o=e_orimm(out,o,im); if d!=0 { o=e_store(out,o,d) } ok=1 } 90 if k==FK_SLLI { o=e_load(out,o,a); o=e_shl(out,o,im); if d!=0 { o=e_store(out,o,d) } ok=1 } 91 if k==FK_MUL { o=e_load(out,o,a); o=e_imul_mem(out,o,b); if d!=0 { o=e_store(out,o,d) } ok=1 } // M ext: MUL (imul, low 64) 92 // *W (32-bit word) ops: compute in EAX, MOVSXD to sign-extend low-32 -> 64. Real `int` loops (i++ = addiw etc.) 93 // are full of these -- without them the JIT never fires on real code. (register-shift + div *W tier to interp.) 94 if k==FK_ADDIW { o=e_load32(out,o,a); o=e_addimm32(out,o,im); o=e_movsxd(out,o); if d!=0 { o=e_store(out,o,d) } ok=1 } 95 if k==FK_ADDW { o=e_load32(out,o,a); o=e_op32_mem(out,o,0x03,b); o=e_movsxd(out,o); if d!=0 { o=e_store(out,o,d) } ok=1 } 96 if k==FK_SUBW { o=e_load32(out,o,a); o=e_op32_mem(out,o,0x2B,b); o=e_movsxd(out,o); if d!=0 { o=e_store(out,o,d) } ok=1 } 97 if k==FK_MULW { o=e_load32(out,o,a); o=e_imul32_mem(out,o,b); o=e_movsxd(out,o); if d!=0 { o=e_store(out,o,d) } ok=1 } 98 if k==FK_SLLIW { o=e_load32(out,o,a); o=e_shl32(out,o,im); o=e_movsxd(out,o); if d!=0 { o=e_store(out,o,d) } ok=1 } 99 if k==FK_SRLIW { o=e_load32(out,o,a); o=e_shr32(out,o,im); o=e_movsxd(out,o); if d!=0 { o=e_store(out,o,d) } ok=1 } 100 if k==FK_SRAIW { o=e_load32(out,o,a); o=e_sar32(out,o,im); o=e_movsxd(out,o); if d!=0 { o=e_store(out,o,d) } ok=1 } 101 // MULH/MULHU: one-operand imul/mul -> rdx:rax; high half in rdx -> mov to rax -> store. rdx is saved/restored by 102 // the prologue push/epilogue pop, so clobbering it here is safe. MULHSU has no x86 form -> tiers to the interpreter. 103 if k==FK_MULH { o=e_load(out,o,a); o=e_mulhi_s(out,o,b); o=e_mov_rax_rdx(out,o); if d!=0 { o=e_store(out,o,d) } ok=1 } 104 if k==FK_MULHU { o=e_load(out,o,a); o=e_mulhi_u(out,o,b); o=e_mov_rax_rdx(out,o); if d!=0 { o=e_store(out,o,d) } ok=1 } 105 if k==FK_SRLI { o=e_load(out,o,a); o=e_shr(out,o,im); if d!=0 { o=e_store(out,o,d) } ok=1 } 106 if k==FK_SRAI { o=e_load(out,o,a); o=e_sar(out,o,im); if d!=0 { o=e_store(out,o,d) } ok=1 } 107 // guest memory: addr = reg[rs1]+imm (in rax); [rsi+rax] is the host location. loads use rax as dest; stores 108 // put reg[rs2] in rdx first. NOTE: a store to the finisher MMIO is NOT special-cased -- JIT'd blocks are pure 109 // compute + RAM; the finisher/halt stays the interpreter's job (blocks with it must not be JIT'd). 110 // each memory op is BOUNDS-CHECKED (e_bcheck): an out-of-range guest address skips the access, exactly like the 111 // interpreter's fk_memok -- the JIT never escapes the guest-memory sandbox. loads: addr in rax, then check, then 112 // load; stores: addr in rax, then check, then load-value-into-rdx + store (so an OOB store skips both). 113 if k==FK_LW { o=e_load(out,o,a); o=e_addimm(out,o,im); o=e_bcheck(out,o,4); let jp: i64=o-1; o=e_lw_mem(out,o); if d!=0 { o=e_store(out,o,d) } out[jp]=((o-(jp+1))&0xff) as u8; ok=1 } 114 if k==FK_LD { o=e_load(out,o,a); o=e_addimm(out,o,im); o=e_bcheck(out,o,8); let jp: i64=o-1; o=e_ld_mem(out,o); if d!=0 { o=e_store(out,o,d) } out[jp]=((o-(jp+1))&0xff) as u8; ok=1 } 115 if k==FK_SW { o=e_load(out,o,a); o=e_addimm(out,o,im); o=e_bcheck(out,o,4); let jp: i64=o-1; o=e_ld_rdx(out,o,b); o=e_sw_mem(out,o); out[jp]=((o-(jp+1))&0xff) as u8; ok=1 } 116 if k==FK_SD { o=e_load(out,o,a); o=e_addimm(out,o,im); o=e_bcheck(out,o,8); let jp: i64=o-1; o=e_ld_rdx(out,o,b); o=e_sd_mem(out,o); out[jp]=((o-(jp+1))&0xff) as u8; ok=1 } 117 // conditional branches, FORWARD or backward. cmp rs1,rs2 (via rax) then jcc; a forward target (not yet 118 // emitted) gets a fixup patched after the pass; a backward target resolves immediately. (tgt out of range -> -1.) 119 if k==FK_BEQ { o=e_load(out,o,a); o=e_opreg(out,o,0x3B,b); let tgt: i64=i+(im/4); if tgt<0 { return 0-1 } if tgt>=nidx { return 0-1 } let js: i64=o; if tgt<=i { o=e_jcc(out,o,0x84,x86off[tgt]-(js+6)) } else { o=e_jcc(out,o,0x84,0); fx_loc[nfx]=js+2; fx_tgt[nfx]=tgt; fx_end[nfx]=js+6; nfx=nfx+1 } ok=1 } 120 if k==FK_BNE { o=e_load(out,o,a); o=e_opreg(out,o,0x3B,b); let tgt: i64=i+(im/4); if tgt<0 { return 0-1 } if tgt>=nidx { return 0-1 } let js: i64=o; if tgt<=i { o=e_jcc(out,o,0x85,x86off[tgt]-(js+6)) } else { o=e_jcc(out,o,0x85,0); fx_loc[nfx]=js+2; fx_tgt[nfx]=tgt; fx_end[nfx]=js+6; nfx=nfx+1 } ok=1 } 121 if k==FK_BLT { o=e_load(out,o,a); o=e_opreg(out,o,0x3B,b); let tgt: i64=i+(im/4); if tgt<0 { return 0-1 } if tgt>=nidx { return 0-1 } let js: i64=o; if tgt<=i { o=e_jcc(out,o,0x8C,x86off[tgt]-(js+6)) } else { o=e_jcc(out,o,0x8C,0); fx_loc[nfx]=js+2; fx_tgt[nfx]=tgt; fx_end[nfx]=js+6; nfx=nfx+1 } ok=1 } 122 if k==FK_BGE { o=e_load(out,o,a); o=e_opreg(out,o,0x3B,b); let tgt: i64=i+(im/4); if tgt<0 { return 0-1 } if tgt>=nidx { return 0-1 } let js: i64=o; if tgt<=i { o=e_jcc(out,o,0x8D,x86off[tgt]-(js+6)) } else { o=e_jcc(out,o,0x8D,0); fx_loc[nfx]=js+2; fx_tgt[nfx]=tgt; fx_end[nfx]=js+6; nfx=nfx+1 } ok=1 } 123 // unconditional jump: JAL x0 = j (rd==0). jal-with-link (rd!=0) unsupported here (no guest PC in the JIT). 124 if k==FK_JAL { if d==0 { let tgt: i64=i+(im/4); if tgt<0 { return 0-1 } if tgt>=nidx { return 0-1 } let js: i64=o; o=j_b(out,o,0xE9); o=j_w32(out,o,0); if tgt<=i { j_w32(out,js+1,x86off[tgt]-(js+5)) } else { fx_loc[nfx]=js+1; fx_tgt[nfx]=tgt; fx_end[nfx]=js+5; nfx=nfx+1 } ok=1 } } 125 if ok==0 { return 0-1 } // unsupported op -> fail-loud (do not silently miscompile) 126 i=i+1 127 } 128 o=j_b(out,o,0x5A); o=j_b(out,o,0x5E); o=e_ret(out,o) // epilogue: pop rdx; pop rsi; ret 129 // patch forward fixups now that every op's x86 offset is known. 130 var f: i64=0 131 while f<nfx { j_w32(out, fx_loc[f], x86off[fx_tgt[f]] - fx_end[f]); f=f+1 } 132 return o 133} 134// mmap the translated block RWX + CALL it with rdi = &reg[0]; the RV64 registers are updated in place. 135func jit_run(x86: *u8, len: i64, reg: *i64) -> i64 { 136 let ex: *u8 = jit_mmap_rwx(len + 64) 137 var i: i64 = 0; while i < len { ex[i] = x86[i]; i = i + 1 } 138 let fp: func(i64) -> i64 = (ex as i64) as func(i64) -> i64 139 fp(reg as i64) 140 return 0 141} 142// ============================================================================================================ 143// REGISTER-ALLOCATED JIT (the SOTA quality step over the memory-per-op JIT): keep the block's hot RV64 registers in 144// x86-64 registers for the whole block -- fill from reg[] at entry, spill at exit -- so a loop body does ZERO memory 145// traffic (the memory-per-op JIT loads/stores every op). Pool = 3 caller-saved x86 regs {rcx,rdx,rsi} (uniform low 146// encoding, no push/pop, no REX.B); rax=scratch, rdi=reg[] base. TIERED + FAIL-SAFE: any op/pattern this fast path 147// does not handle -> returns -2 so the caller falls back to the proven memory JIT (never a miscompile). Handles the 148// common idioms (li via ADDI-from-x0, mv via ADD/OR-from-x0, in-place ADDI, reg-reg ALU, backward branches). 149// ============================================================================================================ 150func ra_pool(s: i64) -> i64 { if s==0 { return 1 } if s==1 { return 2 } return 6 } // rcx, rdx, rsi 151func era_fill(out: *u8, o: i64, preg: i64, disp: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x48); p=j_b(out,p,0x8B); p=j_b(out,p,0x87|(preg<<3)); p=j_w32(out,p,disp); return p } // mov preg,[rdi+disp] 152func era_spill(out: *u8, o: i64, preg: i64, disp: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x48); p=j_b(out,p,0x89); p=j_b(out,p,0x87|(preg<<3)); p=j_w32(out,p,disp); return p } // mov [rdi+disp],preg 153func era_mov_rr(out: *u8, o: i64, dst: i64, src: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x48); p=j_b(out,p,0x8B); p=j_b(out,p,0xC0|(dst<<3)|src); return p } // mov dst,src 154func era_alu_rr(out: *u8, o: i64, opc: i64, dst: i64, src: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x48); p=j_b(out,p,opc); p=j_b(out,p,0xC0|(dst<<3)|src); return p } // <op> dst,src (03/2B/23/0B/3B) 155func era_alu_ri(out: *u8, o: i64, slash: i64, dst: i64, imm: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x48); p=j_b(out,p,0x81); p=j_b(out,p,0xC0|(slash<<3)|dst); p=j_w32(out,p,imm); return p } // add(/0)/or(/1)/and(/4) dst,imm32 156func era_shl_ri(out: *u8, o: i64, dst: i64, sh: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x48); p=j_b(out,p,0xC1); p=j_b(out,p,0xE0|dst); p=j_b(out,p,sh); return p } // shl dst,imm8 157func era_mov_ri(out: *u8, o: i64, dst: i64, imm: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x48); p=j_b(out,p,0xC7); p=j_b(out,p,0xC0|dst); p=j_w32(out,p,imm); return p } // mov dst,imm32(sx) 158func era_jcc(out: *u8, o: i64, cc: i64, rel: i64) -> i64 { var p: i64=o; p=j_b(out,p,0x0F); p=j_b(out,p,cc); p=j_w32(out,p,rel); return p } 159// allocate a pool reg to nonzero RV64 reg r; 0 ok / -2 pool-full. x0 is never allocated. 160func jit_ra_alloc(rvmap: *i64, nbox: *i64, r: i64) -> i64 { if r==0 { return 0 } if rvmap[r]>=0 { return 0 } if nbox[0]>=3 { return 0-2 } rvmap[r]=ra_pool(nbox[0]); nbox[0]=nbox[0]+1; return 0 } 161 162// register-allocated compile. returns x86 length, or -2 (ineligible -> caller uses the memory JIT). rvmap = caller 163// 32-array (filled with the RV64->x86 mapping). x86off = caller nidx-array (per-op x86 offsets for branch resolution). 164func jit_compile_ra(opk: *i64, rd: *i64, rs1: *i64, rs2: *i64, imm: *i64, nidx: i64, out: *u8, x86off: *i64, rvmap: *i64) -> i64 { 165 var i: i64=0; while i<32 { rvmap[i]=-1; i=i+1 } 166 let nbox: *i64 = sys_mmap(16) as *i64; nbox[0]=0 167 // ---- pass 1: eligibility + allocation (FLAT dispatch -- one kind matches; matched==0 -> bail -2). ---- 168 i=0 169 while i<nidx { 170 let k: i64=opk[i]; let d: i64=rd[i]; let a: i64=rs1[i]; let b: i64=rs2[i]; let im: i64=imm[i] 171 var matched: i64=0 172 if k==FK_LUI { if d==0 { return 0-2 } if jit_ra_alloc(rvmap,nbox,d)<0 { return 0-2 } matched=1 } 173 if k==FK_ADDI { if d==0 { return 0-2 } if jit_ra_alloc(rvmap,nbox,d)<0 { return 0-2 } if a!=0 { if jit_ra_alloc(rvmap,nbox,a)<0 { return 0-2 } } matched=1 } 174 if k==FK_ANDI { if d==0 { return 0-2 } if jit_ra_alloc(rvmap,nbox,d)<0 { return 0-2 } if a!=0 { if jit_ra_alloc(rvmap,nbox,a)<0 { return 0-2 } } matched=1 } 175 if k==FK_ORI { if d==0 { return 0-2 } if jit_ra_alloc(rvmap,nbox,d)<0 { return 0-2 } if a!=0 { if jit_ra_alloc(rvmap,nbox,a)<0 { return 0-2 } } matched=1 } 176 if k==FK_SLLI { if d==0 { return 0-2 } if jit_ra_alloc(rvmap,nbox,d)<0 { return 0-2 } if a!=0 { if jit_ra_alloc(rvmap,nbox,a)<0 { return 0-2 } } matched=1 } 177 if k==FK_ADD { if d==0 { return 0-2 } if a==0 { if b==0 { return 0-2 } } if jit_ra_alloc(rvmap,nbox,d)<0 { return 0-2 } if a!=0 { if jit_ra_alloc(rvmap,nbox,a)<0 { return 0-2 } } if b!=0 { if jit_ra_alloc(rvmap,nbox,b)<0 { return 0-2 } } matched=1 } 178 if k==FK_SUB { if d==0 { return 0-2 } if a==0 { return 0-2 } if jit_ra_alloc(rvmap,nbox,d)<0 { return 0-2 } if jit_ra_alloc(rvmap,nbox,a)<0 { return 0-2 } if b!=0 { if jit_ra_alloc(rvmap,nbox,b)<0 { return 0-2 } } matched=1 } 179 if k==FK_AND { if d==0 { return 0-2 } if a==0 { return 0-2 } if b==0 { return 0-2 } if jit_ra_alloc(rvmap,nbox,d)<0 { return 0-2 } if jit_ra_alloc(rvmap,nbox,a)<0 { return 0-2 } if jit_ra_alloc(rvmap,nbox,b)<0 { return 0-2 } matched=1 } 180 if k==FK_OR { if d==0 { return 0-2 } if a==0 { if b==0 { return 0-2 } } if jit_ra_alloc(rvmap,nbox,d)<0 { return 0-2 } if a!=0 { if jit_ra_alloc(rvmap,nbox,a)<0 { return 0-2 } } if b!=0 { if jit_ra_alloc(rvmap,nbox,b)<0 { return 0-2 } } matched=1 } 181 if k==FK_BEQ { let tgt: i64=i+(im/4); if tgt>i { return 0-2 } if a==0 { return 0-2 } if b==0 { return 0-2 } if jit_ra_alloc(rvmap,nbox,a)<0 { return 0-2 } if jit_ra_alloc(rvmap,nbox,b)<0 { return 0-2 } matched=1 } 182 if k==FK_BNE { let tgt: i64=i+(im/4); if tgt>i { return 0-2 } if a==0 { return 0-2 } if b==0 { return 0-2 } if jit_ra_alloc(rvmap,nbox,a)<0 { return 0-2 } if jit_ra_alloc(rvmap,nbox,b)<0 { return 0-2 } matched=1 } 183 if k==FK_BLT { let tgt: i64=i+(im/4); if tgt>i { return 0-2 } if a==0 { return 0-2 } if b==0 { return 0-2 } if jit_ra_alloc(rvmap,nbox,a)<0 { return 0-2 } if jit_ra_alloc(rvmap,nbox,b)<0 { return 0-2 } matched=1 } 184 if k==FK_BGE { let tgt: i64=i+(im/4); if tgt>i { return 0-2 } if a==0 { return 0-2 } if b==0 { return 0-2 } if jit_ra_alloc(rvmap,nbox,a)<0 { return 0-2 } if jit_ra_alloc(rvmap,nbox,b)<0 { return 0-2 } matched=1 } 185 if matched==0 { return 0-2 } 186 i=i+1 187 } 188 // ---- pass 2: prologue = SAVE the pool regs we clobber (push; they may be callee-saved in the caller's ABI) + 189 // FILL them from reg[]; body; epilogue = SPILL back + RESTORE (pop, reverse order) + ret. ---- 190 var o: i64=0 191 var s: i64=0; while s<nbox[0] { o=j_b(out,o, 0x50|ra_pool(s)); s=s+1 } // push rcx/rdx/rsi (the slots in use) 192 var r: i64=0; while r<32 { if rvmap[r]>=0 { o=era_fill(out,o,rvmap[r],r*8) } r=r+1 } 193 i=0 194 while i<nidx { 195 x86off[i]=o 196 let k: i64=opk[i]; let d: i64=rd[i]; let a: i64=rs1[i]; let b: i64=rs2[i]; let im: i64=imm[i] 197 if k==FK_LUI { o=era_mov_ri(out,o,rvmap[d],im) } 198 if k==FK_ADDI { if a==0 { o=era_mov_ri(out,o,rvmap[d],im) } else { if d==a { o=era_alu_ri(out,o,0,rvmap[d],im) } else { o=era_mov_rr(out,o,0,rvmap[a]); o=era_alu_ri(out,o,0,0,im); o=era_mov_rr(out,o,rvmap[d],0) } } } 199 if k==FK_ANDI { if a==0 { o=era_mov_ri(out,o,rvmap[d],0) } else { if d==a { o=era_alu_ri(out,o,4,rvmap[d],im) } else { o=era_mov_rr(out,o,0,rvmap[a]); o=era_alu_ri(out,o,4,0,im); o=era_mov_rr(out,o,rvmap[d],0) } } } 200 if k==FK_ORI { if a==0 { o=era_mov_ri(out,o,rvmap[d],im) } else { if d==a { o=era_alu_ri(out,o,1,rvmap[d],im) } else { o=era_mov_rr(out,o,0,rvmap[a]); o=era_alu_ri(out,o,1,0,im); o=era_mov_rr(out,o,rvmap[d],0) } } } 201 if k==FK_SLLI { if a==0 { o=era_mov_ri(out,o,rvmap[d],0) } else { if d==a { o=era_shl_ri(out,o,rvmap[d],im) } else { o=era_mov_rr(out,o,0,rvmap[a]); o=era_shl_ri(out,o,0,im); o=era_mov_rr(out,o,rvmap[d],0) } } } 202 if k==FK_ADD { if a==0 { o=era_mov_rr(out,o,rvmap[d],rvmap[b]) } else { if b==0 { o=era_mov_rr(out,o,rvmap[d],rvmap[a]) } else { o=era_mov_rr(out,o,0,rvmap[a]); o=era_alu_rr(out,o,0x03,0,rvmap[b]); o=era_mov_rr(out,o,rvmap[d],0) } } } 203 if k==FK_SUB { if b==0 { o=era_mov_rr(out,o,rvmap[d],rvmap[a]) } else { o=era_mov_rr(out,o,0,rvmap[a]); o=era_alu_rr(out,o,0x2B,0,rvmap[b]); o=era_mov_rr(out,o,rvmap[d],0) } } 204 if k==FK_AND { o=era_mov_rr(out,o,0,rvmap[a]); o=era_alu_rr(out,o,0x23,0,rvmap[b]); o=era_mov_rr(out,o,rvmap[d],0) } 205 if k==FK_OR { if a==0 { o=era_mov_rr(out,o,rvmap[d],rvmap[b]) } else { if b==0 { o=era_mov_rr(out,o,rvmap[d],rvmap[a]) } else { o=era_mov_rr(out,o,0,rvmap[a]); o=era_alu_rr(out,o,0x0B,0,rvmap[b]); o=era_mov_rr(out,o,rvmap[d],0) } } } 206 if k==FK_BEQ { o=era_alu_rr(out,o,0x3B,rvmap[a],rvmap[b]); let tgt: i64=i+(im/4); let rel: i64=x86off[tgt]-(o+6); o=era_jcc(out,o,0x84,rel) } 207 if k==FK_BNE { o=era_alu_rr(out,o,0x3B,rvmap[a],rvmap[b]); let tgt: i64=i+(im/4); let rel: i64=x86off[tgt]-(o+6); o=era_jcc(out,o,0x85,rel) } 208 if k==FK_BLT { o=era_alu_rr(out,o,0x3B,rvmap[a],rvmap[b]); let tgt: i64=i+(im/4); let rel: i64=x86off[tgt]-(o+6); o=era_jcc(out,o,0x8C,rel) } 209 if k==FK_BGE { o=era_alu_rr(out,o,0x3B,rvmap[a],rvmap[b]); let tgt: i64=i+(im/4); let rel: i64=x86off[tgt]-(o+6); o=era_jcc(out,o,0x8D,rel) } 210 i=i+1 211 } 212 r=0; while r<32 { if rvmap[r]>=0 { o=era_spill(out,o,rvmap[r],r*8) } r=r+1 } 213 s=nbox[0]-1; while s>=0 { o=j_b(out,o, 0x58|ra_pool(s)); s=s-1 } // pop rsi/rdx/rcx (reverse of push) -> restore 214 o=e_ret(out,o) 215 return o 216} 217func jit_main_ignore() -> i64 { return 0 }