`nx_x86_64_ctx.nx` source

↩ module page · 4094 lines · 220671 B
1// nx_x86_64_ctx.nx -- IR-driven codegen for x86_64 (session 6).
2//
3// Consumes a parsed/opt'd Function and emits AT&T x86_64 asm using
4// the emit primitives in nx_x86_64.nx (sessions 1-5).  Stack-machine
5// layout per nxc2/x86_64.c: every SSA Value that needs storage gets
6// a fixed [rbp - 8*slot] address.  Constants are rematerialised at
7// each use via movabsq; globals via leaq label(%rip).
8//
9// Per cardinal feedback-no-nxc2-c-extension-only-nishilang-forward:
10// the C file stays the read-only comparator; new functionality lives
11// here.
12//
13// nx_safety_envelope:
14//   intended_use:       "IR-driven x86_64 SysV codegen.  Walks a
15//                        Function's blocks/instrs and emits asm via
16//                        the sessions-1-5 emit primitives."
17//   sil_target:         SIL3     (codegen correctness)
18//   asil_target:        QM
19//   dal_target:         DAL B
20//   iec_62304_class:    NONE
21//   evidence:           [no_floating_point_in_layout_logic,
22//                        stack_machine_no_register_allocation,
23//                        per_opcode_dispatch_explicit,
24//                        16_byte_frame_alignment_enforced,
25//                        materialise_handles_const_global_spill]
26//   hazard_register:    [bug-tape-slot-offset-vs-disp-sign,
27//                        bug-tape-syscall-rax-overwritten-by-clobber,
28//                        bug-tape-store-result-skipped-for-unused-value]
29//   residual_risk:      "Subset coverage: arith/bitwise/cmp/branch/
30//                        call/load/store/GEP/syscall/alloca/copy.
31//                        Float / SIMD / atomic / thread ops deferred
32//                        to sessions 8+ (queued per migration index)."
33//   verdict:            NOT_YET_EVALUATED
34
35import "nx_syscalls.nx"
36import "nx_types.nx"
37// Line map: this module reads it for .loc emission. Imported HERE rather than relied on via
38// nx_parse's import list, so this file is self-sufficient regardless of splice order.
39import "nx_linemap.nx"
40import "nx_outbuf.nx"
41import "nx_x86_64.nx"
42import "nx_x86_regalloc.nx"
43
44// ===== per-function compilation context ==========================
45//
46// slot_off[v] = sp-relative byte offset for value v's stack home,
47//               or -1 if v is rematerialised (const / global).
48// alloca_off[v] = byte offset of an OP_ALLOCA result's STORAGE,
49//                 or -1 if v is not an alloca.
50// n_slots = count of values with slot homes.
51// alloca_bytes = total bytes used by allocas.
52// frame_size = total frame including 16-byte alignment.
53
54struct X86Ctx {
  f:             *Function,
  o:             *OutBuf,
  slot_off:      *i64,
  alloca_off:    *i64,
  n_slots:       i64,
  alloca_bytes:  i64,
  frame_size:    i64,
  locs:          *ValueLoc,   // G1: per-value register/spill home (x86_regalloc_function)
  used_cs_mask:  i64,         // G1: callee-saved home bitmask (bit k => home idx k)
  n_saved:       i64,         // G1: popcount(used_cs_mask)
  alloca_home:   *i64,        // G2: alloca STORAGE home reg idx (0..4) or -1 (registerized var)
  elide:         *i64,        // G4: 1 = single-use next-instr temp, slot store skipped
  uses:          *i64,        // G8: operand-appearance count per value (SIB single-use test)
  sib_dead:      *i64,        // G8: 1 = GEP/SHL result folded into a SIB load/store (emit nothing)
  fwd_home:      *i64,        // G10: forwarded load result -> source alloca's home idx, or -1
  chain_home:    *i64,        // G11: chain binop result -> IN-PLACE home idx, or -1
  chain_swap:    *i64,        // G11: 1 = chain value at op1 (commuted); src operand = op0
  next_bb:       i64,         // G12: id of the NEXT emitted block (-1 last); fall-through elision
73}
74
75// FIX-15: keep in EXACT lockstep with the field count -- the mmap in
76// x86ctx_init uses this; a stale value writes new fields past the allocation.
77const NX_X86CTX_BYTES: i64 = 144
78
79// NEGATIVE CONTROL (G14): 1 = also flag-reuse signed LT vs 0 -- the ALU's OF
80// differs from cmp-vs-0's, so (INT_MIN - 1) < 0 evaluates WRONG (adversary T7
81// goes RED). Proves the EQ/NE-only restriction is load-bearing. Ships at 0.
82const X86_NEGCTL_G14_SIGNED: i64 = 0
83
84// NEGATIVE CONTROL (G19 lea-strength): 1 = emit scale=c instead of c-1, so
85// leaq (%n,%n,c) computes (c+1)*n instead of c*n -- every workload that
86// multiplies by 2/3/5/9 checksum-splits vs gcc/clang (matrix RED). Proves the
87// c-1 scale is load-bearing arithmetic, not a free parameter. Ships at 0.
88const X86_NEGCTL_LEA_WRONG_SCALE: i64 = 0
89
90// NEGATIVE CONTROL (G21 commuted rax-consume): 1 = commute WITHOUT the
91// G1_RAX_SLOT match (rax may hold anything at that point) -> a wrong operand
92// rides into commutative binops -> matrix checksum-splits (RED). Proves the
93// G1-slot gate is load-bearing. Ships at 0.
94const X86_NEGCTL_G21_COMMUTE_ANY: i64 = 0
95
96// NEGATIVE CONTROLS (G22 bias-via-cmov division):
97// CMOV_INVERT: 1 = emit cmovns instead of cmovs -- POSITIVE dividends get the
98// bias, negative ones lose it -> divpow2 checksum-splits (RED).
99// RIDE_ALWAYS: 1 = always skip the testq -- cmovs reads whatever stale SF the
100// previous shift/lea left -> wrong side selected -> divpow2 RED. Proves the
101// G14-class producer gate on the flag ride is load-bearing. Both ship at 0.
102const X86_NEGCTL_G22_CMOV_INVERT: i64 = 0
103const X86_NEGCTL_G22_RIDE_ALWAYS: i64 = 0
104
105// NEGATIVE CONTROL (G23 src-direct cmov division): 1 = cmovs instead of
106// cmovns in the HOME-read form -- the select inverts (positives biased,
107// negatives unbiased) -> divpow2 checksum-splits (RED). The elide gate is a
108// perf heuristic (the store_result valve makes a wrong call safe), so only
109// the cmov direction carries a correctness negctl. Ships at 0.
110const X86_NEGCTL_G23_CMOV_INVERT: i64 = 0
111
112// NEGATIVE CONTROL (LN16 f64 ordering): 1 = compare two f64 bit patterns with a
113// RAW signed integer cmpq, which is what this backend did until 2026-08-25. It is
114// correct for two POSITIVE doubles and INVERTED for two negative ones, because the
115// signed-integer order of the bit patterns runs backwards through the sign-magnitude
116// encoding -- so `0.0 - 1.0 < 0.0 - 2.0` answered TRUE. nx_f64_gate MUST go RED when
117// this is 1 and GREEN when it is 0; that pair is what makes the fix non-vacuous.
118// Ships at 0.
119const X86_NEGCTL_F64_CMP_RAW: i64 = 0
120
121// ===== G1 register-residency (intra-block rax reuse) ==============
122// CS:APP SS5.6 "eliminating unneeded memory references": the slot whose
123// value rax currently holds, valid ONLY within straight-line code.
124// SET by store_result(rax); CONSUMED ONCE by an immediately-following
125// load_value_v(spilled->rax) of the same slot (skips the redundant
126// reload); CLEARED at every rax clobber (any load into rax, CALL/
127// SYSCALL/TAIL_CALL) and every basic-block boundary (control-flow join,
128// where rax is not known). -1 = unknown. Soundness rests on clearing at
129// EVERY rax-writer + every block join; see NX_G1_REGALLOC_PHASE0.
130static G1_RAX_SLOT: i64
131
132// G6 cmp+jcc fusion: a compare whose single consumer is the immediately-
133// following BR_COND skips setcc/movzbq/store entirely; the branch consumes
134// the live FLAGS. PENDING_CC = the NX_X64_CC_* code (-1 = none), PENDING_VAL
135// = the cmp result id it covers (belt: br_cond fuses only on an exact match).
136// Reset at function start + every block boundary (module statics are BSS-zero
137// and 0 is a valid CC code -- never rely on the zero init).
138static G1_PENDING_CC: i64
139static G1_PENDING_VAL: i64
140// LN18 (2026-09-01): instructions the overflow fuse has already rendered (the four LN1 intermediates
141// after a fused add/sub); the block loop skips exactly this many. Reset at every block start.
142static G24_SKIP: i64
143static G24_FUSED: i64
144
145func _g1_is_rax(reg: *u8) -> i64 {
  if reg[0] != 114 { return 0 }   // 'r'
  if reg[1] != 97  { return 0 }   // 'a'
  if reg[2] != 120 { return 0 }   // 'x'
  if reg[3] != 0   { return 0 }
  return 1
151}
152
153// ===== helpers ====================================================
154
155func x86ctx_value_at(f: *Function, id: i64) -> *Value {
  let base: i64 = f.values as i64
  return (base + id * 48) as *Value      // Value struct = 48 bytes
158}
159
160func x86ctx_block_at(f: *Function, id: i64) -> *BasicBlock {
  let base: i64 = f.blocks as i64
  return (base + id * 96) as *BasicBlock
163}
164
165func x86ctx_round_up_16(n: i64) -> i64 {
  return (n + 15) & (0 - 16)
167}
168
169// ===== G1 register allocation: frame + home-register helpers =======
170
171// FIX-19: callee-saved homes are saved at the TOP of the frame (rbp-8, rbp-16,
172// ... via x86_home_save_off), and frame_size was already grown by
173// round16(n_saved*8) in x86ctx_init to reserve that region. So value/alloca
174// slots live at the BOTTOM of the (grown) frame -- base = 0-frame_size -- which
175// pushes them down by exactly the reserved amount, guaranteeing no overlap with
176// the save area for any n_saved in 0..5. (n_saved==0 -> identical to before.)
177func x86ctx_value_base(c: *X86Ctx) -> i64 {
  return 0 - c.frame_size
179}
180
181// FIX-1: the register a producing op should compute its result INTO -- the home
182// register if the result is homed (ALLOCATE-not-COPY, zero copy), else rax.
183func x86ctx_result_reg(c: *X86Ctx, v_id: i64) -> *u8 {
  if v_id >= 0 { if v_id < c.f.n_values {
      let lbase: i64 = c.locs as i64
      let l: *ValueLoc = (lbase + v_id * 16) as *ValueLoc
      if l.kind == VL_REGISTER { return x86_home_reg_name(l.idx) }
  } }
  return "rax" as *u8
190}
191
192// Save the callee-saved home registers this function uses into their reserved
193// frame slots (movq, NOT pushq -> rsp unmoved -> 16-byte alignment preserved).
194// Compacted slot ordering (FIX-6). Emits nothing when no home is used.
195func x86ctx_emit_cs_save(c: *X86Ctx) -> i64 {
  if c.used_cs_mask == 0 { return 0 }
  var slot_index: i64 = 0
  var k: i64 = 0
  while k < X86_HOME_CAP {
      if ((c.used_cs_mask >> k) & 1) == 1 {
          x86_emit_store_qword(c.o, x86_home_reg_name(k), "rbp" as *u8, x86_home_save_off(slot_index))
          slot_index = slot_index + 1
      }
      k = k + 1
  }
  return 0
207}
208
209// Restore on every teardown path (epilogue + inline tail-call) -- SAME compacted
210// slot ordering as the save (FIX-6).
211func x86ctx_emit_cs_restore(c: *X86Ctx) -> i64 {
  if c.used_cs_mask == 0 { return 0 }
  var slot_index: i64 = 0
  var k: i64 = 0
  while k < X86_HOME_CAP {
      if ((c.used_cs_mask >> k) & 1) == 1 {
          x86_emit_load_qword(c.o, "rbp" as *u8, x86_home_save_off(slot_index), x86_home_reg_name(k))
          slot_index = slot_index + 1
      }
      k = k + 1
  }
  return 0
223}
224
225// ===== G8: scaled-addressing (SIB) fold ===========================
226// nx_cc emitted `leaq` ONCE program-wide -- every `a[i*N+k]` was shlq+addq+
227// deref (+spills). x86 SIB `(%base,%index,scale)` does base+index*scale in ONE
228// memory operand. This fold recognizes LOAD/STORE(GEP(base, SHL/MUL(idx,2^k)))
229// with SINGLE-USE links, elides the dead GEP+SHL, and emits the SIB form.
230// (nxasm gained SIB support 2026-07-15, gas-matched + additive-proven.)
231
232// Lazy scratch for the 5-i64 probe result (static pointer, mmap-once -- the safe
233// scalar-static pattern; single-threaded compile reuses it immediately per call).
234static G8_SCRATCH: i64
235func x86ctx_sib_scratch() -> *i64 {
  if G8_SCRATCH == 0 { G8_SCRATCH = sys_mmap(64) as i64 }
  return G8_SCRATCH as *i64
238}
239
240// Count operand appearances of every value + clear sib_dead.
241func x86ctx_count_uses(c: *X86Ctx) -> i64 {
  let n: i64 = c.f.n_values
  var v: i64 = 0
  while v < n { c.uses[v] = 0; c.sib_dead[v] = 0; v = v + 1 }
  var bi: i64 = 0
  while bi < c.f.n_blocks {
      let b: *BasicBlock = x86ctx_block_at(c.f, bi)
      var inst: *Instr = b.head
      while inst != (0 as *Instr) {
          let nv: i64 = x86_n_value_operands(inst)
          var k: i64 = 0
          while k < nv {
              let u: i64 = x86_operand_k(inst, k)
              if u >= 0 { if u < n { c.uses[u] = c.uses[u] + 1 } }
              k = k + 1
          }
          inst = inst.next
      }
      bi = bi + 1
  }
  return 0
262}
263
264// Probe: is addr_id = GEP(base, [SHL/MUL idx by 2^k]) with single-use links?
265// Writes out[0]=base_v out[1]=idx_v out[2]=scale(1/2/4/8) out[3]=gep_id
266// out[4]=shl_id(-1 if scale==1, i.e. no shift to elide). Returns 1 on match.
267func x86ctx_sib_probe(c: *X86Ctx, addr_id: i64, out: *i64) -> i64 {
  let n: i64 = c.f.n_values
  if addr_id < 0 { return 0 }
  if addr_id >= n { return 0 }
  if c.uses[addr_id] != 1 { return 0 }            // address value single-use only
  let av: *Value = x86ctx_value_at(c.f, addr_id)
  if av.kind != VK_INSTR { return 0 }
  let gep: *Instr = av.instr
  if gep == (0 as *Instr) { return 0 }
  // The address-forming op: OP_GEP (fixed [N]T arrays -> base is a frame addr)
  // or OP_ADD (pointer indexing `ptr[i]` = ADD(ptr_value, i*esize)). An ADD
  // feeding a load/store's ADDRESS operand IS an address by construction, so
  // folding it is always semantically valid. base=op0, offset=op1 (the parser's
  // convention: ir_emit_gep/binop(OP_ADD, base_first, scaled_offset)).
  if gep.op != OP_GEP { if gep.op != OP_ADD { return 0 } }
  let base_v: i64 = gep.op0
  let off_v: i64 = gep.op1
  out[0] = base_v
  out[1] = off_v
  out[2] = 1
  out[3] = addr_id
  out[4] = 0 - 1
  // out[5]: base kind. GEP base = an ADDRESS (fixed-array alloca -> leaq),
  // load via load_value. OP_ADD base = a pointer VALUE (load via load_value_v).
  // Mirroring the wrong one tripwires on a G2-homed pointer var (as-address of
  // a homed alloca). This is the ONLY difference between the two address ops.
  out[5] = 0
  if gep.op == OP_GEP { out[5] = 1 }
  // Upgrade to a scaled index when the offset is a single-use SHL/MUL by 2^k.
  if off_v >= 0 { if off_v < n { if c.uses[off_v] == 1 {
      let ov: *Value = x86ctx_value_at(c.f, off_v)
      if ov.kind == VK_INSTR {
          let sh: *Instr = ov.instr
          if sh != (0 as *Instr) {
              if sh.op == OP_SHL {
                  let kv: *Value = x86ctx_value_at(c.f, sh.op1)
                  if kv.kind == VK_CONST_INT {
                      if kv.const_int >= 1 { if kv.const_int <= 3 {
                          out[1] = sh.op0
                          out[2] = 1 << kv.const_int
                          out[4] = off_v
                      } }
                  }
              }
              if sh.op == OP_MUL {
                  let mv: *Value = x86ctx_value_at(c.f, sh.op1)
                  if mv.kind == VK_CONST_INT {
                      if mv.const_int == 2 { out[1] = sh.op0; out[2] = 2; out[4] = off_v }
                      if mv.const_int == 4 { out[1] = sh.op0; out[2] = 4; out[4] = off_v }
                      if mv.const_int == 8 { out[1] = sh.op0; out[2] = 8; out[4] = off_v }
                  }
              }
          }
      }
  } } }
  return 1
323}
324
325// Pre-pass: mark the GEP + SHL of every SIB-foldable 8-byte load/store dead.
326func x86ctx_sib_prepass(c: *X86Ctx) -> i64 {
  let out: *i64 = x86ctx_sib_scratch()
  var bi: i64 = 0
  while bi < c.f.n_blocks {
      let b: *BasicBlock = x86ctx_block_at(c.f, bi)
      var inst: *Instr = b.head
      while inst != (0 as *Instr) {
          var addr: i64 = 0 - 1
          if inst.op == OP_LOAD { addr = inst.op0 }
          if inst.op == OP_STORE { addr = inst.op0 }
          if addr >= 0 {
              if x86ctx_type_size(inst.ty) == 8 {
                  if x86ctx_sib_probe(c, addr, out) == 1 {
                      c.sib_dead[out[3]] = 1
                      if out[4] >= 0 { c.sib_dead[out[4]] = 1 }
                      // The SIB fold materializes base/index/value via
                      // load_value(_v) into rcx/rdx/rax -- NOT via the G1
                      // rax-forward path -- so a G4-elided operand (whose slot
                      // store was skipped, value only in rax) would be read
                      // from an empty slot -> the G4 tripwire. Force those
                      // operands to store normally (clearing elide is always
                      // safe: it is the un-optimized default).
                      let n2: i64 = c.f.n_values
                      if out[0] >= 0 { if out[0] < n2 { c.elide[out[0]] = 0 } }
                      if out[1] >= 0 { if out[1] < n2 { c.elide[out[1]] = 0 } }
                      if inst.op == OP_STORE {
                          if inst.op1 >= 0 { if inst.op1 < n2 { c.elide[inst.op1] = 0 } }
                      }
                  }
              }
          }
          inst = inst.next
      }
      bi = bi + 1
  }
  return 0
362}
363
364// Emit `movq (%base,%index,scale),%dst` (load) or `movq %src,(%base,%index,scale)`.
365func x86ctx_emit_sib_mem(c: *X86Ctx, base: *u8, index: *u8, scale: i64) -> i64 {
  out_str(c.o, "(%")
  out_str(c.o, base)
  out_str(c.o, ",%")
  out_str(c.o, index)
  out_str(c.o, ",")
  out_i64(c.o, scale)
  out_str(c.o, ")")
  return 0
374}
375
376// ===== ctx_init ===================================================
377//
378// One pass over Function.values to allocate slots, then one pass
379// over Function.blocks->instrs to find allocas and reserve their
380// storage above the slot area.
381
382func x86ctx_init(f: *Function, o: *OutBuf) -> *X86Ctx {
  let raw: *u8 = sys_mmap(NX_X86CTX_BYTES)
  let c: *X86Ctx = raw as *X86Ctx
  c.f = f
  c.o = o
387
  let n: i64 = f.n_values
  let slot_raw: *u8 = sys_mmap(n * 8 + 16)
  let alloca_raw: *u8 = sys_mmap(n * 8 + 16)
  c.slot_off = slot_raw as *i64
  c.alloca_off = alloca_raw as *i64
393
  var v: i64 = 0
  while v < n {
      c.slot_off[v] = 0 - 1
      c.alloca_off[v] = 0 - 1
      v = v + 1
  }
400
  // Assign a slot to every Value that needs storage.
  var slot: i64 = 0
  var v2: i64 = 0
  while v2 < n {
      let val: *Value = x86ctx_value_at(f, v2)
      var needs: i64 = 1
      if val.kind == VK_CONST_INT { needs = 0 }
      if val.kind == VK_GLOBAL { needs = 0 }
      if val.kind == VK_FUNC_ADDR { needs = 0 }
      if needs == 1 {
          c.slot_off[v2] = slot * 8
          slot = slot + 1
      }
      v2 = v2 + 1
  }
  c.n_slots = slot
417
  // Find allocas and assign their storage above the slot area.
  var ab: i64 = 0
  var b: i64 = 0
  while b < f.n_blocks {
      let bb: *BasicBlock = x86ctx_block_at(f, b)
      var i: *Instr = bb.head
      let II_BUDGET: i64 = 65536
      var iit: i64 = 0
      while i != (0 as *Instr) {
          if iit >= II_BUDGET { i = 0 as *Instr }
          if i != (0 as *Instr) {
              if i.op == OP_ALLOCA {
                  var sz: i64 = 8
                  if i.ty != (0 as *Type) {
                      if i.ty.kind != TY_VOID {
                          sz = i.ty.size
                          if sz <= 0 { sz = 8 }
                      }
                  }
                  sz = (sz + 7) & (0 - 8)
                  c.alloca_off[i.result] = c.n_slots * 8 + ab
                  ab = ab + sz
              }
              i = i.next
          }
          iit = iit + 1
      }
      b = b + 1
  }
  c.alloca_bytes = ab
448
  let raw_size: i64 = c.n_slots * 8 + ab
  c.frame_size = x86ctx_round_up_16(raw_size)
  if c.frame_size == 0 { c.frame_size = 16 }
  // G1 register allocation. Runs AFTER the alloca pass + frame_size so
  // alloca_off is populated (FIX-10). STEP 1a inert: x86_regalloc_function
  // lowers every value to {VL_SPILLED,-1} and returns mask 0, so n_saved=0,
  // the frame is unchanged, and the emit path (untouched) is byte-identical.
  let locs_raw: *u8 = sys_mmap(n * 16 + 16)
  c.locs = locs_raw as *ValueLoc
  let ah_raw: *u8 = sys_mmap(n * 8 + 16)
  c.alloca_home = ah_raw as *i64
  let ge_raw: *u8 = sys_mmap(n * 8 + 16)
  c.elide = ge_raw as *i64
  let us_raw: *u8 = sys_mmap(n * 8 + 16)
  c.uses = us_raw as *i64
  let sd_raw: *u8 = sys_mmap(n * 8 + 16)
  c.sib_dead = sd_raw as *i64
  let fw_raw: *u8 = sys_mmap(n * 8 + 16)
  c.fwd_home = fw_raw as *i64
  let ch_raw: *u8 = sys_mmap(n * 8 + 16)
  c.chain_home = ch_raw as *i64
  let cs_raw: *u8 = sys_mmap(n * 8 + 16)
  c.chain_swap = cs_raw as *i64
  c.next_bb = 0 - 1
  let mask_raw: *u8 = sys_mmap(16)
  let mask_p: *i64 = mask_raw as *i64
  *mask_p = 0
  x86_regalloc_function(c.f, c.alloca_off, c.locs, mask_p, c.alloca_home, c.elide, c.fwd_home, c.chain_home, c.chain_swap)
  // G8: scaled-addressing (SIB) fold. Count operand uses, then mark GEP+SHL
  // chains consumed by an 8-byte load/store as dead (emit nothing) -- the
  // load/store re-derives (base,index,scale) via the SAME probe, so elision
  // and emission cannot diverge.
  x86ctx_count_uses(c)
  x86ctx_sib_prepass(c)
  c.used_cs_mask = *mask_p
  c.n_saved = x86_popcount(c.used_cs_mask)
  c.frame_size = c.frame_size + x86ctx_round_up_16(c.n_saved * 8)
  // FIX-2: a homed value has no stack-slot identity -> the G1_RAX_SLOT peephole
  // (which keys on slot_off) can never alias it, and store_result's slot path
  // is bypassed. The VL_REGISTER branches handle every homed access.
  var hv: i64 = 0
  while hv < n {
      let hl: *ValueLoc = ((c.locs as i64) + hv * 16) as *ValueLoc
      if hl.kind == VL_REGISTER { c.slot_off[hv] = 0 - 1 }
      hv = hv + 1
  }
  return c
496}
497
498// ===== load_value (materialise into a named reg) =================
499//
500// Loads value v into register `reg`.  Handles:
501//   - VK_CONST_INT: movabsq $val, %reg
502//   - VK_GLOBAL: leaq .Lg<id>(%rip), %reg
503//   - OP_ALLOCA result: leaq <rbp-offset>(%rbp), %reg
504//   - everything else: movq <rbp-offset>(%rbp), %reg
505
506func x86ctx_load_value(c: *X86Ctx, v_id: i64, reg: *u8) -> i64 {
  let val: *Value = x86ctx_value_at(c.f, v_id)
  if _g1_is_rax(reg) == 1 {
      if G1_RAX_SLOT >= 0 {
          if val.kind != VK_CONST_INT { if val.kind != VK_GLOBAL { if val.kind != VK_FUNC_ADDR { if c.alloca_off[v_id] < 0 {
              if c.slot_off[v_id] == G1_RAX_SLOT {
                  G1_RAX_SLOT = 0 - 1
                  return 0
              }
          } } } }
      }
      G1_RAX_SLOT = 0 - 1
  }
  // G10: forwarded load result -- for a non-alloca SSA value the as-address
  // and as-value paths are identical (the slot holds the value), so reading
  // the source home is correct here too. Defensive: the audited-consumer
  // whitelist should keep forwarded values out of this path entirely.
  if c.fwd_home[v_id] >= 0 {
      let fwr2: *u8 = x86_home_reg_name(c.fwd_home[v_id])
      if x86_reg_eq(reg, fwr2) == 0 {
          x86_emit_movq_reg_reg(c.o, fwr2, reg)
      }
      return 0
  }
  // G1 (FIX-1): a homed value lives in its home register -> register move.
  let g1l2: *ValueLoc = ((c.locs as i64) + v_id * 16) as *ValueLoc
  if g1l2.kind == VL_REGISTER {
      let g1h2: *u8 = x86_home_reg_name(g1l2.idx)
      if x86_reg_eq(reg, g1h2) == 0 {
          x86_emit_movq_reg_reg(c.o, g1h2, reg)
      }
      return 0
  }
  if val.kind == VK_CONST_INT {
      x86_emit_movabsq(c.o, reg, val.const_int)
      return 0
  }
  if val.kind == VK_GLOBAL {
      out_str(c.o, "    leaq .Lg")
      out_i64(c.o, val.const_int)
      out_str(c.o, "(%rip), %")
      out_str(c.o, reg)
      out_char(c.o, 0x0A)
      return 0
  }
  if val.kind == VK_FUNC_ADDR {
      let fnp: *Function = val.const_int as *Function
      out_str(c.o, "    leaq ")
      out_str(c.o, fnp.name_start as *u8)
      out_str(c.o, "(%rip), %")
      out_str(c.o, reg)
      out_char(c.o, 0x0A)
      return 0
  }
  if c.alloca_off[v_id] >= 0 {
      // G2 tripwire: a homed alloca HAS NO ADDRESS -- reaching the
      // as-address path for one means the eligibility scan missed a use
      // class. Emit an undefined-label jump so the ASSEMBLE fails loud
      // (never a silent miscompile: the 2026-05-30 SEV1 lesson).
      if c.alloca_home[v_id] >= 0 {
          out_str(c.o, "    jmp .G2_addr_of_homed_alloca_bug\n")
          return 0
      }
      let abp_off: i64 = x86ctx_value_base(c) + c.alloca_off[v_id]
      x86_emit_lea_disp(c.o, "rbp" as *u8, abp_off, reg)
      return 0
  }
  // Spilled SSA value.
  // G4 tripwire: an elided temp's slot was never written -- a load from it
  // means the elision criterion missed a consumer; fail the assemble loud.
  if c.elide[v_id] == 1 {
      out_str(c.o, "    jmp .G4_elided_slot_load_bug\n")
      return 0
  }
  let rbp_off: i64 = x86ctx_value_base(c) + c.slot_off[v_id]
  x86_emit_load_qword(c.o, "rbp" as *u8, rbp_off, reg)
  return 0
583}
584
585// ===== store_result (spill reg into value's stack slot) ==========
586
587func x86ctx_store_result(c: *X86Ctx, v_id: i64, reg: *u8) -> i64 {
  if v_id < 0 { return 0 }
  if v_id >= c.f.n_values { return 0 }
  // G11: a chain-fused result already lives IN the home register it was
  // computed into (in place); its single consumer is the next chain step
  // (or the suppressed store-back). Emit nothing, touch no G1 state (the
  // fused op never wrote rax).
  if c.chain_home[v_id] >= 0 { return 0 }
  // G1 (FIX-1/FIX-16): a homed result already lives in its home register (the
  // producing op computed straight into it via x86ctx_result_reg). Emit AT
  // MOST one move -- zero when reg already IS the home -- REPLACING the slot
  // store, never adding to it. Placed before the _g1_is_rax handling (FIX-16).
  let lbase: i64 = c.locs as i64
  let l: *ValueLoc = (lbase + v_id * 16) as *ValueLoc
  if l.kind == VL_REGISTER {
      let h: *u8 = x86_home_reg_name(l.idx)
      if x86_reg_eq(reg, h) == 0 {
          x86_emit_movq_reg_reg(c.o, reg, h)
      }
      G1_RAX_SLOT = 0 - 1
      return 0
  }
  if c.slot_off[v_id] < 0 { return 0 }    // unused result
  // G4: a single-use next-instruction temp's slot store is DEAD -- the
  // consumer reads rax through the G1 forwarding path. Skip the store but
  // keep the G1 contract ("rax holds this slot's value"). Only valid when
  // the result really is in rax; any other producer reg un-flags and falls
  // through to a normal store, keeping the slot-load tripwire exact.
  if c.elide[v_id] == 1 {
      if _g1_is_rax(reg) == 1 {
          G1_RAX_SLOT = c.slot_off[v_id]
          return 0
      }
      c.elide[v_id] = 0
  }
  let rbp_off: i64 = x86ctx_value_base(c) + c.slot_off[v_id]
  x86_emit_store_qword(c.o, reg, "rbp" as *u8, rbp_off)
  // G1: rax now provably holds this value's slot (the store copied it).
  if _g1_is_rax(reg) == 1 { G1_RAX_SLOT = c.slot_off[v_id] }
  return 0
627}
628
629// ===== load_value_v (full materialisation) =======================
630//
631// Defined ahead of its 9 emit_* callers per F7 post-order DFS
632// discipline -- see docs/NISHI_F7_FORWARD_REF_S_CLASS_PLAN.md.
633// Unlike x86ctx_load_value (above) which expects the value already
634// in a register, _v walks alloca/spill/const/global maps and emits
635// the materialising mov.
636
637func x86ctx_load_value_v(c: *X86Ctx, v_id: i64, reg: *u8) -> i64 {
  let val: *Value = x86ctx_value_at(c.f, v_id)
  // G1 (CS:APP SS5.6): rax already holds this spilled SSA value -> skip the
  // reload; else if rax is about to be clobbered by the load -> invalidate.
  if _g1_is_rax(reg) == 1 {
      if G1_RAX_SLOT >= 0 {
          if val.kind != VK_CONST_INT { if val.kind != VK_GLOBAL { if val.kind != VK_FUNC_ADDR { if c.alloca_off[v_id] < 0 {
              if c.slot_off[v_id] == G1_RAX_SLOT {
                  G1_RAX_SLOT = 0 - 1
                  return 0
              }
          } } } }
      }
      G1_RAX_SLOT = 0 - 1
  }
  // G10 (2026-07-15): a home-FORWARDED load result still lives in its source
  // alloca's home register (the fwd scan proved no intervening store/call).
  // Read the home directly -- the load itself emitted NOTHING, its slot was
  // never written. Must precede the spill/elide paths.
  if c.fwd_home[v_id] >= 0 {
      let fwr: *u8 = x86_home_reg_name(c.fwd_home[v_id])
      if x86_reg_eq(reg, fwr) == 0 {
          x86_emit_movq_reg_reg(c.o, fwr, reg)
      }
      return 0
  }
  // G1 (FIX-1): a homed value LIVES in its home register -> a register move,
  // NEVER a reload. Zero-cost when reg already IS the home register.
  let g1l: *ValueLoc = ((c.locs as i64) + v_id * 16) as *ValueLoc
  if g1l.kind == VL_REGISTER {
      let g1h: *u8 = x86_home_reg_name(g1l.idx)
      if x86_reg_eq(reg, g1h) == 0 {
          x86_emit_movq_reg_reg(c.o, g1h, reg)
      }
      return 0
  }
  if val.kind == VK_CONST_INT {
      x86_emit_movabsq(c.o, reg, val.const_int)
      return 0
  }
  if val.kind == VK_GLOBAL {
      out_str(c.o, "    leaq .Lg")
      out_i64(c.o, val.const_int)
      out_str(c.o, "(%rip), %")
      out_str(c.o, reg)
      out_char(c.o, 0x0A)
      return 0
  }
  if val.kind == VK_FUNC_ADDR {
      let fnp: *Function = val.const_int as *Function
      out_str(c.o, "    leaq ")
      out_str(c.o, fnp.name_start as *u8)
      out_str(c.o, "(%rip), %")
      out_str(c.o, reg)
      out_char(c.o, 0x0A)
      return 0
  }
  if c.alloca_off[v_id] >= 0 {
      // G2: a homed alloca's VALUE lives in its home register -> register
      // move (zero-cost when reg already IS the home). This is the deref
      // the comment block below describes, minus the memory.
      if c.alloca_home[v_id] >= 0 {
          let ahreg: *u8 = x86_home_reg_name(c.alloca_home[v_id])
          if x86_reg_eq(reg, ahreg) == 0 {
              x86_emit_movq_reg_reg(c.o, ahreg, reg)
          }
          return 0
      }
      // Peephole 2026-05-20: fold `leaq -off(%rbp), %reg ; movq (%reg), %reg`
      // into single `movq -off(%rbp), %reg`.  Stabilizer-validated 2-4x
      // gap to gcc -O0 is dominated by this load pattern; eliminating
      // the indirection drops ~30% of inner-loop instructions.  Safe:
      // both sequences load the same qword into the same register
      // with no observable intermediate state.
      let abp_off: i64 = x86ctx_value_base(c) + c.alloca_off[v_id]
      x86_emit_load_qword(c.o, "rbp" as *u8, abp_off, reg)
      return 0
  }
  // Spilled SSA value.
  // G4 tripwire (see x86ctx_load_value): elided slots are never loadable.
  if c.elide[v_id] == 1 {
      out_str(c.o, "    jmp .G4_elided_slot_load_bug\n")
      return 0
  }
  let rbp_off: i64 = x86ctx_value_base(c) + c.slot_off[v_id]
  x86_emit_load_qword(c.o, "rbp" as *u8, rbp_off, reg)
  return 0
724}
725
726// G1 consume-side: the home register name if v_id is homed, else null.
727func x86ctx_home_name_or_null(c: *X86Ctx, v_id: i64) -> *u8 {
  if v_id >= 0 { if v_id < c.f.n_values {
      let l: *ValueLoc = ((c.locs as i64) + v_id * 16) as *ValueLoc
      if l.kind == VL_REGISTER { return x86_home_reg_name(l.idx) }
  } }
  return 0 as *u8
733}
734
735// G10 consume-side: the SOURCE home register name for a forwarded load
736// result, else null. Reading it directly at an audited consumer position is
737// ZERO instructions (vs movq %home,%rcx) -- safe because a compare/binop
738// source read never mutates the register.
739func x86ctx_fwd_name_or_null(c: *X86Ctx, v_id: i64) -> *u8 {
  if v_id >= 0 { if v_id < c.f.n_values {
      if c.fwd_home[v_id] >= 0 { return x86_home_reg_name(c.fwd_home[v_id]) }
  } }
  return 0 as *u8
744}
745
746// LN7 (2026-08-23, lang rung "backend consumes its register allocation"): THE
747// consume-side predicate. The register a consumer may read value v_id from IN
748// PLACE -- its own home if the allocator homed it, else the home of the alloca it
749// was forwarded from (x86_fwd_scan), else null (materialize via load_value_v).
750// A value is never both homed and forwarded (the scan releases the home when it
751// forwards), so the two lookups are one question, and every audited consumer
752// position (cmp op0/op1, binop src, SIB index, store value, cmov dividend) asks
753// it here instead of pairing the two lookups by hand. Reading the register as a
754// source operand never mutates it, which is what makes the in-place read sound.
755func x86_use_regalloc(c: *X86Ctx, v_id: i64) -> *u8 {
  let h: *u8 = x86ctx_home_name_or_null(c, v_id)
  if h != (0 as *u8) { return h }
  return x86ctx_fwd_name_or_null(c, v_id)
759}
760
761// Ops whose second operand can be consumed DIRECTLY as the in-place `<op> src,
762// dst` source register (so a homed op1 needs no `movq %home,%rcx`). div/rem
763// (op1 in rcx for idivq) and shifts/rotates (count in cl) are excluded.
764func x86ctx_op1_direct_ok(op: i64) -> i64 {
  if op == OP_ADD { return 1 }
  if op == OP_SUB { return 1 }
  if op == OP_MUL { return 1 }
  if op == OP_AND { return 1 }
  if op == OP_OR  { return 1 }
  if op == OP_XOR { return 1 }
  return 0
772}
773
774// ===== binop dispatch ============================================
775
776// bit position of a power-of-two value (caller guarantees v == 2^k, k in 0..63).
777func x86ctx_log2_i64(v: i64) -> i64 {
  var k: i64 = 0
  var m: i64 = v
  while m > 1 { m = m >> 1; k = k + 1 }
  return k
782}
783
784// ===== LN18: CHECKED ARITHMETIC AT add+jo COST (2026-09-01) ============================
785// nx_parse.nx (LN1, --chkarith) emits every checked i64 add/sub as backend-agnostic IR:
786//   r = a + b ; x1 = a ^ r ; x2 = b ^ r ; a1 = x1 & x2 ; c = a1 < 0 ; br_cond c -> fail, ok
787//   r = a - b ; x1 = a ^ b ; x2 = a ^ r ; a1 = x1 & x2 ; c = a1 < 0 ; br_cond c -> fail, ok
788// which this emitter rendered as six instructions with spills per add -- measured 2026-09-01 on the
789// /compare/lang receipt as 17,011 us for the daily mode against 7,145 us for the default build.
790// The predicate ((a^r)&(b^r))<0 IS the signed-overflow flag of the add just emitted (and
791// ((a^b)&(a^r))<0 of the sub), so when the four intermediates have no other use the whole chain is
792// `addq` (already emitted) + `jo fail` on the live flags: the pending-compare belt G6 already carries a
793// condition code to the following br_cond, and G24_SKIP makes the block loop step over the four
794// intermediates. Single-use is the emitter's OWN G8 operand count (x86ctx_count_uses), so a program
795// that reads x1 or a1 elsewhere keeps the long form -- the fuse can only remove work nothing observes.
796// The store of r happens BEFORE this runs and is a plain movq, which does not touch the flags.
797func x86ctx_ovf_pair(n: *Instr, x: i64, y: i64) -> i64 {
  if n.op0 == x { if n.op1 == y { return 1 } }
  if n.op0 == y { if n.op1 == x { return 1 } }
  return 0
801}
802func x86ctx_ovf_single(c: *X86Ctx, v: i64) -> i64 {
  if v < 0 { return 0 }
  if v >= c.f.n_values { return 0 }
  if c.uses[v] != 1 { return 0 }
  return 1
807}
808func x86ctx_ovf_fuse(c: *X86Ctx, i: *Instr) -> i64 {
  if i.op != OP_ADD { if i.op != OP_SUB { return 0 } }
  let r: i64 = i.result
  if r < 0 { return 0 }
  let a: i64 = i.op0
  let b: i64 = i.op1
  let n1: *Instr = i.next
  if n1 == (0 as *Instr) { return 0 }
  let n2: *Instr = n1.next
  if n2 == (0 as *Instr) { return 0 }
  let n3: *Instr = n2.next
  if n3 == (0 as *Instr) { return 0 }
  let n4: *Instr = n3.next
  if n4 == (0 as *Instr) { return 0 }
  let n5: *Instr = n4.next
  if n5 == (0 as *Instr) { return 0 }
  if n1.op != OP_XOR { return 0 }
  if n2.op != OP_XOR { return 0 }
  if n3.op != OP_AND { return 0 }
  if n4.op != OP_LT_S { return 0 }
  if n5.op != OP_BR_COND { return 0 }
  var shape: i64 = 0
  if i.op == OP_ADD {
      if x86ctx_ovf_pair(n1, a, r) == 1 { if x86ctx_ovf_pair(n2, b, r) == 1 { shape = 1 } }
      if x86ctx_ovf_pair(n1, b, r) == 1 { if x86ctx_ovf_pair(n2, a, r) == 1 { shape = 1 } }
  }
  if i.op == OP_SUB {
      if x86ctx_ovf_pair(n1, a, b) == 1 { if x86ctx_ovf_pair(n2, a, r) == 1 { shape = 1 } }
      if x86ctx_ovf_pair(n1, a, r) == 1 { if x86ctx_ovf_pair(n2, a, b) == 1 { shape = 1 } }
  }
  if shape == 0 { return 0 }
  if x86ctx_ovf_pair(n3, n1.result, n2.result) == 0 { return 0 }
  if n4.op0 != n3.result { return 0 }
  if n4.op1 < 0 { return 0 }
  if n4.op1 >= c.f.n_values { return 0 }
  let zv: *Value = x86ctx_value_at(c.f, n4.op1)
  if zv.kind != VK_CONST_INT { return 0 }
  if zv.const_int != 0 { return 0 }
  if n5.op0 != n4.result { return 0 }
  if x86ctx_ovf_single(c, n1.result) == 0 { return 0 }
  if x86ctx_ovf_single(c, n2.result) == 0 { return 0 }
  if x86ctx_ovf_single(c, n3.result) == 0 { return 0 }
  if x86ctx_ovf_single(c, n4.result) == 0 { return 0 }
  G1_PENDING_CC = NX_X64_CC_O
  G1_PENDING_VAL = n4.result
  G24_SKIP = 4
  G24_FUSED = G24_FUSED + 1
  return 1
856}
857
858func x86ctx_emit_binop(c: *X86Ctx, i: *Instr) -> i64 {
  // G8: a SHL/MUL folded into a SIB load/store is dead -- emit nothing.
  if i.result >= 0 { if i.result < c.f.n_values { if c.sib_dead[i.result] == 1 { return 0 } } }
  // FIX-1 (ALLOCATE-not-COPY): a homed result is computed straight INTO its
  // home register. dst = the home for a homed pure-rax binop, else "rax".
  // Non-homed results (incl. div/rem/shift, never homed in cut-1) get dst=rax,
  // so op0 lands in rax exactly as before and those cases are unchanged.
  // G11: a chain-fused binop operates IN PLACE on the source alloca's home --
  // dst = that home and op0 is NOT materialized (its value IS the home's
  // current content; the head load emitted nothing). Only ADD/SUB/MUL/AND/
  // OR/XOR are ever chain-marked, so the G7/G5a-shift/UMULHI paths below
  // never see a chain dst.
  var g11: i64 = 0 - 1
  if i.result >= 0 { if i.result < c.f.n_values { g11 = c.chain_home[i.result] } }
  var dst: *u8 = x86ctx_result_reg(c, i.result)
  if g11 >= 0 { dst = x86_home_reg_name(g11) }
  // G21 COMMUTED RAX-CONSUME (2026-07-16): op1 of a commutative, non-chain,
  // unhomed-result binop is RIDING IN RAX (G1 contract: rax holds exactly the
  // value of slot G1_RAX_SLOT). Swap operands at emit -- rax IS the dst seed,
  // so the op0-into-dst load (which would clobber the ride) is skipped and
  // op0 becomes the src, resolved by the normal src1 machinery (home-direct /
  // imm-fold / rcx). Kills the op1 rcx-reload; with the mirrored k==1 clause
  // in x86_g4_pos_ok the producer's slot store dies too (spill+reload pair ->
  // nothing). dst is already "rax" here (unhomed result). The exact kind/
  // alloca guards mirror load_value_v's G1 skip. Negctl COMMUTE_ANY drops the
  // G1 match -> garbage rides in -> matrix RED.
  var g21: i64 = 0
  if g11 < 0 {
      if x86_chain_op_commutative(i.op) == 1 {
          if x86ctx_home_name_or_null(c, i.result) == (0 as *u8) {
              if i.op1 >= 0 { if i.op1 < c.f.n_values {
                  let g21v: *Value = x86ctx_value_at(c.f, i.op1)
                  if g21v.kind != VK_CONST_INT { if g21v.kind != VK_GLOBAL { if g21v.kind != VK_FUNC_ADDR {
                      if c.alloca_off[i.op1] < 0 {
                          if X86_NEGCTL_G21_COMMUTE_ANY == 1 { g21 = 1 }
                          if G1_RAX_SLOT >= 0 { if c.slot_off[i.op1] == G1_RAX_SLOT { g21 = 1 } }
                      }
                  } } }
              } }
          }
      }
  }
  // G23 SRC-DIRECT CMOV DIVISION (2026-07-16): a HOMED/FORWARDED sign-unknown
  // dividend is read straight from its register by the cmov form (leaq
  // bias(%h),%rcx; testq %h,%h; cmovns %h,%rcx; sarq $k,%rcx) -- the
  // movq %home,%rax feed dies, and rax (with its G1 ride state) SURVIVES the
  // whole division untouched. Gated on elide[result]==0: an elided result
  // must ride out of rax, which this form never writes. Disjoint from G17
  // (requires val_nonneg==0) and from the k==0 identity (requires 2^k>1).
  var g23: i64 = 0
  var g23h: *u8 = 0 as *u8
  var g23k: i64 = 0
  var g23d: i64 = 0
  if g11 < 0 { if g21 == 0 {
      if i.op == OP_DIV_S {
          let g23v: *Value = x86ctx_value_at(c.f, i.op1)
          if g23v.kind == VK_CONST_INT {
              let g23dd: i64 = g23v.const_int
              if g23dd > 1 { if (g23dd & (g23dd - 1)) == 0 {
                  g23k = x86ctx_log2_i64(g23dd)
                  if g23k <= 31 {
                      if x86_val_nonneg(c.f, i.op0, 8) == 0 {
                          g23h = x86_use_regalloc(c, i.op0)     // LN7: in-place read
                          if g23h != (0 as *u8) {
                              var g23e: i64 = 0
                              if i.result >= 0 { if i.result < c.f.n_values { g23e = c.elide[i.result] } }
                              if g23e == 0 { g23 = 1; g23d = g23dd }
                          }
                      }
                  }
              } }
          }
      }
  } }
  if g11 < 0 { if g21 == 0 { if g23 == 0 { x86ctx_load_value_v(c, i.op0, dst) } } }
  if g21 == 1 { G1_RAX_SLOT = 0 - 1 }
  // G11 commuted chains (MUL(3,n) canonical const-first forms): the chain
  // value sits at op1, so the SOURCE operand of the in-place `op src,%hA` is
  // op0. Everywhere else srcop == op1 -> byte-identical. Only COMMUTATIVE
  // ops are ever swap-marked (dst OP src == src OP dst).
  var srcop: i64 = i.op1
  if g11 >= 0 { if c.chain_swap[i.result] == 1 { srcop = i.op0 } }
  if g21 == 1 { srcop = i.op0 }
  // G7: divide by a CONSTANT POWER OF TWO -> shift. idivq is ~20-40 cycles;
  // a shift is 1. Signed division rounds toward ZERO, so the arithmetic-shift
  // (floor) result needs a bias when the dividend is negative: add (2^k - 1)
  // before the sar. Sequence matches gcc/clang exactly (cross-checked bit-for-
  // bit over negative/positive/INT_MIN by the xlang matrix's signed-div KAT).
  // DIV_S/DIV_U are never homed (dst=="rax"); rcx is free scratch (not a home).
  if i.op == OP_DIV_S {
      let g7v: *Value = x86ctx_value_at(c.f, i.op1)
      if g7v.kind == VK_CONST_INT {
          let g7d: i64 = g7v.const_int
          if g7d > 0 { if (g7d & (g7d - 1)) == 0 {
              let g7k: i64 = x86ctx_log2_i64(g7d)
              if g7k == 0 { x86ctx_store_result(c, i.result, dst); return 0 }
              // G23: home-read cmov form (op0 load was SKIPPED -- dst holds
              // nothing for this op; the branch reads g23h and writes rcx
              // only). Checked FIRST: under g23 the dst-based paths below
              // must not run. Disjointness with G17 is also structural
              // (g23 requires val_nonneg==0).
              if g23 == 1 {
                  out_str(c.o, "    leaq ")
                  out_i64(c.o, g23d - 1)
                  out_str(c.o, "(%")
                  out_str(c.o, g23h)
                  out_str(c.o, "), %rcx\n")
                  out_str(c.o, "    testq %")
                  out_str(c.o, g23h)
                  out_str(c.o, ", %")
                  out_str(c.o, g23h)
                  out_char(c.o, 0x0A)
                  var g23cc: *u8 = "cmovns" as *u8
                  if X86_NEGCTL_G23_CMOV_INVERT == 1 { g23cc = "cmovs" as *u8 }
                  out_str(c.o, "    ")
                  out_str(c.o, g23cc)
                  out_str(c.o, " %")
                  out_str(c.o, g23h)
                  out_str(c.o, ", %rcx\n")
                  out_str(c.o, "    sarq $")
                  out_i64(c.o, g23k)
                  out_str(c.o, ", %rcx\n")
                  x86ctx_store_result(c, i.result, "rcx" as *u8)
                  return 0
              }
              // G17 (2026-07-16): a PROVABLY NON-NEGATIVE dividend needs no
              // sign-bias dance -- one bare shift (the scientist's 2.063x
              // proven spot; gcc -O2 emits the dance wherever it cannot see
              // the range). The lattice excludes every overflow-capable op,
              // and the negctl (nonneg-always) goes RED on the battery.
              if x86_val_nonneg(c.f, i.op0, 8) == 1 {
                  out_str(c.o, "    shrq $")
                  out_i64(c.o, g7k)
                  out_str(c.o, ", %")
                  out_str(c.o, dst)
                  out_char(c.o, 0x0A)
                  x86ctx_store_result(c, i.result, dst)
                  return 0
              }
              // G22 (2026-07-16): bias-via-CMOV -- gcc/clang's own form.
              //     leaq (2^k-1)(%dst), %rcx   ; bias candidate (flag-neutral)
              //     testq %dst, %dst           ; SF := sign(v)  [dies on ride]
              //     cmovs %rcx, %dst           ; v<0 ? v+bias : v
              //     sarq $k, %dst
              // 4 instrs vs the 5-instr shr-dance, dependency depth 3 vs 4.
              // The testq DIES when the textually-previous instruction is an
              // ALU op that produced THIS dividend: SF already equals its
              // sign, and only flag-neutral movs/leas are emitted in between
              // (the exact G14 contract). cmovS reads ONLY SF, so the G14
              // signed-LT OF-hazard does not apply; same producer set
              // {ADD,SUB,AND,OR,XOR} + sib_dead exclusion. Result stays in
              // dst(=rax) so every G1/G4 ride downstream is preserved.
              // k>31 keeps the dance (the bias immediate exceeds leaq's
              // disp32). Negctls: CMOV_INVERT + RIDE_ALWAYS, both matrix-RED.
              if g7k <= 31 {
                  var g22r: i64 = 0
                  let g22p: *Instr = i.prev
                  if g22p != (0 as *Instr) {
                      if g22p.result == i.op0 {
                          var g22ok: i64 = 0
                          if g22p.op == OP_ADD { g22ok = 1 }
                          if g22p.op == OP_SUB { g22ok = 1 }
                          if g22p.op == OP_AND { g22ok = 1 }
                          if g22p.op == OP_OR  { g22ok = 1 }
                          if g22p.op == OP_XOR { g22ok = 1 }
                          if g22ok == 1 {
                              if g22p.result >= 0 { if g22p.result < c.f.n_values {
                                  if c.sib_dead[g22p.result] == 1 { g22ok = 0 }
                              } }
                          }
                          if g22ok == 1 { g22r = 1 }
                      }
                  }
                  if X86_NEGCTL_G22_RIDE_ALWAYS == 1 { g22r = 1 }
                  out_str(c.o, "    leaq ")
                  out_i64(c.o, g7d - 1)
                  out_str(c.o, "(%")
                  out_str(c.o, dst)
                  out_str(c.o, "), %rcx\n")
                  if g22r == 0 {
                      out_str(c.o, "    testq %")
                      out_str(c.o, dst)
                      out_str(c.o, ", %")
                      out_str(c.o, dst)
                      out_char(c.o, 0x0A)
                  }
                  var g22cc: *u8 = "cmovs" as *u8
                  if X86_NEGCTL_G22_CMOV_INVERT == 1 { g22cc = "cmovns" as *u8 }
                  out_str(c.o, "    ")
                  out_str(c.o, g22cc)
                  out_str(c.o, " %rcx, %")
                  out_str(c.o, dst)
                  out_char(c.o, 0x0A)
                  out_str(c.o, "    sarq $")
                  out_i64(c.o, g7k)
                  out_str(c.o, ", %")
                  out_str(c.o, dst)
                  out_char(c.o, 0x0A)
                  x86ctx_store_result(c, i.result, dst)
                  return 0
              }
              x86_emit_movq_reg_reg(c.o, dst, "rcx" as *u8)     // rcx = x
              out_str(c.o, "    sarq $63, %rcx\n")              // rcx = 0 or -1
              out_str(c.o, "    shrq $")
              out_i64(c.o, 64 - g7k)
              out_str(c.o, ", %rcx\n")                          // rcx = 0 or (2^k-1)
              x86_emit_addq_rr(c.o, "rcx" as *u8, dst)          // dst = x + bias
              out_str(c.o, "    sarq $")
              out_i64(c.o, g7k)
              out_str(c.o, ", %")
              out_str(c.o, dst)
              out_char(c.o, 0x0A)
              x86ctx_store_result(c, i.result, dst)
              return 0
          } }
      }
  }
  if i.op == OP_DIV_U {
      let g7uv: *Value = x86ctx_value_at(c.f, i.op1)
      if g7uv.kind == VK_CONST_INT {
          let g7ud: i64 = g7uv.const_int
          if g7ud > 0 { if (g7ud & (g7ud - 1)) == 0 {
              let g7uk: i64 = x86ctx_log2_i64(g7ud)
              if g7uk == 0 { x86ctx_store_result(c, i.result, dst); return 0 }
              out_str(c.o, "    shrq $")
              out_i64(c.o, g7uk)
              out_str(c.o, ", %")
              out_str(c.o, dst)
              out_char(c.o, 0x0A)
              x86ctx_store_result(c, i.result, dst)
              return 0
          } }
      }
  }
  // G5a: imm32 op1 folding -- `addq $imm,%dst` (and shift-by-constant)
  // replaces the movabsq-into-rcx + reg-op form: 2 insns -> 1, and the
  // per-iteration constant re-materialisation disappears. XOR is excluded
  // (nxasm's xorq dispatch is alu_rr-only -- a $imm operand would silently
  // mis-encode); MUL/div/rem/UMULHI/CRC32/PDEP/PEXT keep the register path.
  let g5v1: *Value = x86ctx_value_at(c.f, srcop)
  if g5v1.kind == VK_CONST_INT {
      let g5c: i64 = g5v1.const_int
      var g5mn: *u8 = 0 as *u8
      if i.op == OP_ADD { g5mn = "addq" as *u8 }
      if i.op == OP_SUB { g5mn = "subq" as *u8 }
      if i.op == OP_AND { g5mn = "andq" as *u8 }
      if i.op == OP_OR  { g5mn = "orq" as *u8 }
      if g5mn != (0 as *u8) {
          if g5c <= 2147483647 { if g5c >= (0 - 2147483648) {
              out_str(c.o, "    ")
              out_str(c.o, g5mn)
              out_str(c.o, " $")
              out_i64(c.o, g5c)
              out_str(c.o, ", %")
              out_str(c.o, dst)
              out_char(c.o, 0x0A)
              x86ctx_store_result(c, i.result, dst)
              x86ctx_ovf_fuse(c, i)
              return 0
          } }
      }
      var g5sh: *u8 = 0 as *u8
      if i.op == OP_SHL { g5sh = "shlq" as *u8 }
      if i.op == OP_SHR_S { g5sh = "sarq" as *u8 }
      if i.op == OP_SHR_U { g5sh = "shrq" as *u8 }
      if g5sh != (0 as *u8) {
          if g5c >= 0 { if g5c <= 63 {
              out_str(c.o, "    ")
              out_str(c.o, g5sh)
              out_str(c.o, " $")
              out_i64(c.o, g5c)
              out_str(c.o, ", %rax\n")
              x86ctx_store_result(c, i.result, "rax" as *u8)
              return 0
          } }
      }
      // G19 lea-strength: c*n -> leaq (%n,%n,scale),%n for c in {2,3,5,9},
      // scale=c-1 in {1,2,4,8}. dst already holds op0 (n): the non-chain path
      // loads op0 into dst (~line 741); a chain dst holds the running value;
      // and srcop/g5c is the CONSTANT factor in every path that reaches here,
      // so dst is always the non-const factor n. lea is flag-neutral -- imul
      // leaves ZF undefined anyway and MUL is excluded from the G14 flag-reuse
      // producer set, so no downstream cmp-elision can depend on these flags.
      // c=4/8 are already covered by the shl peephole. The subsequent `+d`
      // (e.g. collatz 3n+1) stays a separate imm-folded addq -- sound with no
      // lookahead; displacement fusion is a later sub-rung. Negctl WRONG_SCALE
      // emits c (=> (c+1)*n, matrix RED).
      if i.op == OP_MUL {
          var g19s: i64 = 0
          if g5c == 2 { g19s = 1 }
          if g5c == 3 { g19s = 2 }
          if g5c == 5 { g19s = 4 }
          if g5c == 9 { g19s = 8 }
          if g19s != 0 {
              if X86_NEGCTL_LEA_WRONG_SCALE == 1 { g19s = g5c }
              out_str(c.o, "    leaq (%")
              out_str(c.o, dst)
              out_str(c.o, ",%")
              out_str(c.o, dst)
              out_str(c.o, ",")
              out_i64(c.o, g19s)
              out_str(c.o, "), %")
              out_str(c.o, dst)
              out_char(c.o, 0x0A)
              x86ctx_store_result(c, i.result, dst)
              return 0
          }
      }
  }
  // G1 consume-side: a homed op1 of a pure-rax op is used DIRECTLY as the src
  // register (no `movq %home,%rcx`). Everything else loads op1 into rcx
  // (div/rem need it there for idivq; shifts/rotates need cl). When nothing is
  // homed, src1 == "rcx" and this is byte-identical to before.
  var src1: *u8 = "rcx" as *u8
  // G10/LN7: a homed OR forwarded src reads its register directly (zero-cost),
  // null when neither. srcop==op1 except for G11 commuted chains (source = op0).
  var op1_home: *u8 = x86_use_regalloc(c, srcop)
  if op1_home != (0 as *u8) {
      if x86ctx_op1_direct_ok(i.op) == 1 {
          src1 = op1_home
      }
      if x86ctx_op1_direct_ok(i.op) == 0 {
          x86ctx_load_value_v(c, srcop, "rcx" as *u8)
      }
  }
  if op1_home == (0 as *u8) {
      // G16: a CHAIN-FUSED op's dst is a home (never rax), so rax is free for
      // the src -- materialize via rax so the G1 store->load forward (and the
      // G4 chain-src elide) collapse `spill+reload` to NOTHING for a producer
      // in the previous instruction. Non-chain ops keep rcx (byte-identical).
      if g11 >= 0 {
          x86ctx_load_value_v(c, srcop, "rax" as *u8)
          src1 = "rax" as *u8
      }
      if g11 < 0 {
          x86ctx_load_value_v(c, srcop, "rcx" as *u8)
      }
  }
  if i.op == OP_ADD {
      x86_emit_addq_rr(c.o, src1, dst)
      x86ctx_store_result(c, i.result, dst)
      x86ctx_ovf_fuse(c, i)
      return 0
  }
  if i.op == OP_SUB {
      x86_emit_subq_rr(c.o, src1, dst)
      x86ctx_store_result(c, i.result, dst)
      x86ctx_ovf_fuse(c, i)
      return 0
  }
  if i.op == OP_MUL {
      x86_emit_imulq_rr(c.o, src1, dst)
      x86ctx_store_result(c, i.result, dst)
      return 0
  }
  if i.op == OP_UMULHI {
      // G2 unsigned 64x64 -> HIGH 64 bits. mulq sets rdx:rax = rax * src1; we
      // keep the rdx half. op0 is in rax (dst -- OP_UMULHI is never homed so
      // dst==rax, the implicit multiplicand); op1 is src1 (rcx, or a home reg).
      x86_emit_mulq_r(c.o, src1)
      x86ctx_store_result(c, i.result, "rdx" as *u8)
      return 0
  }
  if i.op == OP_CRC32 {
      // SSE4.2 CRC-32C accumulate: crc32q %src1,%dst  ->  dst = CRC32C(dst, src1).
      // op0 (the running crc) is in dst; op1 (the data word) is src1. CRC32 is
      // NOT op1-direct-ok, so src1 is always rcx (op1 loaded there above), and
      // dst can be any home reg -- crc32q reg,reg accepts arbitrary GPRs.
      x86_emit_crc32q_rr(c.o, src1, dst)
      x86ctx_store_result(c, i.result, dst)
      return 0
  }
  if i.op == OP_PDEP {
      // BMI2 pdep %src2,%src1,%dst -> deposit src1's low bits into src2's mask.
      // __pdep64(value, mask): op0 (value) is in dst (VEX.vvvv src1 = dst reg),
      // op1 (mask) is in src1==rcx (ModRM.rm src2). PDEP is NOT op1-direct-ok,
      // so the mask is always rcx; the home pool excludes rcx/rdx (FIX-12), so
      // dst is never rcx and dst==src1(vvvv)==result is a legal, collision-free
      // 3-operand form (pdep permits dst == VEX.vvvv).
      x86_emit_pdep_rrr(c.o, src1, dst, dst)
      x86ctx_store_result(c, i.result, dst)
      return 0
  }
  if i.op == OP_PEXT {
      // BMI2 pext %src2,%src1,%dst -> gather src1 bits at src2's mask positions.
      // __pext64(value, mask): op0 (value) in dst (VEX.vvvv src1), op1 (mask) in
      // src1==rcx (ModRM.rm src2). Same collision-free layout as PDEP.
      x86_emit_pext_rrr(c.o, src1, dst, dst)
      x86ctx_store_result(c, i.result, dst)
      return 0
  }
  if i.op == OP_DIV_S {
      x86_emit_cqo(c.o)
      x86_emit_idivq_r(c.o, "rcx" as *u8)
      x86ctx_store_result(c, i.result, "rax" as *u8)
      return 0
  }
  if i.op == OP_REM_S {
      x86_emit_cqo(c.o)
      x86_emit_idivq_r(c.o, "rcx" as *u8)
      x86ctx_store_result(c, i.result, "rdx" as *u8)
      return 0
  }
  if i.op == OP_DIV_U {
      x86_emit_xorq_rr(c.o, "rdx" as *u8, "rdx" as *u8)
      x86_emit_divq_r(c.o, "rcx" as *u8)
      x86ctx_store_result(c, i.result, "rax" as *u8)
      return 0
  }
  if i.op == OP_REM_U {
      x86_emit_xorq_rr(c.o, "rdx" as *u8, "rdx" as *u8)
      x86_emit_divq_r(c.o, "rcx" as *u8)
      x86ctx_store_result(c, i.result, "rdx" as *u8)
      return 0
  }
  if i.op == OP_AND {
      x86_emit_andq_rr(c.o, src1, dst)
      x86ctx_store_result(c, i.result, dst)
      return 0
  }
  if i.op == OP_OR {
      x86_emit_orq_rr(c.o, src1, dst)
      x86ctx_store_result(c, i.result, dst)
      return 0
  }
  if i.op == OP_XOR {
      x86_emit_xorq_rr(c.o, src1, dst)
      x86ctx_store_result(c, i.result, dst)
      return 0
  }
  // shifts: count must be in cl
  if i.op == OP_SHL {
      x86_emit_shlq_cl(c.o, "rax" as *u8)
      x86ctx_store_result(c, i.result, "rax" as *u8)
      return 0
  }
  if i.op == OP_SHR_S {
      x86_emit_sarq_cl(c.o, "rax" as *u8)
      x86ctx_store_result(c, i.result, "rax" as *u8)
      return 0
  }
  if i.op == OP_SHR_U {
      x86_emit_shrq_cl(c.o, "rax" as *u8)
      x86ctx_store_result(c, i.result, "rax" as *u8)
      return 0
  }
  // rotates: value in rax, count in cl (low byte of rcx)
  if i.op == OP_ROTL64 {
      x86_emit_rolq_cl(c.o, "rax" as *u8)
      x86ctx_store_result(c, i.result, "rax" as *u8)
      return 0
  }
  if i.op == OP_ROTR64 {
      x86_emit_rorq_cl(c.o, "rax" as *u8)
      x86ctx_store_result(c, i.result, "rax" as *u8)
      return 0
  }
  out_str(c.o, "    # x86_64: binop opcode ")
  out_i64(c.o, i.op)
  out_str(c.o, " not yet wired\n")
  return 0
1319}
1320
1321// ===== unop dispatch (NEG / NOT / TRUNC / SEXT / ZEXT / BITCAST) ==
1322
1323const OP_TRUNC:   i64 = 17
1324const OP_SEXT:    i64 = 18
1325const OP_ZEXT:    i64 = 19
1326const OP_BITCAST: i64 = 26
1327
1328func x86ctx_emit_unop(c: *X86Ctx, i: *Instr) -> i64 {
  x86ctx_load_value_v(c, i.op0, "rax" as *u8)
  if i.op == OP_NEG {
      x86_emit_negq_r(c.o, "rax" as *u8)
      x86ctx_store_result(c, i.result, "rax" as *u8)
      return 0
  }
  if i.op == OP_NOT {
      x86_emit_notq_r(c.o, "rax" as *u8)
      x86ctx_store_result(c, i.result, "rax" as *u8)
      return 0
  }
  // Scalar bit unops: value already in rax.
  if i.op == OP_BSWAP64 {
      x86_emit_bswapq_rax(c.o)
      x86ctx_store_result(c, i.result, "rax" as *u8)
      return 0
  }
  if i.op == OP_POPCNT64 {
      x86_emit_popcntq_rax(c.o)
      x86ctx_store_result(c, i.result, "rax" as *u8)
      return 0
  }
  if i.op == OP_CLZ32 {
      x86_emit_lzcntl_eax(c.o)
      x86ctx_store_result(c, i.result, "rax" as *u8)
      return 0
  }
  if i.op == OP_CTZ32 {
      x86_emit_tzcntl_eax(c.o)
      x86ctx_store_result(c, i.result, "rax" as *u8)
      return 0
  }
  // __rdtsc(): read the cycle counter.  rdtsc -> EDX:EAX (high:low);
  // combine into a full 64-bit value in rax.  (op0 dummy already loaded
  // into rax above and harmlessly overwritten; rdx clobber is safe in the
  // stack-machine model -- operands are reloaded fresh per instruction.)
  if i.op == OP_RDTSC {
      out_str(c.o, "    rdtsc\n")
      out_str(c.o, "    shlq $32, %rdx\n")
      out_str(c.o, "    orq %rdx, %rax\n")
      x86ctx_store_result(c, i.result, "rax" as *u8)
      return 0
  }
  // TRUNC / SEXT / ZEXT / BITCAST: i64-only stack-machine layer
  // treats all widths as i64, so these are no-ops on rax (the
  // sign-extend at load time already handled width).
  x86ctx_store_result(c, i.result, "rax" as *u8)
  return 0
1377}
1378
1379// ===== atomic dispatch ===========================================
1380// Address -> %r11; conservative-strong ordering (mo operand ignored,
1381// always correct on x86 TSO).  C bootstrap parity (x86_64.c).
1382
1383func x86ctx_emit_atomic(c: *X86Ctx, i: *Instr) -> i64 {
  if i.op == OP_ATOMIC_LOAD_I64 {
      x86ctx_load_value_v(c, i.op0, "r11" as *u8)        // addr
      x86_emit_load_qword(c.o, "r11" as *u8, 0, "rax" as *u8)
      x86ctx_store_result(c, i.result, "rax" as *u8)
      return 0
  }
  if i.op == OP_ATOMIC_STORE_I64 {
      x86ctx_load_value_v(c, i.op0, "r11" as *u8)        // addr
      x86ctx_load_value_v(c, i.op1, "rax" as *u8)        // val
      x86_emit_xchgq_rax_mem_r11(c.o)
      return 0
  }
  if i.op == OP_ATOMIC_CAS_I64 {
      x86ctx_load_value_v(c, i.op0, "r11" as *u8)        // addr
      x86ctx_load_value_v(c, i.op1, "rax" as *u8)        // expected
      x86ctx_load_value_v(c, i.op2, "rcx" as *u8)        // new
      x86_emit_lock_cmpxchgq_rcx_mem_r11(c.o)
      x86_emit_sete_al(c.o)
      x86_emit_movzbq_rr(c.o, "al" as *u8, "rax" as *u8)
      x86ctx_store_result(c, i.result, "rax" as *u8)
      return 0
  }
  if i.op == OP_ATOMIC_FAA_I64 {
      x86ctx_load_value_v(c, i.op0, "r11" as *u8)        // addr
      x86ctx_load_value_v(c, i.op1, "rax" as *u8)        // delta (returns prior)
      x86_emit_lock_xaddq_rax_mem_r11(c.o)
      x86ctx_store_result(c, i.result, "rax" as *u8)
      return 0
  }
  if i.op == OP_ATOMIC_FENCE {
      x86_emit_mfence(c.o)
      return 0
  }
  out_str(c.o, "    # x86_64: atomic opcode not wired\n")
  return 0
1419}
1420
1421// G3 __adc_acc(acc_ptr, lo, hi): add the 128-bit (hi:lo) into the 3-word
1422// accumulator at acc_ptr with carry, as ONE contiguous addq;adcq;adcq block so
1423// CF stays live across the chain (no IR boundary can inject a flag-clobber).
1424// Operand loads (flag-safe movq/movabsq/leaq) FIRST; the block uses only
1425// movq/addq/adcq with constant disp(%r11) (movq preserves CF). Scratch
1426// r11/rax/rcx/rdx are all OUTSIDE the home pool {r12-r15,rbx}, so no homed value
1427// can be corrupted (same discipline as UMULHI/atomic-CAS/thread_clone).
1428func x86ctx_emit_adc_acc(c: *X86Ctx, i: *Instr) -> i64 {
  x86ctx_load_value_v(c, i.op0, "r11" as *u8)        // acc_ptr
  x86ctx_load_value_v(c, i.op1, "rax" as *u8)        // lo
  x86ctx_load_value_v(c, i.op2, "rcx" as *u8)        // hi
  out_str(c.o, "    movq 0(%r11), %rdx\n")           // rdx = acc0
  out_str(c.o, "    addq %rax, %rdx\n")               // acc0 += lo        -> CF
  out_str(c.o, "    movq %rdx, 0(%r11)\n")            // store acc0 (CF preserved)
  out_str(c.o, "    movq 8(%r11), %rdx\n")            // rdx = acc1 (CF preserved)
  out_str(c.o, "    adcq %rcx, %rdx\n")               // acc1 += hi + CF   -> CF
  out_str(c.o, "    movq %rdx, 8(%r11)\n")            // store acc1
  out_str(c.o, "    movq 16(%r11), %rdx\n")           // rdx = acc2 (CF preserved)
  out_str(c.o, "    adcq $0, %rdx\n")                 // acc2 += CF
  out_str(c.o, "    movq %rdx, 16(%r11)\n")           // store acc2
  G1_RAX_SLOT = 0 - 1                                 // FIX-3: block clobbered rax (no store_result to clear it)
  return 0
1443}
1444
1445// G3 gate: __cpuid_ebx(leaf, subleaf) -> the x86 EBX feature register. cpuid
1446// clobbers eax/ebx/ecx/edx, and rbx is a G1 callee-saved HOME -> save/restore it
1447// around the instruction (pushq/popq, net rsp unchanged; no call between, so
1448// 16-byte alignment is irrelevant for cpuid). leaf in eax, subleaf in ecx; the
1449// 32-bit ebx output is zero-extended into rbx, captured into rax.
1450func x86ctx_emit_cpuid_ebx(c: *X86Ctx, i: *Instr) -> i64 {
  x86ctx_load_value_v(c, i.op0, "rax" as *u8)         // leaf  -> eax
  x86ctx_load_value_v(c, i.op1, "rcx" as *u8)         // subleaf -> ecx
  out_str(c.o, "    pushq %rbx\n")                     // save the home (cpuid clobbers rbx)
  out_str(c.o, "    .byte 15, 162\n")                  // cpuid (0F A2) -- emitted as bytes; the
                                                       // sovereign nxasm has no `cpuid` mnemonic
  out_str(c.o, "    movq %rbx, %rax\n")                // capture EBX (zero-extended)
  out_str(c.o, "    popq %rbx\n")                      // restore the home
  x86ctx_store_result(c, i.result, "rax" as *u8)
  G1_RAX_SLOT = 0 - 1
  return 0
1461}
1462
1463// __thread_clone(stack_top, entry_fn, ctx) -> child_tid.  C bootstrap
1464// parity (x86_64.c emit_thread_clone).  SYS_clone(56) + child
1465// trampoline.  Unique label per emission = function name + result id.
1466func x86ctx_emit_clone_label(c: *X86Ctx, rid: i64) -> i64 {
  out_str(c.o, ".Lclone_parent_")
  let name: *u8 = c.f.name_start as *u8
  if name != (0 as *u8) { out_str(c.o, name) }
  out_char(c.o, 0x5F)        // '_'
  out_i64(c.o, rid)
  return 0
1473}
1474
1475func x86ctx_emit_thread_clone(c: *X86Ctx, i: *Instr) -> i64 {
  x86ctx_load_value_v(c, i.op0, "r11" as *u8)    // stack_top
  x86ctx_load_value_v(c, i.op1, "rax" as *u8)    // entry_fn
  x86ctx_load_value_v(c, i.op2, "rcx" as *u8)    // ctx
  out_str(c.o, "    movq %rax, -16(%r11)\n")     // [stk-16] = entry
  out_str(c.o, "    movq %rcx, -8(%r11)\n")      // [stk-8]  = ctx
  out_str(c.o, "    leaq -16(%r11), %rsi\n")     // rsi = child stack
  x86_emit_movabsq(c.o, "rdi" as *u8, 0x50f00)   // CLONE_VM|FS|FILES|SIGHAND|THREAD|SYSVSEM
  out_str(c.o, "    xorq %rdx, %rdx\n")
  out_str(c.o, "    xorq %r10, %r10\n")
  out_str(c.o, "    xorq %r8, %r8\n")
  x86_emit_movabsq(c.o, "rax" as *u8, 56)        // SYS_clone
  out_str(c.o, "    syscall\n")
  out_str(c.o, "    testq %rax, %rax\n")
  // `jne` (== `jnz`, both 0F 85) -- the rest of the compiler emits `jne`, and nxasm's jcc table only
  // knows `jne`; the trampoline previously emitted the `jnz` synonym, which nxasm rejected ("cannot
  // encode: jnz"), blocking sovereign-lane threading. Consistent mnemonic = assembles on both lanes.
  out_str(c.o, "    jne ")
  x86ctx_emit_clone_label(c, i.result)
  out_char(c.o, 0x0A)
  // child: rsp = stack_top-16; pop entry, ctx; call entry(ctx)
  out_str(c.o, "    movq 0(%rsp), %rax\n")       // entry
  out_str(c.o, "    movq 8(%rsp), %rdi\n")       // ctx -> arg0
  // `call *%rax` (indirect) -- nxasm's `call` dispatch handles the K_IND operand; the `callq` suffix
  // form was rejected ("cannot encode: callq *%rax"). GNU as accepts both, so this assembles on both lanes.
  out_str(c.o, "    call *%rax\n")
  out_str(c.o, "    movq %rax, %rdi\n")          // entry returned -> exit thread
  x86_emit_movabsq(c.o, "rax" as *u8, 60)        // SYS_exit
  out_str(c.o, "    syscall\n")
  x86ctx_emit_clone_label(c, i.result)
  out_str(c.o, ":\n")
  x86ctx_store_result(c, i.result, "rax" as *u8)
  return 0
1508}
1509
1510// ===== cmp dispatch ==============================================
1511
1512// LN16 (2026-08-25) -- IEEE-754 binary64 ORDERING KEY, computed in place in `reg`.
1513//
1514// THE DEFECT IT REMOVES. A double rides in an i64 register and x86ctx_emit_cmp
1515// lowered EVERY compare to a signed integer cmpq on the raw bits. binary64 is
1516// SIGN-MAGNITUDE, not two's complement, so among two NEGATIVE doubles the
1517// integer order is the exact REVERSE of the numeric order: bits(-1.0) =
1518// 0xBFF0000000000000 is arithmetically LESS than bits(-2.0) = 0xC000000000000000,
1519// so `0.0 - 1.0 < 0.0 - 2.0` evaluated TRUE. Silent, and invisible to every
1520// positive-only test -- which is why nx_f64_adversary, whose six checks are all
1521// non-negative, ran green over it since 2026-07-16.
1522//
1523// THE TRANSFORM. m = x >> 63 arithmetic (all ones iff the sign bit is set), then
1524//   key = (x XOR (m >>> 1)) - m
1525// For a non-negative double m is 0 and the key IS the bit pattern, which is
1526// already monotone. For a negative double m is -1, so the 63 magnitude bits are
1527// inverted (larger magnitude -> smaller key) while the sign bit stays set (every
1528// negative key stays below every non-negative one), and the final `- m` adds one,
1529// which is what maps -0.0 and +0.0 onto the SAME key. Signed zero comparing equal
1530// is the IEEE answer and is a second defect this fixes: the raw compare called
1531// them different. After the transform the EXISTING signed cmpq and the existing
1532// signed condition codes are correct, so nothing downstream changes.
1533//
1534// WHY A KEY AND NOT ucomisd. The SSE compare needs unsigned and parity condition
1535// codes that neither nx_x86_64's setcc set nor the sovereign assembler carries;
1536// adding them means editing nxasm, a binary every concurrent build lane forks.
1537// Every instruction below is one this backend already emits (the sarq/shrq pair
1538// is the same idiom as the signed-divide bias dance at OP_DIV_S), so the fix
1539// needs no assembler change and no new encoding.
1540//
1541// DECLARED IMPRECISION. A key is a TOTAL order and therefore cannot express IEEE
1542// unorderedness: a comparison against NaN answers as if NaN were an ordered
1543// value, where IEEE says every predicate except `not equal` is false. That is a
1544// strictly smaller residual than what it replaces (which was wrong for every
1545// negative operand) and it is NAMED here, in nx_f64_gate's verdict note and on
1546// the /compare/lang row rather than left for the next reader to find. Ordered-NaN
1547// costs nothing under the estate's integer-deterministic doctrine, where NaN
1548// cannot arise from the sanctioned paths at all.
1549//
1550// r10/r11 are the declared scratch registers (never in the home pool, never live
1551// across this sequence); `reg` is always rax or rcx here, both caller-saved, and
1552// the caller invalidates G1_RAX_SLOT because rax no longer holds its slot value.
1553// Cost: six integer instructions per f64 operand, on f64 compares only. The
1554// integer compare path is not touched, which nx_f64_gate proves by byte-comparing
1555// the assembly of an integer program against a reference compiler.
1556func x86ctx_emit_f64_ordkey(c: *X86Ctx, reg: *u8) -> i64 {
  out_str(c.o, "    movq %")
  out_str(c.o, reg)
  out_str(c.o, ", %r10\n")
  out_str(c.o, "    sarq $63, %r10\n")          // r10 = m: all ones iff negative
  out_str(c.o, "    movq %r10, %r11\n")
  out_str(c.o, "    shrq $1, %r11\n")           // r11 = 0x7fffffffffffffff iff negative
  out_str(c.o, "    xorq %r11, %")
  out_str(c.o, reg)
  out_char(c.o, 0x0A)                            // invert the magnitude bits
  out_str(c.o, "    subq %r10, %")
  out_str(c.o, reg)
  out_char(c.o, 0x0A)                            // + 1 iff negative: ties -0.0 to +0.0
  return 0
1570}
1571
1572// LN38 (2026-09-03): IEEE-754 UNORDERED compares. The ordering-key transform below is a TOTAL order on the
1573// bit pattern, so under it a NaN equalled itself and a sign-set NaN (the x86 default from 0.0/0.0) sorted
1574// BELOW every number: `n != n` read FALSE and `n < 1.0` read TRUE (found by the CM4 quadrature gate NaN
1575// neg-control). IEEE says every ordered predicate on a NaN operand is FALSE and only != is TRUE. The
1576// assembler has no ucomisd/jp, so the guard is integer-only: NaN iff exponent==2047 (bits<<1>>53) AND
1577// mantissa!=0 (bits<<12). When either operand is NaN the ordkey compare is skipped and FLAGS are forced to
1578// the ordered state that gives the IEEE answer for THIS predicate: a GREATER (1 vs 0) makes < <= == false
1579// and != true; a LESS (0 vs 1) makes > >= false. The setcc/branch-fusion tail and G12 inversion read the
1580// same flags unchanged. Labels are unique per emission: function name + result id.
1581func x86ctx_cmp_f64_operands(c: *X86Ctx, i: *Instr) -> i64 {
  x86ctx_load_value_v(c, i.op0, "rax" as *u8)
  x86ctx_load_value_v(c, i.op1, "rcx" as *u8)
  return 0
1585}
1586func x86ctx_emit_f64_nan_label(c: *X86Ctx, i: *Instr, tag: *u8) -> i64 {
  out_str(c.o, ".LN38")
  out_str(c.o, tag)
  out_char(c.o, 0x5F)
  let name: *u8 = c.f.name_start as *u8
  if name != (0 as *u8) { out_str(c.o, name) }
  out_char(c.o, 0x5F)
  out_i64(c.o, i.result)
  return 0
1595}
1596// NaN test on one operand register: jumps to the shared nan label iff REG holds a NaN, else falls through.
1597func x86ctx_emit_f64_nan_guard(c: *X86Ctx, i: *Instr, reg: *u8, oktag: *u8) -> i64 {
  out_str(c.o, "    movq %")
  out_str(c.o, reg)
  out_str(c.o, ", %r10\n")
  out_str(c.o, "    shlq $1, %r10\n")
  out_str(c.o, "    shrq $53, %r10\n")
  out_str(c.o, "    cmpq $2047, %r10\n")
  out_str(c.o, "    jne ")
  x86ctx_emit_f64_nan_label(c, i, oktag)
  out_char(c.o, 0x0A)
  out_str(c.o, "    movq %")
  out_str(c.o, reg)
  out_str(c.o, ", %r11\n")
  out_str(c.o, "    shlq $12, %r11\n")
  out_str(c.o, "    testq %r11, %r11\n")
  out_str(c.o, "    jne ")
  x86ctx_emit_f64_nan_label(c, i, "nan" as *u8)
  out_char(c.o, 0x0A)
  x86ctx_emit_f64_nan_label(c, i, oktag)
  out_str(c.o, ":\n")
  return 0
1618}
1619func x86ctx_emit_f64_cmp(c: *X86Ctx, i: *Instr, cc: i64) -> i64 {
  x86ctx_cmp_f64_operands(c, i)
  x86ctx_emit_f64_nan_guard(c, i, "rax" as *u8, "oka" as *u8)
  x86ctx_emit_f64_nan_guard(c, i, "rcx" as *u8, "okb" as *u8)
  x86ctx_emit_f64_ordkey(c, "rax" as *u8)
  x86ctx_emit_f64_ordkey(c, "rcx" as *u8)
  x86_emit_cmpq_rr(c.o, "rcx" as *u8, "rax" as *u8)
  out_str(c.o, "    jmp ")
  x86ctx_emit_f64_nan_label(c, i, "done" as *u8)
  out_char(c.o, 0x0A)
  x86ctx_emit_f64_nan_label(c, i, "nan" as *u8)
  out_str(c.o, ":\n")
  var force_less: i64 = 0
  if cc == NX_X64_CC_GT_S { force_less = 1 }
  if cc == NX_X64_CC_GE_S { force_less = 1 }
  if force_less == 1 {
  out_str(c.o, "    movq $0, %r10\n")
  out_str(c.o, "    cmpq $1, %r10\n")
  }
  if force_less == 0 {
  out_str(c.o, "    movq $1, %r10\n")
  out_str(c.o, "    cmpq $0, %r10\n")
  }
  x86ctx_emit_f64_nan_label(c, i, "done" as *u8)
  out_str(c.o, ":\n")
  return 0
1645}
1646
1647// 1 iff either operand of this compare is a binary64 value. The RESULT type of a
1648// compare is the boolean, so the precision has to come from the OPERANDS -- the
1649// same reason x86ctx_emit_float reads op0's type for OP_FCAST_F_TO_I. TY_F32 is
1650// deliberately NOT included: a f32 rides in the LOW 32 bits of the carrier, so its
1651// sign bit is 31 and this 64-bit key would be wrong for it. NishiLang has no f32
1652// type a program can declare (nx_parse: "a float rides as the low 32 bits of an
1653// i64 carrier"), so the reachable surface is the __f32_* intrinsic family only;
1654// naming that boundary here beats implying a coverage this has not earned.
1655func x86ctx_cmp_is_f64(c: *X86Ctx, i: *Instr) -> i64 {
  if X86_NEGCTL_F64_CMP_RAW == 1 { return 0 }
  let a: *Value = x86ctx_value_at(c.f, i.op0)
  if a.ty != (0 as *Type) { if a.ty.kind == TY_F64 { return 1 } }
  let b: *Value = x86ctx_value_at(c.f, i.op1)
  if b.ty != (0 as *Type) { if b.ty.kind == TY_F64 { return 1 } }
  return 0
1662}
1663
1664func x86ctx_cmp_to_cc(op: i64) -> i64 {
  if op == OP_EQ   { return NX_X64_CC_EQ }
  if op == OP_NE   { return NX_X64_CC_NE }
  if op == OP_LT_S { return NX_X64_CC_LT_S }
  if op == OP_LE_S { return NX_X64_CC_LE_S }
  if op == OP_GT_S { return NX_X64_CC_GT_S }
  if op == OP_GE_S { return NX_X64_CC_GE_S }
  return 0 - 1
1672}
1673
1674func x86ctx_emit_cmp(c: *X86Ctx, i: *Instr) -> i64 {
  let cc: i64 = x86ctx_cmp_to_cc(i.op)
  if cc < 0 {
      out_str(c.o, "    # x86_64: cmp opcode unknown\n")
      x86_emit_xorq_rr(c.o, "rax" as *u8, "rax" as *u8)
      x86ctx_store_result(c, i.result, "rax" as *u8)
      return 0
  }
  // G14 (2026-07-16): FLAG REUSE -- `cmp X, 0` for EQ/NE where X's producer
  // is the textually PREVIOUS instruction and an ALU op that sets ZF per its
  // result (ADD/SUB/AND/OR/XOR; MUL/shift flags are unreliable) is REDUNDANT
  // -- ZF already reflects X, and only movs (flag-neutral) are emitted after
  // the ALU. Skip BOTH the operand materialization and the cmpq; the fused
  // jcc / setcc tail below reads the live FLAGS. STRICTLY EQ/NE: signed
  // LT/GE read SF^OF, and the ALU's OF differs from cmp-vs-0's (witness:
  // (INT_MIN - 1) < 0 -- adversary T7 + the negctl prove the restriction).
  // A sib-dead producer emitted NOTHING (folded) -> its flags never set ->
  // excluded.
  var g14: i64 = 0
  if i.op == OP_EQ { g14 = 1 }
  if i.op == OP_NE { g14 = 1 }
  if X86_NEGCTL_G14_SIGNED == 1 { if i.op == OP_LT_S { g14 = 1 } }
  if g14 == 1 {
      g14 = 0
      let g14z: *Value = x86ctx_value_at(c.f, i.op1)
      if g14z.kind == VK_CONST_INT { if g14z.const_int == 0 {
          let g14p: *Instr = i.prev
          if g14p != (0 as *Instr) {
              if g14p.result == i.op0 {
                  var zok: i64 = 0
                  if g14p.op == OP_ADD { zok = 1 }
                  if g14p.op == OP_SUB { zok = 1 }
                  if g14p.op == OP_AND { zok = 1 }
                  if g14p.op == OP_OR  { zok = 1 }
                  if g14p.op == OP_XOR { zok = 1 }
                  if zok == 1 {
                      if g14p.result >= 0 { if g14p.result < c.f.n_values {
                          if c.sib_dead[g14p.result] == 1 { zok = 0 }
                      } }
                  }
                  if zok == 1 { g14 = 1 }
              }
          }
      } }
  }
  // ---- LN16 (2026-08-25): f64 ORDERING -----------------------------------
  // A binary64 compare cannot use the raw signed integer cmpq below -- see
  // x86ctx_emit_f64_ordkey for why, and for the two defects it removes. Both
  // operands are materialized into CALLER-SAVED scratch first: a homed or
  // forwarded operand may only be READ in place, and the key transform REWRITES
  // its register, so transforming r12 directly would corrupt that value for
  // every later consumer. Then map each to its ordering key, compare, and tell
  // the generic path that the flags are already live. The setcc / branch-fusion
  // tail below is untouched and needs no change: after the key the SIGNED
  // condition codes are exactly the right ones.
  if x86ctx_cmp_is_f64(c, i) == 1 {
      x86ctx_emit_f64_cmp(c, i, cc)
      G1_RAX_SLOT = 0 - 1
      g14 = 1
  }
  if g14 == 0 {
  // G10: a forwarded or HOMED op0 is compared IN its register directly (cmp
  // writes only FLAGS, never its operands) -- kills the movq %home,%rax.
  // Unforwarded/unhomed op0 keeps the rax materialization (byte-identical).
  var g6left: *u8 = x86_use_regalloc(c, i.op0)     // LN7: in-place read
  if g6left == (0 as *u8) {
      x86ctx_load_value_v(c, i.op0, "rax" as *u8)
      g6left = "rax" as *u8
  }
  // G5a: imm32 op1 -> cmpq $imm,%left; homed/forwarded op1 -> compare the
  // home reg directly; everything else keeps the rcx path.
  var g6imm: i64 = 0
  let g6v1: *Value = x86ctx_value_at(c.f, i.op1)
  if g6v1.kind == VK_CONST_INT {
      if g6v1.const_int <= 2147483647 {
          if g6v1.const_int >= (0 - 2147483648) { g6imm = 1 }
      }
  }
  if g6imm == 1 {
      out_str(c.o, "    cmpq $")
      out_i64(c.o, g6v1.const_int)
      out_str(c.o, ", %")
      out_str(c.o, g6left)
      out_char(c.o, 0x0A)
  }
  if g6imm == 0 {
      var g6src: *u8 = x86_use_regalloc(c, i.op1)     // LN7: in-place read
      if g6src == (0 as *u8) {
          x86ctx_load_value_v(c, i.op1, "rcx" as *u8)
          g6src = "rcx" as *u8
      }
      x86_emit_cmpq_rr(c.o, g6src, g6left)
  }
  }
  // G6: fuse into an immediately-following BR_COND on this result -- the
  // branch consumes the LIVE FLAGS; setcc/movzbq/store are skipped. The
  // elide flag already encodes single-use-in-next-instruction, so the
  // boolean cannot be observed anywhere else.
  if c.elide[i.result] == 1 {
      if i.next != (0 as *Instr) {
          if i.next.op == OP_BR_COND {
              if i.next.op0 == i.result {
                  G1_PENDING_CC = cc
                  G1_PENDING_VAL = i.result
                  return 0
              }
          }
      }
  }
  x86_emit_setcc(c.o, cc, "al" as *u8)
  x86_emit_movzbq_rr(c.o, "al" as *u8, "rax" as *u8)
  x86ctx_store_result(c, i.result, "rax" as *u8)
  return 0
1787}
1788
1789// ===== branch emission ============================================
1790
1791func x86ctx_emit_bb_label(c: *X86Ctx, bb_id: i64) -> i64 {
  out_str(c.o, ".L")
  let name: *u8 = c.f.name_start as *u8
  if name != (0 as *u8) { out_str(c.o, name) }
  out_str(c.o, "_bb")
  out_i64(c.o, bb_id)
  return 0
1798}
1799
1800func x86ctx_emit_jmp_to_bb(c: *X86Ctx, bb_id: i64) -> i64 {
  out_str(c.o, "    jmp ")
  x86ctx_emit_bb_label(c, bb_id)
  out_char(c.o, 0x0A)
  return 0
1805}
1806
1807func x86ctx_emit_jcc_to_bb(c: *X86Ctx, cc: i64, bb_id: i64) -> i64 {
  out_str(c.o, "    j")
  out_str(c.o, x86_cc_suffix(cc))
  out_char(c.o, 0x20)
  x86ctx_emit_bb_label(c, bb_id)
  out_char(c.o, 0x0A)
  return 0
1814}
1815
1816// G12: condition-code inversion (jcc T; jmp F -> j!cc F when T falls through).
1817// Pairs: EQ<->NE, LT_S<->GE_S, LE_S<->GT_S, LT_U<->GE_U, LE_U<->GT_U.
1818func x86ctx_cc_invert(cc: i64) -> i64 {
  if cc == NX_X64_CC_EQ   { return NX_X64_CC_NE }
  if cc == NX_X64_CC_NE   { return NX_X64_CC_EQ }
  if cc == NX_X64_CC_LT_S { return NX_X64_CC_GE_S }
  if cc == NX_X64_CC_LE_S { return NX_X64_CC_GT_S }
  if cc == NX_X64_CC_GT_S { return NX_X64_CC_LE_S }
  if cc == NX_X64_CC_GE_S { return NX_X64_CC_LT_S }
  if cc == NX_X64_CC_LT_U { return NX_X64_CC_GE_U }
  if cc == NX_X64_CC_LE_U { return NX_X64_CC_GT_U }
  if cc == NX_X64_CC_GT_U { return NX_X64_CC_LE_U }
  if cc == NX_X64_CC_GE_U { return NX_X64_CC_LT_U }
  if cc == NX_X64_CC_O    { return NX_X64_CC_NO }
  if cc == NX_X64_CC_NO   { return NX_X64_CC_O }
  return 0 - 1
1832}
1833
1834// G12: emit jcc T / jmp F with FALL-THROUGH ELISION against c.next_bb --
1835// the taken `jmp` per iteration in every branchy loop was 2-3 of the loop's
1836// instructions (gap tool). F==next -> jcc T only; T==next -> j!cc F only.
1837func x86ctx_emit_condjump(c: *X86Ctx, cc: i64, t: i64, fjb: i64) -> i64 {
  if fjb == c.next_bb {
      x86ctx_emit_jcc_to_bb(c, cc, t)
      return 0
  }
  if t == c.next_bb {
      let icc: i64 = x86ctx_cc_invert(cc)
      if icc >= 0 {
          x86ctx_emit_jcc_to_bb(c, icc, fjb)
          return 0
      }
  }
  x86ctx_emit_jcc_to_bb(c, cc, t)
  x86ctx_emit_jmp_to_bb(c, fjb)
  return 0
1852}
1853
1854// G15: block lookup by id (ids may differ from storage order post-opt).
1855func x86ctx_br_target_block(c: *X86Ctx, id: i64) -> *BasicBlock {
  var bi: i64 = 0
  while bi < c.f.n_blocks {
      let b: *BasicBlock = x86ctx_block_at(c.f, bi)
      if b.id == id { return b }
      bi = bi + 1
  }
  return 0 as *BasicBlock
1863}
1864
1865func x86ctx_emit_br(c: *X86Ctx, i: *Instr) -> i64 {
  // G12: unconditional jump to the very next emitted block = fall through.
  if i.op0 == c.next_bb { return 0 }
  // G15 peephole disabled 2026-08-01 see debt 1785569152
  x86ctx_emit_jmp_to_bb(c, i.op0)
  return 0
1871}
1872
1873func x86ctx_emit_br_cond(c: *X86Ctx, i: *Instr) -> i64 {
  // G6: consume a pending fused compare -- branch on the live flags. The
  // value-id match is the belt: any mismatch clears the pending state and
  // falls through to the load+test path (never a stale-flags branch).
  if G1_PENDING_CC >= 0 {
      if G1_PENDING_VAL == i.op0 {
          let g6cc: i64 = G1_PENDING_CC
          G1_PENDING_CC = 0 - 1
          G1_PENDING_VAL = 0 - 1
          x86ctx_emit_condjump(c, g6cc, i.op1, i.op2)
          return 0
      }
      G1_PENDING_CC = 0 - 1
      G1_PENDING_VAL = 0 - 1
  }
  x86ctx_load_value_v(c, i.op0, "rax" as *u8)
  x86_emit_testq_rr(c.o, "rax" as *u8, "rax" as *u8)
  x86ctx_emit_condjump(c, NX_X64_CC_NE, i.op1, i.op2)
  return 0
1892}
1893
1894// ===== load / store / GEP ========================================
1895
1896func x86ctx_type_size(t: *Type) -> i64 {
  if t == (0 as *Type) { return 8 }
  if t.kind == TY_VOID { return 8 }
  if t.size <= 0 { return 8 }
  return t.size
1901}
1902
1903func x86ctx_type_signed(t: *Type) -> i64 {
  // Default to UNSIGNED (0) when type is missing: a missing type
  // on a sub-word load is most often a `*u8`/`*u16` dereference
  // emitted without explicit type tagging.  Sign-extending those
  // corrupts values >= 0x80 to negative i64.  Witness: x509_parse
  // sign-extending the [0] EXPLICIT version tag 0xA0 -> -96 on
  // live Mozilla bundle (diagnosed 2026-05-20).  Unsigned default
  // is strictly safer because 0..127 round-trip either way and
  // 128..255 only preserve via zero-extend.
  if t == (0 as *Type) { return 0 }
  if t.kind == TY_I8  { return 1 }
  if t.kind == TY_I16 { return 1 }
  if t.kind == TY_I32 { return 1 }
  if t.kind == TY_I64 { return 1 }
  return 0
1918}
1919
1920// null-safe sext flag: 1 only for types minted from a SIGNED subword annotation (i8/i16/i32 via
1921// alloc_type_s). u8/u16/u32 and untyped loads stay 0 -> zero-extend (protects the x509 0xA0 witness).
1922func x86ctx_type_sext(t: *Type) -> i64 {
  if t == (0 as *Type) { return 0 }
  return t.sext
1925}
1926
1927func x86ctx_emit_load(c: *X86Ctx, i: *Instr) -> i64 {
  // G10: a home-forwarded load emits NOTHING -- its single consumer reads the
  // source alloca's home register directly (fwd scan proved no intervening
  // store/call; rax and FLAGS untouched, so G1/G6 state stays valid).
  if i.result >= 0 { if i.result < c.f.n_values {
      if c.fwd_home[i.result] >= 0 { return 0 }
  } }
  let sz: i64 = x86ctx_type_size(i.ty)
  // 2026-07-10 debt fix: subword loads SIGN-extend when the pointee was declared signed (sext=1,
  // set by the parser via alloc_type_s for i8/i16/i32) and ZERO-extend otherwise. Previously ALL
  // subword loads zero-extended because u8/i8 were indistinguishable at the IR level (no unsigned
  // kinds); the sext bit now carries the distinction, fixing *i8/*i16/*i32 (witness: SIMD hsum
  // ~4e9 garbage on negative int32 lanes) while leaving *u8 byte code (x509 0xA0 -> 160) untouched.
  // This also ALIGNS x86 with the RV64 backend, which already sign-extended signed loads.
  let sx: i64 = x86ctx_type_sext(i.ty)
  // STAGE 5 (2026-07-15): target the RESULT's home register (rax if unhomed) so
  // a homed 8-byte load lands directly in its reg -- no spill+reload. dst==rax
  // for every unhomed value and every subword load (only 8-byte loads are
  // homeable), so this is byte-identical wherever homing is off.
  let dst: *u8 = x86ctx_result_reg(c, i.result)
1947
  // G8: scaled-addressing fold. LOAD(GEP(base, SHL/MUL(idx,2^k))) single-use
  // -> movq (%base,%idx,scale),%dst (the GEP+SHL are elided). 8-byte only.
  if sz == 8 {
      let lout: *i64 = x86ctx_sib_scratch()
      if x86ctx_sib_probe(c, i.op0, lout) == 1 {
          let lb: i64 = lout[0]
          let li: i64 = lout[1]
          let lsc: i64 = lout[2]
          if lout[5] == 1 { x86ctx_load_value(c, lb, "rcx" as *u8) }      // GEP base = address
          if lout[5] == 0 { x86ctx_load_value_v(c, lb, "rcx" as *u8) }    // ADD base = pointer value
          // G16: a homed/forwarded INDEX is used in the SIB directly (homes
          // are never rcx/rax -> no clash with base or dst-as-rax).
          var lidx: *u8 = x86_use_regalloc(c, li)     // LN7: in-place SIB index
          if lidx == (0 as *u8) {
              x86ctx_load_value_v(c, li, "rax" as *u8)
              lidx = "rax" as *u8
          }
          out_str(c.o, "    movq ")
          x86ctx_emit_sib_mem(c, "rcx" as *u8, lidx, lsc)
          out_str(c.o, ", %")
          out_str(c.o, dst)
          out_char(c.o, 0x0A)
          x86ctx_store_result(c, i.result, dst)
          return 0
      }
  }
1974
  // Peephole 2026-05-20: when the pointer operand is an alloca-result,
  // fold `leaq disp(%rbp), %rcx ; mov(z)? (%rcx), %rax` into a direct
  // `mov(z)? disp(%rbp), %rax`.  No semantic change; eliminates one
  // instruction per stack-variable load -- the dominant pattern in
  // every inner loop measured by paired Stabilizer.
  if i.op0 >= 0 && i.op0 < c.f.n_values && c.alloca_off[i.op0] >= 0 {
      // G2: homed alloca read = one register move (the home IS the
      // storage). Eligibility guarantees sz==8 at every access of a
      // homed alloca, so the subword paths below cannot be reached.
      if c.alloca_home[i.op0] >= 0 {
          let g2h: *u8 = x86_home_reg_name(c.alloca_home[i.op0])
          x86_emit_movq_reg_reg(c.o, g2h, dst)
          x86ctx_store_result(c, i.result, dst)
          return 0
      }
      let abp_off: i64 = x86ctx_value_base(c) + c.alloca_off[i.op0]
      // STAGE 5 ROOT CAUSE (2026-08-14): these MUST write `dst`, not rax.
      // x86ctx_result_reg returns the HOME register for a homed value, and
      // store_result(result, dst) emits nothing when reg already IS the home --
      // so a subword load that wrote rax while dst was a home register left that
      // home NEVER WRITTEN and the result garbage. That is the "miscompiles the
      // compiler on some pattern the KATs don't hit" that reverted STAGE 5: the
      // compiler byte-walks source through *u8/*i32 loads constantly, while the
      // matmul KATs are all 8-byte and cannot reach these branches.
      // Inert until loads are homeable (dst == rax for every unhomed value).
      if sz == 1 {
          if sx == 1 { x86_emit_load_byte_signed(c.o, "rbp" as *u8, abp_off, dst) }
          else { x86_emit_load_byte_unsigned(c.o, "rbp" as *u8, abp_off, dst) }
      }
      if sz == 2 {
          if sx == 1 { x86_emit_load_word_signed(c.o, "rbp" as *u8, abp_off, dst) }
          else { x86_emit_load_word_unsigned(c.o, "rbp" as *u8, abp_off, dst) }
      }
      if sz == 4 {
          if sx == 1 { x86_emit_load_dword_signed(c.o, "rbp" as *u8, abp_off, dst) }
          else { x86_emit_load_dword_unsigned(c.o, "rbp" as *u8, abp_off, dst) }
      }
      if sz == 8 {
          x86_emit_load_qword(c.o, "rbp" as *u8, abp_off, dst)
      }
      x86ctx_store_result(c, i.result, dst)
      return 0
  }
2018
  x86ctx_load_value(c, i.op0, "rcx" as *u8)
  // Same STAGE 5 root cause as the alloca block above: write `dst`, never rax.
  if sz == 1 {
      // sext=1 (declared *i8) sign-extends via movsbq; else (*u8/untyped) zero-extends via movzbq --
      // the x509 0xA0-must-stay-160 witness lives on this default.
      if sx == 1 { x86_emit_load_byte_signed(c.o, "rcx" as *u8, 0, dst) }
      else { x86_emit_load_byte_unsigned(c.o, "rcx" as *u8, 0, dst) }
  }
  if sz == 2 {
      if sx == 1 { x86_emit_load_word_signed(c.o, "rcx" as *u8, 0, dst) }
      else { x86_emit_load_word_unsigned(c.o, "rcx" as *u8, 0, dst) }
  }
  if sz == 4 {
      if sx == 1 { x86_emit_load_dword_signed(c.o, "rcx" as *u8, 0, dst) }
      else { x86_emit_load_dword_unsigned(c.o, "rcx" as *u8, 0, dst) }
  }
  if sz == 8 {
      x86_emit_load_qword(c.o, "rcx" as *u8, 0, dst)
  }
  x86ctx_store_result(c, i.result, dst)
  return 0
2040}
2041
2042func x86ctx_emit_store(c: *X86Ctx, i: *Instr) -> i64 {
  let sz: i64 = x86ctx_type_size(i.ty)
2044
  // G8: scaled-addressing fold, symmetric to emit_load. STORE to
  // GEP(base, SHL/MUL(idx,2^k)) single-use -> movq %rax,(%base,%idx,scale).
  if sz == 8 {
      let sout: *i64 = x86ctx_sib_scratch()
      if x86ctx_sib_probe(c, i.op0, sout) == 1 {
          let sb: i64 = sout[0]
          let si: i64 = sout[1]
          let ssc: i64 = sout[2]
          if sout[5] == 1 { x86ctx_load_value(c, sb, "rcx" as *u8) }      // GEP base = address
          if sout[5] == 0 { x86ctx_load_value_v(c, sb, "rcx" as *u8) }    // ADD base = pointer value
          // G16: homed/forwarded INDEX and VALUE go into the SIB store
          // directly (homes never collide with rcx/rdx/rax).
          var sidx: *u8 = x86_use_regalloc(c, si)     // LN7: in-place SIB index
          if sidx == (0 as *u8) {
              x86ctx_load_value_v(c, si, "rdx" as *u8)
              sidx = "rdx" as *u8
          }
          var sval: *u8 = x86_use_regalloc(c, i.op1)     // LN7: in-place store value
          if sval == (0 as *u8) {
              if i.op1 >= 0 { if i.op1 < c.f.n_values {
                  if c.alloca_home[i.op1] >= 0 { sval = x86_home_reg_name(c.alloca_home[i.op1]) }
              } }
          }
          if sval == (0 as *u8) {
              x86ctx_load_value_v(c, i.op1, "rax" as *u8)
              sval = "rax" as *u8
          }
          out_str(c.o, "    movq %")
          out_str(c.o, sval)
          out_str(c.o, ", ")
          x86ctx_emit_sib_mem(c, "rcx" as *u8, sidx, ssc)
          out_char(c.o, 0x0A)
          return 0
      }
  }
2080
  // Peephole 2026-05-20 (symmetric to emit_load): when the destination
  // pointer is an alloca-result, fold `leaq disp(%rbp), %rcx ; movq
  // %rax, (%rcx)` into direct `movq %rax, disp(%rbp)`.  Eliminates
  // one instruction per stack-variable store.
  if i.op0 >= 0 && i.op0 < c.f.n_values && c.alloca_off[i.op0] >= 0 {
      // G2: homed alloca write = load the value straight into the home.
      // Direct forms (const/global/func-addr/homed-value/homed-alloca)
      // materialise INTO the home in one instruction with rax untouched
      // (G1_RAX_SLOT stays valid). Everything else rides the usual rax
      // path (G1 store->load forwarding applies) + one reg move.
      if c.alloca_home[i.op0] >= 0 {
          // G11: the chain's terminal store-back is a no-op -- the final
          // chain value was computed IN PLACE in this very home register.
          if i.op1 >= 0 { if i.op1 < c.f.n_values {
              if c.chain_home[i.op1] >= 0 {
                  if c.chain_home[i.op1] == c.alloca_home[i.op0] { return 0 }
              }
          } }
          let g2h: *u8 = x86_home_reg_name(c.alloca_home[i.op0])
          let g2v: *Value = x86ctx_value_at(c.f, i.op1)
          var g2direct: i64 = 0
          if g2v.kind == VK_CONST_INT { g2direct = 1 }
          if g2v.kind == VK_GLOBAL { g2direct = 1 }
          if g2v.kind == VK_FUNC_ADDR { g2direct = 1 }
          // G10/LN7: a homed or forwarded op1 materializes straight into the
          // destination home (movq %hsrc,%hdst -- one instruction, rax untouched).
          if x86_use_regalloc(c, i.op1) != (0 as *u8) { g2direct = 1 }
          if i.op1 >= 0 { if i.op1 < c.f.n_values {
              if c.alloca_home[i.op1] >= 0 { g2direct = 1 }
          } }
          if g2direct == 1 {
              x86ctx_load_value_v(c, i.op1, g2h)
              return 0
          }
          x86ctx_load_value_v(c, i.op1, "rax" as *u8)
          x86_emit_movq_reg_reg(c.o, "rax" as *u8, g2h)
          return 0
      }
      let abp_off: i64 = x86ctx_value_base(c) + c.alloca_off[i.op0]
      x86ctx_load_value_v(c, i.op1, "rax" as *u8)
      if sz == 1 { x86_emit_store_byte(c.o, "rax" as *u8, "rbp" as *u8, abp_off) }
      if sz == 2 { x86_emit_store_word(c.o, "rax" as *u8, "rbp" as *u8, abp_off) }
      if sz == 4 { x86_emit_store_dword(c.o, "rax" as *u8, "rbp" as *u8, abp_off) }
      if sz == 8 { x86_emit_store_qword(c.o, "rax" as *u8, "rbp" as *u8, abp_off) }
      return 0
  }
2127
  // op0 is the ADDRESS to store to -- keep load_value (gives address
  // when alloca-result; gives stored ptr when SSA value).
  x86ctx_load_value(c, i.op0, "rcx" as *u8)
  // op1 is the VALUE to store -- use _v so alloca-result is derefed.
  x86ctx_load_value_v(c, i.op1, "rax" as *u8)
  if sz == 1 { x86_emit_store_byte(c.o, "rax" as *u8, "rcx" as *u8, 0) }
  if sz == 2 { x86_emit_store_word(c.o, "rax" as *u8, "rcx" as *u8, 0) }
  if sz == 4 { x86_emit_store_dword(c.o, "rax" as *u8, "rcx" as *u8, 0) }
  if sz == 8 { x86_emit_store_qword(c.o, "rax" as *u8, "rcx" as *u8, 0) }
  return 0
2138}
2139
2140func x86ctx_emit_gep(c: *X86Ctx, i: *Instr) -> i64 {
  // op0 is the BASE ADDRESS. Load it AS-ADDRESS (x86ctx_load_value): for an
  // alloca result that emits `leaq storage(%rbp)` (the address). The as-VALUE
  // variant (load_value_v) DEREFERENCES an alloca (movq storage(%rbp)) -- which
  // read the struct's own bytes as the base pointer => wild store => SIGSEGV
  // (the tagged-enum constructor `Opt::Some(v)` bug, nx_probe_ctor). For every
  // non-alloca base the two variants are byte-identical, so this is a no-op
  // there and correct for alloca bases. op1 (the offset) stays as-value.
  // G8: a GEP folded into a SIB load/store is dead -- emit nothing.
  if i.result >= 0 { if i.result < c.f.n_values { if c.sib_dead[i.result] == 1 { return 0 } } }
  x86ctx_load_value(c, i.op0, "rax" as *u8)
  x86ctx_load_value_v(c, i.op1, "rcx" as *u8)
  x86_emit_gep_add(c.o, "rax" as *u8, "rcx" as *u8)
  x86ctx_store_result(c, i.result, "rax" as *u8)
  return 0
2155}
2156
2157// ===== load_value_v -- derefs alloca-results ======================
2158//
2159// The C-bootstrap parser inserts an implicit "load pointer from
2160// alloca" between an alloca-result and any pointer-USE.  The self-
2161// host parser (nx_parse.nx) does NOT.  So when our codegen sees an
2162// alloca-result Value being USED AS A VALUE (binop / cmp / call /
2163// return / etc.), we need to materialise the missing load here.
2164//
2165// load_value_v is the "as-value" variant; load_value is the "as-
2166// address" variant used by STORE op0 (where we want to store TO the
2167// alloca) and LOAD op0 (where the subsequent movq deref reads
2168// the stored value).
2169
2170// ===== syscall ====================================================
2171//
2172// nxc2 IR: op0 = syscall number, op1..op6 = up to 6 args (Linux ABI).
2173//
2174// The source typically imports nx_syscalls.nx which hard-codes RV64
2175// syscall numbers (read=63, write=64, mmap=222, exit=93, ...).  On
2176// x86_64 those numbers mean different things (or nothing).  When
2177// op0 is a VK_CONST_INT we translate at codegen time via the
2178// rv64->x86_64 map below.  When op0 is a runtime-computed value
2179// the translator is a no-op (caller already loaded x86_64 num).
2180
2181func x86ctx_rv64_to_x86_64_syscall(num: i64) -> i64 {
  if num == 63  { return 0 }     // read
  if num == 64  { return 1 }     // write
  if num == 56  { return 257 }   // openat
  if num == 57  { return 3 }     // close
  if num == 80  { return 5 }     // fstat
  if num == 93  { return 60 }    // exit
  if num == 222 { return 9 }     // mmap
  if num == 220 { return 56 }    // clone
  if num == 221 { return 59 }    // execve
  if num == 260 { return 61 }    // wait4
  if num == 95  { return 247 }   // waitid: WNOWAIT preserves worker identity until watchdog is reaped
  if num == 167 { return 157 }   // prctl: parent-death binding and subreaper ownership
  if num == 198 { return 41 }    // socket
  if num == 200 { return 49 }    // bind
  if num == 201 { return 50 }    // listen
  if num == 202 { return 43 }    // accept
  if num == 203 { return 42 }    // connect
  if num == 206 { return 44 }    // sendto
  if num == 207 { return 45 }    // recvfrom
  if num == 208 { return 54 }    // setsockopt
  if num == 154 { return 109 }   // setpgid -- MEASURED 2026-08-10: bounding a subject that FORKS
                                 // needs a PROCESS-GROUP kill, and rv64/asm-generic 154 fell
                                 // through `return num` to x86_64 154, which is not setpgid. The
                                 // call returned -38 ENOSYS, so the group-kill silently reached
                                 // NOTHING and a 2000ms bound still took 60,060ms. The same shape
                                 // as the getpid/fcntl rows below: an unmapped number is not an
                                 // error, it is a DIFFERENT syscall.
                                 // ACTIVATES ON THE NEXT nx_cc SELF-HOST REBUILD. Until then
                                 // gk_run_cap3's poll deadline bounds the wait without it, so
                                 // nothing depends on this row landing first.
  // ---- ROWS ADDED 2026-07-31 after auditing every rv64 const the runtime actually uses against
  // this table (nx_connect_sweep/xlate_audit). The translator's default is `return num`, so an
  // unmapped number is NOT an error -- it silently becomes a DIFFERENT x86_64 syscall. Each row
  // below was conflict-checked first: nothing calls __syscall with these as x86 numbers.
  // DELIBERATELY NOT ADDED, because live code passes them as X86 numbers already and a row would
  // BREAK it: 51 (nx_upnp_igd uses it as x86 getsockname) and 124 (nx_daemon_gate / nx_thread use
  // it as x86 getsid). Those two need their call sites disambiguated first -- filed, not guessed.
  // ---- 2026-08-01 round 2: the last two audited gaps, unblocked by disambiguating their call
  // sites first. 51 and 124 could not be mapped while live code still passed them as X86 numbers
  // (nx_upnp_igd as getsockname, nx_daemon_gate/nx_thread as getsid). Those now use the RV64
  // numbers 204/156, so each number means ONE thing and the real rows can land.
  if num == 204 { return 51 }    // getsockname (rv64 204 -> x86_64 51)
  if num == 156 { return 124 }   // getsid      (rv64 156 -> x86_64 124)
  if num == 51  { return 161 }   // chroot      (rv64 51  -> x86_64 161). nx_syscalls.nx claimed this
                                 // row shipped long ago; it did not. SYS_CHROOT was falling through
                                 // to x86_64 51 = getsockname on every container path.
  if num == 124 { return 24 }    // sched_yield (rv64 124 -> x86_64 24)
  if num == 210 { return 48 }    // shutdown: rv64 210 has NO x86_64 counterpart, so every
                                 // sys_shutdown() fell through to a nonexistent syscall. Called by
                                 // nx_h2_serve, nx_h2_serve_multi, nx_acme_http, nx_aw_sni_router --
                                 // a half-close that never happened.
  if num == 23  { return 32 }    // dup   (rv64 23 -> x86_64 32; was falling through to x86 select)
  if num == 40  { return 165 }   // mount (rv64 40 -> x86_64 165). nx_syscalls.nx's comment CLAIMS
                                 // this row "was added and shipped FIRST" -- it was not present.
  if num == 241 { return 298 }   // perf_event_open (rv64 241 -> x86_64 298)
  if num == 278 { return 318 }   // getrandom (rv64 278 -> x86_64 318; x86 278 is vmsplice)
  if num == 25  { return 72 }    // fcntl (rv64 25 -> x86_64 72): THE MISSING ROW. Fell through
                                 // `return num` to x86_64 25 = mremap, so EVERY nx_fcntl caller
                                 // silently got -EINVAL: F_GETFL/F_SETFL, and therefore
                                 // nx_fcntl_set_cloexec and nx_fcntl_set_nonblock, were no-ops that
                                 // REPORTED FAILURE nobody checked. Probed live 2026-07-31:
                                 // __syscall(25,fd,3,0)=-22 vs __syscall(72,fd,3,0)=2. Same class as
                                 // the flock row below (silent wrong-syscall via pass-through).
  if num == 73  { return 7 }     // poll (rv64 SYS_POLL const=73 -> x86_64 poll=7; sys_poll's 3-arg fds/nfds/timeout_ms maps directly; the unused 4th arg is harmless)
  if num == 24  { return 292 }   // dup3
  if num == 29  { return 16 }    // ioctl
  if num == 32  { return 73 }    // flock (rv64 32 -> x86_64 73): THE registry-write-race ROOT FIX --
                                 // was falling through `return num` to x86_64 32 = dup2 (a silent no-op
                                 // "lock"), so concurrent writers could not serialize. BOOTSTRAP_MAP s4#2 /
                                 // X-SYSXLATE-FLOCK. Activates on the next nx_cc self-host rebuild.
  if num == 59  { return 22 }    // pipe2 (rv64 59 = pipe2, x86_64 22 = pipe)
  if num == 134 { return 13 }    // rt_sigaction: asm-generic 134 -> x86_64 13
  if num == 113 { return 228 }   // clock_gettime
  if num == 115 { return 230 }   // clock_nanosleep
  if num == 78  { return 217 }   // getdents64 (legacy RV64 syscall number)
  if num == 61  { return 217 }   // getdents64 (current RV64 generic ABI)
  if num == 62  { return 8 }     // lseek
  if num == 33  { return 165 }   // mount (rv64) -> mount (x86_64)
  if num == 161 { return 161 }   // chroot
  if num == 41  { return 272 }   // unshare (rv64) -- mapping rough
  if num == 97  { return 272 }   // unshare (TRUE rv64 asm-generic 97 -> x86 272; R2-A containers.
                                 // nx_ns_probe proved 97 fell through to x86 getrlimit -> EINVAL.
                                 // NOTE 2026-06-09: rows 41/268/33 above are MISLABELED vs the real
                                 // asm-generic table (41=pivot_root, 268=setns, 40=mount) -- kept
                                 // untouched (callers may bind to them); audit + fix as R2-A rung-1.
  if num == 268 { return 155 }   // pivot_root
  if num == 39  { return 16 }    // umount2 (rv64) -- mapping rough
  if num == 116 { return 145 }   // syslog (rv64) -- mapping rough
  if num == 129 { return 62 }    // kill (rv64 129 -> x86_64 62) -- host control plane
  if num == 34  { return 258 }   // mkdirat (rv64 34 -> x86_64 258) -- doc-root creation
  if num == 52  { return 91 }    // fchmod: descriptor-bound permission updates (asm-generic -> x86-64).
  if num == 205 { return 52 }    // getpeername: migrated raw-52 caller retains socket semantics.
  if num == 53  { return 268 }   // fchmodat (rv64 53 -> x86_64 268) -- +x deployed binaries
  if num == 276 { return 316 }   // renameat2 (rv64 276 -> x86_64 316) -- atomic content publish
  if num == 98  { return 202 }   // futex (rv64 98 -> x86_64 202) -- spin-then-BLOCK thread pool
                                 // (2026-07-10): idle pool workers yield-spun forever (nx_chan_recv),
                                 // burning cores + ~1.5ms/dispatch scheduler tax. NOTE x86 202 was only
                                 // reachable as rv64 accept's TARGET before; raw __syscall(202) would
                                 // translate as rv64 accept -> 43. This row gives futex its lawful name.
  if num == 122 { return 203 }   // sched_setaffinity (rv64 122 -> x86_64 203) -- pin pteam workers
                                 // to distinct cores (gcc/OpenMP default; the barrier-quality lever).
                                 // NOTE raw __syscall(203) alone = rv64 CONNECT -> x86 42; the table row
                                 // is the only lawful route, like the futex 98->202 row above.
  if num == 123 { return 204 }   // sched_getaffinity (rv64 123 -> x86_64 204) -- hw cpu-count probe.
                                 // Root of the 2026-07-07 silent-1 bug: nx_hw used 122, which is rv64
                                 // sched_SETaffinity AND passes through to x86_64 setfsgid ("succeeds",
                                 // zero mask, popcount 0 -> 1 CPU reported -> every auto-sized pool
                                 // silently serial). nx_hw runs raw 204 until this row is blessed live,
                                 // then flips to the portable 123.
  // Process-identity family (asm-generic/rv64 -> x86_64). These were MISSING: rv64 getpid(172)
  // fell through `return num` to x86_64 172=iopl -> -ENOSYS. That silently filled a loop-singleton
  // pidfile with -38 and the gate FAILED OPEN -- two daemons raced (X-GETPID-XLATE, see
  // nx_gate_loop_register + nx_proc_kat T4). nx_signal/nx_swarm_queue/nx_model_lane(ml_pid_alive)
  // all wanted getpid via 172 and were silently broken; callers worked around it with a runtime
  // register (untranslated) or /proc/self/stat. These rows give the identity syscalls their lawful
  // name so a const 172/173 translates correctly ecosystem-wide.
  if num == 94  { return 231 }   // exit_group (rv64 94 -> x86_64 231). Was MISSING: 94 fell through
                                 // `return num` to x86_64 94 = getgroups, so a const-numbered
                                 // exit_group silently did NOT exit. sys_exit_group works only
                                 // because it hardcodes the raw x86 231. The compiler-injected
                                 // bounds trap (nx_parse emit_bounds_trap) needs the portable
                                 // RV64 name so the same IR is correct on both backends.
  if num == 172 { return 39 }    // getpid   (rv64 172 -> x86_64 39)
  if num == 173 { return 110 }   // getppid  (rv64 173 -> x86_64 110)
  if num == 174 { return 102 }   // getuid   (rv64 174 -> x86_64 102)
  if num == 175 { return 107 }   // geteuid  (rv64 175 -> x86_64 107)
  if num == 176 { return 104 }   // getgid   (rv64 176 -> x86_64 104)
  if num == 177 { return 108 }   // getegid  (rv64 177 -> x86_64 108)
  if num == 233 { return 28 }    // madvise (rv64/asm-generic 233 -> x86_64 28). Was MISSING: a
                                 // portable const falls through `return num` to x86_64 233 =
                                 // epoll_ctl (the wrong-syscall-not-an-error class, same as the
                                 // setpgid/flock rows above). sys_madvise (nx_syscalls.nx) runs
                                 // the RAW x86 28 until this row is blessed by the next nx_cc
                                 // self-host rebuild -- flip it to the portable const then.
                                 // Conflict-checked 2026-08-12: nothing calls __syscall(233,...)
                                 // and nothing passes 28 as an x86 number.
  return num   // unknown -> pass through (will likely fail at runtime)
2319}
2320
2321func x86ctx_emit_syscall(c: *X86Ctx, i: *Instr) -> i64 {
  let n_args: i64 = i.n_operands - 1
  if n_args >= 1 { x86ctx_load_value_v(c, i.op1, "rdi" as *u8) }
  if n_args >= 2 { x86ctx_load_value_v(c, i.op2, "rsi" as *u8) }
  if n_args >= 3 { x86ctx_load_value_v(c, i.op3, "rdx" as *u8) }
  if n_args >= 4 { x86ctx_load_value_v(c, i.op4, "r10" as *u8) }
  if n_args >= 5 { x86ctx_load_value_v(c, i.op5, "r8"  as *u8) }
  if n_args >= 6 { x86ctx_load_value_v(c, i.op6, "r9"  as *u8) }
2329
  // Translate the syscall number if it's a compile-time constant.
  let num_val: *Value = x86ctx_value_at(c.f, i.op0)
  if num_val.kind == VK_CONST_INT {
      let x86_num: i64 = x86ctx_rv64_to_x86_64_syscall(num_val.const_int)
      x86_emit_movabsq(c.o, "rax" as *u8, x86_num)
  }
  if num_val.kind != VK_CONST_INT {
      // Runtime-computed syscall number -- load as-is.  Caller is
      // responsible for using x86_64 numbers when this path runs.
      x86ctx_load_value(c, i.op0, "rax" as *u8)
  }
  x86_emit_syscall(c.o)
  x86ctx_store_result(c, i.result, "rax" as *u8)
  return 0
2344}
2345
2346// ===== call =======================================================
2347//
2348// V2 (2026-05-20, Task #93): arity 1..8 supported.  Args 1..6 in
2349// registers (SysV ABI rdi/rsi/rdx/rcx/r8/r9); args 7..8 pushed on
2350// stack in REVERSE order (rightmost first), as required by SysV.
2351// Indirect call still deferred.
2352//
2353// Witness of fix:
2354//   - tls13_server_hello_parse (7 args) -- recv_sh SIGSEGV
2355//   - nx_http_resp_parse_header_line (8 args) -- response parse SEGV
2356// Both were dropping args 7+ silently.  Callee read garbage from
2357// stack slots above saved RA, dereferenced as pointers, NULL-faulted.
2358//
2359// Stack alignment: SysV requires 16-aligned %rsp at the call
2360// instruction.  When n_stack_args is odd we push an 8-byte pad
2361// to maintain alignment.
2362
2363func x86ctx_emit_call(c: *X86Ctx, i: *Instr) -> i64 {
  let n_args: i64 = i.n_operands
2365
  // Load register args first (1..6).
  if n_args >= 1 { x86ctx_load_value_v(c, i.op0, "rdi" as *u8) }
  if n_args >= 2 { x86ctx_load_value_v(c, i.op1, "rsi" as *u8) }
  if n_args >= 3 { x86ctx_load_value_v(c, i.op2, "rdx" as *u8) }
  if n_args >= 4 { x86ctx_load_value_v(c, i.op3, "rcx" as *u8) }
  if n_args >= 5 { x86ctx_load_value_v(c, i.op4, "r8"  as *u8) }
  if n_args >= 6 { x86ctx_load_value_v(c, i.op5, "r9"  as *u8) }
2373
  // Stack args 7..24.  n_stack_args bytes pushed; if odd, prepend
  // an 8-byte alignment pad so rsp stays 16-aligned at the call.
  var n_stack_args: i64 = 0
  if n_args >= 7  { n_stack_args = n_stack_args + 1 }
  if n_args >= 8  { n_stack_args = n_stack_args + 1 }
  if n_args >= 9  { n_stack_args = n_stack_args + 1 }
  if n_args >= 10 { n_stack_args = n_stack_args + 1 }
  if n_args >= 11 { n_stack_args = n_stack_args + 1 }
  if n_args >= 12 { n_stack_args = n_stack_args + 1 }
  if n_args >= 13 { n_stack_args = n_stack_args + 1 }
  if n_args >= 14 { n_stack_args = n_stack_args + 1 }
  if n_args >= 15 { n_stack_args = n_stack_args + 1 }
  if n_args >= 16 { n_stack_args = n_stack_args + 1 }
  if n_args >= 17 { n_stack_args = n_stack_args + 1 }
  if n_args >= 18 { n_stack_args = n_stack_args + 1 }
  if n_args >= 19 { n_stack_args = n_stack_args + 1 }
  if n_args >= 20 { n_stack_args = n_stack_args + 1 }
  if n_args >= 21 { n_stack_args = n_stack_args + 1 }
  if n_args >= 22 { n_stack_args = n_stack_args + 1 }
  if n_args >= 23 { n_stack_args = n_stack_args + 1 }
  if n_args >= 24 { n_stack_args = n_stack_args + 1 }
2395
  var pad: i64 = 0
  if n_stack_args > 0 {
      if (n_stack_args & 1) == 1 {
          pad = 8
          out_str(c.o, "    subq $8, %rsp\n")
      }
  }
2403
  // Push in REVERSE order: arg N first (highest), down to arg 7.
  if n_args >= 24 { x86ctx_load_value_v(c, i.op23, "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 23 { x86ctx_load_value_v(c, i.op22, "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 22 { x86ctx_load_value_v(c, i.op21, "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 21 { x86ctx_load_value_v(c, i.op20, "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 20 { x86ctx_load_value_v(c, i.op19, "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 19 { x86ctx_load_value_v(c, i.op18, "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 18 { x86ctx_load_value_v(c, i.op17, "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 17 { x86ctx_load_value_v(c, i.op16, "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 16 { x86ctx_load_value_v(c, i.op15, "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 15 { x86ctx_load_value_v(c, i.op14, "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 14 { x86ctx_load_value_v(c, i.op13, "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 13 { x86ctx_load_value_v(c, i.op12, "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 12 { x86ctx_load_value_v(c, i.op11, "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 11 { x86ctx_load_value_v(c, i.op10, "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 10 { x86ctx_load_value_v(c, i.op9,  "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 9  { x86ctx_load_value_v(c, i.op8,  "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 8  { x86ctx_load_value_v(c, i.op7,  "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 7  { x86ctx_load_value_v(c, i.op6,  "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
2423
  if i.callee != (0 as *Function) {
      let cn: *u8 = i.callee.name_start as *u8
      x86_emit_call_label(c.o, cn)
      x86ctx_store_result(c, i.result, "rax" as *u8)
  }
  if i.callee == (0 as *Function) {
      out_str(c.o, "    # x86_64: indirect call deferred\n")
  }
2432
  // Restore stack: pop stack args + alignment pad.
  let pop_bytes: i64 = n_stack_args * 8 + pad
  if pop_bytes > 0 {
      out_str(c.o, "    addq $")
      out_i64(c.o, pop_bytes)
      out_str(c.o, ", %rsp\n")
  }
  return 0
2441}
2442
2443// fp(args) -- INDIRECT call through a func-pointer VALUE (op0). Args are op1.. (SysV rdi..r9; this MVP caps at
2444// 6 register args -- thread_pool/callbacks use <=6, no stack args). The fn-ptr is spilled to the stack across
2445// the arg-register loads so none can clobber it, then popped into r11 (caller-saved, NOT an arg reg). The
2446// push+pop are balanced, so rsp stays 16-aligned at the call.
2447func x86ctx_emit_call_indirect(c: *X86Ctx, i: *Instr) -> i64 {
  let n_args: i64 = i.n_operands - 1
  // LOUD-FAIL GUARD: the IR carries op0 (fn-ptr) + op1..op23, so 23 is the hard
  // ceiling.  Fail the BUILD, never drop an argument -- dropping is precisely the
  // defect this function shipped with (see the stack-arg note below).
  if n_args > 23 {
      out_str(c.o, "    .error \"nx x86: indirect call with >23 args (IR operand cap)\"\n")
      return 0
  }
  // The fn-ptr is spilled across the arg-register loads so none can clobber it,
  // then popped into r11 -- reserved scratch, never in the allocation pool
  // (FIX-12 invariant).  The pop happens BEFORE the stack args are pushed so those
  // pushes land directly above the return address, where SysV expects them.
  x86ctx_load_value_v(c, i.op0, "rax" as *u8)
  out_str(c.o, "    pushq %rax\n")
  if n_args >= 1 { x86ctx_load_value_v(c, i.op1, "rdi" as *u8) }
  if n_args >= 2 { x86ctx_load_value_v(c, i.op2, "rsi" as *u8) }
  if n_args >= 3 { x86ctx_load_value_v(c, i.op3, "rdx" as *u8) }
  if n_args >= 4 { x86ctx_load_value_v(c, i.op4, "rcx" as *u8) }
  if n_args >= 5 { x86ctx_load_value_v(c, i.op5, "r8"  as *u8) }
  if n_args >= 6 { x86ctx_load_value_v(c, i.op6, "r9"  as *u8) }
  out_str(c.o, "    popq %r11\n")
2469
  // STACK ARGS 7..23 (2026-07-25).  This emitter used to stop at r9 and SILENTLY
  // DROP every argument past the 6th -- the identical failure that cost
  // tls13_server_hello_parse and nx_http_resp_parse_header_line a SEGV each on the
  // DIRECT path (fixed there 2026-06-10; the indirect path kept the bug, and only
  // the parser's 6-arg cap kept it off the road).  Mirrors the direct path exactly:
  // rightmost-first pushes, plus an 8-byte pad when the count is odd so rsp is
  // 16-aligned at the call.  The fn-ptr push/pop above cancel out, so rsp is back
  // to its entry alignment here and the direct path's pad rule applies unchanged.
  var n_stack_args: i64 = 0
  if n_args > 6 { n_stack_args = n_args - 6 }
  var pad: i64 = 0
  if n_stack_args > 0 {
      if (n_stack_args & 1) == 1 {
          pad = 8
          out_str(c.o, "    subq $8, %rsp\n")
      }
  }
  if n_args >= 23 { x86ctx_load_value_v(c, i.op23, "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 22 { x86ctx_load_value_v(c, i.op22, "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 21 { x86ctx_load_value_v(c, i.op21, "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 20 { x86ctx_load_value_v(c, i.op20, "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 19 { x86ctx_load_value_v(c, i.op19, "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 18 { x86ctx_load_value_v(c, i.op18, "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 17 { x86ctx_load_value_v(c, i.op17, "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 16 { x86ctx_load_value_v(c, i.op16, "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 15 { x86ctx_load_value_v(c, i.op15, "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 14 { x86ctx_load_value_v(c, i.op14, "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 13 { x86ctx_load_value_v(c, i.op13, "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 12 { x86ctx_load_value_v(c, i.op12, "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 11 { x86ctx_load_value_v(c, i.op11, "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 10 { x86ctx_load_value_v(c, i.op10, "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 9  { x86ctx_load_value_v(c, i.op9,  "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 8  { x86ctx_load_value_v(c, i.op8,  "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
  if n_args >= 7  { x86ctx_load_value_v(c, i.op7,  "rax" as *u8); out_str(c.o, "    pushq %rax\n") }
2504
  x86_emit_call_indirect(c.o, "r11" as *u8)
  x86ctx_store_result(c, i.result, "rax" as *u8)
2507
  // Restore stack: pop stack args + alignment pad.
  let pop_bytes: i64 = n_stack_args * 8 + pad
  if pop_bytes > 0 {
      out_str(c.o, "    addq $")
      out_i64(c.o, pop_bytes)
      out_str(c.o, ", %rsp\n")
  }
  return 0
2516}
2517
2518// ===== tail_call (session 6b) =====================================
2519//
2520// Same arg-loading as call, but emit `jmp <label>` after tearing
2521// down the current frame.  The callee will execute its own
2522// prologue + epilogue + ret; control returns directly to OUR caller.
2523
2524func x86ctx_emit_tail_call(c: *X86Ctx, i: *Instr) -> i64 {
  let n_args: i64 = i.n_operands
  // LOUD-FAIL GUARD (2026-06-10): stack args (7th+) are impossible
  // after frame teardown; silently dropping them zeroed every TLS
  // Derive-Secret (see _arg7_minrepro.nx).  opt_tail_call no longer
  // converts these, but if one reaches us, fail the BUILD, not the
  // runtime.
  if n_args > 6 {
      out_str(c.o, "    .error \"nx x86: tail_call with >6 args (stack args would be dropped)\"\n")
      return 0
  }
  if n_args >= 1 { x86ctx_load_value_v(c, i.op0, "rdi" as *u8) }
  if n_args >= 2 { x86ctx_load_value_v(c, i.op1, "rsi" as *u8) }
  if n_args >= 3 { x86ctx_load_value_v(c, i.op2, "rdx" as *u8) }
  if n_args >= 4 { x86ctx_load_value_v(c, i.op3, "rcx" as *u8) }
  if n_args >= 5 { x86ctx_load_value_v(c, i.op4, "r8"  as *u8) }
  if n_args >= 6 { x86ctx_load_value_v(c, i.op5, "r9"  as *u8) }
  // G1 FIX-B: this path bypasses x86_emit_epilogue -> restore callee-saved
  // homes here, AFTER the arg reads, BEFORE teardown, or the caller's homes
  // are corrupted (self-host-fatal).
  x86ctx_emit_cs_restore(c)
  // Tear down our frame: restore rsp + rbp, then jmp (not call).
  out_str(c.o, "    movq %rbp, %rsp\n")
  out_str(c.o, "    popq %rbp\n")
  if i.callee != (0 as *Function) {
      let cn: *u8 = i.callee.name_start as *u8
      x86_emit_tail_call_label(c.o, cn)
  }
  if i.callee == (0 as *Function) {
      out_str(c.o, "    # x86_64: indirect tail_call deferred\n")
  }
  return 0
2556}
2557
2558// ===== copy =======================================================
2559
2560func x86ctx_emit_copy(c: *X86Ctx, i: *Instr) -> i64 {
  x86ctx_load_value_v(c, i.op0, "rax" as *u8)
  x86ctx_store_result(c, i.result, "rax" as *u8)
  return 0
2564}
2565
2566// ===== return =====================================================
2567
2568func x86ctx_emit_return(c: *X86Ctx, i: *Instr) -> i64 {
  if i.n_operands >= 1 {
      x86ctx_load_value_v(c, i.op0, "rax" as *u8)
  }
  if i.n_operands == 0 {
      x86_emit_movabsq(c.o, "rax" as *u8, 0)
  }
  x86ctx_emit_cs_restore(c)
  x86_emit_epilogue(c.o)
  return 0
2578}
2579
2580// ===== hardware f32 (SSE scalar-single) ===========================
2581// f32 values are i64-CARRIED bit-patterns (the float lives in the low 32 bits;
2582// NishiLang has no f32 type). Shuttle GPR<->xmm through the SysV red zone (-8(%rsp);
2583// the sequence is call-free, so the 128-byte red zone is safe transient scratch).
2584// rax/rcx are caller-saved scratch (the same regs the integer binop uses) and
2585// xmm0/xmm1 are outside the GPR allocator, so nothing live is clobbered. The dispatch
2586// sets G1_RAX_SLOT=-1 before calling so the rax-tracking peephole is invalidated.
2587func x86ctx_emit_f32(c: *X86Ctx, i: *Instr) -> i64 {
  let op: i64 = i.op
  if op == OP_FCAST_I_TO_F {                          // i64 int -> f32 bits (cvtsi2ss)
      x86ctx_load_value_v(c, i.op0, "rax" as *u8)
      out_str(c.o, "    cvtsi2ss %rax, %xmm0\n")
      out_str(c.o, "    movss %xmm0, -8(%rsp)\n")
      out_str(c.o, "    movl -8(%rsp), %eax\n")
      x86ctx_store_result(c, i.result, "rax" as *u8)
      return 0
  }
  if op == OP_FCAST_F_TO_I {                          // f32 bits -> i64 int, truncate (cvttss2si)
      x86ctx_load_value_v(c, i.op0, "rax" as *u8)
      out_str(c.o, "    movq %rax, -8(%rsp)\n")
      out_str(c.o, "    movss -8(%rsp), %xmm0\n")
      out_str(c.o, "    cvttss2si %xmm0, %rax\n")
      x86ctx_store_result(c, i.result, "rax" as *u8)
      return 0
  }
  // binary: op0,op1 are f32 bits -> xmm0,xmm1 -> SSE compute -> bits back to rax.
  x86ctx_load_value_v(c, i.op0, "rax" as *u8)
  x86ctx_load_value_v(c, i.op1, "rcx" as *u8)
  out_str(c.o, "    movq %rax, -8(%rsp)\n")
  out_str(c.o, "    movss -8(%rsp), %xmm0\n")
  out_str(c.o, "    movq %rcx, -8(%rsp)\n")
  out_str(c.o, "    movss -8(%rsp), %xmm1\n")
  if op == OP_FADD { out_str(c.o, "    addss %xmm1, %xmm0\n") }
  if op == OP_FSUB { out_str(c.o, "    subss %xmm1, %xmm0\n") }
  if op == OP_FMUL { out_str(c.o, "    mulss %xmm1, %xmm0\n") }
  if op == OP_FDIV { out_str(c.o, "    divss %xmm1, %xmm0\n") }
  out_str(c.o, "    movss %xmm0, -8(%rsp)\n")
  out_str(c.o, "    movl -8(%rsp), %eax\n")
  x86ctx_store_result(c, i.result, "rax" as *u8)
  return 0
2620}
2621
2622// ===== hardware f64 (SSE scalar-double) ===========================
2623// f64 values are i64-carried bit-patterns (the double IS the full 64 bits).
2624// Same red-zone shuttle discipline as x86ctx_emit_f32 but with double-precision
2625// instructions (movsd/addsd/... + sqrtsd + cvtsi2sd/cvttsd2si). Dispatched from
2626// the binop router when i.ty.kind == TY_F64 so f64 arithmetic keeps full
2627// precision (the pre-2026-07-16 path forced everything through movss = silent
2628// truncation to f32). rax/rcx caller-saved scratch, xmm0/xmm1 outside the GPR
2629// allocator; G1_RAX_SLOT invalidated by the caller.
2630func x86ctx_emit_f64(c: *X86Ctx, i: *Instr) -> i64 {
  // f64-in-registers (2026-07-16): DIRECT GPR<->xmm movq (SSE2 66 REX.W 0F
  // 6E/7E) replaces the red-zone memory shuttle -- ~7 instrs/op -> ~4, and the
  // memory round-trip (a false dependency chain through -8(%rsp)) is gone. The
  // named spectral-norm perf rung; bit-exact (nx_f64_adversary + the matrix's
  // 1274219991 checksum unchanged). GNU as also accepts `movq %rax,%xmm0`.
  let op: i64 = i.op
  if op == OP_FCAST_I_TO_F {                          // i64 int -> f64 bits (cvtsi2sd)
      x86ctx_load_value_v(c, i.op0, "rax" as *u8)
      out_str(c.o, "    cvtsi2sd %rax, %xmm0\n")
      out_str(c.o, "    movq %xmm0, %rax\n")
      x86ctx_store_result(c, i.result, "rax" as *u8)
      return 0
  }
  if op == OP_FCAST_F_TO_I {                          // f64 bits -> i64 int, truncate (cvttsd2si)
      x86ctx_load_value_v(c, i.op0, "rax" as *u8)
      out_str(c.o, "    movq %rax, %xmm0\n")
      out_str(c.o, "    cvttsd2si %xmm0, %rax\n")
      x86ctx_store_result(c, i.result, "rax" as *u8)
      return 0
  }
  if op == OP_FSQRT {                                 // f64 sqrt (sqrtsd), unary
      x86ctx_load_value_v(c, i.op0, "rax" as *u8)
      out_str(c.o, "    movq %rax, %xmm0\n")
      out_str(c.o, "    sqrtsd %xmm0, %xmm0\n")
      out_str(c.o, "    movq %xmm0, %rax\n")
      x86ctx_store_result(c, i.result, "rax" as *u8)
      return 0
  }
  // binary: op0,op1 are f64 bits -> xmm0,xmm1 -> SSE double compute -> bits back.
  x86ctx_load_value_v(c, i.op0, "rax" as *u8)
  x86ctx_load_value_v(c, i.op1, "rcx" as *u8)
  out_str(c.o, "    movq %rax, %xmm0\n")
  out_str(c.o, "    movq %rcx, %xmm1\n")
  if op == OP_FADD { out_str(c.o, "    addsd %xmm1, %xmm0\n") }
  if op == OP_FSUB { out_str(c.o, "    subsd %xmm1, %xmm0\n") }
  if op == OP_FMUL { out_str(c.o, "    mulsd %xmm1, %xmm0\n") }
  if op == OP_FDIV { out_str(c.o, "    divsd %xmm1, %xmm0\n") }
  out_str(c.o, "    movq %xmm0, %rax\n")
  x86ctx_store_result(c, i.result, "rax" as *u8)
  return 0
2671}
2672
2673// Route an fp op to f64 or f32 codegen by its float precision. For most ops the
2674// RESULT type (i.ty) is the float type; for FCAST_F_TO_I the result is INT, so
2675// the precision comes from the SOURCE operand (op0) instead.
2676func x86ctx_emit_float(c: *X86Ctx, i: *Instr) -> i64 {
  var is64: i64 = 0
  if i.op == OP_FCAST_F_TO_I {
      let sv: *Value = x86ctx_value_at(c.f, i.op0)
      if sv.ty != (0 as *Type) { if sv.ty.kind == TY_F64 { is64 = 1 } }
  }
  if i.op != OP_FCAST_F_TO_I {
      if i.ty != (0 as *Type) { if i.ty.kind == TY_F64 { is64 = 1 } }
  }
  if is64 == 1 { return x86ctx_emit_f64(c, i) }
  return x86ctx_emit_f32(c, i)
2687}
2688
2689// PACKED f32x4 dot: op0,op1 are pointers to 4 CONTIGUOUS 4-byte f32.  movups loads 4 lanes each;
2690// mulps multiplies all 4 pairs in ONE instruction (vs 4 scalar mulss); scalar horizontal-sum via
2691// movss+addss (sidesteps the shufps operand-parse bug, uses only verified ops) -> f32 bits in eax.
2692// This is the compute-physics lever: 4 f32 MACs per mulps instead of 1 per mulss.
2693func x86ctx_emit_f32x4_dot(c: *X86Ctx, i: *Instr) -> i64 {
  x86ctx_load_value_v(c, i.op0, "rax" as *u8)        // rax = a_ptr
  x86ctx_load_value_v(c, i.op1, "rcx" as *u8)        // rcx = b_ptr
  out_str(c.o, "    movups (%rax), %xmm0\n")          // xmm0 = a[0..3]
  out_str(c.o, "    movups (%rcx), %xmm1\n")          // xmm1 = b[0..3]
  out_str(c.o, "    mulps %xmm1, %xmm0\n")            // xmm0 = 4 products (one instruction)
  out_str(c.o, "    movups %xmm0, -16(%rsp)\n")       // spill the 4 lanes to the red zone
  out_str(c.o, "    movss -16(%rsp), %xmm0\n")        // lane0
  out_str(c.o, "    movss -12(%rsp), %xmm1\n")        // lane1
  out_str(c.o, "    addss %xmm1, %xmm0\n")            // +lane1
  out_str(c.o, "    movss -8(%rsp), %xmm1\n")         // lane2
  out_str(c.o, "    addss %xmm1, %xmm0\n")            // +lane2
  out_str(c.o, "    movss -4(%rsp), %xmm1\n")         // lane3
  out_str(c.o, "    addss %xmm1, %xmm0\n")            // +lane3 = dot
  out_str(c.o, "    movss %xmm0, -8(%rsp)\n")         // store result f32 bits
  out_str(c.o, "    movl -8(%rsp), %eax\n")           // eax = f32 bits
  x86ctx_store_result(c, i.result, "rax" as *u8)
  return 0
2711}
2712
2713// PACKED f32x8 dot (AVX2 8-wide): op0,op1 -> 8 contiguous 4-byte f32 each.  vmovups+vmulps do 8 lanes
2714// per instruction (2x the SSE width); vextractf128 folds hi4+lo4 -> xmm, vzeroupper clears the AVX-SSE
2715// transition penalty, then the PROVEN SSE scalar hsum.  VEX bytes via .byte (proven nxasm_vex_kat 7/7;
2716// regs fixed ymm0/ymm1 + ptrs in rax/rcx so every ModRM is constant) -- no nxasm ymm-parser needed.
2717// __f32_i8dot32(a:*i8[32], b:*f32[32]) -> f32 = sum_{j<32} (sext(a[j])) * b[j].
2718// The Q8_0/quantized dequant-dot lever: 32 int8 sign-extended + converted +
2719// multiplied by 32 f32, all SSE, unrolled x8 (4 lanes/iter), hsum once. New
2720// SSE ops (pmovsxbd/cvtdq2ps/movd/mulps/addps/xorps/movaps) emitted as .byte
2721// with FIXED registers (rax=a, rcx=b, xmm0 work, xmm1 A-lanes, xmm2 acc) --
2722// no nxasm SSE-parser needed (the __f32x8_dot AVX-.byte precedent). .byte is
2723// DECIMAL (nxasm axc_emit_bytes = v*10+d). Bit-exact-safe: int8 in [-128,127]
2724// and their products are exact in f32; sum order = lane-parallel (4-wide) then
2725// hsum -- the CALLER (a Q8_0 dequant-dot) owns the block/scale order.
2726// __q5_unpack32(qhqs:*u8[20], out:*i8[32], consts:*u8[80]) -> 0.
2727// Unpacks a Q5_0 block (qh[0..4] u32 high-bits, qs[4..20] 16 nibble-bytes)
2728// into 32 signed int8 = (nibble | (qh_bit<<4)) - 16, in A-order (low
2729// nibbles+qh bits 0..15 -> out[0..15]; high nibbles+qh bits 16..31 ->
2730// out[16..31]).  All SSE via .byte, fixed regs: rax=qhqs rcx=out rdx=consts;
2731// xmm0=qs xmm3=qh xmm4=c_0F xmm6=c_bitmask xmm7=c_10; xmm1=vals xmm2=qh-spread
2732// xmm5=pshuf mask.  consts layout: [0]c_0F [16]pshuf_lo [32]pshuf_hi
2733// [48]bitmask [64]c_10 (each 16B).  Caller builds consts ONCE.
2734func x86ctx_emit_q5unpack32(c: *X86Ctx, i: *Instr) -> i64 {
  x86ctx_load_value_v(c, i.op0, "rax" as *u8)         // rax = qhqs
  x86ctx_load_value_v(c, i.op1, "rcx" as *u8)         // rcx = out
  x86ctx_load_value_v(c, i.op2, "rdx" as *u8)         // rdx = consts
  out_str(c.o, "    .byte 243, 15, 111, 64, 4\n")     // movdqu 4(%rax),%xmm0  (qs)
  out_str(c.o, "    .byte 102, 15, 110, 24\n")        // movd (%rax),%xmm3     (qh)
  out_str(c.o, "    .byte 243, 15, 111, 34\n")        // movdqu (%rdx),%xmm4   (c_0F)
  out_str(c.o, "    .byte 243, 15, 111, 114, 48\n")   // movdqu 48(%rdx),%xmm6 (bitmask)
  out_str(c.o, "    .byte 243, 15, 111, 122, 64\n")   // movdqu 64(%rdx),%xmm7 (c_10)
  // ---- LOW group -> out[0..15] ----
  out_str(c.o, "    .byte 102, 15, 111, 200\n")       // movdqa %xmm0,%xmm1
  out_str(c.o, "    .byte 102, 15, 219, 204\n")       // pand %xmm4,%xmm1   (low nibbles)
  out_str(c.o, "    .byte 243, 15, 111, 106, 16\n")   // movdqu 16(%rdx),%xmm5 (pshuf_lo)
  out_str(c.o, "    .byte 102, 15, 111, 211\n")       // movdqa %xmm3,%xmm2   (qh)
  out_str(c.o, "    .byte 102, 15, 56, 0, 213\n")     // pshufb %xmm5,%xmm2
  out_str(c.o, "    .byte 102, 15, 219, 214\n")       // pand %xmm6,%xmm2     (& bitmask)
  out_str(c.o, "    .byte 102, 15, 116, 214\n")       // pcmpeqb %xmm6,%xmm2  (0xFF if set)
  out_str(c.o, "    .byte 102, 15, 219, 215\n")       // pand %xmm7,%xmm2     (& 0x10)
  out_str(c.o, "    .byte 102, 15, 235, 202\n")       // por %xmm2,%xmm1      (nibble|bit<<4)
  out_str(c.o, "    .byte 102, 15, 248, 207\n")       // psubb %xmm7,%xmm1    (- 16)
  out_str(c.o, "    .byte 243, 15, 127, 9\n")         // movdqu %xmm1,(%rcx)  out[0..15]
  // ---- HIGH group -> out[16..31] ----
  out_str(c.o, "    .byte 102, 15, 111, 200\n")       // movdqa %xmm0,%xmm1
  out_str(c.o, "    .byte 102, 15, 113, 209, 4\n")    // psrlw $4,%xmm1
  out_str(c.o, "    .byte 102, 15, 219, 204\n")       // pand %xmm4,%xmm1     (high nibbles)
  out_str(c.o, "    .byte 243, 15, 111, 106, 32\n")   // movdqu 32(%rdx),%xmm5 (pshuf_hi)
  out_str(c.o, "    .byte 102, 15, 111, 211\n")       // movdqa %xmm3,%xmm2
  out_str(c.o, "    .byte 102, 15, 56, 0, 213\n")     // pshufb %xmm5,%xmm2
  out_str(c.o, "    .byte 102, 15, 219, 214\n")       // pand %xmm6,%xmm2
  out_str(c.o, "    .byte 102, 15, 116, 214\n")       // pcmpeqb %xmm6,%xmm2
  out_str(c.o, "    .byte 102, 15, 219, 215\n")       // pand %xmm7,%xmm2
  out_str(c.o, "    .byte 102, 15, 235, 202\n")       // por %xmm2,%xmm1
  out_str(c.o, "    .byte 102, 15, 248, 207\n")       // psubb %xmm7,%xmm1
  out_str(c.o, "    .byte 243, 15, 127, 73, 16\n")    // movdqu %xmm1,16(%rcx) out[16..31]
  out_str(c.o, "    movabsq $0, %rax\n")
  x86ctx_store_result(c, i.result, "rax" as *u8)
  return 0
2771}
2772
2773// __q4k_unpack32s(qs:*u8[32], out:*i16[64], scpack:i64) -> 0.  AVX2, VEX .byte (DECIMAL, the
2774// __i16x16_madd precedent).  One Q4_K sub-block pair: 32 packed nibble bytes -> 64 i16 lanes
2775// ALREADY multiplied by their 6-bit sub-block scale, so the consumer feeds vpmaddwd directly.
2776//   out[0..31]  = (qs[k] & 15) * sc_lo     (low nibbles, lane k = byte k)
2777//   out[32..63] = (qs[k] >> 4) * sc_hi     (high nibbles)
2778// Fixed regs: rax=qs rcx=out rdx=scpack.  ymm2 = 0x000F per word (built from all-ones >> 12, no
2779// memory constant), ymm3 = sc_lo broadcast, ymm4 = sc_hi broadcast; per 16-byte half: vpmovzxbw
2780// widens bytes to words, vpsrlw $4 yields the high nibble (a word holds one byte, so nothing
2781// crosses a lane), vpand the mask yields the low nibble, vpmullw scales, vmovups stores.
2782// Products <= 15*63 = 945 fit a lane exactly; bit-identical to the scalar spread+multiply it
2783// replaces (nx_nofloat_q4k_gate is the exact-reference proof).  Clobbers ymm0-4 and rdx.
2784func x86ctx_emit_q4kunpack32s(c: *X86Ctx, i: *Instr) -> i64 {
  x86ctx_load_value_v(c, i.op0, "rax" as *u8)         // rax = qs
  x86ctx_load_value_v(c, i.op1, "rcx" as *u8)         // rcx = out
  x86ctx_load_value_v(c, i.op2, "rdx" as *u8)         // rdx = sc_lo | sc_hi<<16
  out_str(c.o, "    .byte 197, 237, 118, 210\n")      // C5 ED 76 D2     vpcmpeqd %ymm2,%ymm2,%ymm2  (all ones)
  out_str(c.o, "    .byte 197, 237, 113, 210, 12\n")  // C5 ED 71 D2 0C  vpsrlw $12,%ymm2,%ymm2      (0x000F per word)
  out_str(c.o, "    .byte 197, 249, 110, 218\n")      // C5 F9 6E DA     vmovd %edx,%xmm3            (sc_lo in word 0)
  out_str(c.o, "    .byte 196, 226, 125, 121, 219\n") // C4 E2 7D 79 DB  vpbroadcastw %xmm3,%ymm3
  out_str(c.o, "    .byte 72, 193, 234, 16\n")        // 48 C1 EA 10     shrq $16,%rdx               (sc_hi to word 0)
  out_str(c.o, "    .byte 197, 249, 110, 226\n")      // C5 F9 6E E2     vmovd %edx,%xmm4
  out_str(c.o, "    .byte 196, 226, 125, 121, 228\n") // C4 E2 7D 79 E4  vpbroadcastw %xmm4,%ymm4
  // ---- bytes 0..15 -> out[0..15] (lo) and out[32..47] (hi) ----
  out_str(c.o, "    .byte 196, 226, 125, 48, 0\n")    // C4 E2 7D 30 00  vpmovzxbw (%rax),%ymm0
  out_str(c.o, "    .byte 197, 245, 113, 208, 4\n")   // C5 F5 71 D0 04  vpsrlw $4,%ymm0,%ymm1       (high nibbles)
  out_str(c.o, "    .byte 197, 253, 219, 194\n")      // C5 FD DB C2     vpand %ymm2,%ymm0,%ymm0     (low nibbles)
  out_str(c.o, "    .byte 197, 253, 213, 195\n")      // C5 FD D5 C3     vpmullw %ymm3,%ymm0,%ymm0   (* sc_lo)
  out_str(c.o, "    .byte 197, 245, 213, 204\n")      // C5 F5 D5 CC     vpmullw %ymm4,%ymm1,%ymm1   (* sc_hi)
  out_str(c.o, "    .byte 197, 252, 17, 1\n")         // C5 FC 11 01     vmovups %ymm0,(%rcx)
  out_str(c.o, "    .byte 197, 252, 17, 73, 64\n")    // C5 FC 11 49 40  vmovups %ymm1,64(%rcx)
  // ---- bytes 16..31 -> out[16..31] (lo) and out[48..63] (hi) ----
  out_str(c.o, "    .byte 196, 226, 125, 48, 64, 16\n") // C4 E2 7D 30 40 10 vpmovzxbw 16(%rax),%ymm0
  out_str(c.o, "    .byte 197, 245, 113, 208, 4\n")   // vpsrlw $4,%ymm0,%ymm1
  out_str(c.o, "    .byte 197, 253, 219, 194\n")      // vpand %ymm2,%ymm0,%ymm0
  out_str(c.o, "    .byte 197, 253, 213, 195\n")      // vpmullw %ymm3,%ymm0,%ymm0
  out_str(c.o, "    .byte 197, 245, 213, 204\n")      // vpmullw %ymm4,%ymm1,%ymm1
  out_str(c.o, "    .byte 197, 252, 17, 65, 32\n")    // C5 FC 11 41 20  vmovups %ymm0,32(%rcx)
  out_str(c.o, "    .byte 197, 252, 17, 73, 96\n")    // C5 FC 11 49 60  vmovups %ymm1,96(%rcx)
  out_str(c.o, "    movabsq $0, %rax\n")
  x86ctx_store_result(c, i.result, "rax" as *u8)
  return 0
2814}
2815
2816// __q4k_sb_dot(sb:*u8[144], col:*i16[256], scpre:*i64[8], out:*i64[5]) -> 0.  ONE WHOLE Q4_K SUPER-BLOCK.
2817// MEASURED reason (2026-09-02): with __q4k_unpack32s the serve profile still put 77 percent of a token in the
2818// fused GEMM at 3.25 ms per 7B call, and the kernel's serial time barely moved when its byte reads were removed --
2819// the cost was the compiler's scalar code around four intrinsic calls per super-block, not the arithmetic. This
2820// intrinsic does the super-block in registers: the 12 scale bytes decoded with shifts (w0 = bytes 0..7, w1 = bytes
2821// 8..15), per sub-block pair the unpack+scale sequence, vpmaddwd straight from the activation lanes in memory,
2822// eight i32 lanes accumulated in ymm5 across all eight sub-blocks, and the dmin term (m_sub * scpre[sub]) summed
2823// in r10. out[0..3] = the accumulator lanes (caller: ds_hsum), out[4] = the dmin sum.
2824// Fixed regs: rax=sb rcx=col (advanced 128 B per group) rdx=scpre rsi=out; r8=w0 r9=w1 r10=msum r11=scpack
2825// rdi,rbx=temps (rbx is a callee-saved home register in this backend, so it is pushed and popped);
2826// ymm0/ymm1 work, ymm2 = 0x000F words, ymm3/ymm4 = scale broadcasts, ymm5 = accumulator.
2827// Lane bound: 32 products per lane per super-block, each <= 945*32767 -> < 2^31.  Bit-identical to nx_q4k_dot_simd2
2828// (nx_q4k_simd2_gate proves it against the scalar row_col ruler on random real-shaped rows).
2829func x86ctx_emit_q4ksbdot(c: *X86Ctx, i: *Instr) -> i64 {
  x86ctx_load_value_v(c, i.op0, "rax" as *u8)         // rax = sb
  x86ctx_load_value_v(c, i.op1, "rcx" as *u8)         // rcx = col lanes of this super-block (512 B)
  x86ctx_load_value_v(c, i.op2, "rdx" as *u8)         // rdx = scpre (8 x i64)
  x86ctx_load_value_v(c, i.op3, "rsi" as *u8)         // rsi = out (5 x i64)
  out_str(c.o, "    pushq %rbx\n")
  out_str(c.o, "    .byte 197, 237, 118, 210\n")      // vpcmpeqd %ymm2,%ymm2,%ymm2
  out_str(c.o, "    .byte 197, 237, 113, 210, 12\n")  // vpsrlw $12,%ymm2,%ymm2  (0x000F per word)
  out_str(c.o, "    .byte 197, 213, 239, 237\n")      // C5 D5 EF ED  vpxor %ymm5,%ymm5,%ymm5
  out_str(c.o, "    movq 4(%rax), %r8\n")             // scale bytes 0..7
  out_str(c.o, "    movq 12(%rax), %r9\n")            // scale bytes 8..11 (+4 qs bytes never selected)
  out_str(c.o, "    xorq %r10, %r10\n")
  var g: i64 = 0
  while g < 4 {
      let is0: i64 = g + g
      let is1: i64 = is0 + 1
      if g < 2 {
          // sub-blocks 0..3: sc = b[is] & 63, m = b[is+4] & 63
          out_str(c.o, "    movq %r8, %r11\n")
          if is0 > 0 { out_str(c.o, "    shrq $"); out_i64(c.o, is0 * 8); out_str(c.o, ", %r11\n") }
          out_str(c.o, "    andq $63, %r11\n")
          out_str(c.o, "    movq %r8, %rdi\n    shrq $"); out_i64(c.o, (is0 + 4) * 8); out_str(c.o, ", %rdi\n    andq $63, %rdi\n")
          out_str(c.o, "    movq "); out_i64(c.o, is0 * 8); out_str(c.o, "(%rdx), %rbx\n    imulq %rbx, %rdi\n    addq %rdi, %r10\n")
          out_str(c.o, "    movq %r8, %rdi\n    shrq $"); out_i64(c.o, is1 * 8); out_str(c.o, ", %rdi\n    andq $63, %rdi\n    shlq $16, %rdi\n    orq %rdi, %r11\n")
          out_str(c.o, "    movq %r8, %rdi\n    shrq $"); out_i64(c.o, (is1 + 4) * 8); out_str(c.o, ", %rdi\n    andq $63, %rdi\n")
          out_str(c.o, "    movq "); out_i64(c.o, is1 * 8); out_str(c.o, "(%rdx), %rbx\n    imulq %rbx, %rdi\n    addq %rdi, %r10\n")
      } else {
          // sub-blocks 4..7 (k = is-4): sc = ((b[k]>>6)&3)<<4 | (b[8+k] & 15), m = ((b[k+4]>>6)&3)<<4 | ((b[8+k]>>4) & 15)
          let k0: i64 = is0 - 4
          let k1: i64 = is1 - 4
          out_str(c.o, "    movq %r8, %r11\n    shrq $"); out_i64(c.o, k0 * 8 + 6); out_str(c.o, ", %r11\n    andq $3, %r11\n    shlq $4, %r11\n")
          out_str(c.o, "    movq %r9, %rdi\n")
          if k0 > 0 { out_str(c.o, "    shrq $"); out_i64(c.o, k0 * 8); out_str(c.o, ", %rdi\n") }
          out_str(c.o, "    andq $15, %rdi\n    orq %rdi, %r11\n")
          out_str(c.o, "    movq %r8, %rdi\n    shrq $"); out_i64(c.o, (k0 + 4) * 8 + 6); out_str(c.o, ", %rdi\n    andq $3, %rdi\n    shlq $4, %rdi\n")
          out_str(c.o, "    movq %r9, %rbx\n    shrq $"); out_i64(c.o, k0 * 8 + 4); out_str(c.o, ", %rbx\n    andq $15, %rbx\n    orq %rbx, %rdi\n")
          out_str(c.o, "    movq "); out_i64(c.o, is0 * 8); out_str(c.o, "(%rdx), %rbx\n    imulq %rbx, %rdi\n    addq %rdi, %r10\n")
          out_str(c.o, "    movq %r8, %rdi\n    shrq $"); out_i64(c.o, k1 * 8 + 6); out_str(c.o, ", %rdi\n    andq $3, %rdi\n    shlq $4, %rdi\n")
          out_str(c.o, "    movq %r9, %rbx\n    shrq $"); out_i64(c.o, k1 * 8); out_str(c.o, ", %rbx\n    andq $15, %rbx\n    orq %rbx, %rdi\n    shlq $16, %rdi\n    orq %rdi, %r11\n")
          out_str(c.o, "    movq %r8, %rdi\n    shrq $"); out_i64(c.o, (k1 + 4) * 8 + 6); out_str(c.o, ", %rdi\n    andq $3, %rdi\n    shlq $4, %rdi\n")
          out_str(c.o, "    movq %r9, %rbx\n    shrq $"); out_i64(c.o, k1 * 8 + 4); out_str(c.o, ", %rbx\n    andq $15, %rbx\n    orq %rbx, %rdi\n")
          out_str(c.o, "    movq "); out_i64(c.o, is1 * 8); out_str(c.o, "(%rdx), %rbx\n    imulq %rbx, %rdi\n    addq %rdi, %r10\n")
      }
      // scale broadcasts: ymm3 = sc_lo (r11 low word), ymm4 = sc_hi (r11 >> 16)
      out_str(c.o, "    .byte 196, 193, 121, 110, 219\n") // C4 C1 79 6E DB  vmovd %r11d,%xmm3
      out_str(c.o, "    .byte 196, 226, 125, 121, 219\n") // vpbroadcastw %xmm3,%ymm3
      out_str(c.o, "    shrq $16, %r11\n")
      out_str(c.o, "    .byte 196, 193, 121, 110, 227\n") // C4 C1 79 6E E3  vmovd %r11d,%xmm4
      out_str(c.o, "    .byte 196, 226, 125, 121, 228\n") // vpbroadcastw %xmm4,%ymm4
      // half 0: qs bytes 16+32g .. +15 -> lanes 0..15 of sub-block is0 (low nibbles, col +0) and is1 (high, col +64)
      out_str(c.o, "    .byte 196, 226, 125, 48, 64, "); out_i64(c.o, 16 + 32 * g); out_str(c.o, "\n")  // vpmovzxbw d8(%rax),%ymm0
      out_str(c.o, "    .byte 197, 245, 113, 208, 4\n")   // vpsrlw $4,%ymm0,%ymm1
      out_str(c.o, "    .byte 197, 253, 219, 194\n")      // vpand %ymm2,%ymm0,%ymm0
      out_str(c.o, "    .byte 197, 253, 213, 195\n")      // vpmullw %ymm3,%ymm0,%ymm0
      out_str(c.o, "    .byte 197, 245, 213, 204\n")      // vpmullw %ymm4,%ymm1,%ymm1
      out_str(c.o, "    .byte 197, 253, 245, 65, 0\n")    // C5 FD F5 41 00  vpmaddwd 0(%rcx),%ymm0,%ymm0
      out_str(c.o, "    .byte 197, 213, 254, 232\n")      // C5 D5 FE E8     vpaddd %ymm0,%ymm5,%ymm5
      out_str(c.o, "    .byte 197, 245, 245, 73, 64\n")   // C5 F5 F5 49 40  vpmaddwd 64(%rcx),%ymm1,%ymm1
      out_str(c.o, "    .byte 197, 213, 254, 233\n")      // C5 D5 FE E9     vpaddd %ymm1,%ymm5,%ymm5
      // half 1: qs bytes 32+32g .. +15 -> lanes 16..31 of is0 (col +32) and is1 (col +96)
      out_str(c.o, "    .byte 196, 226, 125, 48, 64, "); out_i64(c.o, 32 + 32 * g); out_str(c.o, "\n")
      out_str(c.o, "    .byte 197, 245, 113, 208, 4\n")
      out_str(c.o, "    .byte 197, 253, 219, 194\n")
      out_str(c.o, "    .byte 197, 253, 213, 195\n")
      out_str(c.o, "    .byte 197, 245, 213, 204\n")
      out_str(c.o, "    .byte 197, 253, 245, 65, 32\n")   // vpmaddwd 32(%rcx),%ymm0,%ymm0
      out_str(c.o, "    .byte 197, 213, 254, 232\n")
      out_str(c.o, "    .byte 197, 245, 245, 73, 96\n")   // vpmaddwd 96(%rcx),%ymm1,%ymm1
      out_str(c.o, "    .byte 197, 213, 254, 233\n")
      out_str(c.o, "    addq $128, %rcx\n")               // next sub-block pair's 128 B of lanes
      g = g + 1
  }
  out_str(c.o, "    .byte 197, 252, 17, 46\n")            // C5 FC 11 2E  vmovups %ymm5,(%rsi)   acc lanes -> out[0..3]
  out_str(c.o, "    movq %r10, 32(%rsi)\n")               // dmin sum -> out[4]
  out_str(c.o, "    popq %rbx\n")
  out_str(c.o, "    movabsq $0, %rax\n")
  x86ctx_store_result(c, i.result, "rax" as *u8)
  return 0
2907}
2908
2909// __f32_i8dot32a(a:*i8[32], b:*f32[32]) -> f32 : AVX2 256-bit dequant-dot.
2910// 4 blocks of 8 lanes (vs the SSE version's 8 blocks of 4), and TWO
2911// accumulators (ymm4 blocks 0,2 ; ymm5 blocks 1,3) so the vaddps chains run
2912// in parallel instead of an 8-deep serial addps -- the gcc-proven codegen
2913// lever (SSE i8dot32 hit ~5 GB/s; gcc's vectorized code ~15-30).  VEX .byte
2914// (DECIMAL; the __f32x8_dot precedent).  Fixed regs: rax=a rcx=b ; ymm0 work
2915// (converted a), ymm1 (b), ymm4/ymm5 accs.  NOT bit-identical to i8dot32
2916// (2-acc summation order); int8 products are exact in f32 so only the add
2917// ORDER differs -- argmax-robust, gated by nx_i8dot32a_kat + " Paris".
2918// __f32_i8fma32(a:*i8[32], b:*f32[32], d_bits:i64, acc:*f32[8]) -> 0.
2919// DEFERRED-HSUM block: acc[8] += d * (sext(a) . b), 8-lane AVX2, NO hsum.
2920// vfmadd231ps into a persistent 8-lane acc; the caller loops all k/32 blocks
2921// (broadcasting each block's scale d) then hsums the acc ONCE per output
2922// (__f32x8_hsum) -- kills 27/28 per-block hsums (the cold-forward matmul
2923// lever, 2026-07-10; the SSE/AVX2 per-32 hsum, not the dot width, was the
2924// bottleneck).  regs: rax=a rcx=b rdx=d_bits rsi=acc ; ymm7=d broadcast,
2925// ymm6=acc, ymm0 work, ymm1 b.  VEX .byte (decimal).  4 lanes-of-8 unrolled.
2926// unique loop label for the monolithic row kernel (fn name + result id),
2927// mirroring x86ctx_emit_clone_label.
2928func x86ctx_emit_q8row_label(c: *X86Ctx, rid: i64) -> i64 {
  out_str(c.o, ".Lq8row_")
  let name: *u8 = c.f.name_start as *u8
  if name != (0 as *u8) { out_str(c.o, name) }
  out_char(c.o, 0x5F)        // '_'
  out_i64(c.o, rid)
  return 0
2935}
2936
2937// __f32_q8row_dot(qbuf_row:*u8, a_row:*f32, nblocks:i64) -> f32.
2938// MONOLITHIC Q8_0 row dot: acc = sum_b d_b * (sext(int8_b) . a_b), with the
2939// 8-lane ymm6 accumulator REGISTER-RESIDENT across the whole block loop (the
2940// deferred-hsum win the per-block __f32_i8fma32 couldn't get -- that pushed
2941// the acc through memory 28x/output).  F16C vcvtph2ps decodes each block's
2942// f16 scale in ONE instruction.  regs: rax=qbuf_row rcx=a_row rdx=nblocks ;
2943// ymm6=acc ymm7=d-broadcast ymm0/1 work.  NOT bit-identical (8-lane+FMA
2944// order); argmax-robust.  Requires AVX2+F16C (already assumed: vfmadd/vpmaddwd).
2945func x86ctx_emit_q8rowdot(c: *X86Ctx, i: *Instr) -> i64 {
  x86ctx_load_value_v(c, i.op0, "rax" as *u8)         // qbuf_row
  x86ctx_load_value_v(c, i.op1, "rcx" as *u8)         // a_row
  x86ctx_load_value_v(c, i.op2, "rdx" as *u8)         // nblocks
  // FOUR accumulators (ymm2/3/4/5), one per sub-block, to break the serial
  // vfmadd231ps chain (a single acc = 4*nblocks-deep = latency-bound).
  out_str(c.o, "    .byte 197, 236, 87, 210\n")       // vxorps %ymm2,%ymm2,%ymm2
  out_str(c.o, "    .byte 197, 228, 87, 219\n")       // vxorps %ymm3,%ymm3,%ymm3
  out_str(c.o, "    .byte 197, 220, 87, 228\n")       // vxorps %ymm4,%ymm4,%ymm4
  out_str(c.o, "    .byte 197, 212, 87, 237\n")       // vxorps %ymm5,%ymm5,%ymm5
  x86ctx_emit_q8row_label(c, i.result)
  out_str(c.o, ":\n")
  // f16 d at (%rax) -> broadcast into ymm7 (F16C, one instr).
  out_str(c.o, "    .byte 196, 226, 121, 19, 56\n")   // vcvtph2ps (%rax),%xmm7  (lane0=d)
  out_str(c.o, "    .byte 196, 226, 125, 24, 255\n")  // vbroadcastss %xmm7,%ymm7
  // 4 sub-blocks -> 4 accs: int8 at rax+2+j*8 ; f32 at rcx+j*32 ; accN += d*(int8.a)
  out_str(c.o, "    .byte 196, 226, 125, 33, 64, 2\n");  out_str(c.o, "    .byte 197, 252, 91, 192\n"); out_str(c.o, "    .byte 197, 252, 16, 9\n");      out_str(c.o, "    .byte 197, 252, 89, 193\n"); out_str(c.o, "    .byte 196, 226, 125, 184, 215\n")
  out_str(c.o, "    .byte 196, 226, 125, 33, 64, 10\n"); out_str(c.o, "    .byte 197, 252, 91, 192\n"); out_str(c.o, "    .byte 197, 252, 16, 73, 32\n"); out_str(c.o, "    .byte 197, 252, 89, 193\n"); out_str(c.o, "    .byte 196, 226, 125, 184, 223\n")
  out_str(c.o, "    .byte 196, 226, 125, 33, 64, 18\n"); out_str(c.o, "    .byte 197, 252, 91, 192\n"); out_str(c.o, "    .byte 197, 252, 16, 73, 64\n"); out_str(c.o, "    .byte 197, 252, 89, 193\n"); out_str(c.o, "    .byte 196, 226, 125, 184, 231\n")
  out_str(c.o, "    .byte 196, 226, 125, 33, 64, 26\n"); out_str(c.o, "    .byte 197, 252, 91, 192\n"); out_str(c.o, "    .byte 197, 252, 16, 73, 96\n"); out_str(c.o, "    .byte 197, 252, 89, 193\n"); out_str(c.o, "    .byte 196, 226, 125, 184, 239\n")
  out_str(c.o, "    addq $34, %rax\n")                // next block (34 bytes)
  out_str(c.o, "    addq $128, %rcx\n")               // next 32 f32 (128 bytes)
  out_str(c.o, "    subq $1, %rdx\n")
  out_str(c.o, "    jne ")
  x86ctx_emit_q8row_label(c, i.result)
  out_char(c.o, 0x0A)
  // combine 4 accs -> ymm2, then hsum.
  out_str(c.o, "    .byte 197, 236, 88, 211\n")       // vaddps %ymm3,%ymm2,%ymm2
  out_str(c.o, "    .byte 197, 220, 88, 229\n")       // vaddps %ymm5,%ymm4,%ymm4
  out_str(c.o, "    .byte 197, 236, 88, 212\n")       // vaddps %ymm4,%ymm2,%ymm2
  out_str(c.o, "    .byte 197, 252, 40, 194\n")       // vmovaps %ymm2,%ymm0
  out_str(c.o, "    .byte 196, 227, 125, 25, 193, 1\n") // vextractf128 $1,%ymm0,%xmm1
  out_str(c.o, "    .byte 197, 248, 119\n")             // vzeroupper
  out_str(c.o, "    addps %xmm1, %xmm0\n")
  out_str(c.o, "    movups %xmm0, -16(%rsp)\n")
  out_str(c.o, "    movss -16(%rsp), %xmm0\n")
  out_str(c.o, "    movss -12(%rsp), %xmm1\n")
  out_str(c.o, "    addss %xmm1, %xmm0\n")
  out_str(c.o, "    movss -8(%rsp), %xmm1\n")
  out_str(c.o, "    addss %xmm1, %xmm0\n")
  out_str(c.o, "    movss -4(%rsp), %xmm1\n")
  out_str(c.o, "    addss %xmm1, %xmm0\n")
  out_str(c.o, "    movss %xmm0, -8(%rsp)\n")
  out_str(c.o, "    movl -8(%rsp), %eax\n")
  x86ctx_store_result(c, i.result, "rax" as *u8)
  return 0
2991}
2992
2993func x86ctx_emit_i8fma32(c: *X86Ctx, i: *Instr) -> i64 {
  x86ctx_load_value_v(c, i.op0, "rax" as *u8)         // a
  x86ctx_load_value_v(c, i.op1, "rcx" as *u8)         // b
  x86ctx_load_value_v(c, i.op2, "rdx" as *u8)         // d_bits (f32 in low32)
  x86ctx_load_value_v(c, i.op3, "rsi" as *u8)         // acc ptr
  out_str(c.o, "    .byte 102, 15, 110, 250\n")        // movd %edx,%xmm7
  out_str(c.o, "    .byte 196, 226, 125, 24, 255\n")   // vbroadcastss %xmm7,%ymm7 (d in 8 lanes)
  out_str(c.o, "    .byte 197, 252, 16, 54\n")         // vmovups (%rsi),%ymm6   (load acc)
  // 4 blocks of 8: products = int8.b ; acc += products * d
  out_str(c.o, "    .byte 196, 226, 125, 33, 0\n");    out_str(c.o, "    .byte 197, 252, 91, 192\n"); out_str(c.o, "    .byte 197, 252, 16, 9\n");      out_str(c.o, "    .byte 197, 252, 89, 193\n"); out_str(c.o, "    .byte 196, 226, 125, 184, 247\n")
  out_str(c.o, "    .byte 196, 226, 125, 33, 64, 8\n");  out_str(c.o, "    .byte 197, 252, 91, 192\n"); out_str(c.o, "    .byte 197, 252, 16, 73, 32\n"); out_str(c.o, "    .byte 197, 252, 89, 193\n"); out_str(c.o, "    .byte 196, 226, 125, 184, 247\n")
  out_str(c.o, "    .byte 196, 226, 125, 33, 64, 16\n"); out_str(c.o, "    .byte 197, 252, 91, 192\n"); out_str(c.o, "    .byte 197, 252, 16, 73, 64\n"); out_str(c.o, "    .byte 197, 252, 89, 193\n"); out_str(c.o, "    .byte 196, 226, 125, 184, 247\n")
  out_str(c.o, "    .byte 196, 226, 125, 33, 64, 24\n"); out_str(c.o, "    .byte 197, 252, 91, 192\n"); out_str(c.o, "    .byte 197, 252, 16, 73, 96\n"); out_str(c.o, "    .byte 197, 252, 89, 193\n"); out_str(c.o, "    .byte 196, 226, 125, 184, 247\n")
  out_str(c.o, "    .byte 197, 252, 17, 54\n")         // vmovups %ymm6,(%rsi)   (store acc)
  out_str(c.o, "    .byte 197, 248, 119\n")            // vzeroupper
  out_str(c.o, "    xorq %rax, %rax\n")
  x86ctx_store_result(c, i.result, "rax" as *u8)
  return 0
3011}
3012
3013func x86ctx_emit_i8dot32a(c: *X86Ctx, i: *Instr) -> i64 {
  x86ctx_load_value_v(c, i.op0, "rax" as *u8)         // rax = a (i8 ptr)
  x86ctx_load_value_v(c, i.op1, "rcx" as *u8)         // rcx = b (f32 ptr)
  out_str(c.o, "    .byte 197, 220, 87, 228\n")       // vxorps %ymm4,%ymm4,%ymm4  (acc0=0)
  out_str(c.o, "    .byte 197, 212, 87, 237\n")       // vxorps %ymm5,%ymm5,%ymm5  (acc1=0)
  // block j: vpmovsxbd (rax+j*8)->ymm0 ; vcvtdq2ps ; vmovups (rcx+j*32)->ymm1 ;
  // vmulps ymm1,ymm0,ymm0 ; vaddps ymm0,accN,accN.  disp8: i8 j*8, f32 j*32.
  // -- block 0 -> acc0(ymm4)
  out_str(c.o, "    .byte 196, 226, 125, 33, 0\n");    out_str(c.o, "    .byte 197, 252, 91, 192\n"); out_str(c.o, "    .byte 197, 252, 16, 9\n");      out_str(c.o, "    .byte 197, 252, 89, 193\n"); out_str(c.o, "    .byte 197, 220, 88, 224\n")
  // -- block 1 -> acc1(ymm5)
  out_str(c.o, "    .byte 196, 226, 125, 33, 64, 8\n");  out_str(c.o, "    .byte 197, 252, 91, 192\n"); out_str(c.o, "    .byte 197, 252, 16, 73, 32\n"); out_str(c.o, "    .byte 197, 252, 89, 193\n"); out_str(c.o, "    .byte 197, 212, 88, 232\n")
  // -- block 2 -> acc0(ymm4)
  out_str(c.o, "    .byte 196, 226, 125, 33, 64, 16\n"); out_str(c.o, "    .byte 197, 252, 91, 192\n"); out_str(c.o, "    .byte 197, 252, 16, 73, 64\n"); out_str(c.o, "    .byte 197, 252, 89, 193\n"); out_str(c.o, "    .byte 197, 220, 88, 224\n")
  // -- block 3 -> acc1(ymm5)
  out_str(c.o, "    .byte 196, 226, 125, 33, 64, 24\n"); out_str(c.o, "    .byte 197, 252, 91, 192\n"); out_str(c.o, "    .byte 197, 252, 16, 73, 96\n"); out_str(c.o, "    .byte 197, 252, 89, 193\n"); out_str(c.o, "    .byte 197, 212, 88, 232\n")
  out_str(c.o, "    .byte 197, 220, 88, 229\n")       // vaddps %ymm5,%ymm4,%ymm4  (combine accs)
  out_str(c.o, "    .byte 197, 252, 40, 196\n")       // vmovaps %ymm4,%ymm0  (-> ymm0 for hsum)
  // hsum ymm0 -> eax : fold hi128+lo128 then SSE 4-lane hsum (the f32x8 tail).
  out_str(c.o, "    .byte 196, 227, 125, 25, 193, 1\n") // vextractf128 $1,%ymm0,%xmm1
  out_str(c.o, "    .byte 197, 248, 119\n")             // vzeroupper
  out_str(c.o, "    addps %xmm1, %xmm0\n")
  out_str(c.o, "    movups %xmm0, -16(%rsp)\n")
  out_str(c.o, "    movss -16(%rsp), %xmm0\n")
  out_str(c.o, "    movss -12(%rsp), %xmm1\n")
  out_str(c.o, "    addss %xmm1, %xmm0\n")
  out_str(c.o, "    movss -8(%rsp), %xmm1\n")
  out_str(c.o, "    addss %xmm1, %xmm0\n")
  out_str(c.o, "    movss -4(%rsp), %xmm1\n")
  out_str(c.o, "    addss %xmm1, %xmm0\n")
  out_str(c.o, "    movss %xmm0, -8(%rsp)\n")
  out_str(c.o, "    movl -8(%rsp), %eax\n")
  x86ctx_store_result(c, i.result, "rax" as *u8)
  return 0
3046}
3047
3048func x86ctx_emit_i8dot32(c: *X86Ctx, i: *Instr) -> i64 {
  x86ctx_load_value_v(c, i.op0, "rax" as *u8)         // rax = a (i8 ptr)
  x86ctx_load_value_v(c, i.op1, "rcx" as *u8)         // rcx = b (f32 ptr)
  out_str(c.o, "    .byte 15, 87, 210\n")             // xorps %xmm2,%xmm2  (acc=0)
  // 8 unrolled 4-lane blocks; i8 disp = b*4, f32 disp = b*16.
  // movd d(%rax),%xmm0 = 66 0F 6E 40 d ; pmovsxbd = 66 0F 38 21 C0 ;
  // cvtdq2ps = 0F 5B C0 ; movups d(%rcx),%xmm1 = 0F 10 49 d ;
  // mulps %xmm1,%xmm0 = 0F 59 C1 ; addps %xmm0,%xmm2 = 0F 58 D0
  out_str(c.o, "    .byte 102, 15, 110, 64, 0\n");   out_str(c.o, "    .byte 102, 15, 56, 33, 192\n"); out_str(c.o, "    .byte 15, 91, 192\n"); out_str(c.o, "    .byte 15, 16, 73, 0\n");   out_str(c.o, "    .byte 15, 89, 193\n"); out_str(c.o, "    .byte 15, 88, 208\n")
  out_str(c.o, "    .byte 102, 15, 110, 64, 4\n");   out_str(c.o, "    .byte 102, 15, 56, 33, 192\n"); out_str(c.o, "    .byte 15, 91, 192\n"); out_str(c.o, "    .byte 15, 16, 73, 16\n");  out_str(c.o, "    .byte 15, 89, 193\n"); out_str(c.o, "    .byte 15, 88, 208\n")
  out_str(c.o, "    .byte 102, 15, 110, 64, 8\n");   out_str(c.o, "    .byte 102, 15, 56, 33, 192\n"); out_str(c.o, "    .byte 15, 91, 192\n"); out_str(c.o, "    .byte 15, 16, 73, 32\n");  out_str(c.o, "    .byte 15, 89, 193\n"); out_str(c.o, "    .byte 15, 88, 208\n")
  out_str(c.o, "    .byte 102, 15, 110, 64, 12\n");  out_str(c.o, "    .byte 102, 15, 56, 33, 192\n"); out_str(c.o, "    .byte 15, 91, 192\n"); out_str(c.o, "    .byte 15, 16, 73, 48\n");  out_str(c.o, "    .byte 15, 89, 193\n"); out_str(c.o, "    .byte 15, 88, 208\n")
  out_str(c.o, "    .byte 102, 15, 110, 64, 16\n");  out_str(c.o, "    .byte 102, 15, 56, 33, 192\n"); out_str(c.o, "    .byte 15, 91, 192\n"); out_str(c.o, "    .byte 15, 16, 73, 64\n");  out_str(c.o, "    .byte 15, 89, 193\n"); out_str(c.o, "    .byte 15, 88, 208\n")
  out_str(c.o, "    .byte 102, 15, 110, 64, 20\n");  out_str(c.o, "    .byte 102, 15, 56, 33, 192\n"); out_str(c.o, "    .byte 15, 91, 192\n"); out_str(c.o, "    .byte 15, 16, 73, 80\n");  out_str(c.o, "    .byte 15, 89, 193\n"); out_str(c.o, "    .byte 15, 88, 208\n")
  out_str(c.o, "    .byte 102, 15, 110, 64, 24\n");  out_str(c.o, "    .byte 102, 15, 56, 33, 192\n"); out_str(c.o, "    .byte 15, 91, 192\n"); out_str(c.o, "    .byte 15, 16, 73, 96\n");  out_str(c.o, "    .byte 15, 89, 193\n"); out_str(c.o, "    .byte 15, 88, 208\n")
  out_str(c.o, "    .byte 102, 15, 110, 64, 28\n");  out_str(c.o, "    .byte 102, 15, 56, 33, 192\n"); out_str(c.o, "    .byte 15, 91, 192\n"); out_str(c.o, "    .byte 15, 16, 73, 112\n"); out_str(c.o, "    .byte 15, 89, 193\n"); out_str(c.o, "    .byte 15, 88, 208\n")
  out_str(c.o, "    .byte 15, 40, 194\n")             // movaps %xmm2,%xmm0  (acc -> xmm0 for the hsum)
  // hsum xmm0 -> eax (the __f32x4_dot tail, AT&T; nxasm-proven).
  out_str(c.o, "    movups %xmm0, -16(%rsp)\n")
  out_str(c.o, "    movss -16(%rsp), %xmm0\n")
  out_str(c.o, "    movss -12(%rsp), %xmm1\n")
  out_str(c.o, "    addss %xmm1, %xmm0\n")
  out_str(c.o, "    movss -8(%rsp), %xmm1\n")
  out_str(c.o, "    addss %xmm1, %xmm0\n")
  out_str(c.o, "    movss -4(%rsp), %xmm1\n")
  out_str(c.o, "    addss %xmm1, %xmm0\n")
  out_str(c.o, "    movss %xmm0, -8(%rsp)\n")
  out_str(c.o, "    movl -8(%rsp), %eax\n")
  x86ctx_store_result(c, i.result, "rax" as *u8)
  return 0
3078}
3079
3080func x86ctx_emit_f32x8_dot(c: *X86Ctx, i: *Instr) -> i64 {
  x86ctx_load_value_v(c, i.op0, "rax" as *u8)         // rax = a_ptr (8 f32)
  x86ctx_load_value_v(c, i.op1, "rcx" as *u8)         // rcx = b_ptr (8 f32)
  // nxasm .byte parses DECIMAL ONLY (axc_emit_bytes: v*10+d), so VEX bytes are emitted as decimal,
  // NOT 0x-hex (which would parse as just "0" and silently truncate the whole instruction -> SIGSEGV).
  out_str(c.o, "    .byte 197, 252, 16, 0\n")                     // C5 FC 10 00  vmovups (%rax),%ymm0
  out_str(c.o, "    .byte 197, 252, 16, 9\n")                     // C5 FC 10 09  vmovups (%rcx),%ymm1
  out_str(c.o, "    .byte 197, 252, 89, 193\n")                   // C5 FC 59 C1  vmulps %ymm1,%ymm0,%ymm0
  out_str(c.o, "    .byte 196, 227, 125, 25, 193, 1\n")           // C4 E3 7D 19 C1 01  vextractf128 $1,%ymm0,%xmm1
  out_str(c.o, "    .byte 197, 248, 119\n")                       // C5 F8 77  vzeroupper (clear AVX-SSE transition)
  out_str(c.o, "    addps %xmm1, %xmm0\n")                        // xmm0 = lo4 + hi4 = 4 partial sums
  out_str(c.o, "    movups %xmm0, -16(%rsp)\n")
  out_str(c.o, "    movss -16(%rsp), %xmm0\n")
  out_str(c.o, "    movss -12(%rsp), %xmm1\n")
  out_str(c.o, "    addss %xmm1, %xmm0\n")
  out_str(c.o, "    movss -8(%rsp), %xmm1\n")
  out_str(c.o, "    addss %xmm1, %xmm0\n")
  out_str(c.o, "    movss -4(%rsp), %xmm1\n")
  out_str(c.o, "    addss %xmm1, %xmm0\n")
  out_str(c.o, "    movss %xmm0, -8(%rsp)\n")
  out_str(c.o, "    movl -8(%rsp), %eax\n")
  x86ctx_store_result(c, i.result, "rax" as *u8)
  return 0
3103}
3104
3105// FMA vector-accumulate (AVX2): *acc += a*b 8-wide FUSED (vfmadd231ps).  op0=acc op1=a op2=b (pointers).
3106// Accumulator round-trips memory each call (L1-hot) but there is NO per-chunk hsum -- deferred to
3107// x86ctx_emit_f32x8_hsum, ONE hsum per dot.  Pure AVX in the inner loop -> no AVX-SSE transition penalty.
3108func x86ctx_emit_f32x8_fma(c: *X86Ctx, i: *Instr) -> i64 {
  x86ctx_load_value_v(c, i.op0, "rax" as *u8)         // rax = acc ptr
  x86ctx_load_value_v(c, i.op1, "rcx" as *u8)         // rcx = a ptr
  x86ctx_load_value_v(c, i.op2, "rdx" as *u8)         // rdx = b ptr
  out_str(c.o, "    .byte 197, 252, 16, 0\n")         // C5 FC 10 00  vmovups (%rax),%ymm0  (acc)
  out_str(c.o, "    .byte 197, 252, 16, 9\n")         // C5 FC 10 09  vmovups (%rcx),%ymm1  (a)
  out_str(c.o, "    .byte 197, 252, 16, 18\n")        // C5 FC 10 12  vmovups (%rdx),%ymm2  (b)
  out_str(c.o, "    .byte 196, 226, 117, 184, 194\n") // C4 E2 75 B8 C2  vfmadd231ps %ymm2,%ymm1,%ymm0
  out_str(c.o, "    .byte 197, 252, 17, 0\n")         // C5 FC 11 00  vmovups %ymm0,(%rax)  (store acc)
  x86ctx_store_result(c, i.result, "rax" as *u8)
  return 0
3119}
3120// horizontal sum of an 8-wide accumulator -> f32 scalar.  op0 = acc ptr.  vextractf128 folds hi4+lo4, SSE hsum.
3121func x86ctx_emit_f32x8_hsum(c: *X86Ctx, i: *Instr) -> i64 {
  x86ctx_load_value_v(c, i.op0, "rax" as *u8)         // rax = acc ptr
  out_str(c.o, "    .byte 197, 252, 16, 0\n")         // C5 FC 10 00  vmovups (%rax),%ymm0
  out_str(c.o, "    .byte 196, 227, 125, 25, 193, 1\n") // C4 E3 7D 19 C1 01  vextractf128 $1,%ymm0,%xmm1
  out_str(c.o, "    .byte 197, 248, 119\n")           // C5 F8 77  vzeroupper
  out_str(c.o, "    addps %xmm1, %xmm0\n")
  out_str(c.o, "    movups %xmm0, -16(%rsp)\n")
  out_str(c.o, "    movss -16(%rsp), %xmm0\n")
  out_str(c.o, "    movss -12(%rsp), %xmm1\n")
  out_str(c.o, "    addss %xmm1, %xmm0\n")
  out_str(c.o, "    movss -8(%rsp), %xmm1\n")
  out_str(c.o, "    addss %xmm1, %xmm0\n")
  out_str(c.o, "    movss -4(%rsp), %xmm1\n")
  out_str(c.o, "    addss %xmm1, %xmm0\n")
  out_str(c.o, "    movss %xmm0, -8(%rsp)\n")
  out_str(c.o, "    movl -8(%rsp), %eax\n")
  x86ctx_store_result(c, i.result, "rax" as *u8)
  return 0
3139}
3140
3141// NO-FLOAT integer madd-accumulate (AVX2): *acc(i32x8) += vpmaddwd(a(i16x16), b(i16x16)).  op0=acc op1=a
3142// op2=b.  vpmaddwd does 16 SIGNED int16 multiplies + pairwise adds -> 8 int32; vpaddd accumulates into the
3143// int32 vector.  EXACT + DETERMINISTIC (integer add is associative) -- the no-float compute lever.  The
3144// final int64 hsum of the 8 int32 lanes is plain scalar code (caller), no overflow, no hsum intrinsic needed.
3145func x86ctx_emit_i16dot_label(c: *X86Ctx, rid: i64) -> i64 {
  out_str(c.o, ".Li16dot_")
  let name: *u8 = c.f.name_start as *u8
  if name != (0 as *u8) { out_str(c.o, name) }
  out_char(c.o, 0x5F)        // '_'
  out_i64(c.o, rid)
  return 0
3152}
3153
3154// __i16_dot(a:*i16[n], b:*i16[n], n) -> i64 (R0r-b). ONE loop over the whole chunk with the int32x8 accumulator in
3155// %ymm0 -- one vmovups, one vpmaddwd straight from memory, one vpaddd per 16 lanes -- then a single store to the red
3156// zone and an int64 horizontal sum of the eight lanes (movslq each, so the sum is exact past int32). The per-madd
3157// builtin OP_I16X16_MADD loads and stores its accumulator on every call, which is the round trip that bounded the
3158// batched prefill kernel at 450 us per row. regs: rax=a rcx=b rdx=n ; ymm0=acc ymm1=work. n is a POSITIVE multiple
3159// of 16 by contract (a do-while, so n=0 would wrap); the caller bounds n so no int32 lane overflows. AVX2 only.
3160// __q8blk_i16dot(codes:*i8[32], x:*i16[32]) -> i64 (search R0s-b): one Q8_0 block's dot with i16 activation lanes,
3161// the int8 codes sign-extended IN REGISTER (vpmovsxbw) so the block-native kernel never stores an i16 copy of the
3162// weights: two 16-lane halves, each vpmaddwd against its 16 lanes of x, vpaddd, then the same widened int64
3163// horizontal sum __i16_dot uses. EXACT + deterministic: an int32 lane holds two products of at most 127*32767 per
3164// half, four after the add, far inside int32. regs: rax=codes rcx=x ; ymm1/ymm2 halves, ymm0 sum. AVX2 only.
3165// No loop, no counter; encodings mirror x86ctx_emit_i8dot32a (vpmovsx from a disp8 base) and x86ctx_emit_i16dot.
3166func x86ctx_emit_q8blkdot(c: *X86Ctx, i: *Instr) -> i64 {
  x86ctx_load_value_v(c, i.op0, "rax" as *u8)         // rax = codes (int8 lanes)
  x86ctx_load_value_v(c, i.op1, "rcx" as *u8)         // rcx = x (i16 lanes)
  out_str(c.o, "    .byte 196, 226, 125, 32, 8\n")        // C4 E2 7D 20 08     vpmovsxbw (%rax),%ymm1       (codes 0..15 -> i16)
  out_str(c.o, "    .byte 197, 245, 245, 9\n")            // C5 F5 F5 09        vpmaddwd (%rcx),%ymm1,%ymm1   (x 0..15)
  out_str(c.o, "    .byte 196, 226, 125, 32, 80, 16\n")   // C4 E2 7D 20 50 10  vpmovsxbw 16(%rax),%ymm2     (codes 16..31 -> i16)
  out_str(c.o, "    .byte 197, 237, 245, 81, 32\n")       // C5 ED F5 51 20     vpmaddwd 32(%rcx),%ymm2,%ymm2 (x 16..31)
  out_str(c.o, "    .byte 197, 245, 254, 194\n")          // C5 F5 FE C2        vpaddd %ymm2,%ymm1,%ymm0
  out_str(c.o, "    .byte 197, 252, 17, 68, 36, 224\n") // C5 FC 11 44 24 E0  vmovups %ymm0,-32(%rsp)  (red zone)
  out_str(c.o, "    .byte 197, 248, 119\n")             // vzeroupper
  out_str(c.o, "    .byte 72, 99, 68, 36, 224\n")       // movslq -32(%rsp),%rax   lane 0
  out_str(c.o, "    .byte 72, 99, 76, 36, 228\n")       // movslq -28(%rsp),%rcx   lane 1
  out_str(c.o, "    .byte 72, 1, 200\n")                // addq %rcx,%rax
  out_str(c.o, "    .byte 72, 99, 76, 36, 232\n")       // lane 2
  out_str(c.o, "    .byte 72, 1, 200\n")
  out_str(c.o, "    .byte 72, 99, 76, 36, 236\n")       // lane 3
  out_str(c.o, "    .byte 72, 1, 200\n")
  out_str(c.o, "    .byte 72, 99, 76, 36, 240\n")       // lane 4
  out_str(c.o, "    .byte 72, 1, 200\n")
  out_str(c.o, "    .byte 72, 99, 76, 36, 244\n")       // lane 5
  out_str(c.o, "    .byte 72, 1, 200\n")
  out_str(c.o, "    .byte 72, 99, 76, 36, 248\n")       // lane 6
  out_str(c.o, "    .byte 72, 1, 200\n")
  out_str(c.o, "    .byte 72, 99, 76, 36, 252\n")       // lane 7
  out_str(c.o, "    .byte 72, 1, 200\n")
  x86ctx_store_result(c, i.result, "rax" as *u8)
  return 0
3193}
3194
3195func x86ctx_emit_i16dot(c: *X86Ctx, i: *Instr) -> i64 {
  x86ctx_load_value_v(c, i.op0, "rax" as *u8)         // rax = a (i16 lanes)
  x86ctx_load_value_v(c, i.op1, "rcx" as *u8)         // rcx = b (i16 lanes)
  x86ctx_load_value_v(c, i.op2, "rdx" as *u8)         // rdx = n lanes
  out_str(c.o, "    .byte 197, 253, 239, 192\n")      // C5 FD EF C0  vpxor %ymm0,%ymm0,%ymm0  (acc = 0)
  x86ctx_emit_i16dot_label(c, i.result)
  out_str(c.o, ":\n")
  out_str(c.o, "    .byte 197, 252, 16, 8\n")         // C5 FC 10 08  vmovups (%rax),%ymm1   (16 lanes of a)
  out_str(c.o, "    .byte 197, 245, 245, 9\n")        // C5 F5 F5 09  vpmaddwd (%rcx),%ymm1,%ymm1  (8 int32 pair sums)
  out_str(c.o, "    .byte 197, 253, 254, 193\n")      // C5 FD FE C1  vpaddd %ymm1,%ymm0,%ymm0  (acc +=)
  out_str(c.o, "    addq $32, %rax\n")                // next 16 lanes of a
  out_str(c.o, "    addq $32, %rcx\n")                // next 16 lanes of b
  out_str(c.o, "    subq $16, %rdx\n")
  out_str(c.o, "    jne ")
  x86ctx_emit_i16dot_label(c, i.result)
  out_char(c.o, 0x0A)
  out_str(c.o, "    .byte 197, 252, 17, 68, 36, 224\n") // C5 FC 11 44 24 E0  vmovups %ymm0,-32(%rsp)  (red zone)
  out_str(c.o, "    .byte 197, 248, 119\n")             // vzeroupper
  out_str(c.o, "    .byte 72, 99, 68, 36, 224\n")       // movslq -32(%rsp),%rax   lane 0
  out_str(c.o, "    .byte 72, 99, 76, 36, 228\n")       // movslq -28(%rsp),%rcx   lane 1
  out_str(c.o, "    .byte 72, 1, 200\n")                // addq %rcx,%rax
  out_str(c.o, "    .byte 72, 99, 76, 36, 232\n")       // lane 2
  out_str(c.o, "    .byte 72, 1, 200\n")
  out_str(c.o, "    .byte 72, 99, 76, 36, 236\n")       // lane 3
  out_str(c.o, "    .byte 72, 1, 200\n")
  out_str(c.o, "    .byte 72, 99, 76, 36, 240\n")       // lane 4
  out_str(c.o, "    .byte 72, 1, 200\n")
  out_str(c.o, "    .byte 72, 99, 76, 36, 244\n")       // lane 5
  out_str(c.o, "    .byte 72, 1, 200\n")
  out_str(c.o, "    .byte 72, 99, 76, 36, 248\n")       // lane 6
  out_str(c.o, "    .byte 72, 1, 200\n")
  out_str(c.o, "    .byte 72, 99, 76, 36, 252\n")       // lane 7
  out_str(c.o, "    .byte 72, 1, 200\n")
  x86ctx_store_result(c, i.result, "rax" as *u8)
  return 0
3230}
3231
3232func x86ctx_emit_i16x16_madd(c: *X86Ctx, i: *Instr) -> i64 {
  x86ctx_load_value_v(c, i.op0, "rax" as *u8)         // rax = acc (i32x8)
  x86ctx_load_value_v(c, i.op1, "rcx" as *u8)         // rcx = a (i16x16)
  x86ctx_load_value_v(c, i.op2, "rdx" as *u8)         // rdx = b (i16x16)
  out_str(c.o, "    .byte 197, 252, 16, 0\n")         // C5 FC 10 00  vmovups (%rax),%ymm0  (acc)
  out_str(c.o, "    .byte 197, 252, 16, 9\n")         // C5 FC 10 09  vmovups (%rcx),%ymm1  (a)
  out_str(c.o, "    .byte 197, 252, 16, 18\n")        // C5 FC 10 12  vmovups (%rdx),%ymm2  (b)
  out_str(c.o, "    .byte 197, 245, 245, 202\n")      // C5 F5 F5 CA  vpmaddwd %ymm2,%ymm1,%ymm1  (16 int16 -> 8 int32)
  out_str(c.o, "    .byte 197, 253, 254, 193\n")      // C5 FD FE C1  vpaddd %ymm1,%ymm0,%ymm0  (acc += )
  out_str(c.o, "    .byte 197, 252, 17, 0\n")         // C5 FC 11 00  vmovups %ymm0,(%rax)  (store acc)
  x86ctx_store_result(c, i.result, "rax" as *u8)
  return 0
3244}
3245
3246// AES-NI: encrypt the 16-byte block at op0 (state ptr) IN PLACE using the 11 expanded
3247// round keys (176B) at op1 (roundkeys ptr).  State lives in %xmm0 across all 10 rounds
3248// (each round key streamed into %xmm1); ~hardware speed vs the ~0.7 MB/s software path.
3249func x86ctx_emit_aesni(c: *X86Ctx, i: *Instr) -> i64 {
  x86ctx_load_value_v(c, i.op0, "rax" as *u8)        // rax = state ptr (in/out)
  x86ctx_load_value_v(c, i.op1, "rcx" as *u8)        // rcx = round keys ptr
  out_str(c.o, "    movdqu (%rax), %xmm0\n")
  out_str(c.o, "    movdqu (%rcx), %xmm1\n")
  out_str(c.o, "    pxor %xmm1, %xmm0\n")            // AddRoundKey rk0
  out_str(c.o, "    movdqu 16(%rcx), %xmm1\n");  out_str(c.o, "    aesenc %xmm1, %xmm0\n")
  out_str(c.o, "    movdqu 32(%rcx), %xmm1\n");  out_str(c.o, "    aesenc %xmm1, %xmm0\n")
  out_str(c.o, "    movdqu 48(%rcx), %xmm1\n");  out_str(c.o, "    aesenc %xmm1, %xmm0\n")
  out_str(c.o, "    movdqu 64(%rcx), %xmm1\n");  out_str(c.o, "    aesenc %xmm1, %xmm0\n")
  out_str(c.o, "    movdqu 80(%rcx), %xmm1\n");  out_str(c.o, "    aesenc %xmm1, %xmm0\n")
  out_str(c.o, "    movdqu 96(%rcx), %xmm1\n");  out_str(c.o, "    aesenc %xmm1, %xmm0\n")
  out_str(c.o, "    movdqu 112(%rcx), %xmm1\n"); out_str(c.o, "    aesenc %xmm1, %xmm0\n")
  out_str(c.o, "    movdqu 128(%rcx), %xmm1\n"); out_str(c.o, "    aesenc %xmm1, %xmm0\n")
  out_str(c.o, "    movdqu 144(%rcx), %xmm1\n"); out_str(c.o, "    aesenc %xmm1, %xmm0\n")
  out_str(c.o, "    movdqu 160(%rcx), %xmm1\n"); out_str(c.o, "    aesenclast %xmm1, %xmm0\n")
  out_str(c.o, "    movdqu %xmm0, (%rax)\n")          // store encrypted block in place
  out_str(c.o, "    xorq %rax, %rax\n")
  x86ctx_store_result(c, i.result, "rax" as *u8)
  return 0
3269}
3270
3271// Emit `palignr $imm,%xmmSRC,%xmmDST` (SRC,DST both 0..7) as raw bytes.  palignr is
3272// 66 0F 3A 0F /r ib and is NOT a mnemonic the sovereign assembler recognises, so -- exactly
3273// like the AVX2 vpmaddwd path above emits VEX ops via .byte -- we encode it directly.  For
3274// xmm0..xmm7 no REX is needed: ModRM(11,dst,src) = 192 | (dst<<3) | src.  This is the ONLY
3275// SHA-NI instruction that needs .byte; all the others (movdqu/movdqa/pshufd/pshufb/paddd/
3276// punpck*/sha256msg1/msg2/rnds2) are assembler mnemonics.
3277func x86ctx_emit_palignr(c: *X86Ctx, dst: i64, src: i64, imm: i64) -> i64 {
  let modrm: i64 = 192 + (dst * 8) + src
  out_str(c.o, "    .byte 102, 15, 58, 15, ")
  out_i64(c.o, modrm)
  out_str(c.o, ", ")
  out_i64(c.o, imm)
  out_char(c.o, 0x0A)
  return 0
3285}
3286
3287// SHA-NI: one full SHA-256 block compression IN PLACE (the Intel SHA extension).  op0=state ptr
3288// (8 contiguous u32 = working state a..h), op1=block ptr (64 raw big-endian message bytes),
3289// op2=K ptr (64 contiguous u32 round constants).  Register plan mirrors the AES emitter: rax=
3290// state, rcx=block, rdx=K; r8 = scratch GPR for the byte-swap mask build.  xmm layout: MSG=xmm0,
3291// STATE0=xmm1, STATE1=xmm2, MSG0..3=xmm3..6, TMP=xmm7, SHUF=xmm8, ABEF_SAVE=xmm9, CDGH_SAVE=xmm10.
3292// Operand order transcribed from the canonical Intel/Linux SHA-NI sequence; validated bit-exact
3293// against the software sha256_compress oracle (nx_shani_block_probe + NIST KAT).  ~hardware speed.
3294func x86ctx_emit_sha256_ni(c: *X86Ctx, i: *Instr) -> i64 {
  x86ctx_load_value_v(c, i.op0, "rax" as *u8)        // rax = state ptr (in/out)
  x86ctx_load_value_v(c, i.op1, "rcx" as *u8)        // rcx = block ptr
  x86ctx_load_value_v(c, i.op2, "rdx" as *u8)        // rdx = K table ptr (64 u32)
  // --- build the big-endian byte-swap mask 0x0c0d0e0f08090a0b0405060700010203 on the stack ---
  // (there is no RIP-relative movdqu in the assembler, so materialise via two 64-bit movabsq
  //  halves + movq stores, then movdqu into xmm8.  r8 is caller-saved scratch, not a home reg.)
  out_str(c.o, "    subq $16, %rsp\n")
  out_str(c.o, "    movabsq $289644378169868803, %r8\n")   // 0x0405060700010203 (bytes 0..7)
  out_str(c.o, "    movq %r8, (%rsp)\n")
  out_str(c.o, "    movabsq $868365760874482187, %r8\n")   // 0x0c0d0e0f08090a0b (bytes 8..15)
  out_str(c.o, "    movq %r8, 8(%rsp)\n")
  out_str(c.o, "    movdqu (%rsp), %xmm8\n")               // SHUF mask
  out_str(c.o, "    addq $16, %rsp\n")
  // --- load + arrange state (memory a b c d | e f g h -> STATE0=ABEF, STATE1=CDGH) ---
  out_str(c.o, "    movdqu (%rax), %xmm1\n")               // STATE0 = DCBA (a b c d)
  out_str(c.o, "    movdqu 16(%rax), %xmm2\n")             // STATE1 = HGFE (e f g h)
  out_str(c.o, "    movdqa %xmm1, %xmm7\n")                // TMP = STATE0
  out_str(c.o, "    punpcklqdq %xmm2, %xmm1\n")            // STATE0 = FEBA
  out_str(c.o, "    punpckhqdq %xmm7, %xmm2\n")            // STATE1 = DCHG
  out_str(c.o, "    pshufd $27, %xmm1, %xmm1\n")           // 0x1B: STATE0 = ABEF
  out_str(c.o, "    pshufd $177, %xmm2, %xmm2\n")          // 0xB1: STATE1 = CDGH
  out_str(c.o, "    movdqa %xmm1, %xmm9\n")                // ABEF_SAVE
  out_str(c.o, "    movdqa %xmm2, %xmm10\n")               // CDGH_SAVE
  // --- load message quads + byte-swap to big-endian ---
  out_str(c.o, "    movdqu (%rcx), %xmm3\n");   out_str(c.o, "    pshufb %xmm8, %xmm3\n")   // W0..3
  out_str(c.o, "    movdqu 16(%rcx), %xmm4\n"); out_str(c.o, "    pshufb %xmm8, %xmm4\n")   // W4..7
  out_str(c.o, "    movdqu 32(%rcx), %xmm5\n"); out_str(c.o, "    pshufb %xmm8, %xmm5\n")   // W8..11
  out_str(c.o, "    movdqu 48(%rcx), %xmm6\n"); out_str(c.o, "    pshufb %xmm8, %xmm6\n")   // W12..15
  // --- 16 quad-steps, uniform macro ---
  var q: i64 = 0
  while q < 16 {
      let m0: i64 = 3 + (q & 3)
      let m1: i64 = 3 + ((q + 1) & 3)
      let m2: i64 = 3 + ((q + 2) & 3)
      let m3: i64 = 3 + ((q + 3) & 3)
      let koff: i64 = q * 16
      // TMP = K[q..] ; TMP += m0
      out_str(c.o, "    movdqu ")
      out_i64(c.o, koff)
      out_str(c.o, "(%rdx), %xmm7\n")
      x86ctx_emit_sha_paddd(c, 7, m0)            // paddd %m0, %xmm7   (TMP = m0 + K)
      if q < 12 {
          x86ctx_emit_sha_msg1(c, m1, m0)        // sha256msg1 %m1, %m0
      }
      x86ctx_emit_sha_movdqa(c, 0, 7)            // movdqa %xmm7, %xmm0  (MSG = TMP)
      out_str(c.o, "    sha256rnds2 %xmm1, %xmm2\n")   // STATE1 = rnds2(STATE1,STATE0,MSG)
      out_str(c.o, "    pshufd $14, %xmm7, %xmm0\n")   // 0x0E: MSG = high 2 dwords of TMP
      out_str(c.o, "    sha256rnds2 %xmm2, %xmm1\n")   // STATE0 = rnds2(STATE0,STATE1,MSG)
      if q < 12 {
          x86ctx_emit_sha_movdqa(c, 7, m3)       // movdqa %m3, %xmm7  (TMP = m3)
          x86ctx_emit_palignr(c, 7, m2, 4)       // palignr $4, %m2, %xmm7
          x86ctx_emit_sha_paddd(c, m0, 7)        // paddd %xmm7, %m0   (m0 += TMP)
          x86ctx_emit_sha_msg2(c, m3, m0)        // sha256msg2 %m3, %m0
      }
      q = q + 1
  }
  // --- add saved state back ---
  out_str(c.o, "    paddd %xmm9, %xmm1\n")                 // STATE0 += ABEF_SAVE
  out_str(c.o, "    paddd %xmm10, %xmm2\n")                // STATE1 += CDGH_SAVE
  // --- unshuffle + store (inverse of the arrange) ---
  out_str(c.o, "    movdqa %xmm1, %xmm7\n")               // TMP = STATE0 (ABEF)
  out_str(c.o, "    punpcklqdq %xmm2, %xmm1\n")           // STATE0 = GHEF
  out_str(c.o, "    punpckhqdq %xmm7, %xmm2\n")           // STATE1 = ABCD
  out_str(c.o, "    pshufd $177, %xmm1, %xmm1\n")         // 0xB1: STATE0 = HGFE
  out_str(c.o, "    pshufd $27, %xmm2, %xmm2\n")          // 0x1B: STATE1 = DCBA
  out_str(c.o, "    movdqu %xmm2, (%rax)\n")              // state[0..3] = a b c d
  out_str(c.o, "    movdqu %xmm1, 16(%rax)\n")            // state[4..7] = e f g h
  out_str(c.o, "    xorq %rax, %rax\n")
  x86ctx_store_result(c, i.result, "rax" as *u8)
  return 0
3365}
3366
3367// Small helpers so the SHA-NI emitter can address xmm3..xmm10 by register NUMBER (the AES/madd
3368// paths only ever name fixed regs).  Each formats one AT&T SSE mnemonic with numeric xmm operands.
3369func x86ctx_emit_sha_paddd(c: *X86Ctx, dst: i64, src: i64) -> i64 {
  out_str(c.o, "    paddd %xmm"); out_i64(c.o, src)
  out_str(c.o, ", %xmm"); out_i64(c.o, dst); out_char(c.o, 0x0A)
  return 0
3373}
3374func x86ctx_emit_sha_movdqa(c: *X86Ctx, dst: i64, src: i64) -> i64 {
  out_str(c.o, "    movdqa %xmm"); out_i64(c.o, src)
  out_str(c.o, ", %xmm"); out_i64(c.o, dst); out_char(c.o, 0x0A)
  return 0
3378}
3379func x86ctx_emit_sha_msg1(c: *X86Ctx, src: i64, dst: i64) -> i64 {
  out_str(c.o, "    sha256msg1 %xmm"); out_i64(c.o, src)
  out_str(c.o, ", %xmm"); out_i64(c.o, dst); out_char(c.o, 0x0A)
  return 0
3383}
3384func x86ctx_emit_sha_msg2(c: *X86Ctx, src: i64, dst: i64) -> i64 {
  out_str(c.o, "    sha256msg2 %xmm"); out_i64(c.o, src)
  out_str(c.o, ", %xmm"); out_i64(c.o, dst); out_char(c.o, 0x0A)
  return 0
3388}
3389
3390// Hardware CLMUL (PCLMULQDQ): carry-less-multiply a selected 64-bit half of *op0 by a
3391// selected half of *op1; the 128-bit product is written back to *op0 IN PLACE.  imm is the
3392// PCLMULQDQ half-select emitted in DECIMAL (the sovereign assembler's $imm parser is decimal
3393// only): 0 (0x00)=op0.lo*op1.lo, 17 (0x11)=op0.hi*op1.hi, 16 (0x10)=op0.lo*op1.hi,
3394// 1 (0x01)=op0.hi*op1.lo.  xmm0/xmm1 are outside the GPR allocator; rax/rcx are caller-saved
3395// scratch (same as the f32/AES paths).  Dispatch sets G1_RAX_SLOT=-1 first to invalidate the
3396// rax-tracking peephole.  This is the GHASH/GF(2^128) accelerator -- fast AES-GCM auth.
3397func x86ctx_emit_clmul(c: *X86Ctx, i: *Instr, imm: i64) -> i64 {
  x86ctx_load_value_v(c, i.op0, "rax" as *u8)        // rax = p ptr (in/out: holds operand a)
  x86ctx_load_value_v(c, i.op1, "rcx" as *u8)        // rcx = q ptr (operand b)
  out_str(c.o, "    movdqu (%rax), %xmm0\n")          // xmm0 = *p (src1)
  out_str(c.o, "    movdqu (%rcx), %xmm1\n")          // xmm1 = *q (src2)
  if imm == 0  { out_str(c.o, "    pclmulqdq $0, %xmm1, %xmm0\n") }
  if imm == 17 { out_str(c.o, "    pclmulqdq $17, %xmm1, %xmm0\n") }
  if imm == 16 { out_str(c.o, "    pclmulqdq $16, %xmm1, %xmm0\n") }
  if imm == 1  { out_str(c.o, "    pclmulqdq $1, %xmm1, %xmm0\n") }
  out_str(c.o, "    movdqu %xmm0, (%rax)\n")          // *p = 128-bit carry-less product
  out_str(c.o, "    xorq %rax, %rax\n")
  x86ctx_store_result(c, i.result, "rax" as *u8)
  return 0
3410}
3411
3412// Fused 4x64-limb wide multiply (__mul256_wide): *dst(u64[8]) = *a(u64[4]) * *b(u64[4]).
3413// The ADX/BMI2 dual-carry-chain schoolbook kernel -- MULX (flags-free 64x64->128) feeds TWO
3414// independent carry chains: ADCX accumulates the low halves via CF, ADOX the high halves via OF,
3415// so both carries propagate in parallel with no software carry (the exact win the plain 4x64 mulq
3416// path lacked -- it paid +4% on software u64_lt carries).  Register plan: rdx=b[j] (mulx implicit
3417// multiplier), rsi=a ptr, rdi=b ptr, r15=a[i] scratch, rax=mulx-lo/zero-source, rcx=mulx-hi,
3418// r8..r11+rbx+r12..r14 = the 8 result limbs r[0..7].  dst ptr is stashed on the stack.  The
3419// callee-saved regs we use (rbx,r12-r15) are push/pop-balanced.  Per-row carry SETTLE folds the
3420// CF/OF tails into the next-higher limb, which is still zero at that point (rows processed low->high),
3421// so a settle add of {0,1}+{0,1} into a zero limb never cascades.  Correctness is difftest-gated
3422// bit-exact vs the proven 8x32 u256_mul_wide oracle; the software path stays the oracle/fallback.
3423func x86ctx_mul256_reg(k: i64) -> *u8 {
  if k == 0 { return "r8" as *u8 }
  if k == 1 { return "r9" as *u8 }
  if k == 2 { return "r10" as *u8 }
  if k == 3 { return "r11" as *u8 }
  if k == 4 { return "rbx" as *u8 }
  if k == 5 { return "r12" as *u8 }
  if k == 6 { return "r13" as *u8 }
  return "r14" as *u8   // k == 7
3432}
3433func x86ctx_emit_mul256_wide(c: *X86Ctx, i: *Instr) -> i64 {
  // Materialise the three pointers into caller-saved regs BEFORE clobbering the callee-saved homes.
  x86ctx_load_value_v(c, i.op0, "rax" as *u8)   // rax = dst ptr
  x86ctx_load_value_v(c, i.op1, "rsi" as *u8)   // rsi = a ptr
  x86ctx_load_value_v(c, i.op2, "rdi" as *u8)   // rdi = b ptr
  // Save the callee-saved regs used as result limbs r[4..7] + the a[i] scratch (r15).
  out_str(c.o, "    pushq %rbx\n")
  out_str(c.o, "    pushq %r12\n")
  out_str(c.o, "    pushq %r13\n")
  out_str(c.o, "    pushq %r14\n")
  out_str(c.o, "    pushq %r15\n")
  out_str(c.o, "    pushq %rax\n")               // stash dst ptr
  // Zero the 8 result limbs.
  out_str(c.o, "    xorq %r8, %r8\n")
  out_str(c.o, "    xorq %r9, %r9\n")
  out_str(c.o, "    xorq %r10, %r10\n")
  out_str(c.o, "    xorq %r11, %r11\n")
  out_str(c.o, "    xorq %rbx, %rbx\n")
  out_str(c.o, "    xorq %r12, %r12\n")
  out_str(c.o, "    xorq %r13, %r13\n")
  out_str(c.o, "    xorq %r14, %r14\n")
  var j: i64 = 0
  while j < 4 {
      out_str(c.o, "    movq ")
      out_i64(c.o, j * 8)
      out_str(c.o, "(%rdi), %rdx\n")             // rdx = b[j]  (mulx implicit multiplier)
      out_str(c.o, "    xorq %rax, %rax\n")       // CF=0, OF=0 for both carry chains
      var ii: i64 = 0
      while ii < 4 {
          out_str(c.o, "    movq ")
          out_i64(c.o, ii * 8)
          out_str(c.o, "(%rsi), %r15\n")          // r15 = a[i]
          out_str(c.o, "    mulx %r15, %rax, %rcx\n")   // rcx:rax = b[j] * a[i]
          out_str(c.o, "    adcx %rax, %")
          out_str(c.o, x86ctx_mul256_reg(j + ii))
          out_str(c.o, "\n")                       // CF chain: r[j+ii] += lo
          out_str(c.o, "    adox %rcx, %")
          out_str(c.o, x86ctx_mul256_reg(j + ii + 1))
          out_str(c.o, "\n")                       // OF chain: r[j+ii+1] += hi
          ii = ii + 1
      }
      // Settle: rax=0 without disturbing flags, fold CF tail into r[j+4]; for j<3 push the
      // residual OF + CF carries into the still-zero limb r[j+5].
      out_str(c.o, "    movq $0, %rax\n")
      out_str(c.o, "    adcx %rax, %")
      out_str(c.o, x86ctx_mul256_reg(j + 4))
      out_str(c.o, "\n")
      if j < 3 {
          out_str(c.o, "    adox %rax, %")
          out_str(c.o, x86ctx_mul256_reg(j + 5))
          out_str(c.o, "\n")
          out_str(c.o, "    adcx %rax, %")
          out_str(c.o, x86ctx_mul256_reg(j + 5))
          out_str(c.o, "\n")
      }
      j = j + 1
  }
  // Reload dst ptr + store the 8 result limbs (before restoring the callee-saved homes).
  out_str(c.o, "    popq %rax\n")
  var k: i64 = 0
  while k < 8 {
      out_str(c.o, "    movq %")
      out_str(c.o, x86ctx_mul256_reg(k))
      out_str(c.o, ", ")
      out_i64(c.o, k * 8)
      out_str(c.o, "(%rax)\n")
      k = k + 1
  }
  out_str(c.o, "    popq %r15\n")
  out_str(c.o, "    popq %r14\n")
  out_str(c.o, "    popq %r13\n")
  out_str(c.o, "    popq %r12\n")
  out_str(c.o, "    popq %rbx\n")
  out_str(c.o, "    xorq %rax, %rax\n")
  x86ctx_store_result(c, i.result, "rax" as *u8)
  return 0
3509}
3510
3511// ===== opcode dispatch ============================================
3512
3513func x86ctx_emit_instr(c: *X86Ctx, i: *Instr) -> i64 {
  let op: i64 = i.op
  // Binops
  if op == OP_ADD   { x86ctx_emit_binop(c, i); return 0 }
  if op == OP_SUB   { x86ctx_emit_binop(c, i); return 0 }
  if op == OP_MUL   { x86ctx_emit_binop(c, i); return 0 }
  if op == OP_UMULHI { x86ctx_emit_binop(c, i); return 0 }
  if op == OP_CRC32  { x86ctx_emit_binop(c, i); return 0 }
  if op == OP_PDEP   { x86ctx_emit_binop(c, i); return 0 }
  if op == OP_PEXT   { x86ctx_emit_binop(c, i); return 0 }
  if op == OP_DIV_S { x86ctx_emit_binop(c, i); return 0 }
  if op == OP_DIV_U { x86ctx_emit_binop(c, i); return 0 }
  if op == OP_REM_S { x86ctx_emit_binop(c, i); return 0 }
  if op == OP_REM_U { x86ctx_emit_binop(c, i); return 0 }
  if op == OP_AND   { x86ctx_emit_binop(c, i); return 0 }
  if op == OP_OR    { x86ctx_emit_binop(c, i); return 0 }
  if op == OP_XOR   { x86ctx_emit_binop(c, i); return 0 }
  if op == OP_SHL   { x86ctx_emit_binop(c, i); return 0 }
  if op == OP_SHR_S { x86ctx_emit_binop(c, i); return 0 }
  if op == OP_SHR_U { x86ctx_emit_binop(c, i); return 0 }
  if op == OP_ROTL64 { x86ctx_emit_binop(c, i); return 0 }
  if op == OP_ROTR64 { x86ctx_emit_binop(c, i); return 0 }
  // Unops
  if op == OP_NEG     { x86ctx_emit_unop(c, i); return 0 }
  if op == OP_NOT     { x86ctx_emit_unop(c, i); return 0 }
  if op == OP_TRUNC   { x86ctx_emit_unop(c, i); return 0 }
  if op == OP_SEXT    { x86ctx_emit_unop(c, i); return 0 }
  if op == OP_ZEXT    { x86ctx_emit_unop(c, i); return 0 }
  if op == OP_BITCAST { x86ctx_emit_unop(c, i); return 0 }
  if op == OP_BSWAP64  { x86ctx_emit_unop(c, i); return 0 }
  if op == OP_POPCNT64 { x86ctx_emit_unop(c, i); return 0 }
  if op == OP_CLZ32    { x86ctx_emit_unop(c, i); return 0 }
  if op == OP_CTZ32    { x86ctx_emit_unop(c, i); return 0 }
  if op == OP_RDTSC    { x86ctx_emit_unop(c, i); return 0 }
  // Atomics
  if op == OP_ATOMIC_LOAD_I64  { x86ctx_emit_atomic(c, i); return 0 }
  if op == OP_ATOMIC_STORE_I64 { x86ctx_emit_atomic(c, i); return 0 }
  if op == OP_ATOMIC_CAS_I64   { x86ctx_emit_atomic(c, i); return 0 }
  if op == OP_ATOMIC_FAA_I64   { x86ctx_emit_atomic(c, i); return 0 }
  if op == OP_ATOMIC_FENCE     { x86ctx_emit_atomic(c, i); return 0 }
  if op == OP_ADC_ACC          { G1_RAX_SLOT = 0 - 1; x86ctx_emit_adc_acc(c, i); return 0 }
  if op == OP_CPUID_EBX        { G1_RAX_SLOT = 0 - 1; x86ctx_emit_cpuid_ebx(c, i); return 0 }
  if op == OP_THREAD_CLONE     { x86ctx_emit_thread_clone(c, i); return 0 }
  // Address-of: emit the alloca slot ADDRESS (leaq) via the as-address
  // load path -- NOT the auto-loading as-value path that unops use.
  if op == OP_ADDR_OF {
      x86ctx_load_value(c, i.op0, "rax" as *u8)
      x86ctx_store_result(c, i.result, "rax" as *u8)
      return 0
  }
  // Compares
  if op == OP_EQ   { x86ctx_emit_cmp(c, i); return 0 }
  if op == OP_NE   { x86ctx_emit_cmp(c, i); return 0 }
  if op == OP_LT_S { x86ctx_emit_cmp(c, i); return 0 }
  if op == OP_LE_S { x86ctx_emit_cmp(c, i); return 0 }
  if op == OP_GT_S { x86ctx_emit_cmp(c, i); return 0 }
  if op == OP_GE_S { x86ctx_emit_cmp(c, i); return 0 }
  // Branches
  if op == OP_BR      { x86ctx_emit_br(c, i); return 0 }
  if op == OP_BR_COND { x86ctx_emit_br_cond(c, i); return 0 }
  // Memory
  if op == OP_LOAD  { x86ctx_emit_load(c, i); return 0 }
  if op == OP_STORE { x86ctx_emit_store(c, i); return 0 }
  if op == OP_GEP   { x86ctx_emit_gep(c, i); return 0 }
  if op == OP_ALLOCA {
      // Address lazily materialised via load_value; nothing emitted here.
      return 0
  }
  // Control
  if op == OP_RETURN    { x86ctx_emit_return(c, i); return 0 }
  if op == OP_CALL      { G1_RAX_SLOT = 0 - 1; x86ctx_emit_call(c, i); return 0 }
  if op == OP_CALL_INDIRECT { G1_RAX_SLOT = 0 - 1; x86ctx_emit_call_indirect(c, i); return 0 }
  if op == OP_TAIL_CALL { G1_RAX_SLOT = 0 - 1; x86ctx_emit_tail_call(c, i); return 0 }
  if op == OP_SYSCALL   { G1_RAX_SLOT = 0 - 1; x86ctx_emit_syscall(c, i); return 0 }
  // hardware float (SSE scalar) -- carried as i64 bit-patterns, clobbers rax/rcx/xmm.
  // x86ctx_emit_float picks f64 (movsd/addsd/...) vs f32 (movss/...) by precision.
  if op == OP_FADD         { G1_RAX_SLOT = 0 - 1; x86ctx_emit_float(c, i); return 0 }
  if op == OP_FSUB         { G1_RAX_SLOT = 0 - 1; x86ctx_emit_float(c, i); return 0 }
  if op == OP_FMUL         { G1_RAX_SLOT = 0 - 1; x86ctx_emit_float(c, i); return 0 }
  if op == OP_FDIV         { G1_RAX_SLOT = 0 - 1; x86ctx_emit_float(c, i); return 0 }
  if op == OP_FSQRT        { G1_RAX_SLOT = 0 - 1; x86ctx_emit_float(c, i); return 0 }
  if op == OP_FCAST_I_TO_F { G1_RAX_SLOT = 0 - 1; x86ctx_emit_float(c, i); return 0 }
  if op == OP_FCAST_F_TO_I { G1_RAX_SLOT = 0 - 1; x86ctx_emit_float(c, i); return 0 }
  if op == OP_F32X4_DOT    { G1_RAX_SLOT = 0 - 1; x86ctx_emit_f32x4_dot(c, i); return 0 }
  if op == OP_I8DOT32      { G1_RAX_SLOT = 0 - 1; x86ctx_emit_i8dot32(c, i); return 0 }
  if op == OP_I8DOT32A     { G1_RAX_SLOT = 0 - 1; x86ctx_emit_i8dot32a(c, i); return 0 }
  if op == OP_I8FMA32      { G1_RAX_SLOT = 0 - 1; x86ctx_emit_i8fma32(c, i); return 0 }
  if op == OP_Q8ROWDOT     { G1_RAX_SLOT = 0 - 1; x86ctx_emit_q8rowdot(c, i); return 0 }
  if op == OP_Q5UNPACK32   { G1_RAX_SLOT = 0 - 1; x86ctx_emit_q5unpack32(c, i); return 0 }
  if op == OP_Q4KUNPACK32S { G1_RAX_SLOT = 0 - 1; x86ctx_emit_q4kunpack32s(c, i); return 0 }
  if op == OP_Q4KSBDOT     { G1_RAX_SLOT = 0 - 1; x86ctx_emit_q4ksbdot(c, i); return 0 }
  if op == OP_F32X8_DOT    { G1_RAX_SLOT = 0 - 1; x86ctx_emit_f32x8_dot(c, i); return 0 }
  if op == OP_F32X8_FMA    { G1_RAX_SLOT = 0 - 1; x86ctx_emit_f32x8_fma(c, i); return 0 }
  if op == OP_F32X8_HSUM   { G1_RAX_SLOT = 0 - 1; x86ctx_emit_f32x8_hsum(c, i); return 0 }
  if op == OP_I16X16_MADD  { G1_RAX_SLOT = 0 - 1; x86ctx_emit_i16x16_madd(c, i); return 0 }
  if op == OP_I16DOT       { G1_RAX_SLOT = 0 - 1; x86ctx_emit_i16dot(c, i); return 0 }
  if op == OP_Q8BLKDOT     { G1_RAX_SLOT = 0 - 1; x86ctx_emit_q8blkdot(c, i); return 0 }
  if op == OP_AES128_ENC_BLOCK { G1_RAX_SLOT = 0 - 1; x86ctx_emit_aesni(c, i); return 0 }
  if op == OP_SHA256_NI_BLOCK  { G1_RAX_SLOT = 0 - 1; x86ctx_emit_sha256_ni(c, i); return 0 }
  if op == OP_MUL256_WIDE      { G1_RAX_SLOT = 0 - 1; x86ctx_emit_mul256_wide(c, i); return 0 }
  if op == OP_CLMUL_LL { G1_RAX_SLOT = 0 - 1; x86ctx_emit_clmul(c, i, 0);  return 0 }
  if op == OP_CLMUL_HH { G1_RAX_SLOT = 0 - 1; x86ctx_emit_clmul(c, i, 17); return 0 }
  if op == OP_CLMUL_LH { G1_RAX_SLOT = 0 - 1; x86ctx_emit_clmul(c, i, 16); return 0 }
  if op == OP_CLMUL_HL { G1_RAX_SLOT = 0 - 1; x86ctx_emit_clmul(c, i, 1);  return 0 }
  if op == OP_COPY      { x86ctx_emit_copy(c, i); return 0 }
  // Unhandled
  out_str(c.o, "    # x86_64: opcode ")
  out_i64(c.o, op)
  out_str(c.o, " not yet wired (session 6 subset)\n")
  return 0
3623}
3624
3625// ===== block + function emission ==================================
3626//
3627// For each block:
3628//   .L<fn>_bb<id>:
3629//   <emit each instr>
3630// Param prologue is omitted in V1 (stack-machine layer assumes
3631// callers already used rdi..r9 directly; for VK_PARAM values we'd
3632// need to spill them into their slot on entry -- session 6b adds
3633// that).
3634
3635func x86ctx_emit_param_prologue(c: *X86Ctx) -> i64 {
  var p: i64 = 0
  let n: i64 = c.f.n_values
  var pi: i64 = 0
  while pi < n {
      let val: *Value = x86ctx_value_at(c.f, pi)
      if val.kind == VK_PARAM {
          let idx: i64 = val.param_index
          if idx >= 0 {
              if idx < 6 {
                  let reg: *u8 = x86_arg_reg_name(idx)
                  x86ctx_store_result(c, pi, reg)
              }
              // Stack params (idx >= 6): SysV ABI puts them at
              // 16(%rbp) + (idx-6)*8 (above saved RA + saved RBP).
              // Caller's x86ctx_emit_call pushes them in reverse;
              // we load them into %rax then store to the local
              // slot like the register-param case.  2026-05-20 fix
              // per Task #93: was silently dropping params 7+.
              if idx >= 6 {
                  let stack_off: i64 = 16 + (idx - 6) * 8
                  out_str(c.o, "    movq ")
                  out_i64(c.o, stack_off)
                  out_str(c.o, "(%rbp), %rax\n")
                  x86ctx_store_result(c, pi, "rax" as *u8)
              }
          }
          p = p + 1
      }
      pi = pi + 1
  }
  return p
3667}
3668
3669// DDR-006 rung 2: the LAST expanded line already announced by a .loc. A row per instruction would
3670// bloat .debug_line with no added information -- the line-program VM holds a line until told
3671// otherwise -- so we emit only on CHANGE, which is exactly what gcc and clang do.
3672static x86_last_loc_exp: i64
3673
3674func x86ctx_emit_block(c: *X86Ctx, bb: *BasicBlock) -> i64 {
  x86ctx_emit_bb_label(c, bb.id)
  out_str(c.o, ":\n")
  G1_RAX_SLOT = 0 - 1        // block boundary = control-flow join: rax unknown
  G1_PENDING_CC = 0 - 1      // G6: flags never cross a block boundary
  G1_PENDING_VAL = 0 - 1
  G24_SKIP = 0
  var i: *Instr = bb.head
  let BUDGET: i64 = 65536
  var iter: i64 = 0
  while i != (0 as *Instr) {
      if iter >= BUDGET { i = 0 as *Instr }
      if i != (0 as *Instr) {
          // DEBUG LINE INFO, rung 2 -- STATEMENT granularity (DDR-006, 2026-08-07).
          // The line rides OUT OF BAND, keyed by the instruction pointer: alloc_instr hands
          // instructions out of a contiguous per-function pool, so the pointer IS a dense
          // identity and struct Instr never had to grow (its own comments record that widening
          // it moved the stride twice and needed lockstep edits across four modules).
          if lm_debug_on() == 1 {
              let il_map: *LineMap = lm_get_active()
              if (il_map as i64) != 0 {
                  let il_exp: i64 = lm_stmt_lookup(i as i64)
                  if il_exp > 0 {
                      if il_exp != x86_last_loc_exp {
                          let il_sc: *i64 = sys_mmap(32) as *i64
                          let il_fp: *i64 = il_sc
                          let il_sp: *i64 = ((il_sc as i64) + 8) as *i64
                          *il_fp = 0
                          *il_sp = 0
                          // Silent when the map cannot PROVE the location -- same rule as the
                          // function-level emit. A wrong line sends a debugger to innocent code.
                          if lm_lookup(il_map, il_exp, il_fp, il_sp) == 1 {
                              out_str(c.o, "    .loc ")
                              out_i64(c.o, *il_fp + 1)
                              out_str(c.o, " ")
                              out_i64(c.o, *il_sp)
                              out_str(c.o, "\n")
                              x86_last_loc_exp = il_exp
                          }
                      }
                  }
              }
          }
          if G24_SKIP > 0 { G24_SKIP = G24_SKIP - 1 } else { x86ctx_emit_instr(c, i) }
          i = i.next
      }
      iter = iter + 1
  }
  return 0
3723}
3724
3725func x86ctx_emit_function(f: *Function, o: *OutBuf) -> i64 {
  let name: *u8 = f.name_start as *u8
  sys_write(2, "fn=" as *u8, 3)
  if name != (0 as *u8) { sys_write(2, name, f.name_len) }
  sys_write(2, "\n" as *u8, 1)
  let c: *X86Ctx = x86ctx_init(f, o)
  G1_RAX_SLOT = 0 - 1        // fresh function: rax holds nothing known
  G1_PENDING_CC = 0 - 1      // G6: no pending compare (statics are BSS-zero
  G1_PENDING_VAL = 0 - 1     // and 0 is a real CC code -- must init here)
3734
  x86_emit_function_start(o, name)
  // DEBUG LINE INFO, rung 1 -- FUNCTION granularity (DDR-002, 2026-08-06).
  // Emits a gas-compatible `.loc <file> <line>` right after the function's label.
  // WHY HERE AND NOT IN THE COMPILER'S OWN TABLE: .debug_line maps ADDRESSES to lines, and
  // the compiler does not know addresses -- only the assembler does. So the compiler's whole
  // job is to say WHICH LINE this label belongs to and let nxasm pair it with the offset it
  // assigns. That split is forced by the format, which is why GCC and Clang both use it.
  // WHY FUNCTION AND NOT STATEMENT: the IR carries no line numbers at all (Tok does, Instr
  // does not), so statement granularity is a much larger arc. Function granularity needs only
  // what the parser already recorded, and it is what makes a backtrace name its frames.
  // BYTE-NEUTRAL TODAY: nxasm's directive dispatch is a flat if-chain with no else branch, so
  // an unrecognised directive is ignored. The equivalence gate can PROVE that -- identical
  // per-row ea/eb byte counts, not merely a GREEN verdict.
  // OPT-IN (-g). Default OFF so the emitted binary is byte-identical to a no-debug build --
  // see the switch's note in nx_linemap.nx for why that default is load-bearing.
  // Fresh function: nothing has been announced yet, so the first statement must emit its .loc
  // even if it happens to share a line with the previous function's last statement.
  x86_last_loc_exp = 0
  var dl_map: *LineMap = 0 as *LineMap
  if lm_debug_on() == 1 { dl_map = lm_get_active() }
  if (dl_map as i64) != 0 {
      let dl_exp: i64 = lm_fn_lookup(dl_map, name, f.name_len)
      if dl_exp > 0 {
          let dl_sc: *i64 = sys_mmap(32) as *i64
          let dl_fp: *i64 = dl_sc
          let dl_sp: *i64 = ((dl_sc as i64) + 8) as *i64
          *dl_fp = 0
          *dl_sp = 0
          // Silent when the map cannot PROVE the location. A wrong line in debug info sends a
          // debugger to innocent code -- the same class as a caret under the wrong line.
          if lm_lookup(dl_map, dl_exp, dl_fp, dl_sp) == 1 {
              out_str(o, "    .loc ")
              out_i64(o, *dl_fp + 1)
              out_str(o, " ")
              out_i64(o, *dl_sp)
              out_str(o, "\n")
              // Seed the change-tracker so the first statement on this same line does not emit a
              // second, identical .loc immediately after the function label.
              x86_last_loc_exp = dl_exp
          }
      }
  }
  x86_emit_prologue(o, c.frame_size)
  // G1 FIX-17: save callee-saved homes AFTER the prologue but BEFORE the param
  // prologue (a param homed in r12 must not be captured as the caller's r12).
  x86ctx_emit_cs_save(c)
3781
  // Spill params from arg-regs to their stack slots.
  x86ctx_emit_param_prologue(c)
3784
  // G5: load homed BIG constants into their callee-saved homes, once per
  // function (after cs_save -- the caller's register values are already
  // banked; before any block -- every use site reads the home).
  var g5cv: i64 = 0
  while g5cv < f.n_values {
      let g5cl: *ValueLoc = ((c.locs as i64) + g5cv * 16) as *ValueLoc
      if g5cl.kind == VL_REGISTER {
          let g5cval: *Value = x86ctx_value_at(f, g5cv)
          if g5cval.kind == VK_CONST_INT {
              x86_emit_movabsq(o, x86_home_reg_name(g5cl.idx), g5cval.const_int)
          }
      }
      g5cv = g5cv + 1
  }
3799
  // Emit each block. G12: track the NEXT emitted block's id so terminators
  // can elide jumps-to-fall-through (-1 for the last block = never elide).
  var b: i64 = 0
  while b < f.n_blocks {
      let bb: *BasicBlock = x86ctx_block_at(f, b)
      c.next_bb = 0 - 1
      if b + 1 < f.n_blocks {
          let nbb: *BasicBlock = x86ctx_block_at(f, b + 1)
          c.next_bb = nbb.id
      }
      x86ctx_emit_block(c, bb)
      b = b + 1
  }
3813
  // Defensive epilogue if the IR didn't terminate (shouldn't happen
  // for well-formed IR; harmless safety net).
  x86_emit_movabsq(o, "rax" as *u8, 0)
  x86ctx_emit_cs_restore(c)
  x86_emit_epilogue(o)
  x86_emit_function_end(o, name)
  return 0
3821}
3822
3823// ===== module-level globals dump (session 6b) ====================
3824//
3825// Parallels nx_nxc.nx stage 3.5 (and riscv.c's globals dump).  Each
3826// VK_GLOBAL Value references a Module.globals[id] entry; the asm
3827// emitted for the function body says `leaq .Lg<id>(%rip), %reg`,
3828// which is a forward reference.  Without this section emitted at
3829// the end of the module, those leaq refs unresolved.
3830//
3831// Emits:
3832//   .section .rodata
3833//   .Lg0: .asciz "bytes..."
3834//   .Lg1: .asciz "bytes..."
3835//   ...
3836//   .data    (for writable globals)
3837//   <name>: .quad ... or .byte ...
3838//   .bss     (for zero-init globals)
3839//   .lcomm <name>, <len>
3840
3841// LN42 (2026-09-03): does this global carry any non-zero initializer byte?
3842func x86ctx_global_nonzero(g: *Global) -> i64 {
  if g.bytes == (0 as *u8) { return 0 }
  var i: i64 = 0
  while i < g.len {
      if g.bytes[i] != (0 as u8) { return 1 }
      i = i + 1
  }
  return 0
3850}
3851
3852func x86ctx_emit_module_globals(m: *Module, o: *OutBuf) -> i64 {
  if m == (0 as *Module) { return 0 }
  if m.n_globals <= 0 { return 0 }
  let g_base: i64 = m.globals as i64
3856
  // ---- LN42: WRITABLE NON-ZERO STATICS GO IN .data, AND THEY GO FIRST -------------------------
  // Until now a writable static could only be ZERO-initialised: .rodata is read-only on the GNU
  // as/ld lane so a write SIGSEGVs there, and nxasm mapped .data to its ignore bucket, leaving
  // .lcomm (zero-fill, BSS) as the only both-lanes-writable form -- so a non-zero one was refused
  // by a deliberately invalid line. But nxasm's section-1 tail is ALREADY WRITABLE (that is exactly
  // what .lcomm allocates into, "a zero-filled MUTABLE slot ... appended to the data tail"), and
  // .data is writable on the GNU lane by definition. Teaching the assembler .data makes it the
  // both-lanes-correct emission this comment used to say did not exist. Three organs carried the
  // workaround by name (nx_lawpath, nx_rv64_fast, nx_toolsafety_lib) and a DUAL-TARGET module could
  // not declare anything at all -- which is how this was found.
  //
  // WHY FIRST, AND WHY PADDED: an i64 static must be 8-aligned. .lcomm aligns itself; a raw .byte
  // run does not, and the rodata string stream leaves the section cursor at an arbitrary offset.
  // Emitting this block BEFORE any string starts it at cursor 0 (aligned), and padding each entry
  // up to a multiple of 8 keeps every following entry aligned too.
  //
  // BYTE-IDENTICAL WHEN UNUSED: with no writable non-zero global the block is skipped entirely and
  // the stream below is exactly what it was -- checked by rebuilding, not asserted.
  var nzc: i64 = 0
  var nzi: i64 = 0
  while nzi < m.n_globals {
      let gz: *Global = (g_base + nzi * 80) as *Global
      if gz.zero_init == 0 { if gz.writable == 1 { if x86ctx_global_nonzero(gz) == 1 { nzc = nzc + 1 } } }
      nzi = nzi + 1
  }
  if nzc > 0 {
      out_str(o, "    .data
3884")
      var nzk: i64 = 0
      while nzk < m.n_globals {
          let gk: *Global = (g_base + nzk * 80) as *Global
          if gk.zero_init == 0 { if gk.writable == 1 { if x86ctx_global_nonzero(gk) == 1 {
              out_str(o, ".Lg")
              out_i64(o, gk.id)
              out_str(o, ":
  .byte ")
              var nb: i64 = 0
              while nb < gk.len {
                  out_i64(o, gk.bytes[nb])
                  out_str(o, ", ")
                  nb = nb + 1
              }
              // ALWAYS pad to the NEXT 8-boundary -- when the length is already a multiple of 8
              // that is a full 8 bytes, deliberately: the content loop emits a trailing ", " after
              // every byte, so a zero-length pad would leave the line ending in a comma with no
              // operand, and a bare `.byte` is the empty-literal miscompile this file already
              // carries a standing witness for. The cost is at most 8 bytes per non-zero static and
              // it buys 8-alignment for every entry that follows.
              var pad: i64 = (8 - (gk.len & 7)) & 7
              if pad == 0 { pad = 8 }
              var pn: i64 = 0
              while pn < pad {
                  out_i64(o, 0)
                  if pn + 1 < pad { out_str(o, ", ") }
                  pn = pn + 1
              }
              out_char(o, 0x0A)
          } } }
          nzk = nzk + 1
      }
  }
3918
  x86_emit_section_rodata(o)
  var i: i64 = 0
  while i < m.n_globals {
      // Stride 80 -- unified per nx_nxc.nx pool layout.  Wrong
      // stride here caused SIGSEGV after all functions emitted
      // (session 9 bisect 2026-05-17).
      let g: *Global = (g_base + i * 80) as *Global
3926
      if g.zero_init == 0 {
          // X-G3 2026-07-15: WRITABLE data statics (ir_add_global_data --
          // the 07-14 static-init parse change) must NOT land in .rodata:
          // on the GNU as/ld lane .rodata pages are read-only, so the
          // first static write SIGSEGVs (caught by the G2 gauntlet run,
          // ed25519 KAT nx_scratch_init writing .Lg0; error-7 write fault
          // at the rodata page). All-zero payloads emit as the exact
          // pre-regression `.lcomm` (BSS -- writable on BOTH lanes; nxasm
          // IGNORES `.data` sections wholesale (nxasm_x86.nx:810), so
          // .lcomm is the only both-lanes-writable form). A NONZERO-init
          // static has no both-lanes-correct emission yet -> emit a loud
          // invalid line so the build FAILS instead of silently zeroing
          // the initializer (no silent caps). Strings (writable=0) keep
          // the .rodata path byte-identical.
          if g.writable == 1 {
              // LN42: a NON-ZERO writable static was emitted into the .data block above; only the
              // all-zero ones reach .lcomm, and that path is byte-for-byte what it always was.
              if x86ctx_global_nonzero(g) == 0 {
                  out_str(o, "    .lcomm .Lg")
                  out_i64(o, g.id)
                  out_str(o, ", ")
                  out_i64(o, g.len)
                  out_char(o, 0x0A)
              }
          }
          if g.writable == 0 {
              // Anonymous globals always use .Lg<id>.  Named globals
              // are queued for a follow-up; ingest CLI doesn't ship any.
              out_str(o, ".Lg")
              out_i64(o, g.id)
              out_str(o, ":\n    .byte ")
              // Per nx_nxc.nx: use .byte listing rather than .asciz so
              // the assembler can't reinterpret any escape sequences.
              // Raw bytes round-trip safely.
              // Emit "<byte>, " per byte and then an UNCONDITIONAL terminating 0.
              // This produces exactly the same text as the previous
              // "join with ', '" + "append ', 0' when len > 0" form for every
              // NON-EMPTY literal -- but it also emits the terminator for the
              // EMPTY one, which the old shape did not.
              //
              // THE EMPTY-LITERAL MISCOMPILE (fixed here, 2026-07-25): with len == 0
              // the loop wrote nothing and the `len > 0` guard suppressed the
              // terminator, so the emitter produced a bare `.byte` with NO operands.
              // The label .Lg<id> then resolved to the FOLLOWING global's first byte,
              // making `""` silently ALIAS the next literal in the pool: the standing
              // witness runtime/nx_empty_lit_probe.nx measured strlen("") == 4 and
              // ("" as i64) == ("HOLD" as i64). Because pool layout shifts between
              // builds, the damage moved around -- nondeterminism ACROSS builds with
              // determinism WITHIN one binary was the signature. It corrupted a swarm
              // queue with "HOLD" and produced a false "seg_store corruption" verdict.
              var bi: i64 = 0
              while bi < g.len {
                  out_i64(o, g.bytes[bi])
                  out_str(o, ", ")
                  bi = bi + 1
              }
              out_i64(o, 0)
              out_char(o, 0x0A)
          }
      }
      if g.zero_init == 1 {
          out_str(o, "    .lcomm .Lg")
          out_i64(o, g.id)
          out_str(o, ", ")
          out_i64(o, g.len)
          out_char(o, 0x0A)
      }
3994
      i = i + 1
  }
  return 0
3998}
3999
4000// ===== whole-module emission ======================================
4001//
4002// Top-level: emits the standard _start trampoline + every function
4003// + the globals dump.  The session-7 driver invokes this once per
4004// module to produce a complete .s file.
4005
4006func x86ctx_emit_module(m: *Module, o: *OutBuf) -> i64 {
  return x86ctx_emit_module_live(m, o, 0 as *u8)
4008}
4009
4010// Same as x86ctx_emit_module, restricted to the functions a reachability pass marked live.
4011// `live` is a per-slot byte map indexed like m.functions[] (see opt_module_dce_mark, nx_opt.nx);
4012// a NULL map means "emit everything" and is exactly the pre-B1 behaviour -- x86ctx_emit_module
4013// above is that call, so every existing caller is byte-for-byte unchanged. Dead functions are
4014// SKIPPED, never rewritten: the pool, its indices and every callee pointer stay as parsed.
4015// Globals are still dumped in full (a dead function's string literals cost .rodata bytes, not
4016// correctness; trimming them is a separate, measured rung).
4017func x86ctx_emit_module_live(m: *Module, o: *OutBuf, live: *u8) -> i64 {
  if m == (0 as *Module) { return 0 }
4019
  // _start trampoline.  On Linux x86_64 process entry, the SysV ABI
  // for _start places (from sp): argc, argv[0..argc], NULL, envp...
  // To satisfy `main(argc: i64, argv: *i64)`'s SysV calling convention
  // we must MOVE argc into %rdi (1st arg) and pointer-to-argv into
  // %rsi (2nd arg) BEFORE calling main, then exit with main's
  // return value.
  //
  // Prior implementation called main with %rdi / %rsi unset, so
  // path-mode binaries that read argv[1] saw garbage and behaved as
  // though invoked with no args.  Fixed 2026-05-21 during the
  // native-x86_64 self-host bootstrap diagnosis.
4031
  // DWARF FILE TABLE (DDR-002, 2026-08-06). The assembler is a SEPARATE PROCESS and never
  // sees the LineMap, so the .s has to carry the file names itself -- which is precisely why
  // gas grew `.file N "path"`. The `.loc` directives emitted per function index into this
  // table, so the two must agree on numbering: both are 1-based here.
  // FULL path on purpose: the basename is what a HUMAN wants in a diagnostic, but a debugger
  // has to FIND the source. DDR-001 kept the full path in the map for exactly this moment.
  // Byte-neutral today -- nxasm ignores unrecognised directives.
  // OPT-IN (-g), same switch as .loc -- the file table is useless without the .loc rows and
  // both must appear together or neither: an assembler that saw one and not the other would
  // have to guess, and a debug format is exactly where guessing is worst.
  var fl_map: *LineMap = 0 as *LineMap
  if lm_debug_on() == 1 { fl_map = lm_get_active() }
  if (fl_map as i64) != 0 {
      var fli: i64 = 0
      while fli < fl_map.n_files {
          let flp: *u8 = lm_file_path(fl_map, fli)
          if (flp as i64) != 0 {
              out_str(o, "    .file ")
              out_i64(o, fli + 1)
              out_str(o, " \"")
              out_str(o, flp)
              out_str(o, "\"\n")
          }
          fli = fli + 1
      }
  }
4058
  out_str(o, "    .text\n")
  out_str(o, "    .globl _start\n")
  out_str(o, "_start:\n")
  out_str(o, "    movq (%rsp), %rdi\n")     // argc
  out_str(o, "    leaq 8(%rsp), %rsi\n")    // argv
  out_str(o, "    call main\n")
  out_str(o, "    movq %rax, %rdi\n")
  // exit_group, NOT exit (2026-07-07, threading live): nx threads
  // are CLONE_VM tasks with separate PIDs, so plain exit(60) after
  // main returns leaves live pool workers running -- they hold
  // stdout open and wedge any pipeline waiting for EOF (this hung
  // the build lane 22min via the shared-pool dispatcher).  Return-
  // from-main must terminate the WHOLE thread group, same contract
  // as every threaded libc.  Per-thread exit stays sys_exit(93->60)
  // in nx_thread_exit.
  x86_emit_movabsq(o, "rax" as *u8, NX_X64_SYS_EXIT_GROUP)
  x86_emit_syscall(o)
4076
  // Per-function emission (B1: only slots the live map keeps; NULL map keeps all).
  var i: i64 = 0
  while i < m.n_functions {
      let fn_base: i64 = m.functions as i64
      let f: *Function = (fn_base + i * 176) as *Function
      var emit_it: i64 = 1
      if live != (0 as *u8) { if live[i] == 0 { emit_it = 0 } }
      if emit_it == 1 { x86ctx_emit_function(f, o) }
      i = i + 1
  }
4087
  // Globals dump.
  x86ctx_emit_module_globals(m, o)
4090
  // GNU-stack note.
  x86_emit_gnu_stack_note(o)
  return 0
4094}
nx_x86_64_ctx.nx source

`nx_x86_64_ctx.nx` source
