kit

native.c (187438B)

---

 /* src/arch/x64/native.c — x86-64 (SysV / Win64) NativeTarget implementation.
  *
  * Mirrors the rv64 reference (src/arch/rv64/native.c): a physical-emission
  * NativeTarget driven at -O0 by the shared NativeDirectTarget and at -O1+ by
  * the optimizer emit path. ABI decisions route through abi/ and the per-OS
  * X64ABIRegs (x64_abi_for_os); this file owns ISA emission and the x64 frame
  * layout.
  *
  * Frame model (single, rbp-anchored): the prologue does `push rbp; mov rbp,rsp;
  * sub rsp,frame_size`. Local/spill slots live below rbp at positive byte
  * offsets `off` (address = rbp - off). Incoming stack args sit above the saved
  * return address at [rbp + 16 + shadow_space + ...]. Callee-saved GPRs (and, on
  * Win64, XMMs) are saved below the locals; outgoing args sit at [rsp + 0..].
  * The single-pass (-O0) prologue reserves a NOP placeholder patched in func_end
  * once max_outgoing and callee-saves are known.
  *
  * Register model. R10/R11 and XMM14/XMM15 are backend-private hook/asm
  * temporaries and are never optimizer operand locations. RSP/RBP are reserved
  * stack/frame pointers. RAX is reserved too (return value and div/mul implicit
  * operand), but is not in either temporary bank, so inline asm may pin an
  * operand to it (the Linux syscall idiom) — see x64_asm_operand_reg_ok. The O1
  * instruction-scoped operand bank is R8/R9 (int) and XMM4/XMM5 (fp),
  * caller-saved on both SysV and Win64; the O0 direct target happens to use the
  * same registers through its independent `scratch` policy. Both banks are
  * reserved from persistent allocation.
  * Callee-saved set is resolved per-OS via x64_abi_for_os at runtime (the
  * legality masks below are SysV's, the conservative superset that both ABIs'
  * allocators respect — Win64's extra callee-saves RDI/RSI/xmm6-15 only shrink
  * the allocable pool, never grow it). */
 
 #include <string.h>
 
 #include "abi/abi.h"
 #include "arch/x64/asm.h"
 #include "arch/x64/emit.h"
 #include "arch/x64/isa.h"
 #include "arch/x64/regs.h"
 #include "arch/x64/x64.h"
 #include "asm/asm.h"
 #include "asm/asm_lex.h"
 #include "cg/native_argmove.h"
 #include "cg/native_asm.h"
 #include "cg/native_direct_target.h"
 #include "cg/native_frame.h"
 #include "cg/type.h"
 #include "core/arena.h"
 #include "core/bytes.h"
 #include "core/pool.h"
 #include "core/slice.h"
 #include "obj/obj.h"
 
 enum {
   X64_TMP_INT = X64_R10,      /* backend-private int temp (reserved) */
   X64_TMP_INT2 = X64_R11,     /* backend-private int temp (reserved) */
   X64_TMP_FP = X64_XMM0 + 14, /* backend-private fp temp (reserved) */
   X64_TMP_FP2 = X64_XMM15,    /* backend-private fp temp (reserved) */
   X64_MAX_REG_ARG_MOVES = 16u,
   /* Deferred entry register-binds (-O1): bounded by simultaneously-live
    * register-homed param parts, i.e. the allocable register count. */
   X64_MAX_BIND_MOVES = 32u,
   X64_MAX_CS_FP_REGS = 10u, /* Win64 xmm6..xmm15 */
 };
 
 /* ============================ target state ============================ */
 
 /* Frame slots and callee-save records live in the shared NativeFrame
  * bookkeeping (cg/native_frame.h); these aliases keep the x64-local spellings.
  * x64 reads only .reg/.cls of a callee-save (it computes save offsets below the
  * locals rather than homing them in frame slots, so .slot/.type stay unused).
  */
 typedef NativeFrameSlotEntry X64NativeSlot;
 typedef NativeFrameCalleeSave X64CalleeSave;
 
 typedef enum X64PatchKind { X64_PATCH_ALLOCA } X64PatchKind;
 
 typedef struct X64Patch {
   u8 kind; /* X64PatchKind */
   u32 pos; /* byte offset of the disp32 to patch */
 } X64Patch;
 
 typedef struct X64NativeTarget {
   NativeTarget base;
   SrcLoc loc;
   const CGFuncDesc* func;
 
   /* Shared frame bookkeeping: slot table, cum_off, max_outgoing, callee-save
    * set, and the known_frame / has_alloca / frame_final flags. */
   NativeFrame frame;
   u32 frame_size_final;
 
   u32 incoming_stack_size; /* fixed-param stack bytes (tail-call check) */
   u32 next_param_int;
   u32 next_param_fp;
   u32 next_param_stack;
   u8 has_sret;
   u8 is_variadic;
   NativeFrameSlot sret_ptr_slot;
   NativeFrameSlot reg_save_slot; /* SysV variadic 176B __va_list_tag area */
 
   X64Patch* patches;
   u32 npatches;
   u32 patches_cap;
   u32 nalloca;
 
   u32 func_start;
   u32 prologue_pos;
   u32 prologue_nbytes;
   MCLabel epilogue_label;
 
   /* Known-frame (-O1) prologue cost-model tiers, settled in
    * x64_func_begin_known_frame; both 0 on the single-pass path (which can't
    * know the frame up front). Either one suppresses the `sub rsp` reservation;
    * the rbp frame record (push rbp; mov rbp,rsp) and every rbp-relative offset
    * stay unchanged, so the epilogue (`leave`), CFI (CFA = rbp+16), and debug
    * locs are identical to the fat shape. slim_frame   - empty frame (no
    * callee-saves/locals/outgoing/alloca): the `sub rsp` reserved nothing, so it
    * is simply dropped. Safe for non-leaves (push rbp keeps rsp 16-aligned for
    * calls, and nothing lives below rsp). SysV + Win64. redzone_leaf - SysV leaf
    * with a small frame (<= 128B, no alloca, no outgoing args):
    * locals/callee-saves stay at their rbp-relative offsets, which now land in
    * the 128-byte red zone instead of a reserved region. Leaf-only — a call
    * would clobber the red zone. */
   u8 slim_frame;
   u8 redzone_leaf;
 
   /* Optimizer (-O1) entry binds: register-destination param binds are deferred
    * here and resolved as a parallel copy in x64_bind_params_end, since the
    * allocator may rotate params across the incoming arg registers — a
    * permutation the naive per-param move order would clobber. */
   NativeArgMove bind_moves[X64_MAX_BIND_MOVES];
   u32 nbind_moves;
 
   const X64ABIRegs* abi;
 } X64NativeTarget;
 
 static X64NativeTarget* x64_of(NativeTarget* t) { return (X64NativeTarget*)t; }
 
 static _Noreturn void x64_panic(X64NativeTarget* a, const char* msg) {
   compiler_panic(a->base.c, a->loc, "x64 native target: %s", msg);
 }
 
 static X64NativeSlot* x64_slot_get(X64NativeTarget* a, NativeFrameSlot fs) {
   return native_frame_slot_at(&a->frame, fs);
 }
 
 /* ============================ type helpers ============================ */
 
 /* Scalar size/align/mem/class/loc constructors are shared in native_target.h
  * (native_type_size, native_type_align, native_mem_for_type,
  * native_class_for_type_fp_le8, native_loc_reg, native_loc_stack,
  * native_loc_is_fp). loc_reg's mask is arch-specific and stays here. */
 
 /* A scalar value occupies a 64-bit register when it is pointer-sized or wider
  * (drives REX.W selection). */
 static int x64_is_64(NativeTarget* t, KitCgTypeId type) {
   return native_type_size(t, type) >= 8u || cg_type_is_ptr(t->c, type);
 }
 
 /* Scalar byte width of a register operand. Reads the NDT-stamped szinfo
  * descriptor (one byte) when present, else falls back to the live type query —
  * byte-identical to native_type_size for in-range scalars. */
 static u32 loc_size32(NativeTarget* t, NativeLoc loc) {
   if (loc.szinfo & NATIVE_SZINFO_VALID) return native_szinfo_size(loc.szinfo);
   return native_type_size(t, loc.type);
 }
 
 /* Select the physical integer carrier width. Native scalar sizes are
  * 1/2/4/8, while a small direct aggregate may legitimately remain packed in a
  * register under the -O0 cache; its 5/6/7-byte representation needs a 64-bit
  * move even though its semantic size is below eight bytes. */
 static int loc_is_64(NativeTarget* t, NativeLoc loc) {
   return loc_size32(t, loc) > 4u || cg_type_is_ptr(t->c, loc.type);
 }
 
 static u32 loc_reg(NativeLoc loc) { return loc.v.reg & 0xfu; }
 
 /* SSE scalar prefix: F2 (double / 8-byte) vs F3 (single / 4-byte). */
 static u8 sse_scalar_prefix(u32 size) { return size == 8u ? 0xF2u : 0xF3u; }
 
 /* Forward decls for the rel32 branch emitters (used by convert before the
  * control-flow section defines them). */
 static void emit_jmp_rel32(MCEmitter* mc, MCLabel l);
 static void emit_jcc_rel32(MCEmitter* mc, u32 cc, MCLabel l);
 
 /* ============================ register tables ============================ */
 
 #define X64_PHYS_INT_ARG(r)                                                  \
   {.reg = (r),                                                               \
    .cls = NATIVE_REG_INT,                                                    \
    .abi_index = 0xffu,                                                       \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLER_SAVED | NATIVE_REG_ARG, \
    .spill_cost = 1u,                                                         \
    .copy_cost = 1u}
 #define X64_PHYS_INT_ARG_RESERVED(r)                                        \
   {.reg = (r),                                                              \
    .cls = NATIVE_REG_INT,                                                   \
    .abi_index = 0xffu,                                                      \
    .flags = NATIVE_REG_CALLER_SAVED | NATIVE_REG_ARG | NATIVE_REG_RESERVED, \
    .spill_cost = 0u,                                                        \
    .copy_cost = 0u}
 #define X64_PHYS_INT_RET_ARG(r)                                               \
   {.reg = (r),                                                                \
    .cls = NATIVE_REG_INT,                                                     \
    .abi_index = 0xffu,                                                        \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLER_SAVED | NATIVE_REG_ARG | \
             NATIVE_REG_RET,                                                   \
    .spill_cost = 1u,                                                          \
    .copy_cost = 1u}
 #define X64_PHYS_INT_CALLER(r)                              \
   {.reg = (r),                                              \
    .cls = NATIVE_REG_INT,                                   \
    .abi_index = 0xffu,                                      \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLER_SAVED, \
    .spill_cost = 1u,                                        \
    .copy_cost = 1u}
 #define X64_PHYS_INT_CALLEE(r)                              \
   {.reg = (r),                                              \
    .cls = NATIVE_REG_INT,                                   \
    .abi_index = 0xffu,                                      \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLEE_SAVED, \
    .spill_cost = 4u,                                        \
    .copy_cost = 1u}
 #define X64_PHYS_INT_RESERVED_ROLE(r, role) \
   {.reg = (r),                               \
    .cls = NATIVE_REG_INT,                    \
    .abi_index = 0xffu,                       \
    .flags = NATIVE_REG_RESERVED | (role),    \
    .spill_cost = 0u,                         \
    .copy_cost = 0u}
 #define X64_PHYS_INT_RESERVED(r) \
   X64_PHYS_INT_RESERVED_ROLE((r), NATIVE_REG_NONE)
 #define X64_PHYS_INT_RESERVED_CALLER(r) \
   X64_PHYS_INT_RESERVED_ROLE((r), NATIVE_REG_CALLER_SAVED)
 #define X64_PHYS_INT_RESERVED_CALLEE(r) \
   X64_PHYS_INT_RESERVED_ROLE((r), NATIVE_REG_CALLEE_SAVED)
 
 /* The NDT (-O0) value-cache / scratch pool (read only by NativeDirectTarget;
  * the optimizer allocates over the phys[] ALLOCABLE flags instead, which also
  * include rcx/rdx/rsi/rdi). Caller-saved RSI/RDI lead so the cache prefers them
  * (no prologue save) and only spills into callee-saved R13-R15 under pressure
  * (the fixed-size save region covers them). RSI/RDI are safe to cache: they have
  * no implicit x86 use — unlike RCX (shift count) and RAX/RDX (div/mul), which
  * are therefore excluded — and incoming args are spilled to frame homes at entry
  * before any body op runs. R8/R9 are also the O1 operand-temp bank; R10/R11 are
  * backend-private (reserved); RAX is reserved (return/div-mul, asm-pinnable). */
 static const Reg x64_ndt_int_allocable[] = {X64_RSI, X64_RDI, X64_R13,
                                             X64_R14, X64_R15};
 static const Reg x64_int_scratch[] = {X64_R8, X64_R9};
 static const Reg x64_int_asm_temps[] = {X64_TMP_INT, X64_TMP_INT2};
 static const Reg x64_direct_asm_int[] = {X64_RDI, X64_RSI, X64_RDX, X64_RCX};
 
 static const NativePhysRegInfo x64_int_phys[] = {
     X64_PHYS_INT_RESERVED_ROLE(
         X64_RAX, NATIVE_REG_CALLER_SAVED | NATIVE_REG_RET),
     X64_PHYS_INT_ARG(X64_RCX),
     X64_PHYS_INT_RET_ARG(X64_RDX),
     X64_PHYS_INT_RESERVED_CALLEE(X64_RBX),
     X64_PHYS_INT_RESERVED(X64_RSP), /* stack pointer */
     X64_PHYS_INT_RESERVED(X64_RBP), /* frame pointer */
     X64_PHYS_INT_ARG(X64_RSI),
     X64_PHYS_INT_ARG(X64_RDI),
     X64_PHYS_INT_ARG_RESERVED(X64_R8), /* O1 operand temp / O0 scratch */
     X64_PHYS_INT_ARG_RESERVED(X64_R9), /* O1 operand temp / O0 scratch */
     X64_PHYS_INT_RESERVED_CALLER(X64_R10), /* backend-private temp */
     X64_PHYS_INT_RESERVED_CALLER(X64_R11), /* backend-private temp */
     X64_PHYS_INT_RESERVED_CALLEE(X64_R12),
     X64_PHYS_INT_CALLEE(X64_R13),
     X64_PHYS_INT_CALLEE(X64_R14),
     X64_PHYS_INT_CALLEE(X64_R15),
 };
 
 #define X64_PHYS_FP_ARG_RET(r)                                                \
   {.reg = (r),                                                                \
    .cls = NATIVE_REG_FP,                                                      \
    .abi_index = 0xffu,                                                        \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLER_SAVED | NATIVE_REG_ARG | \
             NATIVE_REG_RET,                                                   \
    .spill_cost = 1u,                                                          \
    .copy_cost = 1u}
 #define X64_PHYS_FP_ARG(r)                                                   \
   {.reg = (r),                                                               \
    .cls = NATIVE_REG_FP,                                                     \
    .abi_index = 0xffu,                                                       \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLER_SAVED | NATIVE_REG_ARG, \
    .spill_cost = 1u,                                                         \
    .copy_cost = 1u}
 #define X64_PHYS_FP_ARG_RESERVED(r)                                         \
   {.reg = (r),                                                              \
    .cls = NATIVE_REG_FP,                                                    \
    .abi_index = 0xffu,                                                      \
    .flags = NATIVE_REG_CALLER_SAVED | NATIVE_REG_ARG | NATIVE_REG_RESERVED, \
    .spill_cost = 0u,                                                        \
    .copy_cost = 0u}
 #define X64_PHYS_FP_CALLER(r)                               \
   {.reg = (r),                                              \
    .cls = NATIVE_REG_FP,                                    \
    .abi_index = 0xffu,                                      \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLER_SAVED, \
    .spill_cost = 1u,                                        \
    .copy_cost = 1u}
 #define X64_PHYS_FP_RESERVED(r)                  \
   {.reg = (r),                                   \
    .cls = NATIVE_REG_FP,                         \
    .abi_index = 0xffu,                           \
    .flags = NATIVE_REG_RESERVED |                \
             NATIVE_REG_CALLER_SAVED,             \
    .spill_cost = 0u,                             \
    .copy_cost = 0u}
 
 /* Allocable FP pool: arg/ret xmm0..xmm3 lead so -O0 fp-arg producers
  * materialize directly into the ABI arg registers (Lever 1), then xmm6..xmm11.
  * xmm0/xmm1 are also the fp ret regs but ndt_result_reg_stable is OFF, so the
  * fp result still moves out of xmm0 via the post-call path (often a no-op).
  * xmm4/xmm5 form the O1 operand-temp bank (and O0 scratch policy);
  * xmm14/xmm15 are backend-private. */
 static const Reg x64_ndt_fp_allocable[] = {
     X64_XMM0,      X64_XMM1,      X64_XMM2,     X64_XMM3,
     X64_XMM6,      X64_XMM7,      X64_XMM8,     X64_XMM0 + 9,
     X64_XMM0 + 10, X64_XMM0 + 11};
 static const Reg x64_fp_scratch[] = {X64_XMM4, X64_XMM5};
 static const Reg x64_fp_asm_temps[] = {X64_TMP_FP, X64_TMP_FP2};
 static const Reg x64_direct_asm_fp[] = {
     X64_XMM0,     X64_XMM1,      X64_XMM2,      X64_XMM3,
     X64_XMM6,     X64_XMM7,      X64_XMM8,      X64_XMM0 + 9,
     X64_XMM0 + 10, X64_XMM0 + 11};
 
 static const NativePhysRegInfo x64_fp_phys[] = {
     X64_PHYS_FP_ARG_RET(X64_XMM0),       X64_PHYS_FP_ARG_RET(X64_XMM1),
     X64_PHYS_FP_ARG(X64_XMM2),           X64_PHYS_FP_ARG(X64_XMM3),
     X64_PHYS_FP_ARG_RESERVED(X64_XMM4),  X64_PHYS_FP_ARG_RESERVED(X64_XMM5),
     X64_PHYS_FP_ARG(X64_XMM6),           X64_PHYS_FP_ARG(X64_XMM7),
     X64_PHYS_FP_CALLER(X64_XMM8),        X64_PHYS_FP_CALLER(X64_XMM0 + 9),
     X64_PHYS_FP_CALLER(X64_XMM0 + 10),   X64_PHYS_FP_CALLER(X64_XMM0 + 11),
     X64_PHYS_FP_RESERVED(X64_XMM0 + 12), X64_PHYS_FP_RESERVED(X64_XMM0 + 13),
     X64_PHYS_FP_RESERVED(X64_XMM0 + 14), /* backend-private temp */
     X64_PHYS_FP_RESERVED(X64_XMM15),     /* backend-private temp */
 };
 
 static const NativeAllocClassInfo x64_classes[] = {
     {.cls = NATIVE_REG_INT,
      .ndt_allocable = x64_ndt_int_allocable,
      .ndt_allocable_count =
          sizeof x64_ndt_int_allocable / sizeof x64_ndt_int_allocable[0],
      .scratch = x64_int_scratch,
      .nscratch = sizeof x64_int_scratch / sizeof x64_int_scratch[0],
      .emit_temps = x64_int_scratch,
      .nemit_temps = sizeof x64_int_scratch / sizeof x64_int_scratch[0],
      .asm_temps = x64_int_asm_temps,
      .nasm_temps = sizeof x64_int_asm_temps / sizeof x64_int_asm_temps[0],
      .direct_asm_allocable = x64_direct_asm_int,
      .ndirect_asm_allocable =
          sizeof x64_direct_asm_int / sizeof x64_direct_asm_int[0],
      .emit_cache_mask = (1u << X64_R8) | (1u << X64_R9),
      .phys = x64_int_phys,
      .nphys = sizeof x64_int_phys / sizeof x64_int_phys[0]},
     {.cls = NATIVE_REG_FP,
      .ndt_allocable = x64_ndt_fp_allocable,
      .ndt_allocable_count =
          sizeof x64_ndt_fp_allocable / sizeof x64_ndt_fp_allocable[0],
      .scratch = x64_fp_scratch,
      .nscratch = sizeof x64_fp_scratch / sizeof x64_fp_scratch[0],
      .emit_temps = x64_fp_scratch,
      .nemit_temps = sizeof x64_fp_scratch / sizeof x64_fp_scratch[0],
      .asm_temps = x64_fp_asm_temps,
      .nasm_temps = sizeof x64_fp_asm_temps / sizeof x64_fp_asm_temps[0],
      .direct_asm_allocable = x64_direct_asm_fp,
      .ndirect_asm_allocable =
          sizeof x64_direct_asm_fp / sizeof x64_direct_asm_fp[0],
      .emit_cache_mask = (1u << X64_XMM4) | (1u << X64_XMM5),
      .phys = x64_fp_phys,
      .nphys = sizeof x64_fp_phys / sizeof x64_fp_phys[0]},
 };
 
 /* Resolve a register name ("r10", "xmm3", ...) to its (class, Reg). Powers the
  * optimizer's inline-asm clobber masks and explicit hard-register operands
  * ("{r10}" from a GNU local register variable). GPR names map through the HW
  * encoding; xmm names through the DWARF index table. Returns non-zero for a
  * non-register name (cc/memory/unknown), which the caller skips. */
 static int x64_resolve_name(const NativeRegInfo* ri, Slice name, Reg* out,
                             NativeAllocClass* cls_out) {
   char buf[16];
   uint32_t idx;
   (void)ri;
   if (!name.s || !name.len || name.len >= sizeof buf) return 1;
   memcpy(buf, name.s, name.len);
   buf[name.len] = '\0';
   if (x64_register_hw_index(buf, &idx) == 0 && idx <= 15u) {
     *cls_out = NATIVE_REG_INT;
     *out = (Reg)idx;
     return 0;
   }
   if (x64_register_index(buf, &idx) == 0 && idx >= 17u && idx <= 32u) {
     *cls_out = NATIVE_REG_FP;
     *out = (Reg)(idx - 17u);
     return 0;
   }
   return 1;
 }
 
 static int x64_asm_operand_reg_ok(const NativeRegInfo* ri, NativeAllocClass cls,
                                   Reg reg) {
   (void)ri;
   if (cls == NATIVE_REG_INT) {
     switch (reg) {
       /* RAX is reserved but not an emitter temp, so it is a legal asm pin (the
        * Linux syscall number/return register). R8/R9 are optimizer operand
        * temps and R10/R11 backend/asm temps, so those stay excluded. */
       case X64_RAX:
       case X64_RBX:
       case X64_RCX:
       case X64_RDX:
       case X64_RSI:
       case X64_RDI:
       case X64_R12:
       case X64_R13:
       case X64_R14:
       case X64_R15:
         return 1;
       default:
         return 0;
     }
   }
   if (cls == NATIVE_REG_FP)
     return reg <= X64_XMM0 + 13u && reg != X64_XMM4 && reg != X64_XMM5;
   return 0;
 }
 
 static int x64_asm_constraint_reg(const NativeRegInfo* ri, const char* body,
                                   NativeAllocClass* cls_out, Reg* fixed_out,
                                   u32* allowed_mask_out) {
   (void)ri;
   if (!body || !body[0] || body[1]) return 0;
   if (fixed_out) *fixed_out = REG_NONE;
   if (allowed_mask_out) *allowed_mask_out = 0;
   switch (body[0]) {
     case 'r':
     case 'q':
       if (cls_out) *cls_out = NATIVE_REG_INT;
       return 1;
     case 'a':
       if (cls_out) *cls_out = NATIVE_REG_INT;
       if (fixed_out) *fixed_out = X64_RAX;
       return 1;
     case 'b':
       if (cls_out) *cls_out = NATIVE_REG_INT;
       if (fixed_out) *fixed_out = X64_RBX;
       return 1;
     case 'c':
       if (cls_out) *cls_out = NATIVE_REG_INT;
       if (fixed_out) *fixed_out = X64_RCX;
       return 1;
     case 'd':
       if (cls_out) *cls_out = NATIVE_REG_INT;
       if (fixed_out) *fixed_out = X64_RDX;
       return 1;
     case 'S':
       if (cls_out) *cls_out = NATIVE_REG_INT;
       if (fixed_out) *fixed_out = X64_RSI;
       return 1;
     case 'D':
       if (cls_out) *cls_out = NATIVE_REG_INT;
       if (fixed_out) *fixed_out = X64_RDI;
       return 1;
     case 'x':
     case 'v':
       if (cls_out) *cls_out = NATIVE_REG_FP;
       return 1;
     default:
       return 0;
   }
 }
 
 static const NativeRegInfo x64_reg_info = {
     .classes = x64_classes,
     .nclasses = sizeof x64_classes / sizeof x64_classes[0],
     .resolve_name = x64_resolve_name,
     .asm_operand_reg_ok = x64_asm_operand_reg_ok,
     .asm_constraint_reg = x64_asm_constraint_reg,
 };
 
 /* ============================ legality ============================ */
 
 static int x64_imm_legal(NativeTarget* t, NativeImmUse use, u32 op,
                          KitCgTypeId type, i64 imm) {
   (void)t;
   (void)type;
   switch (use) {
     case NATIVE_IMM_MOVE:
       return 1;
     case NATIVE_IMM_BINOP:
       switch ((BinOp)op) {
         case BO_IADD:
         case BO_ISUB:
         case BO_AND:
         case BO_OR:
         case BO_XOR:
         case BO_IMUL:
           return imm_fits_i32(imm);
         case BO_SHL:
         case BO_SHR_S:
         case BO_SHR_U:
           return imm >= 0 && imm <= 63;
         default:
           return 0;
       }
     case NATIVE_IMM_CMP:
       return imm_fits_i32(imm);
     case NATIVE_IMM_ADDR_OFFSET:
       return imm_fits_i32(imm);
   }
   return 0;
 }
 
 static int x64_addr_legal(NativeTarget* t, const NativeAddr* addr,
                           MemAccess mem) {
   (void)t;
   (void)mem;
   if (!addr) return 0;
   if (addr->base_kind != NATIVE_ADDR_BASE_REG &&
       addr->base_kind != NATIVE_ADDR_BASE_FRAME)
     return 0;
   /* x64 supports [base + index*scale + disp32]; index must be a register. */
   if (addr->index_kind != NATIVE_ADDR_INDEX_NONE &&
       addr->index_kind != NATIVE_ADDR_INDEX_REG)
     return 0;
   return imm_fits_i32(addr->offset);
 }
 
 /* ============================ globals / addresses ============================
  */
 
 static int x64_use_got_for_sym(NativeTarget* t, ObjSymId sym) {
   return obj_symbol_extern_via_got(t->c, t->obj, sym);
 }
 
 /* PC-relative reloc kind for a non-GOT &sym reference. Functions use PLT32 so
  * the linker can route through a PLT; data uses plain PC32. */
 static u32 x64_pcrel_reloc_for_sym(NativeTarget* t, ObjSymId sym) {
   const ObjSym* s = obj_symbol_get(t->obj, sym);
   if (s && (s->kind == SK_FUNC || s->kind == SK_IFUNC)) return R_X64_PLT32;
   return R_PC32;
 }
 
 /* Materialize &sym + addend into dst_reg. Local/static-link symbols use
  * `lea rd, [rip + disp32]`; GOT-routed externs use `mov rd, [rip + GOT]` then
  * add any nonzero addend. */
 static void x64_emit_global_lea(NativeTarget* t, u32 dst_reg, ObjSymId sym,
                                 i64 addend) {
   MCEmitter* mc = t->mc;
   u32 sec = mc->section_id;
   if (x64_use_got_for_sym(t, sym)) {
     u8 op;
     u32 disp_pos;
     emit_rex(mc, 1, dst_reg, 0, 0);
     op = X64_OPC_MOV_R_RM;
     mc_emit_bytes(mc, &op, 1);
     {
       u8 mr = modrm(0u, dst_reg & 7u, 5u); /* [rip + disp32] */
       mc_emit_bytes(mc, &mr, 1);
     }
     disp_pos = mc_pos(mc);
     emit_u32le(mc, 0);
     mc_emit_reloc_at(mc, sec, disp_pos, R_X64_REX_GOTPCRELX, sym, -4, 1, 0);
     if (addend) {
       i32 a = (i32)addend;
       emit_rex(mc, 1, 0, 0, dst_reg);
       if (imm_fits_i8(a)) {
         u8 buf[3] = {X64_OPC_ALU_IMM8, modrm(3u, X64_ALU_SUB_ADD, dst_reg & 7u),
                      (u8)a};
         mc_emit_bytes(mc, buf, 3);
       } else {
         u8 buf[2] = {X64_OPC_ALU_IMM32,
                      modrm(3u, X64_ALU_SUB_ADD, dst_reg & 7u)};
         mc_emit_bytes(mc, buf, 2);
         emit_u32le(mc, (u32)a);
       }
     }
     return;
   }
   {
     u8 op = X64_OPC_LEA;
     u32 disp_pos;
     emit_rex(mc, 1, dst_reg, 0, 0);
     mc_emit_bytes(mc, &op, 1);
     {
       u8 mr = modrm(0u, dst_reg & 7u, 5u); /* [rip + disp32] */
       mc_emit_bytes(mc, &mr, 1);
     }
     disp_pos = mc_pos(mc);
     emit_u32le(mc, 0);
     mc_emit_reloc_at(mc, sec, disp_pos, x64_pcrel_reloc_for_sym(t, sym), sym,
                       addend - 4, 1, 0);
   }
 }
 
 /* Resolve a NativeAddr to (base, index, log2_scale, off). Plain FRAME bases
  * remain rbp-relative. GLOBAL and FRAME_VALUE components use backend-private
  * fixed temps; each FRAME_VALUE load uses its exact base_type/index_type. */
 static u32 x64_resolve_addr(X64NativeTarget* a, const NativeAddr* addr,
                             u32 scratch, u32* idx_out, u32* scale_out,
                             i32* off_out) {
   NativeTarget* t = &a->base;
   u32 base;
   i32 off;
   switch (addr->base_kind) {
     case NATIVE_ADDR_BASE_REG:
       base = addr->base.reg & 0xfu;
       off = addr->offset;
       break;
     case NATIVE_ADDR_BASE_FRAME: {
       X64NativeSlot* s = x64_slot_get(a, addr->base.frame);
       base = X64_RBP;
       off = -(i32)s->off + addr->offset;
       break;
     }
     case NATIVE_ADDR_BASE_FRAME_VALUE: {
       X64NativeSlot* s = x64_slot_get(a, addr->base.frame);
       u32 size;
       if (!addr->base_type) x64_panic(a, "frame-value base has no exact type");
       size = native_type_size(t, addr->base_type);
       emit_mov_load(t->mc, size, 0, scratch, X64_RBP, -(i32)s->off);
       base = scratch;
       off = addr->offset;
       break;
     }
     case NATIVE_ADDR_BASE_GLOBAL:
       x64_emit_global_lea(t, scratch, addr->base.global.sym,
                           addr->base.global.addend);
       base = scratch;
       off = addr->offset;
       break;
     default:
       x64_panic(a, "unsupported address base");
   }
   if (addr->index_kind == NATIVE_ADDR_INDEX_REG) {
     *idx_out = addr->index.reg & 0xfu;
     *scale_out = addr->log2_scale;
   } else if (addr->index_kind == NATIVE_ADDR_INDEX_FRAME_VALUE) {
     X64NativeSlot* s = x64_slot_get(a, addr->index.frame);
     u32 size;
     u32 index_tmp = base == X64_TMP_INT2 ? X64_TMP_INT : X64_TMP_INT2;
     if (!addr->index_type)
       x64_panic(a, "frame-value index has no exact type");
     size = native_type_size(t, addr->index_type);
     emit_mov_load(t->mc, size, 0, index_tmp, X64_RBP, -(i32)s->off);
     *idx_out = index_tmp;
     *scale_out = addr->log2_scale;
   } else {
     *idx_out = REG_NONE;
     *scale_out = 0;
   }
   *off_out = off;
   return base;
 }
 
 /* ============================ memory ============================ */
 
 static int x64_native_part_width(u32 size);
 static u32 x64_addr_to_base_reg(X64NativeTarget* a, NativeAddr addr,
                                 u32 scratch);
 
 /* Central load/store primitive. is_load: 1 load into reg, 0 store reg to mem.
  * Materializes the address through X64_TMP_INT2 (r11) for non-reg bases. */
 static void x64_emit_mem(X64NativeTarget* a, int is_load, NativeLoc reg,
                          NativeAddr addr, MemAccess mem) {
   NativeTarget* t = &a->base;
   MCEmitter* mc = t->mc;
   u32 r = loc_reg(reg);
   int fp = native_loc_is_fp(reg);
   u32 sz = mem.size ? mem.size : loc_size32(t, reg);
   u32 base, idx, scale;
   i32 off;
 
   if (!fp && !x64_native_part_width(sz))
     x64_panic(a, "raw integer memory width is not native");
 
   /* Global base: fold into a single rip-relative access when local. */
   if (addr.base_kind == NATIVE_ADDR_BASE_GLOBAL &&
       addr.index_kind == NATIVE_ADDR_INDEX_NONE &&
       !x64_use_got_for_sym(t, addr.base.global.sym)) {
     ObjSymId sym = addr.base.global.sym;
     i64 ad = addr.base.global.addend + addr.offset;
     u32 sec = mc->section_id;
     u32 disp_pos;
     if (fp) {
       u8 prefix = sse_scalar_prefix(sz);
       mc_emit_bytes(mc, &prefix, 1);
       emit_rex(mc, 0, r, 0, 0);
       {
         u8 op2[2] = {X64_OPC_TWOBYTE, (u8)(is_load ? 0x10u : 0x11u)};
         mc_emit_bytes(mc, op2, 2);
       }
     } else if (sz == 8 || sz == 4) {
       emit_rex(mc, sz == 8, r, 0, 0);
       {
         u8 op = is_load ? X64_OPC_MOV_R_RM : X64_OPC_MOV_RM_R;
         mc_emit_bytes(mc, &op, 1);
       }
     } else if (sz == 2) {
       if (is_load) {
         emit_rex(mc, 0, r, 0, 0);
         {
           u8 op2[2] = {X64_OPC_TWOBYTE, X64_OPC_MOVZX_W};
           mc_emit_bytes(mc, op2, 2);
         }
       } else {
         u8 p = X64_OPSIZE_PFX;
         mc_emit_bytes(mc, &p, 1);
         emit_rex(mc, 0, r, 0, 0);
         {
           u8 op = X64_OPC_MOV_RM_R;
           mc_emit_bytes(mc, &op, 1);
         }
       }
     } else { /* size 1 */
       if (is_load) {
         emit_rex(mc, 0, r, 0, 0);
         {
           u8 op2[2] = {X64_OPC_TWOBYTE, X64_OPC_MOVZX_B};
           mc_emit_bytes(mc, op2, 2);
         }
       } else {
         emit_rex_force(mc, 0, r, 0, 0);
         {
           u8 op = X64_OPC_MOV_RM_R8;
           mc_emit_bytes(mc, &op, 1);
         }
       }
     }
     {
       u8 mr = modrm(0u, r & 7u, 5u);
       mc_emit_bytes(mc, &mr, 1);
     }
     disp_pos = mc_pos(mc);
     emit_u32le(mc, 0);
     mc_emit_reloc_at(mc, sec, disp_pos, x64_pcrel_reloc_for_sym(t, sym), sym,
                       ad - 4, 1, 0);
     return;
   }
 
   base = x64_resolve_addr(a, &addr, X64_TMP_INT2, &idx, &scale, &off);
   if (fp) {
     u8 prefix = sse_scalar_prefix(sz);
     if (is_load)
       emit_sse_load_idx(mc, prefix, 0x10, r, base, idx, scale, off);
     else
       emit_sse_store_idx(mc, prefix, 0x11, r, base, idx, scale, off);
   } else if (is_load) {
     /* Loads narrower than 4 bytes zero-extend (sign-extension is applied by a
      * later CV_SEXT). */
     emit_mov_load_idx(mc, sz, 0, r, base, idx, scale, off);
   } else {
     emit_mov_store_idx(mc, sz, r, base, idx, scale, off);
   }
 }
 
 static int x64_native_part_width(u32 size) {
   return size == 1u || size == 2u || size == 4u || size == 8u;
 }
 
 /* The two integer registers in the backend-private bank. Exact-width helpers
  * assign them explicit address/data roles for their complete local phase. */
 static int x64_private_int_reg(Reg reg) {
   return reg == X64_TMP_INT || reg == X64_TMP_INT2;
 }
 
 static Reg x64_other_private_int(Reg reg) {
   return reg == X64_TMP_INT ? X64_TMP_INT2 : X64_TMP_INT;
 }
 
 /* Resolve the complete effective address into one explicitly selected private
  * register. Exact-width multi-chunk operations keep this address stable while
  * their other private register carries chunk data; callers no longer depend on
  * x64_emit_mem's internal address-scratch choice. */
 static NativeAddr x64_stabilize_part_addr(X64NativeTarget* a, NativeAddr addr,
                                           Reg addr_reg) {
   u32 base;
   KitCgTypeId base_type;
   base_type = addr.base_type;
   base = x64_addr_to_base_reg(a, addr, addr_reg);
   if (base != addr_reg) emit_mov_rr(a->base.mc, 1, addr_reg, base);
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_REG;
   addr.cls = NATIVE_REG_INT;
   addr.base.reg = addr_reg;
   addr.base_type = base_type;
   return addr;
 }
 
 /* Load an exact-width little-endian integer ABI part. x86 has native memory
  * operations only for 1/2/4/8 bytes, so a 3/5/6/7-byte tail is assembled from
  * bounded chunks in the low bits of dst. No chunk crosses the semantic part. */
 static void x64_load_int_part(X64NativeTarget* a, NativeLoc dst,
                               NativeAddr addr, MemAccess mem, u32 size) {
   NativePartChunkIter chunks;
   NativeTarget* t = &a->base;
   Reg rd = (Reg)loc_reg(dst);
   Reg addr_reg;
   Reg data_reg;
   NativeLoc tmp;
   u32 chunk_off, chunk_size;
   int private_dst;
   int first = 1;
 
   if (x64_native_part_width(size)) {
     mem.size = size;
     x64_emit_mem(a, 1, dst, addr, mem);
     return;
   }
   if (size == 0u || size > 8u || native_loc_is_fp(dst))
     x64_panic(a, "invalid exact-width integer part load");
 
   private_dst = x64_private_int_reg(rd);
   addr_reg = private_dst ? x64_other_private_int(rd) : X64_TMP_INT2;
   data_reg = private_dst ? addr_reg : X64_TMP_INT;
   addr = x64_stabilize_part_addr(a, addr, addr_reg);
   /* A private destination consumes both integer temps (one accumulator, one
    * chunk destination). Keep the stable address in the private FP hold register
    * and restore it immediately before each alias-safe memory load. */
   if (private_dst)
     emit_sse_rr_w(t->mc, 0x66, 0x6E, 1, X64_TMP_FP, addr_reg);
   tmp = native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT,
                        data_reg);
   chunks = native_part_chunks(size, 8u);
   while (native_part_chunk_next(&chunks, &chunk_off, &chunk_size)) {
     NativeAddr chunk_addr = addr;
     NativeLoc chunk_dst = first ? dst : tmp;
     chunk_addr.offset += (i32)chunk_off;
     mem.size = chunk_size;
     if (private_dst)
       emit_sse_rr_w(t->mc, 0x66, 0x7E, 1, X64_TMP_FP, addr_reg);
     x64_emit_mem(a, 1, chunk_dst, chunk_addr, mem);
     if (!first) {
       emit_shift_imm(t->mc, 1, X64_SHIFT_SUB_SHL, data_reg,
                      (u8)(chunk_off * 8u));
       emit_alu_rr(t->mc, 1, X64_OPC_ALU_OR, rd, data_reg);
     }
     first = 0;
   }
 }
 
 /* Store exactly the semantic bytes carried in an integer ABI register. The
  * first chunk uses the source directly; later chunks use a shifted private
  * copy, preserving both the source register and adjacent destination bytes. */
 static void x64_store_int_part(X64NativeTarget* a, NativeAddr addr,
                                NativeLoc src, MemAccess mem, u32 size) {
   NativePartChunkIter chunks;
   NativeTarget* t = &a->base;
   Reg rs = (Reg)loc_reg(src);
   Reg addr_reg;
   Reg data_reg;
   NativeLoc tmp;
   u32 chunk_off, chunk_size, shifted = 0;
   int private_src;
   int copied = 0;
 
   if (x64_native_part_width(size)) {
     mem.size = size;
     x64_emit_mem(a, 0, src, addr, mem);
     return;
   }
   if (size == 0u || size > 8u || native_loc_is_fp(src))
     x64_panic(a, "invalid exact-width integer part store");
 
   private_src = x64_private_int_reg(rs);
   addr_reg = private_src ? x64_other_private_int(rs) : X64_TMP_INT2;
   data_reg = private_src ? rs : X64_TMP_INT;
   /* Preserve a backend-private source while complete address stabilization is
    * free to use both integer temps. Restore it after the stores as well: these
    * helpers promise not to consume their source even when another backend
    * marshalling phase supplied it from the private bank. */
   if (private_src)
     emit_sse_rr_w(t->mc, 0x66, 0x6E, 1, X64_TMP_FP, rs);
   addr = x64_stabilize_part_addr(a, addr, addr_reg);
   if (private_src)
     emit_sse_rr_w(t->mc, 0x66, 0x7E, 1, X64_TMP_FP, data_reg);
   tmp = native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT,
                        data_reg);
   chunks = native_part_chunks(size, 8u);
   while (native_part_chunk_next(&chunks, &chunk_off, &chunk_size)) {
     NativeAddr chunk_addr = addr;
     NativeLoc chunk_src = src;
     chunk_addr.offset += (i32)chunk_off;
     if (chunk_off != 0u) {
       if (!copied) {
         emit_mov_rr(t->mc, 1, data_reg, rs);
         copied = 1;
       }
       emit_shift_imm(t->mc, 1, X64_SHIFT_SUB_SHR, data_reg,
                      (u8)((chunk_off - shifted) * 8u));
       shifted = chunk_off;
       chunk_src = tmp;
     }
     mem.size = chunk_size;
     x64_emit_mem(a, 0, chunk_src, chunk_addr, mem);
   }
   if (private_src)
     emit_sse_rr_w(t->mc, 0x66, 0x7E, 1, X64_TMP_FP, rs);
 }
 
 /* ============================ moves / data ============================ */
 
 static void x64_move(NativeTarget* t, NativeLoc dst, NativeLoc src) {
   MCEmitter* mc = t->mc;
   int dfp = native_loc_is_fp(dst), sfp = native_loc_is_fp(src);
   u32 rd = loc_reg(dst), rs = loc_reg(src);
   if (dfp && sfp) {
     if (rd == rs) return;
     emit_sse_rr(mc, sse_scalar_prefix(loc_size32(t, dst)), 0x10, rd,
                 rs);
     return;
   }
   if (dfp && !sfp) { /* movd/movq gpr -> xmm: 66 0F 6E /r */
     int w = loc_size32(t, dst) == 8u;
     emit_sse_rr_w(mc, 0x66, 0x6E, w, rd, rs);
     return;
   }
   if (!dfp && sfp) { /* movd/movq xmm -> gpr: 66 0F 7E /r (xmm is reg field) */
     int w = loc_size32(t, src) == 8u;
     emit_sse_rr_w(mc, 0x66, 0x7E, w, rs, rd);
     return;
   }
   if (rd == rs) return;
   emit_mov_rr(mc, loc_is_64(t, dst) ? 1 : 0, rd, rs);
 }
 
 /* Preserve an indirect call target below rsp while its arguments are
  * marshalled. CFA is rbp-based, so this transient balanced pair needs no CFI
  * adjustment. High registers use the PUSH/POP opcode's REX.B extension. */
 static void x64_push_reg(MCEmitter* mc, Reg reg) {
   if (reg & 8u) emit1(mc, X64_REX_BASE | X64_REX_B);
   emit1(mc, X64_OPC_PUSH_R | (reg & 7u));
 }
 
 static void x64_pop_reg(MCEmitter* mc, Reg reg) {
   if (reg & 8u) emit1(mc, X64_REX_BASE | X64_REX_B);
   emit1(mc, X64_OPC_POP_R | (reg & 7u));
 }
 
 static void x64_load_imm(NativeTarget* t, NativeLoc dst, i64 imm) {
   x64_emit_load_imm(t->mc, loc_is_64(t, dst) ? 1 : 0, loc_reg(dst), imm);
 }
 
 /* FP constant: materialize the bit pattern in a GPR scratch, then movd/movq
  * into the FPR. Integer constant: plain load_imm. */
 static void x64_load_const(NativeTarget* t, NativeLoc dst, ConstBytes cb) {
   u64 v = 0;
   u32 i;
   for (i = 0; i < cb.size && i < 8u; ++i) v |= (u64)cb.bytes[i] << (i * 8u);
   if (!native_loc_is_fp(dst)) {
     x64_load_imm(t, dst, (i64)v);
     return;
   }
   x64_emit_load_imm(t->mc, cb.size == 8u, X64_TMP_INT, (i64)v);
   emit_sse_rr_w(t->mc, 0x66, 0x6E, cb.size == 8u, loc_reg(dst), X64_TMP_INT);
 }
 
 static void x64_load_addr(NativeTarget* t, NativeLoc dst, NativeAddr addr) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst);
   u32 base, idx, scale;
   i32 off;
   if (addr.base_kind == NATIVE_ADDR_BASE_GLOBAL &&
       addr.index_kind == NATIVE_ADDR_INDEX_NONE) {
     x64_emit_global_lea(t, rd, addr.base.global.sym,
                         addr.base.global.addend + addr.offset);
     return;
   }
   base = x64_resolve_addr(a, &addr, rd, &idx, &scale, &off);
   if (idx == REG_NONE) {
     if (base == rd && off == 0) return; /* already &slot in rd */
     emit_lea(mc, rd, base, off);
     return;
   }
   /* lea rd, [base + idx*scale + off] */
   {
     u8 buf[16];
     u32 n = 0;
     n += x64_pack_rex(buf + n, 1, rd, idx, base);
     buf[n++] = X64_OPC_LEA;
     n += x64_pack_mem_sib(buf + n, rd, base, idx, scale, off);
     mc_emit_bytes(mc, buf, n);
   }
 }
 
 static void x64_load(NativeTarget* t, NativeLoc dst, NativeAddr addr,
                      MemAccess mem) {
   u32 size = mem.size ? mem.size : loc_size32(t, dst);
   if (!x64_native_part_width(size) && size <= 8u)
     x64_load_int_part(x64_of(t), dst, addr, mem, size);
   else
     x64_emit_mem(x64_of(t), 1, dst, addr, mem);
 }
 static void x64_store(NativeTarget* t, NativeAddr addr, NativeLoc src,
                       MemAccess mem) {
   u32 size = mem.size ? mem.size : loc_size32(t, src);
   if (!x64_native_part_width(size) && size <= 8u)
     x64_store_int_part(x64_of(t), addr, src, mem, size);
   else
     x64_emit_mem(x64_of(t), 0, src, addr, mem);
 }
 
 /* Resolve an addressable NativeAddr to a bare base register (no index, off 0)
  * by emitting an lea into `scratch` when needed. */
 static u32 x64_addr_to_base_reg(X64NativeTarget* a, NativeAddr addr,
                                 u32 scratch) {
   MCEmitter* mc = a->base.mc;
   u32 base, idx, scale;
   i32 off;
   if (addr.base_kind == NATIVE_ADDR_BASE_GLOBAL &&
       addr.index_kind == NATIVE_ADDR_INDEX_NONE) {
     x64_emit_global_lea(&a->base, scratch, addr.base.global.sym,
                         addr.base.global.addend + addr.offset);
     return scratch;
   }
   base = x64_resolve_addr(a, &addr, scratch, &idx, &scale, &off);
   if (idx == REG_NONE && off == 0) return base;
   if (idx == REG_NONE) {
     emit_lea(mc, scratch, base, off);
     return scratch;
   }
   {
     u8 buf[16];
     u32 n = 0;
     n += x64_pack_rex(buf + n, 1, scratch, idx, base);
     buf[n++] = X64_OPC_LEA;
     n += x64_pack_mem_sib(buf + n, scratch, base, idx, scale, off);
     mc_emit_bytes(mc, buf, n);
   }
   return scratch;
 }
 
 /* Normalize one address at a time, load through R10, preserve the granule in
  * XMM14 while the destination is normalized, then restore it to R10 for the
  * store. This keeps both effective addresses and the transfer value entirely
  * in the backend-private R10/R11/XMM14 bank; no allocator/cache-visible fixed
  * register is touched. */
 static void x64_copy_bytes(NativeTarget* t, NativeAddr dst, NativeAddr src,
                            AggregateAccess access) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   u32 n = access.size, i = 0;
   while (i < n) {
     u32 rem = n - i, s;
     NativeAddr sa = src, da = dst;
     u32 sb, db;
     if (rem >= 8u) {
       s = 8u;
     } else if (rem >= 4u) {
       s = 4u;
     } else if (rem >= 2u) {
       s = 2u;
     } else {
       s = 1u;
     }
     sa.offset += (i32)i;
     da.offset += (i32)i;
     sb = x64_addr_to_base_reg(a, sa, X64_TMP_INT2);
     emit_mov_load(mc, s, 0, X64_TMP_INT, sb, 0);
     emit_sse_rr_w(mc, 0x66, 0x6E, s == 8u, X64_TMP_FP, X64_TMP_INT);
     db = x64_addr_to_base_reg(a, da, X64_TMP_INT2);
     if (db == X64_TMP_INT) {
       emit_mov_rr(mc, 1, X64_TMP_INT2, db);
       db = X64_TMP_INT2;
     }
     emit_sse_rr_w(mc, 0x66, 0x7E, s == 8u, X64_TMP_FP, X64_TMP_INT);
     emit_mov_store(mc, s, X64_TMP_INT, db, 0);
     i += s;
   }
 }
 
 static void x64_set_bytes(NativeTarget* t, NativeAddr dst, NativeLoc byte_value,
                           AggregateAccess access) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   u32 dr = x64_addr_to_base_reg(a, dst, X64_TMP_INT2);
   u32 n = access.size, i = 0;
   /* R10 holds the broadcast value and R11 the resolved destination. Keep both
    * entirely backend-private: the variable case broadcasts the low byte in
    * XMM14 (punpcklbw/punpcklwd/punpckldq), then moves the qword to R10. */
   if (dr == X64_TMP_INT) {
     emit_mov_rr(mc, 1, X64_TMP_INT2, dr);
     dr = X64_TMP_INT2;
   }
   if (byte_value.kind == NATIVE_LOC_IMM) {
     u8 b = (u8)(byte_value.v.imm & 0xffu);
     u64 b64 = b;
     b64 |= b64 << 8;
     b64 |= b64 << 16;
     b64 |= b64 << 32;
     x64_emit_load_imm(mc, 1, X64_TMP_INT, (i64)b64);
   } else {
     emit_mov_rr(mc, 0, X64_TMP_INT, loc_reg(byte_value));
     emit_alu_imm32(mc, 0, X64_ALU_SUB_AND, X64_TMP_INT, 0xff);
     emit_sse_rr_w(mc, 0x66, 0x6E, 0, X64_TMP_FP, X64_TMP_INT);
     emit_sse_rr(mc, 0x66, 0x60, X64_TMP_FP, X64_TMP_FP);
     emit_sse_rr(mc, 0x66, 0x61, X64_TMP_FP, X64_TMP_FP);
     emit_sse_rr(mc, 0x66, 0x62, X64_TMP_FP, X64_TMP_FP);
     emit_sse_rr_w(mc, 0x66, 0x7E, 1, X64_TMP_FP, X64_TMP_INT);
   }
   while (i + 8u <= n) {
     emit_mov_store(mc, 8, X64_TMP_INT, dr, (i32)i);
     i += 8u;
   }
   while (i + 4u <= n) {
     emit_mov_store(mc, 4, X64_TMP_INT, dr, (i32)i);
     i += 4u;
   }
   while (i + 2u <= n) {
     emit_mov_store(mc, 2, X64_TMP_INT, dr, (i32)i);
     i += 2u;
   }
   while (i < n) {
     emit_mov_store(mc, 1, X64_TMP_INT, dr, (i32)i);
     i += 1u;
   }
 }
 
 /* ============================ bitfields ============================ */
 
 static void x64_bitfield_load(NativeTarget* t, NativeLoc dst, NativeAddr ra,
                               BitFieldAccess bf) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   u32 storage_bytes = bf.storage.size ? bf.storage.size : 4u;
   int w = storage_bytes == 8u ? 1 : 0;
   u32 reg_size = w ? 64u : 32u;
   u32 lsb = bf.bit_offset;
   u32 width = bf.bit_width ? bf.bit_width : 1u;
   u32 rd = loc_reg(dst);
   u32 base;
   ra.offset += (i32)bf.storage_offset;
   base = x64_addr_to_base_reg(a, ra, X64_TMP_INT2);
   emit_mov_load(mc, storage_bytes, 0, rd, base, 0);
   {
     u8 left = (u8)(reg_size - lsb - width);
     u8 right = (u8)(reg_size - width);
     if (left) emit_shift_imm(mc, w, X64_SHIFT_SUB_SHL, rd, left);
     if (right)
       emit_shift_imm(mc, w, bf.signed_ ? X64_SHIFT_SUB_SAR : X64_SHIFT_SUB_SHR,
                      rd, right);
   }
 }
 
 static void x64_bitfield_store(NativeTarget* t, NativeAddr ra, NativeLoc src,
                                BitFieldAccess bf) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   u32 storage_bytes = bf.storage.size ? bf.storage.size : 4u;
   int w = storage_bytes == 8u ? 1 : 0;
   u32 lsb = bf.bit_offset;
   u32 width = bf.bit_width ? bf.bit_width : 1u;
   u64 ones = width >= 64u ? ~(u64)0 : (((u64)1 << width) - 1u);
   u64 mask = ones << lsb;
   u32 src_reg = loc_reg(src);
   u32 base;
   ra.offset += (i32)bf.storage_offset;
   /* Resolve the address before borrowing the fixed value temporaries. A bare
    * register address may be returned unchanged, so explicitly stabilize it in
    * r11 when the read-modify-write is about to clobber that register. */
   base = x64_addr_to_base_reg(a, ra, X64_TMP_INT2);
   if (base == X64_RAX || base == X64_RCX || base == X64_RDX) {
     emit_mov_rr(mc, 1, X64_TMP_INT2, base);
     base = X64_TMP_INT2;
   }
   /* A dying source may legally occupy one of the fixed clobber registers.
    * Preserve it after address resolution (which may transiently use r10 for a
    * spilled index) and before loading the storage/mask into rax/rcx. RDX stays
    * live until it has been copied to RCX below, so it needs no staging. */
   if (src_reg == X64_RAX || src_reg == X64_RCX) {
     emit_mov_rr(mc, w, X64_TMP_INT, src_reg);
     src_reg = X64_TMP_INT;
   }
   /* rax = storage; rax &= ~mask. */
   emit_mov_load(mc, storage_bytes, 0, X64_RAX, base, 0);
   x64_emit_load_imm(mc, w, X64_RCX, (i64)~mask);
   emit_alu_rr(mc, w, X64_OPC_ALU_AND, X64_RAX, X64_RCX);
   /* rcx = (src & ones) << lsb. */
   emit_mov_rr(mc, w, X64_RCX, src_reg);
   x64_emit_load_imm(mc, w, X64_RDX, (i64)ones);
   emit_alu_rr(mc, w, X64_OPC_ALU_AND, X64_RCX, X64_RDX);
   if (lsb) emit_shift_imm(mc, w, X64_SHIFT_SUB_SHL, X64_RCX, (u8)lsb);
   emit_alu_rr(mc, w, X64_OPC_ALU_OR, X64_RAX, X64_RCX);
   emit_mov_store(mc, storage_bytes, X64_RAX, base, 0);
 }
 
 /* ============================ arithmetic ============================ */
 
 static void x64_binop(NativeTarget* t, BinOp op, NativeLoc dst, NativeLoc aop,
                       NativeLoc bop) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst);
 
   /* FP binops: two-address. dst = aop op bop. */
   if (op == BO_FADD || op == BO_FSUB || op == BO_FMUL || op == BO_FDIV) {
     u32 ra = loc_reg(aop), rb = loc_reg(bop);
     u8 prefix = sse_scalar_prefix(loc_size32(t, dst));
     u8 opcode;
     switch (op) {
       case BO_FADD:
         opcode = 0x58;
         break;
       case BO_FSUB:
         opcode = 0x5C;
         break;
       case BO_FMUL:
         opcode = 0x59;
         break;
       default:
         opcode = 0x5E;
         break; /* BO_FDIV */
     }
     if (rd == rb && rd != ra) {
       if (op == BO_FADD || op == BO_FMUL) { /* commutative */
         emit_sse_rr(mc, prefix, opcode, rd, ra);
         return;
       }
       /* non-commutative dst==rb: stage rb in fp scratch. */
       emit_sse_rr(mc, prefix, 0x10, X64_TMP_FP2, rb);
       emit_sse_rr(mc, prefix, 0x10, rd, ra);
       emit_sse_rr(mc, prefix, opcode, rd, X64_TMP_FP2);
       return;
     }
     if (rd != ra) emit_sse_rr(mc, prefix, 0x10, rd, ra);
     emit_sse_rr(mc, prefix, opcode, rd, rb);
     return;
   }
 
   {
     int w = loc_is_64(t, dst) ? 1 : 0;
     int b_imm = bop.kind == NATIVE_LOC_IMM;
     i64 imm = b_imm ? bop.v.imm : 0;
     u32 ra = loc_reg(aop);
 
     /* Division: rax/rdx implicit; divisor must avoid rax/rdx. */
     if (op == BO_SDIV || op == BO_UDIV || op == BO_SREM || op == BO_UREM) {
       u32 rb;
       if (b_imm) {
         x64_emit_load_imm(mc, w, X64_R11, imm);
         rb = X64_R11;
       } else {
         rb = loc_reg(bop);
         /* Preserve the divisor before placing the dividend in RAX. This is a
          * real parallel-copy edge: a dying divisor may be in RAX while the
          * dividend is elsewhere, or in RDX before CQO/XOR clears it. */
         if (rb == X64_RAX || rb == X64_RDX) {
           emit_mov_rr(mc, w, X64_R11, rb);
           rb = X64_R11;
         }
       }
       if (ra != X64_RAX) emit_mov_rr(mc, w, X64_RAX, ra);
       if (op == BO_SDIV || op == BO_SREM) {
         emit_cqo_or_cdq(mc, w);
         emit_f7_rm(mc, w, X64_F7_SUB_IDIV, rb);
       } else {
         emit_xor_self(mc, w, X64_RDX);
         emit_f7_rm(mc, w, X64_F7_SUB_DIV, rb);
       }
       {
         u32 result = (op == BO_SREM || op == BO_UREM) ? X64_RDX : X64_RAX;
         if (rd != result) emit_mov_rr(mc, w, rd, result);
       }
       return;
     }
 
     /* Shifts: count in CL or imm8. */
     if (op == BO_SHL || op == BO_SHR_U || op == BO_SHR_S) {
       u32 sub = (op == BO_SHL)     ? X64_SHIFT_SUB_SHL
                 : (op == BO_SHR_U) ? X64_SHIFT_SUB_SHR
                                    : X64_SHIFT_SUB_SAR;
       if (b_imm) {
         u32 wbits = w ? 64u : 32u;
         if (rd != ra) emit_mov_rr(mc, w, rd, ra);
         emit_shift_imm(mc, w, sub, rd, (u8)((u64)imm & (wbits - 1u)));
         return;
       }
       {
         u32 rb = loc_reg(bop);
         /* RCX cannot simultaneously hold a distinct result and the shift
          * count. Compute through backend-private R10, then publish the result
          * after the instruction has consumed CL. */
         if (rd == X64_RCX && (ra != X64_RCX || rb != X64_RCX)) {
           emit_mov_rr(mc, w, X64_TMP_INT, ra);
           if (rb != X64_RCX) emit_mov_rr(mc, 0, X64_RCX, rb);
           emit_shift_cl(mc, w, sub, X64_TMP_INT);
           emit_mov_rr(mc, w, X64_RCX, X64_TMP_INT);
           return;
         }
         /* Place the count in cl and the value in dst. Stage the count through
          * r11 first so neither move clobbers the other when the value already
          * sits in rcx or the count sits in dst. (The optimizer additionally
          * keeps values live across the shift out of rcx — see
          * x64_machine_op_clobbers.) */
         if (rb != X64_RCX) {
           emit_mov_rr(mc, 0, X64_TMP_INT2, rb);
           if (rd != ra) emit_mov_rr(mc, w, rd, ra);
           emit_mov_rr(mc, 0, X64_RCX, X64_TMP_INT2);
         } else if (rd != ra) {
           emit_mov_rr(mc, w, rd, ra);
         }
       }
       emit_shift_cl(mc, w, sub, rd);
       return;
     }
 
     /* IMM-form fast paths (b_imm guaranteed legal by imm_legal: imm32). */
     if (b_imm && (op == BO_IADD || op == BO_ISUB || op == BO_AND ||
                   op == BO_OR || op == BO_XOR || op == BO_IMUL)) {
       if (op == BO_IMUL) {
         if (imm_fits_i8(imm)) {
           emit_imul_imm8(mc, w, rd, ra, (i8)imm);
           return;
         }
         emit_imul_imm32(mc, w, rd, ra, (i32)imm);
         return;
       }
       {
         u32 sub;
         switch (op) {
           case BO_IADD:
             sub = X64_ALU_SUB_ADD;
             break;
           case BO_OR:
             sub = X64_ALU_SUB_OR;
             break;
           case BO_AND:
             sub = X64_ALU_SUB_AND;
             break;
           case BO_ISUB:
             sub = X64_ALU_SUB_SUB;
             break;
           default:
             sub = X64_ALU_SUB_XOR;
             break; /* BO_XOR */
         }
         if (rd != ra) emit_mov_rr(mc, w, rd, ra);
         if (imm_fits_i8(imm))
           emit_alu_imm8(mc, w, sub, rd, (i8)imm);
         else
           emit_alu_imm32(mc, w, sub, rd, (i32)imm);
         return;
       }
     }
 
     /* Generic 2-operand ALU: dst = ra op rb. Preserve rb if dst == rb. */
     {
       u32 rb = loc_reg(bop);
       if (rd == rb && rd != ra) {
         switch (op) {
           case BO_IADD:
             emit_alu_rr(mc, w, X64_OPC_ALU_ADD, rd, ra);
             return;
           case BO_AND:
             emit_alu_rr(mc, w, X64_OPC_ALU_AND, rd, ra);
             return;
           case BO_OR:
             emit_alu_rr(mc, w, X64_OPC_ALU_OR, rd, ra);
             return;
           case BO_XOR:
             emit_alu_rr(mc, w, X64_OPC_ALU_XOR, rd, ra);
             return;
           case BO_IMUL:
             emit_imul_rr(mc, w, rd, ra);
             return;
           default:
             break; /* ISUB falls through: stage rb */
         }
         emit_mov_rr(mc, w, X64_R11, rb);
         rb = X64_R11;
       }
       if (rd != ra) emit_mov_rr(mc, w, rd, ra);
       switch (op) {
         case BO_IADD:
           emit_alu_rr(mc, w, X64_OPC_ALU_ADD, rd, rb);
           break;
         case BO_ISUB:
           emit_alu_rr(mc, w, X64_OPC_ALU_SUB, rd, rb);
           break;
         case BO_AND:
           emit_alu_rr(mc, w, X64_OPC_ALU_AND, rd, rb);
           break;
         case BO_OR:
           emit_alu_rr(mc, w, X64_OPC_ALU_OR, rd, rb);
           break;
         case BO_XOR:
           emit_alu_rr(mc, w, X64_OPC_ALU_XOR, rd, rb);
           break;
         case BO_IMUL:
           emit_imul_rr(mc, w, rd, rb);
           break;
         default:
           x64_panic(a, "unsupported binop");
       }
     }
   }
 }
 
 /* FP sign-mask constant materialized in fp scratch for FNEG. */
 static void x64_unop(NativeTarget* t, UnOp op, NativeLoc dst, NativeLoc src) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst), rs = loc_reg(src);
   if (op == UO_FNEG) {
     int dbl = loc_size32(t, dst) == 8u;
     if (rd != rs)
       emit_sse_rr(mc, sse_scalar_prefix(dbl ? 8u : 4u), 0x10, rd, rs);
     /* sign mask into fp scratch via gpr, then XORPS/XORPD. */
     x64_emit_load_imm(mc, dbl, X64_TMP_INT,
                       dbl ? (i64)0x8000000000000000ull : (i64)0x80000000ull);
     emit_sse_rr_w(mc, 0x66, 0x6E, dbl, X64_TMP_FP2, X64_TMP_INT);
     emit_sse_rr(mc, dbl ? 0x66 : 0, 0x57, rd, X64_TMP_FP2);
     return;
   }
   {
     int w = loc_is_64(t, dst) ? 1 : 0;
     switch (op) {
       case UO_NEG:
         if (rd != rs) emit_mov_rr(mc, w, rd, rs);
         emit_f7_rm(mc, w, X64_F7_SUB_NEG, rd);
         return;
       case UO_BNOT:
         if (rd != rs) emit_mov_rr(mc, w, rd, rs);
         emit_f7_rm(mc, w, X64_F7_SUB_NOT, rd);
         return;
       case UO_NOT:
         /* !x -> (x == 0) as 0/1. */
         emit_test_self(mc, w, rs);
         emit_setcc(mc, X64_CC_E, rd);
         emit_movzx_r32_r8(mc, rd, rd);
         return;
       default:
         x64_panic(a, "unsupported unop");
     }
   }
 }
 
 /* ============================ compares ============================ */
 
 static u32 cmp_to_cc(CmpOp op) {
   switch (op) {
     case CMP_EQ:
       return X64_CC_E;
     case CMP_NE:
       return X64_CC_NE;
     case CMP_LT_U:
       return X64_CC_B;
     case CMP_LE_U:
       return X64_CC_BE;
     case CMP_GT_U:
       return X64_CC_A;
     case CMP_GE_U:
       return X64_CC_AE;
     case CMP_LT_S:
       return X64_CC_L;
     case CMP_LE_S:
       return X64_CC_LE;
     case CMP_GT_S:
       return X64_CC_G;
     case CMP_GE_S:
       return X64_CC_GE;
     default:
       return X64_CC_E;
   }
 }
 
 static int cmp_is_fp(CmpOp op, NativeLoc aop) {
   /* FP-ness is self-describing from the opcode; FP eq/ne are distinct opcodes
    * (CMP_OEQ_F/CMP_UNE_F), so no operand-class sniffing is needed. */
   (void)aop;
   return op >= CMP_OEQ_F;
 }
 
 /* Emit `cmp ra, rb` (or ucomis[sd] for FP), setting flags from ra - rb. */
 static void x64_emit_cmp_flags(NativeTarget* t, NativeLoc aop, NativeLoc bop,
                                int fp) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   if (fp) {
     u8 prefix = loc_size32(t, aop) == 8u ? 0x66u : 0u;
     emit_sse_rr(mc, prefix, 0x2E, loc_reg(aop), loc_reg(bop)); /* ucomis */
     return;
   }
   {
     int w = loc_is_64(t, aop) ? 1 : 0;
     u32 ra = loc_reg(aop);
     if (bop.kind == NATIVE_LOC_IMM) {
       i64 imm = bop.v.imm;
       if (imm_fits_i8(imm))
         emit_alu_imm8(mc, w, X64_ALU_SUB_CMP, ra, (i8)imm);
       else
         emit_alu_imm32(mc, w, X64_ALU_SUB_CMP, ra, (i32)imm);
       return;
     }
     emit_alu_rr(mc, w, X64_OPC_ALU_CMP, ra, loc_reg(bop));
     (void)a;
   }
 }
 
 /* FP ordered setcc: result = (primary cc) && !unordered (NP). */
 static void x64_fp_setcc_ordered(NativeTarget* t, u32 primary, u32 dst) {
   MCEmitter* mc = t->mc;
   emit_setcc(mc, primary, dst);
   emit_movzx_r32_r8(mc, dst, dst);
   emit_setcc(mc, X64_CC_NP, X64_R11);
   emit_movzx_r32_r8(mc, X64_R11, X64_R11);
   emit_alu_rr(mc, 0, X64_OPC_ALU_AND, dst, X64_R11);
 }
 
 /* FP unordered predicate: result = (primary cc) || unordered (P). */
 static void x64_fp_setcc_unord(NativeTarget* t, u32 primary, u32 dst) {
   MCEmitter* mc = t->mc;
   emit_setcc(mc, primary, dst);
   emit_movzx_r32_r8(mc, dst, dst);
   emit_setcc(mc, X64_CC_P, X64_R11);
   emit_movzx_r32_r8(mc, X64_R11, X64_R11);
   emit_alu_rr(mc, 0, X64_OPC_ALU_OR, dst, X64_R11);
 }
 
 static void x64_cmp(NativeTarget* t, CmpOp op, NativeLoc dst, NativeLoc aop,
                     NativeLoc bop) {
   MCEmitter* mc = t->mc;
   u32 d = loc_reg(dst);
   int fp = cmp_is_fp(op, aop);
   x64_emit_cmp_flags(t, aop, bop, fp);
   if (fp) {
     /* ucomis sets ZF/CF and, when unordered (NaN), also PF. Each predicate's
      * flag formula is built explicitly (NOT blindly as !(opposite)):
      *   ordered:    E/B/BE alias {==,<,<=} only when also NP (not-parity);
      *               NE/A/AE already exclude unordered, so they stand alone.
      *   unordered:  E/B/BE already include the unordered case (ZF/CF set on
      *               NaN), so they stand alone; NE/A/AE need an OR with P. */
     switch (op) {
       /* ordered: require not-unordered (NP) on the equality-flag cases */
       case CMP_OEQ_F:
         x64_fp_setcc_ordered(t, X64_CC_E, d);
         return;
       case CMP_OLT_F:
         x64_fp_setcc_ordered(t, X64_CC_B, d);
         return;
       case CMP_OLE_F:
         x64_fp_setcc_ordered(t, X64_CC_BE, d);
         return;
       case CMP_ONE_F:
         emit_setcc(mc, X64_CC_NE, d);
         break;
       case CMP_OGT_F:
         emit_setcc(mc, X64_CC_A, d);
         break;
       case CMP_OGE_F:
         emit_setcc(mc, X64_CC_AE, d);
         break;
       /* unordered: OR-with-P on the cases that exclude unordered */
       case CMP_UEQ_F:
         emit_setcc(mc, X64_CC_E, d);
         break;
       case CMP_ULT_F:
         emit_setcc(mc, X64_CC_B, d);
         break;
       case CMP_ULE_F:
         emit_setcc(mc, X64_CC_BE, d);
         break;
       case CMP_UNE_F:
         x64_fp_setcc_unord(t, X64_CC_NE, d);
         return;
       case CMP_UGT_F:
         x64_fp_setcc_unord(t, X64_CC_A, d);
         return;
       case CMP_UGE_F:
         x64_fp_setcc_unord(t, X64_CC_AE, d);
         return;
       default:
         emit_setcc(mc, cmp_to_cc(op), d);
         break;
     }
     emit_movzx_r32_r8(mc, d, d);
     return;
   }
   emit_setcc(mc, cmp_to_cc(op), d);
   emit_movzx_r32_r8(mc, d, d);
 }
 
 /* ============================ converts ============================ */
 
 static void x64_convert(NativeTarget* t, ConvKind k, NativeLoc dst,
                         NativeLoc src) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst), rs = loc_reg(src);
   switch (k) {
     case CV_SEXT: {
       u32 src_sz = loc_size32(t, src);
       int w = loc_is_64(t, dst) ? 1 : 0;
       emit_extend_rr(mc, w, 1, src_sz, rd, rs);
       return;
     }
     case CV_ZEXT: {
       u32 src_sz = loc_size32(t, src);
       int w = loc_is_64(t, dst) ? 1 : 0;
       emit_extend_rr(mc, w, 0, src_sz, rd, rs);
       return;
     }
     case CV_TRUNC:
       emit_mov_rr(mc, 0, rd, rs); /* low 32 bits; clears high */
       return;
     case CV_ITOF_S:
     case CV_ITOF_U: {
       int w_src = loc_is_64(t, src) ? 1 : 0;
       u8 prefix = sse_scalar_prefix(loc_size32(t, dst));
       if (k == CV_ITOF_U && w_src == 1) {
         MCLabel L_high = mc_label_new(mc);
         MCLabel L_done = mc_label_new(mc);
         emit_test_self(mc, 1, rs);
         emit_jcc_rel32(mc, X64_CC_S, L_high);
         emit_sse_rr_w(mc, prefix, 0x2A, 1, rd, rs);
         emit_jmp_rel32(mc, L_done);
         mc_label_place(mc, L_high);
         emit_mov_rr(mc, 1, X64_R11, rs);
         emit_mov_rr(mc, 1, X64_TMP_INT, rs);
         emit_alu_imm8(mc, 1, X64_ALU_SUB_AND, X64_TMP_INT, 1);
         emit_shift_imm(mc, 1, X64_SHIFT_SUB_SHR, X64_R11, 1);
         emit_alu_rr(mc, 1, X64_OPC_ALU_OR, X64_R11, X64_TMP_INT);
         emit_sse_rr_w(mc, prefix, 0x2A, 1, rd, X64_R11);
         emit_sse_rr(mc, prefix, 0x58, rd, rd);
         mc_label_place(mc, L_done);
         return;
       }
       if (k == CV_ITOF_U) {
         emit_extend_rr(mc, 0, 0, 4, X64_R11, rs); /* zext u32 -> 64 */
         rs = X64_R11;
         w_src = 1;
       }
       emit_sse_rr_w(mc, prefix, 0x2A, w_src, rd, rs);
       return;
     }
     case CV_FTOI_S:
     case CV_FTOI_U: {
       int w_dst = loc_is_64(t, dst) ? 1 : 0;
       u8 prefix = sse_scalar_prefix(loc_size32(t, src));
       /* Unsigned 64-bit FTOI needs the 2^63 bias dance; otherwise cvtt
        * (with the destination widened to 64 for u32) is exact. */
       if (k == CV_FTOI_U && w_dst == 1) {
         int dbl = loc_size32(t, src) == 8u;
         MCLabel L_small = mc_label_new(mc);
         MCLabel L_done = mc_label_new(mc);
         /* limit = 2^63 in fp scratch. */
         x64_emit_load_imm(
             mc, 1, X64_R11,
             dbl ? (i64)0x43E0000000000000ull : (i64)0x5F000000ull);
         emit_sse_rr_w(mc, 0x66, 0x6E, dbl, X64_TMP_FP2, X64_R11);
         emit_sse_rr(mc, dbl ? 0x66 : 0, 0x2E, rs, X64_TMP_FP2); /* ucomis */
         emit_jcc_rel32(mc, X64_CC_B, L_small);
         emit_sse_rr(mc, prefix, 0x10, X64_TMP_FP, rs);
         emit_sse_rr(mc, prefix, 0x5C, X64_TMP_FP, X64_TMP_FP2); /* sub bias */
         emit_sse_rr_w(mc, prefix, 0x2C, 1, rd, X64_TMP_FP);
         x64_emit_load_imm(mc, 1, X64_R11, (i64)0x8000000000000000ull);
         emit_alu_rr(mc, 1, X64_OPC_ALU_XOR, rd, X64_R11);
         emit_jmp_rel32(mc, L_done);
         mc_label_place(mc, L_small);
         emit_sse_rr_w(mc, prefix, 0x2C, 1, rd, rs);
         mc_label_place(mc, L_done);
         return;
       }
       if (k == CV_FTOI_U) w_dst = 1; /* widen u32 result */
       emit_sse_rr_w(mc, prefix, 0x2C, w_dst, rd, rs);
       return;
     }
     case CV_FEXT:
       emit_sse_rr(mc, 0xF3, 0x5A, rd, rs); /* cvtss2sd */
       return;
     case CV_FTRUNC:
       emit_sse_rr(mc, 0xF2, 0x5A, rd, rs); /* cvtsd2ss */
       return;
     case CV_BITCAST:
       if (!native_loc_is_fp(src) && native_loc_is_fp(dst)) {
         emit_sse_rr_w(mc, 0x66, 0x6E, loc_is_64(t, dst), rd, rs);
       } else if (native_loc_is_fp(src) && !native_loc_is_fp(dst)) {
         emit_sse_rr_w(mc, 0x66, 0x7E, loc_is_64(t, src), rs, rd);
       } else {
         x64_move(t, dst, src);
       }
       return;
     default:
       x64_panic(a, "unsupported convert");
   }
 }
 
 /* §E.3 narrow register-only entry points. The NDT crosses 16 B NativeRegLoc
  * here; these reconstruct the NativeLoc the fat hook expects and delegate, so
  * the emitted bytes are identical. Used only on the -O0 NDT path; the opt
  * replay path calls x64_binop/x64_move/x64_cmp/x64_convert directly. */
 static void x64_binop_rr(NativeTarget* t, BinOp op, NativeRegLoc dst,
                          NativeRegLoc a, NativeRegLoc b) {
   x64_binop(t, op, native_loc_from_reg(dst), native_loc_from_reg(a),
             native_loc_from_reg(b));
 }
 
 static void x64_move_rr(NativeTarget* t, NativeRegLoc dst, NativeRegLoc src) {
   x64_move(t, native_loc_from_reg(dst), native_loc_from_reg(src));
 }
 
 static void x64_cmp_rr(NativeTarget* t, CmpOp op, NativeRegLoc dst,
                        NativeRegLoc a, NativeRegLoc b) {
   x64_cmp(t, op, native_loc_from_reg(dst), native_loc_from_reg(a),
           native_loc_from_reg(b));
 }
 
 static void x64_convert_rr(NativeTarget* t, ConvKind op, NativeRegLoc dst,
                            NativeRegLoc src) {
   x64_convert(t, op, native_loc_from_reg(dst), native_loc_from_reg(src));
 }
 
 /* ============================ spill / reload ============================ */
 
 static void x64_spill(NativeTarget* t, NativeLoc src, NativeFrameSlot slot,
                       MemAccess mem) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = slot;
   addr.base_type = src.type;
   x64_emit_mem(x64_of(t), 0, src, addr, mem);
 }
 static void x64_reload(NativeTarget* t, NativeLoc dst, NativeFrameSlot slot,
                        MemAccess mem) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = slot;
   addr.base_type = dst.type;
   x64_emit_mem(x64_of(t), 1, dst, addr, mem);
 }
 
 /* ============================ control flow ============================ */
 
 static void emit_jmp_rel32(MCEmitter* mc, MCLabel l) {
   u8 op = X64_OPC_JMP_REL32;
   mc_emit_bytes(mc, &op, 1);
   emit_u32le(mc, 0);
   mc_emit_label_ref(mc, l, R_PC32, 4, -4);
 }
 static void emit_jcc_rel32(MCEmitter* mc, u32 cc, MCLabel l) {
   u8 op[2] = {X64_OPC_TWOBYTE, (u8)(X64_OPC_JCC_BASE | (cc & 0xfu))};
   mc_emit_bytes(mc, op, 2);
   emit_u32le(mc, 0);
   mc_emit_label_ref(mc, l, R_PC32, 4, -4);
 }
 
 static MCLabel x64_label_new(NativeTarget* t) {
   return mc_label_new(t->mc);
 }
 static void x64_label_place(NativeTarget* t, MCLabel l) {
   mc_label_place(t->mc, l);
 }
 static void x64_jump(NativeTarget* t, MCLabel l) { emit_jmp_rel32(t->mc, l); }
 
 static void x64_cmp_branch(NativeTarget* t, CmpOp op, NativeLoc aop,
                            NativeLoc bop, MCLabel l) {
   MCEmitter* mc = t->mc;
   int fp = cmp_is_fp(op, aop);
   if (fp) {
     /* Materialize the 0/1 result, then branch on nonzero. */
     NativeLoc tmp =
         native_loc_reg(builtin_id(KIT_CG_BUILTIN_I32), NATIVE_REG_INT,
                        X64_TMP_INT);
     x64_cmp(t, op, tmp, aop, bop);
     emit_test_self(mc, 0, X64_TMP_INT);
     emit_jcc_rel32(mc, X64_CC_NE, l);
     return;
   }
   x64_emit_cmp_flags(t, aop, bop, 0);
   emit_jcc_rel32(mc, cmp_to_cc(op), l);
 }
 
 /* Emit an indirect `call`/`jmp r/m64` (opcode FF) against register `r`. The
  * ModRM reg field is the opcode digit: /2 for `call`, /4 for `jmp`. Byte-for-
  * byte the open-coded encoding used by x64_indirect_branch, x64_emit_call's
  * indirect (r11-staged) site, and x64_emit_tail_site's indirect (r11) site. */
 static void x64_emit_indirect_rm(MCEmitter* mc, u32 r, u32 digit) {
   if (r & 8u) {
     u8 rex = X64_REX_BASE | X64_REX_B;
     mc_emit_bytes(mc, &rex, 1);
   }
   {
     u8 buf[2] = {X64_OP_JMP_RM64, modrm(3u, digit, r & 7u)};
     mc_emit_bytes(mc, buf, 2);
   }
 }
 
 static void x64_indirect_branch(NativeTarget* t, NativeLoc addr,
                                 const MCLabel* valid_targets, u32 ntargets) {
   MCEmitter* mc = t->mc;
   u32 r = loc_reg(addr);
   (void)valid_targets;
   (void)ntargets;
   x64_emit_indirect_rm(mc, r, 4u); /* jmp r/m, /4 */
 }
 
 static void x64_load_label_addr(NativeTarget* t, NativeLoc dst, MCLabel l) {
   /* `&&label` address-take: `leaq sym(%rip), rd` with an R_PC32 relocation
    * against the label's per-block local symbol — same form as a global
    * address-take, so a re-encoding assembler recomputes the displacement.
    * (A baked disp32 with no reloc would break once clang re-lays-out the
    * function.) */
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst);
   ObjSymId sym = mc_label_symbol(mc, l);
   u32 disp_pos;
   emit_rex(mc, 1, rd, 0, 0);
   {
     u8 op = X64_OPC_LEA;
     mc_emit_bytes(mc, &op, 1);
   }
   {
     u8 mr = modrm(0u, rd & 7u, 5u); /* [rip + disp32] */
     mc_emit_bytes(mc, &mr, 1);
   }
   disp_pos = mc_pos(mc);
   emit_u32le(mc, 0);
   mc_emit_reloc_at(mc, mc->section_id, disp_pos, R_PC32, sym, -4, 1, 0);
 }
 
 /* ============================ frame / lifecycle ============================
  */
 
 static NativeFrameSlot x64_frame_slot(NativeTarget* t,
                                       const NativeFrameSlotDesc* d) {
   return native_frame_slot_alloc(&x64_of(t)->frame, d);
 }
 
 static void x64_release_frame_slot(NativeTarget* t, NativeFrameSlot slot) {
   native_frame_release_slot(&x64_of(t)->frame, slot);
 }
 
 static int x64_frame_slot_debug_loc(NativeTarget* t, NativeFrameSlot slot,
                                     CGDebugLoc* out) {
   X64NativeTarget* a = x64_of(t);
   X64NativeSlot* s;
   if (!out) return 0;
   memset(out, 0, sizeof *out);
   if (slot == NATIVE_FRAME_SLOT_NONE || slot > a->frame.nslots) return 0;
   s = x64_slot_get(a, slot);
   out->kind = CG_DEBUG_LOC_FRAME;
   /* x64 slots live at RBP - off (exactly how the memory-operand path addresses
    * them). The hosted dbg snapshot seeds the frame base with RBP, so report
    * the RBP-relative offset — mirroring aa64's FP-relative convention. */
   out->v.frame_ofs = -(i32)s->off;
   return 1;
 }
 
 /* xmm save area base (rbp-relative). XMM saves are 16-aligned. */
 static u32 x64_xmm_base(const X64NativeTarget* a, u32 cs_fp) {
   if (cs_fp == 0) return a->frame.cum_off;
   return align_up_u32(a->frame.cum_off, 16u);
 }
 
 static u32 x64_compute_frame_size(const X64NativeTarget* a, u32 cs_int,
                                   u32 cs_fp) {
   u32 xmm_base = x64_xmm_base(a, cs_fp);
   u32 raw = a->frame.max_outgoing + cs_int * 8u + cs_fp * 16u + xmm_base;
   u32 fs = align_up_u32(raw, 16u);
   return fs ? fs : 16u;
 }
 
 /* Collect the callee-saves the body actually used. */
 static u32 x64_collect_int_saves(X64NativeTarget* a, Reg* regs) {
   u32 n = 0, i;
   for (i = 0; i < a->frame.ncallee_saves; ++i)
     if (a->frame.callee_saves[i].cls == NATIVE_REG_INT)
       regs[n++] = a->frame.callee_saves[i].reg;
   return n;
 }
 static u32 x64_collect_fp_saves(X64NativeTarget* a, Reg* regs) {
   u32 n = 0, i;
   for (i = 0; i < a->frame.ncallee_saves; ++i)
     if (a->frame.callee_saves[i].cls == NATIVE_REG_FP)
       regs[n++] = a->frame.callee_saves[i].reg;
   return n;
 }
 
 /* rbp-relative offset of callee-save slot `idx`. The int GPRs sit below the
  * 16-byte-aligned XMM save area (`xmm_base`) and the `n_fp` 16-byte XMM saves;
  * the XMMs sit just below `xmm_base`. Single source for the prologue spill, the
  * epilogue/tail-site restore, and the CFI offsets — all previously open-coded
  * with this exact arithmetic. */
 static inline i32 x64_cs_int_off(u32 xmm_base, u32 n_fp, u32 idx) {
   return -(i32)xmm_base - (i32)n_fp * 16 - (i32)(idx + 1u) * 8;
 }
 static inline i32 x64_cs_fp_off(u32 xmm_base, u32 idx) {
   return -(i32)xmm_base - (i32)(idx + 1u) * 16;
 }
 
 /* Emit the callee-save restores (reverse spill order: XMMs then GPRs) at the
  * current emit cursor. Shared by the function epilogue (x64_func_end) and the
  * tail-call site (x64_emit_tail_site); mirrors aa64's aa_emit_callee_restores.
  * Frame-size-independent, so it needs only the collected save sets. */
 static void x64_emit_callee_restores(X64NativeTarget* a) {
   MCEmitter* mc = a->base.mc;
   Reg cs_int[X64_MAX_CS_INT_REGS], cs_fp[X64_MAX_CS_FP_REGS];
   u32 n_int = x64_collect_int_saves(a, cs_int);
   u32 n_fp = x64_collect_fp_saves(a, cs_fp);
   u32 xmm_base = x64_xmm_base(a, n_fp);
   i32 i;
   for (i = (i32)n_fp - 1; i >= 0; --i)
     emit_sse_load(mc, 0, 0x28, cs_fp[i], X64_RBP,
                   x64_cs_fp_off(xmm_base, (u32)i)); /* movaps */
   for (i = (i32)n_int - 1; i >= 0; --i)
     emit_mov_load(mc, 8, 0, cs_int[i], X64_RBP,
                   x64_cs_int_off(xmm_base, n_fp, (u32)i));
 }
 
 static ObjSymId x64_chkstk_sym(NativeTarget* t) {
   Sym name = pool_intern_slice(t->c->global, SLICE_LIT("__chkstk"));
   ObjSymId s = obj_symbol_find(t->obj, name);
   if (s != 0) return s;
   return obj_symbol(t->obj, name, SB_GLOBAL, SK_UNDEF, OBJ_SEC_NONE, 0, 0);
 }
 
 /* Build the prologue byte sequence into buf. Returns bytes written and, when
  * the chkstk path fires, the disp32 offset of the call site. When `skip_sub` is
  * set (the known-frame slim / red-zone tiers), the `sub rsp` reservation is
  * omitted entirely: the frame record is established but no stack is reserved,
  * either because the frame is empty (slim) or because the locals/saves live in
  * the SysV red zone (redzone_leaf). Callers must only set it when the frame
  * needs no reserved region (no alloca, no outgoing args, and — for the red
  * zone — a leaf frame <= 128 bytes). */
 static u32 x64_build_prologue(X64NativeTarget* a, u8* buf, u32 cap,
                               u32 frame_size, const Reg* cs_int, u32 n_int,
                               const Reg* cs_fp, u32 n_fp, int skip_sub,
                               u32* chkstk_disp_pos_out) {
   u32 wi = 0;
   u32 xmm_base = x64_xmm_base(a, n_fp);
   u32 i;
   /* Page granularity for Windows large-frame probing (0 = no probe needed).
    * Win64 reserves >1-page frames through __chkstk; the same ABI capability
    * the aarch64 backend reads for its inline probe. */
   u32 probe = abi_stack_probe_interval(a->base.c->abi);
   *chkstk_disp_pos_out = (u32)-1;
   if (cap < X64_PROLOGUE_BASE_BYTES)
     x64_panic(a, "prologue placeholder overflow");
   /* push rbp; mov rbp, rsp. */
   buf[wi++] = (u8)(X64_OPC_PUSH_R | (X64_RBP & 7u));
   buf[wi++] = X64_REX_BASE | X64_REX_W;
   buf[wi++] = X64_OPC_MOV_RM_R;
   buf[wi++] = modrm(3u, X64_RSP, X64_RBP);
   /* sub rsp, frame_size (or chkstk on Win64 large frame); skipped by the slim /
    * red-zone tiers, which reserve no stack. */
   if (skip_sub) {
     /* no reservation */
   } else if (probe && frame_size > probe) {
     if (wi + 13u > cap) x64_panic(a, "prologue placeholder overflow");
     buf[wi++] = (u8)(X64_OPC_MOV_RI | (X64_RAX & 7u)); /* mov eax, imm32 */
     wr_u32_le(buf + wi, frame_size);
     wi += 4;
     buf[wi++] = X64_OPC_CALL_REL32;
     *chkstk_disp_pos_out = wi;
     wr_u32_le(buf + wi, 0);
     wi += 4;
     buf[wi++] = X64_REX_BASE | X64_REX_W; /* sub rsp, rax */
     buf[wi++] = X64_OPC_ALU_SUB;
     buf[wi++] = modrm(3u, X64_RAX, X64_RSP);
   } else {
     if (wi + 7u > cap) x64_panic(a, "prologue placeholder overflow");
     buf[wi++] = X64_REX_BASE | X64_REX_W;
     buf[wi++] = X64_OPC_ALU_IMM32;
     buf[wi++] = modrm(3u, X64_ALU_SUB_SUB, X64_RSP);
     wr_u32_le(buf + wi, frame_size);
     wi += 4;
   }
   /* sret: spill the first int arg reg (destination pointer) into its slot.
    * Use the minimal disp encoding (x64_pack_mem) so it matches the body's
    * frame stores and the matching epilogue restore — the `cc -S | as`
    * round-trip can then reproduce these bytes exactly. The -O0 placeholder is
    * NOP-padded to a fixed width, so a shorter prologue is harmless. */
   if (a->has_sret && a->sret_ptr_slot != NATIVE_FRAME_SLOT_NONE) {
     X64NativeSlot* s = x64_slot_get(a, a->sret_ptr_slot);
     u32 sret_reg = a->abi->int_args[0];
     i32 off = -(i32)s->off;
     if (wi + 8u > cap) x64_panic(a, "prologue placeholder overflow");
     buf[wi++] =
         (u8)(X64_REX_BASE | X64_REX_W | ((sret_reg & 8u) ? X64_REX_R : 0u));
     buf[wi++] = X64_OPC_MOV_RM_R;
     wi += x64_pack_mem(buf + wi, sret_reg & 7u, X64_RBP, off);
   }
   /* Spill callee-saved GPRs. */
   for (i = 0; i < n_int; ++i) {
     u32 reg = cs_int[i];
     i32 off = x64_cs_int_off(xmm_base, n_fp, i);
     if (wi + 8u > cap) x64_panic(a, "prologue placeholder overflow");
     buf[wi++] = (u8)(X64_REX_BASE | X64_REX_W | ((reg & 8u) ? X64_REX_R : 0u));
     buf[wi++] = X64_OPC_MOV_RM_R;
     wi += x64_pack_mem(buf + wi, reg & 7u, X64_RBP, off);
   }
   /* Spill callee-saved XMMs (Win64). movaps [rbp+disp], xmm. */
   for (i = 0; i < n_fp; ++i) {
     u32 xmm = cs_fp[i];
     i32 off = x64_cs_fp_off(xmm_base, i);
     u8 rex = (u8)((xmm & 8u) ? (X64_REX_BASE | X64_REX_R) : 0u);
     u32 need = rex ? 9u : 8u;
     if (wi + need > cap) x64_panic(a, "prologue placeholder overflow");
     if (rex) buf[wi++] = rex;
     buf[wi++] = X64_OPC_TWOBYTE;
     buf[wi++] = 0x29; /* MOVAPS r/m128, xmm */
     wi += x64_pack_mem(buf + wi, xmm & 7u, X64_RBP, off);
   }
   return wi;
 }
 
 static u32 x64_signature_stack_bytes(NativeTarget* t, KitCgTypeId fn_type,
                                      int* variadic, u32* nparams);
 
 static void x64_func_begin_common(NativeTarget* t, const CGFuncDesc* fd) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, fd->fn_type);
   a->func = fd;
   a->loc = fd->loc;
   a->abi = x64_abi_for_os(t->c->target.os);
   /* Shared frame bookkeeping: clears the slot table, cum_off, max_outgoing,
    * callee-save set, and known_frame/has_alloca/frame_final. */
   native_frame_reset(&a->frame);
   /* Tail-call legality is queried while the single-pass frontend is emitting
    * the body, so derive the complete incoming stack window from the signature
    * up front rather than waiting for bind_param to walk every declaration.
    * x64_signature_stack_bytes includes Win64 shadow space; keep only the
    * reusable argument suffix here because x64_no_tail adds the common prefix
    * on both sides of its comparison. */
   a->incoming_stack_size =
       x64_signature_stack_bytes(t, fd->fn_type, NULL, NULL) -
       a->abi->shadow_space;
   a->next_param_int = 0;
   a->next_param_fp = 0;
   a->next_param_stack = 0;
   a->has_sret = (abi && abi->has_sret) ? 1u : 0u;
   a->is_variadic = (abi && abi->variadic) ? 1u : 0u;
   a->sret_ptr_slot = NATIVE_FRAME_SLOT_NONE;
   a->reg_save_slot = NATIVE_FRAME_SLOT_NONE;
   a->npatches = 0;
   a->nalloca = 0;
   a->nbind_moves = 0;
   a->slim_frame = 0;
   a->redzone_leaf = 0;
   /* Single-pass (-O0) reservation: the NDT caches only in its allocable pool's
    * bounded callee-saved subset (3 int + 6 fp on Win64), so a far smaller
    * placeholder than the optimizer-worst-case X64_PROLOGUE_BYTES suffices. The
    * known-frame path overrides prologue_nbytes with its exact length. */
   a->prologue_nbytes = a->abi->shadow_space ? X64_NDT_PROLOGUE_BYTES_WIN64
                                             : X64_NDT_PROLOGUE_BYTES;
 
   mc_set_section(mc, fd->text_section_id);
   mc_emit_align(mc, 16, X64_NOP1);
   a->func_start = mc_pos(mc);
   mc_begin_function(mc, fd->sym, fd->text_section_id, a->func_start);
   mc_cfi_startproc(mc);
   a->epilogue_label = mc_label_new(mc);
 }
 
 /* Reserve the sret-pointer slot and (SysV) the 176-byte variadic reg-save
  * area. Advances next_param_int past the sret pointer (a0). */
 static void x64_reserve_entry_saves(X64NativeTarget* a) {
   NativeTarget* t = &a->base;
   if (a->has_sret) {
     NativeFrameSlotDesc sd;
     memset(&sd, 0, sizeof sd);
     sd.type = builtin_id(KIT_CG_BUILTIN_I64);
     sd.size = 8;
     sd.align = 8;
     sd.kind = NATIVE_FRAME_SLOT_SAVE;
     a->sret_ptr_slot = t->frame_slot(t, &sd);
     a->next_param_int = 1;
   }
   if (a->is_variadic && a->abi->emit_sysv_vararg_save) {
     NativeFrameSlotDesc rd;
     memset(&rd, 0, sizeof rd);
     rd.type = builtin_id(KIT_CG_BUILTIN_I64);
     rd.size = 176;
     rd.align = 8;
     rd.kind = NATIVE_FRAME_SLOT_SAVE;
     a->reg_save_slot = t->frame_slot(t, &rd);
   }
 }
 
 static void x64_emit_variadic_reg_saves(X64NativeTarget* a) {
   NativeTarget* t = &a->base;
   MCEmitter* mc = t->mc;
   if (!a->is_variadic) return;
   if (a->abi->emit_sysv_vararg_save) {
     X64NativeSlot* rs = x64_slot_get(a, a->reg_save_slot);
     static const u32 gprs[6] = {X64_RDI, X64_RSI, X64_RDX,
                                 X64_RCX, X64_R8,  X64_R9};
     u32 i;
     for (i = 0; i < 6u; ++i)
       emit_mov_store(mc, 8, gprs[i], X64_RBP, -(i32)rs->off + (i32)(i * 8u));
     for (i = 0; i < 8u; ++i)
       emit_sse_store(mc, 0xF2, 0x11, (u32)(X64_XMM0 + i), X64_RBP,
                      -(i32)rs->off + (i32)(48u + i * 16u));
     return;
   }
   /* Win64 variadic: spill the 4 GPR arg slots to the home space. */
   emit_mov_store(mc, 8, X64_RCX, X64_RBP, 16);
   emit_mov_store(mc, 8, X64_RDX, X64_RBP, 24);
   emit_mov_store(mc, 8, X64_R8, X64_RBP, 32);
   emit_mov_store(mc, 8, X64_R9, X64_RBP, 40);
 }
 
 static void x64_func_begin(NativeTarget* t, const CGFuncDesc* fd) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   u32 i;
   x64_func_begin_common(t, fd);
   a->prologue_pos = mc_pos(mc);
   for (i = 0; i < a->prologue_nbytes; ++i) emit1(mc, X64_NOP1);
   x64_reserve_entry_saves(a);
   x64_emit_variadic_reg_saves(a);
 }
 
 /* x64 homes callee-saves below the locals (offsets computed in
  * x64_compute_frame_size / x64_build_prologue), not in frame slots, so
  * alloc_slots=0: native_frame just records the {reg,cls} set from the masks. */
 static void x64_reserve_callee_saves(NativeTarget* t, const u32* used,
                                      u32 nclasses) {
   native_frame_set_callee_saves(&x64_of(t)->frame, used, nclasses, NULL, 0, 0);
 }
 
 static int x64_reg_is_callee_int(const X64ABIRegs* abi, Reg r);
 static int x64_reg_is_callee_fp(const X64ABIRegs* abi, Reg r);
 
 static u32 x64_live_callee_saved_mask(NativeTarget* t, NativeAllocClass cls) {
   X64NativeTarget* a = x64_of(t);
   const X64ABIRegs* abi = a->abi ? a->abi : x64_abi_for_os(t->c->target.os);
   u32 mask = 0;
   for (Reg r = 0; r < 16u; ++r) {
     if (cls == NATIVE_REG_INT && x64_reg_is_callee_int(abi, r)) mask |= 1u << r;
     if (cls == NATIVE_REG_FP && x64_reg_is_callee_fp(abi, r)) mask |= 1u << r;
   }
   return mask;
 }
 
 static u32 x64_live_caller_saved_mask(NativeTarget* t, NativeAllocClass cls) {
   u32 caller_saved =
       native_reg_info_flag_mask(t->regs, cls, NATIVE_REG_CALLER_SAVED);
   return caller_saved & ~x64_live_callee_saved_mask(t, cls);
 }
 
 static void x64_asm_clobber_masks(Compiler* c, SrcLoc loc, const Sym* clobbers,
                                   u32 nclob, u32* int_mask, u32* fp_mask);
 
 /* abi_clobber_masks is shared as native_asm_abi_clobber_masks
  * (cg/native_asm.h); it reads the target's live ABI masks. */
 
 /* Build the callee-saved set the prologue must preserve: the allocator-assigned
  * callee-saved registers (frame->callee_saved_used) plus any an inline-asm
  * block clobbers. The latter are opaque to the optimizer's operand scan, so it
  * forwards the raw clobber names (frame->asm_clobbers) and the arch-neutral
  * clobber-ABI sets (frame->asm_clobber_abi_sets); we resolve both into masks
  * and keep only the callee-saved ones. x64_reg_is_callee_* follow the live ABI:
  * they exclude rbp (handled by the prologue head) and keep the
  * reserved-but-callee- saved scratch rbx/r12 (which the caller still expects
  * preserved). This is the same register selection the per-block spill used,
  * hoisted into the prologue. */
 static int x64_asm_reg_is_callee_saved(NativeTarget* t, NativeAllocClass cls,
                                        Reg r) {
   X64NativeTarget* a = x64_of(t);
   const X64ABIRegs* abi = a->abi ? a->abi : x64_abi_for_os(t->c->target.os);
   return (cls == NATIVE_REG_INT && x64_reg_is_callee_int(abi, r)) ||
          (cls == NATIVE_REG_FP && x64_reg_is_callee_fp(abi, r));
 }
 
 static u32 x64_known_callee_saves(NativeTarget* t,
                                   const NativeKnownFrameDesc* frame, u32* out,
                                   u32 cap) {
   X64NativeTarget* a = x64_of(t);
   SrcLoc loc = a->func ? a->func->loc : (SrcLoc){0, 0, 0};
   return native_asm_known_callee_saves(
       t, loc, frame, out, cap, x64_asm_clobber_masks,
       x64_asm_reg_is_callee_saved);
 }
 
 /* Optimizer entry point: the full frame is supplied up front, so the prologue
  * is emitted final the moment it is built — no NOP region, no func_end patch
  * (x64_func_end skips patching when known_frame). x64_build_prologue emits the
  * push rbp / sub rsp / sret spill / callee-save spills; the variadic
  * register-save stores are emitted separately, as on the single-pass path. */
 static void x64_func_begin_known_frame(NativeTarget* t, const CGFuncDesc* fd,
                                        const NativeKnownFrameDesc* frame,
                                        NativeFrameSlot* out_slots) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   Reg cs_int[X64_MAX_CS_INT_REGS], cs_fp[X64_MAX_CS_FP_REGS];
   u32 n_int, n_fp, frame_size, nbytes, chkstk_disp_pos, i;
   u8 buf[X64_PROLOGUE_BYTES_WIN64];
   x64_func_begin_common(t, fd);
   a->frame.known_frame = 1;
   if (frame) {
     u32 cs[NATIVE_REG_CLASS_COUNT];
     u32 ncs =
         x64_known_callee_saves(t, frame, cs, NATIVE_REG_CLASS_COUNT);
     a->frame.has_alloca = frame->has_alloca;
     if (ncs) x64_reserve_callee_saves(t, cs, ncs);
     for (i = 0; i < frame->nslots; ++i) {
       NativeFrameSlot slot = x64_frame_slot(t, &frame->slots[i]);
       if (out_slots) out_slots[i] = slot;
     }
     x64_reserve_entry_saves(a);
     native_frame_note_outgoing(&a->frame, frame->max_outgoing);
   }
   /* Frame is final: size and offsets are settled, so emit the exact prologue.
    */
   n_int = x64_collect_int_saves(a, cs_int);
   n_fp = x64_collect_fp_saves(a, cs_fp);
   frame_size = x64_compute_frame_size(a, n_int, n_fp);
   a->frame_size_final = frame_size;
   /* Cost-model tier selection (mirrors aa64's aa_func_begin_known_frame): with
    * the frame final before the body, choose the cheapest valid prologue shape.
    * Both tiers keep the rbp record and only drop the `sub rsp`, so the
    * epilogue/CFI/offset helpers are untouched. x64 needs no
    * `fp_at_bottom`-style fold: `push rbp` already folds the sp-move into the
    * store. */
   a->slim_frame = a->frame.ncallee_saves == 0 && !a->frame.has_alloca &&
                   a->frame.cum_off == 0 && a->frame.max_outgoing == 0;
   /* redzone keeps locals below rsp in the red zone; exclude inline asm, which
    * may issue a `call` (clobbering the red zone) the optimizer can't see. slim
    * needs no such guard: it has no locals there and the return address lives on
    * the stack at [rbp+8], not in a clobberable register. */
   a->redzone_leaf =
       !a->slim_frame &&
       (t->disabled_backend_features & KIT_CG_BACKEND_RED_ZONE) == 0 &&
       a->abi->shadow_space == 0 && frame && frame->is_leaf &&
       !frame->has_asm && !a->frame.has_alloca && a->frame.max_outgoing == 0 &&
       frame_size <= 128u;
   a->prologue_pos = mc_pos(mc);
   nbytes = x64_build_prologue(a, buf, sizeof buf, frame_size, cs_int, n_int,
                               cs_fp, n_fp, a->slim_frame || a->redzone_leaf,
                               &chkstk_disp_pos);
   mc_emit_bytes(mc, buf, nbytes);
   if (chkstk_disp_pos != (u32)-1) {
     ObjSymId chk = x64_chkstk_sym(t);
     mc_emit_reloc_at(mc, mc->section_id, a->prologue_pos + chkstk_disp_pos,
                       R_X64_PLT32, chk, -4, 1, 0);
   }
   a->prologue_nbytes = nbytes; /* exact length: used for the CFI post offset */
   x64_emit_variadic_reg_saves(a);
   native_frame_set_final(&a->frame);
 }
 
 static void x64_func_end(NativeTarget* t) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   ObjBuilder* obj = t->obj;
   ObjSecId sec = a->func->text_section_id;
   Reg cs_int[X64_MAX_CS_INT_REGS], cs_fp[X64_MAX_CS_FP_REGS];
   u32 n_int = x64_collect_int_saves(a, cs_int);
   u32 n_fp = x64_collect_fp_saves(a, cs_fp);
   u32 frame_size = x64_compute_frame_size(a, n_int, n_fp);
   u32 xmm_base = x64_xmm_base(a, n_fp);
   u32 end;
   a->frame_size_final = frame_size;
 
   /* Epilogue. */
   mc_label_place(mc, a->epilogue_label);
   x64_emit_callee_restores(a);
   emit_leave(mc);
   emit_ret(mc);
 
   /* Patch the single-pass prologue placeholder. */
   if (!a->frame.known_frame) {
     u8 buf[X64_PROLOGUE_BYTES_WIN64];
     u32 chkstk_disp_pos;
     u32 nbytes;
     u32 k;
     for (k = 0; k < a->prologue_nbytes; ++k) buf[k] = X64_NOP1;
     /* Single-pass path never selects a slim/red-zone tier (it cannot know the
      * frame up front), so it always emits the full reservation. */
     nbytes = x64_build_prologue(a, buf, a->prologue_nbytes, frame_size, cs_int,
                                 n_int, cs_fp, n_fp, 0, &chkstk_disp_pos);
     (void)nbytes;
     obj_patch(obj, sec, a->prologue_pos, buf, a->prologue_nbytes);
     if (chkstk_disp_pos != (u32)-1) {
       ObjSymId chk = x64_chkstk_sym(t);
       mc_emit_reloc_at(mc, sec, a->prologue_pos + chkstk_disp_pos, R_X64_PLT32,
                         chk, -4, 1, 0);
     }
   }
 
   /* Patch alloca disp32s: lea dst, [rsp + max_outgoing]. */
   {
     u32 mo = align_up_u32(a->frame.max_outgoing, 16u);
     u32 k;
     for (k = 0; k < a->npatches; ++k) {
       u8 dbuf[4];
       wr_u32_le(dbuf, mo);
       obj_patch(obj, sec, a->patches[k].pos, dbuf, 4);
     }
   }
 
   /* CFI: after the prologue, CFA = rbp + 16; rbp at cfa-16, ra at cfa-8. */
   {
     /* Body starts past the prologue. prologue_nbytes is the reserved NOP-region
      * size on the single-pass path and the exact prologue length on the
      * known-frame path (set in x64_func_begin_known_frame). */
     u32 post = a->prologue_pos + a->prologue_nbytes;
     u32 k;
     mc_cfi_set_next_pc_offset(mc, post - a->func_start);
     /* CFI register operands are DWARF numbers, which differ from the x86-64
      * hardware encoding for rbp/rsp/rsi/rdi/rcx/rdx (e.g. rbp is HW 5 but
      * DWARF 6). Map every hardware GPR through x64_dwarf_from_hw_gpr; rip's
      * DWARF number (16) is already correct. */
     mc_cfi_def_cfa(mc, x64_dwarf_from_hw_gpr(X64_RBP), 16);
     mc_cfi_offset(mc, x64_dwarf_from_hw_gpr(X64_RBP), -16);
     mc_cfi_offset(mc, 16u /* rip */, -8);
     for (k = 0; k < n_int; ++k) {
       i32 off = x64_cs_int_off(xmm_base, n_fp, k);
       mc_cfi_offset(mc, x64_dwarf_from_hw_gpr(cs_int[k]), off);
     }
   }
 
   end = mc_pos(mc);
   obj_symbol_define(obj, a->func->sym, sec, (u64)a->func_start,
                     (u64)(end - a->func_start));
   if (a->func->atomize)
     obj_atom_define(obj, sec, a->func_start, end - a->func_start, a->func->sym,
                     0);
   if (mc->debug) debug_func_pc_range(mc->debug, sec, a->func_start, end);
   mc_cfi_endproc(mc);
   mc_end_function(mc);
   a->func = NULL;
 }
 
 /* ============================ params / ABI helpers
  * ============================
  */
 
 /* Win64 shares one arg-slot index across int and FP. Keep cursors in lockstep.
  */
 static void x64_sync_slot(const X64ABIRegs* abi, u32* next_int, u32* next_fp) {
   u32 m;
   if (!abi->slot_shared_int_fp) return;
   m = *next_int > *next_fp ? *next_int : *next_fp;
   *next_int = m;
   *next_fp = m;
 }
 
 static const ABIArgInfo* x64_param_abi(NativeTarget* t, const ABIFuncInfo* abi,
                                        const NativeCallDesc* desc, u32 i,
                                        ABIArgInfo* scratch) {
   int variadic = abi && i >= abi->nparams;
   if (abi && i < abi->nparams) return &abi->params[i];
   (void)variadic;
   memset(scratch, 0, sizeof *scratch);
   scratch->kind = ABI_ARG_DIRECT;
   scratch->nparts = 1;
   scratch->parts = arena_zarray(t->c->tu, ABIArgPart, 1);
   ((ABIArgPart*)scratch->parts)[0].cls =
       cg_type_is_float(t->c, desc->args[i].type) ? ABI_CLASS_FP : ABI_CLASS_INT;
   ((ABIArgPart*)scratch->parts)[0].loc = ABI_LOC_REG;
   ((ABIArgPart*)scratch->parts)[0].size =
       native_type_size(t, desc->args[i].type);
   ((ABIArgPart*)scratch->parts)[0].align =
       native_type_align(t, desc->args[i].type);
   return scratch;
 }
 
 static KitCgTypeId x64_part_scalar_type(const ABIArgPart* part) {
   if (part->cls == ABI_CLASS_FP)
     return part->size <= 4u ? builtin_id(KIT_CG_BUILTIN_F32)
                             : builtin_id(KIT_CG_BUILTIN_F64);
   switch (part->size) {
     case 1u:
       return builtin_id(KIT_CG_BUILTIN_I8);
     case 2u:
       return builtin_id(KIT_CG_BUILTIN_I16);
     case 4u:
       return builtin_id(KIT_CG_BUILTIN_I32);
     default:
       return builtin_id(KIT_CG_BUILTIN_I64);
   }
 }
 
 /* Is the whole DIRECT arg forced to the stack (not enough reg slots)? */
 static int x64_direct_to_stack(const X64ABIRegs* abi, const ABIArgInfo* ai,
                                u32 next_int, u32 next_fp) {
   u32 need_int, need_fp;
   x64_abi_direct_reg_need(ai, &need_int, &need_fp);
   return next_int + need_int > abi->n_int_args ||
          next_fp + need_fp > abi->n_fp_args;
 }
 
 /* Raw outgoing argument bytes a call uses, including Win64 shadow space but
  * excluding the final call-site alignment padding. */
 static u32 x64_call_stack_raw_size(NativeTarget* t,
                                    const NativeCallDesc* desc) {
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, desc->fn_type);
   const X64ABIRegs* aregs = x64_abi_for_os(t->c->target.os);
   u32 next_int = (abi && abi->has_sret) ? 1u : 0u;
   u32 next_fp = 0;
   u32 stack = aregs->shadow_space;
   u32 i;
   x64_sync_slot(aregs, &next_int, &next_fp);
   for (i = 0; i < desc->nargs; ++i) {
     ABIArgInfo tmp;
     const ABIArgInfo* ai = x64_param_abi(t, abi, desc, i, &tmp);
     u32 p;
     if (ai->kind == ABI_ARG_IGNORE) continue;
     if (ai->kind == ABI_ARG_INDIRECT) {
       if (next_int < aregs->n_int_args)
         ++next_int;
       else
         stack += 8u;
       x64_sync_slot(aregs, &next_int, &next_fp);
       continue;
     }
     if (ai->kind == ABI_ARG_DIRECT &&
         x64_direct_to_stack(aregs, ai, next_int, next_fp)) {
       stack += (u32)ai->nparts * 8u;
       continue;
     }
     for (p = 0; p < ai->nparts; ++p) {
       const ABIArgPart* part = &ai->parts[p];
       if (part->cls == ABI_CLASS_FP) {
         if (next_fp < aregs->n_fp_args)
           ++next_fp;
         else
           stack += 8u;
       } else {
         if (next_int < aregs->n_int_args)
           ++next_int;
         else
           stack += 8u;
       }
       x64_sync_slot(aregs, &next_int, &next_fp);
     }
   }
   return stack;
 }
 
 /* Outgoing stack bytes a call uses (16-aligned), per the ABI. */
 static u32 x64_call_stack_size(NativeTarget* t, const NativeCallDesc* desc) {
   return align_up_u32(x64_call_stack_raw_size(t, desc), 16u);
 }
 
 /* A normal indirect/byval argument can point into the caller's local frame,
  * because that frame remains live through a call. A sibling call tears the
  * frame down first, so every such payload needs stable storage in the reusable
  * incoming argument window. Pack those payloads after the raw outgoing ABI
  * slots (using their natural alignment); final call alignment may provide all
  * the space, as in a packed three-byte Win64 argument after one stack slot. */
 static u32 x64_tail_call_stack_size(NativeTarget* t,
                                     const NativeCallDesc* desc) {
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, desc->fn_type);
   u32 payload = x64_call_stack_raw_size(t, desc);
   u32 i;
   for (i = 0; i < desc->nargs; ++i) {
     ABIArgInfo tmp;
     const ABIArgInfo* ai = x64_param_abi(t, abi, desc, i, &tmp);
     u32 align, size;
     if (ai->kind != ABI_ARG_INDIRECT) continue;
     align = ai->indirect_align;
     if (!align) align = native_type_align(t, desc->args[i].type);
     size = native_type_size(t, desc->args[i].type);
     payload = align_up_u32(payload, align ? align : 1u);
     payload += size;
   }
   return align_up_u32(payload, 16u);
 }
 
 static u32 x64_call_stack_bytes(NativeTarget* t, const NativeCallDesc* desc) {
   return (desc->flags & CG_CALL_TAIL) ? x64_tail_call_stack_size(t, desc)
                                       : x64_call_stack_size(t, desc);
 }
 
 static u32 x64_signature_stack_bytes(NativeTarget* t, KitCgTypeId fn_type,
                                      int* variadic, u32* nparams) {
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, fn_type);
   NativeCallDesc d;
   if (variadic) *variadic = abi ? (int)abi->variadic : 0;
   if (nparams) *nparams = abi ? abi->nparams : 0u;
   memset(&d, 0, sizeof d);
   d.fn_type = fn_type;
   d.nargs = abi ? abi->nparams : 0u;
   if (d.nargs) d.args = arena_zarray(t->c->tu, NativeLoc, d.nargs);
   return x64_call_stack_size(t, &d);
 }
 
 /* Resolve a dereferenceable NativeLoc to its storage address. */
 static NativeAddr x64_storage_addr(X64NativeTarget* a, NativeLoc loc,
                                    u32 offset) {
   NativeAddr addr;
   if (!native_loc_storage_addr(loc, (i32)offset, &addr))
     x64_panic(a, "location is not storage");
   return addr;
 }
 
 static void x64_move_part_reg(NativeTarget* t, NativeLoc dst, NativeLoc src,
                               u32 size) {
   MCEmitter* mc = t->mc;
   int dfp = native_loc_is_fp(dst), sfp = native_loc_is_fp(src);
   u32 rd = loc_reg(dst), rs = loc_reg(src);
   if (dfp && sfp) {
     if (size != 4u && size != 8u)
       x64_panic(x64_of(t), "invalid FP register part width");
     if (rd != rs) emit_sse_rr(mc, sse_scalar_prefix(size), 0x10, rd, rs);
     return;
   }
   if (dfp && !sfp) {
     if (size != 4u && size != 8u)
       x64_panic(x64_of(t), "invalid GPR-to-FP part width");
     emit_sse_rr_w(mc, 0x66, 0x6E, size == 8u, rd, rs);
     return;
   }
   if (!dfp && sfp) {
     if (size != 4u && size != 8u)
       x64_panic(x64_of(t), "invalid FP-to-GPR part width");
     emit_sse_rr_w(mc, 0x66, 0x7E, size == 8u, rs, rd);
     return;
   }
   if (rd != rs) emit_mov_rr(mc, size > 4u, rd, rs);
 }
 
 static void x64_load_part(NativeTarget* t, NativeLoc dst, NativeLoc src,
                           u32 offset, u32 size) {
   X64NativeTarget* a = x64_of(t);
   if (src.kind == NATIVE_LOC_REG) {
     x64_move_part_reg(t, dst, src, size);
     return;
   }
   if (native_loc_addr_role(src) == NATIVE_LOC_ADDR_ROLE_VALUE) {
     NativeAddr addr;
     if (offset != 0u || size != t->c->target.ptr_size ||
         !native_loc_address_value(src, &addr))
       x64_panic(a, "split or invalid address-value argument");
     x64_load_addr(t, dst, addr);
     return;
   }
   if (native_loc_addr_role(src) == NATIVE_LOC_ADDR_ROLE_STORAGE) {
     NativeAddr addr = x64_storage_addr(a, src, offset);
     MemAccess mem = native_mem_for_type(t, dst.type, size);
     addr.base_type = dst.type;
     if (!x64_native_part_width(size) && size <= 8u)
       x64_load_int_part(a, dst, addr, mem, size);
     else
       x64_emit_mem(a, 1, dst, addr, mem);
     return;
   }
   if (src.kind == NATIVE_LOC_IMM) {
     i64 part;
     if (!native_loc_imm_part(src, offset, size, &part))
       x64_panic(a, "invalid immediate argument part");
     x64_emit_load_imm(t->mc, size > 4u, loc_reg(dst), part);
     return;
   }
   x64_panic(a, "unsupported part source");
 }
 
 static void x64_store_part(NativeTarget* t, NativeLoc dst, NativeLoc src,
                            u32 offset, u32 size) {
   X64NativeTarget* a = x64_of(t);
   if (native_loc_addr_role(dst) == NATIVE_LOC_ADDR_ROLE_STORAGE) {
     NativeAddr addr = x64_storage_addr(a, dst, offset);
     MemAccess mem = native_mem_for_type(t, src.type, size);
     addr.base_type = src.type;
     if (!x64_native_part_width(size) && size <= 8u)
       x64_store_int_part(a, addr, src, mem, size);
     else
       x64_emit_mem(a, 0, src, addr, mem);
     return;
   }
   if (dst.kind == NATIVE_LOC_REG) {
     x64_move_part_reg(t, dst, src, size);
     return;
   }
   x64_panic(a, "unsupported part destination");
 }
 
 static void x64_addr_of_loc(NativeTarget* t, NativeLoc dst, NativeLoc src) {
   NativeAddr addr = x64_storage_addr(x64_of(t), src, 0);
   x64_load_addr(t, dst, addr);
 }
 
 typedef struct X64CallMarshalCtx {
   u32 rsp_bias;
   Reg cycle_scratch;
   u8 tail;
   u8 callee_parked;
 } X64CallMarshalCtx;
 
 static void x64_store_outgoing_part(NativeTarget* t,
                                     const X64CallMarshalCtx* call,
                                     u32 stack_off, NativeLoc src, u32 size) {
   X64NativeTarget* a = x64_of(t);
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_REG;
   addr.base_type = src.type;
   if (call->tail) {
     /* A sibling call reuses the caller's frame: its outgoing stack args land in
      * the caller's incoming-arg window. `stack_off` already includes the
      * shadow-space prefix (the outgoing cursor starts at shadow_space), so the
      * window address is [rbp + 16 + stack_off] — the same bytes the tail-callee
      * reads once `leave` has restored rsp to the return address. */
     addr.base.reg = X64_RBP;
     addr.offset = (i32)(16u + stack_off);
   } else {
     addr.base.reg = X64_RSP;
     /* Keep semantic outgoing offsets anchored at the pre-marshalling rsp. The
      * per-call context owns any transient callee-save bias explicitly. */
     addr.offset = (i32)(stack_off + call->rsp_bias);
   }
   {
     MemAccess mem = native_mem_for_type(t, src.type, size);
     if (!x64_native_part_width(size) && size <= 8u)
       x64_store_int_part(a, addr, src, mem, size);
     else
       x64_emit_mem(a, 0, src, addr, mem);
   }
 }
 
 /* A stack carrier contains the same byte representation as aggregate storage,
  * so copy a non-power-of-two memory part chunk-by-chunk instead of needlessly
  * packing it into a GPR and unpacking it again. Besides being smaller, this
  * needs only one private register while an indirect callee may occupy r11. */
 static void x64_copy_outgoing_part(NativeTarget* t,
                                    const X64CallMarshalCtx* call, u32 stack_off,
                                    NativeLoc src,
                                    const ABIArgPart* part) {
   NativeAllocClass cls =
       part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
   KitCgTypeId pty = x64_part_scalar_type(part);
   Reg tmp = cls == NATIVE_REG_FP ? X64_TMP_FP : X64_TMP_INT;
   NativeLoc tmpreg = native_loc_reg(pty, cls, tmp);
 
   if (!x64_native_part_width(part->size) && part->size <= 8u &&
       native_loc_addr_role(src) == NATIVE_LOC_ADDR_ROLE_STORAGE) {
     NativePartChunkIter chunks = native_part_chunks(part->size, 8u);
     u32 chunk_off, chunk_size;
     while (native_part_chunk_next(&chunks, &chunk_off, &chunk_size)) {
       x64_load_part(t, tmpreg, src, part->src_offset + chunk_off, chunk_size);
       x64_store_outgoing_part(t, call, stack_off + chunk_off, tmpreg,
                               chunk_size);
     }
     return;
   }
 
   x64_load_part(t, tmpreg, src, part->src_offset, part->size);
   x64_store_outgoing_part(t, call, stack_off, tmpreg, part->size);
 }
 
 /* NativeTarget bind_param: route incoming param (ABI loc) into dst. */
 static void x64_emit_reg_arg_moves(NativeTarget* t, NativeArgMove* moves, u32 n,
                                    Reg int_scratch);
 
 /* Defer a register-destination param bind for the parallel-copy flush in
  * x64_bind_params_end. `src` is the incoming location (an arg register, or a
  * NATIVE_LOC_ADDR for an incoming stack slot). */
 static void x64_defer_reg_bind(X64NativeTarget* a, NativeLoc dst, NativeLoc src,
                                u32 size) {
   NativeArgMove* m;
   if (a->nbind_moves >= X64_MAX_BIND_MOVES)
     x64_panic(a, "too many register parameter binds");
   m = &a->bind_moves[a->nbind_moves++];
   memset(m, 0, sizeof *m);
   m->dst = dst;
   m->src = src;
   m->size = size;
 }
 
 /* Incoming stack-arg source as a NATIVE_LOC_ADDR ([rbp + bias + stack_off]). */
 static NativeLoc x64_incoming_stack_loc(KitCgTypeId type, NativeAllocClass cls,
                                         i32 off) {
   NativeLoc l;
   memset(&l, 0, sizeof l);
   l.kind = NATIVE_LOC_ADDR;
   l.cls = (u8)cls;
   l.type = type;
   l.v.addr.base_kind = NATIVE_ADDR_BASE_REG;
   l.v.addr.base.reg = X64_RBP;
   l.v.addr.base_type = type;
   l.v.addr.offset = off;
   return l;
 }
 
 static void x64_bind_native_param(NativeTarget* t, const CGParamDesc* p,
                                   NativeLoc dst) {
   X64NativeTarget* a = x64_of(t);
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, a->func->fn_type);
   const ABIArgInfo* ai =
       p->index < abi->nparams ? &abi->params[p->index] : NULL;
   int to_reg = dst.kind == NATIVE_LOC_REG;
   /* Incoming stack args sit above the saved rbp + return addr (+16); Win64
    * additionally reserves 32B of home space. */
   i32 incoming_bias = (i32)(16u + a->abi->shadow_space);
   u32 i;
   if (!ai || ai->kind == ABI_ARG_IGNORE) return;
 
   if (ai->kind == ABI_ARG_INDIRECT) {
     /* Incoming pointer to a byval copy: load pointer, memcpy into dst frame. */
     u32 ptr_reg;
     NativeAddr d_addr, from;
     AggregateAccess access;
     if (a->next_param_int < a->abi->n_int_args) {
       ptr_reg = a->abi->int_args[a->next_param_int++];
     } else {
       /* Keep a stack-homed byval pointer out of R10/R11 while copying from
        * it. x64_copy_bytes owns that private bank for transfer/address
        * resolution; in particular, resolving the frame destination through
        * R11 must not overwrite the source pointer between packed chunks. RAX
        * is reserved from allocation and has no live entry value here. */
       ptr_reg = X64_RAX;
       emit_mov_load(t->mc, 8, 0, ptr_reg, X64_RBP,
                     incoming_bias + (i32)a->next_param_stack);
       a->next_param_stack += 8u;
     }
     x64_sync_slot(a->abi, &a->next_param_int, &a->next_param_fp);
     if (dst.kind != NATIVE_LOC_FRAME)
       x64_panic(a, "indirect parameter requires a frame destination");
     memset(&d_addr, 0, sizeof d_addr);
     d_addr.base_kind = NATIVE_ADDR_BASE_FRAME;
     d_addr.base.frame = dst.v.frame;
     d_addr.base_type = p->type;
     memset(&from, 0, sizeof from);
     from.base_kind = NATIVE_ADDR_BASE_REG;
     from.base.reg = ptr_reg;
     from.base_type = p->type;
     memset(&access, 0, sizeof access);
     access.type = p->type;
     access.size = p->size ? p->size : (u32)cg_type_size(t->c, p->type);
     access.align = p->align ? p->align : native_type_align(t, p->type);
     x64_copy_bytes(t, d_addr, from, access);
     return;
   }
 
   if (ai->kind == ABI_ARG_DIRECT &&
       x64_direct_to_stack(a->abi, ai, a->next_param_int, a->next_param_fp)) {
     /* Whole arg on the stack. */
     for (i = 0; i < ai->nparts; ++i) {
       const ABIArgPart* part = &ai->parts[i];
       NativeAllocClass cls =
           part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
       NativeLoc isrc = x64_incoming_stack_loc(
           p->type, cls, incoming_bias + (i32)a->next_param_stack);
       a->next_param_stack += 8u;
       if (dst.kind == NATIVE_LOC_NONE) {
         /* unused */
       } else if (to_reg) {
         /* Defer: a register dst may be another param's incoming reg. */
         x64_defer_reg_bind(
             a,
             native_loc_reg(dst.type ? dst.type : p->type,
                            (NativeAllocClass)dst.cls, (Reg)dst.v.reg),
             isrc, part->size);
       } else {
         /* Frame dst: load to scratch then store (memory dst is never a cycle
          * source, so emit eagerly — it only reads the incoming slot). */
         Reg tmp = cls == NATIVE_REG_FP ? X64_TMP_FP : X64_TMP_INT;
         NativeLoc tloc = native_loc_reg(p->type, cls, tmp);
         x64_load_part(t, tloc, isrc, 0, part->size);
         x64_store_part(
             t, native_loc_stack(p->type, dst.v.frame, (i32)part->src_offset),
             tloc, 0, part->size);
       }
     }
     return;
   }
 
   for (i = 0; i < ai->nparts; ++i) {
     const ABIArgPart* part = &ai->parts[i];
     NativeAllocClass cls =
         part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
     NativeLoc
         src; /* incoming: arg register, or NATIVE_LOC_ADDR for a stack arg */
     if (cls == NATIVE_REG_FP && a->next_param_fp < a->abi->n_fp_args) {
       src = native_loc_reg(p->type, cls, (Reg)(X64_XMM0 + a->next_param_fp++));
     } else if (cls == NATIVE_REG_INT &&
                a->next_param_int < a->abi->n_int_args) {
       src = native_loc_reg(p->type, cls, a->abi->int_args[a->next_param_int++]);
     } else {
       src = x64_incoming_stack_loc(p->type, cls,
                                    incoming_bias + (i32)a->next_param_stack);
       a->next_param_stack += 8u;
     }
     x64_sync_slot(a->abi, &a->next_param_int, &a->next_param_fp);
     if (dst.kind == NATIVE_LOC_NONE) {
       /* unused parameter; cursors advanced */
     } else if (to_reg) {
       /* Defer the register bind: the allocator may rotate params across the
        * incoming arg registers, so a per-param move could clobber a register
        * another bind still needs. x64_bind_params_end resolves them together as
        * a parallel copy. */
       x64_defer_reg_bind(
           a,
           native_loc_reg(dst.type ? dst.type : p->type,
                          (NativeAllocClass)dst.cls, (Reg)dst.v.reg),
           src, part->size);
     } else if (src.kind == NATIVE_LOC_REG) {
       x64_store_part(
           t, native_loc_stack(p->type, dst.v.frame, (i32)part->src_offset), src,
           0, part->size);
     } else {
       /* Stack source -> frame dst: load to scratch, then store. */
       Reg tmp = cls == NATIVE_REG_FP ? X64_TMP_FP : X64_TMP_INT;
       NativeLoc tloc = native_loc_reg(p->type, cls, tmp);
       x64_load_part(t, tloc, src, 0, part->size);
       x64_store_part(
           t, native_loc_stack(p->type, dst.v.frame, (i32)part->src_offset),
           tloc, 0, part->size);
     }
   }
 }
 
 /* Flush the deferred register-destination param binds as a parallel copy (the
  * shared scheduler breaks any cycle the allocator's rotation created through
  * backend-private int/fp cycle temporaries). Frame-dst and indirect binds were emitted eagerly
  * in bind_param — they only read incoming registers, so they precede this. */
 static void x64_bind_params_end(NativeTarget* t) {
   X64NativeTarget* a = x64_of(t);
   /* No callee is staged during entry binds, so r11 is free as the cycle
    * scratch. */
   if (a->nbind_moves)
     x64_emit_reg_arg_moves(t, a->bind_moves, a->nbind_moves, X64_TMP_INT2);
   a->nbind_moves = 0;
 }
 
 /* ============================ calls / returns ============================ */
 
 typedef NativeArgMove X64ArgMove;
 
 static void x64_emit_one_arg_move(NativeTarget* t, const NativeArgMove* m) {
   if (m->is_addr) {
     x64_addr_of_loc(t, m->dst, m->src);
   } else {
     x64_load_part(t, m->dst, m->src, m->src_offset, m->size);
   }
   if (m->dup_to_gpr) {
     /* movq gpr, xmm: 66 REX.W 0F 7E /r (xmm in reg field). */
     emit_sse_rr_w(t->mc, 0x66, 0x7E, 1, loc_reg(m->dst), m->dup_gpr);
   }
 }
 
 /* Parallel-copy register arg moves via the shared scheduler. `int_scratch` is
  * the register used to break an integer cycle: normally r11, but rax when an
  * indirect callee is staged in r11 (rax is never a SysV int arg register and
  * the variadic AL count is written only after the moves). */
 static void x64_emit_reg_arg_moves(NativeTarget* t, NativeArgMove* moves, u32 n,
                                    Reg int_scratch) {
   NativeArgShuffle s;
   if (n > X64_MAX_REG_ARG_MOVES) x64_panic(x64_of(t), "too many register args");
   memset(&s, 0, sizeof s);
   s.t = t;
   s.emit_one = x64_emit_one_arg_move;
   s.reg_move = x64_move;
   s.scratch[NATIVE_REG_INT] = int_scratch;
   s.scratch[NATIVE_REG_FP] = X64_TMP_FP;
   s.scratch_class_mask = (1u << NATIVE_REG_INT) | (1u << NATIVE_REG_FP);
   native_arg_shuffle(&s, moves, n);
 }
 
 static void x64_marshal_call(NativeTarget* t, const NativeCallDesc* desc,
                              NativeCallPhase* plan) {
   X64NativeTarget* a = x64_of(t);
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, desc->fn_type);
   const X64ABIRegs* aregs = a->abi ? a->abi : x64_abi_for_os(t->c->target.os);
   NativeCallPhaseRet* rets;
   X64CallMarshalCtx call;
   KitCgTypeId i64t = builtin_id(KIT_CG_BUILTIN_I64);
   int tail = (desc->flags & CG_CALL_TAIL) != 0;
   /* Right-size the result scratch to the exact number of entries the ret loops
    * below write: nparts on a DIRECT register return, 1 on the !abi fallback,
    * 0 (NULL) otherwise (IGNORE / sret / no results). */
   u32 nrets_cap = (abi && abi->ret.kind == ABI_ARG_DIRECT && desc->nresults)
                       ? abi->ret.nparts
                       : ((!abi && desc->nresults) ? 1u : 0u);
   memset(plan, 0, sizeof *plan);
   memset(&call, 0, sizeof call);
   call.tail = (u8)tail;
   call.cycle_scratch = X64_TMP_INT2;
   rets = nrets_cap ? arena_zarray(t->c->tu, NativeCallPhaseRet, nrets_cap) : NULL;
   plan->callee = desc->callee;
   plan->rets = rets;
   plan->flags = desc->flags;
   plan->has_sret = abi && abi->has_sret;
   plan->is_variadic = abi && abi->variadic;
   plan->stack_arg_size = x64_call_stack_bytes(t, desc);
   if (plan->stack_arg_size > a->frame.max_outgoing)
     a->frame.max_outgoing = plan->stack_arg_size;
   /* Argument transport owns r10 (part data), r11 (address stabilization), and
    * rax (parallel-copy cycle breaking). A register-indirect callee therefore
    * lives in a balanced stack home for the complete marshal phase, independent
    * of its producer register and every allowed NativeLoc argument shape. */
   if (plan->callee.kind == NATIVE_LOC_REG) {
     if ((NativeAllocClass)plan->callee.cls != NATIVE_REG_INT)
       x64_panic(a, "indirect callee is not in an integer register");
     x64_push_reg(t->mc, (Reg)loc_reg(plan->callee));
     call.rsp_bias = 8u;
     call.callee_parked = 1u;
     call.cycle_scratch = X64_RAX;
     plan->callee = native_loc_reg(plan->callee.type, NATIVE_REG_INT, X64_R11);
   }
   {
     u32 next_int = (abi && abi->has_sret) ? 1u : 0u;
     u32 next_fp = 0, stack = aregs->shadow_space, nmoves = 0, i;
     u32 tail_payload = tail ? x64_call_stack_raw_size(t, desc) : 0u;
     u32 p;
     X64ArgMove moves[X64_MAX_REG_ARG_MOVES];
     x64_sync_slot(aregs, &next_int, &next_fp);
     for (i = 0; i < desc->nargs; ++i) {
       ABIArgInfo tmp;
       const ABIArgInfo* ai = x64_param_abi(t, abi, desc, i, &tmp);
       int variadic_arg = abi && i >= abi->nparams;
       if (ai->kind == ABI_ARG_IGNORE) continue;
       if (ai->kind == ABI_ARG_INDIRECT) {
         NativeLoc src = desc->args[i];
         if (tail) {
           AggregateAccess access;
           NativeAddr from, to;
           u32 align = ai->indirect_align;
           u32 size = native_type_size(t, src.type);
           if (!align) align = native_type_align(t, src.type);
           tail_payload = align_up_u32(tail_payload, align ? align : 1u);
           src = x64_incoming_stack_loc(
               src.type, NATIVE_REG_INT, (i32)(16u + tail_payload));
           from = x64_storage_addr(a, desc->args[i], 0);
           to = x64_storage_addr(a, src, 0);
           memset(&access, 0, sizeof access);
           access.type = src.type;
           access.size = size;
           access.align = align ? align : 1u;
           x64_copy_bytes(t, to, from, access);
           tail_payload += size;
         }
         if (next_int < aregs->n_int_args) {
           X64ArgMove* m = &moves[nmoves++];
           memset(m, 0, sizeof *m);
           m->dst =
               native_loc_reg(i64t, NATIVE_REG_INT, aregs->int_args[next_int++]);
           m->src = src;
           m->size = 8;
           m->is_addr = 1;
         } else {
           NativeLoc ptr = native_loc_reg(i64t, NATIVE_REG_INT, X64_RAX);
           x64_addr_of_loc(t, ptr, src);
           x64_store_outgoing_part(t, &call, stack, ptr, 8);
           stack += 8u;
         }
         x64_sync_slot(aregs, &next_int, &next_fp);
         continue;
       }
       if (ai->kind == ABI_ARG_DIRECT &&
           x64_direct_to_stack(aregs, ai, next_int, next_fp)) {
         for (p = 0; p < ai->nparts; ++p) {
           const ABIArgPart* part = &ai->parts[p];
           x64_copy_outgoing_part(t, &call, stack, desc->args[i], part);
           stack += 8u;
         }
         continue;
       }
       for (p = 0; p < ai->nparts; ++p) {
         const ABIArgPart* part = &ai->parts[p];
         NativeAllocClass cls =
             part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
         if (cls == NATIVE_REG_FP && next_fp < aregs->n_fp_args) {
           X64ArgMove* m = &moves[nmoves++];
           u32 slot = next_fp;
           memset(m, 0, sizeof *m);
           m->dst = native_loc_reg(desc->args[i].type, cls,
                                   (Reg)(X64_XMM0 + next_fp++));
           m->src = desc->args[i];
           m->src_offset = part->src_offset;
           m->size = part->size;
           if (aregs->vararg_fp_dup_to_gpr && variadic_arg &&
               slot < aregs->n_int_args) {
             m->dup_to_gpr = 1;
             m->dup_gpr = aregs->int_args[slot];
           }
           x64_sync_slot(aregs, &next_int, &next_fp);
         } else if (cls == NATIVE_REG_INT && next_int < aregs->n_int_args) {
           X64ArgMove* m = &moves[nmoves++];
           memset(m, 0, sizeof *m);
           m->dst = native_loc_reg(desc->args[i].type, cls,
                                   aregs->int_args[next_int++]);
           m->src = desc->args[i];
           m->src_offset = part->src_offset;
           m->size = part->size;
           x64_sync_slot(aregs, &next_int, &next_fp);
         } else {
           x64_copy_outgoing_part(t, &call, stack, desc->args[i], part);
           stack += 8u;
           x64_sync_slot(aregs, &next_int, &next_fp);
         }
       }
     }
     /* If an indirect callee was staged in r11 above, the cycle scratch must
      * avoid it; rax is free here (not an int arg reg; AL count comes later). */
     x64_emit_reg_arg_moves(t, moves, nmoves, call.cycle_scratch);
     if (abi && abi->has_sret) {
       /* sret pointer in the first int-arg reg. A tail call forwards the
        * caller's own incoming sret pointer (spilled at entry); otherwise pass
        * the address of this call's result slot. */
       NativeLoc sret = native_loc_reg(i64t, NATIVE_REG_INT, aregs->int_args[0]);
       if (tail)
         x64_load_part(t, sret, native_loc_stack(i64t, a->sret_ptr_slot, 0), 0,
                       8);
       else if (desc->nresults)
         x64_addr_of_loc(t, sret, desc->results[0]);
     }
     /* Variadic call: AL = number of vector regs used. */
     if (abi && abi->variadic)
       x64_emit_load_imm(t->mc, 0, X64_RAX, (i64)next_fp);
     if (call.callee_parked) {
       x64_pop_reg(t->mc, X64_R11);
       call.rsp_bias = 0u;
       call.callee_parked = 0u;
     }
   }
   /* Return value receipt. */
   if (abi && abi->ret.kind == ABI_ARG_DIRECT && desc->nresults) {
     u32 nr = 0, ni = 0, nf = 0;
     static const u32 ret_int_regs[2] = {X64_RAX, X64_RDX};
     u32 p;
     for (p = 0; p < abi->ret.nparts; ++p) {
       const ABIArgPart* part = &abi->ret.parts[p];
       NativeAllocClass cls =
           part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
       KitCgTypeId pty = x64_part_scalar_type(part);
       Reg rreg = cls == NATIVE_REG_FP ? (Reg)(X64_XMM0 + nf++)
                                       : (Reg)ret_int_regs[ni++];
       rets[nr].src = native_loc_reg(pty, cls, rreg);
       rets[nr].dst = desc->results[0];
       if (rets[nr].dst.kind == NATIVE_LOC_FRAME)
         rets[nr].dst = native_loc_stack(pty, desc->results[0].v.frame,
                                         (i32)part->src_offset);
       else if (rets[nr].dst.kind == NATIVE_LOC_STACK) {
         rets[nr].dst.v.stack.offset += (i32)part->src_offset;
         rets[nr].dst.type = pty;
       }
       rets[nr].mem = native_mem_for_type(t, pty, part->size);
       nr++;
     }
     plan->nrets = nr;
   } else if (abi && abi->ret.kind == ABI_ARG_IGNORE) {
     plan->nrets = 0;
   } else if (!abi && desc->nresults) {
     rets[0].src =
         native_loc_reg(desc->results[0].type, NATIVE_REG_INT, X64_RAX);
     rets[0].dst = desc->results[0];
     rets[0].mem = native_mem_for_type(t, desc->results[0].type, 0);
     plan->nrets = 1;
   }
 }
 
 /* Emit a sibling (tail) call: tear the frame down and jump (no call) to the
  * callee. Outgoing args are already in arg regs / the caller's incoming-arg
  * window. `leave` (mov rsp,rbp; pop rbp) restores the caller's rbp and leaves
  * rsp at the return address — frame_size-independent, so no func_end patch. */
 static void x64_emit_tail_site(NativeTarget* t, NativeLoc callee) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   ObjSecId sec = mc->section_id;
   /* Restore callee-saves before the frame teardown (O1 path; none at -O0).
    * Their rbp-relative offsets are frame-size-independent, and the indirect
    * callee was staged in r11 by marshal_call — a caller-saved scratch — so these
    * restores never clobber it. Mirrors the x64_func_end epilogue. */
   x64_emit_callee_restores(a);
   emit_leave(mc);
   if (callee.kind == NATIVE_LOC_GLOBAL) {
     u8 op = X64_OPC_JMP_REL32;
     u32 disp_pos;
     mc_emit_bytes(mc, &op, 1);
     disp_pos = mc_pos(mc);
     emit_u32le(mc, 0);
     mc_emit_reloc_at(mc, sec, disp_pos, R_X64_PLT32, callee.v.global.sym,
                       callee.v.global.addend - 4, 1, 0);
   } else if (callee.kind == NATIVE_LOC_REG) {
     /* indirect callee was staged in r11 by marshal_call */
     u32 r = loc_reg(callee);
     x64_emit_indirect_rm(mc, r, 4u); /* jmp r/m, /4 */
   } else {
     x64_panic(a, "unsupported tail call target");
   }
 }
 
 static void x64_emit_call(NativeTarget* t, const NativeCallPhase* plan) {
   MCEmitter* mc = t->mc;
   ObjSecId sec = mc->section_id;
   if (plan->flags & CG_CALL_TAIL) {
     x64_emit_tail_site(t, plan->callee);
     return;
   }
   if (plan->callee.kind == NATIVE_LOC_GLOBAL) {
     u8 op = X64_OPC_CALL_REL32;
     u32 disp_pos;
     mc_emit_bytes(mc, &op, 1);
     disp_pos = mc_pos(mc);
     emit_u32le(mc, 0);
     mc_emit_reloc_at(mc, sec, disp_pos, R_X64_PLT32, plan->callee.v.global.sym,
                       plan->callee.v.global.addend - 4, 1, 0);
     return;
   }
   if (plan->callee.kind == NATIVE_LOC_REG) {
     u32 r = loc_reg(plan->callee);
     x64_emit_indirect_rm(mc, r, 2u); /* call r/m, /2 */
     return;
   }
   x64_panic(x64_of(t), "unsupported call target");
 }
 
 static void x64_marshal_ret(NativeTarget* t, const CGFuncDesc* fd,
                             const NativeLoc* value,
                             NativeCallPhaseRet** out_rets, u32* out_nrets) {
   X64NativeTarget* a = x64_of(t);
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, fd->fn_type);
   NativeCallPhaseRet* rets = NULL;
   u32 nr = 0;
   if (value) rets = arena_zarray(t->c->tu, NativeCallPhaseRet, 4);
   if (value && abi && abi->ret.kind == ABI_ARG_INDIRECT) {
     /* sret: reload the destination pointer (spilled at entry) into rax and
      * copy the source aggregate into [rax]. Keep it out of R10/R11: those are
      * x64_copy_bytes' private transfer/address bank, and resolving a stack
      * source through R11 would otherwise overwrite the destination. RAX is
      * also the ABI's sret return register, so it can stay live through the
      * copy and needs no reload afterwards. */
     KitCgTypeId i64t = builtin_id(KIT_CG_BUILTIN_I64);
     NativeLoc dstp = native_loc_reg(i64t, NATIVE_REG_INT, X64_RAX);
     NativeLoc saved = native_loc_stack(i64t, a->sret_ptr_slot, 0);
     NativeAddr dst_addr, src_addr;
     AggregateAccess access;
     x64_load_part(t, dstp, saved, 0, 8);
     memset(&dst_addr, 0, sizeof dst_addr);
     dst_addr.base_kind = NATIVE_ADDR_BASE_REG;
     dst_addr.base.reg = X64_RAX;
     dst_addr.base_type = value->type;
     src_addr = x64_storage_addr(a, *value, 0);
     src_addr.base_type = value->type;
     memset(&access, 0, sizeof access);
     access.type = value->type;
     access.size = (u32)cg_type_size(t->c, value->type);
     access.align = native_type_align(t, value->type);
     x64_copy_bytes(t, dst_addr, src_addr, access);
     *out_rets = NULL;
     *out_nrets = 0;
     return;
   }
   if (value && abi && abi->ret.kind == ABI_ARG_DIRECT) {
     u32 ni = 0, nf = 0;
     static const u32 ret_int_regs[2] = {X64_RAX, X64_RDX};
     u32 p;
     int exact_parts = 0;
     for (p = 0; p < abi->ret.nparts; ++p)
       if (abi->ret.parts[p].cls == ABI_CLASS_INT &&
           !x64_native_part_width(abi->ret.parts[p].size))
         exact_parts = 1;
 
     /* Generic write_phase materializes a register destination from its scalar
      * type, so an I64 carrier for a 3/5/6/7-byte aggregate part would widen the
      * source load to eight bytes. Once one exact part is present, marshal every
      * return part here in ABI order: this preserves already-filled return
      * registers and keeps each memory read bounded by ABIArgPart.size. */
     if (exact_parts) {
       for (p = 0; p < abi->ret.nparts; ++p) {
         const ABIArgPart* part = &abi->ret.parts[p];
         NativeAllocClass cls =
             part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
         KitCgTypeId pty = x64_part_scalar_type(part);
         Reg rreg = cls == NATIVE_REG_FP ? (Reg)(X64_XMM0 + nf++)
                                         : (Reg)ret_int_regs[ni++];
         NativeLoc dst = native_loc_reg(pty, cls, rreg);
         x64_load_part(t, dst, *value, part->src_offset, part->size);
       }
       *out_rets = NULL;
       *out_nrets = 0;
       return;
     }
 
     for (p = 0; p < abi->ret.nparts; ++p) {
       const ABIArgPart* part = &abi->ret.parts[p];
       NativeAllocClass cls =
           part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
       KitCgTypeId pty = x64_part_scalar_type(part);
       Reg rreg = cls == NATIVE_REG_FP ? (Reg)(X64_XMM0 + nf++)
                                       : (Reg)ret_int_regs[ni++];
       rets[nr].src = *value;
       if (rets[nr].src.kind == NATIVE_LOC_FRAME)
         rets[nr].src =
             native_loc_stack(pty, value->v.frame, (i32)part->src_offset);
       else if (rets[nr].src.kind == NATIVE_LOC_STACK) {
         rets[nr].src.v.stack.offset += (i32)part->src_offset;
         rets[nr].src.type = pty;
       }
       rets[nr].dst = native_loc_reg(pty, cls, rreg);
       rets[nr].mem = native_mem_for_type(t, pty, part->size);
       nr++;
     }
   } else if (value) {
     rets[0].src = *value;
     rets[0].dst = native_loc_reg(value->type, NATIVE_REG_INT, X64_RAX);
     rets[0].mem = native_mem_for_type(t, value->type, 0);
     nr = 1;
   }
   *out_rets = rets;
   *out_nrets = nr;
 }
 
 static void x64_ret(NativeTarget* t) {
   X64NativeTarget* a = x64_of(t);
   x64_jump(t, a->epilogue_label);
 }
 
 /* ============================ alloca ============================ */
 
 static void x64_alloca(NativeTarget* t, NativeLoc dst, NativeLoc size,
                        u32 align) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   u32 rsz = loc_reg(size);
   u32 rd = loc_reg(dst);
   u32 al = align ? align : 16u;
   if (al < 16u) al = 16u;
   if (al > 16u) x64_panic(a, "alloca align > 16 not supported");
   if (size.kind == NATIVE_LOC_IMM) {
     u64 aligned = ((u64)size.v.imm + 15u) & ~(u64)15u;
     if (aligned == 0) aligned = 16;
     /* sub rsp, imm32. */
     emit_rex(mc, 1, 0, 0, X64_RSP);
     {
       u8 buf[2] = {X64_OPC_ALU_IMM32, modrm(3u, X64_ALU_SUB_SUB, X64_RSP)};
       mc_emit_bytes(mc, buf, 2);
     }
     emit_u32le(mc, (u32)aligned);
   } else {
     /* r10 = (size + 15) & ~15; sub rsp, r10. */
     emit_lea(mc, X64_TMP_INT, rsz, 15);
     emit_rex(mc, 1, 0, 0, X64_TMP_INT);
     {
       u8 buf[3] = {X64_OPC_ALU_IMM8,
                    modrm(3u, X64_ALU_SUB_AND, X64_TMP_INT), 0xF0};
       mc_emit_bytes(mc, buf, 3);
     }
     emit_alu_rr(mc, 1, X64_OPC_ALU_SUB, X64_RSP, X64_TMP_INT);
   }
   a->frame.has_alloca = 1;
   /* lea dst, [rsp + max_outgoing] — disp32 patched in func_end. */
   if (a->npatches == a->patches_cap) {
     u32 cap = a->patches_cap ? a->patches_cap * 2u : 8u;
     X64Patch* nb = arena_zarray(t->c->tu, X64Patch, cap);
     if (a->patches) memcpy(nb, a->patches, sizeof(*nb) * a->npatches);
     a->patches = nb;
     a->patches_cap = cap;
   }
   emit_rex(mc, 1, rd, 0, X64_RSP);
   {
     u8 op = X64_OPC_LEA;
     mc_emit_bytes(mc, &op, 1);
   }
   {
     u8 mr = modrm(2u, rd & 7u, 4u);
     mc_emit_bytes(mc, &mr, 1);
   }
   {
     u8 s = sib(0u, 4u, X64_RSP);
     mc_emit_bytes(mc, &s, 1);
   }
   a->patches[a->npatches].kind = X64_PATCH_ALLOCA;
   a->patches[a->npatches].pos = mc_pos(mc);
   a->npatches++;
   a->nalloca++;
   emit_u32le(mc, 0); /* placeholder disp32 */
 }
 
 /* ============================ TLS ============================ */
 
 /* Win64 TLS Local-Exec (PE-COFF): TEB pointer -> _tls_index -> TLS block ->
  * lea &sym@SECREL. R11 is scratch. */
 static void x64_tls_addr_of_win64(NativeTarget* t, NativeLoc dst, ObjSymId sym,
                                   i64 addend) {
   MCEmitter* mc = t->mc;
   u32 sec = mc->section_id;
   u32 rd = loc_reg(dst);
   /* (1) mov rd, gs:[0x58]. */
   {
     u8 gs = 0x65;
     mc_emit_bytes(mc, &gs, 1);
     emit_rex(mc, 1, rd, 0, 0);
     {
       u8 op = X64_OPC_MOV_R_RM;
       mc_emit_bytes(mc, &op, 1);
     }
     {
       u8 mr = modrm(0u, rd & 7u, 4u);
       mc_emit_bytes(mc, &mr, 1);
     }
     {
       u8 s = sib(0u, 4u, 5u);
       mc_emit_bytes(mc, &s, 1);
     }
     emit_u32le(mc, 0x58u);
   }
   /* (2) mov r11d, [rip + _tls_index]. */
   {
     Sym idx_name = pool_intern_slice(t->c->global, SLICE_LIT("_tls_index"));
     ObjSymId idx_sym = obj_symbol_find(t->obj, idx_name);
     u8 rex_r, op, mr;
     u32 disp_pos;
     if (idx_sym == 0)
       idx_sym =
           obj_symbol(t->obj, idx_name, SB_GLOBAL, SK_UNDEF, OBJ_SEC_NONE, 0, 0);
     rex_r = X64_REX_BASE | X64_REX_R;
     mc_emit_bytes(mc, &rex_r, 1);
     op = X64_OPC_MOV_R_RM;
     mc_emit_bytes(mc, &op, 1);
     mr = modrm(0u, 3u, 5u); /* r11&7, rip-rel */
     mc_emit_bytes(mc, &mr, 1);
     disp_pos = mc_pos(mc);
     emit_u32le(mc, 0);
     mc_emit_reloc_at(mc, sec, disp_pos, R_PC32, idx_sym, -4, 1, 0);
   }
   /* (3) mov rd, [rd + r11*8]. */
   {
     u8 rex = X64_REX_BASE | X64_REX_W | X64_REX_X;
     u8 op;
     if (rd & 8u) rex |= X64_REX_R | X64_REX_B;
     mc_emit_bytes(mc, &rex, 1);
     op = X64_OPC_MOV_R_RM;
     mc_emit_bytes(mc, &op, 1);
     if ((rd & 7u) == 5u) {
       u8 mr = modrm(1u, rd & 7u, 4u);
       u8 s = sib(3u, 3u, rd & 7u);
       u8 zero = 0;
       mc_emit_bytes(mc, &mr, 1);
       mc_emit_bytes(mc, &s, 1);
       mc_emit_bytes(mc, &zero, 1);
     } else {
       u8 mr = modrm(0u, rd & 7u, 4u);
       u8 s = sib(3u, 3u, rd & 7u);
       mc_emit_bytes(mc, &mr, 1);
       mc_emit_bytes(mc, &s, 1);
     }
   }
   /* (4) lea rd, [rd + sym@SECREL]. */
   {
     u8 rex = X64_REX_BASE | X64_REX_W;
     u8 op;
     u32 disp_pos;
     if (rd & 8u) rex |= X64_REX_R | X64_REX_B;
     mc_emit_bytes(mc, &rex, 1);
     op = X64_OPC_LEA;
     mc_emit_bytes(mc, &op, 1);
     if ((rd & 7u) == 4u) {
       u8 mr = modrm(2u, rd & 7u, 4u);
       u8 s = sib(0u, 4u, rd & 7u);
       mc_emit_bytes(mc, &mr, 1);
       mc_emit_bytes(mc, &s, 1);
     } else {
       u8 mr = modrm(2u, rd & 7u, rd & 7u);
       mc_emit_bytes(mc, &mr, 1);
     }
     disp_pos = mc_pos(mc);
     emit_u32le(mc, 0);
     mc_emit_reloc_at(mc, sec, disp_pos, R_COFF_SECREL, sym, addend, 1, 0);
   }
 }
 
 /* x86-64 Mach-O TLV (descriptor model): load the __thread_ptrs slot holding
  * the TLV descriptor address into %rdi, then call the descriptor's resolver
  * thunk (descriptor[0]); it returns the variable's storage address in %rax.
  * Peer of arm64's aa_tls_addr_of descriptor path. The slot is addressed by an
  * R_X64_TLV reloc (RIP-relative disp32) that the Mach-O linker routes through
  * __thread_ptrs. The Apple TLV thunk preserves every register except %rax, so
  * the only clobbers are %rdi (we load it) and %rax (the result) — modeled in
  * x64_machine_op_clobbers so no value lives across the access in them. Any
  * addend is a byte offset into the variable, applied to %rax after the call. */
 static void x64_tls_addr_of_macho(NativeTarget* t, NativeLoc dst, ObjSymId sym,
                                   i64 addend) {
   MCEmitter* mc = t->mc;
   u32 sec = mc->section_id;
   u32 rd = loc_reg(dst);
   u32 disp_pos;
   /* movq sym@TLVP(%rip), %rdi  (48 8B 3D disp32). */
   emit_rex(mc, 1, X64_RDI, 0, 0);
   {
     u8 op = X64_OPC_MOV_R_RM;
     mc_emit_bytes(mc, &op, 1);
   }
   {
     u8 mr = modrm(0u, X64_RDI & 7u, 5u); /* [rip + disp32] */
     mc_emit_bytes(mc, &mr, 1);
   }
   disp_pos = mc_pos(mc);
   emit_u32le(mc, 0);
   mc_emit_reloc_at(mc, sec, disp_pos, R_X64_TLV, sym, -4, 1, 0);
   /* callq *(%rdi)  (FF /2, mem [rdi]) -> %rax = &storage. */
   {
     u8 call[2] = {0xFFu, modrm(0u, 2u, X64_RDI & 7u)};
     mc_emit_bytes(mc, call, 2);
   }
   /* %rax += addend (byte offset into the variable). */
   if (addend) {
     i32 a = (i32)addend;
     if (a == addend) {
       emit_rex(mc, 1, 0, 0, X64_RAX);
       if (imm_fits_i8(a)) {
         u8 buf[3] = {X64_OPC_ALU_IMM8, modrm(3u, X64_ALU_SUB_ADD, X64_RAX & 7u),
                      (u8)a};
         mc_emit_bytes(mc, buf, 3);
       } else {
         u8 buf[2] = {X64_OPC_ALU_IMM32,
                      modrm(3u, X64_ALU_SUB_ADD, X64_RAX & 7u)};
         mc_emit_bytes(mc, buf, 2);
         emit_u32le(mc, (u32)a);
       }
     } else {
       /* Wide addend: materialize in r11 then add. */
       x64_emit_load_imm(mc, 1, X64_R11, addend);
       emit_alu_rr(mc, 1, X64_OPC_ALU_ADD, X64_RAX, X64_R11);
     }
   }
   if (rd != X64_RAX) emit_mov_rr(mc, 1, rd, X64_RAX);
 }
 
 /* x86-64 TLS Local-Exec: mov rd, fs:0; lea rd, [rd + sym@tpoff]. */
 static void x64_tls_addr_of(NativeTarget* t, NativeLoc dst, ObjSymId sym,
                             i64 addend) {
   MCEmitter* mc = t->mc;
   u32 sec = mc->section_id;
   u32 rd = loc_reg(dst);
   u32 disp_pos;
   if (obj_format_tls_via_descriptor(t->c)) {
     x64_tls_addr_of_macho(t, dst, sym, addend);
     return;
   }
   if (obj_format_tls_model(t->c) == OBJ_TLS_WINDOWS_TEB) {
     x64_tls_addr_of_win64(t, dst, sym, addend);
     return;
   }
   /* mov rd, fs:[0]. */
   {
     u8 fs = 0x64;
     mc_emit_bytes(mc, &fs, 1);
     emit_rex(mc, 1, rd, 0, 0);
     {
       u8 op = X64_OPC_MOV_R_RM;
       mc_emit_bytes(mc, &op, 1);
     }
     {
       u8 mr = modrm(0u, rd & 7u, 4u);
       mc_emit_bytes(mc, &mr, 1);
     }
     {
       u8 s = sib(0u, 4u, 5u);
       mc_emit_bytes(mc, &s, 1);
     }
     emit_u32le(mc, 0);
   }
   /* lea rd, [rd + disp32@tpoff]. */
   emit_rex(mc, 1, rd, 0, rd);
   {
     u8 op = X64_OPC_LEA;
     mc_emit_bytes(mc, &op, 1);
   }
   if ((rd & 7u) == 4u) {
     u8 mr = modrm(2u, rd & 7u, 4u);
     u8 s = sib(0u, 4u, rd & 7u);
     mc_emit_bytes(mc, &mr, 1);
     mc_emit_bytes(mc, &s, 1);
   } else {
     u8 mr = modrm(2u, rd & 7u, rd & 7u);
     mc_emit_bytes(mc, &mr, 1);
   }
   disp_pos = mc_pos(mc);
   emit_u32le(mc, 0);
   mc_emit_reloc_at(mc, sec, disp_pos, R_X64_TPOFF32, sym, addend, 0, 0);
 }
 
 /* ============================ atomics ============================ */
 
 static void emit_lock_prefix(MCEmitter* mc) {
   u8 b = 0xF0;
   mc_emit_bytes(mc, &b, 1);
 }
 static void emit_mfence(MCEmitter* mc) {
   u8 b[3] = {0x0F, 0xAE, 0xF0};
   mc_emit_bytes(mc, b, 3);
 }
 
 /* Resolve an atomic addr to a bare base register (r11) + disp 0. */
 static u32 x64_atomic_base(X64NativeTarget* a, NativeAddr addr) {
   return x64_addr_to_base_reg(a, addr, X64_TMP_INT2);
 }
 
 static void x64_atomic_load(NativeTarget* t, NativeLoc dst, NativeAddr addr,
                             MemAccess mem, KitCgMemOrder mo) {
   X64NativeTarget* a = x64_of(t);
   u32 sz = mem.size ? mem.size : loc_size32(t, dst);
   u32 base;
   (void)mo; /* x86 plain MOV is an acquire load. */
   base = x64_atomic_base(a, addr);
   emit_mov_load(t->mc, sz, 0, loc_reg(dst), base, 0);
 }
 
 static void x64_atomic_store(NativeTarget* t, NativeAddr addr, NativeLoc src,
                              MemAccess mem, KitCgMemOrder mo) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   u32 sz = mem.size ? mem.size : loc_size32(t, src);
   int w = sz == 8u ? 1 : 0;
   u32 base = x64_atomic_base(a, addr);
   u32 sr = loc_reg(src);
   if (mo == KIT_CG_MO_SEQ_CST) {
     /* XCHG writes the prior memory value into its register operand. Stage the
      * semantic source in backend-private R10 so that mutation is invisible to
      * the allocator; R11 holds a materialized address. */
     if (base == X64_TMP_INT) {
       emit_mov_rr(mc, 1, X64_TMP_INT2, base);
       base = X64_TMP_INT2;
     }
     if (sr != X64_TMP_INT) emit_mov_rr(mc, w, X64_TMP_INT, sr);
     emit_lock_prefix(mc);
     emit_rex(mc, w, X64_TMP_INT, 0, base);
     {
       u8 op = 0x87; /* xchg r/m, r */
       mc_emit_bytes(mc, &op, 1);
     }
     emit_mem_operand(mc, X64_TMP_INT, base, 0);
     return;
   }
   emit_mov_store(mc, sz, sr, base, 0);
 }
 
 static void x64_atomic_rmw(NativeTarget* t, KitCgAtomicOp op, NativeLoc dst,
                            NativeAddr addr, NativeLoc val, MemAccess mem,
                            KitCgMemOrder mo) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   u32 sz = mem.size ? mem.size : loc_size32(t, dst);
   int w = sz == 8u ? 1 : 0;
   u32 base = x64_atomic_base(a, addr);
   u32 dr = loc_reg(dst);
   u32 vr = loc_reg(val);
   (void)mo; /* LOCK ops are full barriers. */
   /* The rmw uses fixed rax (prior), rcx (new), rdx (val); the optimizer may
    * have materialized the address into one of them, so keep it out (R11 is
    * backend-private, never an allocated operand). Stage before RDX is loaded.
    */
   if (base == X64_RAX || base == X64_RCX || base == X64_RDX) {
     emit_mov_rr(mc, 1, X64_TMP_INT2, base);
     base = X64_TMP_INT2;
   }
   /* val staged in rdx (rax/rcx used by the cmpxchg loop). */
   emit_mov_rr(mc, w, X64_RDX, vr);
   if (op == KIT_CG_ATOMIC_ADD || op == KIT_CG_ATOMIC_SUB) {
     if (op == KIT_CG_ATOMIC_SUB) emit_f7_rm(mc, w, X64_F7_SUB_NEG, X64_RDX);
     emit_lock_prefix(mc);
     emit_rex(mc, w, X64_RDX, 0, base);
     {
       u8 op2[2] = {X64_OPC_TWOBYTE, 0xC1}; /* xadd */
       mc_emit_bytes(mc, op2, 2);
     }
     emit_mem_operand(mc, X64_RDX, base, 0);
     if (dr != X64_RDX) emit_mov_rr(mc, w, dr, X64_RDX);
     return;
   }
   if (op == KIT_CG_ATOMIC_XCHG) {
     emit_lock_prefix(mc);
     emit_rex(mc, w, X64_RDX, 0, base);
     {
       u8 op2 = 0x87; /* xchg */
       mc_emit_bytes(mc, &op2, 1);
     }
     emit_mem_operand(mc, X64_RDX, base, 0);
     if (dr != X64_RDX) emit_mov_rr(mc, w, dr, X64_RDX);
     return;
   }
   /* AND/OR/XOR/NAND: cmpxchg retry loop. rax=prior, rcx=new, rdx=val. */
   {
     MCLabel retry = mc_label_new(mc);
     emit_mov_load(mc, sz, 0, X64_RAX, base, 0);
     mc_label_place(mc, retry);
     emit_mov_rr(mc, w, X64_RCX, X64_RAX);
     switch (op) {
       case KIT_CG_ATOMIC_AND:
         emit_alu_rr(mc, w, X64_OPC_ALU_AND, X64_RCX, X64_RDX);
         break;
       case KIT_CG_ATOMIC_OR:
         emit_alu_rr(mc, w, X64_OPC_ALU_OR, X64_RCX, X64_RDX);
         break;
       case KIT_CG_ATOMIC_XOR:
         emit_alu_rr(mc, w, X64_OPC_ALU_XOR, X64_RCX, X64_RDX);
         break;
       case KIT_CG_ATOMIC_NAND:
         emit_alu_rr(mc, w, X64_OPC_ALU_AND, X64_RCX, X64_RDX);
         emit_f7_rm(mc, w, X64_F7_SUB_NOT, X64_RCX);
         break;
       default:
         x64_panic(a, "unsupported atomic rmw op");
     }
     emit_lock_prefix(mc);
     emit_rex(mc, w, X64_RCX, 0, base);
     {
       u8 op2[2] = {X64_OPC_TWOBYTE, 0xB1}; /* cmpxchg */
       mc_emit_bytes(mc, op2, 2);
     }
     emit_mem_operand(mc, X64_RCX, base, 0);
     emit_jcc_rel32(mc, X64_CC_NE, retry);
     if (dr != X64_RAX) emit_mov_rr(mc, w, dr, X64_RAX);
   }
 }
 
 static void x64_atomic_cas(NativeTarget* t, NativeLoc prior, NativeLoc ok,
                            NativeAddr addr, NativeLoc expected,
                            NativeLoc desired, MemAccess mem,
                            KitCgMemOrder success, KitCgMemOrder failure) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   u32 sz = mem.size ? mem.size : loc_size32(t, prior);
   int w = sz == 8u ? 1 : 0;
   u32 base = x64_atomic_base(a, addr);
   u32 rprior = loc_reg(prior);
   u32 rok = loc_reg(ok);
   u32 rexp = loc_reg(expected);
   u32 rdes = loc_reg(desired);
   (void)success;
   (void)failure;
   /* cmpxchg uses fixed rax (expected) and rcx (desired). The optimizer may have
    * materialized the address into either; keep it out of both (R11 is
    * backend-private, never an allocated operand). */
   if (base == X64_RAX || base == X64_RCX) {
     emit_mov_rr(mc, 1, X64_TMP_INT2, base);
     base = X64_TMP_INT2;
   }
   /* Place expected -> rax and desired -> rcx as a parallel copy: the allocator
    * may have them in each other's target register (full swap) or desired in rax
    * (expected's target), either of which a naive two-move order would clobber.
    */
   if (rexp == X64_RCX && rdes == X64_RAX) {
     /* Swap rax <-> rcx (xchg needs no temp; base is not rax/rcx here). */
     emit_rex(mc, w, X64_RCX, 0, X64_RAX);
     {
       u8 xchg[2] = {0x87, modrm(3u, X64_RCX, X64_RAX)};
       mc_emit_bytes(mc, xchg, 2);
     }
   } else if (rdes == X64_RAX) {
     /* desired sits in rax; move it to rcx before rax is overwritten. */
     if (rdes != X64_RCX) emit_mov_rr(mc, w, X64_RCX, rdes);
     if (rexp != X64_RAX) emit_mov_rr(mc, w, X64_RAX, rexp);
   } else {
     if (rexp != X64_RAX) emit_mov_rr(mc, w, X64_RAX, rexp);
     if (rdes != X64_RCX) emit_mov_rr(mc, w, X64_RCX, rdes);
   }
   emit_lock_prefix(mc);
   emit_rex(mc, w, X64_RCX, 0, base);
   {
     u8 op2[2] = {X64_OPC_TWOBYTE, 0xB1}; /* cmpxchg [base], rcx */
     mc_emit_bytes(mc, op2, 2);
   }
   emit_mem_operand(mc, X64_RCX, base, 0);
   emit_setcc(mc, X64_CC_E, rok);
   emit_movzx_r32_r8(mc, rok, rok);
   if (rprior != X64_RAX) emit_mov_rr(mc, w, rprior, X64_RAX);
 }
 
 static void x64_fence(NativeTarget* t, KitCgMemOrder mo) {
   if (mo == KIT_CG_MO_SEQ_CST) emit_mfence(t->mc);
 }
 
 /* ============================ variadics ============================
  * SysV: __va_list_tag (gp_offset@0, fp_offset@4, overflow@8, reg_save@16). The
  * prologue filled the 176B reg-save area. Win64: va_list is a single pointer
  * to the next 8-byte slot in the home/overflow area; FP varargs are duplicated
  * into the matching GPR slot at the call site. `ap` addresses the va_list
  * object. */
 
 /* Resolve a va_list address into `scratch`, materializing it there if it is not
  * already, so the backend-private field-value register never aliases it. */
 static u32 x64_va_base(X64NativeTarget* a, NativeAddr ap, u32 scratch) {
   u32 base = x64_addr_to_base_reg(a, ap, scratch);
   if (base != scratch) {
     emit_mov_rr(a->base.mc, 1, scratch, base);
     base = scratch;
   }
   return base;
 }
 
 /* add r/m, imm8 (group-1 /0) directly to a memory field — advances a va_list
  * offset/pointer in place without consuming a register. w selects 64- vs
  * 32-bit. */
 static void x64_add_mem_imm(MCEmitter* mc, int w, u32 base, i32 disp, i8 imm) {
   u8 op = X64_OPC_ALU_IMM8;
   u8 b;
   emit_rex(mc, w, 0, 0, base);
   mc_emit_bytes(mc, &op, 1);
   emit_mem_operand(mc, X64_ALU_SUB_ADD, base, disp); /* modrm.reg = /0 (ADD) */
   b = (u8)imm;
   mc_emit_bytes(mc, &b, 1);
 }
 
 /* add r64, [base+disp] (0x03 /r). */
 static void x64_add_reg_mem(MCEmitter* mc, u32 dst, u32 base, i32 disp) {
   u8 op = 0x03;
   emit_rex(mc, 1, dst, 0, base);
   mc_emit_bytes(mc, &op, 1);
   emit_mem_operand(mc, dst, base, disp);
 }
 
 static void x64_va_start_core(X64NativeTarget* a, NativeAddr ap) {
   NativeTarget* t = &a->base;
   MCEmitter* mc = t->mc;
   u32 ap_base;
   if (!a->is_variadic) x64_panic(a, "va_start: function not variadic");
   ap_base = x64_va_base(a, ap, X64_TMP_INT2);
   if (a->abi->shadow_space) {
     /* Win64: *ap = rbp + 16 + named_int*8 + named_stack. */
     u32 first = 16u + a->next_param_int * 8u + a->next_param_stack;
     emit_lea(mc, X64_TMP_INT, X64_RBP, (i32)first);
     emit_mov_store(mc, 8, X64_TMP_INT, ap_base, 0);
     return;
   }
   {
     X64NativeSlot* rs = x64_slot_get(a, a->reg_save_slot);
     /* gp_offset = next_param_int * 8 */
     x64_emit_load_imm(mc, 0, X64_TMP_INT, (i64)(a->next_param_int * 8u));
     emit_mov_store(mc, 4, X64_TMP_INT, ap_base, 0);
     /* fp_offset = 48 + next_param_fp * 16 */
     x64_emit_load_imm(mc, 0, X64_TMP_INT,
                       (i64)(48u + a->next_param_fp * 16u));
     emit_mov_store(mc, 4, X64_TMP_INT, ap_base, 4);
     /* overflow_arg_area = rbp + 16 + next_param_stack */
     emit_lea(mc, X64_TMP_INT, X64_RBP,
              (i32)(16u + a->next_param_stack));
     emit_mov_store(mc, 8, X64_TMP_INT, ap_base, 8);
     /* reg_save_area = rbp - reg_save_slot.off */
     emit_lea(mc, X64_TMP_INT, X64_RBP, -(i32)rs->off);
     emit_mov_store(mc, 8, X64_TMP_INT, ap_base, 16);
   }
 }
 
 static void x64_va_arg_core(X64NativeTarget* a, NativeLoc dst, NativeAddr ap,
                             KitCgTypeId type) {
   NativeTarget* t = &a->base;
   MCEmitter* mc = t->mc;
   u32 sz = native_type_size(t, type);
   int is_fp = native_loc_is_fp(dst);
   u32 dr = loc_reg(dst);
   u32 ap_base = x64_va_base(a, ap, X64_TMP_INT2); /* r11 */
   /* GPR temporary for the offset/address arithmetic. For integer results the
    * destination is itself a throwaway operand temp — pass_native_emit fetches
    * va_arg into a scoped register and writes it to the real destination — so
    * we reuse `dr` and touch no allocable register at all. FP results keep their
    * value in an XMM register, so they borrow backend-private R10.
    * Either way only r11 (ap_base) and `gp` are used: the va_list fields are
    * advanced in memory (x64_add_mem_imm) and the reg-save base is folded in
    * with x64_add_reg_mem, so no third register is needed. */
   u32 gp = is_fp ? X64_TMP_INT : dr;
   if (a->abi->shadow_space) {
     /* Win64: gp = *ap; load dr from [gp]; *ap += 8. */
     emit_mov_load(mc, 8, 0, gp, ap_base, 0);
     if (is_fp)
       emit_sse_load(mc, sse_scalar_prefix(sz), 0x10, dr, gp, 0);
     else
       emit_mov_load(mc, sz, 0, dr, gp, 0);
     x64_add_mem_imm(mc, 1, ap_base, 0, 8);
     return;
   }
   {
     u32 offs_field = is_fp ? 4u : 0u;
     u32 max_offs = is_fp ? 176u : 48u;
     i8 stride = is_fp ? 16 : 8;
     MCLabel L_stack = mc_label_new(mc);
     MCLabel L_done = mc_label_new(mc);
     /* gp32 = ap[offs]; cmp gp32, max; jae L_stack. Use the imm8 form when the
      * threshold fits (gp_offset max 48) so the encoding is canonical and the
      * `cc -S | as` round-trip reproduces it; fp_offset max 176 needs imm32. */
     emit_mov_load(mc, 4, 0, gp, ap_base, (i32)offs_field);
     if (imm_fits_i8((i64)max_offs))
       emit_alu_imm8(mc, 0, X64_ALU_SUB_CMP, gp, (i8)max_offs);
     else
       emit_alu_imm32(mc, 0, X64_ALU_SUB_CMP, gp, (i32)max_offs);
     emit_jcc_rel32(mc, X64_CC_AE, L_stack);
     /* reg path: ap[offs] += stride; gp = reg_save_area(ap[16]) + offset; load.
      * (The memory increment leaves gp holding the old offset.) */
     x64_add_mem_imm(mc, 0, ap_base, (i32)offs_field, stride);
     x64_add_reg_mem(mc, gp, ap_base, 16);
     if (is_fp)
       emit_sse_load(mc, sse_scalar_prefix(sz), 0x10, dr, gp, 0);
     else
       emit_mov_load(mc, sz, 0, dr, gp, 0);
     emit_jmp_rel32(mc, L_done);
     /* stack path: gp = ap[8] (overflow area); load; ap[8] += 8. */
     mc_label_place(mc, L_stack);
     emit_mov_load(mc, 8, 0, gp, ap_base, 8);
     if (is_fp)
       emit_sse_load(mc, sse_scalar_prefix(sz), 0x10, dr, gp, 0);
     else
       emit_mov_load(mc, sz, 0, dr, gp, 0);
     x64_add_mem_imm(mc, 1, ap_base, 8, 8);
     mc_label_place(mc, L_done);
   }
 }
 
 static void x64_va_copy_core(X64NativeTarget* a, NativeAddr dst_ap,
                              NativeAddr src_ap) {
   NativeTarget* t = &a->base;
   MCEmitter* mc = t->mc;
   /* Resolve dst into R11, src into R10, and copy each qword through
    * backend-private XMM14. Uses only reserved backend registers
    * (R11/R10/XMM14), so the
    * optimizer's register choice for a va_list pointer can never be clobbered
    * and no allocable GPR (previously rdx) is consumed. */
   u32 dst_base = x64_va_base(a, dst_ap, X64_TMP_INT2);
   u32 src_base = x64_va_base(a, src_ap, X64_TMP_INT);
   u32 n = a->abi->shadow_space ? 8u : 24u, i;
   for (i = 0; i < n; i += 8u) {
     emit_sse_load(mc, 0xF2, 0x10, X64_TMP_FP, src_base, (i32)i);  /* movsd */
     emit_sse_store(mc, 0xF2, 0x11, X64_TMP_FP, dst_base, (i32)i); /* movsd */
   }
 }
 
 static NativeAddr x64_va_addr_from_ptr(NativeLoc ap_ptr) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_REG;
   addr.cls = NATIVE_REG_INT;
   addr.base.reg = ap_ptr.v.reg;
   addr.base_type = ap_ptr.type;
   return addr;
 }
 
 static void x64_va_start_native(NativeTarget* t, NativeLoc ap_ptr) {
   x64_va_start_core(x64_of(t), x64_va_addr_from_ptr(ap_ptr));
 }
 static void x64_va_arg_native(NativeTarget* t, NativeLoc dst, NativeLoc ap_ptr,
                               KitCgTypeId type) {
   x64_va_arg_core(x64_of(t), dst, x64_va_addr_from_ptr(ap_ptr), type);
 }
 static void x64_va_end_native(NativeTarget* t, NativeLoc ap_ptr) {
   (void)t;
   (void)ap_ptr;
 }
 static void x64_va_copy_native(NativeTarget* t, NativeLoc dst, NativeLoc src) {
   x64_va_copy_core(x64_of(t), x64_va_addr_from_ptr(dst),
                    x64_va_addr_from_ptr(src));
 }
 
 /* ============================ intrinsics ============================ */
 
 static void emit_popcnt(MCEmitter* mc, int w, u32 dst, u32 src) {
   u8 p = 0xF3;
   mc_emit_bytes(mc, &p, 1);
   emit_rex(mc, w, dst, 0, src);
   {
     u8 op[2] = {X64_OPC_TWOBYTE, 0xB8};
     mc_emit_bytes(mc, op, 2);
   }
   emit_rm_reg(mc, dst, src);
 }
 static void emit_popcount_software(MCEmitter* mc, int w, u32 dst, u32 src) {
   MCLabel loop = mc_label_new(mc);
   MCLabel done = mc_label_new(mc);
   /* Kernighan's bit-count loop. Keep the source in reserved scratch so dst
    * may legally overlap it, and so the baseline x86-64 profile never executes
    * POPCNT unless the resolved target says it is available. */
   emit_mov_rr(mc, w, X64_TMP_INT, src);
   x64_emit_load_imm(mc, w, dst, 0);
   mc_label_place(mc, loop);
   emit_test_self(mc, w, X64_TMP_INT);
   emit_jcc_rel32(mc, X64_CC_E, done);
   emit_alu_imm8(mc, w, X64_ALU_SUB_ADD, dst, 1);
   emit_mov_rr(mc, w, X64_TMP_INT2, X64_TMP_INT);
   emit_alu_imm8(mc, w, X64_ALU_SUB_SUB, X64_TMP_INT2, 1);
   emit_alu_rr(mc, w, X64_OPC_ALU_AND, X64_TMP_INT, X64_TMP_INT2);
   emit_jmp_rel32(mc, loop);
   mc_label_place(mc, done);
 }
 static void emit_bs(MCEmitter* mc, int w, u8 opcode2, u32 dst, u32 src) {
   emit_rex(mc, w, dst, 0, src);
   {
     u8 op[2] = {X64_OPC_TWOBYTE, opcode2};
     mc_emit_bytes(mc, op, 2);
   }
   emit_rm_reg(mc, dst, src);
 }
 static void emit_bswap(MCEmitter* mc, int w, u32 reg) {
   emit_rex(mc, w, 0, 0, reg);
   {
     u8 op[2] = {X64_OPC_TWOBYTE, (u8)(0xC8 + (reg & 7u))};
     mc_emit_bytes(mc, op, 2);
   }
 }
 static void emit_rol16_imm8(MCEmitter* mc, u32 reg, u8 imm) {
   u8 p = X64_OPSIZE_PFX;
   mc_emit_bytes(mc, &p, 1);
   emit_rex(mc, 0, 0, 0, reg);
   {
     u8 buf[3] = {X64_OPC_SHIFT_IMM, modrm(3u, 0u, reg & 7u), imm};
     mc_emit_bytes(mc, buf, 3);
   }
 }
 static void emit_ud2(MCEmitter* mc) {
   u8 b[2] = {0x0F, 0x0B};
   mc_emit_bytes(mc, b, 2);
 }
 
 static void emit_syscall(MCEmitter* mc) {
   u8 b[2] = {0x0F, 0x05};
   mc_emit_bytes(mc, b, 2);
 }
 
 static void emit_rdtsc(MCEmitter* mc) {
   u8 b[2] = {0x0F, 0x31};
   mc_emit_bytes(mc, b, 2);
 }
 
 /* Linux/Android x86-64 stack-protector guard: mov rd, qword ptr fs:[0x28]. */
 static void emit_stack_guard(MCEmitter* mc, u32 rd) {
   u8 fs = 0x64;
   u8 op = X64_OPC_MOV_R_RM;
   u8 mr = modrm(0u, rd & 7u, 4u);
   u8 s = sib(0u, 4u, 5u);
   mc_emit_bytes(mc, &fs, 1);
   emit_rex(mc, 1, rd, 0, 0);
   mc_emit_bytes(mc, &op, 1);
   mc_emit_bytes(mc, &mr, 1);
   mc_emit_bytes(mc, &s, 1);
   emit_u32le(mc, 0x28u);
 }
 
 /* Optimizer folding may put an immediate directly in a native intrinsic even
  * when the instruction form itself requires a register. Materialize those
  * arithmetic operands in the backend-private bank; ordinary MIR/NDT operand
  * locations never occupy R10/R11. Callers pass a distinct scratch for each
  * simultaneously-live argument. */
 static u32 x64_intrinsic_arg_reg(X64NativeTarget* a, NativeLoc arg,
                                  u32 scratch, int w) {
   if (arg.kind == NATIVE_LOC_REG) return loc_reg(arg);
   if (arg.kind == NATIVE_LOC_IMM) {
     x64_emit_load_imm(a->base.mc, w, scratch, arg.v.imm);
     return scratch;
   }
   x64_panic(a, "arithmetic intrinsic operand is not register/immediate");
 }
 
 static int x64_intrinsic_arg_accepts_imm(NativeTarget* t, IntrinKind kind,
                                          u32 arg_index) {
   (void)t;
   if (arg_index > 1u) return 0;
   switch (kind) {
     case INTRIN_SADD_OVERFLOW:
     case INTRIN_UADD_OVERFLOW:
     case INTRIN_SSUB_OVERFLOW:
     case INTRIN_USUB_OVERFLOW:
     case INTRIN_SMUL_OVERFLOW:
     case INTRIN_UMUL_OVERFLOW:
     case INTRIN_SMUL_HIGH:
     case INTRIN_UMUL_HIGH:
       return 1;
     default:
       return 0;
   }
 }
 
 static void x64_intrinsic(NativeTarget* t, IntrinKind kind,
                           const NativeLoc* dsts, u32 ndst,
                           const NativeLoc* args, u32 narg) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   (void)ndst;
   switch (kind) {
     case INTRIN_NONE:
       break;
     case INTRIN_EXPECT:
     case INTRIN_ASSUME_ALIGNED:
       if (args[0].kind == NATIVE_LOC_IMM)
         x64_emit_load_imm(mc, x64_is_64(t, dsts[0].type) ? 1 : 0,
                           loc_reg(dsts[0]), args[0].v.imm);
       else
         x64_move(t, dsts[0], args[0]);
       return;
     case INTRIN_PREFETCH:
       if (narg >= 1u && args[0].kind == NATIVE_LOC_REG) {
         /* GCC locality 0..3 maps from no-temporal through increasing temporal
          * locality: PREFETCHNTA, PREFETCHT2, PREFETCHT1, PREFETCHT0. The rw
          * operand is intentionally ignored: PREFETCHW is not baseline x64. */
         static const u8 hint[4] = {0u, 3u, 2u, 1u};
         u32 locality = 3u;
         u32 base = loc_reg(args[0]);
         u8 op[2] = {0x0F, 0x18};
         if (narg >= 3u && args[2].kind == NATIVE_LOC_IMM)
           locality = (u32)args[2].v.imm;
         if (locality > 3u) locality = 3u;
         emit_rex(mc, 0, hint[locality], 0, base);
         mc_emit_bytes(mc, op, 2);
         emit_mem_operand(mc, hint[locality], base, 0);
       }
       return;
     case INTRIN_TRAP:
       emit_ud2(mc);
       return;
     case INTRIN_READCYCLECOUNTER: {
       /* RDTSC returns the 64-bit timestamp counter split across edx:eax
        * (writing eax/edx zero-extends the upper halves of rax/rdx). Recombine
        * into a single 64-bit value: dst = (rdx << 32) | rax. The rax/rdx
        * clobber is modeled in x64_machine_op_clobbers. */
       u32 rd = loc_reg(dsts[0]);
       emit_rdtsc(mc);
       emit_shift_imm(mc, 1, X64_SHIFT_SUB_SHL, X64_RDX, 32);
       emit_alu_rr(mc, 1, X64_OPC_ALU_OR, X64_RAX, X64_RDX);
       if (rd != X64_RAX) emit_mov_rr(mc, 1, rd, X64_RAX);
       return;
     }
     case INTRIN_STACK_GUARD:
       if (ndst != 1u || narg != 0u)
         x64_panic(a, "stack guard intrinsic has invalid operands");
       emit_stack_guard(mc, loc_reg(dsts[0]));
       return;
     case INTRIN_SYSCALL:
       if (ndst == 1u && narg >= 1u && narg <= 7u) {
         static const u32 syscall_regs[7] = {X64_RAX, X64_RDI, X64_RSI, X64_RDX,
                                             X64_R10, X64_R8,  X64_R9};
         X64ArgMove moves[7];
         for (u32 i = 0; i < narg; ++i) {
           X64ArgMove* m = &moves[i];
           memset(m, 0, sizeof *m);
           m->dst =
               native_loc_reg(dsts[0].type, NATIVE_REG_INT, syscall_regs[i]);
           m->src = args[i];
           m->size = t->c->target.ptr_size;
         }
         x64_emit_reg_arg_moves(t, moves, narg, X64_TMP_INT2);
         emit_syscall(mc);
         x64_move(t, dsts[0],
                  native_loc_reg(dsts[0].type, NATIVE_REG_INT, X64_RAX));
       }
       return;
     case INTRIN_POPCOUNT: {
       int w = x64_is_64(t, args[0].type) ? 1 : 0;
       if (kit_target_has_feature(t->c->target_ref, KIT_SLICE_LIT("popcnt")))
         emit_popcnt(mc, w, loc_reg(dsts[0]), loc_reg(args[0]));
       else
         emit_popcount_software(mc, w, loc_reg(dsts[0]), loc_reg(args[0]));
       return;
     }
     case INTRIN_CTZ:
       emit_bs(mc, x64_is_64(t, args[0].type) ? 1 : 0, 0xBC /* bsf */,
               loc_reg(dsts[0]), loc_reg(args[0]));
       return;
     case INTRIN_CLZ: {
       int w = x64_is_64(t, args[0].type) ? 1 : 0;
       u32 dr = loc_reg(dsts[0]);
       emit_bs(mc, w, 0xBD /* bsr */, dr, loc_reg(args[0]));
       /* clz = (bits-1) - bsr, computed via xor with bits-1. The mask (31/63)
        * fits in imm8, so use the compact 0x83 form to match the canonical
        * encoding (and the assembler's `cc -S | as` round-trip). */
       emit_alu_imm8(mc, w, X64_ALU_SUB_XOR, dr, w ? 63 : 31);
       return;
     }
     case INTRIN_BSWAP: {
       u32 width = abi_cg_sizeof(t->c->abi, dsts[0].type);
       switch (width) {
         case 2: {
           u32 dr = loc_reg(dsts[0]), sr = loc_reg(args[0]);
           if (dr != sr) emit_mov_rr(mc, 0, dr, sr);
           emit_rol16_imm8(mc, dr, 8);
           return;
         }
         case 4: {
           u32 dr = loc_reg(dsts[0]), sr = loc_reg(args[0]);
           if (dr != sr) emit_mov_rr(mc, 0, dr, sr);
           emit_bswap(mc, 0, dr);
           return;
         }
         case 8: {
           u32 dr = loc_reg(dsts[0]), sr = loc_reg(args[0]);
           if (dr != sr) emit_mov_rr(mc, 1, dr, sr);
           emit_bswap(mc, 1, dr);
           return;
         }
         default:
           break;
       }
       return;
     }
     case INTRIN_SADD_OVERFLOW:
     case INTRIN_UADD_OVERFLOW:
     case INTRIN_SSUB_OVERFLOW:
     case INTRIN_USUB_OVERFLOW: {
       int w = x64_is_64(t, dsts[0].type) ? 1 : 0;
       u32 rd = loc_reg(dsts[0]), rovf = loc_reg(dsts[1]);
       u32 ra = x64_intrinsic_arg_reg(a, args[0], X64_TMP_INT, w);
       u32 rb = x64_intrinsic_arg_reg(a, args[1], X64_TMP_INT2, w);
       u8 op = (kind == INTRIN_SADD_OVERFLOW || kind == INTRIN_UADD_OVERFLOW)
                   ? X64_OPC_ALU_ADD
                   : X64_OPC_ALU_SUB;
       u32 cc = (kind == INTRIN_UADD_OVERFLOW || kind == INTRIN_USUB_OVERFLOW)
                    ? X64_CC_B
                    : X64_CC_O;
       if (rd != ra) emit_mov_rr(mc, w, rd, ra);
       emit_alu_rr(mc, w, op, rd, rb);
       emit_setcc(mc, cc, rovf);
       emit_movzx_r32_r8(mc, rovf, rovf);
       return;
     }
     case INTRIN_SMUL_OVERFLOW: {
       int w = x64_is_64(t, dsts[0].type) ? 1 : 0;
       u32 rd = loc_reg(dsts[0]), rovf = loc_reg(dsts[1]);
       u32 ra = x64_intrinsic_arg_reg(a, args[0], X64_TMP_INT, w);
       u32 rb = x64_intrinsic_arg_reg(a, args[1], X64_TMP_INT2, w);
       if (rd != ra) emit_mov_rr(mc, w, rd, ra);
       emit_imul_rr(mc, w, rd, rb);
       emit_setcc(mc, X64_CC_O, rovf);
       emit_movzx_r32_r8(mc, rovf, rovf);
       return;
     }
     case INTRIN_UMUL_OVERFLOW: {
       int w = x64_is_64(t, dsts[0].type) ? 1 : 0;
       u32 rd = loc_reg(dsts[0]), rovf = loc_reg(dsts[1]);
       u32 ra = x64_intrinsic_arg_reg(a, args[0], X64_TMP_INT, w);
       u32 rb = x64_intrinsic_arg_reg(a, args[1], X64_TMP_INT2, w);
       if (rb == X64_RAX || rb == X64_RDX) {
         emit_mov_rr(mc, w, X64_R11, rb);
         rb = X64_R11;
       }
       if (ra != X64_RAX) emit_mov_rr(mc, w, X64_RAX, ra);
       emit_f7_rm(mc, w, X64_F7_SUB_MUL, rb); /* MUL: rdx:rax = rax * rb */
       if (rd != X64_RAX) emit_mov_rr(mc, w, rd, X64_RAX);
       emit_setcc(mc, X64_CC_O, rovf);
       emit_movzx_r32_r8(mc, rovf, rovf);
       return;
     }
     case INTRIN_SMUL_HIGH:
     case INTRIN_UMUL_HIGH: {
       int w = x64_is_64(t, dsts[0].type) ? 1 : 0;
       u32 rd = loc_reg(dsts[0]);
       u32 ra = x64_intrinsic_arg_reg(a, args[0], X64_TMP_INT, w);
       u32 rb = x64_intrinsic_arg_reg(a, args[1], X64_TMP_INT2, w);
       if (rb == X64_RAX || rb == X64_RDX) {
         emit_mov_rr(mc, w, X64_R11, rb);
         rb = X64_R11;
       }
       if (ra != X64_RAX) emit_mov_rr(mc, w, X64_RAX, ra);
       emit_f7_rm(mc, w,
                  kind == INTRIN_SMUL_HIGH ? X64_F7_SUB_IMUL
                                           : X64_F7_SUB_MUL,
                  rb);
       if (rd != X64_RDX) emit_mov_rr(mc, w, rd, X64_RDX);
       return;
     }
     case INTRIN_MEMMOVE: {
       u32 dr, sr, n, i;
       if (narg != 3u || args[0].kind != NATIVE_LOC_REG ||
           args[1].kind != NATIVE_LOC_REG || args[2].kind != NATIVE_LOC_IMM)
         x64_panic(a, "unsupported memory intrinsic operands");
       if (args[2].v.imm < 0 || args[2].v.imm > 0xffffffffll)
         x64_panic(a, "unsupported memory intrinsic size");
       dr = loc_reg(args[0]);
       sr = loc_reg(args[1]);
       n = (u32)args[2].v.imm;
       i = n; /* copy high-to-low so an overlapping dst > src is safe */
       while (i >= 8u) {
         i -= 8u;
         emit_mov_load(mc, 8, 0, X64_TMP_INT, sr, (i32)i);
         emit_mov_store(mc, 8, X64_TMP_INT, dr, (i32)i);
       }
       while (i >= 4u) {
         i -= 4u;
         emit_mov_load(mc, 4, 0, X64_TMP_INT, sr, (i32)i);
         emit_mov_store(mc, 4, X64_TMP_INT, dr, (i32)i);
       }
       while (i >= 2u) {
         i -= 2u;
         emit_mov_load(mc, 2, 0, X64_TMP_INT, sr, (i32)i);
         emit_mov_store(mc, 2, X64_TMP_INT, dr, (i32)i);
       }
       while (i >= 1u) {
         i -= 1u;
         emit_mov_load(mc, 1, 0, X64_TMP_INT, sr, (i32)i);
         emit_mov_store(mc, 1, X64_TMP_INT, dr, (i32)i);
       }
       return;
     }
     case INTRIN_CPU_NOP: {
       u8 b = 0x90; /* NOP */
       mc_emit_bytes(mc, &b, 1);
       return;
     }
     case INTRIN_CPU_YIELD: {
       u8 b[2] = {0xF3, 0x90}; /* PAUSE */
       mc_emit_bytes(mc, b, 2);
       return;
     }
     case INTRIN_DMB:
     case INTRIN_DSB: {
       u8 b[3] = {0x0F, 0xAE, 0xF0}; /* MFENCE: full-system memory barrier */
       mc_emit_bytes(mc, b, 3);
       return;
     }
     case INTRIN_IRQ_DISABLE: {
       u8 b = 0xFA; /* CLI (privileged) */
       mc_emit_bytes(mc, &b, 1);
       return;
     }
     case INTRIN_IRQ_ENABLE: {
       u8 b = 0xFB; /* STI (privileged) */
       mc_emit_bytes(mc, &b, 1);
       return;
     }
     case INTRIN_FRAME_ADDRESS:
     case INTRIN_RETURN_ADDRESS:
       /* Walk the rbp frame-record chain. Every kit prologue keeps the rbp
        * record: [rbp] = caller's rbp, [rbp + 8] = return address pushed by the
        * `call`. The level is a compile-time constant, so the walk unrolls to
        * `level` dependent loads. */
       if (ndst == 1u) {
         u32 level = (narg >= 1u && args[0].kind == NATIVE_LOC_IMM)
                         ? (u32)args[0].v.imm
                         : 0u;
         u32 rd = loc_reg(dsts[0]);
         emit_mov_rr(mc, 1, rd, X64_RBP);
         for (u32 i = 0; i < level; ++i)
           emit_mov_load(mc, 8, 0, rd, rd, 0); /* rd = *(rd) */
         if (kind == INTRIN_RETURN_ADDRESS)
           emit_mov_load(mc, 8, 0, rd, rd, 8); /* rd = *(rd + 8) */
       }
       return;
     default:
       break;
   }
   x64_panic(a, "unsupported compiler intrinsic");
 }
 
 /* ============================ inline asm ============================ */
 
 _Noreturn static void x64_asm_panic_at(Compiler* c, SrcLoc loc,
                                        const char* msg) {
   compiler_panic(c, loc, "x64 inline asm: %s", msg);
 }
 _Noreturn static void x64_asm_panic(NativeDirectTarget* d, const char* msg) {
   x64_asm_panic_at(d->base.c, d->loc, msg);
 }
 
 /* constraint_body / constraint_early / match_index are shared
  * (cg/native_asm.h). */
 
 static void x64_asm_bound_reg(Operand* out, KitCgTypeId type,
                               NativeAllocClass cls, Reg reg) {
   memset(out, 0, sizeof *out);
   out->kind = X64_INLINE_OPK_REG;
   out->pad[0] =
       (cls == NATIVE_REG_FP) ? X64_INLINE_OPCLS_FP : X64_INLINE_OPCLS_INT;
   out->type = type;
   out->v.local = (CGLocal)reg;
 }
 static void x64_asm_bound_mem(Operand* out, KitCgTypeId type, Reg base) {
   memset(out, 0, sizeof *out);
   out->kind = OPK_INDIRECT;
   out->type = type;
   out->v.ind.base = (CGLocal)base;
   out->v.ind.index = CG_LOCAL_NONE;
   out->v.ind.ofs = 0;
 }
 
 /* Parse a clobber register name into (class, reg). Returns 0 for cc/memory.
  * GPR names map to HW encoding via x64_register_hw_index; xmm names map via the
  * DWARF table (xmm0..15 = dwarf 17..32). */
 static int x64_asm_parse_reg_clobber(Compiler* c, SrcLoc loc, Sym name,
                                      NativeAllocClass* cls_out, Reg* reg_out) {
   Slice s = pool_slice(c->global, name);
   char buf[16];
   uint32_t idx;
   if (!s.s || !s.len) return 0;
   if (s.len == 2 && s.s[0] == 'c' && s.s[1] == 'c') return 0;
   if (s.len == 6 && memcmp(s.s, "memory", 6) == 0) return 0;
   if (s.len >= sizeof buf) x64_asm_panic_at(c, loc, "clobber name is too long");
   memcpy(buf, s.s, s.len);
   buf[s.len] = '\0';
   if (x64_register_hw_index(buf, &idx) == 0 && idx <= 15u) {
     *cls_out = NATIVE_REG_INT;
     *reg_out = (Reg)idx;
     return 1;
   }
   if (x64_register_index(buf, &idx) == 0 && idx >= 17u && idx <= 32u) {
     *cls_out = NATIVE_REG_FP;
     *reg_out = (Reg)(idx - 17u);
     return 1;
   }
   x64_asm_panic_at(c, loc, "unknown clobber register");
   return 0;
 }
 
 static void x64_asm_clobber_masks(Compiler* c, SrcLoc loc, const Sym* clobbers,
                                   u32 nclob, u32* int_mask, u32* fp_mask) {
   u32 i;
   *int_mask = 0;
   *fp_mask = 0;
   for (i = 0; i < nclob; ++i) {
     NativeAllocClass cls;
     Reg reg;
     if (!x64_asm_parse_reg_clobber(c, loc, clobbers[i], &cls, &reg)) continue;
     if (cls == NATIVE_REG_INT)
       *int_mask |= 1u << reg;
     else
       *fp_mask |= 1u << reg;
   }
 }
 
 /* Pin resolution + panic is the shared native_asm_bind_direct_operands path. */
 
 /* Direct (-O0) path: resolve a semantic Operand to a NativeAddr. */
 static NativeAddr x64_direct_addr(NativeDirectTarget* d, Operand op) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   switch ((OpKind)op.kind) {
     case OPK_LOCAL:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = d->locals[op.v.local - 1u].home;
       addr.base_type = op.type;
       return addr;
     case OPK_INDIRECT:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME_VALUE;
       addr.base.frame = d->locals[op.v.ind.base - 1u].home;
       addr.cls = d->locals[op.v.ind.base - 1u].cls;
       addr.base_type = d->locals[op.v.ind.base - 1u].type;
       addr.offset = op.v.ind.ofs;
       return addr;
     default:
       x64_asm_panic(d, "operand is not addressable");
   }
 }
 
 static NativeAddr x64_direct_materialize_addr(NativeDirectTarget* d,
                                               Operand op) {
   X64NativeTarget* a = x64_of(d->native);
   NativeAddr addr = x64_direct_addr(d, op);
   if (addr.base_kind == NATIVE_ADDR_BASE_FRAME_VALUE) {
     NativeAddr load;
     memset(&load, 0, sizeof load);
     load.base_kind = NATIVE_ADDR_BASE_FRAME;
     load.base.frame = addr.base.frame;
     load.base_type = addr.base_type;
     emit_mov_load(a->base.mc, 8, 0, X64_TMP_INT2, X64_RBP,
                   -(i32)x64_slot_get(a, addr.base.frame)->off);
     addr.base_kind = NATIVE_ADDR_BASE_REG;
     addr.base.reg = X64_TMP_INT2;
   }
   return addr;
 }
 
 static void x64_direct_load_operand_to_reg(NativeDirectTarget* d, Operand op,
                                            NativeLoc dst) {
   X64NativeTarget* a = x64_of(d->native);
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   switch ((OpKind)op.kind) {
     case OPK_IMM:
       if ((NativeAllocClass)dst.cls != NATIVE_REG_INT)
         x64_asm_panic(d, "floating-point immediate asm input is unsupported");
       d->native->load_imm(d->native, dst, op.v.imm);
       return;
     case OPK_LOCAL:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = d->locals[op.v.local - 1u].home;
       addr.base_type = op.type;
       x64_emit_mem(a, 1, dst, addr, native_mem_for_type(d->native, op.type, 0));
       return;
     case OPK_GLOBAL:
       addr.base_kind = NATIVE_ADDR_BASE_GLOBAL;
       addr.base.global.sym = op.v.global.sym;
       addr.base.global.addend = op.v.global.addend;
       addr.base_type = op.type;
       d->native->load_addr(d->native, dst, addr);
       return;
     case OPK_INDIRECT:
       addr = x64_direct_materialize_addr(d, op);
       x64_emit_mem(a, 1, dst, addr, native_mem_for_type(d->native, op.type, 0));
       return;
   }
   x64_asm_panic(d, "unsupported asm input operand");
 }
 
 static void x64_direct_load_address_to_reg(NativeDirectTarget* d, Operand op,
                                            NativeLoc dst) {
   d->native->load_addr(d->native, dst, x64_direct_addr(d, op));
 }
 
 static void x64_direct_store_reg_to_operand(NativeDirectTarget* d, Operand op,
                                             NativeLoc src) {
   X64NativeTarget* a = x64_of(d->native);
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   if (op.kind == OPK_LOCAL) {
     addr.base_kind = NATIVE_ADDR_BASE_FRAME;
     addr.base.frame = d->locals[op.v.local - 1u].home;
     addr.base_type = op.type;
   } else {
     addr = x64_direct_materialize_addr(d, op);
   }
   x64_emit_mem(a, 0, src, addr, native_mem_for_type(d->native, op.type, 0));
 }
 
 /* Callee-saved registers an asm block clobbers must be saved around the block.
  */
 typedef struct X64AsmSavedClobber {
   NativeFrameSlot slot;
   NativeAllocClass cls;
   Reg reg;
   KitCgTypeId type;
 } X64AsmSavedClobber;
 
 static void x64_asm_save_one(X64NativeTarget* a, X64AsmSavedClobber* s) {
   NativeFrameSlotDesc desc;
   NativeAddr addr;
   memset(&desc, 0, sizeof desc);
   desc.type = s->type;
   desc.size = s->cls == NATIVE_REG_FP ? 16u : 8u;
   desc.align = desc.size;
   desc.kind = NATIVE_FRAME_SLOT_SAVE;
   s->slot = a->base.frame_slot(&a->base, &desc);
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = s->slot;
   addr.base_type = s->type;
   x64_emit_mem(a, 0, native_loc_reg(s->type, s->cls, s->reg), addr,
                native_mem_for_type(&a->base, s->type, desc.size));
 }
 static void x64_asm_restore_one(X64NativeTarget* a,
                                 const X64AsmSavedClobber* s) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = s->slot;
   addr.base_type = s->type;
   x64_emit_mem(a, 1, native_loc_reg(s->type, s->cls, s->reg), addr,
                native_mem_for_type(&a->base, s->type,
                                    s->cls == NATIVE_REG_FP ? 16u : 8u));
 }
 
 /* SysV callee-saved: int rbx,r12-r15; no fp. Win64 adds rdi,rsi + xmm6-15. */
 static int x64_reg_is_callee_int(const X64ABIRegs* abi, Reg r) {
   if (r == X64_RBP) return 0; /* prologue head handles rbp */
   return (abi->cs_int_mask & (1ull << r)) != 0;
 }
 static int x64_reg_is_callee_fp(const X64ABIRegs* abi, Reg r) {
   return (abi->cs_fp_mask & (1ull << r)) != 0;
 }
 
 static X64AsmSavedClobber* x64_asm_save_callee_clobbers(X64NativeTarget* a,
                                                         u32 int_mask,
                                                         u32 fp_mask,
                                                         u32* nsaved_out) {
   X64AsmSavedClobber* saved =
       arena_zarray(a->base.c->tu, X64AsmSavedClobber, 32u);
   KitCgTypeId i64 = builtin_id(KIT_CG_BUILTIN_I64);
   KitCgTypeId f64 = builtin_id(KIT_CG_BUILTIN_F64);
   u32 n = 0;
   Reg r;
   for (r = 0; r <= 15u; ++r) {
     if ((int_mask & (1u << r)) == 0 || !x64_reg_is_callee_int(a->abi, r))
       continue;
     saved[n].cls = NATIVE_REG_INT;
     saved[n].reg = r;
     saved[n].type = i64;
     x64_asm_save_one(a, &saved[n++]);
   }
   for (r = 0; r <= 15u; ++r) {
     if ((fp_mask & (1u << r)) == 0 || !x64_reg_is_callee_fp(a->abi, r))
       continue;
     saved[n].cls = NATIVE_REG_FP;
     saved[n].reg = r;
     saved[n].type = f64;
     x64_asm_save_one(a, &saved[n++]);
   }
   *nsaved_out = n;
   return saved;
 }
 
 /* ---- NativeTarget (optimizer) asm hook ----
  *
  * Register placement, staging, and output writeback belong to the optimized
  * emitter. This hook binds those concrete locations and only materializes
  * memory-constraint bases in backend-private registers. */
 
 static NativeAddr x64_asm_loc_to_addr(X64NativeTarget* a, SrcLoc loc,
                                       NativeLoc src) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_type = src.type;
   switch ((NativeLocKind)src.kind) {
     case NATIVE_LOC_FRAME:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = src.v.frame;
       return addr;
     case NATIVE_LOC_ADDR:
       return src.v.addr;
     case NATIVE_LOC_GLOBAL:
       addr.base_kind = NATIVE_ADDR_BASE_GLOBAL;
       addr.base.global.sym = src.v.global.sym;
       addr.base.global.addend = src.v.global.addend;
       return addr;
     case NATIVE_LOC_REG:
       addr.base_kind = NATIVE_ADDR_BASE_REG;
       addr.cls = NATIVE_REG_INT;
       addr.base.reg = src.v.reg;
       return addr;
     default:
       x64_asm_panic_at(a->base.c, loc, "unsupported memory asm operand");
   }
 }
 
 static Reg x64_asm_native_mem_base(X64NativeTarget* a, SrcLoc loc,
                                    NativeLoc src, u32* ntmp) {
   NativeAddr addr = x64_asm_loc_to_addr(a, loc, src);
   Reg dst;
   if (addr.base_kind == NATIVE_ADDR_BASE_REG && addr.offset == 0 &&
       addr.index_kind == NATIVE_ADDR_INDEX_NONE) {
     if ((addr.base.reg & 0xfu) != X64_TMP_INT &&
         (addr.base.reg & 0xfu) != X64_TMP_INT2)
       return (Reg)(addr.base.reg & 0xfu);
   }
   if (*ntmp >= 2u)
     x64_asm_panic_at(a->base.c, loc, "too many memory asm operands");
   dst = (*ntmp == 0u) ? (Reg)X64_TMP_INT : (Reg)X64_TMP_INT2;
   (*ntmp)++;
   x64_addr_to_base_reg(a, addr, dst);
   return dst;
 }
 
 static void x64_asm_native_panic(NativeTarget* t, SrcLoc loc,
                                  const char* msg) {
   x64_asm_panic_at(t->c, loc, msg);
 }
 
 static Reg x64_asm_native_mem_base_hook(NativeTarget* t, SrcLoc loc,
                                         NativeLoc src, u32* ntmp) {
   return x64_asm_native_mem_base(x64_of(t), loc, src, ntmp);
 }
 
 static void x64_asm_native_run_template_hook(
     NativeTarget* t, const char* tmpl, const AsmConstraint* outs, u32 nout,
     Operand* bound_outs, const AsmConstraint* ins, u32 nin, Operand* bound_ins,
     const Sym* clobbers, u32 nclob) {
   X64Asm* asmh = x64_asm_open(t->c);
   x64_inline_bind(asmh, outs, nout, bound_outs, ins, nin, bound_ins, clobbers,
                   nclob);
   x64_asm_run_template(asmh, t->mc, tmpl);
   x64_asm_close(asmh);
 }
 
 static void x64_asm_block_native(NativeTarget* t, const char* tmpl,
                                  const AsmConstraint* outs, u32 nout,
                                  NativeLoc* out_locs, const AsmConstraint* ins,
                                  u32 nin, const NativeLoc* in_locs,
                                  const Sym* clobbers, u32 nclob) {
   X64NativeTarget* a = x64_of(t);
   SrcLoc loc = a->func ? a->func->loc : (SrcLoc){0, 0, 0};
   static const NativeAsmNativeHooks hooks = {
       .panic = x64_asm_native_panic,
       .bound_reg = x64_asm_bound_reg,
       .bound_mem = x64_asm_bound_mem,
       .mem_base = x64_asm_native_mem_base_hook,
       .run_template = x64_asm_native_run_template_hook,
   };
   native_asm_bind_native_operands(t, loc, tmpl, outs, nout, out_locs, ins, nin,
                                   in_locs, clobbers, nclob, &hooks);
 }
 
 /* file_scope_asm + finalize are shared (cg/native_asm.h). */
 
 static void x64_trap(NativeTarget* t) { emit_ud2(t->mc); }
 static void x64_set_loc(NativeTarget* t, SrcLoc loc) {
   x64_of(t)->loc = loc;
   mc_set_loc(t->mc, loc);
 }
 
 static u32 x64_binop_visible_clobbers(const NativeMachineOp* op) {
   switch ((BinOp)op->binop) {
     case BO_SDIV:
     case BO_UDIV:
     case BO_SREM:
     case BO_UREM:
       return (1u << X64_RAX) | (1u << X64_RDX);
     case BO_SHL:
     case BO_SHR_S:
     case BO_SHR_U:
       return op->second_is_reg ? (1u << X64_RCX) : 0u;
     default:
       return 0u;
   }
 }
 
 static u32 x64_intrinsic_visible_clobbers(IntrinKind kind) {
   if (kind == INTRIN_UMUL_OVERFLOW || kind == INTRIN_SMUL_HIGH ||
       kind == INTRIN_UMUL_HIGH || kind == INTRIN_READCYCLECOUNTER)
     return (1u << X64_RAX) | (1u << X64_RDX);
   if (kind == INTRIN_SYSCALL)
     return (1u << X64_RAX) | (1u << X64_RCX) | (1u << X64_RDX) |
            (1u << X64_RSI) | (1u << X64_RDI) | (1u << X64_R8) |
            (1u << X64_R9);
   return 0u;
 }
 
 /* Physical registers each x86-64 instruction's encoding clobbers as a side
  * effect, so the optimizer keeps values live across them out of those registers
  * (the backend is then free to use them). idiv/div write rax (quotient) and rdx
  * (remainder/sign); a variable shift uses cl; cmpxchg/xadd loops use
  * rax/rcx/rdx; bitfield store uses the same trio for its read-modify-write.
  * All other ordinary helpers use declared results or backend-private R10/R11
  * and XMM14/XMM15 temporaries. */
 static int x64_machine_op_clobbers(NativeTarget* t, const NativeMachineOp* op,
                                    u32 mask[NATIVE_REG_CLASS_COUNT]) {
   mask[0] = mask[1] = mask[2] = 0;
   switch ((NativeMachineOpKind)op->kind) {
     case NATIVE_MOP_TLS_ADDR:
       /* ELF Local-Exec / Windows TEB materialize the address using only the
        * destination register (no fixed clobbers). The Mach-O TLV descriptor
        * sequence loads the descriptor into %rdi and calls its resolver thunk,
        * which returns the storage in %rax and preserves every other register;
        * model the %rdi/%rax clobbers so no value lives across the access in
        * them. */
       if (!obj_format_tls_via_descriptor(t->c)) return 0;
       mask[NATIVE_REG_INT] = (1u << X64_RAX) | (1u << X64_RDI);
       return 1;
     case NATIVE_MOP_BINOP:
       mask[NATIVE_REG_INT] = x64_binop_visible_clobbers(op);
       return mask[NATIVE_REG_INT] != 0u;
     case NATIVE_MOP_BITFIELD_LOAD:
       /* The only non-destination temporary is backend-private R11. */
       return 0;
     case NATIVE_MOP_BITFIELD_STORE:
       mask[NATIVE_REG_INT] =
           (1u << X64_RAX) | (1u << X64_RCX) | (1u << X64_RDX);
       return 1;
     case NATIVE_MOP_ATOMIC_CAS:
     case NATIVE_MOP_ATOMIC_RMW:
       mask[NATIVE_REG_INT] =
           (1u << X64_RAX) | (1u << X64_RCX) | (1u << X64_RDX);
       return 1;
     case NATIVE_MOP_VA_START:
     case NATIVE_MOP_VA_ARG:
       return 0;
     case NATIVE_MOP_INTRINSIC:
       /* The unsigned multiply-overflow intrinsic emits a one-operand MUL, whose
        * rdx:rax product clobbers both registers. The signed variant uses a
        * two-operand IMUL (no fixed-register clobber). Linux syscall writes rax
        * and the CPU instruction itself clobbers rcx/r11; the kernel ABI treats
        * the integer caller-saved syscall registers as volatile. */
       /* R8/R9 are cache-capable emitter temps, so the syscall effect includes
        * them. R10/R11 are pure backend-private temps and remain excluded. */
       mask[NATIVE_REG_INT] =
           x64_intrinsic_visible_clobbers((IntrinKind)op->intrin);
       return mask[NATIVE_REG_INT] != 0u;
     case NATIVE_MOP_COUNT:
       break;
   }
   x64_panic(x64_of(t), "invalid machine-effect operation");
   return 0;
 }
 
 /* ============================ construction ============================ */
 
 NativeTarget* x64_native_target_new(Compiler* c, ObjBuilder* obj,
                                     MCEmitter* mc) {
   X64NativeTarget* a = arena_znew(c->tu, X64NativeTarget);
   NativeTarget* t;
   if (!a) return NULL;
   t = &a->base;
   t->c = c;
   t->obj = obj;
   t->mc = mc;
   native_frame_init(&a->frame, c);
   t->regs = &x64_reg_info;
   t->class_for_type = native_class_for_type_fp_le8;
   t->imm_legal = x64_imm_legal;
   t->addr_legal = x64_addr_legal;
   t->machine_op_clobbers = x64_machine_op_clobbers;
   t->func_begin = x64_func_begin;
   t->func_begin_known_frame = x64_func_begin_known_frame;
   t->bind_params_end = x64_bind_params_end;
   t->note_frame_state = NULL;
   /* Non-NULL so the optimizer emit path (plan_frame) computes the callee-saved
    * set; x64_func_begin_known_frame derives the records from the masks. */
   t->reserve_callee_saves = x64_reserve_callee_saves;
   t->caller_saved_mask = x64_live_caller_saved_mask;
   t->callee_saved_mask = x64_live_callee_saved_mask;
   t->signature_stack_bytes = x64_signature_stack_bytes;
   t->call_stack_bytes = x64_call_stack_bytes;
   t->has_store_zero_reg = 0;
   t->func_end = x64_func_end;
   t->frame_slot = x64_frame_slot;
   t->release_frame_slot = x64_release_frame_slot;
   t->frame_slot_debug_loc = x64_frame_slot_debug_loc;
   t->bind_param = x64_bind_native_param;
   t->label_new = x64_label_new;
   t->label_place = x64_label_place;
   t->jump = x64_jump;
   t->cmp_branch = x64_cmp_branch;
   t->indirect_branch = x64_indirect_branch;
   t->load_label_addr = x64_load_label_addr;
   t->move = x64_move;
   t->load_imm = x64_load_imm;
   t->load_const = x64_load_const;
   t->load_addr = x64_load_addr;
   t->load = x64_load;
   t->store = x64_store;
   t->tls_addr_of = x64_tls_addr_of;
   t->copy_bytes = x64_copy_bytes;
   t->set_bytes = x64_set_bytes;
   t->bitfield_load = x64_bitfield_load;
   t->bitfield_store = x64_bitfield_store;
   t->binop = x64_binop;
   t->unop = x64_unop;
   t->cmp = x64_cmp;
   t->convert = x64_convert;
   t->binop_rr = x64_binop_rr;
   t->move_rr = x64_move_rr;
   t->cmp_rr = x64_cmp_rr;
   t->convert_rr = x64_convert_rr;
   t->alloca_ = x64_alloca;
   t->spill = x64_spill;
   t->reload = x64_reload;
   t->marshal_call = x64_marshal_call;
   t->emit_call = x64_emit_call;
   t->marshal_ret = x64_marshal_ret;
   t->ret = x64_ret;
   t->atomic_load = x64_atomic_load;
   t->atomic_store = x64_atomic_store;
   t->atomic_rmw = x64_atomic_rmw;
   t->atomic_cas = x64_atomic_cas;
   t->fence = x64_fence;
   t->va_start_ = x64_va_start_native;
   t->va_arg_ = x64_va_arg_native;
   t->va_end_ = x64_va_end_native;
   t->va_copy_ = x64_va_copy_native;
   t->intrinsic_arg_accepts_imm = x64_intrinsic_arg_accepts_imm;
   t->intrinsic = x64_intrinsic;
   t->asm_block = x64_asm_block_native;
   t->file_scope_asm = native_file_scope_asm;
   t->trap = x64_trap;
   t->set_loc = x64_set_loc;
   t->finalize = native_finalize;
   return t;
 }
 
 /* ============================ NativeOps (-O0) ============================ */
 
 static void x64_bind_param(NativeDirectTarget* d, const CGParamDesc* p,
                            CGLocal local, NativeDirectLocal* l) {
   NativeLoc dst;
   (void)local;
   memset(&dst, 0, sizeof dst);
   dst.kind = NATIVE_LOC_FRAME;
   dst.type = p->type;
   dst.v.frame = l->home;
   x64_bind_native_param(d->native, p, dst);
 }
 
 /* A sibling call is realizable when its outgoing stack-argument area fits the
  * window the caller itself received. Register-only calls always qualify. */
 static const char* x64_no_tail(NativeDirectTarget* d, const CGCallDesc* call) {
   X64NativeTarget* a = x64_of(d->native);
   NativeCallDesc nd;
   u32 stack;
   if (a->frame.ncallee_saves)
     return "x64 tail call: callee-saved registers in use";
   native_direct_project_tail_call_desc(d, call, &nd);
   stack = x64_tail_call_stack_size(d->native, &nd);
   /* The tail footprint includes the shadow-space prefix; the caller's incoming
    * window has the same prefix, so compare against incoming_stack_size + it. */
   if (stack > a->incoming_stack_size + a->abi->shadow_space)
     return "x64 tail call: stack argument area too small";
   return NULL;
 }
 
 /* Resolve a pointer-typed Operand (the address of a va_list object) into `reg`,
  * returning a register-based NativeAddr. */
 static NativeAddr x64_direct_pointer_addr(NativeDirectTarget* d, Operand op) {
   X64NativeTarget* a = x64_of(d->native);
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   if (op.kind == OPK_LOCAL) {
     emit_mov_load(a->base.mc, 8, 0, X64_R11, X64_RBP,
                   -(i32)x64_slot_get(a, d->locals[op.v.local - 1u].home)->off);
     addr.base_kind = NATIVE_ADDR_BASE_REG;
     addr.base.reg = X64_R11;
     addr.base_type = op.type;
     return addr;
   }
   return x64_direct_materialize_addr(d, op);
 }
 
 static NativeAddr x64_direct_va_base(NativeDirectTarget* d, Operand ap_addr,
                                      Reg reg) {
   NativeLoc dst =
       native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT, reg);
   NativeAddr addr;
   d->native->load_addr(d->native, dst, x64_direct_pointer_addr(d, ap_addr));
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_REG;
   addr.cls = NATIVE_REG_INT;
   addr.base.reg = reg;
   addr.base_type = builtin_id(KIT_CG_BUILTIN_I64);
   return addr;
 }
 
 static void x64_va_start_(NativeDirectTarget* d, Operand ap_addr) {
   /* Hold the va_list base in R11, not RAX: x64_va_start_core materializes the
    * gp/fp_offset and overflow/reg-save-area field values through RAX, which
    * would otherwise clobber the base before the field stores. */
   x64_va_start_core(x64_of(d->native), x64_direct_va_base(d, ap_addr, X64_R11));
 }
 static void x64_va_arg_(NativeDirectTarget* d, Operand dst, Operand ap_addr,
                         KitCgTypeId type) {
   X64NativeTarget* a = x64_of(d->native);
   int is_fp = cg_type_is_float(d->base.c, type);
   NativeLoc res = native_loc_reg(type, is_fp ? NATIVE_REG_FP : NATIVE_REG_INT,
                                  is_fp ? X64_TMP_FP : (Reg)X64_RDX);
   NativeAddr dst_addr;
   /* Base in R11: the core advances/loads through R11 plus one GPR scratch (the
    * integer result reg itself, or RAX for FP results), so R11 must not be RAX.
    */
   x64_va_arg_core(a, res, x64_direct_va_base(d, ap_addr, X64_R11), type);
   dst_addr = x64_direct_addr(d, dst);
   if (dst_addr.base_kind == NATIVE_ADDR_BASE_FRAME_VALUE) {
     emit_mov_load(a->base.mc, 8, 0, X64_R11, X64_RBP,
                   -(i32)x64_slot_get(a, dst_addr.base.frame)->off);
     dst_addr.base_kind = NATIVE_ADDR_BASE_REG;
     dst_addr.base.reg = X64_R11;
   }
   x64_emit_mem(
       a, 0, res, dst_addr,
       native_mem_for_type(d->native, type, native_type_size(d->native, type)));
 }
 static void x64_va_end_(NativeDirectTarget* d, Operand ap_addr) {
   (void)d;
   (void)ap_addr;
 }
 static void x64_va_copy_(NativeDirectTarget* d, Operand dst, Operand src) {
   X64NativeTarget* a = x64_of(d->native);
   NativeAddr src_ap = x64_direct_va_base(d, src, X64_RAX);
   NativeAddr dst_ap = x64_direct_va_base(d, dst, X64_R11);
   x64_va_copy_core(a, dst_ap, src_ap);
 }
 
 /* Hook adapters bridging the arch-typed save/restore + assembler entry to the
  * shared NativeAsmDirectHooks signatures. */
 static void* x64_asm_hook_save_callee_clobbers(NativeDirectTarget* d,
                                                u32 int_mask, u32 fp_mask,
                                                u32* nsaved_out) {
   return x64_asm_save_callee_clobbers(x64_of(d->native), int_mask, fp_mask,
                                       nsaved_out);
 }
 static void x64_asm_hook_restore_one(NativeDirectTarget* d, void* saved,
                                      u32 idx) {
   x64_asm_restore_one(x64_of(d->native), &((X64AsmSavedClobber*)saved)[idx]);
 }
 static void x64_asm_hook_run_template(NativeDirectTarget* d, const char* tmpl,
                                       const AsmConstraint* outs, u32 nout,
                                       Operand* bound_outs,
                                       const AsmConstraint* ins, u32 nin,
                                       Operand* bound_ins, const Sym* clobbers,
                                       u32 nclob) {
   X64Asm* asmh = x64_asm_open(d->base.c);
   x64_inline_bind(asmh, outs, nout, bound_outs, ins, nin, bound_ins, clobbers,
                   nclob);
   x64_asm_run_template(asmh, d->native->mc, tmpl);
   x64_asm_close(asmh);
 }
 
 static void x64_direct_asm_block(NativeDirectTarget* d, const char* tmpl,
                                  const AsmConstraint* outs, u32 nout,
                                  Operand* out_ops, const AsmConstraint* ins,
                                  u32 nin, const Operand* in_ops,
                                  const Sym* clobbers, u32 nclob,
                                  u32 clobber_abi_sets) {
   static const NativeAsmDirectHooks hooks = {
       .opk_reg = X64_INLINE_OPK_REG,
       .opcls_fp = X64_INLINE_OPCLS_FP,
       .panic = x64_asm_panic,
       .bound_reg = x64_asm_bound_reg,
       .bound_mem = x64_asm_bound_mem,
       .clobber_masks = x64_asm_clobber_masks,
       .save_callee_clobbers = x64_asm_hook_save_callee_clobbers,
       .restore_one = x64_asm_hook_restore_one,
       .load_operand_to_reg = x64_direct_load_operand_to_reg,
       .load_address_to_reg = x64_direct_load_address_to_reg,
       .store_reg_to_operand = x64_direct_store_reg_to_operand,
       .run_template = x64_asm_hook_run_template,
   };
   native_asm_bind_direct_operands(d, tmpl, outs, nout, out_ops, ins, nin,
                                   in_ops, clobbers, nclob, clobber_abi_sets,
                                   &hooks);
 }
 
 static const NativeOps x64_direct_ops = {
     .bind_param = x64_bind_param,
     .tail_call_unrealizable_reason = x64_no_tail,
     .va_start_ = x64_va_start_,
     .va_arg_ = x64_va_arg_,
     .va_end_ = x64_va_end_,
     .va_copy_ = x64_va_copy_,
     .asm_block = x64_direct_asm_block,
 };
 
 const NativeOps* x64_native_direct_ops(void) { return &x64_direct_ops; }