kit

native.c (171676B)

---

 /* src/arch/x64/native.c — x86-64 (SysV / Win64) NativeTarget implementation.
  *
  * Mirrors the rv64 reference (src/arch/rv64/native.c): a physical-emission
  * NativeTarget driven at -O0 by the shared NativeDirectTarget and at -O1+ by
  * the optimizer emit path. ABI decisions route through abi/ and the per-OS
  * X64ABIRegs (x64_abi_for_os); this file owns ISA emission and the x64 frame
  * layout.
  *
  * Frame model (single, rbp-anchored): the prologue does `push rbp; mov rbp,rsp;
  * sub rsp,frame_size`. Local/spill slots live below rbp at positive byte
  * offsets `off` (address = rbp - off). Incoming stack args sit above the saved
  * return address at [rbp + 16 + shadow_space + ...]. Callee-saved GPRs (and, on
  * Win64, XMMs) are saved below the locals; outgoing args sit at [rsp + 0..].
  * The single-pass (-O0) prologue reserves a NOP placeholder patched in func_end
  * once max_outgoing and callee-saves are known.
  *
  * Register model. INT scratch (never allocable, never driver scratch): R10 and
  * R11 — the emit paths' fixed temporaries. FP scratch: XMM14 and XMM15. RSP/RBP
  * are reserved (stack/frame pointers). RAX is reserved too (return value, the
  * div/mul implicit operand), but it is NOT an emit temp, so inline asm may pin
  * an operand to it (the Linux syscall idiom) — see x64_asm_operand_reg_ok.
  * The driver scratch pool is R8/R9 (int) and XMM4/XMM5 (fp), caller-saved on
  * both SysV and Win64 and disjoint from the emit temps so a hook never clobbers
  * an operand parked there. Scratch registers are reserved from allocation.
  * Callee-saved set is resolved per-OS via x64_abi_for_os at runtime (the
  * legality masks below are SysV's, the conservative superset that both ABIs'
  * allocators respect — Win64's extra callee-saves RDI/RSI/xmm6-15 only shrink
  * the allocable pool, never grow it). */
 
 #include <string.h>
 
 #include "abi/abi.h"
 #include "arch/x64/asm.h"
 #include "arch/x64/emit.h"
 #include "arch/x64/isa.h"
 #include "arch/x64/regs.h"
 #include "arch/x64/x64.h"
 #include "asm/asm.h"
 #include "asm/asm_lex.h"
 #include "cg/native_argmove.h"
 #include "cg/native_asm.h"
 #include "cg/native_direct_target.h"
 #include "cg/native_frame.h"
 #include "cg/type.h"
 #include "core/arena.h"
 #include "core/bytes.h"
 #include "core/pool.h"
 #include "core/slice.h"
 #include "obj/obj.h"
 
 enum {
   X64_TMP_INT = X64_R10,      /* emit-internal int scratch (reserved) */
   X64_TMP_INT2 = X64_R11,     /* emit-internal int scratch (reserved) */
   X64_TMP_FP = X64_XMM0 + 14, /* emit-internal fp scratch (reserved) */
   X64_TMP_FP2 = X64_XMM15,    /* emit-internal fp scratch (reserved) */
   X64_MAX_REG_ARG_MOVES = 16u,
   /* Deferred entry register-binds (-O1): bounded by simultaneously-live
    * register-homed param parts, i.e. the allocable register count. */
   X64_MAX_BIND_MOVES = 32u,
   X64_MAX_CS_FP_REGS = 10u, /* Win64 xmm6..xmm15 */
 };
 
 /* ============================ target state ============================ */
 
 /* Frame slots and callee-save records live in the shared NativeFrame
  * bookkeeping (cg/native_frame.h); these aliases keep the x64-local spellings.
  * x64 reads only .reg/.cls of a callee-save (it computes save offsets below the
  * locals rather than homing them in frame slots, so .slot/.type stay unused).
  */
 typedef NativeFrameSlotEntry X64NativeSlot;
 typedef NativeFrameCalleeSave X64CalleeSave;
 
 typedef enum X64PatchKind { X64_PATCH_ALLOCA } X64PatchKind;
 
 typedef struct X64Patch {
   u8 kind; /* X64PatchKind */
   u32 pos; /* byte offset of the disp32 to patch */
 } X64Patch;
 
 typedef struct X64NativeTarget {
   NativeTarget base;
   SrcLoc loc;
   const CGFuncDesc* func;
 
   /* Shared frame bookkeeping: slot table, cum_off, max_outgoing, callee-save
    * set, and the known_frame / has_alloca / frame_final flags. */
   NativeFrame frame;
   u32 frame_size_final;
 
   u32 incoming_stack_size; /* fixed-param stack bytes (tail-call check) */
   u32 next_param_int;
   u32 next_param_fp;
   u32 next_param_stack;
   u8 has_sret;
   u8 is_variadic;
   NativeFrameSlot sret_ptr_slot;
   NativeFrameSlot reg_save_slot; /* SysV variadic 176B __va_list_tag area */
 
   X64Patch* patches;
   u32 npatches;
   u32 patches_cap;
   u32 nalloca;
 
   u32 func_start;
   u32 prologue_pos;
   u32 prologue_nbytes;
   MCLabel epilogue_label;
 
   /* Known-frame (-O1) prologue cost-model tiers, settled in
    * x64_func_begin_known_frame; both 0 on the single-pass path (which can't
    * know the frame up front). Either one suppresses the `sub rsp` reservation;
    * the rbp frame record (push rbp; mov rbp,rsp) and every rbp-relative offset
    * stay unchanged, so the epilogue (`leave`), CFI (CFA = rbp+16), and debug
    * locs are identical to the fat shape. slim_frame   - empty frame (no
    * callee-saves/locals/outgoing/alloca): the `sub rsp` reserved nothing, so it
    * is simply dropped. Safe for non-leaves (push rbp keeps rsp 16-aligned for
    * calls, and nothing lives below rsp). SysV + Win64. redzone_leaf - SysV leaf
    * with a small frame (<= 128B, no alloca, no outgoing args):
    * locals/callee-saves stay at their rbp-relative offsets, which now land in
    * the 128-byte red zone instead of a reserved region. Leaf-only — a call
    * would clobber the red zone. */
   u8 slim_frame;
   u8 redzone_leaf;
 
   /* Optimizer (-O1) entry binds: register-destination param binds are deferred
    * here and resolved as a parallel copy in x64_bind_params_end, since the
    * allocator may rotate params across the incoming arg registers — a
    * permutation the naive per-param move order would clobber. */
   NativeArgMove bind_moves[X64_MAX_BIND_MOVES];
   u32 nbind_moves;
 
   const X64ABIRegs* abi;
 } X64NativeTarget;
 
 static X64NativeTarget* x64_of(NativeTarget* t) { return (X64NativeTarget*)t; }
 
 static _Noreturn void x64_panic(X64NativeTarget* a, const char* msg) {
   compiler_panic(a->base.c, a->loc, "x64 native target: %s", msg);
 }
 
 static X64NativeSlot* x64_slot_get(X64NativeTarget* a, NativeFrameSlot fs) {
   return native_frame_slot_at(&a->frame, fs);
 }
 
 static u32 align_up_u32(u32 v, u32 align) {
   u32 mask = align ? align - 1u : 0u;
   return (v + mask) & ~mask;
 }
 
 /* ============================ type helpers ============================ */
 
 /* Scalar size/align/mem/class/loc constructors are shared in native_target.h
  * (native_type_size, native_type_align, native_mem_for_type,
  * native_class_for_type_fp_le8, native_loc_reg, native_loc_stack,
  * native_loc_is_fp). loc_reg's mask is arch-specific and stays here. */
 
 /* A scalar value occupies a 64-bit register when it is pointer-sized or wider
  * (drives REX.W selection). */
 static int x64_is_64(NativeTarget* t, KitCgTypeId type) {
   return native_type_size(t, type) >= 8u || cg_type_is_ptr(t->c, type);
 }
 
 static u32 loc_reg(NativeLoc loc) { return loc.v.reg & 0xfu; }
 
 /* SSE scalar prefix: F2 (double / 8-byte) vs F3 (single / 4-byte). */
 static u8 sse_scalar_prefix(u32 size) { return size == 8u ? 0xF2u : 0xF3u; }
 
 /* Forward decls for the rel32 branch emitters (used by convert before the
  * control-flow section defines them). */
 static void emit_jmp_rel32(MCEmitter* mc, MCLabel l);
 static void emit_jcc_rel32(MCEmitter* mc, u32 cc, MCLabel l);
 
 /* ============================ register tables ============================ */
 
 #define X64_PHYS_INT_ARG(r)                                                  \
   {.reg = (r),                                                               \
    .cls = NATIVE_REG_INT,                                                    \
    .abi_index = 0xffu,                                                       \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLER_SAVED | NATIVE_REG_ARG, \
    .spill_cost = 1u,                                                         \
    .copy_cost = 1u}
 #define X64_PHYS_INT_ARG_RESERVED(r)                                        \
   {.reg = (r),                                                              \
    .cls = NATIVE_REG_INT,                                                   \
    .abi_index = 0xffu,                                                      \
    .flags = NATIVE_REG_CALLER_SAVED | NATIVE_REG_ARG | NATIVE_REG_RESERVED, \
    .spill_cost = 0u,                                                        \
    .copy_cost = 0u}
 #define X64_PHYS_INT_RET_ARG(r)                                               \
   {.reg = (r),                                                                \
    .cls = NATIVE_REG_INT,                                                     \
    .abi_index = 0xffu,                                                        \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLER_SAVED | NATIVE_REG_ARG | \
             NATIVE_REG_RET,                                                   \
    .spill_cost = 1u,                                                          \
    .copy_cost = 1u}
 #define X64_PHYS_INT_CALLER(r)                              \
   {.reg = (r),                                              \
    .cls = NATIVE_REG_INT,                                   \
    .abi_index = 0xffu,                                      \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLER_SAVED, \
    .spill_cost = 1u,                                        \
    .copy_cost = 1u}
 #define X64_PHYS_INT_CALLEE(r)                              \
   {.reg = (r),                                              \
    .cls = NATIVE_REG_INT,                                   \
    .abi_index = 0xffu,                                      \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLEE_SAVED, \
    .spill_cost = 4u,                                        \
    .copy_cost = 1u}
 #define X64_PHYS_INT_RESERVED(r) \
   {.reg = (r),                   \
    .cls = NATIVE_REG_INT,        \
    .abi_index = 0xffu,           \
    .flags = NATIVE_REG_RESERVED, \
    .spill_cost = 0u,             \
    .copy_cost = 0u}
 
 /* Allocable int pool, opt's spill/reload set. R8/R9 are the driver scratch
  * pool; R10/R11 are emit scratch (reserved); RAX is reserved (return / div-mul,
  * asm-pinnable). */
 static const Reg x64_int_allocable[] = {X64_R13, X64_R14, X64_R15};
 static const Reg x64_int_scratch[] = {X64_R8, X64_R9};
 
 static const NativePhysRegInfo x64_int_phys[] = {
     X64_PHYS_INT_RESERVED(X64_RAX), /* return / div-mul (asm-pinnable) */
     X64_PHYS_INT_ARG(X64_RCX),
     X64_PHYS_INT_RET_ARG(X64_RDX),
     X64_PHYS_INT_RESERVED(X64_RBX),
     X64_PHYS_INT_RESERVED(X64_RSP), /* stack pointer */
     X64_PHYS_INT_RESERVED(X64_RBP), /* frame pointer */
     X64_PHYS_INT_ARG(X64_RSI),
     X64_PHYS_INT_ARG(X64_RDI),
     X64_PHYS_INT_ARG_RESERVED(X64_R8), /* driver scratch */
     X64_PHYS_INT_ARG_RESERVED(X64_R9), /* driver scratch */
     X64_PHYS_INT_RESERVED(X64_R10),    /* emit scratch */
     X64_PHYS_INT_RESERVED(X64_R11),    /* emit scratch */
     X64_PHYS_INT_RESERVED(X64_R12),
     X64_PHYS_INT_CALLEE(X64_R13),
     X64_PHYS_INT_CALLEE(X64_R14),
     X64_PHYS_INT_CALLEE(X64_R15),
 };
 
 #define X64_PHYS_FP_ARG_RET(r)                                                \
   {.reg = (r),                                                                \
    .cls = NATIVE_REG_FP,                                                      \
    .abi_index = 0xffu,                                                        \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLER_SAVED | NATIVE_REG_ARG | \
             NATIVE_REG_RET,                                                   \
    .spill_cost = 1u,                                                          \
    .copy_cost = 1u}
 #define X64_PHYS_FP_ARG(r)                                                   \
   {.reg = (r),                                                               \
    .cls = NATIVE_REG_FP,                                                     \
    .abi_index = 0xffu,                                                       \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLER_SAVED | NATIVE_REG_ARG, \
    .spill_cost = 1u,                                                         \
    .copy_cost = 1u}
 #define X64_PHYS_FP_ARG_RESERVED(r)                                         \
   {.reg = (r),                                                              \
    .cls = NATIVE_REG_FP,                                                    \
    .abi_index = 0xffu,                                                      \
    .flags = NATIVE_REG_CALLER_SAVED | NATIVE_REG_ARG | NATIVE_REG_RESERVED, \
    .spill_cost = 0u,                                                        \
    .copy_cost = 0u}
 #define X64_PHYS_FP_CALLER(r)                               \
   {.reg = (r),                                              \
    .cls = NATIVE_REG_FP,                                    \
    .abi_index = 0xffu,                                      \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLER_SAVED, \
    .spill_cost = 1u,                                        \
    .copy_cost = 1u}
 #define X64_PHYS_FP_RESERVED(r)  \
   {.reg = (r),                   \
    .cls = NATIVE_REG_FP,         \
    .abi_index = 0xffu,           \
    .flags = NATIVE_REG_RESERVED, \
    .spill_cost = 0u,             \
    .copy_cost = 0u}
 
 /* Allocable FP pool: xmm6..xmm11 (keep arg/ret xmm0..5 clear). xmm4/xmm5 are
  * driver scratch; xmm14/xmm15 are emit scratch. */
 static const Reg x64_fp_allocable[] = {
     X64_XMM6, X64_XMM7, X64_XMM8, X64_XMM0 + 9, X64_XMM0 + 10, X64_XMM0 + 11};
 static const Reg x64_fp_scratch[] = {X64_XMM4, X64_XMM5};
 
 static const NativePhysRegInfo x64_fp_phys[] = {
     X64_PHYS_FP_ARG_RET(X64_XMM0),       X64_PHYS_FP_ARG_RET(X64_XMM1),
     X64_PHYS_FP_ARG(X64_XMM2),           X64_PHYS_FP_ARG(X64_XMM3),
     X64_PHYS_FP_ARG_RESERVED(X64_XMM4),  X64_PHYS_FP_ARG_RESERVED(X64_XMM5),
     X64_PHYS_FP_CALLER(X64_XMM6),        X64_PHYS_FP_CALLER(X64_XMM7),
     X64_PHYS_FP_CALLER(X64_XMM8),        X64_PHYS_FP_CALLER(X64_XMM0 + 9),
     X64_PHYS_FP_CALLER(X64_XMM0 + 10),   X64_PHYS_FP_CALLER(X64_XMM0 + 11),
     X64_PHYS_FP_RESERVED(X64_XMM0 + 12), X64_PHYS_FP_RESERVED(X64_XMM0 + 13),
     X64_PHYS_FP_RESERVED(X64_XMM0 + 14), /* emit scratch */
     X64_PHYS_FP_RESERVED(X64_XMM15),     /* emit scratch */
 };
 
 static const NativeAllocClassInfo x64_classes[] = {
     {.cls = NATIVE_REG_INT,
      .allocable = x64_int_allocable,
      .nallocable = sizeof x64_int_allocable / sizeof x64_int_allocable[0],
      .scratch = x64_int_scratch,
      .nscratch = sizeof x64_int_scratch / sizeof x64_int_scratch[0],
      .phys = x64_int_phys,
      .nphys = sizeof x64_int_phys / sizeof x64_int_phys[0],
      /* caller-saved: rax,rcx,rdx,rsi,rdi,r8,r9,r10,r11 (SysV) */
      .caller_saved_mask = (1u << X64_RAX) | (1u << X64_RCX) | (1u << X64_RDX) |
                           (1u << X64_RSI) | (1u << X64_RDI) | (1u << X64_R8) |
                           (1u << X64_R9) | (1u << X64_R10) | (1u << X64_R11),
      /* callee-saved: rbx,r12,r13,r14,r15 (rbp handled by prologue head) */
      .callee_saved_mask = (1u << X64_RBX) | (1u << X64_R12) | (1u << X64_R13) |
                           (1u << X64_R14) | (1u << X64_R15),
      /* SysV arg regs rdi,rsi,rdx,rcx,r8,r9 */
      .arg_mask = (1u << X64_RDI) | (1u << X64_RSI) | (1u << X64_RDX) |
                  (1u << X64_RCX) | (1u << X64_R8) | (1u << X64_R9),
      .ret_mask = (1u << X64_RAX) | (1u << X64_RDX),
      /* rax, rsp, rbp reserved; r8/r9 driver scratch; r10/r11 emit scratch */
      .reserved_mask = (1u << X64_RAX) | (1u << X64_RSP) | (1u << X64_RBP) |
                       (1u << X64_R8) | (1u << X64_R9) | (1u << X64_R10) |
                       (1u << X64_R11) | (1u << X64_RBX) | (1u << X64_R12)},
     {.cls = NATIVE_REG_FP,
      .allocable = x64_fp_allocable,
      .nallocable = sizeof x64_fp_allocable / sizeof x64_fp_allocable[0],
      .scratch = x64_fp_scratch,
      .nscratch = sizeof x64_fp_scratch / sizeof x64_fp_scratch[0],
      .phys = x64_fp_phys,
      .nphys = sizeof x64_fp_phys / sizeof x64_fp_phys[0],
      /* All xmm caller-saved on SysV. */
      .caller_saved_mask = 0xffffu,
      .callee_saved_mask = 0u,
      .arg_mask = 0xffu, /* xmm0..xmm7 */
      .ret_mask = (1u << X64_XMM0) | (1u << X64_XMM1),
      /* xmm4/xmm5 driver scratch; xmm14/xmm15 emit scratch. */
      .reserved_mask = (1u << X64_XMM4) | (1u << X64_XMM5) |
                       (1u << (X64_XMM0 + 12)) | (1u << (X64_XMM0 + 13)) |
                       (1u << (X64_XMM0 + 14)) | (1u << X64_XMM15)},
 };
 
 /* Resolve a register name ("r10", "xmm3", ...) to its (class, Reg). Powers the
  * optimizer's inline-asm clobber masks and explicit hard-register operands
  * ("{r10}" from a GNU local register variable). GPR names map through the HW
  * encoding; xmm names through the DWARF index table. Returns non-zero for a
  * non-register name (cc/memory/unknown), which the caller skips. */
 static int x64_resolve_name(const NativeRegInfo* ri, Slice name, Reg* out,
                             NativeAllocClass* cls_out) {
   char buf[16];
   uint32_t idx;
   (void)ri;
   if (!name.s || !name.len || name.len >= sizeof buf) return 1;
   memcpy(buf, name.s, name.len);
   buf[name.len] = '\0';
   if (x64_register_hw_index(buf, &idx) == 0 && idx <= 15u) {
     *cls_out = NATIVE_REG_INT;
     *out = (Reg)idx;
     return 0;
   }
   if (x64_register_index(buf, &idx) == 0 && idx >= 17u && idx <= 32u) {
     *cls_out = NATIVE_REG_FP;
     *out = (Reg)(idx - 17u);
     return 0;
   }
   return 1;
 }
 
 static int x64_asm_operand_reg_ok(const NativeRegInfo* ri, NativeAllocClass cls,
                                   Reg reg) {
   (void)ri;
   if (cls == NATIVE_REG_INT) {
     switch (reg) {
       /* RAX is reserved but not an emit temp, so it is a legal asm pin (the
        * Linux syscall number/return register). R8/R9 are driver scratch and
        * R10/R11 are emit scratch, so those stay excluded. */
       case X64_RAX:
       case X64_RBX:
       case X64_RCX:
       case X64_RDX:
       case X64_RSI:
       case X64_RDI:
       case X64_R12:
       case X64_R13:
       case X64_R14:
       case X64_R15:
         return 1;
       default:
         return 0;
     }
   }
   if (cls == NATIVE_REG_FP)
     return reg <= X64_XMM0 + 13u && reg != X64_XMM4 && reg != X64_XMM5;
   return 0;
 }
 
 static int x64_asm_constraint_reg(const NativeRegInfo* ri, const char* body,
                                   NativeAllocClass* cls_out, Reg* fixed_out,
                                   u32* allowed_mask_out) {
   (void)ri;
   if (!body || !body[0] || body[1]) return 0;
   if (fixed_out) *fixed_out = REG_NONE;
   if (allowed_mask_out) *allowed_mask_out = 0;
   switch (body[0]) {
     case 'r':
     case 'q':
       if (cls_out) *cls_out = NATIVE_REG_INT;
       return 1;
     case 'a':
       if (cls_out) *cls_out = NATIVE_REG_INT;
       if (fixed_out) *fixed_out = X64_RAX;
       return 1;
     case 'b':
       if (cls_out) *cls_out = NATIVE_REG_INT;
       if (fixed_out) *fixed_out = X64_RBX;
       return 1;
     case 'c':
       if (cls_out) *cls_out = NATIVE_REG_INT;
       if (fixed_out) *fixed_out = X64_RCX;
       return 1;
     case 'd':
       if (cls_out) *cls_out = NATIVE_REG_INT;
       if (fixed_out) *fixed_out = X64_RDX;
       return 1;
     case 'S':
       if (cls_out) *cls_out = NATIVE_REG_INT;
       if (fixed_out) *fixed_out = X64_RSI;
       return 1;
     case 'D':
       if (cls_out) *cls_out = NATIVE_REG_INT;
       if (fixed_out) *fixed_out = X64_RDI;
       return 1;
     case 'x':
     case 'v':
       if (cls_out) *cls_out = NATIVE_REG_FP;
       return 1;
     default:
       return 0;
   }
 }
 
 static const NativeRegInfo x64_reg_info = {
     .classes = x64_classes,
     .nclasses = sizeof x64_classes / sizeof x64_classes[0],
     .resolve_name = x64_resolve_name,
     .asm_operand_reg_ok = x64_asm_operand_reg_ok,
     .asm_constraint_reg = x64_asm_constraint_reg,
 };
 
 /* ============================ legality ============================ */
 
 static int x64_imm_legal(NativeTarget* t, NativeImmUse use, u32 op,
                          KitCgTypeId type, i64 imm) {
   (void)t;
   (void)type;
   switch (use) {
     case NATIVE_IMM_MOVE:
       return 1;
     case NATIVE_IMM_BINOP:
       switch ((BinOp)op) {
         case BO_IADD:
         case BO_ISUB:
         case BO_AND:
         case BO_OR:
         case BO_XOR:
         case BO_IMUL:
           return imm_fits_i32(imm);
         case BO_SHL:
         case BO_SHR_S:
         case BO_SHR_U:
           return imm >= 0 && imm <= 63;
         default:
           return 0;
       }
     case NATIVE_IMM_CMP:
       return imm_fits_i32(imm);
     case NATIVE_IMM_ADDR_OFFSET:
       return imm_fits_i32(imm);
   }
   return 0;
 }
 
 static int x64_addr_legal(NativeTarget* t, const NativeAddr* addr,
                           MemAccess mem) {
   (void)t;
   (void)mem;
   if (!addr) return 0;
   if (addr->base_kind != NATIVE_ADDR_BASE_REG &&
       addr->base_kind != NATIVE_ADDR_BASE_FRAME)
     return 0;
   /* x64 supports [base + index*scale + disp32]; index must be a register. */
   if (addr->index_kind != NATIVE_ADDR_INDEX_NONE &&
       addr->index_kind != NATIVE_ADDR_INDEX_REG)
     return 0;
   return imm_fits_i32(addr->offset);
 }
 
 /* ============================ globals / addresses ============================
  */
 
 static int x64_use_got_for_sym(NativeTarget* t, ObjSymId sym) {
   return obj_symbol_extern_via_got(t->c, t->obj, sym);
 }
 
 /* PC-relative reloc kind for a non-GOT &sym reference. Functions use PLT32 so
  * the linker can route through a PLT; data uses plain PC32. */
 static u32 x64_pcrel_reloc_for_sym(NativeTarget* t, ObjSymId sym) {
   const ObjSym* s = obj_symbol_get(t->obj, sym);
   if (s && (s->kind == SK_FUNC || s->kind == SK_IFUNC)) return R_X64_PLT32;
   return R_PC32;
 }
 
 /* Materialize &sym + addend into dst_reg. Local/static-link symbols use
  * `lea rd, [rip + disp32]`; GOT-routed externs use `mov rd, [rip + GOT]` then
  * add any nonzero addend. */
 static void x64_emit_global_lea(NativeTarget* t, u32 dst_reg, ObjSymId sym,
                                 i64 addend) {
   MCEmitter* mc = t->mc;
   u32 sec = mc->section_id;
   if (x64_use_got_for_sym(t, sym)) {
     u8 op;
     u32 disp_pos;
     emit_rex(mc, 1, dst_reg, 0, 0);
     op = X64_OPC_MOV_R_RM;
     mc->emit_bytes(mc, &op, 1);
     {
       u8 mr = modrm(0u, dst_reg & 7u, 5u); /* [rip + disp32] */
       mc->emit_bytes(mc, &mr, 1);
     }
     disp_pos = mc->pos(mc);
     emit_u32le(mc, 0);
     mc->emit_reloc_at(mc, sec, disp_pos, R_X64_REX_GOTPCRELX, sym, -4, 1, 0);
     if (addend) {
       i32 a = (i32)addend;
       emit_rex(mc, 1, 0, 0, dst_reg);
       if (imm_fits_i8(a)) {
         u8 buf[3] = {X64_OPC_ALU_IMM8, modrm(3u, X64_ALU_SUB_ADD, dst_reg & 7u),
                      (u8)a};
         mc->emit_bytes(mc, buf, 3);
       } else {
         u8 buf[2] = {X64_OPC_ALU_IMM32,
                      modrm(3u, X64_ALU_SUB_ADD, dst_reg & 7u)};
         mc->emit_bytes(mc, buf, 2);
         emit_u32le(mc, (u32)a);
       }
     }
     return;
   }
   {
     u8 op = X64_OPC_LEA;
     u32 disp_pos;
     emit_rex(mc, 1, dst_reg, 0, 0);
     mc->emit_bytes(mc, &op, 1);
     {
       u8 mr = modrm(0u, dst_reg & 7u, 5u); /* [rip + disp32] */
       mc->emit_bytes(mc, &mr, 1);
     }
     disp_pos = mc->pos(mc);
     emit_u32le(mc, 0);
     mc->emit_reloc_at(mc, sec, disp_pos, x64_pcrel_reloc_for_sym(t, sym), sym,
                       addend - 4, 1, 0);
   }
 }
 
 /* Resolve a NativeAddr to (base, index, log2_scale, off). Materializes
  * FRAME/FRAME_VALUE/GLOBAL bases into the supplied scratch register. */
 static u32 x64_resolve_addr(X64NativeTarget* a, const NativeAddr* addr,
                             u32 scratch, u32* idx_out, u32* scale_out,
                             i32* off_out) {
   NativeTarget* t = &a->base;
   u32 base;
   i32 off;
   switch (addr->base_kind) {
     case NATIVE_ADDR_BASE_REG:
       base = addr->base.reg & 0xfu;
       off = addr->offset;
       break;
     case NATIVE_ADDR_BASE_FRAME: {
       X64NativeSlot* s = x64_slot_get(a, addr->base.frame);
       base = X64_RBP;
       off = -(i32)s->off + addr->offset;
       break;
     }
     case NATIVE_ADDR_BASE_FRAME_VALUE: {
       X64NativeSlot* s = x64_slot_get(a, addr->base.frame);
       emit_mov_load(t->mc, 8, 0, scratch, X64_RBP, -(i32)s->off);
       base = scratch;
       off = addr->offset;
       break;
     }
     case NATIVE_ADDR_BASE_GLOBAL:
       x64_emit_global_lea(t, scratch, addr->base.global.sym,
                           addr->base.global.addend);
       base = scratch;
       off = addr->offset;
       break;
     default:
       x64_panic(a, "unsupported address base");
   }
   if (addr->index_kind == NATIVE_ADDR_INDEX_REG) {
     *idx_out = addr->index.reg & 0xfu;
     *scale_out = addr->log2_scale;
   } else if (addr->index_kind == NATIVE_ADDR_INDEX_FRAME_VALUE) {
     X64NativeSlot* s = x64_slot_get(a, addr->index.frame);
     emit_mov_load(t->mc, 8, 0, X64_TMP_INT2, X64_RBP, -(i32)s->off);
     *idx_out = X64_TMP_INT2;
     *scale_out = addr->log2_scale;
   } else {
     *idx_out = REG_NONE;
     *scale_out = 0;
   }
   *off_out = off;
   return base;
 }
 
 /* ============================ memory ============================ */
 
 /* Central load/store primitive. is_load: 1 load into reg, 0 store reg to mem.
  * Materializes the address through X64_TMP_INT2 (r11) for non-reg bases. */
 static void x64_emit_mem(X64NativeTarget* a, int is_load, NativeLoc reg,
                          NativeAddr addr, MemAccess mem) {
   NativeTarget* t = &a->base;
   MCEmitter* mc = t->mc;
   u32 r = loc_reg(reg);
   int fp = native_loc_is_fp(reg);
   u32 sz = mem.size ? mem.size : native_type_size(t, reg.type);
   u32 base, idx, scale;
   i32 off;
 
   /* Global base: fold into a single rip-relative access when local. */
   if (addr.base_kind == NATIVE_ADDR_BASE_GLOBAL &&
       addr.index_kind == NATIVE_ADDR_INDEX_NONE &&
       !x64_use_got_for_sym(t, addr.base.global.sym)) {
     ObjSymId sym = addr.base.global.sym;
     i64 ad = addr.base.global.addend + addr.offset;
     u32 sec = mc->section_id;
     u32 disp_pos;
     if (fp) {
       u8 prefix = sse_scalar_prefix(sz);
       mc->emit_bytes(mc, &prefix, 1);
       emit_rex(mc, 0, r, 0, 0);
       {
         u8 op2[2] = {X64_OPC_TWOBYTE, (u8)(is_load ? 0x10u : 0x11u)};
         mc->emit_bytes(mc, op2, 2);
       }
     } else if (sz == 8 || sz == 4) {
       emit_rex(mc, sz == 8, r, 0, 0);
       {
         u8 op = is_load ? X64_OPC_MOV_R_RM : X64_OPC_MOV_RM_R;
         mc->emit_bytes(mc, &op, 1);
       }
     } else if (sz == 2) {
       if (is_load) {
         emit_rex(mc, 0, r, 0, 0);
         {
           u8 op2[2] = {X64_OPC_TWOBYTE, X64_OPC_MOVZX_W};
           mc->emit_bytes(mc, op2, 2);
         }
       } else {
         u8 p = X64_OPSIZE_PFX;
         mc->emit_bytes(mc, &p, 1);
         emit_rex(mc, 0, r, 0, 0);
         {
           u8 op = X64_OPC_MOV_RM_R;
           mc->emit_bytes(mc, &op, 1);
         }
       }
     } else { /* size 1 */
       if (is_load) {
         emit_rex(mc, 0, r, 0, 0);
         {
           u8 op2[2] = {X64_OPC_TWOBYTE, X64_OPC_MOVZX_B};
           mc->emit_bytes(mc, op2, 2);
         }
       } else {
         emit_rex_force(mc, 0, r, 0, 0);
         {
           u8 op = X64_OPC_MOV_RM_R8;
           mc->emit_bytes(mc, &op, 1);
         }
       }
     }
     {
       u8 mr = modrm(0u, r & 7u, 5u);
       mc->emit_bytes(mc, &mr, 1);
     }
     disp_pos = mc->pos(mc);
     emit_u32le(mc, 0);
     mc->emit_reloc_at(mc, sec, disp_pos, x64_pcrel_reloc_for_sym(t, sym), sym,
                       ad - 4, 1, 0);
     return;
   }
 
   base = x64_resolve_addr(a, &addr, X64_TMP_INT2, &idx, &scale, &off);
   if (fp) {
     u8 prefix = sse_scalar_prefix(sz);
     if (is_load)
       emit_sse_load_idx(mc, prefix, 0x10, r, base, idx, scale, off);
     else
       emit_sse_store_idx(mc, prefix, 0x11, r, base, idx, scale, off);
   } else if (is_load) {
     /* Loads narrower than 4 bytes zero-extend (sign-extension is applied by a
      * later CV_SEXT). */
     emit_mov_load_idx(mc, sz, 0, r, base, idx, scale, off);
   } else {
     emit_mov_store_idx(mc, sz, r, base, idx, scale, off);
   }
 }
 
 /* ============================ moves / data ============================ */
 
 static void x64_move(NativeTarget* t, NativeLoc dst, NativeLoc src) {
   MCEmitter* mc = t->mc;
   int dfp = native_loc_is_fp(dst), sfp = native_loc_is_fp(src);
   u32 rd = loc_reg(dst), rs = loc_reg(src);
   if (dfp && sfp) {
     if (rd == rs) return;
     emit_sse_rr(mc, sse_scalar_prefix(native_type_size(t, dst.type)), 0x10, rd,
                 rs);
     return;
   }
   if (dfp && !sfp) { /* movd/movq gpr -> xmm: 66 0F 6E /r */
     int w = native_type_size(t, dst.type) == 8u;
     emit_sse_rr_w(mc, 0x66, 0x6E, w, rd, rs);
     return;
   }
   if (!dfp && sfp) { /* movd/movq xmm -> gpr: 66 0F 7E /r (xmm is reg field) */
     int w = native_type_size(t, src.type) == 8u;
     emit_sse_rr_w(mc, 0x66, 0x7E, w, rs, rd);
     return;
   }
   if (rd == rs) return;
   emit_mov_rr(mc, x64_is_64(t, dst.type) ? 1 : 0, rd, rs);
 }
 
 static void x64_load_imm(NativeTarget* t, NativeLoc dst, i64 imm) {
   x64_emit_load_imm(t->mc, x64_is_64(t, dst.type) ? 1 : 0, loc_reg(dst), imm);
 }
 
 /* FP constant: materialize the bit pattern in a GPR scratch, then movd/movq
  * into the FPR. Integer constant: plain load_imm. */
 static void x64_load_const(NativeTarget* t, NativeLoc dst, ConstBytes cb) {
   u64 v = 0;
   u32 i;
   for (i = 0; i < cb.size && i < 8u; ++i) v |= (u64)cb.bytes[i] << (i * 8u);
   if (!native_loc_is_fp(dst)) {
     x64_load_imm(t, dst, (i64)v);
     return;
   }
   x64_emit_load_imm(t->mc, cb.size == 8u, X64_TMP_INT, (i64)v);
   emit_sse_rr_w(t->mc, 0x66, 0x6E, cb.size == 8u, loc_reg(dst), X64_TMP_INT);
 }
 
 static void x64_load_addr(NativeTarget* t, NativeLoc dst, NativeAddr addr) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst);
   u32 base, idx, scale;
   i32 off;
   if (addr.base_kind == NATIVE_ADDR_BASE_GLOBAL &&
       addr.index_kind == NATIVE_ADDR_INDEX_NONE) {
     x64_emit_global_lea(t, rd, addr.base.global.sym,
                         addr.base.global.addend + addr.offset);
     return;
   }
   base = x64_resolve_addr(a, &addr, rd, &idx, &scale, &off);
   if (idx == REG_NONE) {
     if (base == rd && off == 0) return; /* already &slot in rd */
     emit_lea(mc, rd, base, off);
     return;
   }
   /* lea rd, [base + idx*scale + off] */
   {
     u8 buf[16];
     u32 n = 0;
     n += x64_pack_rex(buf + n, 1, rd, idx, base);
     buf[n++] = X64_OPC_LEA;
     n += x64_pack_mem_sib(buf + n, rd, base, idx, scale, off);
     mc->emit_bytes(mc, buf, n);
   }
 }
 
 static void x64_load(NativeTarget* t, NativeLoc dst, NativeAddr addr,
                      MemAccess mem) {
   x64_emit_mem(x64_of(t), 1, dst, addr, mem);
 }
 static void x64_store(NativeTarget* t, NativeAddr addr, NativeLoc src,
                       MemAccess mem) {
   x64_emit_mem(x64_of(t), 0, src, addr, mem);
 }
 
 /* Resolve an addressable NativeAddr to a bare base register (no index, off 0)
  * by emitting an lea into `scratch` when needed. */
 static u32 x64_addr_to_base_reg(X64NativeTarget* a, NativeAddr addr,
                                 u32 scratch) {
   MCEmitter* mc = a->base.mc;
   u32 base, idx, scale;
   i32 off;
   if (addr.base_kind == NATIVE_ADDR_BASE_GLOBAL &&
       addr.index_kind == NATIVE_ADDR_INDEX_NONE) {
     x64_emit_global_lea(&a->base, scratch, addr.base.global.sym,
                         addr.base.global.addend + addr.offset);
     return scratch;
   }
   base = x64_resolve_addr(a, &addr, scratch, &idx, &scale, &off);
   if (idx == REG_NONE && off == 0) return base;
   if (idx == REG_NONE) {
     emit_lea(mc, scratch, base, off);
     return scratch;
   }
   {
     u8 buf[16];
     u32 n = 0;
     n += x64_pack_rex(buf + n, 1, scratch, idx, base);
     buf[n++] = X64_OPC_LEA;
     n += x64_pack_mem_sib(buf + n, scratch, base, idx, scale, off);
     mc->emit_bytes(mc, buf, n);
   }
   return scratch;
 }
 
 /* copy_bytes: resolve dst into r11 and src into rax (both bare pointers), then
  * unrolled granule copy through rdx. dst is resolved first (its base may live
  * in r11 from a FRAME_VALUE load) and src second so the two never alias. */
 static void x64_copy_bytes(NativeTarget* t, NativeAddr dst, NativeAddr src,
                            AggregateAccess access) {
   X64NativeTarget* a = x64_of(t);
   /* Copy chunk by chunk (8/4/2/1) through the value scratch rax, letting
    * x64_emit_mem resolve each address with its own scratch (r11). Uses only the
    * reserved emit scratch (rax/r11) — no ad-hoc allocable temp (previously
    * rdx), which the optimizer may have live across the copy. */
   KitCgTypeId tys[4];
   u32 n = access.size, i = 0;
   tys[0] = builtin_id(KIT_CG_BUILTIN_I64);
   tys[1] = builtin_id(KIT_CG_BUILTIN_I32);
   tys[2] = builtin_id(KIT_CG_BUILTIN_I16);
   tys[3] = builtin_id(KIT_CG_BUILTIN_I8);
   while (i < n) {
     u32 rem = n - i, s;
     KitCgTypeId ty;
     NativeAddr sa = src, da = dst;
     NativeLoc val;
     MemAccess mem;
     if (rem >= 8u) {
       s = 8u;
       ty = tys[0];
     } else if (rem >= 4u) {
       s = 4u;
       ty = tys[1];
     } else if (rem >= 2u) {
       s = 2u;
       ty = tys[2];
     } else {
       s = 1u;
       ty = tys[3];
     }
     sa.offset += (i32)i;
     sa.base_type = ty;
     da.offset += (i32)i;
     da.base_type = ty;
     val = native_loc_reg(ty, NATIVE_REG_INT, X64_TMP_INT);
     memset(&mem, 0, sizeof mem);
     mem.type = ty;
     mem.size = s;
     mem.align = s;
     x64_emit_mem(a, 1, val, sa, mem); /* rax = [src + i] */
     x64_emit_mem(a, 0, val, da, mem); /* [dst + i] = rax */
     i += s;
   }
 }
 
 static void x64_set_bytes(NativeTarget* t, NativeAddr dst, NativeLoc byte_value,
                           AggregateAccess access) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   u32 dr = x64_addr_to_base_reg(a, dst, X64_TMP_INT2);
   u32 n = access.size, i = 0;
   /* Broadcast the byte across 8 bytes into rax. */
   if (byte_value.kind == NATIVE_LOC_IMM) {
     u8 b = (u8)(byte_value.v.imm & 0xffu);
     u64 b64 = b;
     b64 |= b64 << 8;
     b64 |= b64 << 16;
     b64 |= b64 << 32;
     x64_emit_load_imm(mc, 1, X64_RAX, (i64)b64);
   } else {
     /* Replicate the low byte of a register via multiply by 0x0101..01. */
     x64_emit_load_imm(mc, 1, X64_R11, (i64)0x0101010101010101ll);
     emit_mov_rr(mc, 1, X64_RAX, loc_reg(byte_value));
     emit_imul_rr(mc, 1, X64_RAX, X64_R11);
   }
   while (i + 8u <= n) {
     emit_mov_store(mc, 8, X64_RAX, dr, (i32)i);
     i += 8u;
   }
   while (i + 4u <= n) {
     emit_mov_store(mc, 4, X64_RAX, dr, (i32)i);
     i += 4u;
   }
   while (i + 2u <= n) {
     emit_mov_store(mc, 2, X64_RAX, dr, (i32)i);
     i += 2u;
   }
   while (i < n) {
     emit_mov_store(mc, 1, X64_RAX, dr, (i32)i);
     i += 1u;
   }
 }
 
 /* ============================ bitfields ============================ */
 
 static void x64_bitfield_load(NativeTarget* t, NativeLoc dst, NativeAddr ra,
                               BitFieldAccess bf) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   u32 storage_bytes = bf.storage.size ? bf.storage.size : 4u;
   int w = storage_bytes == 8u ? 1 : 0;
   u32 reg_size = w ? 64u : 32u;
   u32 lsb = bf.bit_offset;
   u32 width = bf.bit_width ? bf.bit_width : 1u;
   u32 rd = loc_reg(dst);
   u32 base;
   ra.offset += (i32)bf.storage_offset;
   base = x64_addr_to_base_reg(a, ra, X64_TMP_INT2);
   emit_mov_load(mc, storage_bytes, 0, rd, base, 0);
   {
     u8 left = (u8)(reg_size - lsb - width);
     u8 right = (u8)(reg_size - width);
     if (left) emit_shift_imm(mc, w, X64_SHIFT_SUB_SHL, rd, left);
     if (right)
       emit_shift_imm(mc, w, bf.signed_ ? X64_SHIFT_SUB_SAR : X64_SHIFT_SUB_SHR,
                      rd, right);
   }
 }
 
 static void x64_bitfield_store(NativeTarget* t, NativeAddr ra, NativeLoc src,
                                BitFieldAccess bf) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   u32 storage_bytes = bf.storage.size ? bf.storage.size : 4u;
   int w = storage_bytes == 8u ? 1 : 0;
   u32 lsb = bf.bit_offset;
   u32 width = bf.bit_width ? bf.bit_width : 1u;
   u64 ones = width >= 64u ? ~(u64)0 : (((u64)1 << width) - 1u);
   u64 mask = ones << lsb;
   u32 src_reg = loc_reg(src);
   u32 base;
   ra.offset += (i32)bf.storage_offset;
   /* Stabilize the base into r11 before consuming rax/rcx/rdx scratch. */
   base = x64_addr_to_base_reg(a, ra, X64_TMP_INT2);
   /* rax = storage; rax &= ~mask. */
   emit_mov_load(mc, storage_bytes, 0, X64_RAX, base, 0);
   x64_emit_load_imm(mc, w, X64_RCX, (i64)~mask);
   emit_alu_rr(mc, w, X64_OPC_ALU_AND, X64_RAX, X64_RCX);
   /* rcx = (src & ones) << lsb. */
   emit_mov_rr(mc, w, X64_RCX, src_reg);
   x64_emit_load_imm(mc, w, X64_RDX, (i64)ones);
   emit_alu_rr(mc, w, X64_OPC_ALU_AND, X64_RCX, X64_RDX);
   if (lsb) emit_shift_imm(mc, w, X64_SHIFT_SUB_SHL, X64_RCX, (u8)lsb);
   emit_alu_rr(mc, w, X64_OPC_ALU_OR, X64_RAX, X64_RCX);
   emit_mov_store(mc, storage_bytes, X64_RAX, base, 0);
 }
 
 /* ============================ arithmetic ============================ */
 
 static void x64_binop(NativeTarget* t, BinOp op, NativeLoc dst, NativeLoc aop,
                       NativeLoc bop) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst);
 
   /* FP binops: two-address. dst = aop op bop. */
   if (op == BO_FADD || op == BO_FSUB || op == BO_FMUL || op == BO_FDIV) {
     u32 ra = loc_reg(aop), rb = loc_reg(bop);
     u8 prefix = sse_scalar_prefix(native_type_size(t, dst.type));
     u8 opcode;
     switch (op) {
       case BO_FADD:
         opcode = 0x58;
         break;
       case BO_FSUB:
         opcode = 0x5C;
         break;
       case BO_FMUL:
         opcode = 0x59;
         break;
       default:
         opcode = 0x5E;
         break; /* BO_FDIV */
     }
     if (rd == rb && rd != ra) {
       if (op == BO_FADD || op == BO_FMUL) { /* commutative */
         emit_sse_rr(mc, prefix, opcode, rd, ra);
         return;
       }
       /* non-commutative dst==rb: stage rb in fp scratch. */
       emit_sse_rr(mc, prefix, 0x10, X64_TMP_FP2, rb);
       emit_sse_rr(mc, prefix, 0x10, rd, ra);
       emit_sse_rr(mc, prefix, opcode, rd, X64_TMP_FP2);
       return;
     }
     if (rd != ra) emit_sse_rr(mc, prefix, 0x10, rd, ra);
     emit_sse_rr(mc, prefix, opcode, rd, rb);
     return;
   }
 
   {
     int w = x64_is_64(t, dst.type) ? 1 : 0;
     int b_imm = bop.kind == NATIVE_LOC_IMM;
     i64 imm = b_imm ? bop.v.imm : 0;
     u32 ra = loc_reg(aop);
 
     /* Division: rax/rdx implicit; divisor must avoid rax/rdx. */
     if (op == BO_SDIV || op == BO_UDIV || op == BO_SREM || op == BO_UREM) {
       u32 rb;
       if (ra != X64_RAX) emit_mov_rr(mc, w, X64_RAX, ra);
       if (b_imm) {
         x64_emit_load_imm(mc, w, X64_R11, imm);
         rb = X64_R11;
       } else {
         rb = loc_reg(bop);
         if (rb == X64_RAX || rb == X64_RDX) {
           emit_mov_rr(mc, w, X64_R11, rb);
           rb = X64_R11;
         }
       }
       if (op == BO_SDIV || op == BO_SREM) {
         emit_cqo_or_cdq(mc, w);
         emit_f7_rm(mc, w, X64_F7_SUB_IDIV, rb);
       } else {
         emit_xor_self(mc, w, X64_RDX);
         emit_f7_rm(mc, w, X64_F7_SUB_DIV, rb);
       }
       {
         u32 result = (op == BO_SREM || op == BO_UREM) ? X64_RDX : X64_RAX;
         if (rd != result) emit_mov_rr(mc, w, rd, result);
       }
       return;
     }
 
     /* Shifts: count in CL or imm8. */
     if (op == BO_SHL || op == BO_SHR_U || op == BO_SHR_S) {
       u32 sub = (op == BO_SHL)     ? X64_SHIFT_SUB_SHL
                 : (op == BO_SHR_U) ? X64_SHIFT_SUB_SHR
                                    : X64_SHIFT_SUB_SAR;
       if (b_imm) {
         u32 wbits = w ? 64u : 32u;
         if (rd != ra) emit_mov_rr(mc, w, rd, ra);
         emit_shift_imm(mc, w, sub, rd, (u8)((u64)imm & (wbits - 1u)));
         return;
       }
       {
         u32 rb = loc_reg(bop);
         /* Place the count in cl and the value in dst. Stage the count through
          * r11 first so neither move clobbers the other when the value already
          * sits in rcx or the count sits in dst. (The optimizer additionally
          * keeps values live across the shift out of rcx — see
          * x64_machine_op_clobbers.) */
         if (rb != X64_RCX) {
           emit_mov_rr(mc, 0, X64_TMP_INT2, rb);
           if (rd != ra) emit_mov_rr(mc, w, rd, ra);
           emit_mov_rr(mc, 0, X64_RCX, X64_TMP_INT2);
         } else if (rd != ra) {
           emit_mov_rr(mc, w, rd, ra);
         }
       }
       emit_shift_cl(mc, w, sub, rd);
       return;
     }
 
     /* IMM-form fast paths (b_imm guaranteed legal by imm_legal: imm32). */
     if (b_imm && (op == BO_IADD || op == BO_ISUB || op == BO_AND ||
                   op == BO_OR || op == BO_XOR || op == BO_IMUL)) {
       if (op == BO_IMUL) {
         if (imm_fits_i8(imm)) {
           emit_imul_imm8(mc, w, rd, ra, (i8)imm);
           return;
         }
         emit_imul_imm32(mc, w, rd, ra, (i32)imm);
         return;
       }
       {
         u32 sub;
         switch (op) {
           case BO_IADD:
             sub = X64_ALU_SUB_ADD;
             break;
           case BO_OR:
             sub = X64_ALU_SUB_OR;
             break;
           case BO_AND:
             sub = X64_ALU_SUB_AND;
             break;
           case BO_ISUB:
             sub = X64_ALU_SUB_SUB;
             break;
           default:
             sub = X64_ALU_SUB_XOR;
             break; /* BO_XOR */
         }
         if (rd != ra) emit_mov_rr(mc, w, rd, ra);
         if (imm_fits_i8(imm))
           emit_alu_imm8(mc, w, sub, rd, (i8)imm);
         else
           emit_alu_imm32(mc, w, sub, rd, (i32)imm);
         return;
       }
     }
 
     /* Generic 2-operand ALU: dst = ra op rb. Preserve rb if dst == rb. */
     {
       u32 rb = loc_reg(bop);
       if (rd == rb && rd != ra) {
         switch (op) {
           case BO_IADD:
             emit_alu_rr(mc, w, X64_OPC_ALU_ADD, rd, ra);
             return;
           case BO_AND:
             emit_alu_rr(mc, w, X64_OPC_ALU_AND, rd, ra);
             return;
           case BO_OR:
             emit_alu_rr(mc, w, X64_OPC_ALU_OR, rd, ra);
             return;
           case BO_XOR:
             emit_alu_rr(mc, w, X64_OPC_ALU_XOR, rd, ra);
             return;
           case BO_IMUL:
             emit_imul_rr(mc, w, rd, ra);
             return;
           default:
             break; /* ISUB falls through: stage rb */
         }
         emit_mov_rr(mc, w, X64_R11, rb);
         rb = X64_R11;
       }
       if (rd != ra) emit_mov_rr(mc, w, rd, ra);
       switch (op) {
         case BO_IADD:
           emit_alu_rr(mc, w, X64_OPC_ALU_ADD, rd, rb);
           break;
         case BO_ISUB:
           emit_alu_rr(mc, w, X64_OPC_ALU_SUB, rd, rb);
           break;
         case BO_AND:
           emit_alu_rr(mc, w, X64_OPC_ALU_AND, rd, rb);
           break;
         case BO_OR:
           emit_alu_rr(mc, w, X64_OPC_ALU_OR, rd, rb);
           break;
         case BO_XOR:
           emit_alu_rr(mc, w, X64_OPC_ALU_XOR, rd, rb);
           break;
         case BO_IMUL:
           emit_imul_rr(mc, w, rd, rb);
           break;
         default:
           x64_panic(a, "unsupported binop");
       }
     }
   }
 }
 
 /* FP sign-mask constant materialized in fp scratch for FNEG. */
 static void x64_unop(NativeTarget* t, UnOp op, NativeLoc dst, NativeLoc src) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst), rs = loc_reg(src);
   if (op == UO_FNEG) {
     int dbl = native_type_size(t, dst.type) == 8u;
     if (rd != rs)
       emit_sse_rr(mc, sse_scalar_prefix(dbl ? 8u : 4u), 0x10, rd, rs);
     /* sign mask into fp scratch via gpr, then XORPS/XORPD. */
     x64_emit_load_imm(mc, dbl, X64_TMP_INT,
                       dbl ? (i64)0x8000000000000000ull : (i64)0x80000000ull);
     emit_sse_rr_w(mc, 0x66, 0x6E, dbl, X64_TMP_FP2, X64_TMP_INT);
     emit_sse_rr(mc, dbl ? 0x66 : 0, 0x57, rd, X64_TMP_FP2);
     return;
   }
   {
     int w = x64_is_64(t, dst.type) ? 1 : 0;
     switch (op) {
       case UO_NEG:
         if (rd != rs) emit_mov_rr(mc, w, rd, rs);
         emit_f7_rm(mc, w, X64_F7_SUB_NEG, rd);
         return;
       case UO_BNOT:
         if (rd != rs) emit_mov_rr(mc, w, rd, rs);
         emit_f7_rm(mc, w, X64_F7_SUB_NOT, rd);
         return;
       case UO_NOT:
         /* !x -> (x == 0) as 0/1. */
         emit_test_self(mc, w, rs);
         emit_setcc(mc, X64_CC_E, rd);
         emit_movzx_r32_r8(mc, rd, rd);
         return;
       default:
         x64_panic(a, "unsupported unop");
     }
   }
 }
 
 /* ============================ compares ============================ */
 
 static u32 cmp_to_cc(CmpOp op) {
   switch (op) {
     case CMP_EQ:
       return X64_CC_E;
     case CMP_NE:
       return X64_CC_NE;
     case CMP_LT_U:
       return X64_CC_B;
     case CMP_LE_U:
       return X64_CC_BE;
     case CMP_GT_U:
       return X64_CC_A;
     case CMP_GE_U:
       return X64_CC_AE;
     case CMP_LT_S:
       return X64_CC_L;
     case CMP_LE_S:
       return X64_CC_LE;
     case CMP_GT_S:
       return X64_CC_G;
     case CMP_GE_S:
       return X64_CC_GE;
     default:
       return X64_CC_E;
   }
 }
 
 static int cmp_is_fp(CmpOp op, NativeLoc aop) {
   /* FP-ness is self-describing from the opcode; FP eq/ne are distinct opcodes
    * (CMP_OEQ_F/CMP_UNE_F), so no operand-class sniffing is needed. */
   (void)aop;
   return op >= CMP_OEQ_F;
 }
 
 /* Emit `cmp ra, rb` (or ucomis[sd] for FP), setting flags from ra - rb. */
 static void x64_emit_cmp_flags(NativeTarget* t, NativeLoc aop, NativeLoc bop,
                                int fp) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   if (fp) {
     u8 prefix = native_type_size(t, aop.type) == 8u ? 0x66u : 0u;
     emit_sse_rr(mc, prefix, 0x2E, loc_reg(aop), loc_reg(bop)); /* ucomis */
     return;
   }
   {
     int w = x64_is_64(t, aop.type) ? 1 : 0;
     u32 ra = loc_reg(aop);
     if (bop.kind == NATIVE_LOC_IMM) {
       i64 imm = bop.v.imm;
       if (imm_fits_i8(imm))
         emit_alu_imm8(mc, w, X64_ALU_SUB_CMP, ra, (i8)imm);
       else
         emit_alu_imm32(mc, w, X64_ALU_SUB_CMP, ra, (i32)imm);
       return;
     }
     emit_alu_rr(mc, w, X64_OPC_ALU_CMP, ra, loc_reg(bop));
     (void)a;
   }
 }
 
 /* FP ordered setcc: result = (primary cc) && !unordered (NP). */
 static void x64_fp_setcc_ordered(NativeTarget* t, u32 primary, u32 dst) {
   MCEmitter* mc = t->mc;
   emit_setcc(mc, primary, dst);
   emit_movzx_r32_r8(mc, dst, dst);
   emit_setcc(mc, X64_CC_NP, X64_R11);
   emit_movzx_r32_r8(mc, X64_R11, X64_R11);
   emit_alu_rr(mc, 0, X64_OPC_ALU_AND, dst, X64_R11);
 }
 
 /* FP unordered predicate: result = (primary cc) || unordered (P). */
 static void x64_fp_setcc_unord(NativeTarget* t, u32 primary, u32 dst) {
   MCEmitter* mc = t->mc;
   emit_setcc(mc, primary, dst);
   emit_movzx_r32_r8(mc, dst, dst);
   emit_setcc(mc, X64_CC_P, X64_R11);
   emit_movzx_r32_r8(mc, X64_R11, X64_R11);
   emit_alu_rr(mc, 0, X64_OPC_ALU_OR, dst, X64_R11);
 }
 
 static void x64_cmp(NativeTarget* t, CmpOp op, NativeLoc dst, NativeLoc aop,
                     NativeLoc bop) {
   MCEmitter* mc = t->mc;
   u32 d = loc_reg(dst);
   int fp = cmp_is_fp(op, aop);
   x64_emit_cmp_flags(t, aop, bop, fp);
   if (fp) {
     /* ucomis sets ZF/CF and, when unordered (NaN), also PF. Each predicate's
      * flag formula is built explicitly (NOT blindly as !(opposite)):
      *   ordered:    E/B/BE alias {==,<,<=} only when also NP (not-parity);
      *               NE/A/AE already exclude unordered, so they stand alone.
      *   unordered:  E/B/BE already include the unordered case (ZF/CF set on
      *               NaN), so they stand alone; NE/A/AE need an OR with P. */
     switch (op) {
       /* ordered: require not-unordered (NP) on the equality-flag cases */
       case CMP_OEQ_F:
         x64_fp_setcc_ordered(t, X64_CC_E, d);
         return;
       case CMP_OLT_F:
         x64_fp_setcc_ordered(t, X64_CC_B, d);
         return;
       case CMP_OLE_F:
         x64_fp_setcc_ordered(t, X64_CC_BE, d);
         return;
       case CMP_ONE_F:
         emit_setcc(mc, X64_CC_NE, d);
         break;
       case CMP_OGT_F:
         emit_setcc(mc, X64_CC_A, d);
         break;
       case CMP_OGE_F:
         emit_setcc(mc, X64_CC_AE, d);
         break;
       /* unordered: OR-with-P on the cases that exclude unordered */
       case CMP_UEQ_F:
         emit_setcc(mc, X64_CC_E, d);
         break;
       case CMP_ULT_F:
         emit_setcc(mc, X64_CC_B, d);
         break;
       case CMP_ULE_F:
         emit_setcc(mc, X64_CC_BE, d);
         break;
       case CMP_UNE_F:
         x64_fp_setcc_unord(t, X64_CC_NE, d);
         return;
       case CMP_UGT_F:
         x64_fp_setcc_unord(t, X64_CC_A, d);
         return;
       case CMP_UGE_F:
         x64_fp_setcc_unord(t, X64_CC_AE, d);
         return;
       default:
         emit_setcc(mc, cmp_to_cc(op), d);
         break;
     }
     emit_movzx_r32_r8(mc, d, d);
     return;
   }
   emit_setcc(mc, cmp_to_cc(op), d);
   emit_movzx_r32_r8(mc, d, d);
 }
 
 /* ============================ converts ============================ */
 
 static void x64_convert(NativeTarget* t, ConvKind k, NativeLoc dst,
                         NativeLoc src) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst), rs = loc_reg(src);
   switch (k) {
     case CV_SEXT: {
       u32 src_sz = native_type_size(t, src.type);
       int w = x64_is_64(t, dst.type) ? 1 : 0;
       emit_extend_rr(mc, w, 1, src_sz, rd, rs);
       return;
     }
     case CV_ZEXT: {
       u32 src_sz = native_type_size(t, src.type);
       int w = x64_is_64(t, dst.type) ? 1 : 0;
       emit_extend_rr(mc, w, 0, src_sz, rd, rs);
       return;
     }
     case CV_TRUNC:
       emit_mov_rr(mc, 0, rd, rs); /* low 32 bits; clears high */
       return;
     case CV_ITOF_S:
     case CV_ITOF_U: {
       int w_src = x64_is_64(t, src.type) ? 1 : 0;
       u8 prefix = sse_scalar_prefix(native_type_size(t, dst.type));
       if (k == CV_ITOF_U && w_src == 1) {
         MCLabel L_high = mc->label_new(mc);
         MCLabel L_done = mc->label_new(mc);
         emit_test_self(mc, 1, rs);
         emit_jcc_rel32(mc, X64_CC_S, L_high);
         emit_sse_rr_w(mc, prefix, 0x2A, 1, rd, rs);
         emit_jmp_rel32(mc, L_done);
         mc->label_place(mc, L_high);
         emit_mov_rr(mc, 1, X64_R11, rs);
         emit_mov_rr(mc, 1, X64_RAX, rs);
         emit_alu_imm8(mc, 1, X64_ALU_SUB_AND, X64_RAX, 1);
         emit_shift_imm(mc, 1, X64_SHIFT_SUB_SHR, X64_R11, 1);
         emit_alu_rr(mc, 1, X64_OPC_ALU_OR, X64_R11, X64_RAX);
         emit_sse_rr_w(mc, prefix, 0x2A, 1, rd, X64_R11);
         emit_sse_rr(mc, prefix, 0x58, rd, rd);
         mc->label_place(mc, L_done);
         return;
       }
       if (k == CV_ITOF_U) {
         emit_extend_rr(mc, 0, 0, 4, X64_R11, rs); /* zext u32 -> 64 */
         rs = X64_R11;
         w_src = 1;
       }
       emit_sse_rr_w(mc, prefix, 0x2A, w_src, rd, rs);
       return;
     }
     case CV_FTOI_S:
     case CV_FTOI_U: {
       int w_dst = x64_is_64(t, dst.type) ? 1 : 0;
       u8 prefix = sse_scalar_prefix(native_type_size(t, src.type));
       /* Unsigned 64-bit FTOI needs the 2^63 bias dance; otherwise cvtt
        * (with the destination widened to 64 for u32) is exact. */
       if (k == CV_FTOI_U && w_dst == 1) {
         int dbl = native_type_size(t, src.type) == 8u;
         MCLabel L_small = mc->label_new(mc);
         MCLabel L_done = mc->label_new(mc);
         /* limit = 2^63 in fp scratch. */
         x64_emit_load_imm(
             mc, 1, X64_R11,
             dbl ? (i64)0x43E0000000000000ull : (i64)0x5F000000ull);
         emit_sse_rr_w(mc, 0x66, 0x6E, dbl, X64_TMP_FP2, X64_R11);
         emit_sse_rr(mc, dbl ? 0x66 : 0, 0x2E, rs, X64_TMP_FP2); /* ucomis */
         emit_jcc_rel32(mc, X64_CC_B, L_small);
         emit_sse_rr(mc, prefix, 0x10, X64_TMP_FP, rs);
         emit_sse_rr(mc, prefix, 0x5C, X64_TMP_FP, X64_TMP_FP2); /* sub bias */
         emit_sse_rr_w(mc, prefix, 0x2C, 1, rd, X64_TMP_FP);
         x64_emit_load_imm(mc, 1, X64_R11, (i64)0x8000000000000000ull);
         emit_alu_rr(mc, 1, X64_OPC_ALU_XOR, rd, X64_R11);
         emit_jmp_rel32(mc, L_done);
         mc->label_place(mc, L_small);
         emit_sse_rr_w(mc, prefix, 0x2C, 1, rd, rs);
         mc->label_place(mc, L_done);
         return;
       }
       if (k == CV_FTOI_U) w_dst = 1; /* widen u32 result */
       emit_sse_rr_w(mc, prefix, 0x2C, w_dst, rd, rs);
       return;
     }
     case CV_FEXT:
       emit_sse_rr(mc, 0xF3, 0x5A, rd, rs); /* cvtss2sd */
       return;
     case CV_FTRUNC:
       emit_sse_rr(mc, 0xF2, 0x5A, rd, rs); /* cvtsd2ss */
       return;
     case CV_BITCAST:
       if (!native_loc_is_fp(src) && native_loc_is_fp(dst)) {
         emit_sse_rr_w(mc, 0x66, 0x6E, x64_is_64(t, dst.type), rd, rs);
       } else if (native_loc_is_fp(src) && !native_loc_is_fp(dst)) {
         emit_sse_rr_w(mc, 0x66, 0x7E, x64_is_64(t, src.type), rs, rd);
       } else {
         x64_move(t, dst, src);
       }
       return;
     default:
       x64_panic(a, "unsupported convert");
   }
 }
 
 /* ============================ spill / reload ============================ */
 
 static void x64_spill(NativeTarget* t, NativeLoc src, NativeFrameSlot slot,
                       MemAccess mem) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = slot;
   addr.base_type = src.type;
   x64_emit_mem(x64_of(t), 0, src, addr, mem);
 }
 static void x64_reload(NativeTarget* t, NativeLoc dst, NativeFrameSlot slot,
                        MemAccess mem) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = slot;
   addr.base_type = dst.type;
   x64_emit_mem(x64_of(t), 1, dst, addr, mem);
 }
 
 /* ============================ control flow ============================ */
 
 static void emit_jmp_rel32(MCEmitter* mc, MCLabel l) {
   u8 op = X64_OPC_JMP_REL32;
   mc->emit_bytes(mc, &op, 1);
   emit_u32le(mc, 0);
   mc->emit_label_ref(mc, l, R_PC32, 4, -4);
 }
 static void emit_jcc_rel32(MCEmitter* mc, u32 cc, MCLabel l) {
   u8 op[2] = {X64_OPC_TWOBYTE, (u8)(X64_OPC_JCC_BASE | (cc & 0xfu))};
   mc->emit_bytes(mc, op, 2);
   emit_u32le(mc, 0);
   mc->emit_label_ref(mc, l, R_PC32, 4, -4);
 }
 
 static MCLabel x64_label_new(NativeTarget* t) {
   return t->mc->label_new(t->mc);
 }
 static void x64_label_place(NativeTarget* t, MCLabel l) {
   t->mc->label_place(t->mc, l);
 }
 static void x64_jump(NativeTarget* t, MCLabel l) { emit_jmp_rel32(t->mc, l); }
 
 static void x64_cmp_branch(NativeTarget* t, CmpOp op, NativeLoc aop,
                            NativeLoc bop, MCLabel l) {
   MCEmitter* mc = t->mc;
   int fp = cmp_is_fp(op, aop);
   if (fp) {
     /* Materialize the 0/1 result, then branch on nonzero. */
     NativeLoc tmp =
         native_loc_reg(builtin_id(KIT_CG_BUILTIN_I32), NATIVE_REG_INT, X64_RAX);
     x64_cmp(t, op, tmp, aop, bop);
     emit_test_self(mc, 0, X64_RAX);
     emit_jcc_rel32(mc, X64_CC_NE, l);
     return;
   }
   x64_emit_cmp_flags(t, aop, bop, 0);
   emit_jcc_rel32(mc, cmp_to_cc(op), l);
 }
 
 static void x64_indirect_branch(NativeTarget* t, NativeLoc addr,
                                 const MCLabel* valid_targets, u32 ntargets) {
   MCEmitter* mc = t->mc;
   u32 r = loc_reg(addr);
   (void)valid_targets;
   (void)ntargets;
   if (r & 8u) {
     u8 rex = X64_REX_BASE | X64_REX_B;
     mc->emit_bytes(mc, &rex, 1);
   }
   {
     u8 buf[2] = {X64_OP_JMP_RM64, modrm(3u, 4u, r & 7u)};
     mc->emit_bytes(mc, buf, 2);
   }
 }
 
 static void x64_load_label_addr(NativeTarget* t, NativeLoc dst, MCLabel l) {
   /* `&&label` address-take: `leaq sym(%rip), rd` with an R_PC32 relocation
    * against the label's per-block local symbol — same form as a global
    * address-take, so a re-encoding assembler recomputes the displacement.
    * (A baked disp32 with no reloc would break once clang re-lays-out the
    * function.) */
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst);
   ObjSymId sym = mc_label_symbol(mc, l);
   u32 disp_pos;
   emit_rex(mc, 1, rd, 0, 0);
   {
     u8 op = X64_OPC_LEA;
     mc->emit_bytes(mc, &op, 1);
   }
   {
     u8 mr = modrm(0u, rd & 7u, 5u); /* [rip + disp32] */
     mc->emit_bytes(mc, &mr, 1);
   }
   disp_pos = mc->pos(mc);
   emit_u32le(mc, 0);
   mc->emit_reloc_at(mc, mc->section_id, disp_pos, R_PC32, sym, -4, 1, 0);
 }
 
 /* ============================ frame / lifecycle ============================
  */
 
 static NativeFrameSlot x64_frame_slot(NativeTarget* t,
                                       const NativeFrameSlotDesc* d) {
   return native_frame_slot_alloc(&x64_of(t)->frame, d);
 }
 
 static int x64_frame_slot_debug_loc(NativeTarget* t, NativeFrameSlot slot,
                                     CGDebugLoc* out) {
   X64NativeTarget* a = x64_of(t);
   X64NativeSlot* s;
   if (!out) return 0;
   memset(out, 0, sizeof *out);
   if (slot == NATIVE_FRAME_SLOT_NONE || slot > a->frame.nslots) return 0;
   s = x64_slot_get(a, slot);
   out->kind = CG_DEBUG_LOC_FRAME;
   /* x64 slots live at RBP - off (exactly how the memory-operand path addresses
    * them). The hosted dbg snapshot seeds the frame base with RBP, so report
    * the RBP-relative offset — mirroring aa64's FP-relative convention. */
   out->v.frame_ofs = -(i32)s->off;
   return 1;
 }
 
 /* xmm save area base (rbp-relative). XMM saves are 16-aligned. */
 static u32 x64_xmm_base(const X64NativeTarget* a, u32 cs_fp) {
   if (cs_fp == 0) return a->frame.cum_off;
   return align_up_u32(a->frame.cum_off, 16u);
 }
 
 static u32 x64_compute_frame_size(const X64NativeTarget* a, u32 cs_int,
                                   u32 cs_fp) {
   u32 xmm_base = x64_xmm_base(a, cs_fp);
   u32 raw = a->frame.max_outgoing + cs_int * 8u + cs_fp * 16u + xmm_base;
   u32 fs = align_up_u32(raw, 16u);
   return fs ? fs : 16u;
 }
 
 /* Collect the callee-saves the body actually used. */
 static u32 x64_collect_int_saves(X64NativeTarget* a, Reg* regs) {
   u32 n = 0, i;
   for (i = 0; i < a->frame.ncallee_saves; ++i)
     if (a->frame.callee_saves[i].cls == NATIVE_REG_INT)
       regs[n++] = a->frame.callee_saves[i].reg;
   return n;
 }
 static u32 x64_collect_fp_saves(X64NativeTarget* a, Reg* regs) {
   u32 n = 0, i;
   for (i = 0; i < a->frame.ncallee_saves; ++i)
     if (a->frame.callee_saves[i].cls == NATIVE_REG_FP)
       regs[n++] = a->frame.callee_saves[i].reg;
   return n;
 }
 
 static ObjSymId x64_chkstk_sym(NativeTarget* t) {
   Sym name = pool_intern_slice(t->c->global, SLICE_LIT("__chkstk"));
   ObjSymId s = obj_symbol_find(t->obj, name);
   if (s != 0) return s;
   return obj_symbol(t->obj, name, SB_GLOBAL, SK_UNDEF, OBJ_SEC_NONE, 0, 0);
 }
 
 /* Build the prologue byte sequence into buf. Returns bytes written and, when
  * the chkstk path fires, the disp32 offset of the call site. When `skip_sub` is
  * set (the known-frame slim / red-zone tiers), the `sub rsp` reservation is
  * omitted entirely: the frame record is established but no stack is reserved,
  * either because the frame is empty (slim) or because the locals/saves live in
  * the SysV red zone (redzone_leaf). Callers must only set it when the frame
  * needs no reserved region (no alloca, no outgoing args, and — for the red
  * zone — a leaf frame <= 128 bytes). */
 static u32 x64_build_prologue(X64NativeTarget* a, u8* buf, u32 cap,
                               u32 frame_size, const Reg* cs_int, u32 n_int,
                               const Reg* cs_fp, u32 n_fp, int skip_sub,
                               u32* chkstk_disp_pos_out) {
   u32 wi = 0;
   u32 xmm_base = x64_xmm_base(a, n_fp);
   u32 i;
   /* Page granularity for Windows large-frame probing (0 = no probe needed).
    * Win64 reserves >1-page frames through __chkstk; the same ABI capability
    * the aarch64 backend reads for its inline probe. */
   u32 probe = abi_stack_probe_interval(a->base.c->abi);
   *chkstk_disp_pos_out = (u32)-1;
   if (cap < X64_PROLOGUE_BASE_BYTES)
     x64_panic(a, "prologue placeholder overflow");
   /* push rbp; mov rbp, rsp. */
   buf[wi++] = (u8)(X64_OPC_PUSH_R | (X64_RBP & 7u));
   buf[wi++] = X64_REX_BASE | X64_REX_W;
   buf[wi++] = X64_OPC_MOV_RM_R;
   buf[wi++] = modrm(3u, X64_RSP, X64_RBP);
   /* sub rsp, frame_size (or chkstk on Win64 large frame); skipped by the slim /
    * red-zone tiers, which reserve no stack. */
   if (skip_sub) {
     /* no reservation */
   } else if (probe && frame_size > probe) {
     if (wi + 13u > cap) x64_panic(a, "prologue placeholder overflow");
     buf[wi++] = (u8)(X64_OPC_MOV_RI | (X64_RAX & 7u)); /* mov eax, imm32 */
     wr_u32_le(buf + wi, frame_size);
     wi += 4;
     buf[wi++] = X64_OPC_CALL_REL32;
     *chkstk_disp_pos_out = wi;
     wr_u32_le(buf + wi, 0);
     wi += 4;
     buf[wi++] = X64_REX_BASE | X64_REX_W; /* sub rsp, rax */
     buf[wi++] = X64_OPC_ALU_SUB;
     buf[wi++] = modrm(3u, X64_RAX, X64_RSP);
   } else {
     if (wi + 7u > cap) x64_panic(a, "prologue placeholder overflow");
     buf[wi++] = X64_REX_BASE | X64_REX_W;
     buf[wi++] = X64_OPC_ALU_IMM32;
     buf[wi++] = modrm(3u, X64_ALU_SUB_SUB, X64_RSP);
     wr_u32_le(buf + wi, frame_size);
     wi += 4;
   }
   /* sret: spill the first int arg reg (destination pointer) into its slot.
    * Use the minimal disp encoding (x64_pack_mem) so it matches the body's
    * frame stores and the matching epilogue restore — the `cc -S | as`
    * round-trip can then reproduce these bytes exactly. The -O0 placeholder is
    * NOP-padded to a fixed width, so a shorter prologue is harmless. */
   if (a->has_sret && a->sret_ptr_slot != NATIVE_FRAME_SLOT_NONE) {
     X64NativeSlot* s = x64_slot_get(a, a->sret_ptr_slot);
     u32 sret_reg = a->abi->int_args[0];
     i32 off = -(i32)s->off;
     if (wi + 8u > cap) x64_panic(a, "prologue placeholder overflow");
     buf[wi++] =
         (u8)(X64_REX_BASE | X64_REX_W | ((sret_reg & 8u) ? X64_REX_R : 0u));
     buf[wi++] = X64_OPC_MOV_RM_R;
     wi += x64_pack_mem(buf + wi, sret_reg & 7u, X64_RBP, off);
   }
   /* Spill callee-saved GPRs. */
   for (i = 0; i < n_int; ++i) {
     u32 reg = cs_int[i];
     i32 off = -(i32)xmm_base - (i32)n_fp * 16 - (i32)(i + 1u) * 8;
     if (wi + 8u > cap) x64_panic(a, "prologue placeholder overflow");
     buf[wi++] = (u8)(X64_REX_BASE | X64_REX_W | ((reg & 8u) ? X64_REX_R : 0u));
     buf[wi++] = X64_OPC_MOV_RM_R;
     wi += x64_pack_mem(buf + wi, reg & 7u, X64_RBP, off);
   }
   /* Spill callee-saved XMMs (Win64). movaps [rbp+disp], xmm. */
   for (i = 0; i < n_fp; ++i) {
     u32 xmm = cs_fp[i];
     i32 off = -(i32)xmm_base - (i32)(i + 1u) * 16;
     u8 rex = (u8)((xmm & 8u) ? (X64_REX_BASE | X64_REX_R) : 0u);
     u32 need = rex ? 9u : 8u;
     if (wi + need > cap) x64_panic(a, "prologue placeholder overflow");
     if (rex) buf[wi++] = rex;
     buf[wi++] = X64_OPC_TWOBYTE;
     buf[wi++] = 0x29; /* MOVAPS r/m128, xmm */
     wi += x64_pack_mem(buf + wi, xmm & 7u, X64_RBP, off);
   }
   return wi;
 }
 
 static void x64_func_begin_common(NativeTarget* t, const CGFuncDesc* fd) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, fd->fn_type);
   a->func = fd;
   a->loc = fd->loc;
   a->abi = x64_abi_for_os(t->c->target.os);
   /* Shared frame bookkeeping: clears the slot table, cum_off, max_outgoing,
    * callee-save set, and known_frame/has_alloca/frame_final. */
   native_frame_reset(&a->frame);
   a->incoming_stack_size = 0;
   a->next_param_int = 0;
   a->next_param_fp = 0;
   a->next_param_stack = 0;
   a->has_sret = (abi && abi->has_sret) ? 1u : 0u;
   a->is_variadic = (abi && abi->variadic) ? 1u : 0u;
   a->sret_ptr_slot = NATIVE_FRAME_SLOT_NONE;
   a->reg_save_slot = NATIVE_FRAME_SLOT_NONE;
   a->npatches = 0;
   a->nalloca = 0;
   a->nbind_moves = 0;
   a->slim_frame = 0;
   a->redzone_leaf = 0;
   a->prologue_nbytes =
       a->abi->shadow_space ? X64_PROLOGUE_BYTES_WIN64 : X64_PROLOGUE_BYTES;
 
   mc->set_section(mc, fd->text_section_id);
   mc->emit_align(mc, 16, X64_NOP1);
   a->func_start = mc->pos(mc);
   mc_begin_function(mc, fd->sym, fd->text_section_id, a->func_start);
   if (mc->cfi_startproc) mc->cfi_startproc(mc);
   a->epilogue_label = mc->label_new(mc);
 }
 
 /* Reserve the sret-pointer slot and (SysV) the 176-byte variadic reg-save
  * area. Advances next_param_int past the sret pointer (a0). */
 static void x64_reserve_entry_saves(X64NativeTarget* a) {
   NativeTarget* t = &a->base;
   if (a->has_sret) {
     NativeFrameSlotDesc sd;
     memset(&sd, 0, sizeof sd);
     sd.type = builtin_id(KIT_CG_BUILTIN_I64);
     sd.size = 8;
     sd.align = 8;
     sd.kind = NATIVE_FRAME_SLOT_SAVE;
     a->sret_ptr_slot = t->frame_slot(t, &sd);
     a->next_param_int = 1;
   }
   if (a->is_variadic && a->abi->emit_sysv_vararg_save) {
     NativeFrameSlotDesc rd;
     memset(&rd, 0, sizeof rd);
     rd.type = builtin_id(KIT_CG_BUILTIN_I64);
     rd.size = 176;
     rd.align = 8;
     rd.kind = NATIVE_FRAME_SLOT_SAVE;
     a->reg_save_slot = t->frame_slot(t, &rd);
   }
 }
 
 static void x64_emit_variadic_reg_saves(X64NativeTarget* a) {
   NativeTarget* t = &a->base;
   MCEmitter* mc = t->mc;
   if (!a->is_variadic) return;
   if (a->abi->emit_sysv_vararg_save) {
     X64NativeSlot* rs = x64_slot_get(a, a->reg_save_slot);
     static const u32 gprs[6] = {X64_RDI, X64_RSI, X64_RDX,
                                 X64_RCX, X64_R8,  X64_R9};
     u32 i;
     for (i = 0; i < 6u; ++i)
       emit_mov_store(mc, 8, gprs[i], X64_RBP, -(i32)rs->off + (i32)(i * 8u));
     for (i = 0; i < 8u; ++i)
       emit_sse_store(mc, 0xF2, 0x11, (u32)(X64_XMM0 + i), X64_RBP,
                      -(i32)rs->off + (i32)(48u + i * 16u));
     return;
   }
   /* Win64 variadic: spill the 4 GPR arg slots to the home space. */
   emit_mov_store(mc, 8, X64_RCX, X64_RBP, 16);
   emit_mov_store(mc, 8, X64_RDX, X64_RBP, 24);
   emit_mov_store(mc, 8, X64_R8, X64_RBP, 32);
   emit_mov_store(mc, 8, X64_R9, X64_RBP, 40);
 }
 
 static void x64_func_begin(NativeTarget* t, const CGFuncDesc* fd) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   u32 i;
   x64_func_begin_common(t, fd);
   a->prologue_pos = mc->pos(mc);
   for (i = 0; i < a->prologue_nbytes; ++i) emit1(mc, X64_NOP1);
   x64_reserve_entry_saves(a);
   x64_emit_variadic_reg_saves(a);
 }
 
 /* x64 homes callee-saves below the locals (offsets computed in
  * x64_compute_frame_size / x64_build_prologue), not in frame slots, so
  * alloc_slots=0: native_frame just records the {reg,cls} set from the masks. */
 static void x64_reserve_callee_saves(NativeTarget* t, const u32* used,
                                      u32 nclasses) {
   native_frame_set_callee_saves(&x64_of(t)->frame, used, nclasses, NULL, 0, 0);
 }
 
 static int x64_reg_is_callee_int(const X64ABIRegs* abi, Reg r);
 static int x64_reg_is_callee_fp(const X64ABIRegs* abi, Reg r);
 
 static u32 x64_live_callee_saved_mask(NativeTarget* t,
                                       NativeAllocClass cls) {
   X64NativeTarget* a = x64_of(t);
   const X64ABIRegs* abi = a->abi ? a->abi : x64_abi_for_os(t->c->target.os);
   u32 mask = 0;
   for (Reg r = 0; r < 16u; ++r) {
     if (cls == NATIVE_REG_INT && x64_reg_is_callee_int(abi, r))
       mask |= 1u << r;
     if (cls == NATIVE_REG_FP && x64_reg_is_callee_fp(abi, r))
       mask |= 1u << r;
   }
   return mask;
 }
 
 static u32 x64_live_caller_saved_mask(NativeTarget* t,
                                       NativeAllocClass cls) {
   const NativeAllocClassInfo* ci = native_target_class_info(t, cls);
   if (!ci) return 0;
   return ci->caller_saved_mask & ~x64_live_callee_saved_mask(t, cls);
 }
 
 static void x64_asm_clobber_masks(Compiler* c, SrcLoc loc, const Sym* clobbers,
                                   u32 nclob, u32* int_mask, u32* fp_mask);
 
 /* abi_clobber_masks is shared as native_asm_abi_clobber_masks
  * (cg/native_asm.h); it reads the target's live ABI masks. */
 
 /* Build the callee-saved set the prologue must preserve: the allocator-assigned
  * callee-saved registers (frame->callee_saved_used) plus any an inline-asm
  * block clobbers. The latter are opaque to the optimizer's operand scan, so it
  * forwards the raw clobber names (frame->asm_clobbers) and the arch-neutral
  * clobber-ABI sets (frame->asm_clobber_abi_sets); we resolve both into masks
  * and keep only the callee-saved ones. x64_reg_is_callee_* follow the live ABI:
  * they exclude rbp (handled by the prologue head) and keep the
  * reserved-but-callee- saved scratch rbx/r12 (which the caller still expects
  * preserved). This is the same register selection the per-block spill used,
  * hoisted into the prologue. */
 static u32 x64_known_callee_saves(NativeTarget* t, const X64ABIRegs* abi,
                                   const NativeKnownFrameDesc* frame, u32* out,
                                   u32 cap) {
   u32 ncls = frame->ncallee_classes;
   u32 clob_int = 0, clob_fp = 0, abi_int, abi_fp;
   if (ncls > cap) ncls = cap;
   for (u32 c = 0; c < ncls; ++c)
     out[c] = frame->callee_saved_used ? frame->callee_saved_used[c] : 0u;
   if (frame->asm_clobbers && frame->nasm_clobbers) {
     X64NativeTarget* a = x64_of(t);
     SrcLoc loc = a->func ? a->func->loc : (SrcLoc){0, 0, 0};
     x64_asm_clobber_masks(t->c, loc, frame->asm_clobbers, frame->nasm_clobbers,
                           &clob_int, &clob_fp);
   }
   native_asm_abi_clobber_masks(t, frame->asm_clobber_abi_sets, &abi_int,
                                &abi_fp);
   clob_int |= abi_int;
   clob_fp |= abi_fp;
   for (Reg r = 0; r < 16u; ++r) {
     if (NATIVE_REG_INT < ncls && (clob_int & (1u << r)) &&
         x64_reg_is_callee_int(abi, r))
       out[NATIVE_REG_INT] |= 1u << r;
     if (NATIVE_REG_FP < ncls && (clob_fp & (1u << r)) &&
         x64_reg_is_callee_fp(abi, r))
       out[NATIVE_REG_FP] |= 1u << r;
   }
   return ncls;
 }
 
 /* Optimizer entry point: the full frame is supplied up front, so the prologue
  * is emitted final the moment it is built — no NOP region, no func_end patch
  * (x64_func_end skips patching when known_frame). x64_build_prologue emits the
  * push rbp / sub rsp / sret spill / callee-save spills; the variadic
  * register-save stores are emitted separately, as on the single-pass path. */
 static void x64_func_begin_known_frame(NativeTarget* t, const CGFuncDesc* fd,
                                        const NativeKnownFrameDesc* frame,
                                        NativeFrameSlot* out_slots) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   Reg cs_int[X64_MAX_CS_INT_REGS], cs_fp[X64_MAX_CS_FP_REGS];
   u32 n_int, n_fp, frame_size, nbytes, chkstk_disp_pos, i;
   u8 buf[X64_PROLOGUE_BYTES_WIN64];
   x64_func_begin_common(t, fd);
   a->frame.known_frame = 1;
   if (frame) {
     u32 cs[NATIVE_CALL_PLAN_CLASSES];
     u32 ncs =
         x64_known_callee_saves(t, a->abi, frame, cs, NATIVE_CALL_PLAN_CLASSES);
     a->frame.has_alloca = frame->has_alloca;
     if (ncs) x64_reserve_callee_saves(t, cs, ncs);
     for (i = 0; i < frame->nslots; ++i) {
       NativeFrameSlot slot = x64_frame_slot(t, &frame->slots[i]);
       if (out_slots) out_slots[i] = slot;
     }
     x64_reserve_entry_saves(a);
     native_frame_note_outgoing(&a->frame, frame->max_outgoing);
   }
   /* Frame is final: size and offsets are settled, so emit the exact prologue.
    */
   n_int = x64_collect_int_saves(a, cs_int);
   n_fp = x64_collect_fp_saves(a, cs_fp);
   frame_size = x64_compute_frame_size(a, n_int, n_fp);
   a->frame_size_final = frame_size;
   /* Cost-model tier selection (mirrors aa64's aa_func_begin_known_frame): with
    * the frame final before the body, choose the cheapest valid prologue shape.
    * Both tiers keep the rbp record and only drop the `sub rsp`, so the
    * epilogue/CFI/offset helpers are untouched. x64 needs no
    * `fp_at_bottom`-style fold: `push rbp` already folds the sp-move into the
    * store. */
   a->slim_frame = a->frame.ncallee_saves == 0 && !a->frame.has_alloca &&
                   a->frame.cum_off == 0 && a->frame.max_outgoing == 0;
   /* redzone keeps locals below rsp in the red zone; exclude inline asm, which
    * may issue a `call` (clobbering the red zone) the optimizer can't see. slim
    * needs no such guard: it has no locals there and the return address lives on
    * the stack at [rbp+8], not in a clobberable register. */
   a->redzone_leaf = !a->slim_frame && a->abi->shadow_space == 0 && frame &&
                     frame->is_leaf && !frame->has_asm && !a->frame.has_alloca &&
                     a->frame.max_outgoing == 0 && frame_size <= 128u;
   a->prologue_pos = mc->pos(mc);
   nbytes = x64_build_prologue(a, buf, sizeof buf, frame_size, cs_int, n_int,
                               cs_fp, n_fp, a->slim_frame || a->redzone_leaf,
                               &chkstk_disp_pos);
   mc->emit_bytes(mc, buf, nbytes);
   if (chkstk_disp_pos != (u32)-1) {
     ObjSymId chk = x64_chkstk_sym(t);
     mc->emit_reloc_at(mc, mc->section_id, a->prologue_pos + chkstk_disp_pos,
                       R_X64_PLT32, chk, -4, 1, 0);
   }
   a->prologue_nbytes = nbytes; /* exact length: used for the CFI post offset */
   x64_emit_variadic_reg_saves(a);
   native_frame_set_final(&a->frame);
 }
 
 static void x64_func_end(NativeTarget* t) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   ObjBuilder* obj = t->obj;
   ObjSecId sec = a->func->text_section_id;
   Reg cs_int[X64_MAX_CS_INT_REGS], cs_fp[X64_MAX_CS_FP_REGS];
   u32 n_int = x64_collect_int_saves(a, cs_int);
   u32 n_fp = x64_collect_fp_saves(a, cs_fp);
   u32 frame_size = x64_compute_frame_size(a, n_int, n_fp);
   u32 xmm_base = x64_xmm_base(a, n_fp);
   u32 end;
   i32 i;
   a->frame_size_final = frame_size;
 
   /* Epilogue. */
   mc->label_place(mc, a->epilogue_label);
   for (i = (i32)n_fp - 1; i >= 0; --i) {
     i32 off = -(i32)xmm_base - (i32)(i + 1) * 16;
     emit_sse_load(mc, 0, 0x28, cs_fp[i], X64_RBP, off); /* movaps */
   }
   for (i = (i32)n_int - 1; i >= 0; --i) {
     i32 off = -(i32)xmm_base - (i32)n_fp * 16 - (i32)(i + 1) * 8;
     emit_mov_load(mc, 8, 0, cs_int[i], X64_RBP, off);
   }
   emit_leave(mc);
   emit_ret(mc);
 
   /* Patch the single-pass prologue placeholder. */
   if (!a->frame.known_frame) {
     u8 buf[X64_PROLOGUE_BYTES_WIN64];
     u32 chkstk_disp_pos;
     u32 nbytes;
     u32 k;
     for (k = 0; k < a->prologue_nbytes; ++k) buf[k] = X64_NOP1;
     /* Single-pass path never selects a slim/red-zone tier (it cannot know the
      * frame up front), so it always emits the full reservation. */
     nbytes = x64_build_prologue(a, buf, a->prologue_nbytes, frame_size, cs_int,
                                 n_int, cs_fp, n_fp, 0, &chkstk_disp_pos);
     (void)nbytes;
     obj_patch(obj, sec, a->prologue_pos, buf, a->prologue_nbytes);
     if (chkstk_disp_pos != (u32)-1) {
       ObjSymId chk = x64_chkstk_sym(t);
       mc->emit_reloc_at(mc, sec, a->prologue_pos + chkstk_disp_pos, R_X64_PLT32,
                         chk, -4, 1, 0);
     }
   }
 
   /* Patch alloca disp32s: lea dst, [rsp + max_outgoing]. */
   {
     u32 mo = align_up_u32(a->frame.max_outgoing, 16u);
     u32 k;
     for (k = 0; k < a->npatches; ++k) {
       u8 dbuf[4];
       wr_u32_le(dbuf, mo);
       obj_patch(obj, sec, a->patches[k].pos, dbuf, 4);
     }
   }
 
   /* CFI: after the prologue, CFA = rbp + 16; rbp at cfa-16, ra at cfa-8. */
   if (mc->cfi_set_next_pc_offset && mc->cfi_def_cfa && mc->cfi_offset) {
     /* Body starts past the prologue. prologue_nbytes is the reserved NOP-region
      * size on the single-pass path and the exact prologue length on the
      * known-frame path (set in x64_func_begin_known_frame). */
     u32 post = a->prologue_pos + a->prologue_nbytes;
     u32 k;
     mc->cfi_set_next_pc_offset(mc, post - a->func_start);
     /* CFI register operands are DWARF numbers, which differ from the x86-64
      * hardware encoding for rbp/rsp/rsi/rdi/rcx/rdx (e.g. rbp is HW 5 but
      * DWARF 6). Map every hardware GPR through x64_dwarf_from_hw_gpr; rip's
      * DWARF number (16) is already correct. */
     mc->cfi_def_cfa(mc, x64_dwarf_from_hw_gpr(X64_RBP), 16);
     mc->cfi_offset(mc, x64_dwarf_from_hw_gpr(X64_RBP), -16);
     mc->cfi_offset(mc, 16u /* rip */, -8);
     for (k = 0; k < n_int; ++k) {
       i32 off = -(i32)xmm_base - (i32)n_fp * 16 - (i32)(k + 1u) * 8;
       mc->cfi_offset(mc, x64_dwarf_from_hw_gpr(cs_int[k]), off);
     }
   }
 
   end = mc->pos(mc);
   obj_symbol_define(obj, a->func->sym, sec, (u64)a->func_start,
                     (u64)(end - a->func_start));
   if (a->func->atomize)
     obj_atom_define(obj, sec, a->func_start, end - a->func_start, a->func->sym,
                     0);
   if (mc->debug) debug_func_pc_range(mc->debug, sec, a->func_start, end);
   if (mc->cfi_endproc) mc->cfi_endproc(mc);
   mc_end_function(mc);
   a->func = NULL;
 }
 
 /* ============================ params / ABI helpers
  * ============================
  */
 
 /* Win64 shares one arg-slot index across int and FP. Keep cursors in lockstep.
  */
 static void x64_sync_slot(const X64ABIRegs* abi, u32* next_int, u32* next_fp) {
   u32 m;
   if (!abi->slot_shared_int_fp) return;
   m = *next_int > *next_fp ? *next_int : *next_fp;
   *next_int = m;
   *next_fp = m;
 }
 
 static const ABIArgInfo* x64_param_abi(NativeTarget* t, const ABIFuncInfo* abi,
                                        const NativeCallDesc* desc, u32 i,
                                        ABIArgInfo* scratch) {
   int variadic = abi && i >= abi->nparams;
   if (abi && i < abi->nparams) return &abi->params[i];
   (void)variadic;
   memset(scratch, 0, sizeof *scratch);
   scratch->kind = ABI_ARG_DIRECT;
   scratch->nparts = 1;
   scratch->parts = arena_zarray(t->c->tu, ABIArgPart, 1);
   ((ABIArgPart*)scratch->parts)[0].cls =
       cg_type_is_float(t->c, desc->args[i].type) ? ABI_CLASS_FP : ABI_CLASS_INT;
   ((ABIArgPart*)scratch->parts)[0].loc = ABI_LOC_REG;
   ((ABIArgPart*)scratch->parts)[0].size =
       native_type_size(t, desc->args[i].type);
   ((ABIArgPart*)scratch->parts)[0].align =
       native_type_align(t, desc->args[i].type);
   return scratch;
 }
 
 static KitCgTypeId x64_part_scalar_type(const ABIArgPart* part) {
   if (part->cls == ABI_CLASS_FP)
     return part->size <= 4u ? builtin_id(KIT_CG_BUILTIN_F32)
                             : builtin_id(KIT_CG_BUILTIN_F64);
   switch (part->size) {
     case 1u:
       return builtin_id(KIT_CG_BUILTIN_I8);
     case 2u:
       return builtin_id(KIT_CG_BUILTIN_I16);
     case 4u:
       return builtin_id(KIT_CG_BUILTIN_I32);
     default:
       return builtin_id(KIT_CG_BUILTIN_I64);
   }
 }
 
 /* Is the whole DIRECT arg forced to the stack (not enough reg slots)? */
 static int x64_direct_to_stack(const X64ABIRegs* abi, const ABIArgInfo* ai,
                                u32 next_int, u32 next_fp) {
   u32 need_int, need_fp;
   x64_abi_direct_reg_need(ai, &need_int, &need_fp);
   return next_int + need_int > abi->n_int_args ||
          next_fp + need_fp > abi->n_fp_args;
 }
 
 /* Outgoing stack bytes a call uses (16-aligned), per the ABI. */
 static u32 x64_call_stack_size(NativeTarget* t, const NativeCallDesc* desc) {
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, desc->fn_type);
   const X64ABIRegs* aregs = x64_abi_for_os(t->c->target.os);
   u32 next_int = (abi && abi->has_sret) ? 1u : 0u;
   u32 next_fp = 0;
   u32 stack = aregs->shadow_space;
   u32 i;
   x64_sync_slot(aregs, &next_int, &next_fp);
   for (i = 0; i < desc->nargs; ++i) {
     ABIArgInfo tmp;
     const ABIArgInfo* ai = x64_param_abi(t, abi, desc, i, &tmp);
     u16 p;
     if (ai->kind == ABI_ARG_IGNORE) continue;
     if (ai->kind == ABI_ARG_INDIRECT) {
       if (next_int < aregs->n_int_args)
         ++next_int;
       else
         stack += 8u;
       x64_sync_slot(aregs, &next_int, &next_fp);
       continue;
     }
     if (ai->kind == ABI_ARG_DIRECT &&
         x64_direct_to_stack(aregs, ai, next_int, next_fp)) {
       stack += (u32)ai->nparts * 8u;
       continue;
     }
     for (p = 0; p < ai->nparts; ++p) {
       const ABIArgPart* part = &ai->parts[p];
       if (part->cls == ABI_CLASS_FP) {
         if (next_fp < aregs->n_fp_args)
           ++next_fp;
         else
           stack += 8u;
       } else {
         if (next_int < aregs->n_int_args)
           ++next_int;
         else
           stack += 8u;
       }
       x64_sync_slot(aregs, &next_int, &next_fp);
     }
   }
   return align_up_u32(stack, 16u);
 }
 
 static u32 x64_call_stack_bytes(NativeTarget* t, const NativeCallDesc* desc) {
   return x64_call_stack_size(t, desc);
 }
 
 static u32 x64_signature_stack_bytes(NativeTarget* t, KitCgTypeId fn_type,
                                      int* variadic, u32* nparams) {
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, fn_type);
   NativeCallDesc d;
   if (variadic) *variadic = abi ? (int)abi->variadic : 0;
   if (nparams) *nparams = abi ? abi->nparams : 0u;
   memset(&d, 0, sizeof d);
   d.fn_type = fn_type;
   d.nargs = abi ? abi->nparams : 0u;
   if (d.nargs) d.args = arena_zarray(t->c->tu, NativeLoc, d.nargs);
   return x64_call_stack_size(t, &d);
 }
 
 /* Resolve a NativeLoc to an addressable NativeAddr (frame/stack/addr). */
 static NativeAddr x64_loc_addr(X64NativeTarget* a, NativeLoc loc, u32 offset) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   switch ((NativeLocKind)loc.kind) {
     case NATIVE_LOC_FRAME:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = loc.v.frame;
       addr.base_type = loc.type;
       addr.offset = (i32)offset;
       return addr;
     case NATIVE_LOC_STACK:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = loc.v.stack.slot;
       addr.base_type = loc.type;
       addr.offset = loc.v.stack.offset + (i32)offset;
       return addr;
     case NATIVE_LOC_ADDR:
       addr = loc.v.addr;
       addr.offset += (i32)offset;
       return addr;
     default:
       x64_panic(a, "location is not addressable");
   }
   return addr;
 }
 
 static void x64_load_part(NativeTarget* t, NativeLoc dst, NativeLoc src,
                           u32 offset, u32 size) {
   X64NativeTarget* a = x64_of(t);
   if (src.kind == NATIVE_LOC_REG) {
     x64_move(t, dst, src);
     return;
   }
   if (src.kind == NATIVE_LOC_FRAME || src.kind == NATIVE_LOC_STACK ||
       src.kind == NATIVE_LOC_ADDR) {
     NativeAddr addr = x64_loc_addr(a, src, offset);
     addr.base_type = dst.type;
     x64_emit_mem(a, 1, dst, addr, native_mem_for_type(t, dst.type, size));
     return;
   }
   if (src.kind == NATIVE_LOC_IMM) {
     x64_emit_load_imm(t->mc, x64_is_64(t, dst.type) ? 1 : 0, loc_reg(dst),
                       src.v.imm);
     return;
   }
   x64_panic(a, "unsupported part source");
 }
 
 static void x64_store_part(NativeTarget* t, NativeLoc dst, NativeLoc src,
                            u32 offset, u32 size) {
   X64NativeTarget* a = x64_of(t);
   if (dst.kind == NATIVE_LOC_FRAME || dst.kind == NATIVE_LOC_STACK ||
       dst.kind == NATIVE_LOC_ADDR) {
     NativeAddr addr = x64_loc_addr(a, dst, offset);
     addr.base_type = src.type;
     x64_emit_mem(a, 0, src, addr, native_mem_for_type(t, src.type, size));
     return;
   }
   if (dst.kind == NATIVE_LOC_REG) {
     x64_move(t, dst, src);
     return;
   }
   x64_panic(a, "unsupported part destination");
 }
 
 static void x64_addr_of_loc(NativeTarget* t, NativeLoc dst, NativeLoc src) {
   NativeAddr addr = x64_loc_addr(x64_of(t), src, 0);
   x64_load_addr(t, dst, addr);
 }
 
 static void x64_store_outgoing_part(NativeTarget* t, int tail_call,
                                     u32 stack_off, NativeLoc src, u32 size) {
   X64NativeTarget* a = x64_of(t);
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_REG;
   addr.base_type = src.type;
   if (tail_call) {
     /* A sibling call reuses the caller's frame: its outgoing stack args land in
      * the caller's incoming-arg window. `stack_off` already includes the
      * shadow-space prefix (the outgoing cursor starts at shadow_space), so the
      * window address is [rbp + 16 + stack_off] — the same bytes the tail-callee
      * reads once `leave` has restored rsp to the return address. */
     addr.base.reg = X64_RBP;
     addr.offset = (i32)(16u + stack_off);
   } else {
     addr.base.reg = X64_RSP;
     addr.offset = (i32)stack_off;
   }
   x64_emit_mem(a, 0, src, addr, native_mem_for_type(t, src.type, size));
 }
 
 /* NativeTarget bind_param: route incoming param (ABI loc) into dst. */
 static void x64_emit_reg_arg_moves(NativeTarget* t, NativeArgMove* moves, u32 n,
                                    Reg int_scratch);
 
 /* Defer a register-destination param bind for the parallel-copy flush in
  * x64_bind_params_end. `src` is the incoming location (an arg register, or a
  * NATIVE_LOC_ADDR for an incoming stack slot). */
 static void x64_defer_reg_bind(X64NativeTarget* a, NativeLoc dst, NativeLoc src,
                                u32 size) {
   NativeArgMove* m;
   if (a->nbind_moves >= X64_MAX_BIND_MOVES)
     x64_panic(a, "too many register parameter binds");
   m = &a->bind_moves[a->nbind_moves++];
   memset(m, 0, sizeof *m);
   m->dst = dst;
   m->src = src;
   m->size = size;
 }
 
 /* Incoming stack-arg source as a NATIVE_LOC_ADDR ([rbp + bias + stack_off]). */
 static NativeLoc x64_incoming_stack_loc(KitCgTypeId type, NativeAllocClass cls,
                                         i32 off) {
   NativeLoc l;
   memset(&l, 0, sizeof l);
   l.kind = NATIVE_LOC_ADDR;
   l.cls = (u8)cls;
   l.type = type;
   l.v.addr.base_kind = NATIVE_ADDR_BASE_REG;
   l.v.addr.base.reg = X64_RBP;
   l.v.addr.base_type = type;
   l.v.addr.offset = off;
   return l;
 }
 
 static void x64_bind_native_param(NativeTarget* t, const CGParamDesc* p,
                                   NativeLoc dst) {
   X64NativeTarget* a = x64_of(t);
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, a->func->fn_type);
   const ABIArgInfo* ai =
       p->index < abi->nparams ? &abi->params[p->index] : NULL;
   int to_reg = dst.kind == NATIVE_LOC_REG;
   /* Incoming stack args sit above the saved rbp + return addr (+16); Win64
    * additionally reserves 32B of home space. */
   i32 incoming_bias = (i32)(16u + a->abi->shadow_space);
   u16 i;
   if (!ai || ai->kind == ABI_ARG_IGNORE) return;
 
   if (ai->kind == ABI_ARG_INDIRECT) {
     /* Incoming pointer to a byval copy: load pointer, memcpy into dst frame. */
     u32 ptr_reg;
     NativeAddr d_addr, from;
     AggregateAccess access;
     if (a->next_param_int < a->abi->n_int_args) {
       ptr_reg = a->abi->int_args[a->next_param_int++];
     } else {
       ptr_reg = X64_R11;
       emit_mov_load(t->mc, 8, 0, ptr_reg, X64_RBP,
                     incoming_bias + (i32)a->next_param_stack);
       a->next_param_stack += 8u;
     }
     x64_sync_slot(a->abi, &a->next_param_int, &a->next_param_fp);
     if (dst.kind != NATIVE_LOC_FRAME)
       x64_panic(a, "indirect parameter requires a frame destination");
     memset(&d_addr, 0, sizeof d_addr);
     d_addr.base_kind = NATIVE_ADDR_BASE_FRAME;
     d_addr.base.frame = dst.v.frame;
     d_addr.base_type = p->type;
     memset(&from, 0, sizeof from);
     from.base_kind = NATIVE_ADDR_BASE_REG;
     from.base.reg = ptr_reg;
     from.base_type = p->type;
     memset(&access, 0, sizeof access);
     access.type = p->type;
     access.size = p->size ? p->size : (u32)cg_type_size(t->c, p->type);
     access.align = p->align ? p->align : native_type_align(t, p->type);
     x64_copy_bytes(t, d_addr, from, access);
     return;
   }
 
   if (ai->kind == ABI_ARG_DIRECT &&
       x64_direct_to_stack(a->abi, ai, a->next_param_int, a->next_param_fp)) {
     /* Whole arg on the stack. */
     for (i = 0; i < ai->nparts; ++i) {
       const ABIArgPart* part = &ai->parts[i];
       NativeAllocClass cls =
           part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
       NativeLoc isrc = x64_incoming_stack_loc(
           p->type, cls, incoming_bias + (i32)a->next_param_stack);
       a->next_param_stack += 8u;
       if (dst.kind == NATIVE_LOC_NONE) {
         /* unused */
       } else if (to_reg) {
         /* Defer: a register dst may be another param's incoming reg. */
         x64_defer_reg_bind(
             a,
             native_loc_reg(dst.type ? dst.type : p->type,
                            (NativeAllocClass)dst.cls, (Reg)dst.v.reg),
             isrc, part->size);
       } else {
         /* Frame dst: load to scratch then store (memory dst is never a cycle
          * source, so emit eagerly — it only reads the incoming slot). */
         Reg tmp = cls == NATIVE_REG_FP ? X64_TMP_FP : X64_TMP_INT;
         NativeLoc tloc = native_loc_reg(p->type, cls, tmp);
         x64_load_part(t, tloc, isrc, 0, part->size);
         x64_store_part(
             t, native_loc_stack(p->type, dst.v.frame, (i32)part->src_offset),
             tloc, 0, part->size);
       }
     }
     return;
   }
 
   for (i = 0; i < ai->nparts; ++i) {
     const ABIArgPart* part = &ai->parts[i];
     NativeAllocClass cls =
         part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
     NativeLoc
         src; /* incoming: arg register, or NATIVE_LOC_ADDR for a stack arg */
     if (cls == NATIVE_REG_FP && a->next_param_fp < a->abi->n_fp_args) {
       src = native_loc_reg(p->type, cls, (Reg)(X64_XMM0 + a->next_param_fp++));
     } else if (cls == NATIVE_REG_INT &&
                a->next_param_int < a->abi->n_int_args) {
       src = native_loc_reg(p->type, cls, a->abi->int_args[a->next_param_int++]);
     } else {
       src = x64_incoming_stack_loc(p->type, cls,
                                    incoming_bias + (i32)a->next_param_stack);
       a->next_param_stack += 8u;
     }
     x64_sync_slot(a->abi, &a->next_param_int, &a->next_param_fp);
     if (dst.kind == NATIVE_LOC_NONE) {
       /* unused parameter; cursors advanced */
     } else if (to_reg) {
       /* Defer the register bind: the allocator may rotate params across the
        * incoming arg registers, so a per-param move could clobber a register
        * another bind still needs. x64_bind_params_end resolves them together as
        * a parallel copy. */
       x64_defer_reg_bind(
           a,
           native_loc_reg(dst.type ? dst.type : p->type,
                          (NativeAllocClass)dst.cls, (Reg)dst.v.reg),
           src, part->size);
     } else if (src.kind == NATIVE_LOC_REG) {
       x64_store_part(
           t, native_loc_stack(p->type, dst.v.frame, (i32)part->src_offset), src,
           0, part->size);
     } else {
       /* Stack source -> frame dst: load to scratch, then store. */
       Reg tmp = cls == NATIVE_REG_FP ? X64_TMP_FP : X64_TMP_INT;
       NativeLoc tloc = native_loc_reg(p->type, cls, tmp);
       x64_load_part(t, tloc, src, 0, part->size);
       x64_store_part(
           t, native_loc_stack(p->type, dst.v.frame, (i32)part->src_offset),
           tloc, 0, part->size);
     }
   }
   a->incoming_stack_size = align_up_u32(a->next_param_stack, 16u);
 }
 
 /* Flush the deferred register-destination param binds as a parallel copy (the
  * shared scheduler breaks any cycle the allocator's rotation created through
  * the int/fp emit scratch). Frame-dst and indirect binds were emitted eagerly
  * in bind_param — they only read incoming registers, so they precede this. */
 static void x64_bind_params_end(NativeTarget* t) {
   X64NativeTarget* a = x64_of(t);
   /* No callee is staged during entry binds, so r11 is free as the cycle
    * scratch. */
   if (a->nbind_moves)
     x64_emit_reg_arg_moves(t, a->bind_moves, a->nbind_moves, X64_TMP_INT2);
   a->nbind_moves = 0;
 }
 
 /* ============================ calls / returns ============================ */
 
 typedef NativeArgMove X64ArgMove;
 
 static void x64_emit_one_arg_move(NativeTarget* t, const NativeArgMove* m) {
   if (m->is_addr) {
     x64_addr_of_loc(t, m->dst, m->src);
   } else {
     x64_load_part(t, m->dst, m->src, m->src_offset, m->size);
   }
   if (m->dup_to_gpr) {
     /* movq gpr, xmm: 66 REX.W 0F 7E /r (xmm in reg field). */
     emit_sse_rr_w(t->mc, 0x66, 0x7E, 1, loc_reg(m->dst), m->dup_gpr);
   }
 }
 
 /* Parallel-copy register arg moves via the shared scheduler. `int_scratch` is
  * the register used to break an integer cycle: normally r11, but rax when an
  * indirect callee is staged in r11 (rax is never a SysV int arg register and
  * the variadic AL count is written only after the moves). */
 static void x64_emit_reg_arg_moves(NativeTarget* t, NativeArgMove* moves, u32 n,
                                    Reg int_scratch) {
   NativeArgShuffle s;
   if (n > X64_MAX_REG_ARG_MOVES) x64_panic(x64_of(t), "too many register args");
   memset(&s, 0, sizeof s);
   s.t = t;
   s.emit_one = x64_emit_one_arg_move;
   s.reg_move = x64_move;
   s.scratch[NATIVE_REG_INT] = int_scratch;
   s.scratch[NATIVE_REG_FP] = X64_TMP_FP;
   native_arg_shuffle(&s, moves, n);
 }
 
 /* Clobber masks: per-call all caller-saved regs are clobbered. */
 static u32 x64_clobber_mask(const X64ABIRegs* abi, NativeAllocClass cls) {
   u32 mask = 0, r;
   if (cls == NATIVE_REG_INT) {
     for (r = 0; r < 16u; ++r) {
       if (r == X64_RSP || r == X64_RBP) continue;
       if ((abi->cs_int_mask & (1ull << r)) == 0) mask |= 1u << r;
     }
   } else if (cls == NATIVE_REG_FP) {
     for (r = 0; r < 16u; ++r)
       if ((abi->cs_fp_mask & (1ull << r)) == 0) mask |= 1u << r;
   }
   return mask;
 }
 
 static u32 x64_return_mask(const ABIFuncInfo* abi, NativeAllocClass cls) {
   u32 mask = 0, ni = 0, nf = 0;
   static const u32 iregs[2] = {X64_RAX, X64_RDX};
   u16 i;
   if (!abi || abi->ret.kind == ABI_ARG_IGNORE ||
       abi->ret.kind == ABI_ARG_INDIRECT)
     return 0;
   for (i = 0; i < abi->ret.nparts; ++i) {
     const ABIArgPart* p = &abi->ret.parts[i];
     if (cls == NATIVE_REG_INT && p->cls == ABI_CLASS_INT && ni < 2)
       mask |= 1u << iregs[ni++];
     else if (cls == NATIVE_REG_FP && p->cls == ABI_CLASS_FP && nf < 2)
       mask |= 1u << (X64_XMM0 + nf++);
   }
   return mask;
 }
 
 static void x64_plan_call(NativeTarget* t, const NativeCallDesc* desc,
                           NativeCallPlan* plan) {
   X64NativeTarget* a = x64_of(t);
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, desc->fn_type);
   const X64ABIRegs* aregs = a->abi ? a->abi : x64_abi_for_os(t->c->target.os);
   NativeCallPlanRet* rets;
   KitCgTypeId i64t = builtin_id(KIT_CG_BUILTIN_I64);
   u32 c;
   memset(plan, 0, sizeof *plan);
   rets = desc->nresults ? arena_zarray(t->c->tu, NativeCallPlanRet, 4) : NULL;
   plan->callee = desc->callee;
   plan->rets = rets;
   plan->flags = desc->flags;
   plan->has_sret = abi && abi->has_sret;
   plan->is_variadic = abi && abi->variadic;
   plan->stack_arg_size = x64_call_stack_size(t, desc);
   if (plan->stack_arg_size > a->frame.max_outgoing)
     a->frame.max_outgoing = plan->stack_arg_size;
   for (c = 0; c < NATIVE_CALL_PLAN_CLASSES; ++c) {
     plan->clobber_mask[c] = x64_clobber_mask(aregs, (NativeAllocClass)c);
     plan->return_mask[c] = x64_return_mask(abi, (NativeAllocClass)c);
   }
   /* Indirect callee in a clobbered/arg register would be lost; stage in r11. */
   if (plan->callee.kind == NATIVE_LOC_REG &&
       (NativeAllocClass)plan->callee.cls == NATIVE_REG_INT &&
       plan->callee.v.reg != X64_R11) {
     NativeLoc scratch =
         native_loc_reg(plan->callee.type, NATIVE_REG_INT, X64_R11);
     x64_move(t, scratch, plan->callee);
     plan->callee = scratch;
   }
   {
     u32 next_int = (abi && abi->has_sret) ? 1u : 0u;
     u32 next_fp = 0, stack = aregs->shadow_space, nmoves = 0, i;
     int tail = (desc->flags & CG_CALL_TAIL) != 0;
     u16 p;
     X64ArgMove moves[X64_MAX_REG_ARG_MOVES];
     x64_sync_slot(aregs, &next_int, &next_fp);
     for (i = 0; i < desc->nargs; ++i) {
       ABIArgInfo tmp;
       const ABIArgInfo* ai = x64_param_abi(t, abi, desc, i, &tmp);
       int variadic_arg = abi && i >= abi->nparams;
       if (ai->kind == ABI_ARG_IGNORE) continue;
       if (ai->kind == ABI_ARG_INDIRECT) {
         if (next_int < aregs->n_int_args) {
           X64ArgMove* m = &moves[nmoves++];
           memset(m, 0, sizeof *m);
           m->dst =
               native_loc_reg(i64t, NATIVE_REG_INT, aregs->int_args[next_int++]);
           m->src = desc->args[i];
           m->size = 8;
           m->is_addr = 1;
         } else {
           NativeLoc ptr = native_loc_reg(i64t, NATIVE_REG_INT, X64_RAX);
           x64_addr_of_loc(t, ptr, desc->args[i]);
           x64_store_outgoing_part(t, tail, stack, ptr, 8);
           stack += 8u;
         }
         x64_sync_slot(aregs, &next_int, &next_fp);
         continue;
       }
       if (ai->kind == ABI_ARG_DIRECT &&
           x64_direct_to_stack(aregs, ai, next_int, next_fp)) {
         for (p = 0; p < ai->nparts; ++p) {
           const ABIArgPart* part = &ai->parts[p];
           NativeAllocClass cls =
               part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
           Reg tmp = cls == NATIVE_REG_FP ? X64_TMP_FP : X64_TMP_INT;
           NativeLoc tmpreg = native_loc_reg(desc->args[i].type, cls, tmp);
           x64_load_part(t, tmpreg, desc->args[i], part->src_offset, part->size);
           x64_store_outgoing_part(t, tail, stack, tmpreg, part->size);
           stack += 8u;
         }
         continue;
       }
       for (p = 0; p < ai->nparts; ++p) {
         const ABIArgPart* part = &ai->parts[p];
         NativeAllocClass cls =
             part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
         if (cls == NATIVE_REG_FP && next_fp < aregs->n_fp_args) {
           X64ArgMove* m = &moves[nmoves++];
           u32 slot = next_fp;
           memset(m, 0, sizeof *m);
           m->dst = native_loc_reg(desc->args[i].type, cls,
                                   (Reg)(X64_XMM0 + next_fp++));
           m->src = desc->args[i];
           m->src_offset = part->src_offset;
           m->size = part->size;
           if (aregs->vararg_fp_dup_to_gpr && variadic_arg &&
               slot < aregs->n_int_args) {
             m->dup_to_gpr = 1;
             m->dup_gpr = aregs->int_args[slot];
           }
           x64_sync_slot(aregs, &next_int, &next_fp);
         } else if (cls == NATIVE_REG_INT && next_int < aregs->n_int_args) {
           X64ArgMove* m = &moves[nmoves++];
           memset(m, 0, sizeof *m);
           m->dst = native_loc_reg(desc->args[i].type, cls,
                                   aregs->int_args[next_int++]);
           m->src = desc->args[i];
           m->src_offset = part->src_offset;
           m->size = part->size;
           x64_sync_slot(aregs, &next_int, &next_fp);
         } else {
           Reg tmp = cls == NATIVE_REG_FP ? X64_TMP_FP : X64_TMP_INT;
           NativeLoc tmpreg = native_loc_reg(desc->args[i].type, cls, tmp);
           x64_load_part(t, tmpreg, desc->args[i], part->src_offset, part->size);
           x64_store_outgoing_part(t, tail, stack, tmpreg, part->size);
           stack += 8u;
           x64_sync_slot(aregs, &next_int, &next_fp);
         }
       }
     }
     /* If an indirect callee was staged in r11 above, the cycle scratch must
      * avoid it; rax is free here (not an int arg reg; AL count comes later). */
     x64_emit_reg_arg_moves(
         t, moves, nmoves,
         (plan->callee.kind == NATIVE_LOC_REG && plan->callee.v.reg == X64_R11)
             ? X64_TMP_INT
             : X64_TMP_INT2);
     if (abi && abi->has_sret) {
       /* sret pointer in the first int-arg reg. A tail call forwards the
        * caller's own incoming sret pointer (spilled at entry); otherwise pass
        * the address of this call's result slot. */
       NativeLoc sret = native_loc_reg(i64t, NATIVE_REG_INT, aregs->int_args[0]);
       if (tail)
         x64_load_part(t, sret, native_loc_stack(i64t, a->sret_ptr_slot, 0), 0,
                       8);
       else if (desc->nresults)
         x64_addr_of_loc(t, sret, desc->results[0]);
     }
     /* Variadic call: AL = number of vector regs used. */
     if (abi && abi->variadic)
       x64_emit_load_imm(t->mc, 0, X64_RAX, (i64)next_fp);
   }
   /* Return value receipt. */
   if (abi && abi->ret.kind == ABI_ARG_DIRECT && desc->nresults) {
     u32 nr = 0, ni = 0, nf = 0;
     static const u32 ret_int_regs[2] = {X64_RAX, X64_RDX};
     u16 p;
     for (p = 0; p < abi->ret.nparts; ++p) {
       const ABIArgPart* part = &abi->ret.parts[p];
       NativeAllocClass cls =
           part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
       KitCgTypeId pty = x64_part_scalar_type(part);
       Reg rreg = cls == NATIVE_REG_FP ? (Reg)(X64_XMM0 + nf++)
                                       : (Reg)ret_int_regs[ni++];
       rets[nr].src = native_loc_reg(pty, cls, rreg);
       rets[nr].dst = desc->results[0];
       if (rets[nr].dst.kind == NATIVE_LOC_FRAME)
         rets[nr].dst = native_loc_stack(pty, desc->results[0].v.frame,
                                         (i32)part->src_offset);
       else if (rets[nr].dst.kind == NATIVE_LOC_STACK) {
         rets[nr].dst.v.stack.offset += (i32)part->src_offset;
         rets[nr].dst.type = pty;
       }
       rets[nr].mem = native_mem_for_type(t, pty, part->size);
       nr++;
     }
     plan->nrets = nr;
   } else if (abi && abi->ret.kind == ABI_ARG_IGNORE) {
     plan->nrets = 0;
   } else if (!abi && desc->nresults) {
     rets[0].src =
         native_loc_reg(desc->results[0].type, NATIVE_REG_INT, X64_RAX);
     rets[0].dst = desc->results[0];
     rets[0].mem = native_mem_for_type(t, desc->results[0].type, 0);
     plan->nrets = 1;
   }
 }
 
 /* Emit a sibling (tail) call: tear the frame down and jump (no call) to the
  * callee. Outgoing args are already in arg regs / the caller's incoming-arg
  * window. `leave` (mov rsp,rbp; pop rbp) restores the caller's rbp and leaves
  * rsp at the return address — frame_size-independent, so no func_end patch. */
 static void x64_emit_tail_site(NativeTarget* t, NativeLoc callee) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   ObjSecId sec = mc->section_id;
   /* Restore callee-saves before the frame teardown (O1 path; none at -O0).
    * Their rbp-relative offsets are frame-size-independent, and the indirect
    * callee was staged in r11 by plan_call — a caller-saved scratch — so these
    * restores never clobber it. Mirrors the x64_func_end epilogue. */
   Reg cs_int[X64_MAX_CS_INT_REGS], cs_fp[X64_MAX_CS_FP_REGS];
   u32 n_int = x64_collect_int_saves(a, cs_int);
   u32 n_fp = x64_collect_fp_saves(a, cs_fp);
   u32 xmm_base = x64_xmm_base(a, n_fp);
   i32 i;
   for (i = (i32)n_fp - 1; i >= 0; --i)
     emit_sse_load(mc, 0, 0x28, cs_fp[i], X64_RBP,
                   -(i32)xmm_base - (i32)(i + 1) * 16); /* movaps */
   for (i = (i32)n_int - 1; i >= 0; --i)
     emit_mov_load(mc, 8, 0, cs_int[i], X64_RBP,
                   -(i32)xmm_base - (i32)n_fp * 16 - (i32)(i + 1) * 8);
   emit_leave(mc);
   if (callee.kind == NATIVE_LOC_GLOBAL) {
     u8 op = X64_OPC_JMP_REL32;
     u32 disp_pos;
     mc->emit_bytes(mc, &op, 1);
     disp_pos = mc->pos(mc);
     emit_u32le(mc, 0);
     mc->emit_reloc_at(mc, sec, disp_pos, R_X64_PLT32, callee.v.global.sym,
                       callee.v.global.addend - 4, 1, 0);
   } else if (callee.kind == NATIVE_LOC_REG) {
     u32 r =
         loc_reg(callee); /* indirect callee was staged in r11 by plan_call */
     if (r & 8u) {
       u8 rex = X64_REX_BASE | X64_REX_B;
       mc->emit_bytes(mc, &rex, 1);
     }
     {
       u8 buf[2] = {X64_OP_JMP_RM64, modrm(3u, 4u, r & 7u)}; /* jmp r/m, /4 */
       mc->emit_bytes(mc, buf, 2);
     }
   } else {
     x64_panic(a, "unsupported tail call target");
   }
 }
 
 static void x64_emit_call(NativeTarget* t, const NativeCallPlan* plan) {
   MCEmitter* mc = t->mc;
   ObjSecId sec = mc->section_id;
   if (plan->flags & CG_CALL_TAIL) {
     x64_emit_tail_site(t, plan->callee);
     return;
   }
   if (plan->callee.kind == NATIVE_LOC_GLOBAL) {
     u8 op = X64_OPC_CALL_REL32;
     u32 disp_pos;
     mc->emit_bytes(mc, &op, 1);
     disp_pos = mc->pos(mc);
     emit_u32le(mc, 0);
     mc->emit_reloc_at(mc, sec, disp_pos, R_X64_PLT32, plan->callee.v.global.sym,
                       plan->callee.v.global.addend - 4, 1, 0);
     return;
   }
   if (plan->callee.kind == NATIVE_LOC_REG) {
     u32 r = loc_reg(plan->callee);
     if (r & 8u) {
       u8 rex = X64_REX_BASE | X64_REX_B;
       mc->emit_bytes(mc, &rex, 1);
     }
     {
       u8 buf[2] = {X64_OP_JMP_RM64, modrm(3u, 2u, r & 7u)}; /* call r/m, /2 */
       mc->emit_bytes(mc, buf, 2);
     }
     return;
   }
   x64_panic(x64_of(t), "unsupported call target");
 }
 
 static void x64_plan_ret(NativeTarget* t, const CGFuncDesc* fd,
                          const NativeLoc* value,
                          NativeCallPlanRet** out_rets, u32* out_nrets) {
   X64NativeTarget* a = x64_of(t);
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, fd->fn_type);
   NativeCallPlanRet* rets = NULL;
   u32 nr = 0;
   if (value) rets = arena_zarray(t->c->tu, NativeCallPlanRet, 4);
   if (value && abi && abi->ret.kind == ABI_ARG_INDIRECT) {
     /* sret: reload destination pointer (spilled at entry) into r11, memcpy the
      * source aggregate into [r11], and convention-return the pointer in rax. */
     KitCgTypeId i64t = builtin_id(KIT_CG_BUILTIN_I64);
     NativeLoc dstp = native_loc_reg(i64t, NATIVE_REG_INT, X64_R11);
     NativeLoc saved = native_loc_stack(i64t, a->sret_ptr_slot, 0);
     NativeAddr dst_addr, src_addr;
     AggregateAccess access;
     x64_load_part(t, dstp, saved, 0, 8);
     memset(&dst_addr, 0, sizeof dst_addr);
     dst_addr.base_kind = NATIVE_ADDR_BASE_REG;
     dst_addr.base.reg = X64_R11;
     dst_addr.base_type = value->type;
     src_addr = x64_loc_addr(a, *value, 0);
     src_addr.base_type = value->type;
     memset(&access, 0, sizeof access);
     access.type = value->type;
     access.size = (u32)cg_type_size(t->c, value->type);
     access.align = native_type_align(t, value->type);
     x64_copy_bytes(t, dst_addr, src_addr, access);
     /* rax = sret pointer. Reload it (copy_bytes clobbered r11/rax). */
     x64_load_part(t, native_loc_reg(i64t, NATIVE_REG_INT, X64_RAX), saved, 0,
                   8);
     *out_rets = NULL;
     *out_nrets = 0;
     return;
   }
   if (value && abi && abi->ret.kind == ABI_ARG_DIRECT) {
     u32 ni = 0, nf = 0;
     static const u32 ret_int_regs[2] = {X64_RAX, X64_RDX};
     u16 p;
     for (p = 0; p < abi->ret.nparts; ++p) {
       const ABIArgPart* part = &abi->ret.parts[p];
       NativeAllocClass cls =
           part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
       KitCgTypeId pty = x64_part_scalar_type(part);
       Reg rreg = cls == NATIVE_REG_FP ? (Reg)(X64_XMM0 + nf++)
                                       : (Reg)ret_int_regs[ni++];
       rets[nr].src = *value;
       if (rets[nr].src.kind == NATIVE_LOC_FRAME)
         rets[nr].src =
             native_loc_stack(pty, value->v.frame, (i32)part->src_offset);
       else if (rets[nr].src.kind == NATIVE_LOC_STACK) {
         rets[nr].src.v.stack.offset += (i32)part->src_offset;
         rets[nr].src.type = pty;
       }
       rets[nr].dst = native_loc_reg(pty, cls, rreg);
       rets[nr].mem = native_mem_for_type(t, pty, part->size);
       nr++;
     }
   } else if (value) {
     rets[0].src = *value;
     rets[0].dst = native_loc_reg(value->type, NATIVE_REG_INT, X64_RAX);
     rets[0].mem = native_mem_for_type(t, value->type, 0);
     nr = 1;
   }
   *out_rets = rets;
   *out_nrets = nr;
 }
 
 static void x64_ret(NativeTarget* t) {
   X64NativeTarget* a = x64_of(t);
   x64_jump(t, a->epilogue_label);
 }
 
 /* ============================ alloca ============================ */
 
 static void x64_alloca(NativeTarget* t, NativeLoc dst, NativeLoc size,
                        u32 align) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   u32 rsz = loc_reg(size);
   u32 rd = loc_reg(dst);
   u32 al = align ? align : 16u;
   if (al < 16u) al = 16u;
   if (al > 16u) x64_panic(a, "alloca align > 16 not supported");
   if (size.kind == NATIVE_LOC_IMM) {
     u64 aligned = ((u64)size.v.imm + 15u) & ~(u64)15u;
     if (aligned == 0) aligned = 16;
     /* sub rsp, imm32. */
     emit_rex(mc, 1, 0, 0, X64_RSP);
     {
       u8 buf[2] = {X64_OPC_ALU_IMM32, modrm(3u, X64_ALU_SUB_SUB, X64_RSP)};
       mc->emit_bytes(mc, buf, 2);
     }
     emit_u32le(mc, (u32)aligned);
   } else {
     /* rax = (size + 15) & ~15; sub rsp, rax. */
     emit_lea(mc, X64_RAX, rsz, 15);
     emit_rex(mc, 1, 0, 0, X64_RAX);
     {
       u8 buf[3] = {X64_OPC_ALU_IMM8, modrm(3u, X64_ALU_SUB_AND, X64_RAX), 0xF0};
       mc->emit_bytes(mc, buf, 3);
     }
     emit_alu_rr(mc, 1, X64_OPC_ALU_SUB, X64_RSP, X64_RAX);
   }
   a->frame.has_alloca = 1;
   /* lea dst, [rsp + max_outgoing] — disp32 patched in func_end. */
   if (a->npatches == a->patches_cap) {
     u32 cap = a->patches_cap ? a->patches_cap * 2u : 8u;
     X64Patch* nb = arena_zarray(t->c->tu, X64Patch, cap);
     if (a->patches) memcpy(nb, a->patches, sizeof(*nb) * a->npatches);
     a->patches = nb;
     a->patches_cap = cap;
   }
   emit_rex(mc, 1, rd, 0, X64_RSP);
   {
     u8 op = X64_OPC_LEA;
     mc->emit_bytes(mc, &op, 1);
   }
   {
     u8 mr = modrm(2u, rd & 7u, 4u);
     mc->emit_bytes(mc, &mr, 1);
   }
   {
     u8 s = sib(0u, 4u, X64_RSP);
     mc->emit_bytes(mc, &s, 1);
   }
   a->patches[a->npatches].kind = X64_PATCH_ALLOCA;
   a->patches[a->npatches].pos = mc->pos(mc);
   a->npatches++;
   a->nalloca++;
   emit_u32le(mc, 0); /* placeholder disp32 */
 }
 
 /* ============================ TLS ============================ */
 
 /* Win64 TLS Local-Exec (PE-COFF): TEB pointer -> _tls_index -> TLS block ->
  * lea &sym@SECREL. R11 is scratch. */
 static void x64_tls_addr_of_win64(NativeTarget* t, NativeLoc dst, ObjSymId sym,
                                   i64 addend) {
   MCEmitter* mc = t->mc;
   u32 sec = mc->section_id;
   u32 rd = loc_reg(dst);
   /* (1) mov rd, gs:[0x58]. */
   {
     u8 gs = 0x65;
     mc->emit_bytes(mc, &gs, 1);
     emit_rex(mc, 1, rd, 0, 0);
     {
       u8 op = X64_OPC_MOV_R_RM;
       mc->emit_bytes(mc, &op, 1);
     }
     {
       u8 mr = modrm(0u, rd & 7u, 4u);
       mc->emit_bytes(mc, &mr, 1);
     }
     {
       u8 s = sib(0u, 4u, 5u);
       mc->emit_bytes(mc, &s, 1);
     }
     emit_u32le(mc, 0x58u);
   }
   /* (2) mov r11d, [rip + _tls_index]. */
   {
     Sym idx_name = pool_intern_slice(t->c->global, SLICE_LIT("_tls_index"));
     ObjSymId idx_sym = obj_symbol_find(t->obj, idx_name);
     u8 rex_r, op, mr;
     u32 disp_pos;
     if (idx_sym == 0)
       idx_sym =
           obj_symbol(t->obj, idx_name, SB_GLOBAL, SK_UNDEF, OBJ_SEC_NONE, 0, 0);
     rex_r = X64_REX_BASE | X64_REX_R;
     mc->emit_bytes(mc, &rex_r, 1);
     op = X64_OPC_MOV_R_RM;
     mc->emit_bytes(mc, &op, 1);
     mr = modrm(0u, 3u, 5u); /* r11&7, rip-rel */
     mc->emit_bytes(mc, &mr, 1);
     disp_pos = mc->pos(mc);
     emit_u32le(mc, 0);
     mc->emit_reloc_at(mc, sec, disp_pos, R_PC32, idx_sym, -4, 1, 0);
   }
   /* (3) mov rd, [rd + r11*8]. */
   {
     u8 rex = X64_REX_BASE | X64_REX_W | X64_REX_X;
     u8 op;
     if (rd & 8u) rex |= X64_REX_R | X64_REX_B;
     mc->emit_bytes(mc, &rex, 1);
     op = X64_OPC_MOV_R_RM;
     mc->emit_bytes(mc, &op, 1);
     if ((rd & 7u) == 5u) {
       u8 mr = modrm(1u, rd & 7u, 4u);
       u8 s = sib(3u, 3u, rd & 7u);
       u8 zero = 0;
       mc->emit_bytes(mc, &mr, 1);
       mc->emit_bytes(mc, &s, 1);
       mc->emit_bytes(mc, &zero, 1);
     } else {
       u8 mr = modrm(0u, rd & 7u, 4u);
       u8 s = sib(3u, 3u, rd & 7u);
       mc->emit_bytes(mc, &mr, 1);
       mc->emit_bytes(mc, &s, 1);
     }
   }
   /* (4) lea rd, [rd + sym@SECREL]. */
   {
     u8 rex = X64_REX_BASE | X64_REX_W;
     u8 op;
     u32 disp_pos;
     if (rd & 8u) rex |= X64_REX_R | X64_REX_B;
     mc->emit_bytes(mc, &rex, 1);
     op = X64_OPC_LEA;
     mc->emit_bytes(mc, &op, 1);
     if ((rd & 7u) == 4u) {
       u8 mr = modrm(2u, rd & 7u, 4u);
       u8 s = sib(0u, 4u, rd & 7u);
       mc->emit_bytes(mc, &mr, 1);
       mc->emit_bytes(mc, &s, 1);
     } else {
       u8 mr = modrm(2u, rd & 7u, rd & 7u);
       mc->emit_bytes(mc, &mr, 1);
     }
     disp_pos = mc->pos(mc);
     emit_u32le(mc, 0);
     mc->emit_reloc_at(mc, sec, disp_pos, R_COFF_SECREL, sym, addend, 1, 0);
   }
 }
 
 /* x86-64 TLS Local-Exec: mov rd, fs:0; lea rd, [rd + sym@tpoff]. */
 static void x64_tls_addr_of(NativeTarget* t, NativeLoc dst, ObjSymId sym,
                             i64 addend) {
   MCEmitter* mc = t->mc;
   u32 sec = mc->section_id;
   u32 rd = loc_reg(dst);
   u32 disp_pos;
   if (obj_format_tls_model(t->c) == OBJ_TLS_WINDOWS_TEB) {
     x64_tls_addr_of_win64(t, dst, sym, addend);
     return;
   }
   /* mov rd, fs:[0]. */
   {
     u8 fs = 0x64;
     mc->emit_bytes(mc, &fs, 1);
     emit_rex(mc, 1, rd, 0, 0);
     {
       u8 op = X64_OPC_MOV_R_RM;
       mc->emit_bytes(mc, &op, 1);
     }
     {
       u8 mr = modrm(0u, rd & 7u, 4u);
       mc->emit_bytes(mc, &mr, 1);
     }
     {
       u8 s = sib(0u, 4u, 5u);
       mc->emit_bytes(mc, &s, 1);
     }
     emit_u32le(mc, 0);
   }
   /* lea rd, [rd + disp32@tpoff]. */
   emit_rex(mc, 1, rd, 0, rd);
   {
     u8 op = X64_OPC_LEA;
     mc->emit_bytes(mc, &op, 1);
   }
   if ((rd & 7u) == 4u) {
     u8 mr = modrm(2u, rd & 7u, 4u);
     u8 s = sib(0u, 4u, rd & 7u);
     mc->emit_bytes(mc, &mr, 1);
     mc->emit_bytes(mc, &s, 1);
   } else {
     u8 mr = modrm(2u, rd & 7u, rd & 7u);
     mc->emit_bytes(mc, &mr, 1);
   }
   disp_pos = mc->pos(mc);
   emit_u32le(mc, 0);
   mc->emit_reloc_at(mc, sec, disp_pos, R_X64_TPOFF32, sym, addend, 0, 0);
 }
 
 /* ============================ atomics ============================ */
 
 static void emit_lock_prefix(MCEmitter* mc) {
   u8 b = 0xF0;
   mc->emit_bytes(mc, &b, 1);
 }
 static void emit_mfence(MCEmitter* mc) {
   u8 b[3] = {0x0F, 0xAE, 0xF0};
   mc->emit_bytes(mc, b, 3);
 }
 
 /* Resolve an atomic addr to a bare base register (r11) + disp 0. */
 static u32 x64_atomic_base(X64NativeTarget* a, NativeAddr addr) {
   return x64_addr_to_base_reg(a, addr, X64_TMP_INT2);
 }
 
 static void x64_atomic_load(NativeTarget* t, NativeLoc dst, NativeAddr addr,
                             MemAccess mem, KitCgMemOrder mo) {
   X64NativeTarget* a = x64_of(t);
   u32 sz = mem.size ? mem.size : native_type_size(t, dst.type);
   u32 base;
   (void)mo; /* x86 plain MOV is an acquire load. */
   base = x64_atomic_base(a, addr);
   emit_mov_load(t->mc, sz, 0, loc_reg(dst), base, 0);
 }
 
 static void x64_atomic_store(NativeTarget* t, NativeAddr addr, NativeLoc src,
                              MemAccess mem, KitCgMemOrder mo) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   u32 sz = mem.size ? mem.size : native_type_size(t, src.type);
   int w = sz == 8u ? 1 : 0;
   u32 base = x64_atomic_base(a, addr);
   u32 sr = loc_reg(src);
   if (mo == KIT_CG_MO_SEQ_CST) {
     /* xchg [mem], r11 implicitly fences. Stage src in rax (r11 holds base). */
     if (sr != X64_RAX) emit_mov_rr(mc, w, X64_RAX, sr);
     emit_lock_prefix(mc);
     emit_rex(mc, w, X64_RAX, 0, base);
     {
       u8 op = 0x87; /* xchg r/m, r */
       mc->emit_bytes(mc, &op, 1);
     }
     emit_mem_operand(mc, X64_RAX, base, 0);
     return;
   }
   emit_mov_store(mc, sz, sr, base, 0);
 }
 
 static void x64_atomic_rmw(NativeTarget* t, KitCgAtomicOp op, NativeLoc dst,
                            NativeAddr addr, NativeLoc val, MemAccess mem,
                            KitCgMemOrder mo) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   u32 sz = mem.size ? mem.size : native_type_size(t, dst.type);
   int w = sz == 8u ? 1 : 0;
   u32 base = x64_atomic_base(a, addr);
   u32 dr = loc_reg(dst);
   u32 vr = loc_reg(val);
   (void)mo; /* LOCK ops are full barriers. */
   /* The rmw uses fixed rax (prior), rcx (new), rdx (val); the optimizer may
    * have materialized the address into one of them, so keep it out (r11 is the
    * int emit scratch, never an allocated operand). Stage before rdx is loaded.
    */
   if (base == X64_RAX || base == X64_RCX || base == X64_RDX) {
     emit_mov_rr(mc, 1, X64_TMP_INT2, base);
     base = X64_TMP_INT2;
   }
   /* val staged in rdx (rax/rcx used by the cmpxchg loop). */
   emit_mov_rr(mc, w, X64_RDX, vr);
   if (op == KIT_CG_ATOMIC_ADD || op == KIT_CG_ATOMIC_SUB) {
     if (op == KIT_CG_ATOMIC_SUB) emit_f7_rm(mc, w, X64_F7_SUB_NEG, X64_RDX);
     emit_lock_prefix(mc);
     emit_rex(mc, w, X64_RDX, 0, base);
     {
       u8 op2[2] = {X64_OPC_TWOBYTE, 0xC1}; /* xadd */
       mc->emit_bytes(mc, op2, 2);
     }
     emit_mem_operand(mc, X64_RDX, base, 0);
     if (dr != X64_RDX) emit_mov_rr(mc, w, dr, X64_RDX);
     return;
   }
   if (op == KIT_CG_ATOMIC_XCHG) {
     emit_lock_prefix(mc);
     emit_rex(mc, w, X64_RDX, 0, base);
     {
       u8 op2 = 0x87; /* xchg */
       mc->emit_bytes(mc, &op2, 1);
     }
     emit_mem_operand(mc, X64_RDX, base, 0);
     if (dr != X64_RDX) emit_mov_rr(mc, w, dr, X64_RDX);
     return;
   }
   /* AND/OR/XOR/NAND: cmpxchg retry loop. rax=prior, rcx=new, rdx=val. */
   {
     MCLabel retry = mc->label_new(mc);
     emit_mov_load(mc, sz, 0, X64_RAX, base, 0);
     mc->label_place(mc, retry);
     emit_mov_rr(mc, w, X64_RCX, X64_RAX);
     switch (op) {
       case KIT_CG_ATOMIC_AND:
         emit_alu_rr(mc, w, X64_OPC_ALU_AND, X64_RCX, X64_RDX);
         break;
       case KIT_CG_ATOMIC_OR:
         emit_alu_rr(mc, w, X64_OPC_ALU_OR, X64_RCX, X64_RDX);
         break;
       case KIT_CG_ATOMIC_XOR:
         emit_alu_rr(mc, w, X64_OPC_ALU_XOR, X64_RCX, X64_RDX);
         break;
       case KIT_CG_ATOMIC_NAND:
         emit_alu_rr(mc, w, X64_OPC_ALU_AND, X64_RCX, X64_RDX);
         emit_f7_rm(mc, w, X64_F7_SUB_NOT, X64_RCX);
         break;
       default:
         x64_panic(a, "unsupported atomic rmw op");
     }
     emit_lock_prefix(mc);
     emit_rex(mc, w, X64_RCX, 0, base);
     {
       u8 op2[2] = {X64_OPC_TWOBYTE, 0xB1}; /* cmpxchg */
       mc->emit_bytes(mc, op2, 2);
     }
     emit_mem_operand(mc, X64_RCX, base, 0);
     emit_jcc_rel32(mc, X64_CC_NE, retry);
     if (dr != X64_RAX) emit_mov_rr(mc, w, dr, X64_RAX);
   }
 }
 
 static void x64_atomic_cas(NativeTarget* t, NativeLoc prior, NativeLoc ok,
                            NativeAddr addr, NativeLoc expected,
                            NativeLoc desired, MemAccess mem,
                            KitCgMemOrder success, KitCgMemOrder failure) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   u32 sz = mem.size ? mem.size : native_type_size(t, prior.type);
   int w = sz == 8u ? 1 : 0;
   u32 base = x64_atomic_base(a, addr);
   u32 rprior = loc_reg(prior);
   u32 rok = loc_reg(ok);
   u32 rexp = loc_reg(expected);
   u32 rdes = loc_reg(desired);
   (void)success;
   (void)failure;
   /* cmpxchg uses fixed rax (expected) and rcx (desired). The optimizer may have
    * materialized the address into either; keep it out of both (r11 is the int
    * emit scratch, never an allocated operand). */
   if (base == X64_RAX || base == X64_RCX) {
     emit_mov_rr(mc, 1, X64_TMP_INT2, base);
     base = X64_TMP_INT2;
   }
   /* Place expected -> rax and desired -> rcx as a parallel copy: the allocator
    * may have them in each other's target register (full swap) or desired in rax
    * (expected's target), either of which a naive two-move order would clobber.
    */
   if (rexp == X64_RCX && rdes == X64_RAX) {
     /* Swap rax <-> rcx (xchg needs no temp; base is not rax/rcx here). */
     emit_rex(mc, w, X64_RCX, 0, X64_RAX);
     {
       u8 xchg[2] = {0x87, modrm(3u, X64_RCX, X64_RAX)};
       mc->emit_bytes(mc, xchg, 2);
     }
   } else if (rdes == X64_RAX) {
     /* desired sits in rax; move it to rcx before rax is overwritten. */
     if (rdes != X64_RCX) emit_mov_rr(mc, w, X64_RCX, rdes);
     if (rexp != X64_RAX) emit_mov_rr(mc, w, X64_RAX, rexp);
   } else {
     if (rexp != X64_RAX) emit_mov_rr(mc, w, X64_RAX, rexp);
     if (rdes != X64_RCX) emit_mov_rr(mc, w, X64_RCX, rdes);
   }
   emit_lock_prefix(mc);
   emit_rex(mc, w, X64_RCX, 0, base);
   {
     u8 op2[2] = {X64_OPC_TWOBYTE, 0xB1}; /* cmpxchg [base], rcx */
     mc->emit_bytes(mc, op2, 2);
   }
   emit_mem_operand(mc, X64_RCX, base, 0);
   emit_setcc(mc, X64_CC_E, rok);
   emit_movzx_r32_r8(mc, rok, rok);
   if (rprior != X64_RAX) emit_mov_rr(mc, w, rprior, X64_RAX);
 }
 
 static void x64_fence(NativeTarget* t, KitCgMemOrder mo) {
   if (mo == KIT_CG_MO_SEQ_CST) emit_mfence(t->mc);
 }
 
 /* ============================ variadics ============================
  * SysV: __va_list_tag (gp_offset@0, fp_offset@4, overflow@8, reg_save@16). The
  * prologue filled the 176B reg-save area. Win64: va_list is a single pointer
  * to the next 8-byte slot in the home/overflow area; FP varargs are duplicated
  * into the matching GPR slot at the call site. `ap` addresses the va_list
  * object. */
 
 /* Resolve a va_list address into `scratch`, materializing it there if it is not
  * already, so the va field-value scratch registers (rax / r10 / rdx) never
  * alias it. At -O1 the optimizer may place the va_list pointer in any register
  * — including those — and the va code would then clobber the pointer
  * mid-sequence. */
 static u32 x64_va_base(X64NativeTarget* a, NativeAddr ap, u32 scratch) {
   u32 base = x64_addr_to_base_reg(a, ap, scratch);
   if (base != scratch) {
     emit_mov_rr(a->base.mc, 1, scratch, base);
     base = scratch;
   }
   return base;
 }
 
 /* add r/m, imm8 (group-1 /0) directly to a memory field — advances a va_list
  * offset/pointer in place without consuming a register. w selects 64- vs
  * 32-bit. */
 static void x64_add_mem_imm(MCEmitter* mc, int w, u32 base, i32 disp, i8 imm) {
   u8 op = X64_OPC_ALU_IMM8;
   u8 b;
   emit_rex(mc, w, 0, 0, base);
   mc->emit_bytes(mc, &op, 1);
   emit_mem_operand(mc, X64_ALU_SUB_ADD, base, disp); /* modrm.reg = /0 (ADD) */
   b = (u8)imm;
   mc->emit_bytes(mc, &b, 1);
 }
 
 /* add r64, [base+disp] (0x03 /r). */
 static void x64_add_reg_mem(MCEmitter* mc, u32 dst, u32 base, i32 disp) {
   u8 op = 0x03;
   emit_rex(mc, 1, dst, 0, base);
   mc->emit_bytes(mc, &op, 1);
   emit_mem_operand(mc, dst, base, disp);
 }
 
 static void x64_va_start_core(X64NativeTarget* a, NativeAddr ap) {
   NativeTarget* t = &a->base;
   MCEmitter* mc = t->mc;
   u32 ap_base;
   if (!a->is_variadic) x64_panic(a, "va_start: function not variadic");
   ap_base = x64_va_base(a, ap, X64_TMP_INT2);
   if (a->abi->shadow_space) {
     /* Win64: *ap = rbp + 16 + named_int*8 + named_stack. */
     u32 first = 16u + a->next_param_int * 8u + a->next_param_stack;
     emit_lea(mc, X64_RAX, X64_RBP, (i32)first);
     emit_mov_store(mc, 8, X64_RAX, ap_base, 0);
     return;
   }
   {
     X64NativeSlot* rs = x64_slot_get(a, a->reg_save_slot);
     /* gp_offset = next_param_int * 8 */
     x64_emit_load_imm(mc, 0, X64_RAX, (i64)(a->next_param_int * 8u));
     emit_mov_store(mc, 4, X64_RAX, ap_base, 0);
     /* fp_offset = 48 + next_param_fp * 16 */
     x64_emit_load_imm(mc, 0, X64_RAX, (i64)(48u + a->next_param_fp * 16u));
     emit_mov_store(mc, 4, X64_RAX, ap_base, 4);
     /* overflow_arg_area = rbp + 16 + next_param_stack */
     emit_lea(mc, X64_RAX, X64_RBP, (i32)(16u + a->next_param_stack));
     emit_mov_store(mc, 8, X64_RAX, ap_base, 8);
     /* reg_save_area = rbp - reg_save_slot.off */
     emit_lea(mc, X64_RAX, X64_RBP, -(i32)rs->off);
     emit_mov_store(mc, 8, X64_RAX, ap_base, 16);
   }
 }
 
 static void x64_va_arg_core(X64NativeTarget* a, NativeLoc dst, NativeAddr ap,
                             KitCgTypeId type) {
   NativeTarget* t = &a->base;
   MCEmitter* mc = t->mc;
   u32 sz = native_type_size(t, type);
   int is_fp = native_loc_is_fp(dst);
   u32 dr = loc_reg(dst);
   u32 ap_base = x64_va_base(a, ap, X64_TMP_INT2); /* r11 */
   /* GPR scratch for the offset/address arithmetic. For integer results the
    * destination is itself a (throwaway) scratch GPR — pass_native_emit fetches
    * va_arg into a scratch and copies it to the real destination afterward — so
    * we reuse `dr` and touch no allocable register at all. FP results keep their
    * value in an XMM register, so they borrow the reserved RAX emit scratch.
    * Either way only r11 (ap_base) and `gp` are used: the va_list fields are
    * advanced in memory (x64_add_mem_imm) and the reg-save base is folded in
    * with x64_add_reg_mem, so no third register is needed. */
   u32 gp = is_fp ? X64_RAX : dr;
   if (a->abi->shadow_space) {
     /* Win64: gp = *ap; load dr from [gp]; *ap += 8. */
     emit_mov_load(mc, 8, 0, gp, ap_base, 0);
     if (is_fp)
       emit_sse_load(mc, sse_scalar_prefix(sz), 0x10, dr, gp, 0);
     else
       emit_mov_load(mc, sz, 0, dr, gp, 0);
     x64_add_mem_imm(mc, 1, ap_base, 0, 8);
     return;
   }
   {
     u32 offs_field = is_fp ? 4u : 0u;
     u32 max_offs = is_fp ? 176u : 48u;
     i8 stride = is_fp ? 16 : 8;
     MCLabel L_stack = mc->label_new(mc);
     MCLabel L_done = mc->label_new(mc);
     /* gp32 = ap[offs]; cmp gp32, max; jae L_stack. Use the imm8 form when the
      * threshold fits (gp_offset max 48) so the encoding is canonical and the
      * `cc -S | as` round-trip reproduces it; fp_offset max 176 needs imm32. */
     emit_mov_load(mc, 4, 0, gp, ap_base, (i32)offs_field);
     if (imm_fits_i8((i64)max_offs))
       emit_alu_imm8(mc, 0, X64_ALU_SUB_CMP, gp, (i8)max_offs);
     else
       emit_alu_imm32(mc, 0, X64_ALU_SUB_CMP, gp, (i32)max_offs);
     emit_jcc_rel32(mc, X64_CC_AE, L_stack);
     /* reg path: ap[offs] += stride; gp = reg_save_area(ap[16]) + offset; load.
      * (The memory increment leaves gp holding the old offset.) */
     x64_add_mem_imm(mc, 0, ap_base, (i32)offs_field, stride);
     x64_add_reg_mem(mc, gp, ap_base, 16);
     if (is_fp)
       emit_sse_load(mc, sse_scalar_prefix(sz), 0x10, dr, gp, 0);
     else
       emit_mov_load(mc, sz, 0, dr, gp, 0);
     emit_jmp_rel32(mc, L_done);
     /* stack path: gp = ap[8] (overflow area); load; ap[8] += 8. */
     mc->label_place(mc, L_stack);
     emit_mov_load(mc, 8, 0, gp, ap_base, 8);
     if (is_fp)
       emit_sse_load(mc, sse_scalar_prefix(sz), 0x10, dr, gp, 0);
     else
       emit_mov_load(mc, sz, 0, dr, gp, 0);
     x64_add_mem_imm(mc, 1, ap_base, 8, 8);
     mc->label_place(mc, L_done);
   }
 }
 
 static void x64_va_copy_core(X64NativeTarget* a, NativeAddr dst_ap,
                              NativeAddr src_ap) {
   NativeTarget* t = &a->base;
   MCEmitter* mc = t->mc;
   /* Resolve dst into r11, src into rax, and copy each qword through the fp emit
    * scratch xmm14. Uses only reserved emit scratch (r11/rax/xmm14), so the
    * optimizer's register choice for a va_list pointer can never be clobbered
    * and no allocable GPR (previously rdx) is consumed. */
   u32 dst_base = x64_va_base(a, dst_ap, X64_TMP_INT2);
   u32 src_base = x64_va_base(a, src_ap, X64_TMP_INT);
   u32 n = a->abi->shadow_space ? 8u : 24u, i;
   for (i = 0; i < n; i += 8u) {
     emit_sse_load(mc, 0xF2, 0x10, X64_TMP_FP, src_base, (i32)i);  /* movsd */
     emit_sse_store(mc, 0xF2, 0x11, X64_TMP_FP, dst_base, (i32)i); /* movsd */
   }
 }
 
 static NativeAddr x64_va_addr_from_ptr(NativeLoc ap_ptr) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_REG;
   addr.cls = NATIVE_REG_INT;
   addr.base.reg = ap_ptr.v.reg;
   addr.base_type = ap_ptr.type;
   return addr;
 }
 
 static void x64_va_start_native(NativeTarget* t, NativeLoc ap_ptr) {
   x64_va_start_core(x64_of(t), x64_va_addr_from_ptr(ap_ptr));
 }
 static void x64_va_arg_native(NativeTarget* t, NativeLoc dst, NativeLoc ap_ptr,
                               KitCgTypeId type) {
   x64_va_arg_core(x64_of(t), dst, x64_va_addr_from_ptr(ap_ptr), type);
 }
 static void x64_va_end_native(NativeTarget* t, NativeLoc ap_ptr) {
   (void)t;
   (void)ap_ptr;
 }
 static void x64_va_copy_native(NativeTarget* t, NativeLoc dst, NativeLoc src) {
   x64_va_copy_core(x64_of(t), x64_va_addr_from_ptr(dst),
                    x64_va_addr_from_ptr(src));
 }
 
 /* ============================ intrinsics ============================ */
 
 static void emit_popcnt(MCEmitter* mc, int w, u32 dst, u32 src) {
   u8 p = 0xF3;
   mc->emit_bytes(mc, &p, 1);
   emit_rex(mc, w, dst, 0, src);
   {
     u8 op[2] = {X64_OPC_TWOBYTE, 0xB8};
     mc->emit_bytes(mc, op, 2);
   }
   emit_rm_reg(mc, dst, src);
 }
 static void emit_bs(MCEmitter* mc, int w, u8 opcode2, u32 dst, u32 src) {
   emit_rex(mc, w, dst, 0, src);
   {
     u8 op[2] = {X64_OPC_TWOBYTE, opcode2};
     mc->emit_bytes(mc, op, 2);
   }
   emit_rm_reg(mc, dst, src);
 }
 static void emit_bswap(MCEmitter* mc, int w, u32 reg) {
   emit_rex(mc, w, 0, 0, reg);
   {
     u8 op[2] = {X64_OPC_TWOBYTE, (u8)(0xC8 + (reg & 7u))};
     mc->emit_bytes(mc, op, 2);
   }
 }
 static void emit_rol16_imm8(MCEmitter* mc, u32 reg, u8 imm) {
   u8 p = X64_OPSIZE_PFX;
   mc->emit_bytes(mc, &p, 1);
   emit_rex(mc, 0, 0, 0, reg);
   {
     u8 buf[3] = {X64_OPC_SHIFT_IMM, modrm(3u, 0u, reg & 7u), imm};
     mc->emit_bytes(mc, buf, 3);
   }
 }
 static void emit_ud2(MCEmitter* mc) {
   u8 b[2] = {0x0F, 0x0B};
   mc->emit_bytes(mc, b, 2);
 }
 
 static void emit_syscall(MCEmitter* mc) {
   u8 b[2] = {0x0F, 0x05};
   mc->emit_bytes(mc, b, 2);
 }
 
 static void x64_intrinsic(NativeTarget* t, IntrinKind kind,
                           const NativeLoc* dsts, u32 ndst,
                           const NativeLoc* args, u32 narg) {
   X64NativeTarget* a = x64_of(t);
   MCEmitter* mc = t->mc;
   (void)ndst;
   switch (kind) {
     case INTRIN_NONE:
       break;
     case INTRIN_EXPECT:
     case INTRIN_ASSUME_ALIGNED:
       if (args[0].kind == NATIVE_LOC_IMM)
         x64_emit_load_imm(mc, x64_is_64(t, dsts[0].type) ? 1 : 0,
                           loc_reg(dsts[0]), args[0].v.imm);
       else
         x64_move(t, dsts[0], args[0]);
       return;
     case INTRIN_PREFETCH:
       return;
     case INTRIN_TRAP:
       emit_ud2(mc);
       return;
     case INTRIN_SYSCALL:
       if (ndst == 1u && narg >= 1u && narg <= 7u) {
         static const u32 syscall_regs[7] = {
             X64_RAX, X64_RDI, X64_RSI, X64_RDX, X64_R10, X64_R8, X64_R9};
         X64ArgMove moves[7];
         for (u32 i = 0; i < narg; ++i) {
           X64ArgMove* m = &moves[i];
           memset(m, 0, sizeof *m);
           m->dst = native_loc_reg(dsts[0].type, NATIVE_REG_INT,
                                   syscall_regs[i]);
           m->src = args[i];
           m->size = t->c->target.ptr_size;
         }
         x64_emit_reg_arg_moves(t, moves, narg, X64_TMP_INT2);
         emit_syscall(mc);
         x64_move(t, dsts[0],
                  native_loc_reg(dsts[0].type, NATIVE_REG_INT, X64_RAX));
       }
       return;
     case INTRIN_POPCOUNT:
       emit_popcnt(mc, x64_is_64(t, args[0].type) ? 1 : 0, loc_reg(dsts[0]),
                   loc_reg(args[0]));
       return;
     case INTRIN_CTZ:
       emit_bs(mc, x64_is_64(t, args[0].type) ? 1 : 0, 0xBC /* bsf */,
               loc_reg(dsts[0]), loc_reg(args[0]));
       return;
     case INTRIN_CLZ: {
       int w = x64_is_64(t, args[0].type) ? 1 : 0;
       u32 dr = loc_reg(dsts[0]);
       emit_bs(mc, w, 0xBD /* bsr */, dr, loc_reg(args[0]));
       /* clz = (bits-1) - bsr, computed via xor with bits-1. The mask (31/63)
        * fits in imm8, so use the compact 0x83 form to match the canonical
        * encoding (and the assembler's `cc -S | as` round-trip). */
       emit_alu_imm8(mc, w, X64_ALU_SUB_XOR, dr, w ? 63 : 31);
       return;
     }
     case INTRIN_BSWAP: {
       u32 width = abi_cg_sizeof(t->c->abi, dsts[0].type);
       switch (width) {
         case 2: {
           u32 dr = loc_reg(dsts[0]), sr = loc_reg(args[0]);
           if (dr != sr) emit_mov_rr(mc, 0, dr, sr);
           emit_rol16_imm8(mc, dr, 8);
           return;
         }
         case 4: {
           u32 dr = loc_reg(dsts[0]), sr = loc_reg(args[0]);
           if (dr != sr) emit_mov_rr(mc, 0, dr, sr);
           emit_bswap(mc, 0, dr);
           return;
         }
         case 8: {
           u32 dr = loc_reg(dsts[0]), sr = loc_reg(args[0]);
           if (dr != sr) emit_mov_rr(mc, 1, dr, sr);
           emit_bswap(mc, 1, dr);
           return;
         }
         default:
           break;
       }
       return;
     }
     case INTRIN_SADD_OVERFLOW:
     case INTRIN_UADD_OVERFLOW:
     case INTRIN_SSUB_OVERFLOW:
     case INTRIN_USUB_OVERFLOW: {
       int w = x64_is_64(t, dsts[0].type) ? 1 : 0;
       u32 rd = loc_reg(dsts[0]), rovf = loc_reg(dsts[1]);
       u32 ra = loc_reg(args[0]), rb = loc_reg(args[1]);
       u8 op = (kind == INTRIN_SADD_OVERFLOW || kind == INTRIN_UADD_OVERFLOW)
                   ? X64_OPC_ALU_ADD
                   : X64_OPC_ALU_SUB;
       u32 cc = (kind == INTRIN_UADD_OVERFLOW || kind == INTRIN_USUB_OVERFLOW)
                    ? X64_CC_B
                    : X64_CC_O;
       if (rd != ra) emit_mov_rr(mc, w, rd, ra);
       emit_alu_rr(mc, w, op, rd, rb);
       emit_setcc(mc, cc, rovf);
       emit_movzx_r32_r8(mc, rovf, rovf);
       return;
     }
     case INTRIN_SMUL_OVERFLOW: {
       int w = x64_is_64(t, dsts[0].type) ? 1 : 0;
       u32 rd = loc_reg(dsts[0]), rovf = loc_reg(dsts[1]);
       u32 ra = loc_reg(args[0]), rb = loc_reg(args[1]);
       if (rd != ra) emit_mov_rr(mc, w, rd, ra);
       emit_imul_rr(mc, w, rd, rb);
       emit_setcc(mc, X64_CC_O, rovf);
       emit_movzx_r32_r8(mc, rovf, rovf);
       return;
     }
     case INTRIN_UMUL_OVERFLOW: {
       int w = x64_is_64(t, dsts[0].type) ? 1 : 0;
       u32 rd = loc_reg(dsts[0]), rovf = loc_reg(dsts[1]);
       u32 ra = loc_reg(args[0]), rb = loc_reg(args[1]);
       if (rb == X64_RAX || rb == X64_RDX) {
         emit_mov_rr(mc, w, X64_R11, rb);
         rb = X64_R11;
       }
       if (ra != X64_RAX) emit_mov_rr(mc, w, X64_RAX, ra);
       emit_f7_rm(mc, w, X64_F7_SUB_MUL, rb); /* MUL: rdx:rax = rax * rb */
       if (rd != X64_RAX) emit_mov_rr(mc, w, rd, X64_RAX);
       emit_setcc(mc, X64_CC_O, rovf);
       emit_movzx_r32_r8(mc, rovf, rovf);
       return;
     }
     case INTRIN_MEMMOVE: {
       u32 dr, sr, n, i;
       if (narg != 3u || args[0].kind != NATIVE_LOC_REG ||
           args[1].kind != NATIVE_LOC_REG || args[2].kind != NATIVE_LOC_IMM)
         x64_panic(a, "unsupported memory intrinsic operands");
       if (args[2].v.imm < 0 || args[2].v.imm > 0xffffffffll)
         x64_panic(a, "unsupported memory intrinsic size");
       dr = loc_reg(args[0]);
       sr = loc_reg(args[1]);
       n = (u32)args[2].v.imm;
       i = n; /* copy high-to-low so an overlapping dst > src is safe */
       while (i >= 8u) {
         i -= 8u;
         emit_mov_load(mc, 8, 0, X64_RAX, sr, (i32)i);
         emit_mov_store(mc, 8, X64_RAX, dr, (i32)i);
       }
       while (i >= 4u) {
         i -= 4u;
         emit_mov_load(mc, 4, 0, X64_RAX, sr, (i32)i);
         emit_mov_store(mc, 4, X64_RAX, dr, (i32)i);
       }
       while (i >= 2u) {
         i -= 2u;
         emit_mov_load(mc, 2, 0, X64_RAX, sr, (i32)i);
         emit_mov_store(mc, 2, X64_RAX, dr, (i32)i);
       }
       while (i >= 1u) {
         i -= 1u;
         emit_mov_load(mc, 1, 0, X64_RAX, sr, (i32)i);
         emit_mov_store(mc, 1, X64_RAX, dr, (i32)i);
       }
       return;
     }
     case INTRIN_CPU_NOP: {
       u8 b = 0x90; /* NOP */
       mc->emit_bytes(mc, &b, 1);
       return;
     }
     case INTRIN_CPU_YIELD: {
       u8 b[2] = {0xF3, 0x90}; /* PAUSE */
       mc->emit_bytes(mc, b, 2);
       return;
     }
     case INTRIN_DMB:
     case INTRIN_DSB: {
       u8 b[3] = {0x0F, 0xAE, 0xF0}; /* MFENCE: full-system memory barrier */
       mc->emit_bytes(mc, b, 3);
       return;
     }
     case INTRIN_IRQ_DISABLE: {
       u8 b = 0xFA; /* CLI (privileged) */
       mc->emit_bytes(mc, &b, 1);
       return;
     }
     case INTRIN_IRQ_ENABLE: {
       u8 b = 0xFB; /* STI (privileged) */
       mc->emit_bytes(mc, &b, 1);
       return;
     }
     case INTRIN_FRAME_ADDRESS:
     case INTRIN_RETURN_ADDRESS:
       /* Walk the rbp frame-record chain. Every kit prologue keeps the rbp
        * record: [rbp] = caller's rbp, [rbp + 8] = return address pushed by the
        * `call`. The level is a compile-time constant, so the walk unrolls to
        * `level` dependent loads. */
       if (ndst == 1u) {
         u32 level = (narg >= 1u && args[0].kind == NATIVE_LOC_IMM)
                         ? (u32)args[0].v.imm
                         : 0u;
         u32 rd = loc_reg(dsts[0]);
         emit_mov_rr(mc, 1, rd, X64_RBP);
         for (u32 i = 0; i < level; ++i)
           emit_mov_load(mc, 8, 0, rd, rd, 0); /* rd = *(rd) */
         if (kind == INTRIN_RETURN_ADDRESS)
           emit_mov_load(mc, 8, 0, rd, rd, 8); /* rd = *(rd + 8) */
       }
       return;
     default:
       break;
   }
   x64_panic(a, "unsupported compiler intrinsic");
 }
 
 /* ============================ inline asm ============================ */
 
 _Noreturn static void x64_asm_panic_at(Compiler* c, SrcLoc loc,
                                        const char* msg) {
   compiler_panic(c, loc, "x64 inline asm: %s", msg);
 }
 _Noreturn static void x64_asm_panic(NativeDirectTarget* d, const char* msg) {
   x64_asm_panic_at(d->base.c, d->loc, msg);
 }
 
 /* constraint_body / constraint_early / match_index are shared
  * (cg/native_asm.h). */
 
 static void x64_asm_bound_reg(Operand* out, KitCgTypeId type,
                               NativeAllocClass cls, Reg reg) {
   memset(out, 0, sizeof *out);
   out->kind = X64_INLINE_OPK_REG;
   out->pad[0] =
       (cls == NATIVE_REG_FP) ? X64_INLINE_OPCLS_FP : X64_INLINE_OPCLS_INT;
   out->type = type;
   out->v.local = (CGLocal)reg;
 }
 static void x64_asm_bound_mem(Operand* out, KitCgTypeId type, Reg base) {
   memset(out, 0, sizeof *out);
   out->kind = OPK_INDIRECT;
   out->type = type;
   out->v.ind.base = (CGLocal)base;
   out->v.ind.index = CG_LOCAL_NONE;
   out->v.ind.ofs = 0;
 }
 
 /* Parse a clobber register name into (class, reg). Returns 0 for cc/memory.
  * GPR names map to HW encoding via x64_register_hw_index; xmm names map via the
  * DWARF table (xmm0..15 = dwarf 17..32). */
 static int x64_asm_parse_reg_clobber(Compiler* c, SrcLoc loc, Sym name,
                                      NativeAllocClass* cls_out, Reg* reg_out) {
   Slice s = pool_slice(c->global, name);
   char buf[16];
   uint32_t idx;
   if (!s.s || !s.len) return 0;
   if (s.len == 2 && s.s[0] == 'c' && s.s[1] == 'c') return 0;
   if (s.len == 6 && memcmp(s.s, "memory", 6) == 0) return 0;
   if (s.len >= sizeof buf) x64_asm_panic_at(c, loc, "clobber name is too long");
   memcpy(buf, s.s, s.len);
   buf[s.len] = '\0';
   if (x64_register_hw_index(buf, &idx) == 0 && idx <= 15u) {
     *cls_out = NATIVE_REG_INT;
     *reg_out = (Reg)idx;
     return 1;
   }
   if (x64_register_index(buf, &idx) == 0 && idx >= 17u && idx <= 32u) {
     *cls_out = NATIVE_REG_FP;
     *reg_out = (Reg)(idx - 17u);
     return 1;
   }
   x64_asm_panic_at(c, loc, "unknown clobber register");
   return 0;
 }
 
 static void x64_asm_clobber_masks(Compiler* c, SrcLoc loc, const Sym* clobbers,
                                   u32 nclob, u32* int_mask, u32* fp_mask) {
   u32 i;
   *int_mask = 0;
   *fp_mask = 0;
   for (i = 0; i < nclob; ++i) {
     NativeAllocClass cls;
     Reg reg;
     if (!x64_asm_parse_reg_clobber(c, loc, clobbers[i], &cls, &reg)) continue;
     if (cls == NATIVE_REG_INT)
       *int_mask |= 1u << reg;
     else
       *fp_mask |= 1u << reg;
   }
 }
 
 static int x64_asm_resolve_pin_or_panic(NativeDirectTarget* d, Sym reg,
                                         const char* constraint,
                                         NativeAsmRegPin* pin) {
   NativeAsmRegPinStatus st =
       native_asm_resolve_pin(d->native, reg, constraint, pin);
   if (st == NATIVE_ASM_REG_PIN_ABSENT) return 0;
   if (st != NATIVE_ASM_REG_PIN_OK)
     x64_asm_panic(d, native_asm_pin_status_message(st));
   return 1;
 }
 
 /* Pick a free register from caller-saved allocable pools for an asm operand the
  * direct path self-allocates. */
 static Reg x64_asm_alloc_reg(NativeDirectTarget* d, NativeAllocClass cls,
                              u32 allowed_mask, u32* used_int, u32* used_fp) {
   static const Reg int_pool[] = {X64_RDI, X64_RSI, X64_RDX,
                                  X64_RCX, X64_R8,  X64_R9};
   static const Reg fp_pool[] = {
       X64_XMM0, X64_XMM1, X64_XMM2, X64_XMM3,     X64_XMM4,      X64_XMM5,
       X64_XMM6, X64_XMM7, X64_XMM8, X64_XMM0 + 9, X64_XMM0 + 10, X64_XMM0 + 11};
   const Reg* pool = cls == NATIVE_REG_FP ? fp_pool : int_pool;
   u32 n = cls == NATIVE_REG_FP ? (u32)(sizeof fp_pool / sizeof fp_pool[0])
                                : (u32)(sizeof int_pool / sizeof int_pool[0]);
   u32* used = cls == NATIVE_REG_FP ? used_fp : used_int;
   u32 i;
   for (i = 0; i < n; ++i) {
     Reg r = pool[i];
     if (allowed_mask && (allowed_mask & (1u << r)) == 0) continue;
     if ((*used & (1u << r)) != 0) continue;
     *used |= 1u << r;
     return r;
   }
   x64_asm_panic(d, "out of registers for asm operands");
   return REG_NONE;
 }
 
 /* Direct (-O0) path: resolve a semantic Operand to a NativeAddr. */
 static NativeAddr x64_direct_addr(NativeDirectTarget* d, Operand op) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   switch ((OpKind)op.kind) {
     case OPK_LOCAL:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = d->locals[op.v.local - 1u].home;
       addr.base_type = op.type;
       return addr;
     case OPK_INDIRECT:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME_VALUE;
       addr.base.frame = d->locals[op.v.ind.base - 1u].home;
       addr.cls = d->locals[op.v.ind.base - 1u].cls;
       addr.base_type = d->locals[op.v.ind.base - 1u].type;
       addr.offset = op.v.ind.ofs;
       return addr;
     default:
       x64_asm_panic(d, "operand is not addressable");
   }
 }
 
 static NativeAddr x64_direct_materialize_addr(NativeDirectTarget* d,
                                               Operand op) {
   X64NativeTarget* a = x64_of(d->native);
   NativeAddr addr = x64_direct_addr(d, op);
   if (addr.base_kind == NATIVE_ADDR_BASE_FRAME_VALUE) {
     NativeAddr load;
     memset(&load, 0, sizeof load);
     load.base_kind = NATIVE_ADDR_BASE_FRAME;
     load.base.frame = addr.base.frame;
     load.base_type = addr.base_type;
     emit_mov_load(a->base.mc, 8, 0, X64_TMP_INT2, X64_RBP,
                   -(i32)x64_slot_get(a, addr.base.frame)->off);
     addr.base_kind = NATIVE_ADDR_BASE_REG;
     addr.base.reg = X64_TMP_INT2;
   }
   return addr;
 }
 
 static void x64_direct_load_operand_to_reg(NativeDirectTarget* d, Operand op,
                                            NativeLoc dst) {
   X64NativeTarget* a = x64_of(d->native);
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   switch ((OpKind)op.kind) {
     case OPK_IMM:
       if ((NativeAllocClass)dst.cls != NATIVE_REG_INT)
         x64_asm_panic(d, "floating-point immediate asm input is unsupported");
       d->native->load_imm(d->native, dst, op.v.imm);
       return;
     case OPK_LOCAL:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = d->locals[op.v.local - 1u].home;
       addr.base_type = op.type;
       x64_emit_mem(a, 1, dst, addr, native_mem_for_type(d->native, op.type, 0));
       return;
     case OPK_GLOBAL:
       addr.base_kind = NATIVE_ADDR_BASE_GLOBAL;
       addr.base.global.sym = op.v.global.sym;
       addr.base.global.addend = op.v.global.addend;
       addr.base_type = op.type;
       d->native->load_addr(d->native, dst, addr);
       return;
     case OPK_INDIRECT:
       addr = x64_direct_materialize_addr(d, op);
       x64_emit_mem(a, 1, dst, addr, native_mem_for_type(d->native, op.type, 0));
       return;
   }
   x64_asm_panic(d, "unsupported asm input operand");
 }
 
 static void x64_direct_load_address_to_reg(NativeDirectTarget* d, Operand op,
                                            NativeLoc dst) {
   d->native->load_addr(d->native, dst, x64_direct_addr(d, op));
 }
 
 static void x64_direct_store_reg_to_operand(NativeDirectTarget* d, Operand op,
                                             NativeLoc src) {
   X64NativeTarget* a = x64_of(d->native);
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   if (op.kind == OPK_LOCAL) {
     addr.base_kind = NATIVE_ADDR_BASE_FRAME;
     addr.base.frame = d->locals[op.v.local - 1u].home;
     addr.base_type = op.type;
   } else {
     addr = x64_direct_materialize_addr(d, op);
   }
   x64_emit_mem(a, 0, src, addr, native_mem_for_type(d->native, op.type, 0));
 }
 
 /* Callee-saved registers an asm block clobbers must be saved around the block.
  */
 typedef struct X64AsmSavedClobber {
   NativeFrameSlot slot;
   NativeAllocClass cls;
   Reg reg;
   KitCgTypeId type;
 } X64AsmSavedClobber;
 
 static void x64_asm_save_one(X64NativeTarget* a, X64AsmSavedClobber* s) {
   NativeFrameSlotDesc desc;
   NativeAddr addr;
   memset(&desc, 0, sizeof desc);
   desc.type = s->type;
   desc.size = s->cls == NATIVE_REG_FP ? 16u : 8u;
   desc.align = desc.size;
   desc.kind = NATIVE_FRAME_SLOT_SAVE;
   s->slot = a->base.frame_slot(&a->base, &desc);
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = s->slot;
   addr.base_type = s->type;
   x64_emit_mem(a, 0, native_loc_reg(s->type, s->cls, s->reg), addr,
                native_mem_for_type(&a->base, s->type, desc.size));
 }
 static void x64_asm_restore_one(X64NativeTarget* a,
                                 const X64AsmSavedClobber* s) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = s->slot;
   addr.base_type = s->type;
   x64_emit_mem(a, 1, native_loc_reg(s->type, s->cls, s->reg), addr,
                native_mem_for_type(&a->base, s->type,
                                    s->cls == NATIVE_REG_FP ? 16u : 8u));
 }
 
 /* SysV callee-saved: int rbx,r12-r15; no fp. Win64 adds rdi,rsi + xmm6-15. */
 static int x64_reg_is_callee_int(const X64ABIRegs* abi, Reg r) {
   if (r == X64_RBP) return 0; /* prologue head handles rbp */
   return (abi->cs_int_mask & (1ull << r)) != 0;
 }
 static int x64_reg_is_callee_fp(const X64ABIRegs* abi, Reg r) {
   return (abi->cs_fp_mask & (1ull << r)) != 0;
 }
 
 static X64AsmSavedClobber* x64_asm_save_callee_clobbers(X64NativeTarget* a,
                                                         u32 int_mask,
                                                         u32 fp_mask,
                                                         u32* nsaved_out) {
   X64AsmSavedClobber* saved =
       arena_zarray(a->base.c->tu, X64AsmSavedClobber, 32u);
   KitCgTypeId i64 = builtin_id(KIT_CG_BUILTIN_I64);
   KitCgTypeId f64 = builtin_id(KIT_CG_BUILTIN_F64);
   u32 n = 0;
   Reg r;
   for (r = 0; r <= 15u; ++r) {
     if ((int_mask & (1u << r)) == 0 || !x64_reg_is_callee_int(a->abi, r))
       continue;
     saved[n].cls = NATIVE_REG_INT;
     saved[n].reg = r;
     saved[n].type = i64;
     x64_asm_save_one(a, &saved[n++]);
   }
   for (r = 0; r <= 15u; ++r) {
     if ((fp_mask & (1u << r)) == 0 || !x64_reg_is_callee_fp(a->abi, r))
       continue;
     saved[n].cls = NATIVE_REG_FP;
     saved[n].reg = r;
     saved[n].type = f64;
     x64_asm_save_one(a, &saved[n++]);
   }
   *nsaved_out = n;
   return saved;
 }
 
 /* ---- NativeTarget (optimizer) asm hook ---- */
 
 static NativeAddr x64_asm_loc_to_addr(X64NativeTarget* a, SrcLoc loc,
                                       NativeLoc src) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_type = src.type;
   switch ((NativeLocKind)src.kind) {
     case NATIVE_LOC_FRAME:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = src.v.frame;
       return addr;
     case NATIVE_LOC_ADDR:
       return src.v.addr;
     case NATIVE_LOC_GLOBAL:
       addr.base_kind = NATIVE_ADDR_BASE_GLOBAL;
       addr.base.global.sym = src.v.global.sym;
       addr.base.global.addend = src.v.global.addend;
       return addr;
     case NATIVE_LOC_REG:
       addr.base_kind = NATIVE_ADDR_BASE_REG;
       addr.cls = NATIVE_REG_INT;
       addr.base.reg = src.v.reg;
       return addr;
     default:
       x64_asm_panic_at(a->base.c, loc, "unsupported memory asm operand");
   }
 }
 
 static Reg x64_asm_native_mem_base(X64NativeTarget* a, SrcLoc loc,
                                    NativeLoc src, u32* ntmp) {
   NativeAddr addr = x64_asm_loc_to_addr(a, loc, src);
   Reg dst;
   if (addr.base_kind == NATIVE_ADDR_BASE_REG && addr.offset == 0 &&
       addr.index_kind == NATIVE_ADDR_INDEX_NONE) {
     if ((addr.base.reg & 0xfu) != X64_TMP_INT &&
         (addr.base.reg & 0xfu) != X64_TMP_INT2)
       return (Reg)(addr.base.reg & 0xfu);
   }
   if (*ntmp >= 2u)
     x64_asm_panic_at(a->base.c, loc, "too many memory asm operands");
   dst = (*ntmp == 0u) ? (Reg)X64_TMP_INT : (Reg)X64_TMP_INT2;
   (*ntmp)++;
   x64_addr_to_base_reg(a, addr, dst);
   return dst;
 }
 
 static void x64_asm_load_loc_to_reg(X64NativeTarget* a, SrcLoc loc,
                                     NativeLoc src, NativeLoc dst) {
   NativeTarget* t = &a->base;
   NativeAllocClass cls = (NativeAllocClass)dst.cls;
   if (src.kind == NATIVE_LOC_REG) {
     if (src.v.reg != dst.v.reg || src.cls != dst.cls) t->move(t, dst, src);
     return;
   }
   if (src.kind == NATIVE_LOC_IMM) {
     if (cls != NATIVE_REG_INT)
       x64_asm_panic_at(t->c, loc,
                        "floating-point immediate asm input is unsupported");
     t->load_imm(t, dst, src.v.imm);
     return;
   }
   x64_emit_mem(a, 1, dst, x64_asm_loc_to_addr(a, loc, src),
                native_mem_for_type(t, dst.type, native_type_size(t, dst.type)));
 }
 
 static void x64_asm_store_reg_to_loc(X64NativeTarget* a, SrcLoc loc,
                                      NativeLoc dst, NativeLoc src) {
   NativeTarget* t = &a->base;
   if (dst.kind == NATIVE_LOC_REG) {
     if (dst.v.reg != src.v.reg || dst.cls != src.cls) t->move(t, dst, src);
     return;
   }
   x64_emit_mem(a, 0, src, x64_asm_loc_to_addr(a, loc, dst),
                native_mem_for_type(t, src.type, native_type_size(t, src.type)));
 }
 
 static void x64_asm_bind_native(X64NativeTarget* a, SrcLoc loc, Operand* out,
                                 const char* constraint, KitCgTypeId type,
                                 NativeLoc src, u32* ntmp) {
   const char* body = native_asm_constraint_body(constraint);
   NativeAsmConstraintInfo info;
   if (native_asm_constraint_reg_info(&a->base, constraint, &info)) {
     if (src.kind != NATIVE_LOC_REG)
       x64_asm_panic_at(a->base.c, loc,
                        "register asm operand not in a register");
     if (info.fixed_reg != REG_NONE && info.fixed_reg != (Reg)src.v.reg)
       x64_asm_panic_at(a->base.c, loc,
                        "fixed-register asm operand in wrong register");
     if (info.allowed_mask &&
         ((Reg)src.v.reg >= 32 ||
          (info.allowed_mask & (1u << (Reg)src.v.reg)) == 0))
       x64_asm_panic_at(a->base.c, loc,
                        "register asm operand violates constraint register set");
     x64_asm_bound_reg(out, type, info.cls, (Reg)src.v.reg);
   } else if (body[0] == 'i') {
     if (src.kind != NATIVE_LOC_IMM)
       x64_asm_panic_at(a->base.c, loc,
                        "immediate asm operand is not immediate");
     memset(out, 0, sizeof *out);
     out->kind = OPK_IMM;
     out->type = type;
     out->v.imm = src.v.imm;
   } else if (body[0] == 'm') {
     x64_asm_bound_mem(out, type, x64_asm_native_mem_base(a, loc, src, ntmp));
   } else {
     x64_asm_panic_at(a->base.c, loc, "unsupported asm constraint");
   }
 }
 
 static void x64_asm_block_native(NativeTarget* t, const char* tmpl,
                                  const AsmConstraint* outs, u32 nout,
                                  NativeLoc* out_locs, const AsmConstraint* ins,
                                  u32 nin, const NativeLoc* in_locs,
                                  const Sym* clobbers, u32 nclob) {
   X64NativeTarget* a = x64_of(t);
   Compiler* c = t->c;
   SrcLoc loc = a->func ? a->func->loc : (SrcLoc){0, 0, 0};
   Operand* bound_outs = nout ? arena_zarray(c->tu, Operand, nout) : NULL;
   Operand* bound_ins = nin ? arena_zarray(c->tu, Operand, nin) : NULL;
   u8* staged_outs = nout ? arena_zarray(c->tu, u8, nout) : NULL;
   u32 ntmp = 0, i;
   X64Asm* asmh;
 
   for (i = 0; i < nout; ++i) {
     KitCgTypeId type = outs[i].type ? outs[i].type : out_locs[i].type;
     NativeLoc outloc = out_locs[i];
     NativeAsmPinnedLoc pinned =
         native_asm_prepare_pinned_loc(t, outs[i].reg, outs[i].str, type, outloc);
     if (pinned.has_pin) {
       if (pinned.pin_status != NATIVE_ASM_REG_PIN_OK)
         x64_asm_panic_at(c, loc,
                          native_asm_pin_status_message(pinned.pin_status));
       if (pinned.wrong_reg)
         x64_asm_panic_at(c, loc, "hard-register asm operand in wrong register");
       outloc = pinned.loc;
       if (pinned.needs_stage) {
         staged_outs[i] = 1u;
         if (outs[i].dir == KIT_CG_ASM_INOUT)
           x64_asm_load_loc_to_reg(a, loc, out_locs[i], outloc);
       }
     }
     x64_asm_bind_native(a, loc, &bound_outs[i], outs[i].str, type, outloc,
                         &ntmp);
   }
   for (i = 0; i < nin; ++i) {
     const char* body = native_asm_constraint_body(ins[i].str);
     int matched = native_asm_match_index(body);
     KitCgTypeId type;
     NativeLoc inloc;
     if (matched >= 0) {
       if ((u32)matched >= nout)
         x64_asm_panic_at(c, loc, "matching constraint out of range");
       bound_ins[i] = bound_outs[matched];
       continue;
     }
     type = ins[i].type ? ins[i].type : in_locs[i].type;
     inloc = in_locs[i];
     {
       NativeAsmPinnedLoc pinned =
           native_asm_prepare_pinned_loc(t, ins[i].reg, ins[i].str, type, inloc);
       if (pinned.has_pin) {
         if (pinned.pin_status != NATIVE_ASM_REG_PIN_OK)
           x64_asm_panic_at(c, loc,
                            native_asm_pin_status_message(pinned.pin_status));
         if (pinned.wrong_reg)
           x64_asm_panic_at(c, loc,
                            "hard-register asm operand in wrong register");
         inloc = pinned.loc;
         if (pinned.needs_stage)
           x64_asm_load_loc_to_reg(a, loc, in_locs[i], inloc);
       } else if ((body[0] == 'r') && inloc.kind != NATIVE_LOC_REG) {
         Reg r;
         if (ntmp >= 2u) x64_asm_panic_at(c, loc, "too many memory asm operands");
         r = (ntmp == 0u) ? (Reg)X64_TMP_INT : (Reg)X64_TMP_INT2;
         ntmp++;
         inloc = native_loc_reg(type, NATIVE_REG_INT, r);
         x64_emit_mem(a, 1, inloc, x64_asm_loc_to_addr(a, loc, in_locs[i]),
                      native_mem_for_type(t, type, native_type_size(t, type)));
       }
     }
     x64_asm_bind_native(a, loc, &bound_ins[i], ins[i].str, type, inloc, &ntmp);
   }
 
   /* No per-block callee-saved spill here: plan_frame forwarded the asm clobber
    * masks and x64_known_callee_saves folded the callee-saved ones into the
    * function's saved set, so the prologue/epilogue already preserve them. */
   asmh = x64_asm_open(c);
   x64_inline_bind(asmh, outs, nout, bound_outs, ins, nin, bound_ins, clobbers,
                   nclob);
   x64_asm_run_template(asmh, t->mc, tmpl);
   x64_asm_close(asmh);
 
   for (i = 0; i < nout; ++i) {
     NativeAllocClass cls;
     NativeLoc src;
     if (!staged_outs || !staged_outs[i]) continue;
     if (bound_outs[i].kind != X64_INLINE_OPK_REG) continue;
     cls = bound_outs[i].pad[0] == X64_INLINE_OPCLS_FP ? NATIVE_REG_FP
                                                        : NATIVE_REG_INT;
     src = native_loc_reg(bound_outs[i].type, cls, (Reg)bound_outs[i].v.local);
     x64_asm_store_reg_to_loc(a, loc, out_locs[i], src);
   }
 }
 
 /* file_scope_asm + finalize are shared (cg/native_asm.h). */
 
 static void x64_trap(NativeTarget* t) { emit_ud2(t->mc); }
 static void x64_set_loc(NativeTarget* t, SrcLoc loc) {
   x64_of(t)->loc = loc;
   if (t->mc->set_loc) t->mc->set_loc(t->mc, loc);
 }
 
 /* Physical registers each x86-64 instruction's encoding clobbers as a side
  * effect, so the optimizer keeps values live across them out of those registers
  * (the backend is then free to use them). idiv/div write rax (quotient) and rdx
  * (remainder/sign); a variable shift uses cl; cmpxchg/xadd loops use
  * rax/rcx/rdx; an FP va_arg borrows rax for the gp/fp offset (an integer va_arg
  * does the offset arithmetic in its own destination register, so it clobbers
  * nothing). */
 static int x64_machine_op_clobbers(NativeTarget* t, const NativeMachineOp* op,
                                    u32 mask[NATIVE_CALL_PLAN_CLASSES]) {
   (void)t;
   mask[0] = mask[1] = mask[2] = 0;
   switch ((NativeMachineOpKind)op->kind) {
     case NATIVE_MOP_BINOP:
       switch ((BinOp)op->binop) {
         case BO_SDIV:
         case BO_UDIV:
         case BO_SREM:
         case BO_UREM:
           mask[NATIVE_REG_INT] = (1u << X64_RAX) | (1u << X64_RDX);
           return 1;
         case BO_SHL:
         case BO_SHR_S:
         case BO_SHR_U:
           if (op->second_is_reg) {
             mask[NATIVE_REG_INT] = (1u << X64_RCX);
             return 1;
           }
           return 0;
         default:
           return 0;
       }
     case NATIVE_MOP_ATOMIC_CAS:
     case NATIVE_MOP_ATOMIC_RMW:
       mask[NATIVE_REG_INT] =
           (1u << X64_RAX) | (1u << X64_RCX) | (1u << X64_RDX);
       return 1;
     case NATIVE_MOP_VA_START:
       /* x64_va_start_core materializes the va_list field values through RAX
        * (the ap pointer itself lands in the reserved r11 scratch). RAX is the
        * return register, so the allocator may otherwise keep a live value there
        * across the op. */
       mask[NATIVE_REG_INT] = (1u << X64_RAX);
       return 1;
     case NATIVE_MOP_VA_ARG:
       if (!op->result_is_fp) return 0;
       mask[NATIVE_REG_INT] = (1u << X64_RAX);
       return 1;
     case NATIVE_MOP_INTRINSIC:
       /* The unsigned multiply-overflow intrinsic emits a one-operand MUL, whose
        * rdx:rax product clobbers both registers. The signed variant uses a
        * two-operand IMUL (no fixed-register clobber). Linux syscall writes rax
        * and the CPU instruction itself clobbers rcx/r11; the kernel ABI treats
        * the integer caller-saved syscall registers as volatile. */
       if ((IntrinKind)op->intrin == INTRIN_UMUL_OVERFLOW) {
         mask[NATIVE_REG_INT] = (1u << X64_RAX) | (1u << X64_RDX);
         return 1;
       }
       if ((IntrinKind)op->intrin == INTRIN_SYSCALL) {
         mask[NATIVE_REG_INT] =
             (1u << X64_RAX) | (1u << X64_RCX) | (1u << X64_RDX) |
             (1u << X64_RSI) | (1u << X64_RDI) | (1u << X64_R8) |
             (1u << X64_R9) | (1u << X64_R10) | (1u << X64_R11);
         return 1;
       }
       return 0;
     default:
       return 0;
   }
 }
 
 /* ============================ construction ============================ */
 
 NativeTarget* x64_native_target_new(Compiler* c, ObjBuilder* obj,
                                     MCEmitter* mc) {
   X64NativeTarget* a = arena_znew(c->tu, X64NativeTarget);
   NativeTarget* t;
   if (!a) return NULL;
   t = &a->base;
   t->c = c;
   t->obj = obj;
   t->mc = mc;
   native_frame_init(&a->frame, c);
   t->regs = &x64_reg_info;
   t->class_for_type = native_class_for_type_fp_le8;
   t->imm_legal = x64_imm_legal;
   t->addr_legal = x64_addr_legal;
   t->machine_op_clobbers = x64_machine_op_clobbers;
   t->func_begin = x64_func_begin;
   t->func_begin_known_frame = x64_func_begin_known_frame;
   t->bind_params_end = x64_bind_params_end;
   t->note_frame_state = NULL;
   /* Non-NULL so the optimizer emit path (plan_frame) computes the callee-saved
    * set; x64_func_begin_known_frame derives the records from the masks. */
   t->reserve_callee_saves = x64_reserve_callee_saves;
   t->caller_saved_mask = x64_live_caller_saved_mask;
   t->callee_saved_mask = x64_live_callee_saved_mask;
   t->signature_stack_bytes = x64_signature_stack_bytes;
   t->call_stack_bytes = x64_call_stack_bytes;
   t->has_store_zero_reg = 0;
   t->func_end = x64_func_end;
   t->frame_slot = x64_frame_slot;
   t->frame_slot_debug_loc = x64_frame_slot_debug_loc;
   t->bind_param = x64_bind_native_param;
   t->label_new = x64_label_new;
   t->label_place = x64_label_place;
   t->jump = x64_jump;
   t->cmp_branch = x64_cmp_branch;
   t->indirect_branch = x64_indirect_branch;
   t->load_label_addr = x64_load_label_addr;
   t->move = x64_move;
   t->load_imm = x64_load_imm;
   t->load_const = x64_load_const;
   t->load_addr = x64_load_addr;
   t->load = x64_load;
   t->store = x64_store;
   t->tls_addr_of = x64_tls_addr_of;
   t->copy_bytes = x64_copy_bytes;
   t->set_bytes = x64_set_bytes;
   t->bitfield_load = x64_bitfield_load;
   t->bitfield_store = x64_bitfield_store;
   t->binop = x64_binop;
   t->unop = x64_unop;
   t->cmp = x64_cmp;
   t->convert = x64_convert;
   t->alloca_ = x64_alloca;
   t->spill = x64_spill;
   t->reload = x64_reload;
   t->plan_call = x64_plan_call;
   t->emit_call = x64_emit_call;
   t->plan_ret = x64_plan_ret;
   t->ret = x64_ret;
   t->atomic_load = x64_atomic_load;
   t->atomic_store = x64_atomic_store;
   t->atomic_rmw = x64_atomic_rmw;
   t->atomic_cas = x64_atomic_cas;
   t->fence = x64_fence;
   t->va_start_ = x64_va_start_native;
   t->va_arg_ = x64_va_arg_native;
   t->va_end_ = x64_va_end_native;
   t->va_copy_ = x64_va_copy_native;
   t->intrinsic = x64_intrinsic;
   t->asm_block = x64_asm_block_native;
   t->file_scope_asm = native_file_scope_asm;
   t->trap = x64_trap;
   t->set_loc = x64_set_loc;
   t->finalize = native_finalize;
   return t;
 }
 
 /* ============================ NativeOps (-O0) ============================ */
 
 static void x64_bind_param(NativeDirectTarget* d, const CGParamDesc* p,
                            CGLocal local, NativeDirectLocal* l) {
   NativeLoc dst;
   (void)local;
   memset(&dst, 0, sizeof dst);
   dst.kind = NATIVE_LOC_FRAME;
   dst.type = p->type;
   dst.v.frame = l->home;
   x64_bind_native_param(d->native, p, dst);
 }
 
 /* A sibling call is realizable when its outgoing stack-argument area fits the
  * window the caller itself received. Register-only calls always qualify. */
 static const char* x64_no_tail(NativeDirectTarget* d, const CGCallDesc* call) {
   X64NativeTarget* a = x64_of(d->native);
   NativeCallDesc nd;
   NativeLoc* args = NULL;
   NativeLoc* results = NULL;
   u32 i, stack;
   if (a->frame.ncallee_saves)
     return "x64 tail call: callee-saved registers in use";
   memset(&nd, 0, sizeof nd);
   u32 nresults = call->result != CG_LOCAL_NONE ? 1u : 0u;
   if (call->nargs) args = arena_zarray(d->base.c->tu, NativeLoc, call->nargs);
   if (nresults) results = arena_zarray(d->base.c->tu, NativeLoc, nresults);
   for (i = 0; i < call->nargs; ++i) {
     args[i].kind = NATIVE_LOC_FRAME;
     args[i].type = d->locals[call->args[i] - 1u].type;
     args[i].cls = d->locals[call->args[i] - 1u].cls;
     args[i].v.frame = d->locals[call->args[i] - 1u].home;
   }
   if (nresults) {
     results[0].kind = NATIVE_LOC_FRAME;
     results[0].type = d->locals[call->result - 1u].type;
     results[0].cls = d->locals[call->result - 1u].cls;
     results[0].v.frame = d->locals[call->result - 1u].home;
   }
   nd.fn_type = call->fn_type;
   nd.args = args;
   nd.results = results;
   nd.nargs = call->nargs;
   nd.nresults = nresults;
   stack = x64_call_stack_size(d->native, &nd);
   /* x64_call_stack_size includes the shadow-space prefix; the caller's incoming
    * window has the same prefix, so compare against incoming_stack_size + it. */
   if (stack > a->incoming_stack_size + a->abi->shadow_space)
     return "x64 tail call: stack argument area too small";
   return NULL;
 }
 
 /* Resolve a pointer-typed Operand (the address of a va_list object) into `reg`,
  * returning a register-based NativeAddr. */
 static NativeAddr x64_direct_pointer_addr(NativeDirectTarget* d, Operand op) {
   X64NativeTarget* a = x64_of(d->native);
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   if (op.kind == OPK_LOCAL) {
     emit_mov_load(a->base.mc, 8, 0, X64_R11, X64_RBP,
                   -(i32)x64_slot_get(a, d->locals[op.v.local - 1u].home)->off);
     addr.base_kind = NATIVE_ADDR_BASE_REG;
     addr.base.reg = X64_R11;
     addr.base_type = op.type;
     return addr;
   }
   return x64_direct_materialize_addr(d, op);
 }
 
 static NativeAddr x64_direct_va_base(NativeDirectTarget* d, Operand ap_addr,
                                      Reg reg) {
   NativeLoc dst =
       native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT, reg);
   NativeAddr addr;
   d->native->load_addr(d->native, dst, x64_direct_pointer_addr(d, ap_addr));
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_REG;
   addr.cls = NATIVE_REG_INT;
   addr.base.reg = reg;
   addr.base_type = builtin_id(KIT_CG_BUILTIN_I64);
   return addr;
 }
 
 static void x64_va_start_(NativeDirectTarget* d, Operand ap_addr) {
   /* Hold the va_list base in R11, not RAX: x64_va_start_core materializes the
    * gp/fp_offset and overflow/reg-save-area field values through RAX, which
    * would otherwise clobber the base before the field stores. */
   x64_va_start_core(x64_of(d->native), x64_direct_va_base(d, ap_addr, X64_R11));
 }
 static void x64_va_arg_(NativeDirectTarget* d, Operand dst, Operand ap_addr,
                         KitCgTypeId type) {
   X64NativeTarget* a = x64_of(d->native);
   int is_fp = cg_type_is_float(d->base.c, type);
   NativeLoc res = native_loc_reg(type, is_fp ? NATIVE_REG_FP : NATIVE_REG_INT,
                                  is_fp ? X64_TMP_FP : (Reg)X64_RDX);
   NativeAddr dst_addr;
   /* Base in R11: the core advances/loads through R11 plus one GPR scratch (the
    * integer result reg itself, or RAX for FP results), so R11 must not be RAX.
    */
   x64_va_arg_core(a, res, x64_direct_va_base(d, ap_addr, X64_R11), type);
   dst_addr = x64_direct_addr(d, dst);
   if (dst_addr.base_kind == NATIVE_ADDR_BASE_FRAME_VALUE) {
     emit_mov_load(a->base.mc, 8, 0, X64_R11, X64_RBP,
                   -(i32)x64_slot_get(a, dst_addr.base.frame)->off);
     dst_addr.base_kind = NATIVE_ADDR_BASE_REG;
     dst_addr.base.reg = X64_R11;
   }
   x64_emit_mem(
       a, 0, res, dst_addr,
       native_mem_for_type(d->native, type, native_type_size(d->native, type)));
 }
 static void x64_va_end_(NativeDirectTarget* d, Operand ap_addr) {
   (void)d;
   (void)ap_addr;
 }
 static void x64_va_copy_(NativeDirectTarget* d, Operand dst, Operand src) {
   X64NativeTarget* a = x64_of(d->native);
   NativeAddr src_ap = x64_direct_va_base(d, src, X64_RAX);
   NativeAddr dst_ap = x64_direct_va_base(d, dst, X64_R11);
   x64_va_copy_core(a, dst_ap, src_ap);
 }
 
 static void x64_direct_asm_block(NativeDirectTarget* d, const char* tmpl,
                                  const AsmConstraint* outs, u32 nout,
                                  Operand* out_ops, const AsmConstraint* ins,
                                  u32 nin, const Operand* in_ops,
                                  const Sym* clobbers, u32 nclob,
                                  u32 clobber_abi_sets) {
   X64NativeTarget* a = x64_of(d->native);
   Compiler* c = d->base.c;
   Operand* bound_outs = nout ? arena_zarray(c->tu, Operand, nout) : NULL;
   Operand* bound_ins = nin ? arena_zarray(c->tu, Operand, nin) : NULL;
   u32 clob_int, clob_fp, abi_int, abi_fp, used_int, used_fp;
   X64AsmSavedClobber* saved;
   u32 nsaved, i;
   X64Asm* asmh;
 
   x64_asm_clobber_masks(c, d->loc, clobbers, nclob, &clob_int, &clob_fp);
   native_asm_abi_clobber_masks(d->native, clobber_abi_sets, &abi_int, &abi_fp);
   clob_int |= abi_int;
   clob_fp |= abi_fp;
   /* Reserve emit scratch (r10,r11), driver scratch (r8,r9), rax (reserved;
    * only self-allocated here when explicitly pinned), sp/bp, and clobbers. */
   used_int = clob_int | (1u << X64_RAX) | (1u << X64_R11) | (1u << X64_RSP) |
              (1u << X64_RBP) | (1u << X64_R8) | (1u << X64_R9) |
              (1u << X64_R10);
   used_fp = clob_fp | (1u << X64_XMM4) | (1u << X64_XMM5) |
             (1u << (X64_XMM0 + 14)) | (1u << X64_XMM15);
 
   for (i = 0; i < nout; ++i) {
     const char* body = native_asm_constraint_body(outs[i].str);
     KitCgTypeId type = outs[i].type ? outs[i].type : out_ops[i].type;
     NativeAsmRegPin pin;
     if (x64_asm_resolve_pin_or_panic(d, outs[i].reg, outs[i].str, &pin)) {
       /* GNU local register variable: pin to the named hard register. */
       if (pin.cls == NATIVE_REG_FP) {
         used_fp |= 1u << pin.reg;
         clob_fp |= 1u << pin.reg;
       } else {
         used_int |= 1u << pin.reg;
         clob_int |= 1u << pin.reg;
       }
       x64_asm_bound_reg(&bound_outs[i], type, pin.cls, pin.reg);
     } else {
       NativeAsmConstraintInfo info;
       if (native_asm_constraint_reg_info(d->native, outs[i].str, &info)) {
         Reg reg = info.fixed_reg != REG_NONE
                       ? info.fixed_reg
                       : x64_asm_alloc_reg(d, info.cls, info.allowed_mask,
                                           &used_int, &used_fp);
         if (info.cls == NATIVE_REG_FP) {
           used_fp |= 1u << reg;
           if (info.fixed_reg != REG_NONE) clob_fp |= 1u << reg;
         } else {
           used_int |= 1u << reg;
           if (info.fixed_reg != REG_NONE) clob_int |= 1u << reg;
         }
         x64_asm_bound_reg(&bound_outs[i], type, info.cls, reg);
       } else if (body[0] == 'm') {
         Reg reg = x64_asm_alloc_reg(d, NATIVE_REG_INT, 0, &used_int, &used_fp);
         x64_asm_bound_mem(&bound_outs[i], type, reg);
       } else {
         x64_asm_panic(d, "unsupported output constraint");
       }
     }
   }
 
   for (i = 0; i < nin; ++i) {
     const char* body = native_asm_constraint_body(ins[i].str);
     int matched = native_asm_match_index(body);
     KitCgTypeId type = ins[i].type ? ins[i].type : in_ops[i].type;
     if (matched >= 0) {
       if ((u32)matched >= nout)
         x64_asm_panic(d, "matching constraint out of range");
       if (native_asm_constraint_early(outs[matched].str))
         x64_asm_panic(d, "matching input names early-clobber output");
       if (bound_outs[matched].kind != X64_INLINE_OPK_REG)
         x64_asm_panic(d, "matching constraint requires register output");
       bound_ins[i] = bound_outs[matched];
       continue;
     }
     NativeAsmRegPin pin;
     if (x64_asm_resolve_pin_or_panic(d, ins[i].reg, ins[i].str, &pin)) {
       /* GNU local register variable: pin to the named hard register. */
       if (pin.cls == NATIVE_REG_FP) {
         used_fp |= 1u << pin.reg;
         clob_fp |= 1u << pin.reg;
       } else {
         used_int |= 1u << pin.reg;
         clob_int |= 1u << pin.reg;
       }
       x64_asm_bound_reg(&bound_ins[i], type, pin.cls, pin.reg);
     } else {
       NativeAsmConstraintInfo info;
       if (native_asm_constraint_reg_info(d->native, ins[i].str, &info)) {
         Reg reg = info.fixed_reg != REG_NONE
                       ? info.fixed_reg
                       : x64_asm_alloc_reg(d, info.cls, info.allowed_mask,
                                           &used_int, &used_fp);
         if (info.cls == NATIVE_REG_FP) {
           used_fp |= 1u << reg;
           if (info.fixed_reg != REG_NONE) clob_fp |= 1u << reg;
         } else {
           used_int |= 1u << reg;
           if (info.fixed_reg != REG_NONE) clob_int |= 1u << reg;
         }
         x64_asm_bound_reg(&bound_ins[i], type, info.cls, reg);
       } else if (body[0] == 'i') {
         if (in_ops[i].kind != OPK_IMM)
           x64_asm_panic(d, "immediate constraint requires immediate operand");
         bound_ins[i] = in_ops[i];
       } else if (body[0] == 'm') {
         Reg reg = x64_asm_alloc_reg(d, NATIVE_REG_INT, 0, &used_int, &used_fp);
         x64_asm_bound_mem(&bound_ins[i], type, reg);
       } else {
         x64_asm_panic(d, "unsupported input constraint");
       }
     }
   }
 
   saved = x64_asm_save_callee_clobbers(a, clob_int, clob_fp, &nsaved);
   for (i = 0; i < nout; ++i) {
     if (bound_outs[i].kind == X64_INLINE_OPK_REG) {
       NativeAllocClass cls = bound_outs[i].pad[0] == X64_INLINE_OPCLS_FP
                                  ? NATIVE_REG_FP
                                  : NATIVE_REG_INT;
       if (outs[i].dir == KIT_CG_ASM_INOUT) {
         x64_direct_load_operand_to_reg(
             d, out_ops[i],
             native_loc_reg(bound_outs[i].type, cls,
                            (Reg)bound_outs[i].v.local));
       }
     } else if (bound_outs[i].kind == OPK_INDIRECT) {
       NativeLoc loc =
           native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT,
                          (Reg)bound_outs[i].v.ind.base);
       x64_direct_load_address_to_reg(d, out_ops[i], loc);
     }
   }
   for (i = 0; i < nin; ++i) {
     if (bound_ins[i].kind == X64_INLINE_OPK_REG) {
       NativeAllocClass cls = bound_ins[i].pad[0] == X64_INLINE_OPCLS_FP
                                  ? NATIVE_REG_FP
                                  : NATIVE_REG_INT;
       x64_direct_load_operand_to_reg(
           d, in_ops[i],
           native_loc_reg(bound_ins[i].type, cls, (Reg)bound_ins[i].v.local));
     } else if (bound_ins[i].kind == OPK_INDIRECT) {
       NativeLoc loc =
           native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT,
                          (Reg)bound_ins[i].v.ind.base);
       x64_direct_load_address_to_reg(d, in_ops[i], loc);
     }
   }
   asmh = x64_asm_open(c);
   x64_inline_bind(asmh, outs, nout, bound_outs, ins, nin, bound_ins, clobbers,
                   nclob);
   x64_asm_run_template(asmh, d->native->mc, tmpl);
   x64_asm_close(asmh);
 
   for (i = 0; i < nout; ++i) {
     NativeAllocClass cls;
     NativeLoc src;
     if (bound_outs[i].kind != X64_INLINE_OPK_REG) continue;
     cls = bound_outs[i].pad[0] == X64_INLINE_OPCLS_FP ? NATIVE_REG_FP
                                                       : NATIVE_REG_INT;
     src = native_loc_reg(bound_outs[i].type, cls, (Reg)bound_outs[i].v.local);
     x64_direct_store_reg_to_operand(d, out_ops[i], src);
   }
   for (i = nsaved; i > 0; --i) x64_asm_restore_one(a, &saved[i - 1u]);
 }
 
 static const NativeOps x64_direct_ops = {
     .bind_param = x64_bind_param,
     .tail_call_unrealizable_reason = x64_no_tail,
     .va_start_ = x64_va_start_,
     .va_arg_ = x64_va_arg_,
     .va_end_ = x64_va_end_,
     .va_copy_ = x64_va_copy_,
     .asm_block = x64_direct_asm_block,
 };
 
 const NativeOps* x64_native_direct_ops(void) { return &x64_direct_ops; }