kit

native.c (210050B)

---

 /* aa64 NativeTarget production-readiness checklist:
  * - ABI completeness: finish AAPCS64/Linux va_list and register-save-area
  *   lowering, verify Apple/AAPCS64/Windows arm64 differences, handle all
  *   homogeneous aggregates, indirect/byval/sret corner cases, small aggregate
  *   splitting, multi-register returns, stack alignment, and ABI diagnostics.
  * - Calls and returns: replace call-plus-return tail handling with true direct
  *   and indirect sibling calls, preserve musttail ABI guarantees, support stack
  *   argument reshuffling without clobbering live inputs, and cover all sret,
  *   variadic, FP, aggregate, and many-argument combinations.
  * - Frame lowering: implement known-frame/prologue integration for optimized
  *   emission, spill/reload hooks, callee-save tracking for integer and FP/SIMD
  *   registers, large-frame probing/materialization as needed by each platform,
  *   dynamic alloca restoration, and unwind/debug frame metadata.
  * - Operations and intrinsics: fill remaining scalar, FP, conversion, rounding,
  *   overflow, bit, vector/SIMD, trap, prefetch, and target-specific intrinsics;
  *   validate NaN/ordered/unordered FP compare semantics and integer narrowing
  *   behavior for every supported width.
  * - Aggregates and memory: support large constants, overlap-safe memmove,
  *   optimized bulk copy/set selection, bitfield load/store, packed/unaligned
  *   accesses, volatile access constraints, and record/slice edge cases across
  *   direct and optimized lowering.
  * - Atomics: replace ordinary load/store RMW/CAS sequences with correct LL/SC
  *   or LSE loops, implement acquire/release/seq_cst mappings precisely, handle
  *   failure ordering, byte/halfword/word/dword widths, and retry/clobber rules.
  * - Inline and file-scope asm: complete register/memory/immediate constraints,
  *   named operands, tied operands, early-clobber and clobber validation, hard
  *   register conflicts, memory barriers, outputs for aggregates/FP values, and
  *   file-scope asm integration. */
 
 #include <string.h>
 
 #include "abi/abi.h"
 #include "arch/aa64/aa64.h"
 #include "arch/aa64/asm.h"
 #include "arch/aa64/isa.h"
 #include "arch/aa64/regs.h"
 #include "asm/asm.h"
 #include "asm/asm_lex.h"
 #include "cg/native_argmove.h"
 #include "cg/native_asm.h"
 #include "cg/native_direct_target.h"
 #include "cg/native_frame.h"
 #include "cg/type.h"
 #include "core/arena.h"
 #include "core/bytes.h"
 #include "core/core.h"
 #include "core/pool.h"
 #include "core/slice.h"
 #include "obj/obj.h"
 
 #if defined(__GNUC__) || defined(__clang__)
 #define AA_UNUSED_FN __attribute__((unused))
 #else
 #define AA_UNUSED_FN
 #endif
 
 enum {
   AA_X8 = 8u, /* indirect-result (sret) register; usable as a copy base that
                  aa_copy_bytes (which scratches only x16/x17) never clobbers */
   AA_TMP0 = 16u,
   AA_TMP1 = 17u,
   AA_FP = 29u,
   AA_LR = 30u,
   AA_SP = 31u,
   AA_FRAME_SAVE_SIZE = 16u,
   /* Worst-case reserved prologue region (NDT single-pass path patches it in
    * place; the optimizer path reserves exactly what it emits). Sized to hold
    * the fat prologue plus the Windows large-frame stack probe (≤7 words, see
    * aa_words_stack_probe). */
   AA_PROLOGUE_WORDS = 32u,
   AA_TAIL_WORDS = 32u,
 };
 
 /* Windows/AArch64 TLS Local-Exec. The TEB pointer lives in the reserved
  * platform register x18 (never allocated; see AA_PHYS_INT_RESERVED(18)), and
  * the thread's TLS-array pointer (TEB.ThreadLocalStoragePointer) sits at
  * TEB+0x58 — same offset as on Win64/x86-64. */
 enum {
   AA_WIN_TEB_REG = 18u,
   AA_WIN_TEB_TLS_PTR_OFF = 0x58u,
 };
 
 /* ============================================================================
  * AAPCS64 frame layout
  *
  * Two layouts. Every fp- or sp-relative offset in this file is computed via one
  * of the aa_fp_off_ / aa_sp_off_ helpers below — no site does bare arithmetic
  * on AA_FP / AA_SP, and no site outside those helpers branches on the layout.
  *
  * TOP-RECORD (default — single-pass -O0, fat frames, and out_stack>0 small
  * frames). fp anchors at the caller's saved-pair address near the top; sp at
  * the bottom of the outgoing-arg area. Offsets are frame-size-independent.
  *
  *   high addr   caller's stack frame
  *               +------------------------------+
  *               | incoming stack args          |  aa_fp_off_in_arg(a,i) = 16+i
  *               +------------------------------+
  *      fp  -->  | saved x29 (prev fp)          |  aa_fp_off_saved_fp() = 0
  *               | saved x30 (prev lr)          |  aa_fp_off_saved_lr() = 8
  *               +------------------------------+
  *               | frame slots                  |  aa_fp_off_slot(a,off) = -off
  *               |   (callee-saves + locals     |
  *               |    + spills + sret/variadic) |
  *               +------------------------------+
  *               | outgoing args                |  aa_sp_off_out_arg(i)
  *      sp  -->  +------------------------------+
  *   low addr                                       CFA = fp + 16
  *
  * BOTTOM-RECORD (fp_at_bottom — known-frame -O1 small frames with
  * callee-saves/locals and out_stack==0). The record moves to the bottom so the
  * sp adjustment folds into a pre/post-indexed stp/ldp (−2 insns/call). fp = sp;
  * slots/callee-saves stack ABOVE the record at positive offsets. Offsets depend
  * on frame_size (hence known-frame only, where the frame is final before body).
  *
  *   high addr   caller's stack frame
  *               +------------------------------+
  *               | incoming stack args          |  aa_fp_off_in_arg(a,i) = N+i
  *               +------------------------------+   <- caller's sp = CFA = fp +
  * N | frame slots (+ align pad)    |  aa_fp_off_slot(a,off)=N-off |
  * (callee-saves + locals …)  |     (in [16, N), above record)
  *               +------------------------------+
  *  fp = sp -->  | saved x29 (prev fp)          |  aa_fp_off_saved_fp() = 0
  *               | saved x30 (prev lr)          |  aa_fp_off_saved_lr() = 8
  *   low addr    +------------------------------+   (N = frame_size;
  * out_stack==0)
  *
  * frame_size (N) = align16(AA_FRAME_SAVE_SIZE + slot_bytes + out_stack).
  * Tail calls write outgoing args into the caller's incoming-args window —
  * physically the same address, expressed via aa_fp_off_tail_out_arg.
  * ========================================================================== */
 
 static u32 align_up_u32(u32 v, u32 align);
 
 typedef struct AAFrameLayout {
   u32 slot_bytes; /* sum of aa_frame_slot reservations (callee-saves + locals
                    * + spills + sret/variadic) */
   u32 out_stack;  /* max outgoing-arg bytes across all calls in this function */
   u32 top_home;   /* Windows-variadic GP register home area, reserved between
                    * the saved pair and the incoming stack args so the
                    * plain-pointer va_list walks register then stack varargs as
                    * one contiguous block (0 on every other ABI). */
   u32 frame_size; /* align16(AA_FRAME_SAVE_SIZE + top_home + slot_bytes +
                    * out_stack) */
 } AAFrameLayout;
 
 static inline AAFrameLayout aa_build_layout(u32 slot_bytes, u32 out_stack,
                                             u32 top_home) {
   AAFrameLayout L;
   L.slot_bytes = slot_bytes;
   L.out_stack = out_stack;
   L.top_home = top_home;
   L.frame_size =
       align_up_u32(AA_FRAME_SAVE_SIZE + top_home + slot_bytes + out_stack, 16u);
   return L;
 }
 
 /* FP-relative byte offsets. The saved-pair is at [fp]/[fp+8] in both the
  * top-record and bottom-record (fp_at_bottom) layouts, so these two are
  * layout-independent. The frame-size-dependent helpers — aa_fp_off_in_arg,
  * aa_fp_off_slot, aa_fp_off_tail_out_arg — branch on a->fp_at_bottom and are
  * defined after AANativeTarget (see aa_fp_off_* below aa_of). */
 static inline i32 aa_fp_off_saved_fp(void) { return 0; }
 static inline i32 aa_fp_off_saved_lr(void) { return 8; }
 
 /* SP-relative byte offsets. */
 static inline i32 aa_sp_off_out_arg(u32 byte_off) { return (i32)byte_off; }
 static inline u32 aa_sp_off_saved_pair(const AAFrameLayout* L) {
   return L->frame_size - AA_FRAME_SAVE_SIZE - L->top_home;
 }
 
 /* Frame slots and callee-save records are owned by the shared NativeFrame
  * bookkeeping (cg/native_frame.h); these aliases keep the aa64-local spellings.
  */
 typedef NativeFrameSlotEntry AANativeSlot;
 
 /* Deferred in-function patches, all resolved in aa_func_end once the frame
  * layout (max_outgoing, callee-saves) is final. One growable list carries both
  * kinds; each entry patches a disjoint, fixed code position, so insertion order
  * is irrelevant. The prologue region is patched separately (exactly one per
  * function, fixed position) and is not a list entry. */
 typedef enum AAPatchKind {
   AA_PATCH_ALLOCA, /* single instr: add dst, sp, #max_outgoing */
   AA_PATCH_TAIL,   /* AA_TAIL_WORDS region: callee restores + frame + br/b */
 } AAPatchKind;
 
 typedef struct AAPatch {
   AAPatchKind kind;
   u32 pos;
   union {
     u32 dst_reg;      /* AA_PATCH_ALLOCA */
     NativeLoc callee; /* AA_PATCH_TAIL */
   } u;
 } AAPatch;
 
 typedef NativeFrameCalleeSave AACalleeSave;
 
 typedef struct AANativeTarget {
   NativeTarget base;
   SrcLoc loc;
   const CGFuncDesc* func;
 
   /* Shared frame bookkeeping: slot table, cumulative offset, max-outgoing,
    * callee-save set, and the known_frame / has_alloca / frame_final flags. */
   NativeFrame frame;
   /* Final frame size, set once in aa_func_begin_known_frame when fp_at_bottom
    * is decided. Read by the fp-relative offset helpers in the bottom-record
    * layout (where slot/incoming-arg offsets depend on frame_size); meaningless
    * and unread on the single-pass path, which never sets fp_at_bottom. */
   u32 frame_size_final;
   u32 incoming_stack_size;
   /* Windows-variadic GP register home area size (gp_reg_count * gp_slot_size,
    * 64 today; 0 on every other ABI). When nonzero the function takes the fat
    * top-record layout and homes x0..x7 into [fp + AA_FRAME_SAVE_SIZE ..] so the
    * plain-pointer va_list can walk register then stack varargs contiguously. */
   u32 top_home_bytes;
   u32 next_param_int;
   u32 next_param_fp;
   u32 next_param_stack;
   NativeFrameSlot sret_ptr_slot;
   NativeFrameSlot saved_tmp_slot;
   NativeFrameSlot va_gr_slot;
   NativeFrameSlot va_vr_slot;
 
   AAPatch* patches;
   u32 npatches;
   u32 patches_cap;
   u32 nalloca; /* count of AA_PATCH_ALLOCA entries; gates slim prologue/frame */
 
   u32 func_start;
   u32 prologue_pos;
   u32 minimal_prologue_words; /* opt path: exact prologue length, else 0 */
   MCLabel epilogue_label;
 
   /* Set at func_end when this function qualifies for the slim prologue/epilogue
    * (Tier A: no body locals/spills, no callee-saves, no alloca, no outgoing
    * stack args, no sret/variadic). When set, the prologue patch and epilogue
    * emit a 2-insn `stp x29,x30,[sp,#-16]! ; mov x29,sp` and matching `ldp
    * x29,x30,[sp],#16 ; ret` instead of the fat 4+3-insn FP-frame form. */
   u8 slim_prologue;
   /* Set at func_end when frame_size - 16 fits stp's signed 7-bit scaled
    * immediate (frame_size <= 520). Skips the `add x17, sp, #(N-16)` scratch
    * materialization in the prologue (stp x29,x30,[sp,#N-16] instead) and
    * the matching `add x10, fp, #0` in the epilogue (ldp x29,x30,[sp,#N-16]
    * + add sp,sp,#N). Mutually exclusive with `slim_prologue` (Tier A wins
    * when both would apply) and `fp_at_bottom` (which wins for out_stack==0).
    * Now only reached for small frames with outgoing stack args (out_stack>0),
    * where the record cannot move to the bottom. Keeps the top-record layout. */
   u8 slim_small_frame;
   /* Set by aa_func_begin_known_frame for a small frame with callee-saves/locals
    * and no outgoing stack args: the frame record moves to the bottom of the
    * frame (fp = sp, `mov x29,sp`) so the sp adjustment folds into a pre-indexed
    * `stp x29,x30,[sp,#-N]!` entry and post-indexed `ldp x29,x30,[sp],#N` exit
    * (−2 insns/call vs slim_small_frame). Slots and callee-saves stack ABOVE the
    * record at positive fp offsets; incoming args sit at fp+frame_size; CFA =
    * fp+frame_size. The frame-size-dependent offsets are the reason this is only
    * available on the known-frame path (frame final before the body). Mutually
    * exclusive with slim_prologue (Tier A) and slim_small_frame; gated on
    * out_stack==0 && !has_alloca && frame_size <= 504. */
   u8 fp_at_bottom;
 } AANativeTarget;
 
 static AANativeTarget* aa_of(NativeTarget* t) { return (AANativeTarget*)t; }
 
 /* Layout-aware FP-relative offsets. Every frame use site goes through these;
  * the fp_at_bottom test lives here and nowhere else.
  *
  *   top-record (default):  record near the top, fp anchored at the saved pair.
  *     incoming args at fp+16+b, slots below fp at -off.  CFA = fp+16.
  *   bottom-record (fp_at_bottom): record at the bottom, fp = sp.
  *     incoming args at fp+frame_size+b, slots above the record at
  *     frame_size-off (in [16, frame_size), never overlapping the 16-byte
  *     record since frame_size = align16(16+cum_off) >= 16+cum_off).
  *     CFA = fp+frame_size. */
 static inline i32 aa_fp_off_in_arg(const AANativeTarget* a, u32 byte_off) {
   /* top-record incoming args sit above the saved pair and the (usually empty)
    * Windows-variadic GP home area; bottom-record never carries a home area. */
   u32 base = a->fp_at_bottom ? a->frame_size_final
                              : AA_FRAME_SAVE_SIZE + a->top_home_bytes;
   return (i32)(base + byte_off);
 }
 static inline i32 aa_fp_off_slot(const AANativeTarget* a, u32 slot_off) {
   return a->fp_at_bottom ? (i32)a->frame_size_final - (i32)slot_off
                          : -(i32)slot_off;
 }
 /* Outgoing stack args on a tail call land in the caller's incoming-arg window —
  * the same physical address the tail-callee will read via aa_fp_off_in_arg.
  * Same helper, distinct name for site-side intent. */
 static inline i32 aa_fp_off_tail_out_arg(const AANativeTarget* a,
                                          u32 byte_off) {
   return aa_fp_off_in_arg(a, byte_off);
 }
 /* CFA = caller's sp, expressed as an fp-relative offset (fp+16 top-record,
  * fp+frame_size bottom-record). Named so the CFI emit site stays layout-blind.
  */
 static inline i32 aa_cfa_off(const AANativeTarget* a) {
   return a->fp_at_bottom
              ? (i32)a->frame_size_final
              : (i32)(AA_FRAME_SAVE_SIZE + a->top_home_bytes);
 }
 
 /* fp-relative offset of GP home slot `i` (Windows variadic only). The home area
  * sits just above the saved pair and just below the incoming stack args, so
  * slot gp_reg_count coincides with incoming-arg byte 0 (top-record only — a
  * function with a home area never takes a slim/bottom layout). */
 static inline i32 aa_fp_off_home_slot(u32 i) {
   return (i32)(AA_FRAME_SAVE_SIZE + i * 8u);
 }
 
 static void aa_panic(AANativeTarget* a, const char* msg) {
   compiler_panic(a->base.c, a->loc, "aarch64 native target: %s", msg);
 }
 
 /* Declared locally rather than pulling in debug/debug.h, keeping the
  * backend's dependency on the Debug producer to this one entry point —
  * same pattern as the x64/rv64 emit TUs (see arch/mc.h). */
 extern void debug_emit_row(Debug*, ObjSecId text_section, u32 offset, SrcLoc);
 extern void debug_func_pc_range(Debug*, ObjSecId text_section, u32 begin_ofs,
                                 u32 end_ofs);
 
 static void aa_emit32(MCEmitter* mc, u32 word) {
   u8 b[4];
   u32 ofs = obj_pos(mc->obj, mc->section_id);
   wr_u32_le(b, word);
   mc->emit_bytes(mc, b, sizeof b);
   /* Record one line-table row per instruction start (no-op when not -g). */
   if (mc->debug) debug_emit_row(mc->debug, mc->section_id, ofs, mc->loc);
 }
 
 static void aa_patch32(ObjBuilder* obj, ObjSecId sec, u32 off, u32 word) {
   u8 b[4];
   wr_u32_le(b, word);
   obj_patch(obj, sec, off, b, sizeof b);
 }
 
 static u32 align_up_u32(u32 v, u32 align) {
   u32 mask = align ? align - 1u : 0u;
   return (v + mask) & ~mask;
 }
 
 static u32 type_size32(NativeTarget* t, KitCgTypeId type) {
   u64 n = type ? cg_type_size(t->c, type) : 8u;
   if (n == 0) n = 8u;
   if (n > 16u)
     compiler_panic(t->c, (SrcLoc){0, 0, 0},
                    "aarch64 native target: scalar too large");
   return (u32)n;
 }
 
 static u32 type_align32(NativeTarget* t, KitCgTypeId type) {
   u64 n = type ? cg_type_align(t->c, type) : 8u;
   if (n == 0) n = 1u;
   if (n > 16u) n = 16u;
   return (u32)n;
 }
 
 static u32 size_idx(u32 n) {
   if (n <= 1u) return 0u;
   if (n <= 2u) return 1u;
   if (n <= 4u) return 2u;
   return 3u;
 }
 
 static u32 loc_reg(NativeLoc loc) { return loc.v.reg & 0x1fu; }
 
 static int loc_is_64(NativeTarget* t, NativeLoc loc) {
   return type_size32(t, loc.type) == 8u || cg_type_is_ptr(t->c, loc.type);
 }
 
 /* native_loc_is_fp is shared in native_target.h. */
 
 static __attribute__((unused)) int aa_use_got_for_sym(NativeTarget* t,
                                                       ObjSymId sym) {
   return obj_symbol_extern_via_got(t->c, t->obj, sym);
 }
 
 static __attribute__((unused)) RelocKind aa_ldst_reloc_for_size(u32 size) {
   switch (size) {
     case 0:
       return R_AARCH64_LDST8_ABS_LO12_NC;
     case 1:
       return R_AARCH64_LDST16_ABS_LO12_NC;
     case 2:
       return R_AARCH64_LDST32_ABS_LO12_NC;
     case 3:
       return R_AARCH64_LDST64_ABS_LO12_NC;
     default:
       return R_AARCH64_LDST64_ABS_LO12_NC;
   }
 }
 
 static u32 aa_load_imm_words(u32* out, u32 cap, u32 sf, u32 rd, i64 imm) {
   u64 v = (u64)imm;
   u32 words = sf ? 4u : 2u;
   u32 n = 0;
   for (u32 i = 0; i < words; ++i) {
     u32 part = (u32)((v >> (i * 16u)) & 0xffffu);
     if (!part && n) continue;
     if (n >= cap) return 0;
     out[n] = n ? aa64_movk(sf, rd, part, i) : aa64_movz(sf, rd, part, i);
     ++n;
   }
   if (!n) {
     if (!cap) return 0;
     out[n++] = aa64_movz(sf, rd, 0, 0);
   }
   return n;
 }
 
 static void aa_emit_load_imm(MCEmitter* mc, u32 sf, u32 rd, i64 imm) {
   u32 words[4];
   u32 n = aa_load_imm_words(words, 4u, sf, rd, imm);
   for (u32 i = 0; i < n; ++i) aa_emit32(mc, words[i]);
 }
 
 static void aa_emit_add_imm(AANativeTarget* a, u32 rd, u32 rn, i32 off) {
   u32 imm12, sh;
   MCEmitter* mc = a->base.mc;
   if (off >= 0 && aa64_addsub_imm_fits(off, &imm12, &sh)) {
     aa_emit32(mc, aa64_add_imm(1, rd, rn, imm12, sh));
     return;
   }
   if (off < 0 && aa64_addsub_imm_fits(-(i64)off, &imm12, &sh)) {
     aa_emit32(mc, aa64_sub_imm(1, rd, rn, imm12, sh));
     return;
   }
   aa_emit_load_imm(mc, 1, rd, off);
   aa_emit32(mc, aa64_add(1, rd, rn, rd));
 }
 
 static __attribute__((unused)) void aa_emit_add_i64(AANativeTarget* a, u32 rd,
                                                     u32 rn, i64 off) {
   u32 imm12, sh;
   MCEmitter* mc = a->base.mc;
   if (off >= 0 && aa64_addsub_imm_fits(off, &imm12, &sh)) {
     aa_emit32(mc, aa64_add_imm(1, rd, rn, imm12, sh));
     return;
   }
   if (off < 0 && aa64_addsub_imm_fits(-off, &imm12, &sh)) {
     aa_emit32(mc, aa64_sub_imm(1, rd, rn, imm12, sh));
     return;
   }
   aa_emit_load_imm(mc, 1, rd, off);
   aa_emit32(mc, aa64_add(1, rd, rn, rd));
 }
 
 static u32 aa_ldur_v(u32 size, u32 v, u32 rt, u32 rn, i32 simm9) {
   return aa64_ldst_simm9_pack((AA64LdStSimm9){.size = size,
                                               .V = v,
                                               .opc = AA64_LDST_OPC_LDR,
                                               .imm9 = (u32)simm9 & 0x1ffu,
                                               .Rn = rn,
                                               .Rt = rt});
 }
 
 static u32 aa_stur_v(u32 size, u32 v, u32 rt, u32 rn, i32 simm9) {
   return aa64_ldst_simm9_pack((AA64LdStSimm9){.size = size,
                                               .V = v,
                                               .opc = AA64_LDST_OPC_STR,
                                               .imm9 = (u32)simm9 & 0x1ffu,
                                               .Rn = rn,
                                               .Rt = rt});
 }
 
 static u32 aa_ldr_uimm_v(u32 size, u32 v, u32 rt, u32 rn, u32 byte_off) {
   u32 sc = byte_off >> size;
   return aa64_ldst_uimm_pack((AA64LdStUimm){.size = size,
                                             .V = v,
                                             .opc = AA64_LDST_OPC_LDR,
                                             .imm12 = sc,
                                             .Rn = rn,
                                             .Rt = rt});
 }
 
 static u32 aa_str_uimm_v(u32 size, u32 v, u32 rt, u32 rn, u32 byte_off) {
   u32 sc = byte_off >> size;
   return aa64_ldst_uimm_pack((AA64LdStUimm){.size = size,
                                             .V = v,
                                             .opc = AA64_LDST_OPC_STR,
                                             .imm12 = sc,
                                             .Rn = rn,
                                             .Rt = rt});
 }
 
 static u32 aa_ldr_uimm(u32 size, u32 rt, u32 rn, u32 byte_off) {
   return aa_ldr_uimm_v(size, 0, rt, rn, byte_off);
 }
 
 static __attribute__((unused)) u32 aa_str_uimm(u32 size, u32 rt, u32 rn,
                                                u32 byte_off) {
   return aa_str_uimm_v(size, 0, rt, rn, byte_off);
 }
 
 static __attribute__((unused)) u32 aa_ldst_regoff_v(u32 size, u32 v, u32 load,
                                                     u32 rt, u32 rn, u32 rm,
                                                     u32 scaled) {
   return ((size & 3u) << 30) | 0x38200800u | ((v & 1u) << 26) |
          ((load ? AA64_LDST_OPC_LDR : AA64_LDST_OPC_STR) << 22) |
          ((rm & 0x1fu) << 16) | (3u << 13) | ((scaled & 1u) << 12) |
          ((rn & 0x1fu) << 5) | (rt & 0x1fu);
 }
 
 static __attribute__((unused)) u32 aa_ldr_lit64(u32 rt, u32 imm19) {
   return 0x58000000u | ((imm19 & 0x7ffffu) << 5) | (rt & 0x1fu);
 }
 
 static __attribute__((unused)) u32 aa_mrs_tpidr_el0(u32 rt) {
   return 0xd53bd040u | (rt & 0x1fu);
 }
 
 static u32 aa_fp_bin(u32 op, u32 is_double, u32 rd, u32 rn, u32 rm) {
   return (is_double ? 0x1e600000u : 0x1e200000u) | op | ((rm & 0x1fu) << 16) |
          ((rn & 0x1fu) << 5) | (rd & 0x1fu);
 }
 
 static u32 aa_fcmp(u32 is_double, u32 rn, u32 rm) {
   return (is_double ? 0x1e602000u : 0x1e202000u) | ((rm & 0x1fu) << 16) |
          ((rn & 0x1fu) << 5);
 }
 
 static u32 aa_fneg(u32 is_double, u32 rd, u32 rn) {
   return (is_double ? 0x1e614000u : 0x1e214000u) | ((rn & 0x1fu) << 5) |
          (rd & 0x1fu);
 }
 
 static u32 aa_fmov_fp(u32 is_double, u32 rd, u32 rn) {
   return (is_double ? 0x1e604000u : 0x1e204000u) | ((rn & 0x1fu) << 5) |
          (rd & 0x1fu);
 }
 
 /* MOV Vd.16B, Vn.16B (alias of ORR Vd.16B, Vn.16B, Vn.16B): a full 128-bit
  * SIMD register copy. Used to move binary128 / long double values, which fmov
  * (scalar, max 64-bit) would truncate. */
 static u32 aa_mov_vec16(u32 rd, u32 rn) {
   return 0x4ea01c00u | ((rn & 0x1fu) << 16) | ((rn & 0x1fu) << 5) |
          (rd & 0x1fu);
 }
 
 static u32 aa_scvtf(u32 is_double_dst, u32 is64_src, u32 fd, u32 rn) {
   return (is64_src ? 0x9e220000u : 0x1e220000u) |
          (is_double_dst ? 0x00400000u : 0) | ((rn & 0x1fu) << 5) | (fd & 0x1fu);
 }
 
 static u32 aa_ucvtf(u32 is_double_dst, u32 is64_src, u32 fd, u32 rn) {
   return (is64_src ? 0x9e230000u : 0x1e230000u) |
          (is_double_dst ? 0x00400000u : 0) | ((rn & 0x1fu) << 5) | (fd & 0x1fu);
 }
 
 static u32 aa_fcvtzs(u32 is64_dst, u32 is_double_src, u32 rd, u32 fn) {
   return (is64_dst ? 0x9e380000u : 0x1e380000u) |
          (is_double_src ? 0x00400000u : 0) | ((fn & 0x1fu) << 5) | (rd & 0x1fu);
 }
 
 static u32 aa_fcvtzu(u32 is64_dst, u32 is_double_src, u32 rd, u32 fn) {
   return (is64_dst ? 0x9e390000u : 0x1e390000u) |
          (is_double_src ? 0x00400000u : 0) | ((fn & 0x1fu) << 5) | (rd & 0x1fu);
 }
 
 static u32 aa_fcvt_d_s(u32 rd, u32 rn) {
   return 0x1e22c000u | ((rn & 0x1fu) << 5) | (rd & 0x1fu);
 }
 
 static u32 aa_fcvt_s_d(u32 rd, u32 rn) {
   return 0x1e624000u | ((rn & 0x1fu) << 5) | (rd & 0x1fu);
 }
 
 static u32 aa_fmov_gpr_to_fp(u32 is64, u32 fd, u32 rn) {
   return (is64 ? 0x9e670000u : 0x1e270000u) | ((rn & 0x1fu) << 5) |
          (fd & 0x1fu);
 }
 
 static u32 aa_fmov_fp_to_gpr(u32 is64, u32 rd, u32 fn) {
   return (is64 ? 0x9e660000u : 0x1e260000u) | ((fn & 0x1fu) << 5) |
          (rd & 0x1fu);
 }
 
 static u32 aa_clz(u32 sf, u32 rd, u32 rn) {
   return (sf ? 0xdac01000u : 0x5ac01000u) | ((rn & 0x1fu) << 5) | (rd & 0x1fu);
 }
 
 static u32 aa_rbit(u32 sf, u32 rd, u32 rn) {
   return (sf ? 0xdac00000u : 0x5ac00000u) | ((rn & 0x1fu) << 5) | (rd & 0x1fu);
 }
 
 static u32 aa_rev(u32 sf, u32 rd, u32 rn) {
   return (sf ? 0xdac00c00u : 0x5ac00800u) | ((rn & 0x1fu) << 5) | (rd & 0x1fu);
 }
 
 static u32 aa_sbfm(u32 sf, u32 rd, u32 rn, u32 immr, u32 imms) {
   return (sf ? 0x93400000u : 0x13000000u) | ((immr & 0x3fu) << 16) |
          ((imms & 0x3fu) << 10) | ((rn & 0x1fu) << 5) | (rd & 0x1fu);
 }
 
 static __attribute__((unused)) u32 aa_ubfm(u32 sf, u32 rd, u32 rn, u32 immr,
                                            u32 imms) {
   return (sf ? 0xd3400000u : 0x53000000u) | ((immr & 0x3fu) << 16) |
          ((imms & 0x3fu) << 10) | ((rn & 0x1fu) << 5) | (rd & 0x1fu);
 }
 
 static __attribute__((unused)) u32 aa_ldaxr(u32 size, u32 rt, u32 rn) {
   return (size << 30) | 0x085ffc00u | ((rn & 0x1fu) << 5) | (rt & 0x1fu);
 }
 
 static __attribute__((unused)) u32 aa_ldxr(u32 size, u32 rt, u32 rn) {
   return (size << 30) | 0x085f7c00u | ((rn & 0x1fu) << 5) | (rt & 0x1fu);
 }
 
 static __attribute__((unused)) u32 aa_stlxr(u32 size, u32 rs, u32 rt, u32 rn) {
   return (size << 30) | 0x0800fc00u | ((rs & 0x1fu) << 16) |
          ((rn & 0x1fu) << 5) | (rt & 0x1fu);
 }
 
 static __attribute__((unused)) u32 aa_stxr(u32 size, u32 rs, u32 rt, u32 rn) {
   return (size << 30) | 0x08007c00u | ((rs & 0x1fu) << 16) |
          ((rn & 0x1fu) << 5) | (rt & 0x1fu);
 }
 
 static __attribute__((unused)) u32 aa_ldar(u32 size, u32 rt, u32 rn) {
   return (size << 30) | 0x08dffc00u | ((rn & 0x1fu) << 5) | (rt & 0x1fu);
 }
 
 static __attribute__((unused)) u32 aa_stlr(u32 size, u32 rt, u32 rn) {
   return (size << 30) | 0x089ffc00u | ((rn & 0x1fu) << 5) | (rt & 0x1fu);
 }
 
 static u32 aa_umaddl(u32 rd, u32 rn, u32 rm, u32 ra) {
   return 0x9ba00000u | ((rm & 0x1fu) << 16) | ((ra & 0x1fu) << 10) |
          ((rn & 0x1fu) << 5) | (rd & 0x1fu);
 }
 
 static u32 aa_smaddl(u32 rd, u32 rn, u32 rm, u32 ra) {
   return 0x9b200000u | ((rm & 0x1fu) << 16) | ((ra & 0x1fu) << 10) |
          ((rn & 0x1fu) << 5) | (rd & 0x1fu);
 }
 
 static u32 aa_smulh(u32 rd, u32 rn, u32 rm) {
   return 0x9b407c00u | ((rm & 0x1fu) << 16) | ((rn & 0x1fu) << 5) |
          (rd & 0x1fu);
 }
 
 static u32 aa_umulh(u32 rd, u32 rn, u32 rm) {
   return 0x9bc07c00u | ((rm & 0x1fu) << 16) | ((rn & 0x1fu) << 5) |
          (rd & 0x1fu);
 }
 
 static u32 aa_subs_reg(u32 sf, u32 rd, u32 rn, u32 rm) {
   return aa64_addsubsr_pack(
       (AA64AddSubSR){.sf = sf, .op = 1, .S = 1, .Rm = rm, .Rn = rn, .Rd = rd});
 }
 
 static void aa_emit_cmp_to_flags(NativeTarget* t, NativeLoc lhs, NativeLoc rhs);
 
 static u32 aa_add_lsl(u32 rd, u32 rn, u32 rm, u32 shift) {
   return aa64_addsubsr_pack((AA64AddSubSR){.sf = 1,
                                            .op = 0,
                                            .S = 0,
                                            .shift = 0,
                                            .Rm = rm,
                                            .imm6 = shift,
                                            .Rn = rn,
                                            .Rd = rd});
 }
 
 static u32 aa_cset(u32 sf, u32 rd, u32 cond) {
   return aa64_csinc_enc(sf, rd, AA64_ZR, AA64_ZR, cond ^ 1u);
 }
 
 static u32 cmp_cond(CmpOp op) {
   switch (op) {
     case CMP_EQ:
       return 0x0u;
     case CMP_NE:
       return 0x1u;
     case CMP_LT_U:
       return 0x3u;
     case CMP_LE_U:
       return 0x9u;
     case CMP_GT_U:
       return 0x8u;
     case CMP_GE_U:
       return 0x2u;
     case CMP_LT_S:
       return 0xbu;
     case CMP_LE_S:
       return 0xdu;
     case CMP_GT_S:
       return 0xcu;
     case CMP_GE_S:
       return 0xau;
     /* FP predicates after FCMP set NZCV as: a<b -> N; a==b -> Z,C; a>b -> C;
      * unordered -> C,V. Each maps to a single condition except CMP_ONE_F /
      * CMP_UEQ_F (synthesized with two instructions in aa_cmp/aa_cmp_branch,
      * which intercept them before calling cmp_cond). */
     case CMP_OEQ_F:
       return 0x0u; /* EQ */
     case CMP_OLT_F:
       return 0x4u; /* MI */
     case CMP_OLE_F:
       return 0x9u; /* LS */
     case CMP_OGT_F:
       return 0xcu; /* GT */
     case CMP_OGE_F:
       return 0xau; /* GE */
     case CMP_UNE_F:
       return 0x1u; /* NE  (unordered or not-equal) */
     case CMP_ULT_F:
       return 0xbu; /* LT  (unordered or less-than) */
     case CMP_ULE_F:
       return 0xdu; /* LE  (unordered or less-or-equal) */
     case CMP_UGT_F:
       return 0x8u; /* HI  (unordered or greater-than) */
     case CMP_UGE_F:
       return 0x2u; /* CS  (unordered or greater-or-equal) */
     default:
       return 0x0u;
   }
 }
 
 static AANativeSlot* aa_slot(AANativeTarget* a, NativeFrameSlot slot) {
   return native_frame_slot_at(&a->frame, slot);
 }
 
 static void aa_addr_base(AANativeTarget* a, NativeAddr addr, u32* base_out,
                          i32* off_out) {
   *base_out = AA_TMP0;
   *off_out = addr.offset;
   switch ((NativeAddrBaseKind)addr.base_kind) {
     case NATIVE_ADDR_BASE_REG:
       *base_out = addr.base.reg;
       return;
     case NATIVE_ADDR_BASE_FRAME: {
       AANativeSlot* s = aa_slot(a, addr.base.frame);
       *base_out = AA_FP;
       *off_out = aa_fp_off_slot(a, s->off) + addr.offset;
       return;
     }
     case NATIVE_ADDR_BASE_GLOBAL: {
       NativeLoc tmp;
       memset(&tmp, 0, sizeof tmp);
       tmp.kind = NATIVE_LOC_REG;
       tmp.cls = NATIVE_REG_INT;
       tmp.type = builtin_id(KIT_CG_BUILTIN_I64);
       tmp.v.reg = AA_TMP0;
       a->base.load_addr(&a->base, tmp, addr);
       *base_out = AA_TMP0;
       *off_out = 0;
       return;
     }
     default:
       aa_panic(a, "unsupported address base");
   }
 }
 
 static u32 aa_ldst_q_uimm(int load, u32 rt, u32 rn, u32 byte_off);
 static u32 aa_ldst_q_simm9(int load, u32 rt, u32 rn, i32 byte_off);
 
 static void aa_emit_mem_q(AANativeTarget* a, int load, NativeLoc reg,
                           NativeAddr addr) {
   u32 base, rt;
   i32 off;
   MCEmitter* mc = a->base.mc;
   if (addr.index_kind != NATIVE_ADDR_INDEX_NONE)
     aa_panic(a, "unsupported q-register indexed memory access");
   aa_addr_base(a, addr, &base, &off);
   rt = loc_reg(reg);
   if (off >= 0 && (((u32)off & 15u) == 0) && ((u32)off >> 4) <= 0xfffu) {
     aa_emit32(mc, aa_ldst_q_uimm(load, rt, base, (u32)off));
     return;
   }
   if (off >= -256 && off <= 255) {
     aa_emit32(mc, aa_ldst_q_simm9(load, rt, base, off));
     return;
   }
   aa_emit_add_imm(a, AA_TMP1, base, off);
   aa_emit32(mc, aa_ldst_q_uimm(load, rt, AA_TMP1, 0));
 }
 
 static void aa_emit_mem(AANativeTarget* a, int load, NativeLoc reg,
                         NativeAddr addr, MemAccess mem) {
   u32 base, rt, sz;
   i32 off;
   MCEmitter* mc = a->base.mc;
   rt = loc_reg(reg);
   sz = size_idx(mem.size
                     ? mem.size
                     : type_size32(&a->base, reg.type ? reg.type : mem.type));
   if (native_loc_is_fp(reg) &&
       (mem.size
            ? mem.size
            : type_size32(&a->base, reg.type ? reg.type : mem.type)) == 16u) {
     aa_emit_mem_q(a, load, reg, addr);
     return;
   }
   if (native_loc_is_fp(reg) && sz < 2u) sz = 2u;
   if (addr.base_kind == NATIVE_ADDR_BASE_GLOBAL &&
       addr.index_kind == NATIVE_ADDR_INDEX_NONE) {
     i64 addend = addr.base.global.addend + (i64)addr.offset;
     u32 scratch = (!load && rt == AA_TMP0) ? AA_TMP1 : AA_TMP0;
     u32 pos = mc->pos(mc);
     if (aa_use_got_for_sym(&a->base, addr.base.global.sym)) {
       aa_emit32(mc, aa64_adrp(scratch, 0, 0));
       mc->emit_reloc_at(mc, mc->section_id, pos, R_AARCH64_ADR_GOT_PAGE,
                         addr.base.global.sym, 0, 0, 0);
       pos = mc->pos(mc);
       aa_emit32(mc, aa_ldr_uimm(3, scratch, scratch, 0));
       mc->emit_reloc_at(mc, mc->section_id, pos, R_AARCH64_LD64_GOT_LO12_NC,
                         addr.base.global.sym, 0, 0, 0);
       if (addend) aa_emit_add_i64(a, scratch, scratch, addend);
       aa_emit32(mc, load
                         ? aa_ldur_v(sz, native_loc_is_fp(reg), rt, scratch, 0)
                         : aa_stur_v(sz, native_loc_is_fp(reg), rt, scratch, 0));
       return;
     }
     aa_emit32(mc, aa64_adrp(scratch, 0, 0));
     mc->emit_reloc_at(mc, mc->section_id, pos, R_AARCH64_ADR_PREL_PG_HI21,
                       addr.base.global.sym, addend, 0, 0);
     pos = mc->pos(mc);
     aa_emit32(mc,
               load ? aa_ldr_uimm_v(sz, native_loc_is_fp(reg), rt, scratch, 0)
                    : aa_str_uimm_v(sz, native_loc_is_fp(reg), rt, scratch, 0));
     mc->emit_reloc_at(mc, mc->section_id, pos, aa_ldst_reloc_for_size(sz),
                       addr.base.global.sym, addend, 0, 0);
     return;
   }
   aa_addr_base(a, addr, &base, &off);
   if (addr.index_kind != NATIVE_ADDR_INDEX_NONE) {
     u32 use_base = base;
     u32 scaled = 0;
     if (addr.index_kind != NATIVE_ADDR_INDEX_REG)
       aa_panic(a, "unsupported address index");
     if (off) {
       use_base = AA_TMP1;
       aa_emit_add_imm(a, use_base, base, off);
     }
     if (addr.log2_scale == 0) {
       scaled = 0;
     } else if (addr.log2_scale == sz) {
       scaled = 1;
     } else {
       aa_panic(a, "unsupported memory address scale");
     }
     aa_emit32(mc, aa_ldst_regoff_v(sz, native_loc_is_fp(reg), load, rt,
                                    use_base, addr.index.reg, scaled));
     return;
   }
   if (off >= 0 && (((u32)off & ((1u << sz) - 1u)) == 0) &&
       ((u32)off >> sz) <= 0xfffu) {
     aa_emit32(
         mc, load
                 ? aa_ldr_uimm_v(sz, native_loc_is_fp(reg), rt, base, (u32)off)
                 : aa_str_uimm_v(sz, native_loc_is_fp(reg), rt, base, (u32)off));
     return;
   }
   if (off >= -256 && off <= 255) {
     aa_emit32(mc, load ? aa_ldur_v(sz, native_loc_is_fp(reg), rt, base, off)
                        : aa_stur_v(sz, native_loc_is_fp(reg), rt, base, off));
     return;
   }
   aa_emit_add_imm(a, AA_TMP1, base, off);
   aa_emit32(mc, load ? aa_ldur_v(sz, native_loc_is_fp(reg), rt, AA_TMP1, 0)
                      : aa_stur_v(sz, native_loc_is_fp(reg), rt, AA_TMP1, 0));
 }
 
 static NativeAllocClass aa_class_for_type(NativeTarget* t, KitCgTypeId type) {
   if (type && cg_type_is_float(t->c, type) && cg_type_size(t->c, type) <= 8u)
     return NATIVE_REG_FP;
   return NATIVE_REG_INT;
 }
 
 static int aa_addr_legal(NativeTarget* t, const NativeAddr* addr,
                          MemAccess mem) {
   u32 sz;
   (void)t;
   if (!addr) return 0;
   if (addr->index_kind == NATIVE_ADDR_INDEX_NONE) return 1;
   if (addr->index_kind != NATIVE_ADDR_INDEX_REG) return 0;
   if (addr->log2_scale == 0) return 1;
   sz = size_idx(mem.size ? mem.size : 8u);
   return addr->log2_scale == sz;
 }
 
 /* True if `mul Rd, Rn, #c` can be replaced by a single non-mul aarch64
  * instruction using only Rn as a source (no extra scratch reg). Constants
  * that match: 0, 1, -1, +/-2^k, 2^k+1, 1-2^k for k in [1..width-1]. The
  * shift exponent must fit imm6 for the operand width (width = 32 if !sf
  * else 64). The emit side is aa_emit_mul_const_imm. */
 static int aa64_imul_strength_reducible(u32 sf, i64 imm) {
   u32 max_sh = sf ? 63u : 31u;
   u64 a;
   if (imm == 0 || imm == 1 || imm == -1) return 1;
   /* +2^k */
   a = (u64)imm;
   if (imm > 0 && (a & (a - 1u)) == 0u) {
     u32 k = (u32)__builtin_ctzll(a);
     return k <= max_sh;
   }
   /* -2^k */
   if (imm < 0) {
     a = (u64)(-imm);
     if (a && (a & (a - 1u)) == 0u) {
       u32 k = (u32)__builtin_ctzll(a);
       return k >= 1u && k <= max_sh;
     }
   }
   /* 2^k + 1 (k >= 1, so c >= 3) */
   if (imm >= 3) {
     u64 m = (u64)(imm - 1);
     if ((m & (m - 1u)) == 0u) {
       u32 k = (u32)__builtin_ctzll(m);
       return k >= 1u && k <= max_sh;
     }
   }
   /* 1 - 2^k (k >= 1, so c <= -1) */
   if (imm <= -1) {
     u64 m = (u64)(1 - imm);
     if (m && (m & (m - 1u)) == 0u) {
       u32 k = (u32)__builtin_ctzll(m);
       return k >= 1u && k <= max_sh;
     }
   }
   return 0;
 }
 
 /* Which constant operands the backend can fold directly into an instruction
  * (so the optimizer can leave them as immediates instead of materializing a
  * register). Currently: add/sub/cmp 12-bit immediates (optionally <<12),
  * any value for a plain register move (movz/movk synthesizes it), and
  * strength-reducible mul constants (handled in aa_binop via shift / shifted
  * add or sub). */
 static int aa_imm_legal(NativeTarget* t, NativeImmUse use, u32 op,
                         KitCgTypeId type, i64 imm) {
   u32 imm12, sh;
   switch (use) {
     case NATIVE_IMM_BINOP:
       if ((BinOp)op == BO_IADD || (BinOp)op == BO_ISUB)
         return aa64_addsub_imm_fits(imm < 0 ? -imm : imm, &imm12, &sh);
       if ((BinOp)op == BO_IMUL) {
         u32 sf = type_size32(t, type) == 8u ? 1u : 0u;
         return aa64_imul_strength_reducible(sf, imm);
       }
       /* LSL/LSR/ASR #imm via the UBFM/SBFM aliases: shift count in range. */
       if ((BinOp)op == BO_SHL || (BinOp)op == BO_SHR_S ||
           (BinOp)op == BO_SHR_U) {
         u32 bits = type_size32(t, type) == 8u ? 64u : 32u;
         return imm >= 0 && (u64)imm < (u64)bits;
       }
       /* AND/ORR/EOR #bitmask: encodable as an AArch64 logical immediate. */
       if ((BinOp)op == BO_AND || (BinOp)op == BO_OR || (BinOp)op == BO_XOR) {
         u32 sf = type_size32(t, type) == 8u ? 1u : 0u;
         u32 N, immr, imms;
         return aa64_logimm_encode((u64)imm, sf, &N, &immr, &imms);
       }
       return 0;
     case NATIVE_IMM_CMP:
       /* cmp lowers to subs #imm12; cmn (negative) is not wired, so require a
        * non-negative immediate. */
       return imm >= 0 && aa64_addsub_imm_fits(imm, &imm12, &sh);
     case NATIVE_IMM_ADDR_OFFSET:
       return aa64_addsub_imm_fits(imm < 0 ? -imm : imm, &imm12, &sh);
     case NATIVE_IMM_MOVE:
       return 1;
   }
   return 0;
 }
 
 static void aa_apply_index(AANativeTarget* a, u32 rd, const NativeAddr* addr) {
   if (addr->index_kind == NATIVE_ADDR_INDEX_NONE) return;
   if (addr->index_kind != NATIVE_ADDR_INDEX_REG)
     aa_panic(a, "unsupported address index");
   if (addr->log2_scale > 4u) aa_panic(a, "unsupported address scale");
   aa_emit32(a->base.mc, aa_add_lsl(rd, rd, addr->index.reg, addr->log2_scale));
 }
 
 static void aa_materialize_frame_index(AANativeTarget* a, NativeAddr* addr,
                                        u32 avoid_reg) {
   NativeAddr load;
   NativeLoc idx;
   MemAccess mem;
   u32 reg;
   if (addr->index_kind != NATIVE_ADDR_INDEX_FRAME_VALUE) return;
   reg = avoid_reg == AA_TMP1 ? AA_TMP0 : AA_TMP1;
   memset(&load, 0, sizeof load);
   load.base_kind = NATIVE_ADDR_BASE_FRAME;
   load.base.frame = addr->index.frame;
   load.base_type =
       addr->index_type ? addr->index_type : builtin_id(KIT_CG_BUILTIN_I64);
   memset(&idx, 0, sizeof idx);
   idx.kind = NATIVE_LOC_REG;
   idx.cls = NATIVE_REG_INT;
   idx.type = load.base_type;
   idx.v.reg = reg;
   memset(&mem, 0, sizeof mem);
   mem.type = load.base_type;
   mem.size = 8;
   mem.align = 8;
   aa_emit_mem(a, 1, idx, load, mem);
   addr->index_kind = NATIVE_ADDR_INDEX_REG;
   addr->index.reg = reg;
 }
 
 static NativeLoc native_loc_reg(KitCgTypeId type, NativeAllocClass cls,
                                 Reg reg);
 
 static u32 aa_ldst_q_uimm(int load, u32 rt, u32 rn, u32 byte_off) {
   return aa64_ldst_uimm_pack((AA64LdStUimm){.size = 0,
                                             .V = 1,
                                             .opc = load ? 3u : 2u,
                                             .imm12 = byte_off >> 4,
                                             .Rn = rn,
                                             .Rt = rt});
 }
 
 static u32 aa_ldst_q_simm9(int load, u32 rt, u32 rn, i32 byte_off) {
   return aa64_ldst_simm9_pack((AA64LdStSimm9){.size = 0,
                                               .V = 1,
                                               .opc = load ? 3u : 2u,
                                               .imm9 = (u32)byte_off & 0x1ffu,
                                               .Rn = rn,
                                               .Rt = rt});
 }
 
 static void aa_emit_q_frame(AANativeTarget* a, int load, u32 qreg,
                             NativeFrameSlot slot, u32 offset) {
   AANativeSlot* s = aa_slot(a, slot);
   i32 off = aa_fp_off_slot(a, s->off) + (i32)offset;
   MCEmitter* mc = a->base.mc;
   if (off >= 0 && ((u32)off & 15u) == 0 && ((u32)off >> 4) <= 0xfffu) {
     aa_emit32(mc, aa_ldst_q_uimm(load, qreg, AA_FP, (u32)off));
     return;
   }
   if (off >= -256 && off <= 255) {
     aa_emit32(mc, aa_ldst_q_simm9(load, qreg, AA_FP, off));
     return;
   }
   aa_emit_add_imm(a, AA_TMP1, AA_FP, off);
   aa_emit32(mc, aa_ldst_q_uimm(load, qreg, AA_TMP1, 0));
 }
 
 /* Reserve the variadic register-save-area frame slots (gp then fp). Split from
  * the store emission so the known-frame path can fix the full frame — including
  * these slots — before the prologue, then emit the stores after it. */
 static void aa_reserve_variadic_reg_saves(AANativeTarget* a) {
   NativeFrameSlotDesc sd;
   KitCgTypeId i64 = builtin_id(KIT_CG_BUILTIN_I64);
   ABIVaListInfo vai = abi_va_list_layout(a->base.c->abi);
   if (vai.kind != ABI_VA_LIST_AAPCS64) return;
   memset(&sd, 0, sizeof sd);
   sd.type = i64;
   sd.size = vai.gp_reg_count * vai.gp_slot_size;
   sd.align = 8;
   sd.kind = NATIVE_FRAME_SLOT_SAVE;
   a->va_gr_slot = a->base.frame_slot(&a->base, &sd);
   sd.size = vai.fp_reg_count * vai.fp_slot_size;
   sd.align = 16;
   a->va_vr_slot = a->base.frame_slot(&a->base, &sd);
 }
 
 /* Emit the stores into the variadic register-save area. For AAPCS64 these land
  * in the reserved gr/vr frame slots (aa_reserve_variadic_reg_saves); for the
  * Windows GP home area they land in [fp + AA_FRAME_SAVE_SIZE ..], the
  * top-of-frame block contiguous with the incoming stack args. */
 static void aa_emit_variadic_reg_save_stores(AANativeTarget* a) {
   NativeAddr addr;
   MemAccess mem;
   KitCgTypeId i64 = builtin_id(KIT_CG_BUILTIN_I64);
   ABIVaListInfo vai = abi_va_list_layout(a->base.c->abi);
   if (vai.kind == ABI_VA_LIST_POINTER && a->top_home_bytes) {
     /* Windows: home x0..x{gp_reg_count-1} so the plain-pointer va_list walks
      * register then stack varargs as one block. The named leading registers are
      * homed too (harmless): va_start skips past them. */
     memset(&mem, 0, sizeof mem);
     mem.type = i64;
     mem.size = 8;
     mem.align = 8;
     memset(&addr, 0, sizeof addr);
     addr.base_kind = NATIVE_ADDR_BASE_REG;
     addr.base.reg = AA_FP;
     addr.base_type = i64;
     for (u32 r = 0; r < vai.gp_reg_count && r < 8u; ++r) {
       NativeLoc src = native_loc_reg(i64, NATIVE_REG_INT, r);
       addr.offset = aa_fp_off_home_slot(r);
       aa_emit_mem(a, 0, src, addr, mem);
     }
     return;
   }
   if (vai.kind != ABI_VA_LIST_AAPCS64) return;
   memset(&mem, 0, sizeof mem);
   mem.type = i64;
   mem.size = 8;
   mem.align = 8;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = a->va_gr_slot;
   addr.base_type = i64;
   for (u32 r = 0; r < vai.gp_reg_count && r < 8u; ++r) {
     NativeLoc src = native_loc_reg(i64, NATIVE_REG_INT, r);
     addr.offset = (i32)(r * vai.gp_slot_size);
     aa_emit_mem(a, 0, src, addr, mem);
   }
   for (u32 r = 0; r < vai.fp_reg_count && r < 8u; ++r)
     aa_emit_q_frame(a, 0, r, a->va_vr_slot, r * vai.fp_slot_size);
 }
 
 static void aa_emit_entry_saves(AANativeTarget* a);
 
 /* Per-function state reset + function-symbol / cfi / prologue-anchor setup
  * shared by both entry points (aa_func_begin for the single-pass path,
  * aa_func_begin_known_frame for the optimizer path). Emits no prologue. */
 static void aa_func_begin_common(NativeTarget* t, const CGFuncDesc* fd) {
   AANativeTarget* a = aa_of(t);
   MCEmitter* mc = t->mc;
   a->func = fd;
   /* Shared frame bookkeeping: clears the slot table, cum_off, max_outgoing,
    * callee-save set, and known_frame/has_alloca/frame_final. cum_off counts
    * frame-slot bytes below fp; the saved fp/lr pair (16 bytes at [fp, fp+8]) is
    * *not* part of it — aa_build_layout adds it in aa_func_end. */
   native_frame_reset(&a->frame);
   a->incoming_stack_size = 0;
   a->next_param_int = 0;
   a->next_param_fp = 0;
   /* 0-based byte cursor for incoming stack args (also reported as the
    * caller's incoming_stack_size for tail-call realizability). bind_param
    * forms its fp-relative address via aa_fp_off_in_arg(next_param_stack),
    * which adds the saved-pair offset. */
   a->next_param_stack = 0;
   a->sret_ptr_slot = NATIVE_FRAME_SLOT_NONE;
   a->saved_tmp_slot = NATIVE_FRAME_SLOT_NONE;
   a->va_gr_slot = NATIVE_FRAME_SLOT_NONE;
   a->va_vr_slot = NATIVE_FRAME_SLOT_NONE;
   a->npatches = 0;
   a->nalloca = 0;
   a->slim_prologue = 0;
   a->slim_small_frame = 0;
   a->fp_at_bottom = 0;
   a->frame_size_final = 0;
   /* Windows variadic functions reserve a GP register home area at the top of
    * the frame (just below the incoming stack args). The plain-pointer va_list
    * then walks register-passed then stack-passed varargs as one block. Other
    * ABIs leave gp_reg_count 0 here: Apple ARM64 routes all varargs to the
    * stack, AAPCS64 uses a struct va_list with separate reg-save pointers. */
   {
     const ABIFuncInfo* fi = abi_cg_func_info(t->c->abi, fd->fn_type);
     ABIVaListInfo vai = abi_va_list_layout(t->c->abi);
     a->top_home_bytes = (fi && fi->variadic && vai.kind == ABI_VA_LIST_POINTER)
                             ? vai.gp_reg_count * vai.gp_slot_size
                             : 0u;
   }
   mc->set_section(mc, fd->text_section_id);
   mc->emit_align(mc, 4, 0);
   a->func_start = mc->pos(mc);
   mc_begin_function(mc, fd->sym, fd->text_section_id, a->func_start);
   if (mc->cfi_startproc) mc->cfi_startproc(mc);
   a->prologue_pos = mc->pos(mc);
   a->minimal_prologue_words = 0;
   a->epilogue_label = mc->label_new(mc);
 }
 
 /* Single-pass (NativeDirectTarget) entry point: the frame is not known up
  * front, so reserve a worst-case prologue region (patched in aa_func_end once
  * max_outgoing / callee-saves are final) and emit the entry saves now. */
 static void aa_func_begin(NativeTarget* t, const CGFuncDesc* fd) {
   AANativeTarget* a = aa_of(t);
   MCEmitter* mc = t->mc;
   aa_func_begin_common(t, fd);
   for (u32 i = 0; i < AA_PROLOGUE_WORDS; ++i) aa_emit32(mc, 0xd503201fu);
   aa_emit_entry_saves(a);
 }
 
 /* Reserve the entry-save frame slots: the sret-pointer home (x8) and, for
  * variadic functions, the argument register-save area. Reserving is split from
  * emitting so the known-frame path can fix the full frame before the prologue;
  * the single-pass path runs both back to back via aa_emit_entry_saves. */
 static void aa_reserve_entry_saves(AANativeTarget* a) {
   NativeTarget* t = &a->base;
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, a->func->fn_type);
   if (abi && abi->has_sret) {
     NativeFrameSlotDesc sd;
     memset(&sd, 0, sizeof sd);
     sd.type = builtin_id(KIT_CG_BUILTIN_I64);
     sd.size = 8;
     sd.align = 8;
     sd.kind = NATIVE_FRAME_SLOT_SAVE;
     a->sret_ptr_slot = t->frame_slot(t, &sd);
   }
   if (abi && abi->variadic) aa_reserve_variadic_reg_saves(a);
 }
 
 /* Emit the entry-save stores (x8 → sret slot, then the variadic reg-save area).
  * Slots must already be reserved (aa_reserve_entry_saves). */
 static void aa_emit_entry_save_stores(AANativeTarget* a) {
   NativeTarget* t = &a->base;
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, a->func->fn_type);
   if (abi && abi->has_sret) {
     NativeAddr addr;
     NativeLoc src;
     MemAccess mem;
     KitCgTypeId i64 = builtin_id(KIT_CG_BUILTIN_I64);
     memset(&addr, 0, sizeof addr);
     addr.base_kind = NATIVE_ADDR_BASE_FRAME;
     addr.base.frame = a->sret_ptr_slot;
     addr.base_type = i64;
     memset(&src, 0, sizeof src);
     src.kind = NATIVE_LOC_REG;
     src.cls = NATIVE_REG_INT;
     src.type = i64;
     src.v.reg = 8u;
     memset(&mem, 0, sizeof mem);
     mem.type = i64;
     mem.size = 8;
     mem.align = 8;
     aa_emit_mem(a, 0, src, addr, mem);
   }
   if (abi && abi->variadic) aa_emit_variadic_reg_save_stores(a);
 }
 
 /* Reserve + emit the entry saves back to back. Single-pass (NativeDirectTarget)
  * path, where the prologue region is a reserved worst-case block and slot
  * offsets need not be final before it. */
 static void aa_emit_entry_saves(AANativeTarget* a) {
   aa_reserve_entry_saves(a);
   aa_emit_entry_save_stores(a);
 }
 
 static void aa_note_frame_state(NativeTarget* t,
                                 const NativeFramePatchState* state) {
   AANativeTarget* a = aa_of(t);
   if (state && state->max_outgoing > a->frame.max_outgoing)
     a->frame.max_outgoing = state->max_outgoing;
 }
 
 /* Reserve a save slot for each callee-saved register the allocator used. Runs
  * before frame-slot mapping so these slots get the lowest offsets, keeping the
  * prologue stores within stur's signed-9-bit range. The prologue/epilogue
  * save/restore is emitted from this list in aa_patch_prologue / aa_func_end. */
 static void aa_reserve_callee_saves(NativeTarget* t, const u32* used,
                                     u32 nclasses) {
   AANativeTarget* a = aa_of(t);
   /* aa64 homes each callee-save in its own 8-byte frame slot (reserved before
    * the body slots so they sit nearest fp, in stur range), so alloc_slots=1.
    * Adjacent integer slots are later paired into stp/ldp. */
   NativeFrameSaveSpec spec[NATIVE_REG_VEC + 1];
   memset(spec, 0, sizeof spec);
   spec[NATIVE_REG_INT].size = 8;
   spec[NATIVE_REG_INT].align = 8;
   spec[NATIVE_REG_INT].type = builtin_id(KIT_CG_BUILTIN_I64);
   spec[NATIVE_REG_FP].size = 8;
   spec[NATIVE_REG_FP].align = 8;
   spec[NATIVE_REG_FP].type = builtin_id(KIT_CG_BUILTIN_F64);
   native_frame_set_callee_saves(&a->frame, used, nclasses, spec,
                                 NATIVE_REG_VEC + 1, 1);
 }
 
 static MemAccess aa_mem_for_type(NativeTarget* t, KitCgTypeId type, u32 size);
 static void aa_words_callee_saves(AANativeTarget* a, int save, u32* words,
                                   u32 cap, u32* n);
 
 static void aa_emit_callee_restores(AANativeTarget* a) {
   u32 words[AA_PROLOGUE_WORDS];
   u32 n = 0;
   aa_words_callee_saves(a, 0, words, AA_PROLOGUE_WORDS, &n);
   for (u32 i = 0; i < n; ++i) aa_emit32(a->base.mc, words[i]);
 }
 
 static void aa_words_load_imm(AANativeTarget* a, u32* words, u32 cap, u32* n,
                               u32 rd, i64 imm) {
   u32 tmp[4];
   u32 m = aa_load_imm_words(tmp, 4u, 1, rd, imm);
   if (!m || *n + m > cap) aa_panic(a, "instruction patch too small");
   for (u32 i = 0; i < m; ++i) words[(*n)++] = tmp[i];
 }
 
 /* Windows large-frame stack probe. kit's prologue reserves the whole frame in
  * one `sub sp, sp, #N`, but Windows grows a thread stack one guard page at a
  * time: a sub that jumps SP more than a page past the guard page leaves the
  * skipped pages uncommitted, and the first store into them faults (and, since
  * SP itself is then in uncommitted memory, the fault can't even be delivered).
  * Touch every page the frame spans, top-down, so each guard page commits in
  * turn before the sub. Inlined (no external __chkstk symbol / no reloc in the
  * patched prologue region); mirrors the linker's aa64_coff_chkstk body. Only
  * x16/x17 are clobbered — the following sub-sp / saved-pair material re-derives
  * both. Emitted only when frame_size > interval (one page). */
 static void aa_words_stack_probe(AANativeTarget* a, u32* words, u32 cap, u32* n,
                                  u32 frame_size, u32 interval) {
   u32 imm12, sh;
   if (!aa64_addsub_imm_fits(interval, &imm12, &sh))
     aa_panic(a, "stack-probe interval not an addsub immediate");
   /* x16 = frame_size ; x17 = sp */
   aa_words_load_imm(a, words, cap, n, AA_TMP0, frame_size);
   if (*n + 5u > cap) aa_panic(a, "instruction patch too small");
   words[(*n)++] = aa64_add_imm(1, AA_TMP1, AA_SP, 0, 0); /* mov x17, sp */
   /* loop: x17 -= page ; x16 -= page (sets flags) ; touch [x17] ; b.gt loop */
   words[(*n)++] = aa64_sub_imm(1, AA_TMP1, AA_TMP1, imm12, sh);
   words[(*n)++] = aa64_subs_imm12(1, AA_TMP0, AA_TMP0, imm12, sh);
   words[(*n)++] = aa64_ldr64_uimm12(31, AA_TMP1, 0); /* ldr xzr, [x17] */
   /* branch back to the `sub x17` three words above while x16 stays positive */
   words[(*n)++] =
       aa64_brcond_pack((AA64BrCond){.imm19 = (u32)(-3), .cond = 0xcu /* GT */});
 }
 
 static void aa_words_sub_sp_frame(AANativeTarget* a, u32* words, u32 cap,
                                   u32* n, u32 frame_size) {
   u32 imm12, sh;
   if (aa64_addsub_imm_fits(frame_size, &imm12, &sh)) {
     if (*n >= cap) aa_panic(a, "instruction patch too small");
     words[(*n)++] = aa64_sub_imm(1, AA_SP, AA_SP, imm12, sh);
     return;
   }
   aa_words_load_imm(a, words, cap, n, AA_TMP0, frame_size);
   if (*n + 3u > cap) aa_panic(a, "instruction patch too small");
   words[(*n)++] = aa64_add_imm(1, AA_TMP1, AA_SP, 0, 0);
   words[(*n)++] = aa64_sub(1, AA_TMP1, AA_TMP1, AA_TMP0);
   words[(*n)++] = aa64_add_imm(1, AA_SP, AA_TMP1, 0, 0);
 }
 
 /* Anchor fp at the AAPCS64 saved-pair address (= sp + saved-pair offset).
  * The slim_prologue path achieves the same anchor in a single insn via
  * `add x29, sp, #0` after the pre-decrement stp moves sp to the saved-pair. */
 static void aa_words_frame_ptr_from_sp(AANativeTarget* a, u32* words, u32 cap,
                                        u32* n, const AAFrameLayout* L) {
   u32 imm12, sh;
   u32 anchor = aa_sp_off_saved_pair(L);
   if (aa64_addsub_imm_fits(anchor, &imm12, &sh)) {
     if (*n >= cap) aa_panic(a, "instruction patch too small");
     words[(*n)++] = aa64_add_imm(1, AA_FP, AA_SP, imm12, sh);
     return;
   }
   aa_words_load_imm(a, words, cap, n, AA_TMP0, anchor);
   if (*n + 2u > cap) aa_panic(a, "instruction patch too small");
   words[(*n)++] = aa64_add_imm(1, AA_TMP1, AA_SP, 0, 0);
   words[(*n)++] = aa64_add(1, AA_FP, AA_TMP1, AA_TMP0);
 }
 
 /* x17 = address of the saved-pair slot (= sp + saved-pair offset). Used by
  * the fat prologue to materialize the stp destination when the offset
  * doesn't fit stp's signed-7-bit-scaled immediate. */
 static void aa_words_saved_pair_addr(AANativeTarget* a, u32* words, u32 cap,
                                      u32* n, const AAFrameLayout* L) {
   u32 save_off = aa_sp_off_saved_pair(L);
   u32 imm12, sh;
   if (aa64_addsub_imm_fits(save_off, &imm12, &sh)) {
     if (*n >= cap) aa_panic(a, "instruction patch too small");
     words[(*n)++] = aa64_add_imm(1, AA_TMP1, AA_SP, imm12, sh);
     return;
   }
   aa_words_load_imm(a, words, cap, n, AA_TMP0, save_off);
   if (*n + 2u > cap) aa_panic(a, "instruction patch too small");
   words[(*n)++] = aa64_add_imm(1, AA_TMP1, AA_SP, 0, 0);
   words[(*n)++] = aa64_add(1, AA_TMP1, AA_TMP1, AA_TMP0);
 }
 
 static void aa_words_restore_frame(AANativeTarget* a, u32* words, u32 cap,
                                    u32* n, const AAFrameLayout* L) {
   if (!L->frame_size) return;
   if (a->slim_prologue) {
     if (*n + 1u > cap) aa_panic(a, "instruction patch too small");
     /* `ldp x29, x30, [sp], #16` — pop saved pair, restore sp. */
     words[(*n)++] = aa64_ldp64_post(AA_FP, AA_LR, AA_SP, 2);
     return;
   }
   if (a->fp_at_bottom) {
     /* Bottom-record fold: `ldp x29,x30,[sp],#N` reloads the pair from the
      * bottom AND releases the whole frame in one insn. Callee-saves were
      * already restored by aa_emit_callee_restores. -1 insn vs slim_small_frame
      * (which needs a separate `add sp`). N <= 504 holds the post-index imm. */
     if (*n + 1u > cap) aa_panic(a, "instruction patch too small");
     words[(*n)++] =
         aa64_ldp64_post(AA_FP, AA_LR, AA_SP, (i32)(L->frame_size / 8u));
     return;
   }
   if (a->slim_small_frame) {
     /* `ldp x29,x30,[sp,#saved_pair] ; add sp,sp,#frame_size` — load through
      * sp avoids the fat path's `add x10, fp, #0` scratch, and the subsequent
      * `add sp` unwinds without depending on the (now-clobbered) old fp. */
     u32 save_off = aa_sp_off_saved_pair(L);
     u32 imm12, sh;
     if (*n + 2u > cap) aa_panic(a, "instruction patch too small");
     words[(*n)++] = aa64_ldp64_soff(AA_FP, AA_LR, AA_SP, (i32)(save_off / 8u));
     if (!aa64_addsub_imm_fits(L->frame_size, &imm12, &sh))
       aa_panic(a, "slim_small_frame: frame_size out of addsub imm range");
     words[(*n)++] = aa64_add_imm(1, AA_SP, AA_SP, imm12, sh);
     return;
   }
   if (*n + 3u > cap) aa_panic(a, "instruction patch too small");
   /* AAPCS64: fp is the saved-pair address. Reload pair from [fp], then restore
    * sp to fp + CFA-offset (= caller's original sp = CFA). The CFA offset is
    * AA_FRAME_SAVE_SIZE normally, plus the Windows-variadic GP home area when
    * present. */
   words[(*n)++] = aa64_add_imm(1, AA_TMP0, AA_FP, 0, 0);
   words[(*n)++] = aa64_ldp64_soff(AA_FP, AA_LR, AA_TMP0, 0);
   words[(*n)++] = aa64_add_imm(1, AA_SP, AA_TMP0, (u32)aa_cfa_off(a), 0);
 }
 
 /* Emit callee-save store (save=1) or restore (save=0) words into `words`,
  * pairing adjacent integer registers into a single stp/ldp.
  * reserve_callee_saves allocates consecutive 8-byte slots in order, so
  * callee_saves[i] sits 8 bytes above callee_saves[i+1]; for an int pair the
  * lower-addressed reg[i+1] is the stp's Rt and reg[i] is Rt2. FP registers (and
  * an unpaired trailing int) use the single-register stur/ldur form. */
 static void aa_words_callee_saves(AANativeTarget* a, int save, u32* words,
                                   u32 cap, u32* n) {
   for (u32 i = 0; i < a->frame.ncallee_saves;) {
     const AACalleeSave* cs = &a->frame.callee_saves[i];
     i32 off = aa_fp_off_slot(a, aa_slot(a, cs->slot)->off);
     if (i + 1u < a->frame.ncallee_saves && cs->cls == (u8)NATIVE_REG_INT &&
         a->frame.callee_saves[i + 1u].cls == (u8)NATIVE_REG_INT) {
       const AACalleeSave* cs2 = &a->frame.callee_saves[i + 1u];
       i32 off2 = aa_fp_off_slot(a, aa_slot(a, cs2->slot)->off);
       /* cs2 is reserved after cs (larger slot.off), so it is the lower address
        * in both layouts (off2 = off - 8): stp's Rt = cs2, Rt2 = cs, base off2.
        * stp/ldp's signed-7-bit scaled immediate reaches ±504. */
       if (off2 < -512 || off2 > 504)
         aa_panic(a, "callee-save pair offset out of prologue range");
       if (*n >= cap) aa_panic(a, "prologue too large");
       words[(*n)++] = save
                           ? aa64_stp64_soff(cs2->reg, cs->reg, AA_FP, off2 / 8)
                           : aa64_ldp64_soff(cs2->reg, cs->reg, AA_FP, off2 / 8);
       i += 2u;
     } else {
       u32 v = cs->cls == (u8)NATIVE_REG_FP ? 1u : 0u;
       if (*n >= cap) aa_panic(a, "prologue too large");
       if (a->fp_at_bottom) {
         /* Positive, 8-aligned offset above the record (up to frame_size-8 ≤
          * 496): the unscaled stur (±256) can't reach it, so use the scaled
          * unsigned-imm str/ldr. */
         if (off < 0 || (u32)off > 0x7ff8u)
           aa_panic(a, "callee-save offset out of prologue range");
         words[(*n)++] = save ? aa_str_uimm_v(3, v, cs->reg, AA_FP, (u32)off)
                              : aa_ldr_uimm_v(3, v, cs->reg, AA_FP, (u32)off);
       } else {
         if (off < -256 || off > 255)
           aa_panic(a, "callee-save offset out of prologue range");
         words[(*n)++] = save ? aa_stur_v(3, v, cs->reg, AA_FP, off)
                              : aa_ldur_v(3, v, cs->reg, AA_FP, off);
       }
       i += 1u;
     }
   }
 }
 
 /* Build the prologue instruction words for `L` into `words` (capacity `cap`),
  * returning the count. Shared by the NativeDirectTarget patch path (reserves
  * a fixed worst-case region, then patches it here) and the optimizer path
  * (aa_func_begin_known_frame emits exactly these words up front).
  *
  * All variants establish a post-prologue state defined by L: saved x29/x30 at
  * [fp]/[fp+8], callee-saves at aa_fp_off_slot of each. The top-record variants
  * leave fp = sp + aa_sp_off_saved_pair(L) (saved-pair near the top); the
  * bottom-record variant leaves fp = sp (saved-pair at the bottom). */
 static u32 aa_build_prologue_words(AANativeTarget* a, const AAFrameLayout* L,
                                    u32* words, u32 cap) {
   u32 n = 0;
   if (!L->frame_size) return 0;
   if (a->slim_prologue) {
     if (cap < 2u) aa_panic(a, "prologue too large");
     /* `stp x29, x30, [sp, #-16]!; add x29, sp, #0` — the pre-decrement stp
      * moves sp down to the saved-pair address, so a no-op add anchors fp
      * there directly. AAPCS64 frame record. */
     words[n++] = aa64_stp64_pre(AA_FP, AA_LR, AA_SP, -2);
     words[n++] = aa64_add_imm(1, AA_FP, AA_SP, 0, 0);
     return n;
   }
   if (a->fp_at_bottom) {
     /* Bottom-record fold: `stp x29,x30,[sp,#-N]!` decrements sp by the whole
      * frame AND saves the pair at the new bottom in one insn; `mov x29,sp`
      * (add #0) anchors fp there. Callee-saves then stack above the record at
      * positive offsets. -2 insns/call vs the top-record slim_small_frame. */
     if (n + 2u > cap) aa_panic(a, "prologue too large");
     words[n++] =
         aa64_stp64_pre(AA_FP, AA_LR, AA_SP, -(i32)(L->frame_size / 8u));
     words[n++] = aa64_add_imm(1, AA_FP, AA_SP, 0, 0);
     aa_words_callee_saves(a, 1, words, cap, &n);
     return n;
   }
   /* On targets that don't auto-grow the stack (Windows), probe each page the
    * frame spans before the single large `sub sp` jumps past the guard page.
    * slim_prologue/fp_at_bottom returned above — their frames are bounded to
    * one page (≤16 / ≤504 bytes), so only this path can exceed `interval`. */
   {
     u32 interval = abi_stack_probe_interval(a->base.c->abi);
     if (interval && L->frame_size > interval)
       aa_words_stack_probe(a, words, cap, &n, L->frame_size, interval);
   }
   aa_words_sub_sp_frame(a, words, cap, &n, L->frame_size);
   if (a->slim_small_frame) {
     /* `stp x29, x30, [sp, #saved_pair_off]` — skip the `add x17, sp, #...`
      * scratch the fat path needs. Valid when the offset fits stp's
      * signed-7-bit scaled immediate (saved_pair_off <= 504). */
     u32 save_off = aa_sp_off_saved_pair(L);
     if (n >= cap) aa_panic(a, "prologue too large");
     words[n++] = aa64_stp64_soff(AA_FP, AA_LR, AA_SP, (i32)(save_off / 8u));
   } else {
     aa_words_saved_pair_addr(a, words, cap, &n, L);
     if (n >= cap) aa_panic(a, "prologue too large");
     words[n++] = aa64_stp64_soff(AA_FP, AA_LR, AA_TMP1, 0); /* fp,lr @ [x17] */
   }
   aa_words_frame_ptr_from_sp(a, words, cap, &n, L);
   /* Save callee-saved registers the allocator used (fp-relative; their slots
    * were reserved first by aa_reserve_callee_saves so offsets fit stur). */
   aa_words_callee_saves(a, 1, words, cap, &n);
   return n;
 }
 
 /* Patch the reserved prologue region (`region` words at prologue_pos) with the
  * real prologue for `L`. Used by the NativeDirectTarget single-pass path,
  * which reserves AA_PROLOGUE_WORDS up front before the frame is known. The
  * optimizer path reserves exactly the words it needs, so `region` equals
  * the real prologue length and no tail remains. */
 static void aa_patch_prologue(AANativeTarget* a, const AAFrameLayout* L,
                               u32 region) {
   u32 words[AA_PROLOGUE_WORDS];
   u32 n;
   ObjSecId sec = a->func->text_section_id;
   if (region > AA_PROLOGUE_WORDS) aa_panic(a, "prologue region too large");
   memset(words, 0, sizeof words);
   n = aa_build_prologue_words(a, L, words, region);
   /* If the real prologue is shorter than the reserved region (the worst-case
    * NDT reservation), branch straight to the body rather than leaving the
    * trailing slots as NOPs that fall through and execute on every call. */
   if (n < region) {
     words[n] = aa64_b(region - n);
     for (u32 i = n + 1u; i < region; ++i) words[i] = 0xd503201fu;
   }
   for (u32 i = 0; i < region; ++i)
     aa_patch32(a->base.obj, sec, a->prologue_pos + i * 4u, words[i]);
 }
 
 static void aa_emit_restore_frame(AANativeTarget* a, const AAFrameLayout* L) {
   MCEmitter* mc = a->base.mc;
   u32 words[AA_PROLOGUE_WORDS];
   u32 n = 0;
   if (!L->frame_size) return;
   aa_words_restore_frame(a, words, AA_PROLOGUE_WORDS, &n, L);
   for (u32 i = 0; i < n; ++i) aa_emit32(mc, words[i]);
 }
 
 /* Reserve one entry in the deferred-patch list, growing (arena-doubling) as
  * needed. The returned pointer is stable until the next aa_patch_alloc. */
 static AAPatch* aa_patch_alloc(AANativeTarget* a) {
   if (a->npatches == a->patches_cap) {
     u32 cap = a->patches_cap ? a->patches_cap * 2u : 8u;
     AAPatch* nb = arena_zarray(a->base.c->tu, AAPatch, cap);
     if (a->patches) memcpy(nb, a->patches, sizeof(*nb) * a->npatches);
     a->patches = nb;
     a->patches_cap = cap;
   }
   return &a->patches[a->npatches++];
 }
 
 /* Append FP-relative loads that restore the saved callee registers (stp/ldp
  * paired, same as the prologue saves). Shared by the tail-call patch; the
  * function epilogue uses aa_emit_callee_restores. */
 static void aa_words_callee_restores(AANativeTarget* a, u32* words, u32 cap,
                                      u32* n) {
   aa_words_callee_saves(a, 0, words, cap, n);
 }
 
 /* Drain the deferred-patch list. Each entry targets a disjoint, fixed code
  * position, so insertion order does not affect output. */
 static void aa_apply_patches(AANativeTarget* a, const AAFrameLayout* L) {
   ObjSecId sec = a->func->text_section_id;
   for (u32 i = 0; i < a->npatches; ++i) {
     AAPatch* p = &a->patches[i];
     if (p->kind == AA_PATCH_ALLOCA) {
       u32 imm12, sh;
       if (!aa64_addsub_imm_fits(a->frame.max_outgoing, &imm12, &sh))
         aa_panic(a, "outgoing area too large for alloca result");
       aa_patch32(a->base.obj, sec, p->pos,
                  aa64_add_imm(1, p->u.dst_reg, AA_SP, imm12, sh));
     } else { /* AA_PATCH_TAIL */
       NativeLoc callee = p->u.callee;
       u32 words[AA_TAIL_WORDS];
       u32 n = 0;
       memset(words, 0, sizeof words);
       aa_words_callee_restores(a, words, AA_TAIL_WORDS, &n);
       aa_words_restore_frame(a, words, AA_TAIL_WORDS, &n, L);
       if (n >= AA_TAIL_WORDS) aa_panic(a, "tail patch too small");
       if (callee.kind == NATIVE_LOC_REG) {
         words[n++] = aa64_br(loc_reg(callee));
       } else if (callee.kind == NATIVE_LOC_GLOBAL) {
         while (n + 1u < AA_TAIL_WORDS) words[n++] = 0xd503201fu;
         words[n++] = aa64_b(0);
       } else {
         aa_panic(a, "unsupported tail target");
       }
       while (n < AA_TAIL_WORDS) words[n++] = 0xd503201fu;
       for (u32 w = 0; w < AA_TAIL_WORDS; ++w)
         aa_patch32(a->base.obj, sec, p->pos + w * 4u, words[w]);
     }
   }
 }
 
 static void aa_func_end(NativeTarget* t) {
   AANativeTarget* a = aa_of(t);
   MCEmitter* mc = t->mc;
   AAFrameLayout L =
       aa_build_layout(a->frame.cum_off, a->frame.max_outgoing, a->top_home_bytes);
   /* known_frame (optimizer): prologue, allocas, and tail epilogues were emitted
    * final and slim eligibility was settled in aa_func_begin_known_frame — there
    * is nothing to patch. Single-pass (NDT): a worst-case prologue region was
    * reserved and the deferred patches recorded; resolve them now that the frame
    * is final. The NDT path always uses the fat prologue/epilogue (slim_* left 0
    * by aa_func_begin_common, since its reserved region is much larger). */
   u32 prologue_region =
       a->frame.known_frame ? a->minimal_prologue_words : AA_PROLOGUE_WORDS;
   mc->label_place(mc, a->epilogue_label);
   aa_emit_callee_restores(a);
   aa_emit_restore_frame(a, &L);
   aa_emit32(mc, aa64_ret(AA_LR));
   if (a->frame.known_frame) {
     /* The frame-planning pre-pass plus final prologue/alloca/tail emission must
      * leave nothing deferred; a stray patch would mean a body-time frame change
      * the final prologue never saw. */
     if (a->npatches != 0) aa_panic(a, "known-frame path left deferred patches");
   } else {
     aa_patch_prologue(a, &L, prologue_region);
     aa_apply_patches(a, &L);
   }
   if (mc->cfi_set_next_pc_offset && mc->cfi_def_cfa && mc->cfi_offset) {
     i32 cfa = aa_cfa_off(a);
     mc->cfi_set_next_pc_offset(mc, prologue_region * 4u);
     /* CFA = caller's sp, an fp-relative offset that depends on the layout:
      * fp+16 (top-record) or fp+frame_size (bottom-record). saved fp/lr live at
      * [fp]/[fp+8] in both, hence at CFA-cfa / CFA-cfa+8. */
     mc->cfi_def_cfa(mc, AA_FP, cfa);
     mc->cfi_offset(mc, AA_FP, aa_fp_off_saved_fp() - cfa);
     mc->cfi_offset(mc, AA_LR, aa_fp_off_saved_lr() - cfa);
   }
   obj_symbol_define(t->obj, a->func->sym, a->func->text_section_id,
                     a->func_start, mc->pos(mc) - a->func_start);
   if (a->func->atomize) {
     obj_atom_define(t->obj, a->func->text_section_id, a->func_start,
                     mc->pos(mc) - a->func_start, a->func->sym, 0);
   }
   /* Hand the function's PC range to the Debug producer so its line program
    * (and DW_AT_low_pc/high_pc) cover this function — emit_section_line skips
    * functions without a recorded range. */
   if (mc->debug)
     debug_func_pc_range(mc->debug, a->func->text_section_id, a->func_start,
                         mc->pos(mc));
   if (mc->cfi_endproc) mc->cfi_endproc(mc);
   mc_end_function(mc);
   a->func = NULL;
 }
 
 static NativeFrameSlot aa_frame_slot(NativeTarget* t,
                                      const NativeFrameSlotDesc* d) {
   return native_frame_slot_alloc(&aa_of(t)->frame, d);
 }
 
 static int aa_frame_slot_debug_loc(NativeTarget* t, NativeFrameSlot slot,
                                    CGDebugLoc* out) {
   AANativeTarget* a = aa_of(t);
   AANativeSlot* s;
   i32 fp_off;
   if (!out) return 0;
   memset(out, 0, sizeof *out);
   if (slot == NATIVE_FRAME_SLOT_NONE || slot > a->frame.nslots) return 0;
   s = aa_slot(a, slot);
   fp_off = aa_fp_off_slot(a, s->off);
   out->kind = CG_DEBUG_LOC_FRAME;
   /* The hosted dbg stop snapshot currently carries x29/fp as the frame base
    * for variable materialization, so report the same FP-relative slot offset
    * used by native memory operands. */
   out->v.frame_ofs = fp_off;
   return 1;
 }
 
 static void aa_asm_clobber_masks(Compiler* c, SrcLoc loc, const Sym* clobbers,
                                  u32 nclob, u32* int_mask, u32* fp_mask);
 /* abi_clobber_masks is shared as native_asm_abi_clobber_masks
  * (cg/native_asm.h); it reads the masks from t->regs->classes. */
 
 /* Build the callee-saved set the prologue must preserve: the allocator-assigned
  * callee-saved registers (frame->callee_saved_used) plus any an inline-asm
  * block clobbers. The latter are opaque to the optimizer's operand scan, so it
  * forwards the raw clobber names (frame->asm_clobbers) and the arch-neutral
  * clobber-ABI sets (frame->asm_clobber_abi_sets); we resolve both into masks
  * and keep only the callee-saved ones, per AAPCS64: x19..x28 and the low 64
  * bits of v8..v15 (x29/x30 are the frame pointer and link register, handled by
  * the prologue head, not as ordinary callee-saves). This is the same register
  * selection the per-block spill used, hoisted into the prologue. */
 static u32 aa_known_callee_saves(NativeTarget* t,
                                  const NativeKnownFrameDesc* frame, u32* out,
                                  u32 cap) {
   u32 ncls = frame->ncallee_classes;
   u32 clob_int = 0, clob_fp = 0, abi_int, abi_fp;
   if (ncls > cap) ncls = cap;
   for (u32 c = 0; c < ncls; ++c)
     out[c] = frame->callee_saved_used ? frame->callee_saved_used[c] : 0u;
   if (frame->asm_clobbers && frame->nasm_clobbers) {
     AANativeTarget* a = aa_of(t);
     SrcLoc loc = a->func ? a->func->loc : (SrcLoc){0, 0, 0};
     aa_asm_clobber_masks(t->c, loc, frame->asm_clobbers, frame->nasm_clobbers,
                          &clob_int, &clob_fp);
   }
   native_asm_abi_clobber_masks(t, frame->asm_clobber_abi_sets, &abi_int,
                                &abi_fp);
   clob_int |= abi_int;
   clob_fp |= abi_fp;
   for (Reg r = 0; r < 32u; ++r) {
     if (NATIVE_REG_INT < ncls && (clob_int & (1u << r)) && r >= 19u && r <= 28u)
       out[NATIVE_REG_INT] |= 1u << r;
     if (NATIVE_REG_FP < ncls && (clob_fp & (1u << r)) && r >= 8u && r <= 15u)
       out[NATIVE_REG_FP] |= 1u << r;
   }
   return ncls;
 }
 
 /* Optimizer entry point: the full frame is supplied up front, so the prologue,
  * entry saves, slim-form eligibility, allocas, and tail epilogues are all final
  * the moment they are emitted — no back-patching (aa_func_end skips the patch
  * passes when a->frame.known_frame). Slot creation order matches the
  * single-pass path (callee-saves first for stur range, then the static slots,
  * then sret/variadic entry saves) so offsets are identical to what the patch
  * path would produce. */
 static void aa_func_begin_known_frame(NativeTarget* t, const CGFuncDesc* fd,
                                       const NativeKnownFrameDesc* frame,
                                       NativeFrameSlot* out_slots) {
   AANativeTarget* a = aa_of(t);
   AAFrameLayout L;
   u32 words[AA_PROLOGUE_WORDS];
   u32 n;
   aa_func_begin_common(t, fd);
   a->frame.known_frame = 1;
   if (frame) {
     u32 cs[NATIVE_CALL_PLAN_CLASSES];
     u32 ncs = aa_known_callee_saves(t, frame, cs, NATIVE_CALL_PLAN_CLASSES);
     a->frame.has_alloca = frame->has_alloca;
     if (ncs) aa_reserve_callee_saves(t, cs, ncs);
     for (u32 i = 0; i < frame->nslots; ++i) {
       NativeFrameSlot slot = aa_frame_slot(t, &frame->slots[i]);
       if (out_slots) out_slots[i] = slot;
     }
     aa_reserve_entry_saves(a);
     /* Reserve the atomic-RMW scratch spill last (matching its lazy position in
      * the single-pass path), so aa_saved_tmp_spill reuses it instead of growing
      * the frame mid-body. */
     if (frame->needs_scratch_spill) {
       NativeFrameSlotDesc sd;
       memset(&sd, 0, sizeof sd);
       sd.type = builtin_id(KIT_CG_BUILTIN_I64);
       sd.size = 8;
       sd.align = 8;
       sd.kind = NATIVE_FRAME_SLOT_SPILL;
       a->saved_tmp_slot = a->base.frame_slot(&a->base, &sd);
     }
     if (frame->max_outgoing > a->frame.max_outgoing)
       a->frame.max_outgoing = frame->max_outgoing;
   }
   /* Frame is final: slot_bytes (cum_off) and out_stack (max_outgoing) are both
    * known, so the prologue immediates and slim-form choice are settled here.
    * frame_size_final must be set before aa_build_prologue_words / entry saves,
    * since the bottom-record offset helpers read it. */
   L = aa_build_layout(a->frame.cum_off, a->frame.max_outgoing,
                       a->top_home_bytes);
   a->frame_size_final = L.frame_size;
   /* Slim Tier A: no callee-saves, no alloca, no body slots, no outgoing stack
    * args — the whole frame is the 16-byte record. fp_at_bottom: a small frame
    * with callee-saves/locals and no outgoing stack args; the record moves to
    * the bottom (fp = sp) so sp adjustment folds into the pre/post-indexed
    * stp/ldp (frame_size <= 504 keeps the post-index ldp imm in range).
    * Otherwise slim_small_frame keeps the top-record layout but skips the
    * x17/x10 scratch (out_stack>0 small frames land here). A Windows-variadic
    * home area forces the fat top-record layout: it lives above the saved pair,
    * which neither the slim forms (saved pair at the very top) nor the
    * bottom-record (saved pair at the very bottom) leave room for. (See
    * aa_func_end for the single-pass path, which never takes any slim form.) */
   a->slim_prologue = a->frame.ncallee_saves == 0 && !a->frame.has_alloca &&
                      L.slot_bytes == 0 && L.out_stack == 0 && !a->top_home_bytes;
   a->fp_at_bottom = !a->slim_prologue && !a->frame.has_alloca &&
                     L.out_stack == 0 && L.frame_size <= 504u &&
                     !a->top_home_bytes;
   a->slim_small_frame = !a->slim_prologue && !a->fp_at_bottom &&
                         !a->frame.has_alloca && !a->top_home_bytes &&
                         aa_sp_off_saved_pair(&L) <= 504u;
   n = aa_build_prologue_words(a, &L, words, AA_PROLOGUE_WORDS);
   for (u32 i = 0; i < n; ++i) aa_emit32(t->mc, words[i]);
   a->minimal_prologue_words = n;
   a->frame.frame_final = 1;
   aa_emit_entry_save_stores(a);
 }
 
 static void aa_spill(NativeTarget* t, NativeLoc src, NativeFrameSlot slot,
                      MemAccess mem) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = slot;
   addr.base_type = src.type;
   aa_emit_mem(aa_of(t), 0, src, addr, mem);
 }
 
 static void aa_reload(NativeTarget* t, NativeLoc dst, NativeFrameSlot slot,
                       MemAccess mem) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = slot;
   addr.base_type = dst.type;
   aa_emit_mem(aa_of(t), 1, dst, addr, mem);
 }
 
 static MCLabel aa_label_new(NativeTarget* t) { return t->mc->label_new(t->mc); }
 
 static void aa_label_place(NativeTarget* t, MCLabel label) {
   t->mc->label_place(t->mc, label);
 }
 
 static void aa_jump(NativeTarget* t, MCLabel label) {
   aa_emit32(t->mc, aa64_b(0));
   t->mc->emit_label_ref(t->mc, label, R_AARCH64_JUMP26, 4, 0);
 }
 
 static void aa_cmp_branch(NativeTarget* t, CmpOp op, NativeLoc lhs,
                           NativeLoc rhs, MCLabel label) {
   aa_emit_cmp_to_flags(t, lhs, rhs);
   /* CMP_ONE_F / CMP_UEQ_F have no single FP condition: take the branch from a
    * pair of conditional branches to the same label (no scratch register). */
   if (op == CMP_ONE_F) {
     /* ordered & !=: branch if a<b (MI) or a>b (GT). */
     aa_emit32(t->mc, aa64_brcond_pack((AA64BrCond){.cond = 0x4u})); /* MI */
     t->mc->emit_label_ref(t->mc, label, R_AARCH64_CONDBR19, 4, 0);
     aa_emit32(t->mc, aa64_brcond_pack((AA64BrCond){.cond = 0xcu})); /* GT */
     t->mc->emit_label_ref(t->mc, label, R_AARCH64_CONDBR19, 4, 0);
     return;
   }
   if (op == CMP_UEQ_F) {
     /* unordered | ==: branch if a==b (EQ) or unordered (VS). */
     aa_emit32(t->mc, aa64_brcond_pack((AA64BrCond){.cond = 0x0u})); /* EQ */
     t->mc->emit_label_ref(t->mc, label, R_AARCH64_CONDBR19, 4, 0);
     aa_emit32(t->mc, aa64_brcond_pack((AA64BrCond){.cond = 0x6u})); /* VS */
     t->mc->emit_label_ref(t->mc, label, R_AARCH64_CONDBR19, 4, 0);
     return;
   }
   aa_emit32(t->mc, aa64_brcond_pack((AA64BrCond){.cond = cmp_cond(op)}));
   t->mc->emit_label_ref(t->mc, label, R_AARCH64_CONDBR19, 4, 0);
 }
 
 static void aa_indirect_branch(NativeTarget* t, NativeLoc addr,
                                const MCLabel* valid_targets, u32 ntargets) {
   (void)valid_targets;
   (void)ntargets;
   aa_emit32(t->mc, aa64_br(loc_reg(addr)));
 }
 
 static void aa_load_label_addr(NativeTarget* t, NativeLoc dst, MCLabel target) {
   /* `&&label` address-take: adrp/add with the ADR_PREL_PG_HI21 +
    * ADD_ABS_LO12_NC relocation pair against the label's per-block local symbol
    * — the same form used to address a global — so the reference is genuinely
    * relocatable (reaches ±4 GiB) and any assembler resolves it from the symbol.
    * Replaces the old 16-byte INTRA-label sequence with a baked offset. */
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst);
   ObjSymId sym = mc_label_symbol(mc, target);
   u32 pos = mc->pos(mc);
   aa_emit32(mc, aa64_adrp(rd, 0, 0));
   mc->emit_reloc_at(mc, mc->section_id, pos, R_AARCH64_ADR_PREL_PG_HI21, sym, 0,
                     0, 0);
   pos = mc->pos(mc);
   aa_emit32(mc, aa64_add_imm(1, rd, rd, 0, 0));
   mc->emit_reloc_at(mc, mc->section_id, pos, R_AARCH64_ADD_ABS_LO12_NC, sym, 0,
                     0, 0);
 }
 
 static void aa_move(NativeTarget* t, NativeLoc dst, NativeLoc src) {
   /* Identity move elision: same-class same-reg is a no-op on aarch64
    * regardless of width (mov xN,xN and mov wN,wN both leave the low bits
    * untouched). Catches no-op IR_CONVERT (BITCAST, ZEXT/SEXT with
    * src_bits>=dst_bits, FEXT/FTRUNC across-class) when the allocator put
    * dst and src in the same hard reg — common post #2.5 return-reg
    * coalescing, e.g. `convert opnds=[v0,v0]` after a pointer-returning call
    * was emitting `mov x0,x0`. Cross-class (fp<->gpr) bitcasts are not
    * elided here even when the reg numbers match — the register files are
    * disjoint. */
   if (dst.kind == NATIVE_LOC_REG && src.kind == NATIVE_LOC_REG &&
       native_loc_is_fp(dst) == native_loc_is_fp(src) && dst.v.reg == src.v.reg)
     return;
   if (native_loc_is_fp(dst) && native_loc_is_fp(src)) {
     if (type_size32(t, dst.type) == 16u)
       aa_emit32(t->mc, aa_mov_vec16(loc_reg(dst), loc_reg(src)));
     else
       aa_emit32(t->mc, aa_fmov_fp(type_size32(t, dst.type) == 8u, loc_reg(dst),
                                   loc_reg(src)));
   } else if (native_loc_is_fp(dst)) {
     aa_emit32(t->mc,
               aa_fmov_gpr_to_fp(loc_is_64(t, src), loc_reg(dst), loc_reg(src)));
   } else if (native_loc_is_fp(src)) {
     aa_emit32(t->mc,
               aa_fmov_fp_to_gpr(loc_is_64(t, dst), loc_reg(dst), loc_reg(src)));
   } else {
     aa_emit32(t->mc,
               aa64_mov_reg(loc_is_64(t, dst), loc_reg(dst), loc_reg(src)));
   }
 }
 
 static NativeLoc aa_tmp_loc(KitCgTypeId type, Reg reg);
 
 static void aa_load_imm_native(NativeTarget* t, NativeLoc dst, i64 imm) {
   aa_emit_load_imm(t->mc, loc_is_64(t, dst), loc_reg(dst), imm);
 }
 
 static void aa_load_const(NativeTarget* t, NativeLoc dst, ConstBytes cbytes) {
   u64 v = 0;
   if (cbytes.size > 8u)
     compiler_panic(t->c, ((AANativeTarget*)t)->loc,
                    "aarch64 native target: byte constant too large");
   for (u32 i = 0; i < cbytes.size; ++i) v |= (u64)cbytes.bytes[i] << (i * 8u);
   if (native_loc_is_fp(dst)) {
     NativeLoc tmp = aa_tmp_loc(cbytes.type, AA_TMP0);
     aa_emit_load_imm(t->mc, cbytes.size == 8u, AA_TMP0, (i64)v);
     aa_move(t, dst, tmp);
   } else {
     aa_emit_load_imm(t->mc, loc_is_64(t, dst), loc_reg(dst), (i64)v);
   }
 }
 
 static void aa_load_addr(NativeTarget* t, NativeLoc dst, NativeAddr addr) {
   AANativeTarget* a = aa_of(t);
   u32 rd = loc_reg(dst);
   aa_materialize_frame_index(a, &addr, rd);
   switch ((NativeAddrBaseKind)addr.base_kind) {
     case NATIVE_ADDR_BASE_FRAME: {
       AANativeSlot* s = aa_slot(a, addr.base.frame);
       aa_emit_add_imm(a, rd, AA_FP, aa_fp_off_slot(a, s->off) + addr.offset);
       aa_apply_index(a, rd, &addr);
       return;
     }
     case NATIVE_ADDR_BASE_FRAME_VALUE: {
       NativeAddr load;
       MemAccess mem;
       memset(&load, 0, sizeof load);
       load.base_kind = NATIVE_ADDR_BASE_FRAME;
       load.base.frame = addr.base.frame;
       load.base_type =
           addr.base_type ? addr.base_type : builtin_id(KIT_CG_BUILTIN_I64);
       memset(&mem, 0, sizeof mem);
       mem.type = load.base_type;
       mem.size = 8;
       mem.align = 8;
       aa_emit_mem(a, 1, dst, load, mem);
       if (addr.offset) aa_emit_add_imm(a, rd, rd, addr.offset);
       aa_apply_index(a, rd, &addr);
       return;
     }
     case NATIVE_ADDR_BASE_REG:
       aa_emit_add_imm(a, rd, addr.base.reg, addr.offset);
       aa_apply_index(a, rd, &addr);
       return;
     case NATIVE_ADDR_BASE_GLOBAL: {
       i64 addend = addr.base.global.addend + (i64)addr.offset;
       u32 pos = t->mc->pos(t->mc);
       if (aa_use_got_for_sym(t, addr.base.global.sym)) {
         aa_emit32(t->mc, aa64_adrp(rd, 0, 0));
         t->mc->emit_reloc_at(t->mc, t->mc->section_id, pos,
                              R_AARCH64_ADR_GOT_PAGE, addr.base.global.sym, 0, 0,
                              0);
         pos = t->mc->pos(t->mc);
         aa_emit32(t->mc, aa_ldr_uimm(3, rd, rd, 0));
         t->mc->emit_reloc_at(t->mc, t->mc->section_id, pos,
                              R_AARCH64_LD64_GOT_LO12_NC, addr.base.global.sym,
                              0, 0, 0);
         if (addend) aa_emit_add_i64(a, rd, rd, addend);
         aa_apply_index(a, rd, &addr);
         return;
       }
       aa_emit32(t->mc, aa64_adrp(rd, 0, 0));
       t->mc->emit_reloc_at(t->mc, t->mc->section_id, pos,
                            R_AARCH64_ADR_PREL_PG_HI21, addr.base.global.sym,
                            addend, 0, 0);
       pos = t->mc->pos(t->mc);
       aa_emit32(t->mc, aa64_add_imm(1, rd, rd, 0, 0));
       t->mc->emit_reloc_at(t->mc, t->mc->section_id, pos,
                            R_AARCH64_ADD_ABS_LO12_NC, addr.base.global.sym,
                            addend, 0, 0);
       aa_apply_index(a, rd, &addr);
       return;
     }
     default:
       aa_panic(a, "unsupported load_addr");
   }
 }
 
 static void aa_load_native(NativeTarget* t, NativeLoc dst, NativeAddr addr,
                            MemAccess mem) {
   aa_emit_mem(aa_of(t), 1, dst, addr, mem);
 }
 
 static void aa_store_native(NativeTarget* t, NativeAddr addr, NativeLoc src,
                             MemAccess mem) {
   aa_emit_mem(aa_of(t), 0, src, addr, mem);
 }
 
 /* Windows/AArch64 TLS Local-Exec (PE-COFF). Mirrors x64_tls_addr_of_win64:
  *   ldr  rd,  [x18, #0x58]            ; TEB.ThreadLocalStoragePointer
  *   adrp x16, _tls_index              ; PAGEBASE_REL21
  *   add  x16, x16, :lo12:_tls_index   ; PAGEOFFSET_12A
  *   ldr  w16, [x16]                   ; module's TLS index
  *   ldr  rd,  [rd, x16, lsl #3]       ; this module's TLS block base
  *   add  rd,  rd, #:secrel_hi12:sym   ; SECREL_HIGH12A (sh=1)
  *   add  rd,  rd, #:secrel_lo12:sym   ; SECREL_LOW12A  (sh=0)
  * We materialize &_tls_index via ADRP+ADD (not LDR :lo12:) on purpose: the
  * COFF reader collapses LDST32→LDST64 width, so an LDR :lo12: form would be
  * mis-scaled at link time; ADD_ABS_LO12_NC carries no width and round-trips
  * cleanly. AA_TMP0 (x16) is the reserved scratch; rd is an allocated reg
  * distinct from x16/x17/x18. */
 static void aa_tls_addr_of_win(NativeTarget* t, NativeLoc dst, ObjSymId sym,
                                i64 addend) {
   MCEmitter* mc = t->mc;
   u32 sec = mc->section_id;
   u32 rd = loc_reg(dst);
   u32 pos;
   Sym idx_name = pool_intern_slice(t->c->global, SLICE_LIT("_tls_index"));
   ObjSymId idx_sym = obj_symbol_find(t->obj, idx_name);
   if (idx_sym == 0)
     idx_sym =
         obj_symbol(t->obj, idx_name, SB_GLOBAL, SK_UNDEF, OBJ_SEC_NONE, 0, 0);
   /* (1) rd = TEB.ThreadLocalStoragePointer. */
   aa_emit32(mc, aa_ldr_uimm(3, rd, AA_WIN_TEB_REG, AA_WIN_TEB_TLS_PTR_OFF));
   /* (2)+(3) x16 = &_tls_index via ADRP + ADD. */
   pos = mc->pos(mc);
   aa_emit32(mc, aa64_adrp(AA_TMP0, 0, 0));
   mc->emit_reloc_at(mc, sec, pos, R_AARCH64_ADR_PREL_PG_HI21, idx_sym, 0, 0, 0);
   pos = mc->pos(mc);
   aa_emit32(mc, aa64_add_imm(1, AA_TMP0, AA_TMP0, 0, 0));
   mc->emit_reloc_at(mc, sec, pos, R_AARCH64_ADD_ABS_LO12_NC, idx_sym, 0, 0, 0);
   /* (4) w16 = _tls_index (the loaded value). */
   aa_emit32(mc, aa_ldr_uimm(2, AA_TMP0, AA_TMP0, 0));
   /* (5) rd = TLS array slot for this module: ldr rd, [rd, x16, lsl #3]. */
   aa_emit32(mc, aa_ldst_regoff_v(3, 0, 1, rd, rd, AA_TMP0, 1));
   /* (6) rd += :secrel_hi12:sym  (ADD with sh=1; linker patches imm12). */
   pos = mc->pos(mc);
   aa_emit32(mc, aa64_add_imm(1, rd, rd, 0, 1));
   mc->emit_reloc_at(mc, sec, pos, R_COFF_AARCH64_SECREL_HIGH12A, sym, addend, 1,
                     0);
   /* (7) rd += :secrel_lo12:sym  (ADD with sh=0). */
   pos = mc->pos(mc);
   aa_emit32(mc, aa64_add_imm(1, rd, rd, 0, 0));
   mc->emit_reloc_at(mc, sec, pos, R_COFF_AARCH64_SECREL_LOW12A, sym, addend, 1,
                     0);
 }
 
 static void aa_tls_addr_of(NativeTarget* t, NativeLoc dst, ObjSymId sym,
                            i64 addend) {
   AANativeTarget* a = aa_of(t);
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst);
   u32 pos;
   if (obj_format_tls_via_descriptor(t->c)) {
     aa_emit32(mc, aa64_adrp(0, 0, 0));
     pos = mc->pos(mc) - 4u;
     mc->emit_reloc_at(mc, mc->section_id, pos, R_AARCH64_TLVP_LOAD_PAGE21, sym,
                       0, 0, 0);
     aa_emit32(mc, aa_ldr_uimm(3, 0, 0, 0));
     pos = mc->pos(mc) - 4u;
     mc->emit_reloc_at(mc, mc->section_id, pos, R_AARCH64_TLVP_LOAD_PAGEOFF12,
                       sym, 0, 0, 0);
     aa_emit32(mc, aa_ldr_uimm(3, AA_TMP0, 0, 0));
     aa_emit32(mc, aa64_blr(AA_TMP0));
     if (addend) aa_emit_add_i64(a, 0, 0, addend);
     if (rd != 0) aa_emit32(mc, aa64_mov_reg(1, rd, 0));
     return;
   }
   if (obj_format_tls_model(t->c) == OBJ_TLS_WINDOWS_TEB) {
     aa_tls_addr_of_win(t, dst, sym, addend);
     return;
   }
   if (t->c->target.obj != KIT_OBJ_ELF) {
     aa_panic(a, "unsupported TLS object format");
   }
   aa_emit32(mc, aa_mrs_tpidr_el0(rd));
   pos = mc->pos(mc);
   aa_emit32(mc, aa64_add_imm(1, rd, rd, 0, 1));
   mc->emit_reloc_at(mc, mc->section_id, pos, R_AARCH64_TLSLE_ADD_TPREL_HI12,
                     sym, addend, 0, 0);
   pos = mc->pos(mc);
   aa_emit32(mc, aa64_add_imm(1, rd, rd, 0, 0));
   mc->emit_reloc_at(mc, mc->section_id, pos, R_AARCH64_TLSLE_ADD_TPREL_LO12_NC,
                     sym, addend, 0, 0);
 }
 
 static NativeLoc aa_tmp_loc(KitCgTypeId type, Reg reg) {
   NativeLoc loc;
   memset(&loc, 0, sizeof loc);
   loc.kind = NATIVE_LOC_REG;
   loc.cls = NATIVE_REG_INT;
   loc.type = type;
   loc.v.reg = reg;
   return loc;
 }
 
 static NativeAddr aa_addr_plus(NativeAddr addr, u32 off) {
   addr.offset += (i32)off;
   return addr;
 }
 
 static void aa_copy_bytes_dir(NativeTarget* t, NativeAddr dst, NativeAddr src,
                               AggregateAccess access, int backward) {
   KitCgTypeId i64 = builtin_id(KIT_CG_BUILTIN_I64);
   KitCgTypeId i32 = builtin_id(KIT_CG_BUILTIN_I32);
   KitCgTypeId i16 = builtin_id(KIT_CG_BUILTIN_I16);
   KitCgTypeId i8 = builtin_id(KIT_CG_BUILTIN_I8);
   NativeLoc tmp = aa_tmp_loc(i64, AA_TMP0);
   u32 off = 0;
   while (off < access.size) {
     u32 rem = access.size - off;
     u32 pos;
     MemAccess mem = access.mem;
     if (rem >= 8u) {
       mem.type = i64;
       mem.size = 8u;
     } else if (rem >= 4u) {
       mem.type = i32;
       mem.size = 4u;
       tmp.type = i32;
     } else if (rem >= 2u) {
       mem.type = i16;
       mem.size = 2u;
       tmp.type = i16;
     } else {
       mem.type = i8;
       mem.size = 1u;
       tmp.type = i8;
     }
     mem.align = mem.size;
     pos = backward ? access.size - off - mem.size : off;
     aa_load_native(t, tmp, aa_addr_plus(src, pos), mem);
     aa_store_native(t, aa_addr_plus(dst, pos), tmp, mem);
     off += mem.size;
     tmp.type = i64;
   }
 }
 
 static void aa_copy_bytes(NativeTarget* t, NativeAddr dst, NativeAddr src,
                           AggregateAccess access) {
   aa_copy_bytes_dir(t, dst, src, access, 0);
 }
 
 static void aa_set_bytes(NativeTarget* t, NativeAddr dst, NativeLoc byte_value,
                          AggregateAccess access) {
   KitCgTypeId i8 = builtin_id(KIT_CG_BUILTIN_I8);
   NativeLoc byte = byte_value;
   MemAccess mem = access.mem;
   mem.type = i8;
   mem.size = 1u;
   mem.align = 1u;
   byte.type = i8;
   for (u32 off = 0; off < access.size; ++off)
     aa_store_native(t, aa_addr_plus(dst, off), byte, mem);
 }
 
 static void aa_lsl_imm(NativeTarget* t, u32 sf, u32 rd, u32 rn, u32 sh);
 static void aa_lsr_imm(NativeTarget* t, u32 sf, u32 rd, u32 rn, u32 sh);
 static void aa_asr_imm(NativeTarget* t, u32 sf, u32 rd, u32 rn, u32 sh);
 
 /* Strength-reduce `mul rd, rn, #imm` for the constants accepted by
  * aa64_imul_strength_reducible into a single non-mul instruction. Callers
  * must gate on aa64_imul_strength_reducible(sf, imm) — this routine panics
  * on unhandled constants. */
 static void aa_emit_mul_const_imm(NativeTarget* t, u32 sf, u32 rd, u32 rn,
                                   i64 imm) {
   u64 a;
   if (imm == 0) {
     aa_emit32(t->mc, aa64_mov_reg(sf, rd, AA64_ZR));
     return;
   }
   if (imm == 1) {
     if (rd != rn) aa_emit32(t->mc, aa64_mov_reg(sf, rd, rn));
     return;
   }
   if (imm == -1) {
     aa_emit32(t->mc, aa64_neg(sf, rd, rn));
     return;
   }
   /* +2^k: lsl rd, rn, #k */
   a = (u64)imm;
   if (imm > 0 && (a & (a - 1u)) == 0u) {
     u32 k = (u32)__builtin_ctzll(a);
     aa_lsl_imm(t, sf, rd, rn, k);
     return;
   }
   /* -2^k: sub rd, xzr, rn, lsl #k */
   if (imm < 0) {
     a = (u64)(-imm);
     if (a && (a & (a - 1u)) == 0u) {
       u32 k = (u32)__builtin_ctzll(a);
       aa_emit32(t->mc, aa64_addsubsr_pack((AA64AddSubSR){.sf = sf,
                                                          .op = 1u,
                                                          .S = 0u,
                                                          .shift = 0u,
                                                          .Rm = rn,
                                                          .imm6 = k,
                                                          .Rn = AA64_ZR,
                                                          .Rd = rd}));
       return;
     }
   }
   /* 2^k + 1: add rd, rn, rn, lsl #k */
   if (imm >= 3) {
     u64 m = (u64)(imm - 1);
     if ((m & (m - 1u)) == 0u) {
       u32 k = (u32)__builtin_ctzll(m);
       aa_emit32(t->mc, aa64_addsubsr_pack((AA64AddSubSR){.sf = sf,
                                                          .op = 0u,
                                                          .S = 0u,
                                                          .shift = 0u,
                                                          .Rm = rn,
                                                          .imm6 = k,
                                                          .Rn = rn,
                                                          .Rd = rd}));
       return;
     }
   }
   /* 1 - 2^k: sub rd, rn, rn, lsl #k */
   if (imm <= -1) {
     u64 m = (u64)(1 - imm);
     if (m && (m & (m - 1u)) == 0u) {
       u32 k = (u32)__builtin_ctzll(m);
       aa_emit32(t->mc, aa64_addsubsr_pack((AA64AddSubSR){.sf = sf,
                                                          .op = 1u,
                                                          .S = 0u,
                                                          .shift = 0u,
                                                          .Rm = rn,
                                                          .imm6 = k,
                                                          .Rn = rn,
                                                          .Rd = rd}));
       return;
     }
   }
   aa_panic(aa_of(t), "aa_emit_mul_const_imm: unhandled constant");
 }
 
 static void aa_binop(NativeTarget* t, BinOp op, NativeLoc dst, NativeLoc lhs,
                      NativeLoc rhs) {
   u32 sf = loc_is_64(t, dst) ? 1u : 0u;
   u32 rd = loc_reg(dst), rn = loc_reg(lhs), rm = loc_reg(rhs);
   if (native_loc_is_fp(dst)) {
     u32 d = type_size32(t, dst.type) == 8u;
     switch (op) {
       case BO_FADD:
         aa_emit32(t->mc, aa_fp_bin(0x002800u, d, rd, rn, rm));
         return;
       case BO_FSUB:
         aa_emit32(t->mc, aa_fp_bin(0x003800u, d, rd, rn, rm));
         return;
       case BO_FMUL:
         aa_emit32(t->mc, aa_fp_bin(0x000800u, d, rd, rn, rm));
         return;
       case BO_FDIV:
         aa_emit32(t->mc, aa_fp_bin(0x001800u, d, rd, rn, rm));
         return;
       default:
         aa_panic(aa_of(t), "unsupported floating binary op");
     }
   }
   if (rhs.kind == NATIVE_LOC_IMM && (op == BO_IADD || op == BO_ISUB)) {
     i64 imm = rhs.v.imm;
     int is_add = (op == BO_IADD);
     u32 imm12, sh;
     if (imm < 0) {
       is_add = !is_add;
       imm = -imm;
     }
     if (!aa64_addsub_imm_fits(imm, &imm12, &sh))
       aa_panic(aa_of(t), "binop immediate not encodable");
     aa_emit32(t->mc, is_add ? aa64_add_imm(sf, rd, rn, imm12, sh)
                             : aa64_sub_imm(sf, rd, rn, imm12, sh));
     return;
   }
   if (rhs.kind == NATIVE_LOC_IMM && op == BO_IMUL) {
     aa_emit_mul_const_imm(t, sf, rd, rn, rhs.v.imm);
     return;
   }
   if (rhs.kind == NATIVE_LOC_IMM &&
       (op == BO_SHL || op == BO_SHR_U || op == BO_SHR_S)) {
     u32 shamt = (u32)rhs.v.imm; /* imm_legal guarantees 0 <= imm < datasize */
     if (op == BO_SHL)
       aa_lsl_imm(t, sf, rd, rn, shamt);
     else if (op == BO_SHR_U)
       aa_lsr_imm(t, sf, rd, rn, shamt);
     else
       aa_asr_imm(t, sf, rd, rn, shamt);
     return;
   }
   if (rhs.kind == NATIVE_LOC_IMM &&
       (op == BO_AND || op == BO_OR || op == BO_XOR)) {
     u32 N, immr, imms;
     if (!aa64_logimm_encode((u64)rhs.v.imm, sf, &N, &immr, &imms))
       aa_panic(aa_of(t), "logical immediate not encodable");
     if (op == BO_AND)
       aa_emit32(t->mc, aa64_and_imm(sf, rd, rn, N, immr, imms));
     else if (op == BO_OR)
       aa_emit32(t->mc, aa64_orr_imm(sf, rd, rn, N, immr, imms));
     else
       aa_emit32(t->mc, aa64_eor_imm(sf, rd, rn, N, immr, imms));
     return;
   }
   switch (op) {
     case BO_IADD:
       aa_emit32(t->mc, aa64_add(sf, rd, rn, rm));
       return;
     case BO_ISUB:
       aa_emit32(t->mc, aa64_sub(sf, rd, rn, rm));
       return;
     case BO_IMUL:
       aa_emit32(t->mc, aa64_mul(sf, rd, rn, rm));
       return;
     case BO_SDIV:
       aa_emit32(t->mc, aa64_sdiv(sf, rd, rn, rm));
       return;
     case BO_UDIV:
       aa_emit32(t->mc, aa64_udiv(sf, rd, rn, rm));
       return;
     case BO_SREM:
       aa_emit32(t->mc, aa64_sdiv(sf, AA_TMP0, rn, rm));
       aa_emit32(t->mc, aa64_mul(sf, AA_TMP0, AA_TMP0, rm));
       aa_emit32(t->mc, aa64_sub(sf, rd, rn, AA_TMP0));
       return;
     case BO_UREM:
       aa_emit32(t->mc, aa64_udiv(sf, AA_TMP0, rn, rm));
       aa_emit32(t->mc, aa64_mul(sf, AA_TMP0, AA_TMP0, rm));
       aa_emit32(t->mc, aa64_sub(sf, rd, rn, AA_TMP0));
       return;
     case BO_AND:
       aa_emit32(t->mc, aa64_and(sf, rd, rn, rm));
       return;
     case BO_OR:
       aa_emit32(t->mc, aa64_orr(sf, rd, rn, rm));
       return;
     case BO_XOR:
       aa_emit32(t->mc, aa64_eor(sf, rd, rn, rm));
       return;
     case BO_SHL:
       aa_emit32(t->mc, aa64_lslv(sf, rd, rn, rm));
       return;
     case BO_SHR_U:
       aa_emit32(t->mc, aa64_lsrv(sf, rd, rn, rm));
       return;
     case BO_SHR_S:
       aa_emit32(t->mc, aa64_asrv(sf, rd, rn, rm));
       return;
     default:
       aa_panic(aa_of(t), "unsupported binary op");
   }
 }
 
 static void aa_unop(NativeTarget* t, UnOp op, NativeLoc dst, NativeLoc src) {
   u32 sf = loc_is_64(t, dst) ? 1u : 0u;
   if (native_loc_is_fp(dst)) {
     switch (op) {
       case UO_FNEG:
       case UO_NEG:
         aa_emit32(t->mc, aa_fneg(type_size32(t, dst.type) == 8u, loc_reg(dst),
                                  loc_reg(src)));
         return;
       default:
         aa_panic(aa_of(t), "unsupported floating unary op");
     }
   }
   switch (op) {
     case UO_NEG:
       aa_emit32(t->mc, aa64_neg(sf, loc_reg(dst), loc_reg(src)));
       return;
     case UO_BNOT:
       aa_emit32(t->mc, aa64_mvn(sf, loc_reg(dst), loc_reg(src)));
       return;
     case UO_NOT:
       aa_emit32(t->mc, aa64_subs_imm12(sf, AA64_ZR, loc_reg(src), 0, 0));
       aa_emit32(t->mc, aa_cset(sf, loc_reg(dst), 0x0u));
       return;
     default:
       aa_panic(aa_of(t), "unsupported unary op");
   }
 }
 
 static void aa_emit_cmp_to_flags(NativeTarget* t, NativeLoc lhs,
                                  NativeLoc rhs) {
   if (native_loc_is_fp(lhs)) {
     aa_emit32(t->mc, aa_fcmp(type_size32(t, lhs.type) == 8u, loc_reg(lhs),
                              loc_reg(rhs)));
     return;
   }
   {
     u32 sf = loc_is_64(t, lhs) ? 1u : 0u;
     if (rhs.kind == NATIVE_LOC_IMM) {
       u32 imm12 = 0, sh = 0;
       if (rhs.v.imm < 0 || !aa64_addsub_imm_fits(rhs.v.imm, &imm12, &sh))
         aa_panic(aa_of(t), "cmp immediate not encodable");
       aa_emit32(t->mc, aa64_subs_imm12(sf, AA64_ZR, loc_reg(lhs), imm12, sh));
       return;
     }
     aa_emit32(t->mc, aa_subs_reg(sf, AA64_ZR, loc_reg(lhs), loc_reg(rhs)));
   }
 }
 
 static void aa_cmp(NativeTarget* t, CmpOp op, NativeLoc dst, NativeLoc lhs,
                    NativeLoc rhs) {
   u32 sf = loc_is_64(t, dst);
   u32 rd = loc_reg(dst);
   aa_emit_cmp_to_flags(t, lhs, rhs);
   /* CMP_ONE_F (ordered & !=) and CMP_UEQ_F (unordered | ==) have no single
    * AArch64 FP condition. After FCMP, unordered sets V (and Z=0), so VC
    * (V==0) selects "ordered". */
   if (op == CMP_ONE_F) {
     /* ordered & not-equal: NE masked to the ordered case. */
     aa_emit32(t->mc, aa_cset(sf, rd, 0x1u)); /* cset  rd, NE */
     aa_emit32(t->mc,
               aa64_csel_enc(sf, rd, rd, AA64_ZR, 0x7u)); /* csel  rd,rd,zr,VC */
     return;
   }
   if (op == CMP_UEQ_F) {
     /* equal, or forced to 1 when unordered. */
     aa_emit32(t->mc, aa_cset(sf, rd, 0x0u)); /* cset  rd, EQ */
     aa_emit32(t->mc, aa64_csinc_enc(sf, rd, rd, AA64_ZR,
                                     0x7u)); /* csinc rd,rd,zr,VC */
     return;
   }
   aa_emit32(t->mc, aa_cset(sf, rd, cmp_cond(op)));
 }
 
 static void aa_convert(NativeTarget* t, ConvKind op, NativeLoc dst,
                        NativeLoc src) {
   int dst_fp = native_loc_is_fp(dst);
   int src_fp = native_loc_is_fp(src);
   switch (op) {
     case CV_TRUNC:
     case CV_BITCAST:
       aa_move(t, dst, src);
       return;
     case CV_ZEXT: {
       u32 src_bits = type_size32(t, src.type) * 8u;
       u32 dst_bits = type_size32(t, dst.type) * 8u;
       u32 sf = dst_bits > 32u;
       if (src_bits >= dst_bits) {
         aa_move(t, dst, src);
       } else if (src_bits >= 32u) {
         aa_emit32(t->mc, aa64_mov_reg(0, loc_reg(dst), loc_reg(src)));
       } else {
         aa_emit32(t->mc,
                   aa_ubfm(sf, loc_reg(dst), loc_reg(src), 0, src_bits - 1u));
       }
       return;
     }
     case CV_SEXT: {
       u32 src_bits = type_size32(t, src.type) * 8u;
       u32 dst_bits = type_size32(t, dst.type) * 8u;
       u32 sf = dst_bits > 32u;
       if (src_bits >= dst_bits) {
         aa_move(t, dst, src);
       } else {
         aa_emit32(t->mc,
                   aa_sbfm(sf, loc_reg(dst), loc_reg(src), 0, src_bits - 1u));
       }
       return;
     }
     case CV_ITOF_S:
       aa_emit32(t->mc, aa_scvtf(type_size32(t, dst.type) == 8u,
                                 loc_is_64(t, src), loc_reg(dst), loc_reg(src)));
       return;
     case CV_ITOF_U:
       aa_emit32(t->mc, aa_ucvtf(type_size32(t, dst.type) == 8u,
                                 loc_is_64(t, src), loc_reg(dst), loc_reg(src)));
       return;
     case CV_FTOI_S:
       aa_emit32(t->mc,
                 aa_fcvtzs(loc_is_64(t, dst), type_size32(t, src.type) == 8u,
                           loc_reg(dst), loc_reg(src)));
       return;
     case CV_FTOI_U:
       aa_emit32(t->mc,
                 aa_fcvtzu(loc_is_64(t, dst), type_size32(t, src.type) == 8u,
                           loc_reg(dst), loc_reg(src)));
       return;
     case CV_FEXT:
       if (dst_fp && src_fp)
         aa_emit32(t->mc, aa_fcvt_d_s(loc_reg(dst), loc_reg(src)));
       else
         aa_move(t, dst, src);
       return;
     case CV_FTRUNC:
       if (dst_fp && src_fp)
         aa_emit32(t->mc, aa_fcvt_s_d(loc_reg(dst), loc_reg(src)));
       else
         aa_move(t, dst, src);
       return;
     default:
       aa_panic(aa_of(t), "unsupported conversion");
   }
 }
 
 static void aa_alloca(NativeTarget* t, NativeLoc dst, NativeLoc size,
                       u32 align) {
   AANativeTarget* a = aa_of(t);
   u32 use_align = align < 16u ? 16u : align;
   if (use_align & (use_align - 1u)) aa_panic(a, "alloca alignment not pow2");
   aa_emit_add_imm(a, AA_TMP0, loc_reg(size), (i32)(use_align - 1u));
   aa_emit_load_imm(t->mc, 1, AA_TMP1, -(i64)use_align);
   aa_emit32(t->mc, aa64_and(1, AA_TMP0, AA_TMP0, AA_TMP1));
   aa_emit32(t->mc, aa64_add_imm(1, AA_TMP1, AA_SP, 0, 0));
   aa_emit32(t->mc, aa64_sub(1, AA_TMP1, AA_TMP1, AA_TMP0));
   aa_emit32(t->mc, aa64_add_imm(1, AA_SP, AA_TMP1, 0, 0));
   /* The alloca result is sp + outgoing-area bytes. On the known-frame path
    * max_outgoing is already final, so emit the final `add dst, sp, #N` here; on
    * the single-pass path it is not known yet, so record a patch. */
   if (a->frame.known_frame) {
     u32 imm12, sh;
     if (!aa64_addsub_imm_fits(a->frame.max_outgoing, &imm12, &sh))
       aa_panic(a, "outgoing area too large for alloca result");
     aa_emit32(t->mc, aa64_add_imm(1, loc_reg(dst), AA_SP, imm12, sh));
   } else {
     AAPatch* p = aa_patch_alloc(a);
     p->kind = AA_PATCH_ALLOCA;
     p->pos = t->mc->pos(t->mc);
     p->u.dst_reg = loc_reg(dst);
     a->nalloca++;
     aa_emit32(t->mc, aa64_add_imm(1, loc_reg(dst), AA_SP, 0, 0));
   }
 }
 
 static MemAccess aa_mem_for_type(NativeTarget* t, KitCgTypeId type, u32 size) {
   MemAccess mem;
   memset(&mem, 0, sizeof mem);
   mem.type = type;
   mem.size = size ? size : type_size32(t, type);
   mem.align = type_align32(t, type);
   if (mem.align > mem.size && mem.size) mem.align = mem.size;
   return mem;
 }
 
 /* native_loc_reg / native_loc_stack are shared in native_target.h. */
 
 static NativeAddr aa_loc_addr(AANativeTarget* a, NativeLoc loc, u32 offset) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   switch ((NativeLocKind)loc.kind) {
     case NATIVE_LOC_FRAME:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = loc.v.frame;
       addr.base_type = loc.type;
       addr.offset = (i32)offset;
       return addr;
     case NATIVE_LOC_STACK:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = loc.v.stack.slot;
       addr.base_type = loc.type;
       addr.offset = loc.v.stack.offset + (i32)offset;
       return addr;
     case NATIVE_LOC_ADDR:
       addr = loc.v.addr;
       addr.offset += (i32)offset;
       return addr;
     default:
       aa_panic(a, "location is not addressable");
   }
   return addr;
 }
 
 static void aa_addr_of_loc(NativeTarget* t, NativeLoc dst, NativeLoc src) {
   AANativeTarget* a = aa_of(t);
   NativeAddr addr = aa_loc_addr(a, src, 0);
   aa_load_addr(t, dst, addr);
 }
 
 static void aa_load_part(NativeTarget* t, NativeLoc dst, NativeLoc src,
                          u32 offset, u32 size) {
   AANativeTarget* a = aa_of(t);
   MemAccess mem = aa_mem_for_type(t, dst.type, size);
   if (src.kind == NATIVE_LOC_REG) {
     aa_move(t, dst, src);
     return;
   }
   if (src.kind == NATIVE_LOC_FRAME || src.kind == NATIVE_LOC_STACK ||
       src.kind == NATIVE_LOC_ADDR) {
     NativeAddr addr = aa_loc_addr(a, src, offset);
     addr.base_type = dst.type;
     aa_emit_mem(a, 1, dst, addr, mem);
     return;
   }
   if (src.kind == NATIVE_LOC_IMM) {
     aa_emit_load_imm(t->mc, loc_is_64(t, dst), loc_reg(dst), src.v.imm);
     return;
   }
   aa_panic(a, "unsupported call argument source");
 }
 
 static void aa_store_part(NativeTarget* t, NativeLoc dst, NativeLoc src,
                           u32 offset, u32 size) {
   AANativeTarget* a = aa_of(t);
   MemAccess mem = aa_mem_for_type(t, src.type, size);
   if (dst.kind == NATIVE_LOC_FRAME || dst.kind == NATIVE_LOC_STACK ||
       dst.kind == NATIVE_LOC_ADDR) {
     NativeAddr addr = aa_loc_addr(a, dst, offset);
     addr.base_type = src.type;
     aa_emit_mem(a, 0, src, addr, mem);
     return;
   }
   if (dst.kind == NATIVE_LOC_REG) {
     aa_move(t, dst, src);
     return;
   }
   aa_panic(a, "unsupported call return destination");
 }
 
 static void aa_store_outgoing_part(NativeTarget* t, int tail_call,
                                    u32 stack_off, NativeLoc src, u32 size) {
   NativeAddr addr;
   MemAccess mem = aa_mem_for_type(t, src.type, size);
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_REG;
   addr.base.reg = tail_call ? AA_FP : AA_SP;
   addr.base_type = src.type;
   /* Tail calls write outgoing args into the caller's incoming-args window
    * (= [fp + 16 + off], same address the tail-callee will read via
    * aa_fp_off_in_arg). Non-tail calls write to the sp-anchored outgoing
    * area at the bottom of the caller's frame. */
   addr.offset = tail_call ? aa_fp_off_tail_out_arg(aa_of(t), stack_off)
                           : aa_sp_off_out_arg(stack_off);
   aa_emit_mem(aa_of(t), 0, src, addr, mem);
 }
 
 static const ABIArgInfo* aa_param_abi(NativeTarget* t, const ABIFuncInfo* abi,
                                       const NativeCallDesc* desc, u32 i,
                                       ABIArgInfo* scratch) {
   if (abi && i < abi->nparams) return &abi->params[i];
   memset(scratch, 0, sizeof *scratch);
   scratch->kind = ABI_ARG_DIRECT;
   scratch->flags = ABI_AF_NONE;
   scratch->nparts = 1;
   scratch->parts = arena_zarray(t->c->tu, ABIArgPart, 1);
   /* Windows ARM64 routes variadic floating-point arguments through the integer
    * registers/stack (the classifier's remap_fp_parts_to_int does the same for
    * the *named* params of a variadic function); the value's bit pattern moves
    * via fmov x,d. Every other ABI keeps the `...` FP args in v registers. */
   ((ABIArgPart*)scratch->parts)[0].cls =
       (cg_type_is_float(t->c, desc->args[i].type) &&
        !(abi && abi->vararg_fp_via_int))
           ? ABI_CLASS_FP
           : ABI_CLASS_INT;
   ((ABIArgPart*)scratch->parts)[0].loc = ABI_LOC_REG;
   ((ABIArgPart*)scratch->parts)[0].size = type_size32(t, desc->args[i].type);
   ((ABIArgPart*)scratch->parts)[0].align = type_align32(t, desc->args[i].type);
   ((ABIArgPart*)scratch->parts)[0].src_offset = 0;
   return scratch;
 }
 
 /* Stack footprint of a single argument part. AAPCS64 uses 8-byte slots. Apple
  * ARM64 uses compact 4-byte slots for fixed stack-passed int32-sized values,
  * but its forced stack variadics still use 8-byte slots. */
 static u32 aa_stack_arg_min_align(const ABIFuncInfo* abi) {
   return (abi && abi->stack_arg_min_align) ? abi->stack_arg_min_align : 8u;
 }
 
 static u32 aa_vararg_stack_arg_min_align(const ABIFuncInfo* abi) {
   if (abi && abi->vararg_stack_arg_min_align)
     return abi->vararg_stack_arg_min_align;
   return aa_stack_arg_min_align(abi);
 }
 
 static u32 aa_vararg_stack_start(const ABIFuncInfo* abi, u32 cursor) {
   return align_up_u32(cursor, aa_vararg_stack_arg_min_align(abi));
 }
 
 /* Natural stack alignment of a part, capped at 16 (binary128). */
 static u32 aa_part_stack_align_min(u32 min_align, const ABIArgPart* part) {
   u32 al = part->align ? part->align : 8u;
   if (al < min_align) al = min_align;
   if (al > 16u) al = 16u;
   return al;
 }
 
 static u32 aa_part_stack_align(const ABIFuncInfo* abi,
                                const ABIArgPart* part) {
   return aa_part_stack_align_min(aa_stack_arg_min_align(abi), part);
 }
 
 static u32 aa_part_vararg_stack_align(const ABIFuncInfo* abi,
                                       const ABIArgPart* part) {
   return aa_part_stack_align_min(aa_vararg_stack_arg_min_align(abi), part);
 }
 
 static u32 aa_part_stack_size(const ABIFuncInfo* abi,
                               const ABIArgPart* part) {
   return align_up_u32(part->size ? part->size : 8u,
                       aa_part_stack_align(abi, part));
 }
 
 static u32 aa_part_vararg_stack_size(const ABIFuncInfo* abi,
                                      const ABIArgPart* part) {
   return align_up_u32(part->size ? part->size : 8u,
                       aa_part_vararg_stack_align(abi, part));
 }
 
 /* The scalar type used to move one ABI part through a register. Aggregate
  * args/results are split into parts; each part must move at its own width, not
  * the (possibly >8-byte) aggregate width. */
 static KitCgTypeId aa_part_scalar_type(const ABIArgPart* part) {
   if (part->cls == ABI_CLASS_FP) {
     if (part->size <= 4u) return builtin_id(KIT_CG_BUILTIN_F32);
     if (part->size <= 8u) return builtin_id(KIT_CG_BUILTIN_F64);
     return builtin_id(KIT_CG_BUILTIN_F128);
   }
   switch (part->size) {
     case 1u:
       return builtin_id(KIT_CG_BUILTIN_I8);
     case 2u:
       return builtin_id(KIT_CG_BUILTIN_I16);
     case 4u:
       return builtin_id(KIT_CG_BUILTIN_I32);
     default:
       return builtin_id(KIT_CG_BUILTIN_I64);
   }
 }
 
 static u32 aa_class_vararg_stack_size(const ABIFuncInfo* abi,
                                       const ABIArgInfo* ai) {
   u32 total = 0;
   u32 min_align = aa_vararg_stack_arg_min_align(abi);
   if (!ai || ai->kind == ABI_ARG_IGNORE) return 0;
   if (ai->kind == ABI_ARG_INDIRECT) return 8u;
   for (u32 p = 0; p < ai->nparts; ++p) {
     total = align_up_u32(total, aa_part_vararg_stack_align(abi, &ai->parts[p]));
     total += aa_part_vararg_stack_size(abi, &ai->parts[p]);
   }
   return align_up_u32(total ? total : min_align, min_align);
 }
 
 static u32 aa_call_stack_size(NativeTarget* t, const NativeCallDesc* desc) {
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, desc->fn_type);
   u32 next_int = 0, next_fp = 0, stack = 0;
   for (u32 i = 0; i < desc->nargs; ++i) {
     ABIArgInfo tmp;
     const ABIArgInfo* ai = aa_param_abi(t, abi, desc, i, &tmp);
     int force_stack =
         abi && abi->variadic && abi->vararg_on_stack && i >= abi->nparams;
     if (ai->kind == ABI_ARG_IGNORE) continue;
     if (force_stack) {
       stack = aa_vararg_stack_start(abi, stack);
       stack += aa_class_vararg_stack_size(abi, ai);
       continue;
     }
     if (ai->kind == ABI_ARG_INDIRECT) {
       if (next_int < 8u)
         next_int++;
       else
         stack += 8u;
       continue;
     }
     for (u32 p = 0; p < ai->nparts; ++p) {
       const ABIArgPart* part = &ai->parts[p];
       if (part->cls == ABI_CLASS_FP) {
         if (next_fp < 8u)
           next_fp++;
         else {
           stack = align_up_u32(stack, aa_part_stack_align(abi, part));
           stack += aa_part_stack_size(abi, part);
         }
       } else {
         if (next_int < 8u)
           next_int++;
         else {
           stack = align_up_u32(stack, aa_part_stack_align(abi, part));
           stack += aa_part_stack_size(abi, part);
         }
       }
     }
   }
   return align_up_u32(stack, 16u);
 }
 
 /* Stack-argument bytes a call with `fn_type`'s fixed parameters uses. Reuses
  * aa_call_stack_size by routing the declared params through it (their ABI
  * classification is independent of the actual operand locations, which
  * aa_call_stack_size ignores for register/stack placement). */
 static u32 aa_signature_stack_bytes(NativeTarget* t, KitCgTypeId fn_type,
                                     int* variadic, u32* nparams) {
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, fn_type);
   NativeCallDesc d;
   if (variadic) *variadic = abi ? (int)abi->variadic : 0;
   if (nparams) *nparams = abi ? abi->nparams : 0u;
   memset(&d, 0, sizeof d);
   d.fn_type = fn_type;
   d.nargs = abi ? abi->nparams : 0u;
   if (d.nargs) d.args = arena_zarray(t->c->tu, NativeLoc, d.nargs);
   return aa_call_stack_size(t, &d);
 }
 
 /* Pure NativeTarget.call_stack_bytes: outgoing stack bytes for a full call
  * descriptor (handles variadic stack args, unlike signature_stack_bytes which
  * sees only the fixed params). aa_call_stack_size reads only fn_type and each
  * args[i].type, so the frame-planning pre-pass can call this before emitting.
  */
 static u32 aa_call_stack_bytes(NativeTarget* t, const NativeCallDesc* desc) {
   return aa_call_stack_size(t, desc);
 }
 
 /* One register-passed call argument: write `src` (or its address) into the
  * argument register `dst`. Collected during planning and emitted as a batch so
  * the backend can order them as a parallel copy (see aa_emit_reg_arg_moves). */
 typedef NativeArgMove AAArgMove;
 
 /* AAPCS64/Apple permit at most 8 GP + 8 FP register-passed argument slots. */
 #define AA_MAX_REG_ARG_MOVES 16u
 
 static void aa_emit_one_arg_move(NativeTarget* t, const NativeArgMove* m) {
   if (m->is_addr)
     aa_addr_of_loc(t, m->dst, m->src);
   else
     aa_load_part(t, m->dst, m->src, m->src_offset, m->size);
 }
 
 /* Emit register-argument moves as a parallel copy via the shared scheduler:
  * every register is read by all moves that source it before any move overwrites
  * it; a true cycle is broken through a scratch. The allocator usually arranges
  * a conflict-free order, but not always (notably variadic args, where it can
  * leave a prior call's result in x0 even though x0 is this call's first arg
  * register), so the backend must not assume a safe order. Cycle scratch is
  * AA_TMP1 (x17) for int and v16 for fp — distinct from x16 (AA_TMP0), which may
  * hold a stashed indirect callee. */
 static void aa_emit_reg_arg_moves(NativeTarget* t, NativeArgMove* moves,
                                   u32 n) {
   NativeArgShuffle s;
   if (n > AA_MAX_REG_ARG_MOVES)
     aa_panic(aa_of(t), "too many register arguments");
   memset(&s, 0, sizeof s);
   s.t = t;
   s.emit_one = aa_emit_one_arg_move;
   s.reg_move = aa_move;
   s.scratch[NATIVE_REG_INT] = AA_TMP1;
   s.scratch[NATIVE_REG_FP] = 16u;
   native_arg_shuffle(&s, moves, n);
 }
 
 static void aa_plan_call(NativeTarget* t, const NativeCallDesc* desc,
                          NativeCallPlan* plan) {
   NativeCallPlanRet* rets;
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, desc->fn_type);
   memset(plan, 0, sizeof *plan);
   rets = desc->nresults ? arena_zarray(t->c->tu, NativeCallPlanRet, 4) : NULL;
   plan->callee = desc->callee;
   plan->rets = rets;
   plan->flags = desc->flags;
   plan->has_sret = abi && abi->has_sret;
   plan->is_variadic = abi && abi->variadic;
   plan->stack_arg_size = aa_call_stack_size(t, desc);
   native_frame_note_outgoing(&aa_of(t)->frame, plan->stack_arg_size);
   /* Indirect call whose callee lives in x0..x7: the upcoming arg-load loop
    * writes those same registers and would clobber the function pointer
    * before blr reads it. Stash callee into AA_TMP0 (x16) up front and
    * retarget the call. (AA_TMP0 is a backend scratch, never an arg reg.) */
   if (plan->callee.kind == NATIVE_LOC_REG &&
       (NativeAllocClass)plan->callee.cls == NATIVE_REG_INT &&
       plan->callee.v.reg < 8u) {
     NativeLoc scratch =
         native_loc_reg(plan->callee.type, NATIVE_REG_INT, AA_TMP0);
     aa_move(t, scratch, plan->callee);
     plan->callee = scratch;
   }
   {
     u32 next_int = 0, next_fp = 0, stack = 0, nmoves = 0;
     int tail_call = (desc->flags & CG_CALL_TAIL) != 0;
     AAArgMove moves[AA_MAX_REG_ARG_MOVES];
     /* Stack-passed arguments are stored inline as we walk, *before* any
      * argument register is written, so a stack-arg source that the allocator
      * left in an arg register (e.g. a prior call's result still in x0, consumed
      * as a variadic stack arg) is read while it is still live. Stack stores
      * only touch memory and the AA_TMP0/v16 scratch, never an arg-register
      * source, so emitting them first cannot clobber a register-arg source.
      * Register-passed arguments are collected and emitted afterward as a
      * parallel copy (aa_emit_reg_arg_moves) so they likewise never overwrite a
      * register another argument still needs to read. */
     for (u32 i = 0; i < desc->nargs; ++i) {
       ABIArgInfo tmp;
       const ABIArgInfo* ai = aa_param_abi(t, abi, desc, i, &tmp);
       int force_stack =
           abi && abi->variadic && abi->vararg_on_stack && i >= abi->nparams;
       if (ai->kind == ABI_ARG_IGNORE) continue;
       if (force_stack) {
         NativeLoc tmpreg =
             native_loc_reg(desc->args[i].type, NATIVE_REG_INT, AA_TMP0);
         u32 n = aa_class_vararg_stack_size(abi, ai);
         u32 off = 0;
         stack = aa_vararg_stack_start(abi, stack);
         while (off < n) {
           u32 chunk = (n - off > 8u) ? 8u : (n - off);
           aa_load_part(t, tmpreg, desc->args[i], off, chunk);
           aa_store_outgoing_part(t, tail_call, stack + off, tmpreg, chunk);
           off += chunk;
         }
         stack += n;
         continue;
       }
       if (ai->kind == ABI_ARG_INDIRECT) {
         if (next_int < 8u) {
           AAArgMove* m = &moves[nmoves++];
           m->dst = native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64),
                                   NATIVE_REG_INT, next_int++);
           m->src = desc->args[i];
           m->src_offset = 0;
           m->size = 8;
           m->is_addr = 1;
         } else {
           NativeLoc ptr = native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64),
                                          NATIVE_REG_INT, AA_TMP0);
           aa_addr_of_loc(t, ptr, desc->args[i]);
           aa_store_outgoing_part(t, tail_call, stack, ptr, 8);
           stack += 8u;
         }
         continue;
       }
       for (u32 p = 0; p < ai->nparts; ++p) {
         const ABIArgPart* part = &ai->parts[p];
         NativeAllocClass cls =
             part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
         if ((cls == NATIVE_REG_FP && next_fp < 8u) ||
             (cls == NATIVE_REG_INT && next_int < 8u)) {
           AAArgMove* m = &moves[nmoves++];
           m->dst =
               native_loc_reg(desc->args[i].type, cls,
                              cls == NATIVE_REG_FP ? next_fp++ : next_int++);
           m->src = desc->args[i];
           m->src_offset = part->src_offset;
           m->size = part->size;
           m->is_addr = 0;
         } else {
           NativeLoc tmpreg = native_loc_reg(
               desc->args[i].type, cls, cls == NATIVE_REG_FP ? 16u : AA_TMP0);
           aa_load_part(t, tmpreg, desc->args[i], part->src_offset, part->size);
           stack = align_up_u32(stack, aa_part_stack_align(abi, part));
           aa_store_outgoing_part(t, tail_call, stack, tmpreg, part->size);
           stack += aa_part_stack_size(abi, part);
         }
       }
     }
     aa_emit_reg_arg_moves(t, moves, nmoves);
     /* Set the indirect-result register (x8) *after* the argument loads: an
      * argument source may have been allocated to x8, and the sret pointer load
      * would otherwise clobber it before it is moved into its argument
      * register. */
     if (abi && abi->has_sret) {
       NativeLoc x8 =
           native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT, 8u);
       if (desc->flags & CG_CALL_TAIL) {
         AANativeTarget* a = aa_of(t);
         NativeLoc saved = native_loc_stack(x8.type, a->sret_ptr_slot, 0);
         aa_load_part(t, x8, saved, 0, 8);
       } else if (desc->nresults) {
         aa_addr_of_loc(t, x8, desc->results[0]);
       }
     }
   }
   if (abi && abi->ret.kind == ABI_ARG_DIRECT && desc->nresults) {
     u32 nr = 0, ni = 0, nf = 0;
     for (u32 p = 0; p < abi->ret.nparts; ++p) {
       const ABIArgPart* part = &abi->ret.parts[p];
       NativeAllocClass cls =
           part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
       KitCgTypeId pty = aa_part_scalar_type(part);
       rets[nr].src =
           native_loc_reg(pty, cls, cls == NATIVE_REG_FP ? nf++ : ni++);
       rets[nr].dst = desc->results[0];
       if (rets[nr].dst.kind == NATIVE_LOC_FRAME)
         rets[nr].dst = native_loc_stack(pty, desc->results[0].v.frame,
                                         (i32)part->src_offset);
       else if (rets[nr].dst.kind == NATIVE_LOC_STACK) {
         rets[nr].dst.v.stack.offset += (i32)part->src_offset;
         rets[nr].dst.type = pty;
       } else if (rets[nr].dst.kind == NATIVE_LOC_ADDR) {
         rets[nr].dst.v.addr.offset += (i32)part->src_offset;
         rets[nr].dst.type = pty;
       }
       rets[nr].mem = aa_mem_for_type(t, pty, part->size);
       nr++;
     }
     plan->nrets = nr;
   } else if (abi && abi->ret.kind == ABI_ARG_IGNORE) {
     plan->nrets = 0;
   } else if (!abi && desc->nresults) {
     rets[0].src = native_loc_reg(desc->results[0].type, NATIVE_REG_INT, 0);
     rets[0].dst = desc->results[0];
     rets[0].mem = aa_mem_for_type(t, desc->results[0].type, 0);
     plan->nrets = 1;
   }
 }
 
 static void aa_ret(NativeTarget* t);
 
 static void aa_emit_tail_site(NativeTarget* t, NativeLoc callee) {
   AANativeTarget* a = aa_of(t);
   if (a->frame.known_frame) {
     /* Frame is final: emit the tail epilogue (callee restores + frame restore +
      * branch) directly, exactly the words aa_apply_patches would patch in but
      * without the reserved NOP padding. */
     AAFrameLayout L =
       aa_build_layout(a->frame.cum_off, a->frame.max_outgoing, a->top_home_bytes);
     u32 words[AA_TAIL_WORDS];
     u32 n = 0;
     aa_words_callee_restores(a, words, AA_TAIL_WORDS, &n);
     aa_words_restore_frame(a, words, AA_TAIL_WORDS, &n, &L);
     if (n >= AA_TAIL_WORDS) aa_panic(a, "tail epilogue too large");
     for (u32 i = 0; i < n; ++i) aa_emit32(t->mc, words[i]);
     if (callee.kind == NATIVE_LOC_REG) {
       aa_emit32(t->mc, aa64_br(loc_reg(callee)));
     } else if (callee.kind == NATIVE_LOC_GLOBAL) {
       u32 pos = t->mc->pos(t->mc);
       aa_emit32(t->mc, aa64_b(0));
       t->mc->emit_reloc_at(t->mc, t->mc->section_id, pos, R_AARCH64_JUMP26,
                            callee.v.global.sym, callee.v.global.addend, 0, 0);
     } else {
       aa_panic(a, "unsupported tail target");
     }
     return;
   }
   /* Single-pass: reserve a worst-case region and record a patch; the callee
    * restores and frame restore depend on the not-yet-final frame layout. */
   AAPatch* p = aa_patch_alloc(a);
   p->kind = AA_PATCH_TAIL;
   p->pos = t->mc->pos(t->mc);
   p->u.callee = callee;
   for (u32 i = 0; i < AA_TAIL_WORDS; ++i) aa_emit32(t->mc, 0xd503201fu);
   if (callee.kind == NATIVE_LOC_GLOBAL) {
     t->mc->emit_reloc_at(t->mc, t->mc->section_id,
                          p->pos + (AA_TAIL_WORDS - 1u) * 4u, R_AARCH64_JUMP26,
                          callee.v.global.sym, callee.v.global.addend, 0, 0);
   }
 }
 
 static void aa_emit_call(NativeTarget* t, const NativeCallPlan* plan) {
   int is_tail = (plan->flags & CG_CALL_TAIL) != 0;
   if (is_tail) {
     if (plan->callee.kind != NATIVE_LOC_GLOBAL &&
         plan->callee.kind != NATIVE_LOC_REG)
       aa_panic(aa_of(t), "unsupported tail target");
     aa_emit_tail_site(t, plan->callee);
     return;
   }
   if (plan->callee.kind == NATIVE_LOC_GLOBAL) {
     aa_emit32(t->mc, aa64_bl(0));
     t->mc->emit_reloc_at(t->mc, t->mc->section_id, t->mc->pos(t->mc) - 4u,
                          R_AARCH64_CALL26, plan->callee.v.global.sym,
                          plan->callee.v.global.addend, 0, 0);
     return;
   }
   if (plan->callee.kind == NATIVE_LOC_REG) {
     aa_emit32(t->mc, aa64_blr(loc_reg(plan->callee)));
     return;
   }
   aa_panic(aa_of(t), "unsupported call target");
 }
 
 static void aa_plan_ret(NativeTarget* t, const CGFuncDesc* fd,
                         const NativeLoc* value,
                         NativeCallPlanRet** out_rets, u32* out_nrets) {
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, fd->fn_type);
   NativeCallPlanRet* rets = NULL;
   u32 nr = 0;
   if (value) rets = arena_zarray(t->c->tu, NativeCallPlanRet, 4);
   if (value && abi && abi->ret.kind == ABI_ARG_INDIRECT) {
     AANativeTarget* a = aa_of(t);
     /* Hold the sret destination pointer in x8, not AA_TMP1: aa_copy_bytes
      * materializes out-of-range source/dest frame offsets into AA_TMP1, which
      * would clobber the destination base mid-copy (only triggered once a frame
      * is large enough that the source offset escapes stur's signed-9 range). */
     NativeLoc dstp =
         native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT, AA_X8);
     NativeLoc saved = native_loc_stack(dstp.type, a->sret_ptr_slot, 0);
     NativeAddr dst_addr, src_addr;
     AggregateAccess access;
     aa_load_part(t, dstp, saved, 0, 8);
     memset(&dst_addr, 0, sizeof dst_addr);
     dst_addr.base_kind = NATIVE_ADDR_BASE_REG;
     dst_addr.base.reg = AA_X8;
     dst_addr.base_type = value->type;
     src_addr = aa_loc_addr(a, *value, 0);
     src_addr.base_type = value->type;
     memset(&access, 0, sizeof access);
     access.type = value->type;
     access.size = (u32)cg_type_size(t->c, value->type);
     access.align = type_align32(t, value->type);
     aa_copy_bytes(t, dst_addr, src_addr, access);
     *out_rets = NULL;
     *out_nrets = 0;
     return;
   }
   if (value && abi && abi->ret.kind == ABI_ARG_DIRECT) {
     u32 ni = 0, nf = 0;
     for (u32 p = 0; p < abi->ret.nparts; ++p) {
       const ABIArgPart* part = &abi->ret.parts[p];
       NativeAllocClass cls =
           part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
       KitCgTypeId pty = aa_part_scalar_type(part);
       rets[nr].src = *value;
       if (rets[nr].src.kind == NATIVE_LOC_FRAME)
         rets[nr].src =
             native_loc_stack(pty, value->v.frame, (i32)part->src_offset);
       else if (rets[nr].src.kind == NATIVE_LOC_STACK) {
         rets[nr].src.v.stack.offset += (i32)part->src_offset;
         rets[nr].src.type = pty;
       } else if (rets[nr].src.kind == NATIVE_LOC_ADDR) {
         rets[nr].src.v.addr.offset += (i32)part->src_offset;
         rets[nr].src.type = pty;
       }
       rets[nr].dst =
           native_loc_reg(pty, cls, cls == NATIVE_REG_FP ? nf++ : ni++);
       rets[nr].mem = aa_mem_for_type(t, pty, part->size);
       nr++;
     }
   } else if (value) {
     rets[0].src = *value;
     rets[0].dst = native_loc_reg(value->type, NATIVE_REG_INT, 0);
     rets[0].mem = aa_mem_for_type(t, value->type, 0);
     nr = 1;
   }
   *out_rets = rets;
   *out_nrets = nr;
 }
 
 static void aa_ret(NativeTarget* t) {
   AANativeTarget* a = aa_of(t);
   aa_jump(t, a->epilogue_label);
 }
 
 static u32 aa_bit_storage_reg_bits(u32 storage_bytes) {
   return storage_bytes == 8u ? 64u : 32u;
 }
 
 static void aa_lsl_imm(NativeTarget* t, u32 sf, u32 rd, u32 rn, u32 sh) {
   u32 bits = sf ? 64u : 32u;
   if (!sh) {
     if (rd != rn) aa_emit32(t->mc, aa64_mov_reg(sf, rd, rn));
     return;
   }
   aa_emit32(t->mc, aa_ubfm(sf, rd, rn, bits - sh, bits - 1u - sh));
 }
 
 static void aa_lsr_imm(NativeTarget* t, u32 sf, u32 rd, u32 rn, u32 sh) {
   if (!sh) {
     if (rd != rn) aa_emit32(t->mc, aa64_mov_reg(sf, rd, rn));
     return;
   }
   aa_emit32(t->mc, aa_ubfm(sf, rd, rn, sh, sf ? 63u : 31u));
 }
 
 static void aa_asr_imm(NativeTarget* t, u32 sf, u32 rd, u32 rn, u32 sh) {
   if (!sh) {
     if (rd != rn) aa_emit32(t->mc, aa64_mov_reg(sf, rd, rn));
     return;
   }
   aa_emit32(t->mc, aa_sbfm(sf, rd, rn, sh, sf ? 63u : 31u));
 }
 
 static void aa_bitfield_load(NativeTarget* t, NativeLoc dst, NativeAddr addr,
                              BitFieldAccess bf) {
   u32 storage = bf.storage.size ? bf.storage.size : 4u;
   u32 bits = aa_bit_storage_reg_bits(storage);
   u32 width = bf.bit_width ? bf.bit_width : 1u;
   u32 sf = bits == 64u;
   NativeAddr saddr = aa_addr_plus(addr, bf.storage_offset);
   NativeLoc tmp = dst;
   tmp.type = bf.storage.type ? bf.storage.type : dst.type;
   aa_load_native(t, tmp, saddr, bf.storage);
   aa_lsl_imm(t, sf, loc_reg(dst), loc_reg(dst),
              bits - (u32)bf.bit_offset - width);
   if (bf.signed_)
     aa_asr_imm(t, sf, loc_reg(dst), loc_reg(dst), bits - width);
   else
     aa_lsr_imm(t, sf, loc_reg(dst), loc_reg(dst), bits - width);
 }
 
 static void aa_bitfield_store(NativeTarget* t, NativeAddr addr, NativeLoc src,
                               BitFieldAccess bf) {
   u32 storage = bf.storage.size ? bf.storage.size : 4u;
   u32 bits = aa_bit_storage_reg_bits(storage);
   u32 width = bf.bit_width ? bf.bit_width : 1u;
   u32 sf = bits == 64u;
   u64 ones = width >= 64u ? ~(u64)0 : ((1ull << width) - 1ull);
   u64 field_mask = ones << bf.bit_offset;
   NativeAddr saddr = aa_addr_plus(addr, bf.storage_offset);
   NativeLoc word =
       aa_tmp_loc(bf.storage.type ? bf.storage.type : src.type, AA_TMP0);
   aa_load_native(t, word, saddr, bf.storage);
   aa_emit_load_imm(t->mc, sf, AA_TMP1, (i64)~field_mask);
   aa_emit32(t->mc, aa64_and(sf, AA_TMP0, AA_TMP0, AA_TMP1));
   aa_emit32(t->mc, aa_ubfm(sf, AA_TMP1, loc_reg(src), 0, width - 1u));
   aa_lsl_imm(t, sf, AA_TMP1, AA_TMP1, bf.bit_offset);
   aa_emit32(t->mc, aa64_orr(sf, AA_TMP0, AA_TMP0, AA_TMP1));
   aa_store_native(t, saddr, word, bf.storage);
 }
 
 static void aa_trap(NativeTarget* t);
 
 static int aa_order_acquire(KitCgMemOrder order) {
   return order == KIT_CG_MO_CONSUME || order == KIT_CG_MO_ACQUIRE ||
          order == KIT_CG_MO_ACQ_REL || order == KIT_CG_MO_SEQ_CST;
 }
 
 static int aa_order_release(KitCgMemOrder order) {
   return order == KIT_CG_MO_RELEASE || order == KIT_CG_MO_ACQ_REL ||
          order == KIT_CG_MO_SEQ_CST;
 }
 
 static NativeLoc aa_i64_reg_loc(u32 reg) {
   return native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT, reg);
 }
 
 static void aa_atomic_addr_reg(NativeTarget* t, NativeAddr addr, u32 reg) {
   NativeLoc dst = aa_i64_reg_loc(reg);
   t->load_addr(t, dst, addr);
 }
 
 static u32 aa_saved_tmp_pick(u32 a, u32 b, u32 c) {
   static const u32 regs[] = {11u, 12u, 13u, 14u, 15u};
   for (u32 i = 0; i < sizeof regs / sizeof regs[0]; ++i) {
     if (regs[i] != a && regs[i] != b && regs[i] != c) return regs[i];
   }
   return 15u;
 }
 
 static void aa_saved_tmp_spill(AANativeTarget* a, u32 reg) {
   NativeFrameSlotDesc sd;
   NativeAddr addr;
   MemAccess mem;
   memset(&sd, 0, sizeof sd);
   if (a->saved_tmp_slot == NATIVE_FRAME_SLOT_NONE) {
     sd.type = builtin_id(KIT_CG_BUILTIN_I64);
     sd.size = 8;
     sd.align = 8;
     sd.kind = NATIVE_FRAME_SLOT_SPILL;
     a->saved_tmp_slot = a->base.frame_slot(&a->base, &sd);
   }
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = a->saved_tmp_slot;
   addr.base_type = builtin_id(KIT_CG_BUILTIN_I64);
   mem = aa_mem_for_type(&a->base, addr.base_type, 8);
   aa_store_native(&a->base, addr, aa_i64_reg_loc(reg), mem);
 }
 
 static void aa_saved_tmp_restore(AANativeTarget* a, u32 reg) {
   NativeAddr addr;
   MemAccess mem;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = a->saved_tmp_slot;
   addr.base_type = builtin_id(KIT_CG_BUILTIN_I64);
   mem = aa_mem_for_type(&a->base, addr.base_type, 8);
   aa_load_native(&a->base, aa_i64_reg_loc(reg), addr, mem);
 }
 
 static void aa_atomic_load(NativeTarget* t, NativeLoc dst, NativeAddr addr,
                            MemAccess mem, KitCgMemOrder order) {
   u32 base = AA_TMP0;
   u32 sz = size_idx(mem.size ? mem.size : type_size32(t, dst.type));
   aa_atomic_addr_reg(t, addr, base);
   aa_emit32(t->mc, aa_order_acquire(order)
                        ? aa_ldar(sz, loc_reg(dst), base)
                        : aa_ldr_uimm(sz, loc_reg(dst), base, 0));
   if (order == KIT_CG_MO_SEQ_CST)
     aa_emit32(t->mc, aa64_dmb(AA64_BARRIER_OPT_ISH));
 }
 
 static void aa_atomic_store(NativeTarget* t, NativeAddr addr, NativeLoc src,
                             MemAccess mem, KitCgMemOrder order) {
   u32 base = AA_TMP0;
   u32 sz = size_idx(mem.size ? mem.size : type_size32(t, src.type));
   if (order == KIT_CG_MO_SEQ_CST)
     aa_emit32(t->mc, aa64_dmb(AA64_BARRIER_OPT_ISH));
   aa_atomic_addr_reg(t, addr, base);
   aa_emit32(t->mc, aa_order_release(order)
                        ? aa_stlr(sz, loc_reg(src), base)
                        : aa_str_uimm(sz, loc_reg(src), base, 0));
   if (order == KIT_CG_MO_SEQ_CST)
     aa_emit32(t->mc, aa64_dmb(AA64_BARRIER_OPT_ISH));
 }
 
 static void aa_atomic_rmw(NativeTarget* t, KitCgAtomicOp op, NativeLoc dst,
                           NativeAddr addr, NativeLoc val, MemAccess mem,
                           KitCgMemOrder order) {
   AANativeTarget* a = aa_of(t);
   u32 base = AA_TMP0;
   u32 next_reg = AA_TMP1;
   u32 status = aa_saved_tmp_pick(loc_reg(dst), loc_reg(val), base);
   NativeLoc next = aa_tmp_loc(dst.type, next_reg);
   MCLabel retry = t->mc->label_new(t->mc);
   u32 sz = size_idx(mem.size ? mem.size : type_size32(t, dst.type));
   if (order == KIT_CG_MO_SEQ_CST)
     aa_emit32(t->mc, aa64_dmb(AA64_BARRIER_OPT_ISH));
   aa_saved_tmp_spill(a, status);
   aa_atomic_addr_reg(t, addr, base);
   t->mc->label_place(t->mc, retry);
   aa_emit32(t->mc, aa_order_acquire(order) ? aa_ldaxr(sz, loc_reg(dst), base)
                                            : aa_ldxr(sz, loc_reg(dst), base));
   switch (op) {
     case KIT_CG_ATOMIC_XCHG:
       aa_move(t, next, val);
       break;
     case KIT_CG_ATOMIC_ADD:
       aa_binop(t, BO_IADD, next, dst, val);
       break;
     case KIT_CG_ATOMIC_SUB:
       aa_binop(t, BO_ISUB, next, dst, val);
       break;
     case KIT_CG_ATOMIC_AND:
       aa_binop(t, BO_AND, next, dst, val);
       break;
     case KIT_CG_ATOMIC_OR:
       aa_binop(t, BO_OR, next, dst, val);
       break;
     case KIT_CG_ATOMIC_XOR:
       aa_binop(t, BO_XOR, next, dst, val);
       break;
     case KIT_CG_ATOMIC_NAND:
       aa_binop(t, BO_AND, next, dst, val);
       aa_unop(t, UO_BNOT, next, next);
       break;
     default:
       aa_panic(a, "unsupported atomic rmw op");
   }
   aa_emit32(t->mc, aa_order_release(order)
                        ? aa_stlxr(sz, status, next_reg, base)
                        : aa_stxr(sz, status, next_reg, base));
   aa_emit32(t->mc, aa64_cbnz_imm(0, status, 0));
   t->mc->emit_label_ref(t->mc, retry, R_AARCH64_CONDBR19, 4, 0);
   aa_saved_tmp_restore(a, status);
   if (order == KIT_CG_MO_SEQ_CST)
     aa_emit32(t->mc, aa64_dmb(AA64_BARRIER_OPT_ISH));
 }
 
 static void aa_atomic_cas(NativeTarget* t, NativeLoc prior, NativeLoc ok,
                           NativeAddr addr, NativeLoc expected,
                           NativeLoc desired, MemAccess mem,
                           KitCgMemOrder success, KitCgMemOrder failure) {
   u32 base = AA_TMP0;
   u32 status = AA_TMP1;
   u32 sz = size_idx(mem.size ? mem.size : type_size32(t, prior.type));
   u32 sf = sz == 3u;
   int acquire = aa_order_acquire(success) || aa_order_acquire(failure);
   int release = aa_order_release(success);
   MCLabel retry = t->mc->label_new(t->mc);
   MCLabel fail = t->mc->label_new(t->mc);
   MCLabel done = t->mc->label_new(t->mc);
   if (success == KIT_CG_MO_SEQ_CST || failure == KIT_CG_MO_SEQ_CST)
     aa_emit32(t->mc, aa64_dmb(AA64_BARRIER_OPT_ISH));
   aa_atomic_addr_reg(t, addr, base);
   t->mc->label_place(t->mc, retry);
   aa_emit32(t->mc, acquire ? aa_ldaxr(sz, loc_reg(prior), base)
                            : aa_ldxr(sz, loc_reg(prior), base));
   aa_emit32(t->mc, aa_subs_reg(sf, AA64_ZR, loc_reg(prior), loc_reg(expected)));
   aa_emit32(t->mc, aa64_brcond_pack((AA64BrCond){.cond = cmp_cond(CMP_NE)}));
   t->mc->emit_label_ref(t->mc, fail, R_AARCH64_CONDBR19, 4, 0);
   aa_emit32(t->mc, release ? aa_stlxr(sz, status, loc_reg(desired), base)
                            : aa_stxr(sz, status, loc_reg(desired), base));
   aa_emit32(t->mc, aa64_cbnz_imm(0, status, 0));
   t->mc->emit_label_ref(t->mc, retry, R_AARCH64_CONDBR19, 4, 0);
   aa_emit_load_imm(t->mc, loc_is_64(t, ok), loc_reg(ok), 1);
   aa_jump(t, done);
   t->mc->label_place(t->mc, fail);
   aa_emit32(t->mc, aa64_clrex(AA64_BARRIER_OPT_SY));
   aa_emit_load_imm(t->mc, loc_is_64(t, ok), loc_reg(ok), 0);
   t->mc->label_place(t->mc, done);
   if (success == KIT_CG_MO_SEQ_CST || failure == KIT_CG_MO_SEQ_CST)
     aa_emit32(t->mc, aa64_dmb(AA64_BARRIER_OPT_ISH));
 }
 
 static void aa_fence(NativeTarget* t, KitCgMemOrder order) {
   if (order != KIT_CG_MO_RELAXED)
     aa_emit32(t->mc, aa64_dmb(AA64_BARRIER_OPT_ISH));
 }
 
 /* Map a KitCgBarrierScope (passed as an immediate arg to DMB/DSB) onto an
  * AArch64 barrier domain option. Defaults to full-system (SY) when the scope
  * is absent or unrecognized. */
 static u32 aa_barrier_opt(const NativeLoc* args, u32 narg) {
   if (narg < 1u || args[0].kind != NATIVE_LOC_IMM) return AA64_BARRIER_OPT_SY;
   switch ((KitCgBarrierScope)args[0].v.imm) {
     case KIT_CG_BARRIER_FULL:
       return AA64_BARRIER_OPT_SY;
     case KIT_CG_BARRIER_INNER:
       return AA64_BARRIER_OPT_ISH;
     case KIT_CG_BARRIER_INNER_STORE:
       return AA64_BARRIER_OPT_ISHST;
     case KIT_CG_BARRIER_OUTER:
       return AA64_BARRIER_OPT_OSH;
     case KIT_CG_BARRIER_OUTER_STORE:
       return AA64_BARRIER_OPT_OSHST;
     case KIT_CG_BARRIER_NON_SHARE:
       return AA64_BARRIER_OPT_NSH;
   }
   return AA64_BARRIER_OPT_SY;
 }
 
 static void aa_intrinsic(NativeTarget* t, IntrinKind kind,
                          const NativeLoc* dsts, u32 ndst, const NativeLoc* args,
                          u32 narg) {
   AggregateAccess access;
   NativeAddr dst_addr;
   NativeAddr src_addr;
   memset(&access, 0, sizeof access);
   memset(&dst_addr, 0, sizeof dst_addr);
   memset(&src_addr, 0, sizeof src_addr);
   switch (kind) {
     case INTRIN_NONE:
       if (ndst == 1u && narg == 3u && native_loc_is_fp(dsts[0])) {
         u32 d = type_size32(t, dsts[0].type) == 8u;
         aa_emit32(t->mc, aa_fp_bin(0x000800u, d, loc_reg(dsts[0]),
                                    loc_reg(args[0]), loc_reg(args[1])));
         aa_emit32(t->mc, aa_fp_bin(0x002800u, d, loc_reg(dsts[0]),
                                    loc_reg(dsts[0]), loc_reg(args[2])));
         return;
       }
       break;
     case INTRIN_CLZ:
       if (ndst == 1u && narg == 1u) {
         aa_emit32(t->mc, aa_clz(loc_is_64(t, args[0]), loc_reg(dsts[0]),
                                 loc_reg(args[0])));
         return;
       }
       break;
     case INTRIN_CTZ:
       if (ndst == 1u && narg == 1u) {
         u32 sf = loc_is_64(t, args[0]);
         aa_emit32(t->mc, aa_rbit(sf, loc_reg(dsts[0]), loc_reg(args[0])));
         aa_emit32(t->mc, aa_clz(sf, loc_reg(dsts[0]), loc_reg(dsts[0])));
         return;
       }
       break;
     case INTRIN_POPCOUNT:
       if (ndst == 1u && narg == 1u) {
         u32 sf = loc_is_64(t, args[0]);
         u32 rd = loc_reg(dsts[0]);
         u32 rn = loc_reg(args[0]);
         MCLabel loop = t->mc->label_new(t->mc);
         MCLabel done = t->mc->label_new(t->mc);
         aa_emit_load_imm(t->mc, sf, rd, 0);
         aa_emit32(t->mc, aa64_mov_reg(sf, AA_TMP0, rn));
         t->mc->label_place(t->mc, loop);
         aa_emit32(t->mc, aa64_cbz(sf, AA_TMP0, 0));
         t->mc->emit_label_ref(t->mc, done, R_AARCH64_CONDBR19, 4, 0);
         aa_emit_load_imm(t->mc, sf, AA_TMP1, 1);
         aa_emit32(t->mc, aa64_and(sf, AA_TMP1, AA_TMP0, AA_TMP1));
         aa_emit32(t->mc, aa64_add(sf, rd, rd, AA_TMP1));
         aa_emit_load_imm(t->mc, sf, AA_TMP1, 1);
         aa_emit32(t->mc, aa64_lsrv(sf, AA_TMP0, AA_TMP0, AA_TMP1));
         aa_jump(t, loop);
         t->mc->label_place(t->mc, done);
         return;
       }
       break;
     case INTRIN_BSWAP:
       if (ndst == 1u && narg == 1u) {
         u32 width = abi_cg_sizeof(t->c->abi, dsts[0].type);
         switch (width) {
           case 2: {
             u32 sf = 0;
             aa_emit32(t->mc, aa_rev(sf, loc_reg(dsts[0]), loc_reg(args[0])));
             aa_emit_load_imm(t->mc, 0, AA_TMP0, 16);
             aa_emit32(t->mc, aa64_lsrv(0, loc_reg(dsts[0]), loc_reg(dsts[0]),
                                        AA_TMP0));
             return;
           }
           case 4: {
             u32 sf = 0;
             aa_emit32(t->mc, aa_rev(sf, loc_reg(dsts[0]), loc_reg(args[0])));
             return;
           }
           case 8: {
             u32 sf = 1;
             aa_emit32(t->mc, aa_rev(sf, loc_reg(dsts[0]), loc_reg(args[0])));
             return;
           }
           default:
             break;
         }
       }
       break;
     case INTRIN_SADD_OVERFLOW:
     case INTRIN_UADD_OVERFLOW:
     case INTRIN_SSUB_OVERFLOW:
     case INTRIN_USUB_OVERFLOW:
       if (ndst == 2u && narg == 2u) {
         u32 sf = loc_is_64(t, dsts[0]);
         u32 rd = loc_reg(dsts[0]);
         if (kind == INTRIN_SADD_OVERFLOW || kind == INTRIN_UADD_OVERFLOW)
           aa_emit32(t->mc,
                     aa64_addsubsr_pack((AA64AddSubSR){.sf = sf,
                                                       .op = 0,
                                                       .S = 1,
                                                       .Rm = loc_reg(args[1]),
                                                       .Rn = loc_reg(args[0]),
                                                       .Rd = rd}));
         else
           aa_emit32(t->mc,
                     aa64_addsubsr_pack((AA64AddSubSR){.sf = sf,
                                                       .op = 1,
                                                       .S = 1,
                                                       .Rm = loc_reg(args[1]),
                                                       .Rn = loc_reg(args[0]),
                                                       .Rd = rd}));
         aa_emit32(t->mc,
                   aa_cset(loc_is_64(t, dsts[1]), loc_reg(dsts[1]),
                           (kind == INTRIN_SADD_OVERFLOW ||
                            kind == INTRIN_SSUB_OVERFLOW)
                               ? 0x6u
                               : (kind == INTRIN_UADD_OVERFLOW ? 0x2u : 0x3u)));
         return;
       }
       break;
     case INTRIN_SMUL_OVERFLOW:
     case INTRIN_UMUL_OVERFLOW:
       if (ndst == 2u && narg == 2u) {
         u32 sf = loc_is_64(t, dsts[0]);
         if (sf) {
           if (kind == INTRIN_SMUL_OVERFLOW) {
             aa_emit32(t->mc,
                       aa_smulh(AA_TMP0, loc_reg(args[0]), loc_reg(args[1])));
             aa_emit32(t->mc, aa64_mul(1, loc_reg(dsts[0]), loc_reg(args[0]),
                                       loc_reg(args[1])));
             aa_emit32(t->mc, aa_sbfm(1, AA_TMP1, loc_reg(dsts[0]), 63, 63));
             aa_emit32(t->mc, aa_subs_reg(1, AA64_ZR, AA_TMP0, AA_TMP1));
             aa_emit32(t->mc, aa_cset(0, loc_reg(dsts[1]), cmp_cond(CMP_NE)));
           } else {
             aa_emit32(t->mc,
                       aa_umulh(AA_TMP0, loc_reg(args[0]), loc_reg(args[1])));
             aa_emit32(t->mc, aa64_mul(1, loc_reg(dsts[0]), loc_reg(args[0]),
                                       loc_reg(args[1])));
             aa_emit32(t->mc, aa_subs_reg(1, AA64_ZR, AA_TMP0, AA64_ZR));
             aa_emit32(t->mc, aa_cset(0, loc_reg(dsts[1]), cmp_cond(CMP_NE)));
           }
         } else if (kind == INTRIN_SMUL_OVERFLOW) {
           aa_emit32(t->mc, aa_smaddl(AA_TMP0, loc_reg(args[0]),
                                      loc_reg(args[1]), AA64_ZR));
           aa_emit32(t->mc, aa64_mov_reg(0, loc_reg(dsts[0]), AA_TMP0));
           aa_emit32(t->mc, aa_sbfm(1, AA_TMP1, loc_reg(dsts[0]), 0, 31));
           aa_emit32(t->mc, aa_subs_reg(1, AA64_ZR, AA_TMP0, AA_TMP1));
           aa_emit32(t->mc, aa_cset(0, loc_reg(dsts[1]), cmp_cond(CMP_NE)));
         } else {
           aa_emit32(t->mc, aa_umaddl(AA_TMP0, loc_reg(args[0]),
                                      loc_reg(args[1]), AA64_ZR));
           aa_emit32(t->mc, aa64_mov_reg(0, loc_reg(dsts[0]), AA_TMP0));
           aa_emit_load_imm(t->mc, 1, AA_TMP1, 32);
           aa_emit32(t->mc, aa64_lsrv(1, AA_TMP1, AA_TMP0, AA_TMP1));
           aa_emit32(t->mc, aa_subs_reg(1, AA64_ZR, AA_TMP1, AA64_ZR));
           aa_emit32(t->mc, aa_cset(0, loc_reg(dsts[1]), cmp_cond(CMP_NE)));
         }
         return;
       }
       break;
     case INTRIN_MEMMOVE: {
       MCLabel forward = t->mc->label_new(t->mc);
       MCLabel done = t->mc->label_new(t->mc);
       if (narg != 3u || args[0].kind != NATIVE_LOC_REG ||
           args[1].kind != NATIVE_LOC_REG || args[2].kind != NATIVE_LOC_IMM)
         aa_panic(aa_of(t), "unsupported memory intrinsic operands");
       if (args[2].v.imm < 0 || args[2].v.imm > 0xffffffffll)
         aa_panic(aa_of(t), "unsupported memory intrinsic size");
       access.size = (u32)args[2].v.imm;
       access.align = 1u;
       dst_addr.base_kind = NATIVE_ADDR_BASE_REG;
       dst_addr.base.reg = args[0].v.reg;
       src_addr.base_kind = NATIVE_ADDR_BASE_REG;
       src_addr.base.reg = args[1].v.reg;
       aa_emit32(t->mc, aa_subs_reg(1, AA64_ZR, args[0].v.reg, args[1].v.reg));
       aa_emit32(t->mc,
                 aa64_brcond_pack((AA64BrCond){.cond = cmp_cond(CMP_LT_U)}));
       t->mc->emit_label_ref(t->mc, forward, R_AARCH64_CONDBR19, 4, 0);
       aa_copy_bytes_dir(t, dst_addr, src_addr, access, 1);
       aa_jump(t, done);
       t->mc->label_place(t->mc, forward);
       aa_copy_bytes_dir(t, dst_addr, src_addr, access, 0);
       t->mc->label_place(t->mc, done);
       return;
     }
     case INTRIN_EXPECT:
     case INTRIN_ASSUME_ALIGNED:
       if (ndst == 1u && narg >= 1u) {
         if (args[0].kind == NATIVE_LOC_IMM)
           aa_load_imm_native(t, dsts[0], args[0].v.imm);
         else
           aa_move(t, dsts[0], args[0]);
       }
       return;
     case INTRIN_PREFETCH:
       return;
     case INTRIN_TRAP:
       aa_trap(t);
       return;
     case INTRIN_SYSCALL:
       if (ndst == 1u && narg >= 1u && narg <= 7u) {
         static const u32 syscall_regs[7] = {AA_X8, 0u, 1u, 2u, 3u, 4u, 5u};
         AAArgMove moves[7];
         for (u32 i = 0; i < narg; ++i) {
           AAArgMove* m = &moves[i];
           memset(m, 0, sizeof *m);
           m->dst =
               native_loc_reg(dsts[0].type, NATIVE_REG_INT, syscall_regs[i]);
           m->src = args[i];
           m->size = t->c->target.ptr_size;
         }
         aa_emit_reg_arg_moves(t, moves, narg);
         aa_emit32(t->mc, aa64_svc(0));
         aa_move(t, dsts[0], native_loc_reg(dsts[0].type, NATIVE_REG_INT, 0));
       }
       return;
     case INTRIN_CPU_NOP:
       aa_emit32(t->mc, aa64_hint(AA64_HINT_OP_NOP));
       return;
     case INTRIN_CPU_YIELD:
       aa_emit32(t->mc, aa64_hint(AA64_HINT_OP_YIELD));
       return;
     case INTRIN_WFI:
       aa_emit32(t->mc, aa64_hint(AA64_HINT_OP_WFI));
       return;
     case INTRIN_WFE:
       aa_emit32(t->mc, aa64_hint(AA64_HINT_OP_WFE));
       return;
     case INTRIN_SEV:
       aa_emit32(t->mc, aa64_hint(AA64_HINT_OP_SEV));
       return;
     case INTRIN_ISB:
       aa_emit32(t->mc, aa64_isb(AA64_BARRIER_OPT_SY));
       return;
     case INTRIN_DMB:
       aa_emit32(t->mc, aa64_dmb(aa_barrier_opt(args, narg)));
       return;
     case INTRIN_DSB:
       aa_emit32(t->mc, aa64_dsb(aa_barrier_opt(args, narg)));
       return;
     case INTRIN_IRQ_SAVE:
       /* Read the interrupt-mask state, then mask D,A,I,F. */
       if (ndst == 1u) {
         aa_emit32(t->mc, aa64_mrs_daif(loc_reg(dsts[0])));
         aa_emit32(t->mc, aa64_msr_daifset(AA64_DAIF_ALL));
       }
       return;
     case INTRIN_IRQ_RESTORE:
       if (narg == 1u) aa_emit32(t->mc, aa64_msr_daif(loc_reg(args[0])));
       return;
     case INTRIN_IRQ_DISABLE:
       aa_emit32(t->mc, aa64_msr_daifset(AA64_DAIF_ALL));
       return;
     case INTRIN_IRQ_ENABLE:
       aa_emit32(t->mc, aa64_msr_daifclr(AA64_DAIF_ALL));
       return;
     case INTRIN_FRAME_ADDRESS:
     case INTRIN_RETURN_ADDRESS:
       /* Walk the AAPCS64 frame-record chain. Every kit prologue stores
        * {x29, x30} and anchors x29 at the record: [x29] = caller's x29,
        * [x29 + 8] = saved x30 (this frame's return address). The level is a
        * compile-time constant, so the walk unrolls to `level` dependent loads. */
       if (ndst == 1u) {
         u32 level = (narg >= 1u && args[0].kind == NATIVE_LOC_IMM)
                         ? (u32)args[0].v.imm
                         : 0u;
         u32 rd = loc_reg(dsts[0]);
         aa_emit32(t->mc, aa64_mov_reg(1, rd, AA_FP));
         for (u32 i = 0; i < level; ++i)
           aa_emit32(t->mc, aa64_ldr64_uimm12(rd, rd, 0)); /* rd = *(rd) */
         if (kind == INTRIN_RETURN_ADDRESS)
           aa_emit32(t->mc, aa64_ldr64_uimm12(rd, rd, 1)); /* rd = *(rd + 8) */
       }
       return;
     default:
       aa_panic(aa_of(t), "unsupported compiler intrinsic");
   }
 }
 
 static void aa_trap(NativeTarget* t) { aa_emit32(t->mc, aa64_brk(0)); }
 
 /* file_scope_asm + finalize are shared (cg/native_asm.h). */
 
 static int aa_machine_op_clobbers(NativeTarget* t, const NativeMachineOp* op,
                                   u32 mask[NATIVE_CALL_PLAN_CLASSES]) {
   mask[0] = mask[1] = mask[2] = 0;
   if ((NativeMachineOpKind)op->kind == NATIVE_MOP_TLS_ADDR) {
     /* ELF Local-Exec materializes the address using only the destination
      * register (mrs tpidr_el0 + add/add into rd) — no extra clobbers. The
      * Mach-O TLV sequence loads the descriptor into x0 and calls the resolver
      * thunk through x16, clobbering x0/x16/x17 and the link register; the JIT
      * relaxation of that same sequence keeps the x0/x16/x17 footprint. Model
      * the descriptor-model clobbers so a value live across a TLS access is not
      * left in one of these registers. */
     if (!obj_format_tls_via_descriptor(t->c)) return 0;
     mask[NATIVE_REG_INT] =
         (1u << 0) | (1u << 16) | (1u << 17) | (1u << AA_LR);
     return 1;
   }
   if ((NativeMachineOpKind)op->kind != NATIVE_MOP_INTRINSIC ||
       (IntrinKind)op->intrin != INTRIN_SYSCALL)
     return 0;
   mask[NATIVE_REG_INT] = (1u << 0) | (1u << 1) | (1u << 2) | (1u << 3) |
                          (1u << 4) | (1u << 5) | (1u << AA_X8);
   return 1;
 }
 
 static void aa_set_loc(NativeTarget* t, SrcLoc loc) {
   AANativeTarget* a = aa_of(t);
   a->loc = loc;
   if (t->mc && t->mc->set_loc) t->mc->set_loc(t->mc, loc);
 }
 
 static void aa_bind_native_param(NativeTarget* t, const CGParamDesc* p,
                                  NativeLoc dst);
 
 /* Caller-saved allocables come first so the allocator prefers them (lower
  * spill_cost); callee-saved x19..x28 / v8..v15 are appended and only chosen
  * under register pressure, after which the prologue saves/restores them. */
 static const Reg aa_int_allocable[] = {8u,  11u, 12u, 13u, 14u, 15u, 19u, 20u,
                                        21u, 22u, 23u, 24u, 25u, 26u, 27u, 28u};
 /* Three int scratch registers, not two: a 3-operand op (e.g. `binop dst, a, b`
  * or `store [base+index], value`) whose dst/sources all spill needs three
  * distinct scratch regs at emit time — the IR spill-rewrite round-robins
  * operands across this pool and the native emitter materializes each into one.
  * With only two, an immediate operand of an all-spilled binop had nowhere to
  * land. x9/x10/x11 are all caller-saved temporaries reserved out of the
  * allocable set below. */
 static const Reg aa_int_scratch[] = {9u, 10u, 11u};
 static const Reg aa_fp_allocable[] = {18u, 19u, 8u,  9u,  10u,
                                       11u, 12u, 13u, 14u, 15u};
 static const Reg aa_fp_scratch[] = {20u, 21u};
 
 #define AA_PHYS_INT_ALLOC(r)                                \
   {.reg = (r),                                              \
    .cls = NATIVE_REG_INT,                                   \
    .abi_index = 0xffu,                                      \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLER_SAVED, \
    .spill_cost = 1u,                                        \
    .copy_cost = 1u}
 #define AA_PHYS_INT_CALLER(r)        \
   {.reg = (r),                       \
    .cls = NATIVE_REG_INT,            \
    .abi_index = 0xffu,               \
    .flags = NATIVE_REG_CALLER_SAVED, \
    .spill_cost = 1u,                 \
    .copy_cost = 1u}
 #define AA_PHYS_INT_ARG(r)                             \
   {.reg = (r),                                         \
    .cls = NATIVE_REG_INT,                              \
    .abi_index = (r),                                   \
    .flags = NATIVE_REG_CALLER_SAVED | NATIVE_REG_ARG | \
             ((r) < 2u ? NATIVE_REG_RET : 0),           \
    .spill_cost = 1u,                                   \
    .copy_cost = 1u}
 #define AA_PHYS_INT_CALLEE(r)                               \
   {.reg = (r),                                              \
    .cls = NATIVE_REG_INT,                                   \
    .abi_index = 0xffu,                                      \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLEE_SAVED, \
    .spill_cost = 4u,                                        \
    .copy_cost = 1u}
 #define AA_PHYS_INT_RESERVED(r)  \
   {.reg = (r),                   \
    .cls = NATIVE_REG_INT,        \
    .abi_index = 0xffu,           \
    .flags = NATIVE_REG_RESERVED, \
    .spill_cost = 0u,             \
    .copy_cost = 0u}
 
 static const NativePhysRegInfo aa_int_phys[] = {
     AA_PHYS_INT_ARG(0u),       AA_PHYS_INT_ARG(1u),
     AA_PHYS_INT_ARG(2u),       AA_PHYS_INT_ARG(3u),
     AA_PHYS_INT_ARG(4u),       AA_PHYS_INT_ARG(5u),
     AA_PHYS_INT_ARG(6u),       AA_PHYS_INT_ARG(7u),
     AA_PHYS_INT_ALLOC(8u),     AA_PHYS_INT_RESERVED(9u),
     AA_PHYS_INT_RESERVED(10u), AA_PHYS_INT_RESERVED(11u),
     AA_PHYS_INT_ALLOC(12u),    AA_PHYS_INT_ALLOC(13u),
     AA_PHYS_INT_ALLOC(14u),    AA_PHYS_INT_ALLOC(15u),
     AA_PHYS_INT_RESERVED(16u), AA_PHYS_INT_RESERVED(17u),
     AA_PHYS_INT_RESERVED(18u), AA_PHYS_INT_CALLEE(19u),
     AA_PHYS_INT_CALLEE(20u),   AA_PHYS_INT_CALLEE(21u),
     AA_PHYS_INT_CALLEE(22u),   AA_PHYS_INT_CALLEE(23u),
     AA_PHYS_INT_CALLEE(24u),   AA_PHYS_INT_CALLEE(25u),
     AA_PHYS_INT_CALLEE(26u),   AA_PHYS_INT_CALLEE(27u),
     AA_PHYS_INT_CALLEE(28u),   AA_PHYS_INT_RESERVED(29u),
     AA_PHYS_INT_RESERVED(30u), AA_PHYS_INT_RESERVED(31u),
 };
 
 #define AA_PHYS_FP_ALLOC(r)                                 \
   {.reg = (r),                                              \
    .cls = NATIVE_REG_FP,                                    \
    .abi_index = 0xffu,                                      \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLER_SAVED, \
    .spill_cost = 1u,                                        \
    .copy_cost = 1u}
 #define AA_PHYS_FP_CALLER(r)         \
   {.reg = (r),                       \
    .cls = NATIVE_REG_FP,             \
    .abi_index = 0xffu,               \
    .flags = NATIVE_REG_CALLER_SAVED, \
    .spill_cost = 1u,                 \
    .copy_cost = 1u}
 #define AA_PHYS_FP_ARG(r)                              \
   {.reg = (r),                                         \
    .cls = NATIVE_REG_FP,                               \
    .abi_index = (r),                                   \
    .flags = NATIVE_REG_CALLER_SAVED | NATIVE_REG_ARG | \
             ((r) < 4u ? NATIVE_REG_RET : 0),           \
    .spill_cost = 1u,                                   \
    .copy_cost = 1u}
 #define AA_PHYS_FP_CALLEE(r)                                \
   {.reg = (r),                                              \
    .cls = NATIVE_REG_FP,                                    \
    .abi_index = 0xffu,                                      \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLEE_SAVED, \
    .spill_cost = 4u,                                        \
    .copy_cost = 1u}
 #define AA_PHYS_FP_RESERVED(r)   \
   {.reg = (r),                   \
    .cls = NATIVE_REG_FP,         \
    .abi_index = 0xffu,           \
    .flags = NATIVE_REG_RESERVED, \
    .spill_cost = 0u,             \
    .copy_cost = 0u}
 
 static const NativePhysRegInfo aa_fp_phys[] = {
     AA_PHYS_FP_ARG(0u),       AA_PHYS_FP_ARG(1u),     AA_PHYS_FP_ARG(2u),
     AA_PHYS_FP_ARG(3u),       AA_PHYS_FP_ARG(4u),     AA_PHYS_FP_ARG(5u),
     AA_PHYS_FP_ARG(6u),       AA_PHYS_FP_ARG(7u),     AA_PHYS_FP_CALLEE(8u),
     AA_PHYS_FP_CALLEE(9u),    AA_PHYS_FP_CALLEE(10u), AA_PHYS_FP_CALLEE(11u),
     AA_PHYS_FP_CALLEE(12u),   AA_PHYS_FP_CALLEE(13u), AA_PHYS_FP_CALLEE(14u),
     AA_PHYS_FP_CALLEE(15u),   AA_PHYS_FP_CALLER(16u), AA_PHYS_FP_CALLER(17u),
     AA_PHYS_FP_ALLOC(18u),    AA_PHYS_FP_ALLOC(19u),  AA_PHYS_FP_RESERVED(20u),
     AA_PHYS_FP_RESERVED(21u), AA_PHYS_FP_CALLER(22u), AA_PHYS_FP_CALLER(23u),
     AA_PHYS_FP_CALLER(24u),   AA_PHYS_FP_CALLER(25u), AA_PHYS_FP_CALLER(26u),
     AA_PHYS_FP_CALLER(27u),   AA_PHYS_FP_CALLER(28u), AA_PHYS_FP_CALLER(29u),
     AA_PHYS_FP_CALLER(30u),   AA_PHYS_FP_CALLER(31u),
 };
 
 static const NativeAllocClassInfo aa_classes[] = {
     {.cls = NATIVE_REG_INT,
      .allocable = aa_int_allocable,
      .nallocable = sizeof aa_int_allocable / sizeof aa_int_allocable[0],
      .scratch = aa_int_scratch,
      .nscratch = sizeof aa_int_scratch / sizeof aa_int_scratch[0],
      .phys = aa_int_phys,
      .nphys = sizeof aa_int_phys / sizeof aa_int_phys[0],
      .caller_saved_mask = 0x0007ffffu,
      .callee_saved_mask = 0x1ff80000u,
      .arg_mask = 0x000000ffu,
      .ret_mask = 0x00000003u,
      .reserved_mask =
          (1u << AA_TMP0) | (1u << AA_TMP1) | (1u << AA_FP) | (1u << AA_LR)},
     {.cls = NATIVE_REG_FP,
      .allocable = aa_fp_allocable,
      .nallocable = sizeof aa_fp_allocable / sizeof aa_fp_allocable[0],
      .scratch = aa_fp_scratch,
      .nscratch = sizeof aa_fp_scratch / sizeof aa_fp_scratch[0],
      .phys = aa_fp_phys,
      .nphys = sizeof aa_fp_phys / sizeof aa_fp_phys[0],
      /* v8..v15 are callee-saved (low 64 bits per AAPCS64); the rest are
       * caller-saved. */
      .caller_saved_mask = 0xffff00ffu,
      .callee_saved_mask = 0x0000ff00u,
      .arg_mask = 0x000000ffu,
      .ret_mask = 0x0000000fu},
 };
 
 /* Resolve a register name ("x8", "v3", ...) to its (class, Reg). Powers the
  * optimizer's inline-asm clobber masks and explicit hard-register operands
  * ("{x8}" from a GNU local register variable). x0..x30 are DWARF 0..30; the
  * SIMD/FP bank v0..v31 is DWARF 64..95. Returns non-zero for a non-register
  * name (cc/memory/unknown), which the caller skips. */
 static int aa_resolve_name(const NativeRegInfo* ri, Slice name, Reg* out,
                            NativeAllocClass* cls_out) {
   char buf[16];
   uint32_t dwarf;
   (void)ri;
   if (!name.s || !name.len || name.len >= sizeof buf) return 1;
   memcpy(buf, name.s, name.len);
   buf[name.len] = '\0';
   if (aa64_register_index(buf, &dwarf) != 0) return 1;
   if (dwarf <= 30u) {
     *cls_out = NATIVE_REG_INT;
     *out = (Reg)dwarf;
     return 0;
   }
   if (dwarf >= 64u && dwarf <= 95u) {
     *cls_out = NATIVE_REG_FP;
     *out = (Reg)(dwarf - 64u);
     return 0;
   }
   return 1;
 }
 
 static int aa_asm_operand_reg_ok(const NativeRegInfo* ri, NativeAllocClass cls,
                                  Reg reg) {
   (void)ri;
   if (cls == NATIVE_REG_INT) {
     if (reg <= 8u) return 1;
     if (reg >= 12u && reg <= 15u) return 1;
     if (reg >= 19u && reg <= 28u) return 1;
     return 0;
   }
   if (cls == NATIVE_REG_FP) {
     if (reg <= 19u) return 1;
     if (reg >= 22u && reg <= 31u) return 1;
   }
   return 0;
 }
 
 static int aa_asm_constraint_reg(const NativeRegInfo* ri, const char* body,
                                  NativeAllocClass* cls_out, Reg* fixed_out,
                                  u32* allowed_mask_out) {
   (void)ri;
   if (!body || !body[0] || body[1]) return 0;
   if (fixed_out) *fixed_out = REG_NONE;
   if (allowed_mask_out) *allowed_mask_out = 0;
   switch (body[0]) {
     case 'r':
       if (cls_out) *cls_out = NATIVE_REG_INT;
       return 1;
     case 'w':
       if (cls_out) *cls_out = NATIVE_REG_FP;
       return 1;
     case 'x':
       if (cls_out) *cls_out = NATIVE_REG_FP;
       if (allowed_mask_out) *allowed_mask_out = 0x0000ffffu; /* v0..v15 */
       return 1;
     case 'y':
       if (cls_out) *cls_out = NATIVE_REG_FP;
       if (allowed_mask_out) *allowed_mask_out = 0x000000ffu; /* v0..v7 */
       return 1;
     default:
       return 0;
   }
 }
 
 static const NativeRegInfo aa_reg_info = {
     .classes = aa_classes,
     .nclasses = sizeof aa_classes / sizeof aa_classes[0],
     .resolve_name = aa_resolve_name,
     .asm_operand_reg_ok = aa_asm_operand_reg_ok,
     .asm_constraint_reg = aa_asm_constraint_reg,
 };
 
 static void aa_va_start_native(NativeTarget* t, NativeLoc ap_ptr);
 static void aa_va_arg_native(NativeTarget* t, NativeLoc dst, NativeLoc ap_ptr,
                              KitCgTypeId type);
 static void aa_va_end_native(NativeTarget* t, NativeLoc ap_ptr);
 static void aa_va_copy_native(NativeTarget* t, NativeLoc dst_ap_ptr,
                               NativeLoc src_ap_ptr);
 static void aa_asm_block_native(NativeTarget* t, const char* tmpl,
                                 const AsmConstraint* outs, u32 nout,
                                 NativeLoc* out_locs, const AsmConstraint* ins,
                                 u32 nin, const NativeLoc* in_locs,
                                 const Sym* clobbers, u32 nclob);
 
 NativeTarget* aa64_native_target_new(Compiler* c, ObjBuilder* obj,
                                      MCEmitter* mc) {
   AANativeTarget* a = arena_znew(c->tu, AANativeTarget);
   NativeTarget* t;
   if (!a) return NULL;
   t = &a->base;
   t->c = c;
   t->obj = obj;
   t->mc = mc;
   native_frame_init(&a->frame, c);
   t->regs = &aa_reg_info;
   t->class_for_type = aa_class_for_type;
   t->imm_legal = aa_imm_legal;
   t->addr_legal = aa_addr_legal;
   t->machine_op_clobbers = aa_machine_op_clobbers;
   t->func_begin = aa_func_begin;
   t->func_begin_known_frame = aa_func_begin_known_frame;
   t->note_frame_state = aa_note_frame_state;
   t->reserve_callee_saves = aa_reserve_callee_saves;
   t->signature_stack_bytes = aa_signature_stack_bytes;
   t->call_stack_bytes = aa_call_stack_bytes;
   t->has_store_zero_reg = 1;
   t->store_zero_reg = 31u; /* wzr/xzr in the Rt position of a store */
   t->func_end = aa_func_end;
   t->frame_slot = aa_frame_slot;
   t->frame_slot_debug_loc = aa_frame_slot_debug_loc;
   t->bind_param = aa_bind_native_param;
   t->label_new = aa_label_new;
   t->label_place = aa_label_place;
   t->jump = aa_jump;
   t->cmp_branch = aa_cmp_branch;
   t->indirect_branch = aa_indirect_branch;
   t->load_label_addr = aa_load_label_addr;
   t->move = aa_move;
   t->load_imm = aa_load_imm_native;
   t->load_const = aa_load_const;
   t->load_addr = aa_load_addr;
   t->load = aa_load_native;
   t->store = aa_store_native;
   t->tls_addr_of = aa_tls_addr_of;
   t->copy_bytes = aa_copy_bytes;
   t->set_bytes = aa_set_bytes;
   t->bitfield_load = aa_bitfield_load;
   t->bitfield_store = aa_bitfield_store;
   t->binop = aa_binop;
   t->unop = aa_unop;
   t->cmp = aa_cmp;
   t->convert = aa_convert;
   t->alloca_ = aa_alloca;
   t->spill = aa_spill;
   t->reload = aa_reload;
   t->plan_call = aa_plan_call;
   t->emit_call = aa_emit_call;
   t->plan_ret = aa_plan_ret;
   t->ret = aa_ret;
   t->atomic_load = aa_atomic_load;
   t->atomic_store = aa_atomic_store;
   t->atomic_rmw = aa_atomic_rmw;
   t->atomic_cas = aa_atomic_cas;
   t->fence = aa_fence;
   t->va_start_ = aa_va_start_native;
   t->va_arg_ = aa_va_arg_native;
   t->va_end_ = aa_va_end_native;
   t->va_copy_ = aa_va_copy_native;
   t->intrinsic = aa_intrinsic;
   t->asm_block = aa_asm_block_native;
   t->file_scope_asm = native_file_scope_asm;
   t->trap = aa_trap;
   t->set_loc = aa_set_loc;
   t->finalize = native_finalize;
   return t;
 }
 
 /* Place the incoming parameter into `dst`: a hard register (the common
  * register-allocated scalar case -> a single arg-reg move, or a stack load
  * straight into the register), a frame slot (address-taken / aggregate /
  * spilled), or nowhere (unused). Incoming arg registers are never allocable,
  * so a register dst never aliases an incoming arg register. */
 static void aa_bind_native_param(NativeTarget* t, const CGParamDesc* p,
                                  NativeLoc dst) {
   AANativeTarget* a = aa_of(t);
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, a->func->fn_type);
   const ABIArgInfo* ai =
       p->index < abi->nparams ? &abi->params[p->index] : NULL;
   int to_reg = dst.kind == NATIVE_LOC_REG;
   if (!ai || ai->kind == ABI_ARG_IGNORE) return;
   if (ai->kind == ABI_ARG_INDIRECT) {
     NativeAddr d_addr, from;
     AggregateAccess access;
     NativeLoc src =
         native_loc_reg(p->type, NATIVE_REG_INT,
                        a->next_param_int < 8u ? a->next_param_int++ : AA_TMP0);
     if (src.v.reg == AA_TMP0) {
       NativeAddr saddr;
       memset(&saddr, 0, sizeof saddr);
       saddr.base_kind = NATIVE_ADDR_BASE_REG;
       saddr.base.reg = AA_FP;
       saddr.offset = aa_fp_off_in_arg(a, a->next_param_stack);
       aa_emit_mem(a, 1, src, saddr, aa_mem_for_type(t, p->type, 8));
       a->next_param_stack += 8u;
     }
     if (dst.kind != NATIVE_LOC_FRAME)
       aa_panic(a, "indirect parameter requires a frame destination");
     memset(&d_addr, 0, sizeof d_addr);
     d_addr.base_kind = NATIVE_ADDR_BASE_FRAME;
     d_addr.base.frame = dst.v.frame;
     d_addr.base_type = p->type;
     memset(&from, 0, sizeof from);
     from.base_kind = NATIVE_ADDR_BASE_REG;
     from.base.reg = src.v.reg;
     from.base_type = p->type;
     memset(&access, 0, sizeof access);
     access.type = p->type;
     access.size = p->size ? p->size : (u32)cg_type_size(t->c, p->type);
     access.align = p->align ? p->align : type_align32(t, p->type);
     aa_copy_bytes(t, d_addr, from, access);
     return;
   }
   for (u32 i = 0; i < ai->nparts; ++i) {
     const ABIArgPart* part = &ai->parts[i];
     NativeAllocClass cls =
         part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
     int reg_dst = to_reg && (NativeAllocClass)dst.cls == cls;
     NativeLoc src;
     if (cls == NATIVE_REG_FP && a->next_param_fp < 8u) {
       src = native_loc_reg(p->type, cls, a->next_param_fp++);
     } else if (cls == NATIVE_REG_INT && a->next_param_int < 8u) {
       src = native_loc_reg(p->type, cls, a->next_param_int++);
     } else {
       /* Stack-passed part: load straight into the dst register when possible,
        * otherwise a scratch for the store-to-frame path. */
       Reg tmp =
           reg_dst ? (Reg)dst.v.reg : (cls == NATIVE_REG_FP ? 16u : AA_TMP0);
       NativeAddr saddr;
       src = native_loc_reg(p->type, cls, tmp);
       a->next_param_stack =
           align_up_u32(a->next_param_stack, aa_part_stack_align(abi, part));
       memset(&saddr, 0, sizeof saddr);
       saddr.base_kind = NATIVE_ADDR_BASE_REG;
       saddr.base.reg = AA_FP;
       saddr.base_type = p->type;
       saddr.offset = aa_fp_off_in_arg(a, a->next_param_stack);
       aa_emit_mem(a, 1, src, saddr, aa_mem_for_type(t, p->type, part->size));
       a->next_param_stack += aa_part_stack_size(abi, part);
     }
     if (dst.kind == NATIVE_LOC_NONE) {
       /* Unused parameter: only the ABI cursor advances. */
     } else if (to_reg) {
       NativeLoc d = native_loc_reg(dst.type ? dst.type : p->type,
                                    (NativeAllocClass)dst.cls, (Reg)dst.v.reg);
       if (!(src.kind == NATIVE_LOC_REG && src.v.reg == d.v.reg &&
             (NativeAllocClass)src.cls == (NativeAllocClass)d.cls))
         aa_move(t, d, src);
     } else {
       aa_store_part(
           t, native_loc_stack(p->type, dst.v.frame, (i32)part->src_offset), src,
           0, part->size);
     }
   }
   a->incoming_stack_size = align_up_u32(a->next_param_stack, 16u);
 }
 
 static void aa_bind_param(NativeDirectTarget* d, const CGParamDesc* p,
                           CGLocal local, NativeDirectLocal* l) {
   NativeLoc dst;
   (void)local;
   memset(&dst, 0, sizeof dst);
   dst.kind = NATIVE_LOC_FRAME;
   dst.type = p->type;
   dst.v.frame = l->home;
   aa_bind_native_param(d->native, p, dst);
 }
 
 static const char* aa_no_tail(NativeDirectTarget* d, const CGCallDesc* call) {
   NativeCallDesc nd;
   NativeLoc* args = NULL;
   NativeLoc* results = NULL;
   u32 stack;
   memset(&nd, 0, sizeof nd);
   u32 nresults = call->result != CG_LOCAL_NONE ? 1u : 0u;
   if (call->nargs) args = arena_zarray(d->base.c->tu, NativeLoc, call->nargs);
   if (nresults) results = arena_zarray(d->base.c->tu, NativeLoc, nresults);
   for (u32 i = 0; i < call->nargs; ++i) {
     args[i].kind = NATIVE_LOC_FRAME;
     args[i].type = d->locals[call->args[i] - 1u].type;
     args[i].cls = d->locals[call->args[i] - 1u].cls;
     args[i].v.frame = d->locals[call->args[i] - 1u].home;
   }
   if (nresults) {
     results[0].kind = NATIVE_LOC_FRAME;
     results[0].type = d->locals[call->result - 1u].type;
     results[0].cls = d->locals[call->result - 1u].cls;
     results[0].v.frame = d->locals[call->result - 1u].home;
   }
   nd.fn_type = call->fn_type;
   nd.args = args;
   nd.results = results;
   nd.nargs = call->nargs;
   nd.nresults = nresults;
   stack = aa_call_stack_size(d->native, &nd);
   if (stack > aa_of(d->native)->incoming_stack_size)
     return "aarch64 tail call: stack argument area too small";
   return NULL;
 }
 
 static NativeAddr aa_direct_addr(NativeDirectTarget* d, Operand op) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   switch ((OpKind)op.kind) {
     case OPK_LOCAL:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = d->locals[op.v.local - 1u].home;
       addr.base_type = op.type;
       return addr;
     case OPK_INDIRECT:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME_VALUE;
       addr.base.frame = d->locals[op.v.ind.base - 1u].home;
       addr.cls = d->locals[op.v.ind.base - 1u].cls;
       addr.base_type = d->locals[op.v.ind.base - 1u].type;
       addr.offset = op.v.ind.ofs;
       return addr;
     default:
       compiler_panic(d->base.c, d->loc,
                      "aarch64 native target: operand is not addressable");
   }
 }
 
 static NativeAddr aa_direct_materialize_addr(NativeDirectTarget* d,
                                              Operand op) {
   NativeAddr addr = aa_direct_addr(d, op);
   if (addr.base_kind == NATIVE_ADDR_BASE_FRAME_VALUE) {
     NativeLoc base = native_loc_reg(addr.base_type, NATIVE_REG_INT, AA_TMP1);
     NativeAddr load;
     memset(&load, 0, sizeof load);
     load.base_kind = NATIVE_ADDR_BASE_FRAME;
     load.base.frame = addr.base.frame;
     load.base_type = addr.base_type;
     aa_emit_mem(aa_of(d->native), 1, base, load,
                 aa_mem_for_type(d->native, addr.base_type, 8));
     addr.base_kind = NATIVE_ADDR_BASE_REG;
     addr.base.reg = AA_TMP1;
   }
   return addr;
 }
 
 static NativeAddr aa_direct_pointer_addr(NativeDirectTarget* d, Operand op) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   if (op.kind == OPK_LOCAL) {
     NativeLoc base = native_loc_reg(op.type, NATIVE_REG_INT, AA_TMP1);
     NativeAddr load;
     memset(&load, 0, sizeof load);
     load.base_kind = NATIVE_ADDR_BASE_FRAME;
     load.base.frame = d->locals[op.v.local - 1u].home;
     load.base_type = op.type;
     aa_emit_mem(aa_of(d->native), 1, base, load,
                 aa_mem_for_type(d->native, op.type, 8));
     addr.base_kind = NATIVE_ADDR_BASE_REG;
     addr.base.reg = AA_TMP1;
     addr.base_type = op.type;
     return addr;
   }
   return aa_direct_materialize_addr(d, op);
 }
 
 static NativeAddr aa_reg_addr(KitCgTypeId type, u32 reg, i32 offset) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_REG;
   addr.base.reg = reg;
   addr.base_type = type;
   addr.offset = offset;
   return addr;
 }
 
 static void aa_load_ap_addr(NativeDirectTarget* d, Operand ap_addr,
                             u32 dst_reg) {
   NativeLoc dst =
       native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT, dst_reg);
   NativeAddr ap = aa_direct_pointer_addr(d, ap_addr);
   d->native->load_addr(d->native, dst, ap);
 }
 
 /* The va cores use only non-allocable registers for their temporaries
  * (scratch x9/x10, reserved x16=TMP0 / x17=TMP1, vector v16) so they never
  * clobber a value the optimizer's register allocator may hold live across the
  * op. The va_list base register is supplied by the caller (ap.base.reg), which
  * the optimizer materializes into a safe register before the call. */
 static u32 aa_va_base_reg(AANativeTarget* a, NativeAddr ap) {
   if (ap.base_kind != NATIVE_ADDR_BASE_REG)
     compiler_panic(a->base.c, a->func ? a->func->loc : (SrcLoc){0, 0, 0},
                    "aarch64 native target: va_list pointer not in register");
   return ap.base.reg;
 }
 
 /* va_list layout is queried from the ABI; the optimizer/direct callers pass the
  * va_list pointer opaquely. `ap` addresses the va_list object itself. */
 static void aa_va_start_core(AANativeTarget* a, NativeAddr ap) {
   NativeTarget* t = &a->base;
   const ABIFuncInfo* abi =
       a->func ? abi_cg_func_info(t->c->abi, a->func->fn_type) : NULL;
   ABIVaListInfo vai = abi_va_list_layout(t->c->abi);
   NativeLoc ptr =
       native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT, AA_TMP0);
   if (vai.kind == ABI_VA_LIST_POINTER) {
     if (a->top_home_bytes) {
       /* Windows: `va_list = &<first vararg>` inside the contiguous
        * [GP home area | incoming stack args] block. Named args consume the
        * leading slots; next_param_int (FP params remapped to GP included) plus
        * next_param_stack locate the first unnamed slot. Home slot
        * gp_reg_count coincides with incoming-arg byte 0, so a single formula
        * spans both regions. */
       i32 off =
           aa_fp_off_home_slot(a->next_param_int) + (i32)a->next_param_stack;
       aa_emit_add_imm(a, AA_TMP0, AA_FP, off);
       aa_emit_mem(a, 0, ptr, ap, aa_mem_for_type(t, ptr.type, 8));
       return;
     }
     /* `va_list = &<first vararg>`. Variadic stack args follow the fixed
      * incoming params in the same caller window. Apple ARM64 compact fixed
      * stack args may leave this cursor at +4, while the first variadic slot
      * starts at the next 8-byte boundary. */
     u32 stack = aa_vararg_stack_start(abi, a->next_param_stack);
     aa_emit_add_imm(a, AA_TMP0, AA_FP, aa_fp_off_in_arg(a, stack));
     aa_emit_mem(a, 0, ptr, ap, aa_mem_for_type(t, ptr.type, 8));
     return;
   }
   if (vai.kind == ABI_VA_LIST_AAPCS64) {
     KitCgTypeId i32_ty = builtin_id(KIT_CG_BUILTIN_I32);
     NativeLoc i32tmp = native_loc_reg(i32_ty, NATIVE_REG_INT, AA_TMP1);
     MemAccess ptr_mem = aa_mem_for_type(t, ptr.type, 8);
     MemAccess i32_mem = aa_mem_for_type(t, i32_ty, 4);
     AANativeSlot* gr = aa_slot(a, a->va_gr_slot);
     AANativeSlot* vr = aa_slot(a, a->va_vr_slot);
     u32 base = aa_va_base_reg(a, ap);
     u32 used_gr = a->next_param_int < vai.gp_reg_count ? a->next_param_int
                                                        : vai.gp_reg_count;
     u32 used_vr = a->next_param_fp < vai.fp_reg_count ? a->next_param_fp
                                                       : vai.fp_reg_count;
     /* __stack points at the incoming stack args, which sit above the saved
      * fp/lr pair — the same address bind_param uses (aa_fp_off_in_arg), not the
      * raw next_param_stack cursor. */
     aa_emit_add_imm(a, AA_TMP0, AA_FP,
                     aa_fp_off_in_arg(a, a->next_param_stack));
     aa_emit_mem(a, 0, ptr, aa_reg_addr(ptr.type, base, (i32)vai.stack_offset),
                 ptr_mem);
     aa_emit_add_imm(a, AA_TMP0, AA_FP,
                     aa_fp_off_slot(a, gr->off) +
                         (i32)(vai.gp_reg_count * vai.gp_slot_size));
     aa_emit_mem(a, 0, ptr, aa_reg_addr(ptr.type, base, (i32)vai.gr_top_offset),
                 ptr_mem);
     aa_emit_add_imm(a, AA_TMP0, AA_FP,
                     aa_fp_off_slot(a, vr->off) +
                         (i32)(vai.fp_reg_count * vai.fp_slot_size));
     aa_emit_mem(a, 0, ptr, aa_reg_addr(ptr.type, base, (i32)vai.vr_top_offset),
                 ptr_mem);
     aa_emit_load_imm(t->mc, 0, AA_TMP1,
                      -(i32)((vai.gp_reg_count - used_gr) * vai.gp_slot_size));
     aa_emit_mem(a, 0, i32tmp,
                 aa_reg_addr(i32_ty, base, (i32)vai.gr_offs_offset), i32_mem);
     aa_emit_load_imm(t->mc, 0, AA_TMP1,
                      -(i32)((vai.fp_reg_count - used_vr) * vai.fp_slot_size));
     aa_emit_mem(a, 0, i32tmp,
                 aa_reg_addr(i32_ty, base, (i32)vai.vr_offs_offset), i32_mem);
     return;
   }
   compiler_panic(t->c, a->func ? a->func->loc : (SrcLoc){0, 0, 0},
                  "aarch64 native target: unsupported va_list layout");
 }
 
 static void aa_va_arg_core(AANativeTarget* a, NativeLoc dst, NativeAddr ap,
                            KitCgTypeId type) {
   NativeTarget* t = &a->base;
   ABIVaListInfo vai = abi_va_list_layout(t->c->abi);
   NativeLoc cur =
       native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT, AA_TMP0);
   /* The fetched value is written directly into the caller-provided register
    * `dst`, which the caller guarantees is distinct from the va_list base
    * register. Only TMP0/TMP1 are used as private scratch. */
   NativeLoc val = dst;
   NativeAddr src;
   MemAccess ptr_mem = aa_mem_for_type(t, cur.type, 8);
   MemAccess val_mem = aa_mem_for_type(t, type, type_size32(t, type));
   if (dst.kind != NATIVE_LOC_REG)
     compiler_panic(t->c, a->func ? a->func->loc : (SrcLoc){0, 0, 0},
                    "aarch64 native target: va_arg destination must be a "
                    "register");
   if (vai.kind == ABI_VA_LIST_POINTER) {
     aa_emit_mem(a, 1, cur, ap, ptr_mem);
     src = aa_reg_addr(type, AA_TMP0, 0);
     {
       const ABIFuncInfo* abi =
           a->func ? abi_cg_func_info(t->c->abi, a->func->fn_type) : NULL;
       ABIArgPart part;
       memset(&part, 0, sizeof part);
       part.cls = cg_type_is_float(t->c, type) ? ABI_CLASS_FP : ABI_CLASS_INT;
       part.size = type_size32(t, type);
       part.align = type_align32(t, type);
       aa_emit_add_imm(a, AA_TMP1, AA_TMP0,
                       (i32)aa_part_vararg_stack_size(abi, &part));
     }
     aa_emit_mem(a, 0, native_loc_reg(cur.type, NATIVE_REG_INT, AA_TMP1), ap,
                 ptr_mem);
     aa_emit_mem(a, 1, val, src, val_mem);
     return;
   }
   if (vai.kind == ABI_VA_LIST_AAPCS64) {
     KitCgTypeId i32_ty = builtin_id(KIT_CG_BUILTIN_I32);
     NativeLoc off = native_loc_reg(i32_ty, NATIVE_REG_INT, AA_TMP1);
     MemAccess i32_mem = aa_mem_for_type(t, i32_ty, 4);
     int is_fp = cg_type_is_float(t->c, type);
     u32 base = aa_va_base_reg(a, ap);
     u32 offs_field = is_fp ? vai.vr_offs_offset : vai.gr_offs_offset;
     u32 top_field = is_fp ? vai.vr_top_offset : vai.gr_top_offset;
     u32 slot_size = is_fp ? vai.fp_slot_size : vai.gp_slot_size;
     MCLabel stack_label = t->mc->label_new(t->mc);
     MCLabel done_label = t->mc->label_new(t->mc);
     aa_emit_mem(a, 1, off, aa_reg_addr(i32_ty, base, (i32)offs_field), i32_mem);
     aa_emit32(t->mc, aa64_subs_imm12(0, AA64_ZR, AA_TMP1, 0, 0));
     aa_emit32(t->mc,
               aa64_brcond_pack((AA64BrCond){.cond = cmp_cond(CMP_GE_S)}));
     t->mc->emit_label_ref(t->mc, stack_label, R_AARCH64_CONDBR19, 4, 0);
     aa_emit_mem(a, 1, cur, aa_reg_addr(cur.type, base, (i32)top_field),
                 ptr_mem);
     aa_emit32(t->mc, aa_sbfm(1, AA_TMP1, AA_TMP1, 0, 31));
     aa_emit32(t->mc, aa64_add(1, AA_TMP0, AA_TMP0, AA_TMP1));
     aa_emit_mem(a, 1, val, aa_reg_addr(type, AA_TMP0, 0), val_mem);
     aa_emit_add_imm(a, AA_TMP1, AA_TMP1, (i32)slot_size);
     aa_emit_mem(a, 0, off, aa_reg_addr(i32_ty, base, (i32)offs_field), i32_mem);
     aa_emit32(t->mc, aa64_b(0));
     t->mc->emit_label_ref(t->mc, done_label, R_AARCH64_JUMP26, 4, 0);
     t->mc->label_place(t->mc, stack_label);
     aa_emit_mem(a, 1, cur, aa_reg_addr(cur.type, base, (i32)vai.stack_offset),
                 ptr_mem);
     aa_emit_mem(a, 1, val, aa_reg_addr(type, AA_TMP0, 0), val_mem);
     aa_emit_add_imm(a, AA_TMP0, AA_TMP0, 8);
     aa_emit_mem(a, 0, cur, aa_reg_addr(cur.type, base, (i32)vai.stack_offset),
                 ptr_mem);
     t->mc->label_place(t->mc, done_label);
     return;
   }
   compiler_panic(t->c, a->func ? a->func->loc : (SrcLoc){0, 0, 0},
                  "aarch64 native target: unsupported va_list layout");
 }
 
 static void aa_va_copy_core(AANativeTarget* a, NativeAddr dst_ap,
                             NativeAddr src_ap) {
   NativeTarget* t = &a->base;
   ABIVaListInfo vai = abi_va_list_layout(t->c->abi);
   NativeLoc tmp =
       native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT, AA_TMP0);
   MemAccess mem = aa_mem_for_type(t, tmp.type, 8);
   if (vai.kind == ABI_VA_LIST_POINTER) {
     aa_emit_mem(a, 1, tmp, src_ap, mem);
     aa_emit_mem(a, 0, tmp, dst_ap, mem);
     return;
   }
   if (vai.kind == ABI_VA_LIST_AAPCS64) {
     u32 sb = aa_va_base_reg(a, src_ap);
     u32 db = aa_va_base_reg(a, dst_ap);
     for (u32 off = 0; off < vai.type.size; off += 8u) {
       aa_emit_mem(a, 1, tmp, aa_reg_addr(tmp.type, sb, (i32)off), mem);
       aa_emit_mem(a, 0, tmp, aa_reg_addr(tmp.type, db, (i32)off), mem);
     }
     return;
   }
   compiler_panic(t->c, a->func ? a->func->loc : (SrcLoc){0, 0, 0},
                  "aarch64 native target: unsupported va_list layout");
 }
 
 /* ---- Direct-path (NativeDirectTarget) wrappers: convert semantic operands to
  * NativeAddr/NativeLoc, then call the shared cores above. ---- */
 
 /* The cores reserve x16/x17 (TMP0/TMP1) as private scratch and require the
  * va_list base register(s) to be distinct from those. aa_direct_pointer_addr
  * returns the pointer in TMP1, so the direct wrappers first relocate it into
  * x9/x10 before calling the cores. */
 static NativeAddr aa_direct_va_base(NativeDirectTarget* d, Operand ap_addr,
                                     u32 reg) {
   aa_load_ap_addr(d, ap_addr, reg);
   return aa_reg_addr(builtin_id(KIT_CG_BUILTIN_I64), reg, 0);
 }
 
 static void aa_va_start_(NativeDirectTarget* d, Operand ap_addr) {
   aa_va_start_core(aa_of(d->native), aa_direct_va_base(d, ap_addr, 10u));
 }
 
 static void aa_va_arg_(NativeDirectTarget* d, Operand dst_op, Operand ap_addr,
                        KitCgTypeId type) {
   AANativeTarget* a = aa_of(d->native);
   int is_fp = cg_type_is_float(d->base.c, type);
   NativeLoc res = native_loc_reg(type, is_fp ? NATIVE_REG_FP : NATIVE_REG_INT,
                                  is_fp ? 16u : 9u);
   MemAccess val_mem =
       aa_mem_for_type(d->native, type, type_size32(d->native, type));
   NativeAddr dst;
   aa_va_arg_core(a, res, aa_direct_va_base(d, ap_addr, 10u), type);
   dst = aa_direct_materialize_addr(d, dst_op);
   aa_emit_mem(a, 0, res, dst, val_mem);
 }
 
 static void aa_va_end_(NativeDirectTarget* d, Operand ap_addr) {
   (void)d;
   (void)ap_addr;
 }
 
 static void aa_va_copy_(NativeDirectTarget* d, Operand dst_ap_addr,
                         Operand src_ap_addr) {
   AANativeTarget* a = aa_of(d->native);
   NativeAddr src = aa_direct_va_base(d, src_ap_addr, 9u);
   NativeAddr dst = aa_direct_va_base(d, dst_ap_addr, 10u);
   aa_va_copy_core(a, dst, src);
 }
 
 /* ---- NativeTarget (optimizer) hooks: the optimizer passes the va_list
  * pointer as a materialized register; layout is resolved inside the cores. ----
  */
 
 static NativeAddr aa_va_addr_from_ptr(NativeLoc ap_ptr) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_REG;
   addr.cls = NATIVE_REG_INT;
   addr.base.reg = ap_ptr.v.reg;
   addr.base_type = ap_ptr.type;
   return addr;
 }
 
 static void aa_va_start_native(NativeTarget* t, NativeLoc ap_ptr) {
   aa_va_start_core(aa_of(t), aa_va_addr_from_ptr(ap_ptr));
 }
 
 static void aa_va_arg_native(NativeTarget* t, NativeLoc dst, NativeLoc ap_ptr,
                              KitCgTypeId type) {
   aa_va_arg_core(aa_of(t), dst, aa_va_addr_from_ptr(ap_ptr), type);
 }
 
 static void aa_va_end_native(NativeTarget* t, NativeLoc ap_ptr) {
   (void)t;
   (void)ap_ptr;
 }
 
 static void aa_va_copy_native(NativeTarget* t, NativeLoc dst_ap_ptr,
                               NativeLoc src_ap_ptr) {
   aa_va_copy_core(aa_of(t), aa_va_addr_from_ptr(dst_ap_ptr),
                   aa_va_addr_from_ptr(src_ap_ptr));
 }
 
 /* constraint_body / constraint_early / match_index are shared
  * (cg/native_asm.h). */
 
 _Noreturn static void aa_asm_panic_at(Compiler* c, SrcLoc loc,
                                       const char* msg) {
   compiler_panic(c, loc, "aarch64 inline asm: %s", msg);
 }
 
 _Noreturn static void aa_asm_panic(NativeDirectTarget* d, const char* msg) {
   aa_asm_panic_at(d->base.c, d->loc, msg);
 }
 
 AA_UNUSED_FN static void aa_asm_bound_reg(Operand* out, KitCgTypeId type,
                                           NativeAllocClass cls, Reg reg) {
   memset(out, 0, sizeof *out);
   out->kind = AA64_INLINE_OPK_REG;
   out->pad[0] =
       (cls == NATIVE_REG_FP) ? AA64_INLINE_OPCLS_FP : AA64_INLINE_OPCLS_INT;
   out->type = type;
   out->v.local = (CGLocal)reg;
 }
 
 AA_UNUSED_FN static void aa_asm_bound_mem(Operand* out, KitCgTypeId type,
                                           Reg base) {
   memset(out, 0, sizeof *out);
   out->kind = OPK_INDIRECT;
   out->type = type;
   out->v.ind.base = (CGLocal)base;
   out->v.ind.index = CG_LOCAL_NONE;
 }
 
 static int aa_asm_parse_reg_clobber(Compiler* c, SrcLoc loc, Sym name,
                                     NativeAllocClass* cls_out, Reg* reg_out) {
   Slice s = pool_slice(c->global, name);
   char buf[16];
   uint32_t dwarf;
   if (!s.s || !s.len) return 0;
   if (s.len == 2 && s.s[0] == 'c' && s.s[1] == 'c') return 0;
   if (s.len == 6 && memcmp(s.s, "memory", 6) == 0) return 0;
   if (s.len >= sizeof buf) aa_asm_panic_at(c, loc, "clobber name is too long");
   memcpy(buf, s.s, s.len);
   buf[s.len] = '\0';
   if (aa64_register_index(buf, &dwarf) != 0)
     aa_asm_panic_at(c, loc, "unknown clobber register");
   if (dwarf <= 30u) {
     *cls_out = NATIVE_REG_INT;
     *reg_out = (Reg)dwarf;
     return 1;
   }
   if (dwarf >= 64u && dwarf <= 95u) {
     *cls_out = NATIVE_REG_FP;
     *reg_out = (Reg)(dwarf - 64u);
     return 1;
   }
   aa_asm_panic_at(c, loc, "unsupported clobber register");
   return 0;
 }
 
 static void aa_asm_clobber_masks(Compiler* c, SrcLoc loc, const Sym* clobbers,
                                  u32 nclob, u32* int_mask, u32* fp_mask) {
   *int_mask = 0;
   *fp_mask = 0;
   for (u32 i = 0; i < nclob; ++i) {
     NativeAllocClass cls;
     Reg reg;
     if (!aa_asm_parse_reg_clobber(c, loc, clobbers[i], &cls, &reg)) continue;
     if (cls == NATIVE_REG_INT)
       *int_mask |= 1u << reg;
     else if (cls == NATIVE_REG_FP)
       *fp_mask |= 1u << reg;
   }
 }
 
 AA_UNUSED_FN static Reg aa_asm_alloc_reg(NativeDirectTarget* d,
                                          NativeAllocClass cls,
                                          u32 allowed_mask, u32* used_int,
                                          u32* used_fp) {
   static const Reg int_pool[] = {0u, 1u, 2u,  3u,  4u,  5u,  6u,
                                  7u, 8u, 11u, 12u, 13u, 14u, 15u};
   static const Reg fp_pool[] = {0u,  1u,  2u,  3u,  4u,  5u,  6u,  7u,
                                 16u, 17u, 18u, 19u, 22u, 23u, 24u, 25u,
                                 26u, 27u, 28u, 29u, 30u, 31u};
   const Reg* pool = cls == NATIVE_REG_FP ? fp_pool : int_pool;
   u32 n = cls == NATIVE_REG_FP ? (u32)(sizeof fp_pool / sizeof fp_pool[0])
                                : (u32)(sizeof int_pool / sizeof int_pool[0]);
   u32* used = cls == NATIVE_REG_FP ? used_fp : used_int;
   for (u32 i = 0; i < n; ++i) {
     Reg r = pool[i];
     if (allowed_mask && (allowed_mask & (1u << r)) == 0) continue;
     if ((*used & (1u << r)) != 0) continue;
     *used |= 1u << r;
     return r;
   }
   aa_asm_panic(d, "out of registers for asm operands");
   return REG_NONE;
 }
 
 static int aa_asm_resolve_pin_or_panic(NativeDirectTarget* d, Sym reg,
                                        const char* constraint,
                                        NativeAsmRegPin* pin) {
   NativeAsmRegPinStatus st =
       native_asm_resolve_pin(d->native, reg, constraint, pin);
   if (st == NATIVE_ASM_REG_PIN_ABSENT) return 0;
   if (st != NATIVE_ASM_REG_PIN_OK)
     aa_asm_panic(d, native_asm_pin_status_message(st));
   return 1;
 }
 
 AA_UNUSED_FN static void aa_direct_load_operand_to_reg(NativeDirectTarget* d,
                                                        Operand op,
                                                        NativeLoc dst) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   switch ((OpKind)op.kind) {
     case OPK_IMM:
       if ((NativeAllocClass)dst.cls != NATIVE_REG_INT)
         aa_asm_panic(d, "floating-point immediate asm input is unsupported");
       d->native->load_imm(d->native, dst, op.v.imm);
       return;
     case OPK_LOCAL:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = d->locals[op.v.local - 1u].home;
       addr.base_type = op.type;
       aa_emit_mem(aa_of(d->native), 1, dst, addr,
                   aa_mem_for_type(d->native, op.type, 0));
       return;
     case OPK_GLOBAL:
       addr.base_kind = NATIVE_ADDR_BASE_GLOBAL;
       addr.base.global.sym = op.v.global.sym;
       addr.base.global.addend = op.v.global.addend;
       addr.base_type = op.type;
       d->native->load_addr(d->native, dst, addr);
       return;
     case OPK_INDIRECT:
       addr = aa_direct_materialize_addr(d, op);
       aa_emit_mem(aa_of(d->native), 1, dst, addr,
                   aa_mem_for_type(d->native, op.type, 0));
       return;
   }
   aa_asm_panic(d, "unsupported asm input operand");
 }
 
 AA_UNUSED_FN static void aa_direct_load_address_to_reg(NativeDirectTarget* d,
                                                        Operand op,
                                                        NativeLoc dst) {
   NativeAddr addr = aa_direct_addr(d, op);
   d->native->load_addr(d->native, dst, addr);
 }
 
 AA_UNUSED_FN static void aa_direct_store_reg_to_operand(NativeDirectTarget* d,
                                                         Operand op,
                                                         NativeLoc src) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   if (op.kind == OPK_LOCAL) {
     addr.base_kind = NATIVE_ADDR_BASE_FRAME;
     addr.base.frame = d->locals[op.v.local - 1u].home;
     addr.base_type = op.type;
   } else {
     addr = aa_direct_materialize_addr(d, op);
   }
   aa_emit_mem(aa_of(d->native), 0, src, addr,
               aa_mem_for_type(d->native, op.type, 0));
 }
 
 typedef struct AAAsmSavedClobber {
   NativeFrameSlot slot;
   NativeAllocClass cls;
   Reg reg;
   KitCgTypeId type;
 } AAAsmSavedClobber;
 
 static void aa_asm_save_one(AANativeTarget* a, AAAsmSavedClobber* s) {
   NativeFrameSlotDesc desc;
   NativeAddr addr;
   NativeLoc reg;
   memset(&desc, 0, sizeof desc);
   desc.type = s->type;
   desc.size = 8;
   desc.align = 8;
   desc.kind = NATIVE_FRAME_SLOT_SAVE;
   s->slot = a->base.frame_slot(&a->base, &desc);
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = s->slot;
   addr.base_type = s->type;
   reg = native_loc_reg(s->type, s->cls, s->reg);
   aa_emit_mem(a, 0, reg, addr, aa_mem_for_type(&a->base, s->type, 8));
 }
 
 AA_UNUSED_FN static void aa_asm_restore_one(AANativeTarget* a,
                                             const AAAsmSavedClobber* s) {
   NativeAddr addr;
   NativeLoc reg = native_loc_reg(s->type, s->cls, s->reg);
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = s->slot;
   addr.base_type = s->type;
   aa_emit_mem(a, 1, reg, addr, aa_mem_for_type(&a->base, s->type, 8));
 }
 
 AA_UNUSED_FN static AAAsmSavedClobber* aa_asm_save_callee_clobbers(
     AANativeTarget* a, u32 int_mask, u32 fp_mask, u32* nsaved_out) {
   AAAsmSavedClobber* saved =
       arena_zarray(a->base.c->tu, AAAsmSavedClobber, 20u);
   u32 n = 0;
   KitCgTypeId i64 = builtin_id(KIT_CG_BUILTIN_I64);
   KitCgTypeId f64 = builtin_id(KIT_CG_BUILTIN_F64);
   for (Reg r = 19u; r <= 28u; ++r) {
     if ((int_mask & (1u << r)) == 0) continue;
     saved[n].cls = NATIVE_REG_INT;
     saved[n].reg = r;
     saved[n].type = i64;
     aa_asm_save_one(a, &saved[n++]);
   }
   for (Reg r = 8u; r <= 15u; ++r) {
     if ((fp_mask & (1u << r)) == 0) continue;
     saved[n].cls = NATIVE_REG_FP;
     saved[n].reg = r;
     saved[n].type = f64;
     aa_asm_save_one(a, &saved[n++]);
   }
   *nsaved_out = n;
   return saved;
 }
 
 static void aa_direct_asm_block(NativeDirectTarget* d, const char* tmpl,
                                 const AsmConstraint* outs, u32 nout,
                                 Operand* out_ops, const AsmConstraint* ins,
                                 u32 nin, const Operand* in_ops,
                                 const Sym* clobbers, u32 nclob,
                                 u32 clobber_abi_sets) {
   Operand* bound_outs =
       nout ? arena_zarray(d->base.c->tu, Operand, nout) : NULL;
   Operand* bound_ins = nin ? arena_zarray(d->base.c->tu, Operand, nin) : NULL;
   u32 clob_int, clob_fp, abi_int, abi_fp, used_int, used_fp;
   AAAsmSavedClobber* saved;
   u32 nsaved;
   AA64Asm* a;
 
   aa_asm_clobber_masks(d->base.c, d->loc, clobbers, nclob, &clob_int, &clob_fp);
   native_asm_abi_clobber_masks(d->native, clobber_abi_sets, &abi_int, &abi_fp);
   clob_int |= abi_int;
   clob_fp |= abi_fp;
   used_int = clob_int | (1u << AA_TMP0) | (1u << AA_TMP1) | (1u << 18u) |
              (1u << AA_FP) | (1u << AA_LR) | (1u << AA_SP);
   used_fp = clob_fp | (1u << 20u) | (1u << 21u);
 
   for (u32 i = 0; i < nout; ++i) {
     const char* body = native_asm_constraint_body(outs[i].str);
     NativeAsmRegPin pin;
     if (aa_asm_resolve_pin_or_panic(d, outs[i].reg, outs[i].str, &pin)) {
       /* GNU local register variable: pin to the named hard register. */
       KitCgTypeId type = outs[i].type ? outs[i].type : out_ops[i].type;
       if (pin.cls == NATIVE_REG_FP) {
         used_fp |= 1u << pin.reg;
         clob_fp |= 1u << pin.reg;
       } else {
         used_int |= 1u << pin.reg;
         clob_int |= 1u << pin.reg;
       }
       aa_asm_bound_reg(&bound_outs[i], type, pin.cls, pin.reg);
     } else {
       NativeAsmConstraintInfo info;
       if (native_asm_constraint_reg_info(d->native, outs[i].str, &info)) {
         Reg reg = info.fixed_reg != REG_NONE
                       ? info.fixed_reg
                       : aa_asm_alloc_reg(d, info.cls, info.allowed_mask,
                                          &used_int, &used_fp);
         KitCgTypeId type = outs[i].type ? outs[i].type : out_ops[i].type;
         if (info.cls == NATIVE_REG_FP) {
           used_fp |= 1u << reg;
           if (info.fixed_reg != REG_NONE) clob_fp |= 1u << reg;
         } else {
           used_int |= 1u << reg;
           if (info.fixed_reg != REG_NONE) clob_int |= 1u << reg;
         }
         aa_asm_bound_reg(&bound_outs[i], type, info.cls, reg);
       } else if (body[0] == 'm') {
         Reg reg = aa_asm_alloc_reg(d, NATIVE_REG_INT, 0, &used_int, &used_fp);
         KitCgTypeId type = outs[i].type ? outs[i].type : out_ops[i].type;
         aa_asm_bound_mem(&bound_outs[i], type, reg);
       } else {
         aa_asm_panic(d, "unsupported output constraint");
       }
     }
   }
 
   for (u32 i = 0; i < nin; ++i) {
     const char* body = native_asm_constraint_body(ins[i].str);
     int matched = native_asm_match_index(body);
     if (matched >= 0) {
       if ((u32)matched >= nout)
         aa_asm_panic(d, "matching constraint out of range");
       if (native_asm_constraint_early(outs[matched].str))
         aa_asm_panic(d, "matching input names early-clobber output");
       if (bound_outs[matched].kind != AA64_INLINE_OPK_REG)
         aa_asm_panic(d, "matching constraint requires register output");
       bound_ins[i] = bound_outs[matched];
       continue;
     }
     NativeAsmRegPin pin;
     if (aa_asm_resolve_pin_or_panic(d, ins[i].reg, ins[i].str, &pin)) {
       /* GNU local register variable: pin to the named hard register. */
       KitCgTypeId type = ins[i].type ? ins[i].type : in_ops[i].type;
       if (pin.cls == NATIVE_REG_FP) {
         used_fp |= 1u << pin.reg;
         clob_fp |= 1u << pin.reg;
       } else {
         used_int |= 1u << pin.reg;
         clob_int |= 1u << pin.reg;
       }
       aa_asm_bound_reg(&bound_ins[i], type, pin.cls, pin.reg);
     } else {
       NativeAsmConstraintInfo info;
       if (native_asm_constraint_reg_info(d->native, ins[i].str, &info)) {
         Reg reg = info.fixed_reg != REG_NONE
                       ? info.fixed_reg
                       : aa_asm_alloc_reg(d, info.cls, info.allowed_mask,
                                          &used_int, &used_fp);
         KitCgTypeId type = ins[i].type ? ins[i].type : in_ops[i].type;
         if (info.cls == NATIVE_REG_FP) {
           used_fp |= 1u << reg;
           if (info.fixed_reg != REG_NONE) clob_fp |= 1u << reg;
         } else {
           used_int |= 1u << reg;
           if (info.fixed_reg != REG_NONE) clob_int |= 1u << reg;
         }
         aa_asm_bound_reg(&bound_ins[i], type, info.cls, reg);
       } else if (body[0] == 'i') {
         if (in_ops[i].kind != OPK_IMM)
           aa_asm_panic(d, "immediate constraint requires immediate operand");
         bound_ins[i] = in_ops[i];
       } else if (body[0] == 'm') {
         Reg reg = aa_asm_alloc_reg(d, NATIVE_REG_INT, 0, &used_int, &used_fp);
         KitCgTypeId type = ins[i].type ? ins[i].type : in_ops[i].type;
         aa_asm_bound_mem(&bound_ins[i], type, reg);
       } else {
         aa_asm_panic(d, "unsupported input constraint");
       }
     }
   }
 
   saved =
       aa_asm_save_callee_clobbers(aa_of(d->native), clob_int, clob_fp, &nsaved);
   for (u32 i = 0; i < nout; ++i) {
     if (bound_outs[i].kind == AA64_INLINE_OPK_REG) {
       NativeAllocClass cls = bound_outs[i].pad[0] == AA64_INLINE_OPCLS_FP
                                  ? NATIVE_REG_FP
                                  : NATIVE_REG_INT;
       if (outs[i].dir == KIT_CG_ASM_INOUT) {
         aa_direct_load_operand_to_reg(
             d, out_ops[i],
             native_loc_reg(bound_outs[i].type, cls,
                            (Reg)bound_outs[i].v.local));
       }
     } else if (bound_outs[i].kind == OPK_INDIRECT) {
       NativeLoc loc =
           native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT,
                          (Reg)bound_outs[i].v.ind.base);
       aa_direct_load_address_to_reg(d, out_ops[i], loc);
     }
   }
   for (u32 i = 0; i < nin; ++i) {
     if (bound_ins[i].kind == AA64_INLINE_OPK_REG) {
       NativeAllocClass cls = bound_ins[i].pad[0] == AA64_INLINE_OPCLS_FP
                                  ? NATIVE_REG_FP
                                  : NATIVE_REG_INT;
       aa_direct_load_operand_to_reg(
           d, in_ops[i],
           native_loc_reg(bound_ins[i].type, cls, (Reg)bound_ins[i].v.local));
     } else if (bound_ins[i].kind == OPK_INDIRECT) {
       NativeLoc loc =
           native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT,
                          (Reg)bound_ins[i].v.ind.base);
       aa_direct_load_address_to_reg(d, in_ops[i], loc);
     }
   }
   a = aa64_asm_open(d->base.c);
   aa64_inline_bind(a, outs, nout, bound_outs, ins, nin, bound_ins, clobbers,
                    nclob);
   aa64_asm_run_template(a, d->native->mc, tmpl);
   aa64_asm_close(a);
 
   for (u32 i = 0; i < nout; ++i) {
     NativeAllocClass cls;
     NativeLoc src;
     if (bound_outs[i].kind != AA64_INLINE_OPK_REG) continue;
     cls = bound_outs[i].pad[0] == AA64_INLINE_OPCLS_FP ? NATIVE_REG_FP
                                                        : NATIVE_REG_INT;
     src = native_loc_reg(bound_outs[i].type, cls, (Reg)bound_outs[i].v.local);
     aa_direct_store_reg_to_operand(d, out_ops[i], src);
   }
   for (u32 i = nsaved; i > 0; --i)
     aa_asm_restore_one(aa_of(d->native), &saved[i - 1u]);
 }
 
 /* ---- NativeTarget (optimizer) asm hook ----
  *
  * The optimizer has already allocated every operand register and arranged the
  * surrounding data flow (inputs are live in their registers on entry, outputs
  * are consumed from their registers on exit; the asm's clobber_mask kept the
  * allocator from holding live values in clobbered registers). So unlike the
  * direct path this hook does NOT self-allocate registers and does NOT load
  * inputs / store outputs -- it only binds the pre-allocated registers to the
  * template, materializing memory-operand base addresses into the reserved
  * scratch registers and saving/restoring callee-saved registers the asm
  * clobbers (the only ABI obligation the allocator cannot discharge itself). */
 
 static NativeAddr aa_asm_loc_to_addr(AANativeTarget* a, SrcLoc loc,
                                      NativeLoc src) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_type = src.type;
   switch ((NativeLocKind)src.kind) {
     case NATIVE_LOC_FRAME:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = src.v.frame;
       return addr;
     case NATIVE_LOC_ADDR:
       return src.v.addr;
     case NATIVE_LOC_GLOBAL:
       addr.base_kind = NATIVE_ADDR_BASE_GLOBAL;
       addr.base.global.sym = src.v.global.sym;
       addr.base.global.addend = src.v.global.addend;
       return addr;
     case NATIVE_LOC_REG:
       addr.base_kind = NATIVE_ADDR_BASE_REG;
       addr.cls = NATIVE_REG_INT;
       addr.base.reg = src.v.reg;
       return addr;
     default:
       aa_asm_panic_at(a->base.c, loc, "unsupported memory asm operand");
   }
 }
 
 /* Resolve a memory-constraint operand to a single base register with zero
  * offset, folding any frame/global/offset into a scratch register. At most the
  * two reserved scratch registers are used across one asm block. */
 static Reg aa_asm_native_mem_base(AANativeTarget* a, SrcLoc loc, NativeLoc src,
                                   u32* ntmp) {
   NativeAddr addr = aa_asm_loc_to_addr(a, loc, src);
   u32 base;
   i32 off;
   Reg dst;
   if (addr.index_kind != NATIVE_ADDR_INDEX_NONE)
     aa_asm_panic_at(a->base.c, loc, "indexed memory asm operand unsupported");
   aa_addr_base(a, addr, &base, &off);
   if (off == 0) return (Reg)base;
   if (*ntmp >= 2u)
     aa_asm_panic_at(a->base.c, loc, "too many memory asm operands");
   dst = (*ntmp == 0u) ? AA_TMP0 : AA_TMP1;
   (*ntmp)++;
   aa_emit_add_imm(a, dst, base, off);
   return dst;
 }
 
 static void aa_asm_load_loc_to_reg(AANativeTarget* a, SrcLoc loc, NativeLoc src,
                                    NativeLoc dst) {
   NativeTarget* t = &a->base;
   NativeAllocClass cls = (NativeAllocClass)dst.cls;
   if (src.kind == NATIVE_LOC_REG) {
     if (src.v.reg != dst.v.reg || src.cls != dst.cls) t->move(t, dst, src);
     return;
   }
   if (src.kind == NATIVE_LOC_IMM) {
     if (cls != NATIVE_REG_INT)
       aa_asm_panic_at(t->c, loc,
                       "floating-point immediate asm input is unsupported");
     t->load_imm(t, dst, src.v.imm);
     return;
   }
   aa_emit_mem(a, 1, dst, aa_asm_loc_to_addr(a, loc, src),
               aa_mem_for_type(t, dst.type, type_size32(t, dst.type)));
 }
 
 static void aa_asm_store_reg_to_loc(AANativeTarget* a, SrcLoc loc,
                                     NativeLoc dst, NativeLoc src) {
   NativeTarget* t = &a->base;
   if (dst.kind == NATIVE_LOC_REG) {
     if (dst.v.reg != src.v.reg || dst.cls != src.cls) t->move(t, dst, src);
     return;
   }
   aa_emit_mem(a, 0, src, aa_asm_loc_to_addr(a, loc, dst),
               aa_mem_for_type(t, src.type, type_size32(t, src.type)));
 }
 
 static void aa_asm_bind_native(AANativeTarget* a, SrcLoc loc, Operand* out,
                                const char* constraint, KitCgTypeId type,
                                NativeLoc src, u32* ntmp) {
   const char* body = native_asm_constraint_body(constraint);
   NativeAsmConstraintInfo info;
   if (native_asm_constraint_reg_info(&a->base, constraint, &info)) {
     if (src.kind != NATIVE_LOC_REG)
       aa_asm_panic_at(a->base.c, loc, "register asm operand not in a register");
     if (info.fixed_reg != REG_NONE && info.fixed_reg != (Reg)src.v.reg)
       aa_asm_panic_at(a->base.c, loc,
                       "fixed-register asm operand in wrong register");
     if (info.allowed_mask &&
         ((Reg)src.v.reg >= 32 ||
          (info.allowed_mask & (1u << (Reg)src.v.reg)) == 0))
       compiler_panic(
           a->base.c, loc,
           "aarch64 inline asm: constraint %s got cls%u reg%u outside %08x",
           constraint, (unsigned)info.cls, (unsigned)src.v.reg,
           (unsigned)info.allowed_mask);
     aa_asm_bound_reg(out, type, info.cls, (Reg)src.v.reg);
   } else if (body[0] == 'i') {
     if (src.kind != NATIVE_LOC_IMM)
       aa_asm_panic_at(a->base.c, loc, "immediate asm operand is not immediate");
     memset(out, 0, sizeof *out);
     out->kind = OPK_IMM;
     out->type = type;
     out->v.imm = src.v.imm;
   } else if (body[0] == 'm') {
     aa_asm_bound_mem(out, type, aa_asm_native_mem_base(a, loc, src, ntmp));
   } else {
     aa_asm_panic_at(a->base.c, loc, "unsupported asm constraint");
   }
 }
 
 static void aa_asm_block_native(NativeTarget* t, const char* tmpl,
                                 const AsmConstraint* outs, u32 nout,
                                 NativeLoc* out_locs, const AsmConstraint* ins,
                                 u32 nin, const NativeLoc* in_locs,
                                 const Sym* clobbers, u32 nclob) {
   AANativeTarget* a = aa_of(t);
   Compiler* c = t->c;
   SrcLoc loc = a->func ? a->func->loc : (SrcLoc){0, 0, 0};
   Operand* bound_outs = nout ? arena_zarray(c->tu, Operand, nout) : NULL;
   Operand* bound_ins = nin ? arena_zarray(c->tu, Operand, nin) : NULL;
   u8* staged_outs = nout ? arena_zarray(c->tu, u8, nout) : NULL;
   u32 ntmp = 0;
   AA64Asm* asmh;
 
   for (u32 i = 0; i < nout; ++i) {
     KitCgTypeId type = outs[i].type ? outs[i].type : out_locs[i].type;
     NativeLoc outloc = out_locs[i];
     NativeAsmPinnedLoc pinned =
         native_asm_prepare_pinned_loc(t, outs[i].reg, outs[i].str, type, outloc);
     if (pinned.has_pin) {
       if (pinned.pin_status != NATIVE_ASM_REG_PIN_OK)
         aa_asm_panic_at(c, loc,
                         native_asm_pin_status_message(pinned.pin_status));
       if (pinned.wrong_reg)
         aa_asm_panic_at(c, loc, "hard-register asm operand in wrong register");
       outloc = pinned.loc;
       if (pinned.needs_stage) {
         staged_outs[i] = 1u;
         if (outs[i].dir == KIT_CG_ASM_INOUT)
           aa_asm_load_loc_to_reg(a, loc, out_locs[i], outloc);
       }
     }
     aa_asm_bind_native(a, loc, &bound_outs[i], outs[i].str, type, outloc, &ntmp);
   }
   for (u32 i = 0; i < nin; ++i) {
     const char* body = native_asm_constraint_body(ins[i].str);
     int matched = native_asm_match_index(body);
     KitCgTypeId type;
     if (matched >= 0) {
       if ((u32)matched >= nout)
         aa_asm_panic_at(c, loc, "matching constraint out of range");
       bound_ins[i] = bound_outs[matched];
       continue;
     }
     type = ins[i].type ? ins[i].type : in_locs[i].type;
     {
       const char* in_body = native_asm_constraint_body(ins[i].str);
       NativeAsmConstraintInfo info;
       NativeLoc inloc = in_locs[i];
       NativeAsmPinnedLoc pinned =
           native_asm_prepare_pinned_loc(t, ins[i].reg, ins[i].str, type, inloc);
       /* A register-constrained input whose value is an address-taken local
        * arrives in a frame slot: the optimizer cannot keep an address-taken
        * local live in a register across the block, so the "inputs are already
        * in registers" contract does not hold for it. Load it into a reserved
        * scratch register (as the direct path does) before binding. With no
        * hard pin, only unrestricted integer constraints can use this scratch;
        * restricted register sets must already arrive in an allowed hard
        * register. */
       if (pinned.has_pin) {
         if (pinned.pin_status != NATIVE_ASM_REG_PIN_OK)
           aa_asm_panic_at(c, loc,
                           native_asm_pin_status_message(pinned.pin_status));
         if (pinned.wrong_reg)
           aa_asm_panic_at(c, loc,
                           "hard-register asm operand in wrong register");
         inloc = pinned.loc;
         if (pinned.needs_stage)
           aa_asm_load_loc_to_reg(a, loc, in_locs[i], inloc);
       } else if (native_asm_constraint_reg_info(t, ins[i].str, &info) &&
                  info.cls == NATIVE_REG_INT && info.allowed_mask == 0 &&
                  inloc.kind != NATIVE_LOC_REG) {
         Reg r;
         if (ntmp >= 2u) aa_asm_panic_at(c, loc, "too many memory asm operands");
         r = (ntmp == 0u) ? AA_TMP0 : AA_TMP1;
         ntmp++;
         inloc = native_loc_reg(type, NATIVE_REG_INT, r);
         aa_emit_mem(a, 1, inloc, aa_asm_loc_to_addr(a, loc, in_locs[i]),
                     aa_mem_for_type(t, type, type_size32(t, type)));
       }
       (void)in_body;
       aa_asm_bind_native(a, loc, &bound_ins[i], ins[i].str, type, inloc, &ntmp);
     }
   }
 
   /* No per-block callee-saved spill here: plan_frame forwarded the asm clobber
    * masks and aa_known_callee_saves folded the callee-saved ones into the
    * function's saved set, so the prologue/epilogue already preserve them. */
   asmh = aa64_asm_open(c);
   aa64_inline_bind(asmh, outs, nout, bound_outs, ins, nin, bound_ins, clobbers,
                    nclob);
   aa64_asm_run_template(asmh, t->mc, tmpl);
   aa64_asm_close(asmh);
 
   for (u32 i = 0; i < nout; ++i) {
     NativeAllocClass cls;
     NativeLoc src;
     if (!staged_outs || !staged_outs[i]) continue;
     if (bound_outs[i].kind != AA64_INLINE_OPK_REG) continue;
     cls = bound_outs[i].pad[0] == AA64_INLINE_OPCLS_FP ? NATIVE_REG_FP
                                                        : NATIVE_REG_INT;
     src = native_loc_reg(bound_outs[i].type, cls, (Reg)bound_outs[i].v.local);
     aa_asm_store_reg_to_loc(a, loc, out_locs[i], src);
   }
 }
 
 static const NativeOps aa_direct_ops = {
     .bind_param = aa_bind_param,
     .tail_call_unrealizable_reason = aa_no_tail,
     .va_start_ = aa_va_start_,
     .va_arg_ = aa_va_arg_,
     .va_end_ = aa_va_end_,
     .va_copy_ = aa_va_copy_,
     .asm_block = aa_direct_asm_block,
 };
 
 const NativeOps* aa64_native_direct_ops(void) { return &aa_direct_ops; }