kit

native.c (250345B)

---

 /* aa64 NativeTarget production-readiness checklist:
  * - ABI completeness: finish AAPCS64/Linux va_list and register-save-area
  *   lowering, verify Apple/AAPCS64/Windows arm64 differences, handle all
  *   homogeneous aggregates, indirect/byval/sret corner cases, small aggregate
  *   splitting, multi-register returns, stack alignment, and ABI diagnostics.
  * - Calls and returns: replace call-plus-return tail handling with true direct
  *   and indirect sibling calls, preserve musttail ABI guarantees, support stack
  *   argument reshuffling without clobbering live inputs, and cover all sret,
  *   variadic, FP, aggregate, and many-argument combinations.
  * - Frame lowering: implement known-frame/prologue integration for optimized
  *   emission, spill/reload hooks, callee-save tracking for integer and FP/SIMD
  *   registers, large-frame probing/materialization as needed by each platform,
  *   dynamic alloca restoration, and unwind/debug frame metadata.
  * - Operations and intrinsics: fill remaining scalar, FP, conversion, rounding,
  *   overflow, bit, vector/SIMD, trap, prefetch, and target-specific intrinsics;
  *   validate NaN/ordered/unordered FP compare semantics and integer narrowing
  *   behavior for every supported width.
  * - Aggregates and memory: support large constants, overlap-safe memmove,
  *   optimized bulk copy/set selection, bitfield load/store, packed/unaligned
  *   accesses, volatile access constraints, and record/slice edge cases across
  *   direct and optimized lowering.
  * - Atomics: replace ordinary load/store RMW/CAS sequences with correct LL/SC
  *   or LSE loops, implement acquire/release/seq_cst mappings precisely, handle
  *   failure ordering, byte/halfword/word/dword widths, and retry/clobber rules.
  * - Inline and file-scope asm: complete register/memory/immediate constraints,
  *   named operands, tied operands, early-clobber and clobber validation, hard
  *   register conflicts, memory barriers, outputs for aggregates/FP values, and
  *   file-scope asm integration. */
 
 #include <string.h>
 
 #include "abi/abi.h"
 #include "arch/aa64/aa64.h"
 #include "arch/aa64/asm.h"
 #include "arch/aa64/isa.h"
 #include "arch/aa64/regs.h"
 #include "asm/asm.h"
 #include "asm/asm_lex.h"
 #include "cg/native_argmove.h"
 #include "cg/native_asm.h"
 #include "cg/native_direct_target.h"
 #include "cg/native_frame.h"
 #include "cg/type.h"
 #include "core/arena.h"
 #include "core/bytes.h"
 #include "core/core.h"
 #include "core/pool.h"
 #include "core/slice.h"
 #include "obj/obj.h"
 
 #if defined(__GNUC__) || defined(__clang__)
 #define AA_UNUSED_FN __attribute__((unused))
 #else
 #define AA_UNUSED_FN
 #endif
 
 enum {
   AA_X8 = 8u, /* indirect-result (sret) register; usable as a copy base that
                  aa_copy_bytes (which scratches only x16/x17) never clobbers */
   /* Tail epilogues restore x30 and may use x16/x17 while rebuilding the frame,
    * so an indirect tail target is moved here only after argument marshalling.
    * x9 is an instruction-scoped emitter temp and no later operand phase runs
    * between marshal_call and the tail branch. */
   AA_TAIL_TARGET = 9u,
   AA_TMP0 = 16u,
   AA_TMP1 = 17u,
   /* Callee-saved frame-base anchor. The single-pass far-slot fast path
    * (aa_emit_mem / AA_PATCH_SLOT) addresses fixed slots at positive scaled
    * offsets from a stable bottom anchor: sp itself when the function never
    * moves sp, else this register, set to the frame base in the prologue and
    * saved / restored like a callee-save (only when the function calls alloca).
    */
   AA_FRAME_BASE = 28u,
   AA_FP = 29u,
   AA_LR = 30u,
   AA_SP = 31u,
   AA_FRAME_SAVE_SIZE = 16u,
   /* Worst-case reserved prologue region (NDT single-pass path patches it in
    * place; the optimizer path reserves exactly what it emits). Sized to hold
    * the fat prologue plus the Windows large-frame stack probe (≤7 words, see
    * aa_words_stack_probe). */
   /* Worst-case prologue/epilogue word counts. The W1.1 bottom-record general
    * path can stack: a Windows page probe (≤7), the robust `sub sp` (≤5), the
    * far saved-pair address build + stp (≤3), the x29 anchor (≤3), a callee-save
    * base build for huge frames (≤4), and up to 18 callee-save store/restores
    * (~5 int stp pairs + an int single + 8 fp singles ≈ 14 insns) — round up. */
   AA_PROLOGUE_WORDS = 48u,
   AA_TAIL_WORDS = 48u,
   /* Single-pass (-O0) tcc-style prologue. The frame-independent entry
    * `stp x29,x30,[sp,#-(16+top_home)]! ; mov x29,sp` (AA_NDT_FIXED_ENTRY_WORDS)
    * is emitted live in aa_func_begin; only the `sub sp` that grows the rest of
    * the frame below the saved pair is deferred (patched once frame_size is
    * final), plus — on stack-probing targets (Windows) — the page probe. The
    * single-pass path uses no callee-saved registers (enforced in
    * nd_scratch_acquire), so nothing else is deferred except the optional
    * alloca frame-base anchor setup: worst-case sub = load_imm(2)+3 = 5; +probe
    * load_imm(2)+5 = 7; +frame-base save/mov = 2. */
   AA_NDT_FIXED_ENTRY_WORDS = 2u,
   AA_NDT_SUB_WORDS = 14u,        /* worst case (probe + sub + frame-base) */
   AA_NDT_SUB_WORDS_NOPROBE = 7u, /* reserved region when no stack probe */
 };
 
 /* Windows/AArch64 TLS Local-Exec. The TEB pointer lives in the reserved
  * platform register x18 (never allocated; see AA_PHYS_INT_RESERVED(18)), and
  * the thread's TLS-array pointer (TEB.ThreadLocalStoragePointer) sits at
  * TEB+0x58 — same offset as on Win64/x86-64. */
 enum {
   AA_WIN_TEB_REG = 18u,
   AA_WIN_TEB_TLS_PTR_OFF = 0x58u,
 };
 
 /* ============================================================================
  * AAPCS64 frame layout
  *
  * Two layouts. Every fp- or sp-relative offset in this file is computed via one
  * of the aa_fp_off_ / aa_sp_off_ helpers below — no site does bare arithmetic
  * on AA_FP / AA_SP, and no site outside those helpers branches on the layout.
  *
  * TOP-RECORD (default — single-pass -O0, fat frames, and out_stack>0 small
  * frames). fp anchors at the caller's saved-pair address near the top; sp at
  * the bottom of the outgoing-arg area. Offsets are frame-size-independent.
  *
  *   high addr   caller's stack frame
  *               +------------------------------+
  *               | incoming stack args          |  aa_fp_off_in_arg(a,i) = 16+i
  *               +------------------------------+
  *      fp  -->  | saved x29 (prev fp)          |  aa_fp_off_saved_fp() = 0
  *               | saved x30 (prev lr)          |  aa_fp_off_saved_lr() = 8
  *               +------------------------------+
  *               | frame slots                  |  aa_fp_off_slot(a,off) = -off
  *               |   (callee-saves + locals     |
  *               |    + spills + sret/variadic) |
  *               +------------------------------+
  *               | outgoing args                |  aa_sp_off_out_arg(i)
  *      sp  -->  +------------------------------+
  *   low addr                                       CFA = fp + 16
  *
  * BOTTOM-RECORD (fp_at_bottom — the W1.1 uniform x29-at-bottom layout: every
  * known frame on the -O1 path EXCEPT slim Tier A and the Windows-variadic
  * top_home case). The frame record + slots stack ABOVE x29, anchored just above
  * the outgoing-arg area: x29 = sp + out_stack. Every slot is then a positive
  * `ldr/str [x29,#k]` — one instruction within the 32 KB scaled reach (add-build
  * `add x16,x29,#hi ; ldr [x16,#lo]` only past it), replacing the top-record
  * `sub x17,x29,#k ; ldur` fallback. x29 is the frame pointer (reserved,
  * alloca-stable, set once) so alloca falls out for free and outgoing args stay
  * sp-relative (calls after an alloca still address [sp,#k]). Offsets depend on
  * frame_size (hence known-frame only, where the frame is final before the body).
  * Let N = frame_size, os = out_stack:
  *
  *   high addr   caller's stack frame
  *               +------------------------------+
  *               | incoming stack args          |  aa_fp_off_in_arg(a,i)=(N-os)+i
  *               +------------------------------+   <- caller's sp = CFA = x29+(N-os)
  *               | frame slots (+ align pad)    |  aa_fp_off_slot(a,off)=(N-os)-off
  *               |   (callee-saves + locals …)  |     (in [16, N-os), above record)
  *               +------------------------------+
  *  x29 = sp+os->| saved x29 (prev fp)          |  aa_fp_off_saved_fp() = 0
  *               | saved x30 (prev lr)          |  aa_fp_off_saved_lr() = 8
  *               +------------------------------+
  *               | outgoing args (os)           |  aa_sp_off_out_arg(i)  ([sp,#i])
  *      sp  -->  +------------------------------+
  *   low addr
  *
  * frame_size (N) = align16(AA_FRAME_SAVE_SIZE + slot_bytes + out_stack).
  * The saved pair sits AT x29 so [x29]=caller fp, [x29+8]=caller lr — the
  * frame-pointer chain kit's unwinder / __kit_backtrace walks (uniform
  * fp[0]/fp[1]); non-negotiable. When os==0 this reduces to fp = sp and the
  * earlier folded `stp [sp,#-N]! ; mov x29,sp` encoding (the original
  * fp_at_bottom fast path). Tail calls write outgoing args into the caller's
  * incoming-args window — physically the same address, expressed via
  * aa_fp_off_tail_out_arg.
  * ========================================================================== */
 
 typedef struct AAFrameLayout {
   u32 slot_bytes; /* sum of aa_frame_slot reservations (callee-saves + locals
                    * + spills + sret/variadic) */
   u32 out_stack;  /* max outgoing-arg bytes across all calls in this function */
   u32 top_home;   /* Windows-variadic GP register home area, reserved between
                    * the saved pair and the incoming stack args so the
                    * plain-pointer va_list walks register then stack varargs as
                    * one contiguous block (0 on every other ABI). */
   u32 frame_size; /* align16(AA_FRAME_SAVE_SIZE + top_home + slot_bytes +
                    * out_stack) */
 } AAFrameLayout;
 
 static inline AAFrameLayout aa_build_layout(u32 slot_bytes, u32 out_stack,
                                             u32 top_home) {
   AAFrameLayout L;
   L.slot_bytes = slot_bytes;
   L.out_stack = out_stack;
   L.top_home = top_home;
   L.frame_size =
       align_up_u32(AA_FRAME_SAVE_SIZE + top_home + slot_bytes + out_stack, 16u);
   return L;
 }
 
 /* FP-relative byte offsets. The saved-pair is at [fp]/[fp+8] in both the
  * top-record and bottom-record (fp_at_bottom) layouts, so these two are
  * layout-independent. The frame-size-dependent helpers — aa_fp_off_in_arg,
  * aa_fp_off_slot, aa_fp_off_tail_out_arg — branch on a->fp_at_bottom and are
  * defined after AANativeTarget (see aa_fp_off_* below aa_of). */
 static inline i32 aa_fp_off_saved_fp(void) { return 0; }
 static inline i32 aa_fp_off_saved_lr(void) { return 8; }
 
 /* SP-relative byte offsets. */
 static inline i32 aa_sp_off_out_arg(u32 byte_off) { return (i32)byte_off; }
 static inline u32 aa_sp_off_saved_pair(const AAFrameLayout* L) {
   return L->frame_size - AA_FRAME_SAVE_SIZE - L->top_home;
 }
 
 /* Frame slots and callee-save records are owned by the shared NativeFrame
  * bookkeeping (cg/native_frame.h); these aliases keep the aa64-local spellings.
  */
 typedef NativeFrameSlotEntry AANativeSlot;
 
 /* Deferred in-function patches, all resolved in aa_func_end once the frame
  * layout (max_outgoing, callee-saves) is final. One growable list carries both
  * kinds; each entry patches a disjoint, fixed code position, so insertion order
  * is irrelevant. The prologue region is patched separately (exactly one per
  * function, fixed position) and is not a list entry. */
 typedef enum AAPatchKind {
   AA_PATCH_ALLOCA, /* single instr: add dst, sp, #max_outgoing */
   AA_PATCH_TAIL,   /* AA_TAIL_WORDS region: callee restores + frame + br/b */
   AA_PATCH_SLOT,   /* single instr: far fixed-slot ldr/str at a positive scaled
                     * offset from the frame base (sp, or AA_FRAME_BASE under
                     * alloca), resolved once frame_size is final. */
 } AAPatchKind;
 
 /* A deferred far fixed-slot access. The body emits a one-word placeholder
  * `ldr/str [sp,#0]`; aa_apply_patches rewrites the base register and scaled
  * offset once the frame is final. slot_off + extra is the slot's byte position
  * (as fed to aa_fp_off_slot); the positive frame-base offset is
  * aa_sp_off_saved_pair(L) + aa_fp_off_slot(slot)+extra (top-record). */
 typedef struct AASlotPatch {
   u32 slot_off; /* aa_slot(...)->off */
   i32 extra;    /* addr.offset added to the slot base */
   u8 sz;        /* size_idx: 0 (byte) / 1 (half) / 2 (word) / 3 (dword). sz<2
                  * reserves a 2nd placeholder word (overflow fallback). */
   u8 vbit;      /* SIMD/FP register (always 0 for sz<2) */
   u8 load;      /* 1 = ldr, 0 = str */
   u8 rt;        /* transfer register */
 } AASlotPatch;
 
 typedef struct AAPatch {
   AAPatchKind kind;
   u32 pos;
   union {
     u32 dst_reg;      /* AA_PATCH_ALLOCA */
     NativeLoc callee; /* AA_PATCH_TAIL */
     AASlotPatch slot; /* AA_PATCH_SLOT */
   } u;
 } AAPatch;
 
 typedef NativeFrameCalleeSave AACalleeSave;
 
 typedef struct AANativeTarget {
   NativeTarget base;
   SrcLoc loc;
   const CGFuncDesc* func;
 
   /* Shared frame bookkeeping: slot table, cumulative offset, max-outgoing,
    * callee-save set, and the known_frame / has_alloca / frame_final flags. */
   NativeFrame frame;
   /* Final frame size, set once in aa_func_begin_known_frame when fp_at_bottom
    * is decided. Read by the fp-relative offset helpers in the bottom-record
    * layout (where slot/incoming-arg offsets depend on frame_size); meaningless
    * and unread on the single-pass path, which never sets fp_at_bottom. */
   u32 frame_size_final;
   /* Final outgoing-arg area (= max_outgoing), set with frame_size_final in
    * aa_func_begin_known_frame. In the bottom-record layout x29 is anchored
    * `out_stack` bytes above sp, so every fp-relative offset is measured from
    * `frame_size - out_stack` (the bytes above the anchor). Zero on the
    * single-pass path (top-record) and on os==0 known frames (where the formula
    * reduces to the original fp_at_bottom one). */
   u32 out_stack_final;
   u32 incoming_stack_size;
   /* Windows-variadic GP register home area size (gp_reg_count * gp_slot_size,
    * 64 today; 0 on every other ABI). When nonzero the function takes the fat
    * top-record layout and homes x0..x7 into [fp + AA_FRAME_SAVE_SIZE ..] so the
    * plain-pointer va_list can walk register then stack varargs contiguously. */
   u32 top_home_bytes;
   u32 next_param_int;
   u32 next_param_fp;
   u32 next_param_stack;
   NativeFrameSlot sret_ptr_slot;
   NativeFrameSlot saved_tmp_slot;
   NativeFrameSlot va_gr_slot;
   NativeFrameSlot va_vr_slot;
 
   AAPatch* patches;
   u32 npatches;
   u32 patches_cap;
   u32 nalloca; /* count of AA_PATCH_ALLOCA entries; gates slim prologue/frame */
 
   u32 func_start;
   u32 prologue_pos;
   u32 prologue_region_words;  /* single-pass: reserved deferred-`sub` region */
   u32 minimal_prologue_words; /* opt path: exact prologue length, else 0 */
   MCLabel epilogue_label;
 
   /* Set at func_end when this function qualifies for the slim prologue/epilogue
    * (Tier A: no body locals/spills, no callee-saves, no alloca, no outgoing
    * stack args, no sret/variadic). When set, the prologue patch and epilogue
    * emit a 2-insn `stp x29,x30,[sp,#-16]! ; mov x29,sp` and matching `ldp
    * x29,x30,[sp],#16 ; ret` instead of the fat 4+3-insn FP-frame form. */
   u8 slim_prologue;
   /* Set by aa_func_begin_known_frame for the W1.1 uniform x29-at-bottom layout:
    * every known frame except slim Tier A and the Windows-variadic top_home
    * case. x29 is anchored out_stack bytes above sp with the saved pair AT x29;
    * every slot is a positive `ldr/str [x29,#k]`, slots/callee-saves stack ABOVE
    * the record, incoming args at x29 + (frame_size - out_stack) = CFA. Covers
    * out_stack>0, alloca (x29 is stable across the floated sp), and arbitrarily
    * large frames. Two prologue/epilogue encodings: a folded `stp [sp,#-N]!`
    * fast path when out_stack==0 && frame_size<=504 && !alloca, and the general
    * `sub sp ; stp [sp,#os] ; add x29,sp,#os` / `mov x16,x29 ; ldp [x16] ;
    * add sp,x16,#(N-os)` form otherwise. The frame-size-dependent offsets are
    * the reason this is only available on the known-frame path (frame final
    * before the body). Mutually exclusive with slim_prologue (Tier A). */
   u8 fp_at_bottom;
   /* L10: a stricter case of slim Tier A — a true frameless leaf. Set by
    * aa_func_begin_known_frame when, on top of the slim conditions (no
    * callee-saves, no alloca, no body slots, no outgoing stack args, no
    * sret/variadic), the function is also a LEAF (no call of any kind, so x30/LR
    * is never clobbered), contains no inline asm (which could clobber LR or make
    * a call opaquely), and never reads its own frame chain
    * (__builtin_frame_address / __builtin_return_address). Such a function needs
    * no frame record at all: the prologue/epilogue emit nothing but `ret`, and
    * the CFA stays at sp with the return address live in LR (the aa64 CIE
    * default). Implies slim_prologue=0 and fp_at_bottom=0. */
   u8 frameless;
 
   /* Single-pass far-slot fast path (Lever 1 / Fix B). When set, fixed-slot
    * loads/stores whose top-record fp offset falls outside stur's ±256 range are
    * emitted as a one-word positive scaled `ldr/str [base,#scaled]` placeholder
    * plus an AA_PATCH_SLOT, instead of the multi-insn `sub xN,x29,#off ; ldur`
    * address build. The base is a stable bottom anchor: sp (the common case) or
    * AA_FRAME_BASE when the function moves sp via alloca. Decided at func_begin
    * (off when a Windows GP home area shifts the layout); the frame-base
    * fallback is armed lazily on the first alloca. Only 4/8-byte int/fp slots
    * take this path — byte/half stay on the existing path (their positive scaled
    * reach, 4 KB/8 KB, is too small to guarantee a one-word patch). */
   u8 slot_sp_base;
   u8 uses_frame_base; /* an alloca fired: patch AA_PATCH_SLOT against
                        * AA_FRAME_BASE and set it up in the prologue. */
   NativeFrameSlot frame_base_slot; /* home for the saved caller AA_FRAME_BASE */
 } AANativeTarget;
 
 static AANativeTarget* aa_of(NativeTarget* t) { return (AANativeTarget*)t; }
 
 /* Layout-aware FP-relative offsets. Every frame use site goes through these;
  * the fp_at_bottom test lives here and nowhere else.
  *
  *   top-record (default):  record near the top, fp anchored at the saved pair.
  *     incoming args at fp+16+b, slots below fp at -off.  CFA = fp+16.
  *   bottom-record (fp_at_bottom): x29 = sp + out_stack, the saved pair AT x29,
  *     slots above the record at (frame_size - out_stack) - off, incoming args
  *     at x29 + (frame_size - out_stack).  CFA = x29 + (frame_size - out_stack).
  *     The "bytes above the anchor" is bsz = frame_size - out_stack = align16(16
  *     + slot_bytes) padding aside, so slots land in [16, bsz) and never overlap
  *     the 16-byte record. When out_stack==0, bsz = frame_size (fp = sp). */
 static inline u32 aa_fp_bottom_above(const AANativeTarget* a) {
   return a->frame_size_final - a->out_stack_final;
 }
 static inline i32 aa_fp_off_in_arg(const AANativeTarget* a, u32 byte_off) {
   /* top-record incoming args sit above the saved pair and the (usually empty)
    * Windows-variadic GP home area; bottom-record never carries a home area. */
   u32 base = a->fp_at_bottom ? aa_fp_bottom_above(a)
                              : AA_FRAME_SAVE_SIZE + a->top_home_bytes;
   return (i32)(base + byte_off);
 }
 static inline i32 aa_fp_off_slot(const AANativeTarget* a, u32 slot_off) {
   return a->fp_at_bottom ? (i32)aa_fp_bottom_above(a) - (i32)slot_off
                          : -(i32)slot_off;
 }
 /* Outgoing stack args on a tail call land in the caller's incoming-arg window —
  * the same physical address the tail-callee will read via aa_fp_off_in_arg.
  * Same helper, distinct name for site-side intent. */
 static inline i32 aa_fp_off_tail_out_arg(const AANativeTarget* a,
                                          u32 byte_off) {
   return aa_fp_off_in_arg(a, byte_off);
 }
 /* CFA = caller's sp, expressed as an fp-relative offset (fp+16 top-record,
  * x29 + (frame_size - out_stack) bottom-record). Named so the CFI emit site
  * stays layout-blind. */
 static inline i32 aa_cfa_off(const AANativeTarget* a) {
   return a->fp_at_bottom ? (i32)aa_fp_bottom_above(a)
                          : (i32)(AA_FRAME_SAVE_SIZE + a->top_home_bytes);
 }
 
 /* fp-relative offset of GP home slot `i` (Windows variadic only). The home area
  * sits just above the saved pair and just below the incoming stack args, so
  * slot gp_reg_count coincides with incoming-arg byte 0 (top-record only — a
  * function with a home area never takes a slim/bottom layout). */
 static inline i32 aa_fp_off_home_slot(u32 i) {
   return (i32)(AA_FRAME_SAVE_SIZE + i * 8u);
 }
 
 static _Noreturn void aa_panic(AANativeTarget* a, const char* msg) {
   compiler_panic(a->base.c, a->loc, "aarch64 native target: %s", msg);
 }
 
 /* Declared locally rather than pulling in debug/debug.h, keeping the
  * backend's dependency on the Debug producer to this one entry point —
  * same pattern as the x64/rv64 emit TUs (see arch/mc.h). */
 extern void debug_emit_row(Debug*, ObjSecId text_section, u32 offset, SrcLoc);
 extern void debug_func_pc_range(Debug*, ObjSecId text_section, u32 begin_ofs,
                                 u32 end_ofs);
 
 static void aa_emit32(MCEmitter* mc, u32 word) {
   /* obj_pos is a section lookup + buf_pos; only the -g line table needs it, so
    * skip it on the common no-debug compile (one fewer lookup per instruction).
    * The pre-write offset must be read BEFORE mc_emit32 advances the cursor. */
   if (mc->debug) {
     u32 ofs = obj_pos(mc->obj, mc->section_id);
     mc_emit32(mc, word);
     debug_emit_row(mc->debug, mc->section_id, ofs, mc->loc);
   } else {
     mc_emit32(mc, word);
   }
 }
 
 static void aa_patch32(ObjBuilder* obj, ObjSecId sec, u32 off, u32 word) {
   u8 b[4];
   wr_u32_le(b, word);
   obj_patch(obj, sec, off, b, sizeof b);
 }
 
 static u32 type_size32(NativeTarget* t, KitCgTypeId type) {
   u64 n = type ? cg_type_size(t->c, type) : 8u;
   if (n == 0) n = 8u;
   if (n > 16u)
     compiler_panic(t->c, (SrcLoc){0, 0, 0},
                    "aarch64 native target: scalar too large");
   return (u32)n;
 }
 
 static u32 type_align32(NativeTarget* t, KitCgTypeId type) {
   u64 n = type ? cg_type_align(t->c, type) : 8u;
   if (n == 0) n = 1u;
   if (n > 16u) n = 16u;
   return (u32)n;
 }
 
 static u32 size_idx(u32 n) {
   if (n <= 1u) return 0u;
   if (n <= 2u) return 1u;
   if (n <= 4u) return 2u;
   return 3u;
 }
 
 static u32 loc_reg(NativeLoc loc) { return loc.v.reg & 0x1fu; }
 
 /* Scalar byte width of a register operand. Reads the NDT-stamped szinfo
  * descriptor (one byte) when present, else falls back to the live type query —
  * so partial adoption stays byte-identical. */
 static u32 loc_size32(NativeTarget* t, NativeLoc loc) {
   if (loc.szinfo & NATIVE_SZINFO_VALID) return native_szinfo_size(loc.szinfo);
   return type_size32(t, loc.type);
 }
 
 /* The original predicate exactly, but sourcing the width from the cached
  * descriptor when stamped: `size == 8 || is_ptr`. is_ptr is short-circuited
  * away whenever the cached size is already 8 (the pointer case on this arch),
  * so the common path becomes a single byte test. Byte-identical to the old
  * type_size32+cg_type_is_ptr form. */
 static int loc_is_64(NativeTarget* t, NativeLoc loc) {
   return loc_size32(t, loc) == 8u || cg_type_is_ptr(t->c, loc.type);
 }
 
 /* native_loc_is_fp is shared in native_target.h. */
 
 static int aa_use_got_for_sym(NativeTarget* t, ObjSymId sym) {
   return obj_symbol_extern_via_got(t->c, t->obj, sym);
 }
 
 static RelocKind aa_ldst_reloc_for_size(u32 size) {
   switch (size) {
     case 0:
       return R_AARCH64_LDST8_ABS_LO12_NC;
     case 1:
       return R_AARCH64_LDST16_ABS_LO12_NC;
     case 2:
       return R_AARCH64_LDST32_ABS_LO12_NC;
     case 3:
       return R_AARCH64_LDST64_ABS_LO12_NC;
     default:
       return R_AARCH64_LDST64_ABS_LO12_NC;
   }
 }
 
 static u32 aa_load_imm_words(u32* out, u32 cap, u32 sf, u32 rd, i64 imm) {
   u64 v = (u64)imm;
   u32 words = sf ? 4u : 2u;
   u32 n = 0;
   for (u32 i = 0; i < words; ++i) {
     u32 part = (u32)((v >> (i * 16u)) & 0xffffu);
     if (!part && n) continue;
     if (n >= cap) return 0;
     out[n] = n ? aa64_movk(sf, rd, part, i) : aa64_movz(sf, rd, part, i);
     ++n;
   }
   if (!n) {
     if (!cap) return 0;
     out[n++] = aa64_movz(sf, rd, 0, 0);
   }
   return n;
 }
 
 static void aa_emit_load_imm(MCEmitter* mc, u32 sf, u32 rd, i64 imm) {
   u32 words[4];
   u32 n = aa_load_imm_words(words, 4u, sf, rd, imm);
   for (u32 i = 0; i < n; ++i) aa_emit32(mc, words[i]);
 }
 
 static void aa_emit_add_imm(AANativeTarget* a, u32 rd, u32 rn, i32 off) {
   u32 imm12, sh;
   MCEmitter* mc = a->base.mc;
   if (off >= 0 && aa64_addsub_imm_fits(off, &imm12, &sh)) {
     aa_emit32(mc, aa64_add_imm(1, rd, rn, imm12, sh));
     return;
   }
   if (off < 0 && aa64_addsub_imm_fits(-(i64)off, &imm12, &sh)) {
     aa_emit32(mc, aa64_sub_imm(1, rd, rn, imm12, sh));
     return;
   }
   u32 tmp = rd == rn ? (rd == AA_TMP0 ? AA_TMP1 : AA_TMP0) : rd;
   aa_emit_load_imm(mc, 1, tmp, off);
   aa_emit32(mc, aa64_add(1, rd, rn, tmp));
 }
 
 static void aa_emit_add_i64(AANativeTarget* a, u32 rd, u32 rn, i64 off) {
   u32 imm12, sh;
   MCEmitter* mc = a->base.mc;
   if (off >= 0 && aa64_addsub_imm_fits(off, &imm12, &sh)) {
     aa_emit32(mc, aa64_add_imm(1, rd, rn, imm12, sh));
     return;
   }
   if (off < 0 && aa64_addsub_imm_fits(-off, &imm12, &sh)) {
     aa_emit32(mc, aa64_sub_imm(1, rd, rn, imm12, sh));
     return;
   }
   u32 tmp = rd == rn ? (rd == AA_TMP0 ? AA_TMP1 : AA_TMP0) : rd;
   aa_emit_load_imm(mc, 1, tmp, off);
   aa_emit32(mc, aa64_add(1, rd, rn, tmp));
 }
 
 /* Unscaled load with an explicit load opcode (AA64_LDST_OPC_LDR for a plain
  * zero-extending load, AA64_LDST_OPC_LDRS_X for a sign-extending ldursb/ldursh
  * into the X register). */
 static u32 aa_ldur_op_v(u32 size, u32 v, u32 ld_opc, u32 rt, u32 rn,
                         i32 simm9) {
   return aa64_ldst_simm9_pack((AA64LdStSimm9){.size = size,
                                               .V = v,
                                               .opc = ld_opc,
                                               .imm9 = (u32)simm9 & 0x1ffu,
                                               .Rn = rn,
                                               .Rt = rt});
 }
 
 static u32 aa_ldur_v(u32 size, u32 v, u32 rt, u32 rn, i32 simm9) {
   return aa_ldur_op_v(size, v, AA64_LDST_OPC_LDR, rt, rn, simm9);
 }
 
 static u32 aa_stur_v(u32 size, u32 v, u32 rt, u32 rn, i32 simm9) {
   return aa64_ldst_simm9_pack((AA64LdStSimm9){.size = size,
                                               .V = v,
                                               .opc = AA64_LDST_OPC_STR,
                                               .imm9 = (u32)simm9 & 0x1ffu,
                                               .Rn = rn,
                                               .Rt = rt});
 }
 
 /* Scaled load with an explicit load opcode (see aa_ldur_op_v). The scale shift
  * is the access size for plain LDR; a sign-extending narrow load (opc=LDRS_X)
  * uses the same byte-size scale (the encoded imm12 is byte_off >> size). */
 static u32 aa_ldr_uimm_op_v(u32 size, u32 v, u32 ld_opc, u32 rt, u32 rn,
                             u32 byte_off) {
   u32 sc = byte_off >> size;
   return aa64_ldst_uimm_pack((AA64LdStUimm){
       .size = size, .V = v, .opc = ld_opc, .imm12 = sc, .Rn = rn, .Rt = rt});
 }
 
 static u32 aa_ldr_uimm_v(u32 size, u32 v, u32 rt, u32 rn, u32 byte_off) {
   return aa_ldr_uimm_op_v(size, v, AA64_LDST_OPC_LDR, rt, rn, byte_off);
 }
 
 static u32 aa_str_uimm_v(u32 size, u32 v, u32 rt, u32 rn, u32 byte_off) {
   u32 sc = byte_off >> size;
   return aa64_ldst_uimm_pack((AA64LdStUimm){.size = size,
                                             .V = v,
                                             .opc = AA64_LDST_OPC_STR,
                                             .imm12 = sc,
                                             .Rn = rn,
                                             .Rt = rt});
 }
 
 static u32 aa_ldr_uimm(u32 size, u32 rt, u32 rn, u32 byte_off) {
   return aa_ldr_uimm_v(size, 0, rt, rn, byte_off);
 }
 
 static u32 aa_str_uimm(u32 size, u32 rt, u32 rn, u32 byte_off) {
   return aa_str_uimm_v(size, 0, rt, rn, byte_off);
 }
 
 /* PRFM (immediate), zero byte offset. prfop is the architectural Rt field:
  * type[4:3] | level[2:1] | policy[0]. */
 static u32 aa_prfm(u32 prfop, u32 rn) {
   return 0xF9800000u | ((rn & 31u) << 5) | (prfop & 31u);
 }
 
 /* Register-offset load/store with an explicit load opcode (ld_opc is consulted
  * only when load != 0; AA64_LDST_OPC_LDRS_X gives a sign-extending ldrsb/ldrsh
  * into the X register) and an explicit index-extend `option`
  * (AA64_LDST_OPTION_*): LSL/UXTX (full X index, the default), or SXTW/UXTW for
  * a 32-bit W index widened by the addressing mode (O1-PATTERNS L8). */
 static u32 aa_ldst_regoff_opt_v(u32 size, u32 v, u32 load, u32 ld_opc, u32 rt,
                                 u32 rn, u32 rm, u32 option, u32 scaled) {
   return aa64_ldst_regoff_pack((AA64LdStRegOff){
       .size = size & 3u,
       .V = v & 1u,
       .opc = (load ? ld_opc : AA64_LDST_OPC_STR),
       .Rm = rm & 0x1fu,
       .option = option & 7u,
       .S = scaled & 1u,
       .Rn = rn & 0x1fu,
       .Rt = rt & 0x1fu});
 }
 
 static u32 aa_ldst_regoff_op_v(u32 size, u32 v, u32 load, u32 ld_opc, u32 rt,
                                u32 rn, u32 rm, u32 scaled) {
   return aa_ldst_regoff_opt_v(size, v, load, ld_opc, rt, rn, rm,
                               AA64_LDST_OPTION_LSL, scaled);
 }
 
 static u32 aa_ldst_regoff_v(u32 size, u32 v, u32 load, u32 rt, u32 rn, u32 rm,
                             u32 scaled) {
   return aa_ldst_regoff_op_v(size, v, load, AA64_LDST_OPC_LDR, rt, rn, rm,
                              scaled);
 }
 
 static u32 aa_mrs_tpidr_el0(u32 rt) { return 0xd53bd040u | (rt & 0x1fu); }
 
 /* The scalar-FP / bit-packing encoders below delegate to the single-source
  * isa.h encoders so encode/decode stay in lockstep. ftype 0=single, 1=double;
  * the historical aa_* signatures (and call sites) are preserved as thin
  * wrappers. */
 static u32 aa_fp_bin(u32 op, u32 is_double, u32 rd, u32 rn, u32 rm) {
   return aa64_fp_dp2(is_double, op, rd, rn, rm);
 }
 
 static u32 aa_fcmp(u32 is_double, u32 rn, u32 rm) {
   return aa64_fcmp_reg(is_double, rn, rm);
 }
 
 static u32 aa_fneg(u32 is_double, u32 rd, u32 rn) {
   return aa64_fp_dp1(is_double, AA64_FP_DP1_FNEG, rd, rn);
 }
 
 static u32 aa_fmov_fp(u32 is_double, u32 rd, u32 rn) {
   return aa64_fp_dp1(is_double, AA64_FP_DP1_FMOV, rd, rn);
 }
 
 /* MOV Vd.16B, Vn.16B (alias of ORR Vd.16B, Vn.16B, Vn.16B): a full 128-bit
  * SIMD register copy. Used to move binary128 / long double values, which fmov
  * (scalar, max 64-bit) would truncate. */
 static u32 aa_mov_vec16(u32 rd, u32 rn) {
   return 0x4ea01c00u | ((rn & 0x1fu) << 16) | ((rn & 0x1fu) << 5) |
          (rd & 0x1fu);
 }
 
 /* The FP<->int conversion family: sf selects the GPR width (1=64-bit), ftype
  * the FP width (1=double). Their roles flip between the convert-from-int
  * (s/ucvtf: sf=src GPR, ftype=dst FP) and convert-to-int (fcvtz*: sf=dst GPR,
  * ftype=src FP) directions, but the encoding is the same shape. */
 static u32 aa_scvtf(u32 is_double_dst, u32 is64_src, u32 fd, u32 rn) {
   return aa64_fp_int_cvt(is64_src, is_double_dst, AA64_FP_ICVT_SCVTF, fd, rn);
 }
 
 static u32 aa_ucvtf(u32 is_double_dst, u32 is64_src, u32 fd, u32 rn) {
   return aa64_fp_int_cvt(is64_src, is_double_dst, AA64_FP_ICVT_UCVTF, fd, rn);
 }
 
 static u32 aa_fcvtzs(u32 is64_dst, u32 is_double_src, u32 rd, u32 fn) {
   return aa64_fp_int_cvt(is64_dst, is_double_src, AA64_FP_ICVT_FCVTZS, rd, fn);
 }
 
 static u32 aa_fcvtzu(u32 is64_dst, u32 is_double_src, u32 rd, u32 fn) {
   return aa64_fp_int_cvt(is64_dst, is_double_src, AA64_FP_ICVT_FCVTZU, rd, fn);
 }
 
 static u32 aa_fcvt_d_s(u32 rd, u32 rn) {
   return aa64_fcvt_prec(/*src=single*/ 0, /*dst=double*/ 1, rd, rn);
 }
 
 static u32 aa_fcvt_s_d(u32 rd, u32 rn) {
   return aa64_fcvt_prec(/*src=double*/ 1, /*dst=single*/ 0, rd, rn);
 }
 
 /* fmov between GPR and FP reg: the FP ftype tracks the operand width (64-bit
  * GPR <-> double, 32-bit GPR <-> single), so sf and ftype move together. */
 static u32 aa_fmov_gpr_to_fp(u32 is64, u32 fd, u32 rn) {
   return aa64_fp_int_cvt(is64, is64, AA64_FP_ICVT_FMOV_TO_FP, fd, rn);
 }
 
 static u32 aa_fmov_fp_to_gpr(u32 is64, u32 rd, u32 fn) {
   return aa64_fp_int_cvt(is64, is64, AA64_FP_ICVT_FMOV_TO_GPR, rd, fn);
 }
 
 static u32 aa_clz(u32 sf, u32 rd, u32 rn) {
   return aa64_dp1(sf, AA64_DP1_CLZ, rd, rn);
 }
 
 static u32 aa_rbit(u32 sf, u32 rd, u32 rn) {
   return aa64_dp1(sf, AA64_DP1_RBIT, rd, rn);
 }
 
 /* REV reverses all bytes of the operand, so the 32-bit form is REV(32) and the
  * 64-bit form is REV(64) — the opcode2 follows sf rather than being constant.
  */
 static u32 aa_rev(u32 sf, u32 rd, u32 rn) {
   return aa64_dp1(sf, sf ? AA64_DP1_REV64 : AA64_DP1_REV32, rd, rn);
 }
 
 static u32 aa_sbfm(u32 sf, u32 rd, u32 rn, u32 immr, u32 imms) {
   return aa64_bitfield(sf, /*SBFM*/ 0u, immr, imms, rd, rn);
 }
 
 static u32 aa_ubfm(u32 sf, u32 rd, u32 rn, u32 immr, u32 imms) {
   return aa64_bitfield(sf, /*UBFM*/ 2u, immr, imms, rd, rn);
 }
 
 static u32 aa_ldaxr(u32 size, u32 rt, u32 rn) {
   return (size << 30) | 0x085ffc00u | ((rn & 0x1fu) << 5) | (rt & 0x1fu);
 }
 
 static u32 aa_ldxr(u32 size, u32 rt, u32 rn) {
   return (size << 30) | 0x085f7c00u | ((rn & 0x1fu) << 5) | (rt & 0x1fu);
 }
 
 static u32 aa_stlxr(u32 size, u32 rs, u32 rt, u32 rn) {
   return (size << 30) | 0x0800fc00u | ((rs & 0x1fu) << 16) |
          ((rn & 0x1fu) << 5) | (rt & 0x1fu);
 }
 
 static u32 aa_stxr(u32 size, u32 rs, u32 rt, u32 rn) {
   return (size << 30) | 0x08007c00u | ((rs & 0x1fu) << 16) |
          ((rn & 0x1fu) << 5) | (rt & 0x1fu);
 }
 
 static u32 aa_ldar(u32 size, u32 rt, u32 rn) {
   return (size << 30) | 0x08dffc00u | ((rn & 0x1fu) << 5) | (rt & 0x1fu);
 }
 
 static u32 aa_stlr(u32 size, u32 rt, u32 rn) {
   return (size << 30) | 0x089ffc00u | ((rn & 0x1fu) << 5) | (rt & 0x1fu);
 }
 
 static u32 aa_umaddl(u32 rd, u32 rn, u32 rm, u32 ra) {
   return 0x9ba00000u | ((rm & 0x1fu) << 16) | ((ra & 0x1fu) << 10) |
          ((rn & 0x1fu) << 5) | (rd & 0x1fu);
 }
 
 static u32 aa_smaddl(u32 rd, u32 rn, u32 rm, u32 ra) {
   return 0x9b200000u | ((rm & 0x1fu) << 16) | ((ra & 0x1fu) << 10) |
          ((rn & 0x1fu) << 5) | (rd & 0x1fu);
 }
 
 static u32 aa_smulh(u32 rd, u32 rn, u32 rm) {
   return 0x9b407c00u | ((rm & 0x1fu) << 16) | ((rn & 0x1fu) << 5) |
          (rd & 0x1fu);
 }
 
 static u32 aa_umulh(u32 rd, u32 rn, u32 rm) {
   return 0x9bc07c00u | ((rm & 0x1fu) << 16) | ((rn & 0x1fu) << 5) |
          (rd & 0x1fu);
 }
 
 static u32 aa_subs_reg(u32 sf, u32 rd, u32 rn, u32 rm) {
   return aa64_addsubsr_pack(
       (AA64AddSubSR){.sf = sf, .op = 1, .S = 1, .Rm = rm, .Rn = rn, .Rd = rd});
 }
 
 static void aa_emit_cmp_to_flags(NativeTarget* t, NativeLoc lhs, NativeLoc rhs);
 
 static u32 aa_add_lsl(u32 rd, u32 rn, u32 rm, u32 shift) {
   return aa64_addsubsr_pack((AA64AddSubSR){.sf = 1,
                                            .op = 0,
                                            .S = 0,
                                            .shift = 0,
                                            .Rm = rm,
                                            .imm6 = shift,
                                            .Rn = rn,
                                            .Rd = rd});
 }
 
 /* L7 shifted-register ALU forms: rd = rn <op> (rm << shift). shift type is
  * always LSL (the .shift field, 0); the shift amount is imm6. add/sub use the
  * add/sub shifted-register family, and/orr/eor the logical shifted-register
  * family. `sf` selects the 32- vs 64-bit operand width. */
 static u32 aa_addsub_lsl(u32 sf, u32 op, u32 rd, u32 rn, u32 rm, u32 shift) {
   return aa64_addsubsr_pack((AA64AddSubSR){.sf = sf,
                                            .op = op,
                                            .S = 0,
                                            .shift = 0,
                                            .Rm = rm,
                                            .imm6 = shift,
                                            .Rn = rn,
                                            .Rd = rd});
 }
 
 static u32 aa_logsr_lsl(u32 sf, u32 opc, u32 rd, u32 rn, u32 rm, u32 shift) {
   return aa64_logsr_pack((AA64LogSR){.sf = sf,
                                      .opc = opc,
                                      .shift = 0,
                                      .N = 0,
                                      .Rm = rm,
                                      .imm6 = shift,
                                      .Rn = rn,
                                      .Rd = rd});
 }
 
 static u32 aa_cset(u32 sf, u32 rd, u32 cond) {
   return aa64_csinc_enc(sf, rd, AA64_ZR, AA64_ZR, cond ^ 1u);
 }
 
 static u32 cmp_cond(CmpOp op) {
   switch (op) {
     case CMP_EQ:
       return 0x0u;
     case CMP_NE:
       return 0x1u;
     case CMP_LT_U:
       return 0x3u;
     case CMP_LE_U:
       return 0x9u;
     case CMP_GT_U:
       return 0x8u;
     case CMP_GE_U:
       return 0x2u;
     case CMP_LT_S:
       return 0xbu;
     case CMP_LE_S:
       return 0xdu;
     case CMP_GT_S:
       return 0xcu;
     case CMP_GE_S:
       return 0xau;
     /* FP predicates after FCMP set NZCV as: a<b -> N; a==b -> Z,C; a>b -> C;
      * unordered -> C,V. Each maps to a single condition except CMP_ONE_F /
      * CMP_UEQ_F (synthesized with two instructions in aa_cmp/aa_cmp_branch,
      * which intercept them before calling cmp_cond). */
     case CMP_OEQ_F:
       return 0x0u; /* EQ */
     case CMP_OLT_F:
       return 0x4u; /* MI */
     case CMP_OLE_F:
       return 0x9u; /* LS */
     case CMP_OGT_F:
       return 0xcu; /* GT */
     case CMP_OGE_F:
       return 0xau; /* GE */
     case CMP_UNE_F:
       return 0x1u; /* NE  (unordered or not-equal) */
     case CMP_ULT_F:
       return 0xbu; /* LT  (unordered or less-than) */
     case CMP_ULE_F:
       return 0xdu; /* LE  (unordered or less-or-equal) */
     case CMP_UGT_F:
       return 0x8u; /* HI  (unordered or greater-than) */
     case CMP_UGE_F:
       return 0x2u; /* CS  (unordered or greater-or-equal) */
     default:
       return 0x0u;
   }
 }
 
 static AANativeSlot* aa_slot(AANativeTarget* a, NativeFrameSlot slot) {
   return native_frame_slot_at(&a->frame, slot);
 }
 
 /* x16/x17 are the backend-private integer temporaries available to memory
  * primitives. Address formation must not silently overwrite a store payload
  * or a register index that the final memory instruction still consumes. Keep
  * those conflicts explicit at the address-materialization boundary. */
 typedef u8 AATmpAvoid;
 enum {
   AA_TMP_AVOID_0 = 1u << 0,
   AA_TMP_AVOID_1 = 1u << 1,
 };
 
 static AATmpAvoid aa_tmp_avoid_reg(Reg reg) {
   if (reg == AA_TMP0) return AA_TMP_AVOID_0;
   if (reg == AA_TMP1) return AA_TMP_AVOID_1;
   return 0u;
 }
 
 static Reg aa_addr_tmp(AANativeTarget* a, AATmpAvoid avoid) {
   if (!(avoid & AA_TMP_AVOID_0)) return AA_TMP0;
   if (!(avoid & AA_TMP_AVOID_1)) return AA_TMP1;
   aa_panic(a, "address formation has no scratch register");
   return AA_TMP0;
 }
 
 static Reg aa_addr_adjust_tmp(AANativeTarget* a, Reg base,
                               AATmpAvoid avoid) {
   AATmpAvoid base_bit = aa_tmp_avoid_reg(base);
   if (base_bit && !(avoid & base_bit)) return base;
   return aa_addr_tmp(a, avoid);
 }
 
 static AATmpAvoid aa_mem_addr_avoid(int load, NativeLoc reg,
                                     NativeAddr addr) {
   AATmpAvoid avoid = 0u;
   /* v16/v17 and x16/x17 are separate register files. Only an integer store
    * keeps its Rt live across integer address materialization. */
   if (!load && !native_loc_is_fp(reg))
     avoid |= aa_tmp_avoid_reg(loc_reg(reg));
   if (addr.index_kind == NATIVE_ADDR_INDEX_REG)
     avoid |= aa_tmp_avoid_reg(addr.index.reg);
   return avoid;
 }
 
 static void aa_addr_base(AANativeTarget* a, NativeAddr addr,
                          AATmpAvoid avoid, u32* base_out, i32* off_out) {
   *base_out = AA_TMP0;
   *off_out = addr.offset;
   switch ((NativeAddrBaseKind)addr.base_kind) {
     case NATIVE_ADDR_BASE_REG:
       *base_out = addr.base.reg;
       return;
     case NATIVE_ADDR_BASE_FRAME: {
       AANativeSlot* s = aa_slot(a, addr.base.frame);
       *base_out = AA_FP;
       *off_out = aa_fp_off_slot(a, s->off) + addr.offset;
       return;
     }
     case NATIVE_ADDR_BASE_FRAME_VALUE: {
       NativeLoc tmp;
       NativeAddr base_addr = addr;
       Reg addr_tmp = aa_addr_tmp(a, avoid);
       if (!addr.base_type)
         aa_panic(a, "frame-value base has no exact type");
       base_addr.index_kind = NATIVE_ADDR_INDEX_NONE;
       base_addr.index_ext = NATIVE_ADDR_IDX_EXT_NONE;
       base_addr.log2_scale = 0;
       memset(&tmp, 0, sizeof tmp);
       tmp.kind = NATIVE_LOC_REG;
       tmp.cls = NATIVE_REG_INT;
       tmp.type = addr.base_type;
       tmp.v.reg = addr_tmp;
       a->base.load_addr(&a->base, tmp, base_addr);
       *base_out = addr_tmp;
       *off_out = 0;
       return;
     }
     case NATIVE_ADDR_BASE_GLOBAL: {
       NativeLoc tmp;
       NativeAddr base_addr = addr;
       Reg addr_tmp = aa_addr_tmp(a, avoid);
       base_addr.index_kind = NATIVE_ADDR_INDEX_NONE;
       base_addr.index_ext = NATIVE_ADDR_IDX_EXT_NONE;
       base_addr.log2_scale = 0;
       memset(&tmp, 0, sizeof tmp);
       tmp.kind = NATIVE_LOC_REG;
       tmp.cls = NATIVE_REG_INT;
       tmp.type = builtin_id(KIT_CG_BUILTIN_I64);
       tmp.v.reg = addr_tmp;
       a->base.load_addr(&a->base, tmp, base_addr);
       *base_out = addr_tmp;
       *off_out = 0;
       return;
     }
     default:
       aa_panic(a, "unsupported address base");
   }
 }
 
 static u32 aa_ldst_q_uimm(int load, u32 rt, u32 rn, u32 byte_off);
 static u32 aa_ldst_q_simm9(int load, u32 rt, u32 rn, i32 byte_off);
 static AAPatch* aa_patch_alloc(AANativeTarget* a); /* far-slot deferral */
 
 static void aa_emit_mem_q(AANativeTarget* a, int load, NativeLoc reg,
                           NativeAddr addr) {
   u32 base, rt;
   i32 off;
   MCEmitter* mc = a->base.mc;
   if (addr.index_kind != NATIVE_ADDR_INDEX_NONE)
     aa_panic(a, "unsupported q-register indexed memory access");
   aa_addr_base(a, addr, 0u, &base, &off);
   rt = loc_reg(reg);
   if (off >= 0 && (((u32)off & 15u) == 0) && ((u32)off >> 4) <= 0xfffu) {
     aa_emit32(mc, aa_ldst_q_uimm(load, rt, base, (u32)off));
     return;
   }
   if (off >= -256 && off <= 255) {
     aa_emit32(mc, aa_ldst_q_simm9(load, rt, base, off));
     return;
   }
   aa_emit_add_imm(a, AA_TMP1, base, off);
   aa_emit32(mc, aa_ldst_q_uimm(load, rt, AA_TMP1, 0));
 }
 
 static void aa_emit_mem_native(AANativeTarget* a, int load, NativeLoc reg,
                                NativeAddr addr, MemAccess mem) {
   u32 base, rt, sz, bytes, ld_opc;
   AATmpAvoid addr_avoid;
   i32 off;
   MCEmitter* mc = a->base.mc;
   rt = loc_reg(reg);
   addr_avoid = aa_mem_addr_avoid(load, reg, addr);
   bytes = mem.size
               ? mem.size
               : type_size32(&a->base, reg.type ? reg.type : mem.type);
   if (native_loc_is_fp(reg)) {
     if (bytes != 4u && bytes != 8u && bytes != 16u)
       aa_panic(a, "unsupported native fp memory width");
   } else if (bytes != 1u && bytes != 2u && bytes != 4u && bytes != 8u) {
     aa_panic(a, "unsupported native integer memory width");
   }
   sz = size_idx(bytes);
   if (native_loc_is_fp(reg) && bytes == 16u) {
     aa_emit_mem_q(a, load, reg, addr);
     return;
   }
   if (native_loc_is_fp(reg) && sz < 2u) sz = 2u;
   /* Lever 4: a signed narrow integer load (MF_SEXT_LOAD, byte/half, integer
    * register) becomes a sign-extending load into the X register (ldrsb/ldrsh,
    * opc=10) so it fills the whole register in one instruction — the cg layer
    * then drops the redundant CV_SEXT. Every other load (incl. fp, word, dword)
    * keeps the plain zero-extending LDR. */
   ld_opc =
       (load && (mem.flags & MF_SEXT_LOAD) && !native_loc_is_fp(reg) && sz <= 1u)
           ? AA64_LDST_OPC_LDRS_X
           : AA64_LDST_OPC_LDR;
   /* Far fixed-slot fast path (Lever 1/3 / Fix B). A plain frame slot whose
    * top-record fp offset is past stur's ±256 range would otherwise cost `sub
    * xN,x29,#off (+movk) ; ldur` (2-4 insns). Instead emit a positive scaled
    * `ldr/str [sp,#0]` placeholder and defer the offset: once the frame is
    * final, aa_apply_patches rewrites it to `[base, #frame_size-...]`, base = sp
    * (stable) or AA_FRAME_BASE (alloca).
    *
    * The base-relative offset is frame_size - 16 - slot_off, unknown here
    * (single-pass: the frame is still growing) and able to exceed the scaled
    * ldr/str reach (4 KB/8 KB/16 KB/32 KB by size) for ANY access size — a
    * top-of-frame 8-byte slot sits ~frame_size off sp, and kit's own
    * src/api/package.c reaches a ~69 KB frame. So every size reserves TWO words:
    * when the resolved offset fits it is `ldr/str [base,#scaled] ; nop`, and when
    * it overflows the resolver emits the address-build fallback `add x17,base,#hi
    * ; ldr/str [x17,#lo]` (correct for any frame < 16 MB).
    *
    * Lever 3/4 interaction: a far SIGNED narrow load (ld_opc==LDRS_X) is
    * excluded from this scaled-slot path — the slot patch records only a plain
    * LDR opcode, so it would zero-extend while the cg layer dropped the CV_SEXT.
    * Such loads fall through to the general path below, which honors ld_opc
    * (ldrsb/ldrsh with an address build). Rare (a far signed char/short slot);
    * correctness over the one-insn win. */
   int sext_far = load && ld_opc == AA64_LDST_OPC_LDRS_X;
   if (a->slot_sp_base && addr.base_kind == NATIVE_ADDR_BASE_FRAME &&
       addr.index_kind == NATIVE_ADDR_INDEX_NONE && sz <= 3u && !sext_far) {
     AANativeSlot* s = aa_slot(a, addr.base.frame);
     i32 fp_off = aa_fp_off_slot(a, s->off) + addr.offset;
     /* The placeholder resolves to a scaled `ldr/str [base,#imm<<sz]`, whose
      * immediate must be a multiple of the access size. The frame base
      * (sp / AA_FRAME_BASE) is 16-aligned at this point (slot_sp_base is off
      * whenever a Windows GP home area shifts the layout), so the resolved
      * offset is a multiple of the access size exactly when fp_off is. An
      * aggregate copy chunks an under-aligned record (e.g. a 4-aligned 12-byte
      * struct copied as 8+4 bytes) into an access whose frame offset is not a
      * multiple of the chunk size; such an access can't use the scaled form, so
      * it falls through to the fp-relative address-build path below, which
      * encodes any offset. */
     u32 acc = 1u << sz;
     if (fp_off < -256 && ((u32)fp_off & (acc - 1u)) == 0u) {
       u32 vbit = native_loc_is_fp(reg) ? 1u : 0u;
       AAPatch* p = aa_patch_alloc(a);
       p->kind = AA_PATCH_SLOT;
       p->pos = mc_pos(mc);
       p->u.slot.slot_off = s->off;
       p->u.slot.extra = addr.offset;
       p->u.slot.sz = (u8)sz;
       p->u.slot.vbit = (u8)vbit;
       p->u.slot.load = (u8)(load ? 1u : 0u);
       p->u.slot.rt = (u8)rt;
       aa_emit32(mc, load ? aa_ldr_uimm_v(sz, vbit, rt, AA_SP, 0)
                          : aa_str_uimm_v(sz, vbit, rt, AA_SP, 0));
       /* Reserve a second word for the address-build fallback. The final
        * base-relative offset is frame_size - 16 - slot_off, which is unknown
        * here (single-pass: the frame keeps growing) and can exceed the scaled
        * ldr/str reach for ANY access size in a large frame (a top-of-frame
        * 8-byte slot sits ~frame_size off sp; kit's own src/api/package.c has a
        * ~69 KB frame). So every size reserves the fallback word; aa_apply_patches
        * leaves it a nop when the one-word scaled form fits. */
       aa_emit32(mc, aa64_nop());
       return;
     }
   }
   if (addr.base_kind == NATIVE_ADDR_BASE_GLOBAL &&
       addr.index_kind == NATIVE_ADDR_INDEX_NONE) {
     i64 addend = addr.base.global.addend + (i64)addr.offset;
     u32 scratch = aa_addr_tmp(a, addr_avoid);
     u32 pos = mc_pos(mc);
     if (aa_use_got_for_sym(&a->base, addr.base.global.sym)) {
       aa_emit32(mc, aa64_adrp(scratch, 0, 0));
       mc_emit_reloc_at(mc, mc->section_id, pos, R_AARCH64_ADR_GOT_PAGE,
                        addr.base.global.sym, 0, 0, 0);
       pos = mc_pos(mc);
       aa_emit32(mc, aa_ldr_uimm(3, scratch, scratch, 0));
       mc_emit_reloc_at(mc, mc->section_id, pos, R_AARCH64_LD64_GOT_LO12_NC,
                        addr.base.global.sym, 0, 0, 0);
       if (addend) aa_emit_add_i64(a, scratch, scratch, addend);
       aa_emit32(
           mc,
           load ? aa_ldur_op_v(sz, native_loc_is_fp(reg), ld_opc, rt, scratch, 0)
                : aa_stur_v(sz, native_loc_is_fp(reg), rt, scratch, 0));
       return;
     }
     aa_emit32(mc, aa64_adrp(scratch, 0, 0));
     mc_emit_reloc_at(mc, mc->section_id, pos, R_AARCH64_ADR_PREL_PG_HI21,
                      addr.base.global.sym, addend, 0, 0);
     pos = mc_pos(mc);
     aa_emit32(mc,
               load ? aa_ldr_uimm_op_v(sz, native_loc_is_fp(reg), ld_opc, rt,
                                       scratch, 0)
                    : aa_str_uimm_v(sz, native_loc_is_fp(reg), rt, scratch, 0));
     mc_emit_reloc_at(mc, mc->section_id, pos, aa_ldst_reloc_for_size(sz),
                      addr.base.global.sym, addend, 0, 0);
     return;
   }
   aa_addr_base(a, addr, addr_avoid, &base, &off);
   if (addr.index_kind != NATIVE_ADDR_INDEX_NONE) {
     u32 use_base = base;
     u32 scaled = 0;
     /* L8 index extend: a 32-bit W index widened by the addressing mode. The
      * Rm field still names the same register number; only the option bits
      * change (010=UXTW, 110=SXTW). NONE keeps the full-width LSL/UXTX form. */
     u32 option = AA64_LDST_OPTION_LSL;
     if (addr.index_kind != NATIVE_ADDR_INDEX_REG)
       aa_panic(a, "unsupported address index");
     if (addr.index_ext == NATIVE_ADDR_IDX_EXT_SXTW)
       option = AA64_LDST_OPTION_SXTW;
     else if (addr.index_ext == NATIVE_ADDR_IDX_EXT_UXTW)
       option = AA64_LDST_OPTION_UXTW;
     if (off) {
       use_base = aa_addr_adjust_tmp(a, base, addr_avoid);
       aa_emit_add_imm(a, use_base, base, off);
     }
     if (addr.log2_scale == 0) {
       scaled = 0;
     } else if (addr.log2_scale == sz) {
       scaled = 1;
     } else {
       aa_panic(a, "unsupported memory address scale");
     }
     aa_emit32(mc, aa_ldst_regoff_opt_v(sz, native_loc_is_fp(reg), load, ld_opc,
                                        rt, use_base, addr.index.reg, option,
                                        scaled));
     return;
   }
   if (off >= 0 && (((u32)off & ((1u << sz) - 1u)) == 0) &&
       ((u32)off >> sz) <= 0xfffu) {
     aa_emit32(mc, load ? aa_ldr_uimm_op_v(sz, native_loc_is_fp(reg), ld_opc, rt,
                                           base, (u32)off)
                        : aa_str_uimm_v(sz, native_loc_is_fp(reg), rt, base,
                                        (u32)off));
     return;
   }
   if (off >= -256 && off <= 255) {
     aa_emit32(mc, load ? aa_ldur_op_v(sz, native_loc_is_fp(reg), ld_opc, rt,
                                       base, off)
                        : aa_stur_v(sz, native_loc_is_fp(reg), rt, base, off));
     return;
   }
   {
     u32 scratch = aa_addr_adjust_tmp(a, base, addr_avoid);
     aa_emit_add_imm(a, scratch, base, off);
     aa_emit32(mc, load ? aa_ldur_op_v(sz, native_loc_is_fp(reg), ld_opc, rt,
                                       scratch, 0)
                        : aa_stur_v(sz, native_loc_is_fp(reg), rt, scratch, 0));
   }
 }
 
 static u32 aa_part_transfer_tmp(AANativeTarget* a, NativeLoc reg,
                                 NativeAddr addr) {
   u32 used = 1u << loc_reg(reg);
   if (addr.base_kind == NATIVE_ADDR_BASE_REG) used |= 1u << addr.base.reg;
   if (addr.index_kind == NATIVE_ADDR_INDEX_REG) used |= 1u << addr.index.reg;
   if (!(used & (1u << AA_TMP0))) return AA_TMP0;
   if (!(used & (1u << AA_TMP1))) return AA_TMP1;
   aa_panic(a, "exact ABI part transfer has no scratch register");
   return AA_TMP0;
 }
 
 /* A register carrier is 8 bytes, but the final part of a small aggregate may
  * carry any byte count from 1 through 8. Keep the one-instruction native-width
  * path; decompose 3/5/6/7-byte memory transfers into exact low-to-high chunks.
  * This is deliberately below call marshalling so incoming binds, outgoing
  * args, and generic NativeCallPhaseRet write phases all share the same rule. */
 static void aa_emit_mem(AANativeTarget* a, int load, NativeLoc reg,
                         NativeAddr addr, MemAccess mem) {
   u32 bytes = mem.size
                   ? mem.size
                   : type_size32(&a->base, reg.type ? reg.type : mem.type);
   NativePartChunkIter it;
   NativeLoc lane;
   u32 chunk_off, chunk_size, regno, tmp;
   if (bytes == 1u || bytes == 2u || bytes == 4u || bytes == 8u ||
       (native_loc_is_fp(reg) && bytes == 16u)) {
     aa_emit_mem_native(a, load, reg, addr, mem);
     return;
   }
   if (native_loc_is_fp(reg) || bytes == 0u || bytes > 8u)
     aa_panic(a, "unsupported exact ABI part memory width");
 
   regno = loc_reg(reg);
   /* A native-width load may overwrite its address register because it is one
    * instruction (`ldr x0, [x0]`). An exact-width load is a sequence, so retain
    * the effective address in a backend-private register when dst aliases its
    * base or index. The other private register remains available for packing
    * later chunks into dst. */
   if (load &&
       ((addr.base_kind == NATIVE_ADDR_BASE_REG && addr.base.reg == regno) ||
        (addr.index_kind == NATIVE_ADDR_INDEX_REG &&
         addr.index.reg == regno))) {
     NativeLoc addr_loc;
     NativeAddr retained;
     u32 addr_reg = regno == AA_TMP0 ? AA_TMP1 : AA_TMP0;
     if (regno == AA_TMP0 || regno == AA_TMP1)
       aa_panic(a, "exact load destination aliases private address register");
     addr_loc = native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64),
                               NATIVE_REG_INT, addr_reg);
     a->base.load_addr(&a->base, addr_loc, addr);
     memset(&retained, 0, sizeof retained);
     retained.base_kind = NATIVE_ADDR_BASE_REG;
     retained.base.reg = addr_reg;
     retained.base_type = addr_loc.type;
     addr = retained;
   }
   tmp = aa_part_transfer_tmp(a, reg, addr);
   it = native_part_chunks(bytes, 8u);
   while (native_part_chunk_next(&it, &chunk_off, &chunk_size)) {
     NativeAddr chunk_addr = addr;
     MemAccess chunk_mem = mem;
     chunk_addr.offset += (i32)chunk_off;
     chunk_mem.size = chunk_size;
     if (chunk_mem.align > chunk_size) chunk_mem.align = chunk_size;
     lane = reg;
     if (chunk_off == 0u) {
       lane.v.reg = regno;
     } else {
       lane.v.reg = tmp;
       if (!load)
         aa_emit32(a->base.mc,
                   aa_ubfm(1u, tmp, regno, chunk_off * 8u, 63u));
     }
     aa_emit_mem_native(a, load, lane, chunk_addr, chunk_mem);
     if (load && chunk_off != 0u)
       aa_emit32(a->base.mc,
                 aa_logsr_lsl(1u, 1u, regno, regno, tmp, chunk_off * 8u));
   }
 }
 
 static NativeAllocClass aa_class_for_type(NativeTarget* t, KitCgTypeId type) {
   if (type && cg_type_is_float(t->c, type) && cg_type_size(t->c, type) <= 8u)
     return NATIVE_REG_FP;
   return NATIVE_REG_INT;
 }
 
 static int aa_addr_legal(NativeTarget* t, const NativeAddr* addr,
                          MemAccess mem) {
   u32 sz;
   (void)t;
   if (!addr) return 0;
   if (addr->index_kind == NATIVE_ADDR_INDEX_NONE) return 1;
   if (addr->index_kind != NATIVE_ADDR_INDEX_REG) return 0;
   /* The SXTW/UXTW index-extend (L8) is encoded in the same regoff form as the
    * plain LSL/UXTX index, so the scale legality is identical: scale 0 (no
    * shift) or scale == access size (S bit). */
   if (addr->log2_scale == 0) return 1;
   sz = size_idx(mem.size ? mem.size : 8u);
   return addr->log2_scale == sz;
 }
 
 /* O1-PATTERNS rider capabilities (see native_target.h). aa64 emits both folded
  * forms: the shifted-register ALU op (L7, aa_binop) and the SXTW/UXTW
  * index-extend addressing mode (L8, aa_emit_mem). */
 static int aa_can_fold_shift_into_alu(NativeTarget* t) {
   (void)t;
   return 1;
 }
 
 static int aa_can_fold_extend_into_addr(NativeTarget* t) {
   (void)t;
   return 1;
 }
 
 /* True if `mul Rd, Rn, #c` can be replaced by a single non-mul aarch64
  * instruction using only Rn as a source (no extra scratch reg). Constants
  * that match: 0, 1, -1, +/-2^k, 2^k+1, 1-2^k for k in [1..width-1]. The
  * shift exponent must fit imm6 for the operand width (width = 32 if !sf
  * else 64). The emit side is aa_emit_mul_const_imm. */
 static int aa64_imul_strength_reducible(u32 sf, i64 imm) {
   u32 max_sh = sf ? 63u : 31u;
   u64 a;
   if (imm == 0 || imm == 1 || imm == -1) return 1;
   /* +2^k */
   a = (u64)imm;
   if (imm > 0 && (a & (a - 1u)) == 0u) {
     u32 k = (u32)__builtin_ctzll(a);
     return k <= max_sh;
   }
   /* -2^k */
   if (imm < 0) {
     a = (u64)(-imm);
     if (a && (a & (a - 1u)) == 0u) {
       u32 k = (u32)__builtin_ctzll(a);
       return k >= 1u && k <= max_sh;
     }
   }
   /* 2^k + 1 (k >= 1, so c >= 3) */
   if (imm >= 3) {
     u64 m = (u64)(imm - 1);
     if ((m & (m - 1u)) == 0u) {
       u32 k = (u32)__builtin_ctzll(m);
       return k >= 1u && k <= max_sh;
     }
   }
   /* 1 - 2^k (k >= 1, so c <= -1) */
   if (imm <= -1) {
     u64 m = (u64)(1 - imm);
     if (m && (m & (m - 1u)) == 0u) {
       u32 k = (u32)__builtin_ctzll(m);
       return k >= 1u && k <= max_sh;
     }
   }
   return 0;
 }
 
 /* Which constant operands the backend can fold directly into an instruction
  * (so the optimizer can leave them as immediates instead of materializing a
  * register). Currently: add/sub/cmp 12-bit immediates (optionally <<12),
  * any value for a plain register move (movz/movk synthesizes it), and
  * strength-reducible mul constants (handled in aa_binop via shift / shifted
  * add or sub). */
 static int aa_imm_legal(NativeTarget* t, NativeImmUse use, u32 op,
                         KitCgTypeId type, i64 imm) {
   u32 imm12, sh;
   switch (use) {
     case NATIVE_IMM_BINOP:
       if ((BinOp)op == BO_IADD || (BinOp)op == BO_ISUB)
         return aa64_addsub_imm_fits(imm < 0 ? -imm : imm, &imm12, &sh);
       if ((BinOp)op == BO_IMUL) {
         u32 sf = type_size32(t, type) == 8u ? 1u : 0u;
         return aa64_imul_strength_reducible(sf, imm);
       }
       /* LSL/LSR/ASR #imm via the UBFM/SBFM aliases: shift count in range. */
       if ((BinOp)op == BO_SHL || (BinOp)op == BO_SHR_S ||
           (BinOp)op == BO_SHR_U) {
         u32 bits = type_size32(t, type) == 8u ? 64u : 32u;
         return imm >= 0 && (u64)imm < (u64)bits;
       }
       /* AND/ORR/EOR #bitmask: encodable as an AArch64 logical immediate. */
       if ((BinOp)op == BO_AND || (BinOp)op == BO_OR || (BinOp)op == BO_XOR) {
         u32 sf = type_size32(t, type) == 8u ? 1u : 0u;
         u32 N, immr, imms;
         return aa64_logimm_encode((u64)imm, sf, &N, &immr, &imms);
       }
       return 0;
     case NATIVE_IMM_CMP:
       /* cmp lowers to subs #imm12; cmn (negative) is not wired, so require a
        * non-negative immediate. */
       return imm >= 0 && aa64_addsub_imm_fits(imm, &imm12, &sh);
     case NATIVE_IMM_ADDR_OFFSET:
       return aa64_addsub_imm_fits(imm < 0 ? -imm : imm, &imm12, &sh);
     case NATIVE_IMM_MOVE:
       return 1;
   }
   return 0;
 }
 
 static int aa_addr_index_is_reg(const NativeAddr* addr, u32 reg) {
   return addr->index_kind == NATIVE_ADDR_INDEX_REG && addr->index.reg == reg;
 }
 
 static u32 aa_tmp_avoiding(u32 reg) {
   return reg == AA_TMP0 ? AA_TMP1 : AA_TMP0;
 }
 
 static void aa_load_addr_from_base(AANativeTarget* a, u32 rd, u32 base, i64 off,
                                    const NativeAddr* addr) {
   if (addr->index_kind == NATIVE_ADDR_INDEX_NONE) {
     if (rd != base || off) aa_emit_add_i64(a, rd, base, off);
     return;
   }
   if (addr->index_kind != NATIVE_ADDR_INDEX_REG)
     aa_panic(a, "unsupported address index");
   if (addr->log2_scale > 4u) aa_panic(a, "unsupported address scale");
 
   /* Read the index before writing rd. O1 can legally select the same physical
    * register for an address result and its dead-after-use index; materializing
    * the base into rd first would turn `base + index` into `base + base`. */
   u32 idx_reg = addr->index.reg;
   if (addr->index_ext != NATIVE_ADDR_IDX_EXT_NONE) {
     /* L8 extend rider on an address-materialization (LEA-shaped) use: the
      * shifted-register add takes a full-width X index, so first widen the
      * 32-bit W index into a scratch (sxtw/uxtw) before the scaled add. The
      * regoff memory form (aa_emit_mem) folds the extend directly; this path is
      * the fallback when the indexed indirect is materialized as an address. */
     u32 tmp = aa_tmp_avoiding(rd == base ? base : rd);
     if (tmp == base) tmp = aa_tmp_avoiding(base);
     if (addr->index_ext == NATIVE_ADDR_IDX_EXT_SXTW)
       aa_emit32(a->base.mc, aa_sbfm(1, tmp, idx_reg, 0, 31));
     else
       aa_emit32(a->base.mc, aa_ubfm(1, tmp, idx_reg, 0, 31));
     idx_reg = tmp;
   }
   aa_emit32(a->base.mc, aa_add_lsl(rd, base, idx_reg, addr->log2_scale));
   if (off) aa_emit_add_i64(a, rd, rd, off);
 }
 
 static void aa_materialize_frame_index(AANativeTarget* a, NativeAddr* addr,
                                        u32 avoid_reg) {
   NativeAddr load;
   NativeLoc idx;
   MemAccess mem;
   u32 reg;
   if (addr->index_kind != NATIVE_ADDR_INDEX_FRAME_VALUE) return;
   if (!addr->index_type)
     aa_panic(a, "frame-value index has no exact type");
   reg = avoid_reg == AA_TMP1 ? AA_TMP0 : AA_TMP1;
   memset(&load, 0, sizeof load);
   load.base_kind = NATIVE_ADDR_BASE_FRAME;
   load.base.frame = addr->index.frame;
   load.base_type = addr->index_type;
   memset(&idx, 0, sizeof idx);
   idx.kind = NATIVE_LOC_REG;
   idx.cls = NATIVE_REG_INT;
   idx.type = load.base_type;
   idx.v.reg = reg;
   memset(&mem, 0, sizeof mem);
   mem.type = load.base_type;
   mem.size = type_size32(&a->base, load.base_type);
   mem.align = type_align32(&a->base, load.base_type);
   aa_emit_mem(a, 1, idx, load, mem);
   addr->index_kind = NATIVE_ADDR_INDEX_REG;
   addr->index.reg = reg;
 }
 
 static NativeLoc native_loc_reg(KitCgTypeId type, NativeAllocClass cls,
                                 Reg reg);
 
 static u32 aa_ldst_q_uimm(int load, u32 rt, u32 rn, u32 byte_off) {
   return aa64_ldst_uimm_pack((AA64LdStUimm){.size = 0,
                                             .V = 1,
                                             .opc = load ? 3u : 2u,
                                             .imm12 = byte_off >> 4,
                                             .Rn = rn,
                                             .Rt = rt});
 }
 
 static u32 aa_ldst_q_simm9(int load, u32 rt, u32 rn, i32 byte_off) {
   return aa64_ldst_simm9_pack((AA64LdStSimm9){.size = 0,
                                               .V = 1,
                                               .opc = load ? 3u : 2u,
                                               .imm9 = (u32)byte_off & 0x1ffu,
                                               .Rn = rn,
                                               .Rt = rt});
 }
 
 static void aa_emit_q_frame(AANativeTarget* a, int load, u32 qreg,
                             NativeFrameSlot slot, u32 offset) {
   AANativeSlot* s = aa_slot(a, slot);
   i32 off = aa_fp_off_slot(a, s->off) + (i32)offset;
   MCEmitter* mc = a->base.mc;
   if (off >= 0 && ((u32)off & 15u) == 0 && ((u32)off >> 4) <= 0xfffu) {
     aa_emit32(mc, aa_ldst_q_uimm(load, qreg, AA_FP, (u32)off));
     return;
   }
   if (off >= -256 && off <= 255) {
     aa_emit32(mc, aa_ldst_q_simm9(load, qreg, AA_FP, off));
     return;
   }
   aa_emit_add_imm(a, AA_TMP1, AA_FP, off);
   aa_emit32(mc, aa_ldst_q_uimm(load, qreg, AA_TMP1, 0));
 }
 
 /* Reserve the variadic register-save-area frame slots (gp then fp). Split from
  * the store emission so the known-frame path can fix the full frame — including
  * these slots — before the prologue, then emit the stores after it. */
 static void aa_reserve_variadic_reg_saves(AANativeTarget* a) {
   NativeFrameSlotDesc sd;
   KitCgTypeId i64 = builtin_id(KIT_CG_BUILTIN_I64);
   ABIVaListInfo vai = abi_va_list_layout(a->base.c->abi);
   if (vai.kind != ABI_VA_LIST_AAPCS64) return;
   memset(&sd, 0, sizeof sd);
   sd.type = i64;
   sd.size = vai.gp_reg_count * vai.gp_slot_size;
   sd.align = 8;
   sd.kind = NATIVE_FRAME_SLOT_SAVE;
   a->va_gr_slot = a->base.frame_slot(&a->base, &sd);
   sd.size = vai.fp_reg_count * vai.fp_slot_size;
   sd.align = 16;
   a->va_vr_slot = a->base.frame_slot(&a->base, &sd);
 }
 
 /* Emit the stores into the variadic register-save area. For AAPCS64 these land
  * in the reserved gr/vr frame slots (aa_reserve_variadic_reg_saves); for the
  * Windows GP home area they land in [fp + AA_FRAME_SAVE_SIZE ..], the
  * top-of-frame block contiguous with the incoming stack args. */
 static void aa_emit_variadic_reg_save_stores(AANativeTarget* a) {
   NativeAddr addr;
   MemAccess mem;
   KitCgTypeId i64 = builtin_id(KIT_CG_BUILTIN_I64);
   ABIVaListInfo vai = abi_va_list_layout(a->base.c->abi);
   if (vai.kind == ABI_VA_LIST_POINTER && a->top_home_bytes) {
     /* Windows: home x0..x{gp_reg_count-1} so the plain-pointer va_list walks
      * register then stack varargs as one block. The named leading registers are
      * homed too (harmless): va_start skips past them. */
     memset(&mem, 0, sizeof mem);
     mem.type = i64;
     mem.size = 8;
     mem.align = 8;
     memset(&addr, 0, sizeof addr);
     addr.base_kind = NATIVE_ADDR_BASE_REG;
     addr.base.reg = AA_FP;
     addr.base_type = i64;
     for (u32 r = 0; r < vai.gp_reg_count && r < 8u; ++r) {
       NativeLoc src = native_loc_reg(i64, NATIVE_REG_INT, r);
       addr.offset = aa_fp_off_home_slot(r);
       aa_emit_mem(a, 0, src, addr, mem);
     }
     return;
   }
   if (vai.kind != ABI_VA_LIST_AAPCS64) return;
   memset(&mem, 0, sizeof mem);
   mem.type = i64;
   mem.size = 8;
   mem.align = 8;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = a->va_gr_slot;
   addr.base_type = i64;
   for (u32 r = 0; r < vai.gp_reg_count && r < 8u; ++r) {
     NativeLoc src = native_loc_reg(i64, NATIVE_REG_INT, r);
     addr.offset = (i32)(r * vai.gp_slot_size);
     aa_emit_mem(a, 0, src, addr, mem);
   }
   for (u32 r = 0; r < vai.fp_reg_count && r < 8u; ++r)
     aa_emit_q_frame(a, 0, r, a->va_vr_slot, r * vai.fp_slot_size);
 }
 
 static void aa_emit_entry_saves(AANativeTarget* a);
 
 /* Per-function state reset + function-symbol / cfi / prologue-anchor setup
  * shared by both entry points (aa_func_begin for the single-pass path,
  * aa_func_begin_known_frame for the optimizer path). Emits no prologue. */
 static void aa_func_begin_common(NativeTarget* t, const CGFuncDesc* fd) {
   AANativeTarget* a = aa_of(t);
   MCEmitter* mc = t->mc;
   a->func = fd;
   /* Shared frame bookkeeping: clears the slot table, cum_off, max_outgoing,
    * callee-save set, and known_frame/has_alloca/frame_final. cum_off counts
    * frame-slot bytes below fp; the saved fp/lr pair (16 bytes at [fp, fp+8]) is
    * *not* part of it — aa_build_layout adds it in aa_func_end. */
   native_frame_reset(&a->frame);
   a->incoming_stack_size = 0;
   a->next_param_int = 0;
   a->next_param_fp = 0;
   /* 0-based byte cursor for incoming stack args (also reported as the
    * caller's incoming_stack_size for tail-call realizability). bind_param
    * forms its fp-relative address via aa_fp_off_in_arg(next_param_stack),
    * which adds the saved-pair offset. */
   a->next_param_stack = 0;
   a->sret_ptr_slot = NATIVE_FRAME_SLOT_NONE;
   a->saved_tmp_slot = NATIVE_FRAME_SLOT_NONE;
   a->va_gr_slot = NATIVE_FRAME_SLOT_NONE;
   a->va_vr_slot = NATIVE_FRAME_SLOT_NONE;
   a->npatches = 0;
   a->nalloca = 0;
   a->slim_prologue = 0;
   a->fp_at_bottom = 0;
   a->frameless = 0;
   a->frame_size_final = 0;
   a->out_stack_final = 0;
   a->slot_sp_base = 0;
   a->uses_frame_base = 0;
   a->frame_base_slot = NATIVE_FRAME_SLOT_NONE;
   /* Windows variadic functions reserve a GP register home area at the top of
    * the frame (just below the incoming stack args). The plain-pointer va_list
    * then walks register-passed then stack-passed varargs as one block. Other
    * ABIs leave gp_reg_count 0 here: Apple ARM64 routes all varargs to the
    * stack, AAPCS64 uses a struct va_list with separate reg-save pointers. */
   {
     const ABIFuncInfo* fi = abi_cg_func_info(t->c->abi, fd->fn_type);
     ABIVaListInfo vai = abi_va_list_layout(t->c->abi);
     a->top_home_bytes = (fi && fi->variadic && vai.kind == ABI_VA_LIST_POINTER)
                             ? vai.gp_reg_count * vai.gp_slot_size
                             : 0u;
   }
   mc_set_section(mc, fd->text_section_id);
   mc_emit_align(mc, 4, 0);
   a->func_start = mc_pos(mc);
   mc_begin_function(mc, fd->sym, fd->text_section_id, a->func_start);
   mc_cfi_startproc(mc);
   a->prologue_pos = mc_pos(mc);
   a->minimal_prologue_words = 0;
   a->epilogue_label = mc_label_new(mc);
 }
 
 /* Single-pass (NativeDirectTarget) entry point: the frame is not known up
  * front, so reserve a worst-case prologue region (patched in aa_func_end once
  * max_outgoing / callee-saves are final) and emit the entry saves now. */
 static void aa_func_begin(NativeTarget* t, const CGFuncDesc* fd) {
   AANativeTarget* a = aa_of(t);
   MCEmitter* mc = t->mc;
   u32 region;
   aa_func_begin_common(t, fd);
   /* tcc-style prologue. Emit the frame-independent entry live: save the fp/lr
    * pair via a (16 + top_home)-byte pre-decrement and anchor fp at the pair.
    * top_home (the Windows-variadic GP home area, 0 elsewhere) is known here, so
    * the pair and fp land at exactly the addresses every fp-relative offset
    * already assumes — the frame is byte-identical to the old single big-`sub`
    * prologue, only the instruction sequence changes. */
   aa_emit32(mc, aa64_stp64_pre(
                     AA_FP, AA_LR, AA_SP,
                     -(i32)((AA_FRAME_SAVE_SIZE + a->top_home_bytes) / 8u)));
   aa_emit32(mc, aa64_add_imm(1, AA_FP, AA_SP, 0, 0)); /* mov x29, sp */
   /* Arm the far-slot positive-scaled fast path for the common top-record
    * layout. A Windows GP home area (top_home_bytes) places incoming args/home
    * above the saved pair and is rare; leave those on the fp-relative path. */
   a->slot_sp_base = (a->top_home_bytes == 0u);
   /* Reserve only the deferred `sub sp` (+ Windows probe), patched in
    * aa_func_end. The region starts here; record it for the patch and CFI. */
   region = abi_stack_probe_interval(a->base.c->abi) ? AA_NDT_SUB_WORDS
                                                     : AA_NDT_SUB_WORDS_NOPROBE;
   a->prologue_pos = mc_pos(mc);
   a->prologue_region_words = region;
   {
     u8 nops[AA_NDT_SUB_WORDS * 4u];
     for (u32 i = 0; i < region; ++i) wr_u32_le(nops + i * 4u, aa64_nop());
     if (mc->debug) {
       u32 ofs = obj_pos(mc->obj, mc->section_id);
       mc_emit_bytes(mc, nops, region * 4u);
       for (u32 i = 0; i < region; ++i)
         debug_emit_row(mc->debug, mc->section_id, ofs + i * 4u, mc->loc);
     } else {
       mc_emit_bytes(mc, nops, region * 4u);
     }
   }
   aa_emit_entry_saves(a);
 }
 
 /* Reserve the entry-save frame slots: the sret-pointer home (x8) and, for
  * variadic functions, the argument register-save area. Reserving is split from
  * emitting so the known-frame path can fix the full frame before the prologue;
  * the single-pass path runs both back to back via aa_emit_entry_saves. */
 static void aa_reserve_entry_saves(AANativeTarget* a) {
   NativeTarget* t = &a->base;
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, a->func->fn_type);
   if (abi && abi->has_sret) {
     NativeFrameSlotDesc sd;
     memset(&sd, 0, sizeof sd);
     sd.type = builtin_id(KIT_CG_BUILTIN_I64);
     sd.size = 8;
     sd.align = 8;
     sd.kind = NATIVE_FRAME_SLOT_SAVE;
     a->sret_ptr_slot = t->frame_slot(t, &sd);
   }
   if (abi && abi->variadic) aa_reserve_variadic_reg_saves(a);
 }
 
 /* Emit the entry-save stores (x8 → sret slot, then the variadic reg-save area).
  * Slots must already be reserved (aa_reserve_entry_saves). */
 static void aa_emit_entry_save_stores(AANativeTarget* a) {
   NativeTarget* t = &a->base;
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, a->func->fn_type);
   if (abi && abi->has_sret) {
     NativeAddr addr;
     NativeLoc src;
     MemAccess mem;
     KitCgTypeId i64 = builtin_id(KIT_CG_BUILTIN_I64);
     memset(&addr, 0, sizeof addr);
     addr.base_kind = NATIVE_ADDR_BASE_FRAME;
     addr.base.frame = a->sret_ptr_slot;
     addr.base_type = i64;
     memset(&src, 0, sizeof src);
     src.kind = NATIVE_LOC_REG;
     src.cls = NATIVE_REG_INT;
     src.type = i64;
     src.v.reg = 8u;
     memset(&mem, 0, sizeof mem);
     mem.type = i64;
     mem.size = 8;
     mem.align = 8;
     aa_emit_mem(a, 0, src, addr, mem);
   }
   if (abi && abi->variadic) aa_emit_variadic_reg_save_stores(a);
 }
 
 /* Reserve + emit the entry saves back to back. Single-pass (NativeDirectTarget)
  * path, where the prologue region is a reserved worst-case block and slot
  * offsets need not be final before it. */
 static void aa_emit_entry_saves(AANativeTarget* a) {
   aa_reserve_entry_saves(a);
   aa_emit_entry_save_stores(a);
 }
 
 static void aa_note_frame_state(NativeTarget* t,
                                 const NativeFramePatchState* state) {
   AANativeTarget* a = aa_of(t);
   if (state && state->max_outgoing > a->frame.max_outgoing)
     a->frame.max_outgoing = state->max_outgoing;
 }
 
 /* Reserve a save slot for each callee-saved register the allocator used. Runs
  * before frame-slot mapping so these slots get the lowest offsets, keeping the
  * prologue stores within stur's signed-9-bit range. The prologue/epilogue
  * save/restore is emitted from this list in aa_patch_prologue / aa_func_end. */
 static void aa_reserve_callee_saves(NativeTarget* t, const u32* used,
                                     u32 nclasses) {
   AANativeTarget* a = aa_of(t);
   /* aa64 homes each callee-save in its own 8-byte frame slot (reserved before
    * the body slots so they sit nearest fp, in stur range), so alloc_slots=1.
    * Adjacent integer slots are later paired into stp/ldp. */
   NativeFrameSaveSpec spec[NATIVE_REG_VEC + 1];
   memset(spec, 0, sizeof spec);
   spec[NATIVE_REG_INT].size = 8;
   spec[NATIVE_REG_INT].align = 8;
   spec[NATIVE_REG_INT].type = builtin_id(KIT_CG_BUILTIN_I64);
   spec[NATIVE_REG_FP].size = 8;
   spec[NATIVE_REG_FP].align = 8;
   spec[NATIVE_REG_FP].type = builtin_id(KIT_CG_BUILTIN_F64);
   native_frame_set_callee_saves(&a->frame, used, nclasses, spec,
                                 NATIVE_REG_VEC + 1, 1);
 }
 
 static MemAccess aa_mem_for_type(NativeTarget* t, KitCgTypeId type, u32 size);
 static void aa_words_callee_saves(AANativeTarget* a, int save, u32* words,
                                   u32 cap, u32* n);
 
 static void aa_emit_callee_restores(AANativeTarget* a) {
   u32 words[AA_PROLOGUE_WORDS];
   u32 n = 0;
   aa_words_callee_saves(a, 0, words, AA_PROLOGUE_WORDS, &n);
   for (u32 i = 0; i < n; ++i) aa_emit32(a->base.mc, words[i]);
 }
 
 static void aa_words_load_imm(AANativeTarget* a, u32* words, u32 cap, u32* n,
                               u32 rd, i64 imm) {
   u32 tmp[4];
   u32 m = aa_load_imm_words(tmp, 4u, 1, rd, imm);
   if (!m || *n + m > cap) aa_panic(a, "instruction patch too small");
   for (u32 i = 0; i < m; ++i) words[(*n)++] = tmp[i];
 }
 
 /* Windows large-frame stack probe. kit's prologue reserves the whole frame in
  * one `sub sp, sp, #N`, but Windows grows a thread stack one guard page at a
  * time: a sub that jumps SP more than a page past the guard page leaves the
  * skipped pages uncommitted, and the first store into them faults (and, since
  * SP itself is then in uncommitted memory, the fault can't even be delivered).
  * Touch every page the frame spans, top-down, so each guard page commits in
  * turn before the sub. Inlined (no external __chkstk symbol / no reloc in the
  * patched prologue region); mirrors the linker's aa64_coff_chkstk body. Only
  * x16/x17 are clobbered — the following sub-sp / saved-pair material re-derives
  * both. Emitted only when frame_size > interval (one page). */
 static void aa_words_stack_probe(AANativeTarget* a, u32* words, u32 cap, u32* n,
                                  u32 frame_size, u32 interval) {
   u32 imm12, sh;
   if (!aa64_addsub_imm_fits(interval, &imm12, &sh))
     aa_panic(a, "stack-probe interval not an addsub immediate");
   /* x16 = frame_size ; x17 = sp */
   aa_words_load_imm(a, words, cap, n, AA_TMP0, frame_size);
   if (*n + 5u > cap) aa_panic(a, "instruction patch too small");
   words[(*n)++] = aa64_add_imm(1, AA_TMP1, AA_SP, 0, 0); /* mov x17, sp */
   /* loop: x17 -= page ; x16 -= page (sets flags) ; touch [x17] ; b.gt loop */
   words[(*n)++] = aa64_sub_imm(1, AA_TMP1, AA_TMP1, imm12, sh);
   words[(*n)++] = aa64_subs_imm12(1, AA_TMP0, AA_TMP0, imm12, sh);
   words[(*n)++] = aa64_ldr64_uimm12(31, AA_TMP1, 0); /* ldr xzr, [x17] */
   /* branch back to the `sub x17` three words above while x16 stays positive */
   words[(*n)++] =
       aa64_brcond_pack((AA64BrCond){.imm19 = (u32)(-3), .cond = 0xcu /* GT */});
 }
 
 static void aa_words_sub_sp_frame(AANativeTarget* a, u32* words, u32 cap,
                                   u32* n, u32 frame_size) {
   u32 imm12, sh;
   if (aa64_addsub_imm_fits(frame_size, &imm12, &sh)) {
     if (*n >= cap) aa_panic(a, "instruction patch too small");
     words[(*n)++] = aa64_sub_imm(1, AA_SP, AA_SP, imm12, sh);
     return;
   }
   aa_words_load_imm(a, words, cap, n, AA_TMP0, frame_size);
   if (*n + 3u > cap) aa_panic(a, "instruction patch too small");
   words[(*n)++] = aa64_add_imm(1, AA_TMP1, AA_SP, 0, 0);
   words[(*n)++] = aa64_sub(1, AA_TMP1, AA_TMP1, AA_TMP0);
   words[(*n)++] = aa64_add_imm(1, AA_SP, AA_TMP1, 0, 0);
 }
 
 /* Anchor fp at the AAPCS64 saved-pair address (= sp + saved-pair offset).
  * The slim_prologue path achieves the same anchor in a single insn via
  * `add x29, sp, #0` after the pre-decrement stp moves sp to the saved-pair. */
 static void aa_words_frame_ptr_from_sp(AANativeTarget* a, u32* words, u32 cap,
                                        u32* n, const AAFrameLayout* L) {
   u32 imm12, sh;
   u32 anchor = aa_sp_off_saved_pair(L);
   if (aa64_addsub_imm_fits(anchor, &imm12, &sh)) {
     if (*n >= cap) aa_panic(a, "instruction patch too small");
     words[(*n)++] = aa64_add_imm(1, AA_FP, AA_SP, imm12, sh);
     return;
   }
   aa_words_load_imm(a, words, cap, n, AA_TMP0, anchor);
   if (*n + 2u > cap) aa_panic(a, "instruction patch too small");
   words[(*n)++] = aa64_add_imm(1, AA_TMP1, AA_SP, 0, 0);
   words[(*n)++] = aa64_add(1, AA_FP, AA_TMP1, AA_TMP0);
 }
 
 /* x17 = address of the saved-pair slot (= sp + saved-pair offset). Used by
  * the fat prologue to materialize the stp destination when the offset
  * doesn't fit stp's signed-7-bit-scaled immediate. */
 static void aa_words_saved_pair_addr(AANativeTarget* a, u32* words, u32 cap,
                                      u32* n, const AAFrameLayout* L) {
   u32 save_off = aa_sp_off_saved_pair(L);
   u32 imm12, sh;
   if (aa64_addsub_imm_fits(save_off, &imm12, &sh)) {
     if (*n >= cap) aa_panic(a, "instruction patch too small");
     words[(*n)++] = aa64_add_imm(1, AA_TMP1, AA_SP, imm12, sh);
     return;
   }
   aa_words_load_imm(a, words, cap, n, AA_TMP0, save_off);
   if (*n + 2u > cap) aa_panic(a, "instruction patch too small");
   words[(*n)++] = aa64_add_imm(1, AA_TMP1, AA_SP, 0, 0);
   words[(*n)++] = aa64_add(1, AA_TMP1, AA_TMP1, AA_TMP0);
 }
 
 /* Positive frame-base offset of the slot homing the caller's AA_FRAME_BASE
  * (alloca functions only). Same positive-scaled form the far-slot patches use,
  * so it is one word and fits any frame within dword scaled reach (32 KB). */
 static u32 aa_frame_base_save_off(AANativeTarget* a, const AAFrameLayout* L) {
   i32 off = (i32)aa_sp_off_saved_pair(L) +
             aa_fp_off_slot(a, aa_slot(a, a->frame_base_slot)->off);
   if (off < 0 || ((u32)off >> 3) > 0xfffu)
     aa_panic(a, "frame-base save slot out of positive scaled range");
   return (u32)off;
 }
 
 static void aa_words_restore_frame(AANativeTarget* a, u32* words, u32 cap,
                                    u32* n, const AAFrameLayout* L) {
   /* L10: frameless leaf — nothing to tear down (no record, sp untouched). The
    * caller emits the bare `ret`. */
   if (a->frameless) return;
   if (!L->frame_size) return;
   /* Restore the caller's AA_FRAME_BASE before tearing the frame down. It still
    * holds the frame base here (untouched by the body), so the slot is reachable
    * as a positive scaled offset off itself; the load then overwrites it with
    * the saved caller value. Alloca single-pass functions only. */
   if (a->uses_frame_base) {
     u32 off = aa_frame_base_save_off(a, L);
     if (*n >= cap) aa_panic(a, "epilogue too small for frame-base restore");
     words[(*n)++] = aa_ldr_uimm_v(3, 0, AA_FRAME_BASE, AA_FRAME_BASE, off);
   }
   if (a->slim_prologue) {
     if (*n + 1u > cap) aa_panic(a, "instruction patch too small");
     /* `ldp x29, x30, [sp], #16` — pop saved pair, restore sp. */
     words[(*n)++] = aa64_ldp64_post(AA_FP, AA_LR, AA_SP, 2);
     return;
   }
   if (a->fp_at_bottom) {
     u32 os = L->out_stack;
     if (os == 0u && !a->frame.has_alloca && L->frame_size <= 504u) {
       /* Folded fast path (matches the os==0 prologue fold): the saved pair sits
        * at the very bottom (= sp, since no alloca floated it), so
        * `ldp x29,x30,[sp],#N` reloads the pair AND releases the whole frame in
        * one insn. Callee-saves were already restored by aa_emit_callee_restores.
        * N <= 504 holds the post-index imm. */
       if (*n + 1u > cap) aa_panic(a, "instruction patch too small");
       words[(*n)++] =
           aa64_ldp64_post(AA_FP, AA_LR, AA_SP, (i32)(L->frame_size / 8u));
       return;
     }
     /* General bottom-record teardown, correct with OR without alloca (the
      * post-indexed `ldp [sp],#N` can't be used once alloca floated sp): recover
      * sp from the stable anchor x29, not from sp. `mov x16,x29 ; ldp x29,x30,
      * [x16] ; add sp,x16,#(fs-os)` — x16 holds the anchor across the pair reload
      * so the final add restores sp = x29 + (fs-os) = caller's sp = CFA. */
     {
       u32 bsz = L->frame_size - os; /* = aa_cfa_off in this layout */
       u32 imm12, sh;
       if (*n + 2u > cap) aa_panic(a, "instruction patch too small");
       words[(*n)++] = aa64_add_imm(1, AA_TMP0, AA_FP, 0, 0); /* mov x16, x29 */
       words[(*n)++] = aa64_ldp64_soff(AA_FP, AA_LR, AA_TMP0, 0);
       if (aa64_addsub_imm_fits(bsz, &imm12, &sh)) {
         if (*n >= cap) aa_panic(a, "instruction patch too small");
         words[(*n)++] = aa64_add_imm(1, AA_SP, AA_TMP0, imm12, sh);
       } else {
         /* bsz exceeds the addsub-imm window. The 3-register add cannot target
          * SP (Rd=31 means XZR there), so sum into x16 (a normal register) then
          * copy to sp via the immediate `add sp, x16, #0` form (Rd=31 == SP). */
         aa_words_load_imm(a, words, cap, n, AA_TMP1, bsz);
         if (*n + 2u > cap) aa_panic(a, "instruction patch too small");
         words[(*n)++] = aa64_add(1, AA_TMP0, AA_TMP0, AA_TMP1);
         words[(*n)++] = aa64_add_imm(1, AA_SP, AA_TMP0, 0, 0);
       }
       return;
     }
   }
   if (aa_cfa_off(a) == AA_FRAME_SAVE_SIZE) {
     /* Common top-record case (CFA == 16: no Windows-variadic GP home area). The
      * saved pair sits at [fp] and the caller's sp is fp+16, so `mov sp,x29`
      * followed by a post-indexed `ldp x29,x30,[sp],#16` restores the pair and
      * sp in two insns with no scratch — vs the fat path's three (`mov x16,fp;
      * ldp [x16]; add sp,x16,#16`). Correct under alloca: `mov sp,x29` resets sp
      * from the fp anchor before the post-index pops the pair. */
     if (*n + 2u > cap) aa_panic(a, "instruction patch too small");
     words[(*n)++] = aa64_add_imm(1, AA_SP, AA_FP, 0, 0);
     words[(*n)++] = aa64_ldp64_post(AA_FP, AA_LR, AA_SP, 2);
     return;
   }
   if (*n + 3u > cap) aa_panic(a, "instruction patch too small");
   /* AAPCS64: fp is the saved-pair address. Reload pair from [fp], then restore
    * sp to fp + CFA-offset (= caller's original sp = CFA). The CFA offset is
    * AA_FRAME_SAVE_SIZE normally, plus the Windows-variadic GP home area when
    * present (the only case that still reaches here). */
   words[(*n)++] = aa64_add_imm(1, AA_TMP0, AA_FP, 0, 0);
   words[(*n)++] = aa64_ldp64_soff(AA_FP, AA_LR, AA_TMP0, 0);
   words[(*n)++] = aa64_add_imm(1, AA_SP, AA_TMP0, (u32)aa_cfa_off(a), 0);
 }
 
 /* Emit callee-save store (save=1) or restore (save=0) words into `words`,
  * pairing adjacent integer registers into a single stp/ldp.
  * reserve_callee_saves allocates consecutive 8-byte slots in order, so
  * callee_saves[i] sits 8 bytes above callee_saves[i+1]; for an int pair the
  * lower-addressed reg[i+1] is the stp's Rt and reg[i] is Rt2. FP registers (and
  * an unpaired trailing int) use the single-register stur/ldur form.
  *
  * Layout reach. Top-record callee-saves sit nearest fp at small NEGATIVE offsets
  * (reserved first), so they always fit stur/stp directly. Bottom-record (W1.1)
  * anchors x29 at the bottom, so callee-saves stack near the TOP at large
  * POSITIVE offsets — past stp's +504 reach in any frame above ~512 bytes (and
  * past the scaled str reach of 32 KB in a huge frame). When the highest
  * callee-save offset exceeds stp's reach we materialize a base register
  * `x16 = x29 + adj` once (adj 16-aligned so it stays addsub-encodable) and
  * address the whole callee-save block relative to it at small offsets — x16 is
  * dead here on both the prologue (after the x29 anchor) and epilogue (callee
  * restores precede the teardown that reuses x16) paths. */
 static void aa_words_callee_saves(AANativeTarget* a, int save, u32* words,
                                   u32 cap, u32* n) {
   u32 base = AA_FP;
   i32 adj = 0;
   if (a->fp_at_bottom && a->frame.ncallee_saves) {
     /* The first reserved callee-save (smallest slot.off) gets the largest fp
      * offset; that bound decides whether a base register is needed. */
     i32 hi = aa_fp_off_slot(a, aa_slot(a, a->frame.callee_saves[0].slot)->off);
     if (hi > 504) {
       u32 imm12, sh;
       adj = (i32)((u32)(hi - 504) & ~15u); /* 16-aligned, leaves hi-adj <= 504 */
       if (!aa64_addsub_imm_fits((u32)adj, &imm12, &sh)) {
         /* adj is a multiple of 4096 worth of frame; build it via x16 load_imm. */
         aa_words_load_imm(a, words, cap, n, AA_TMP0, adj);
         if (*n + 1u > cap) aa_panic(a, "prologue too large");
         words[(*n)++] = aa64_add(1, AA_TMP0, AA_FP, AA_TMP0);
       } else {
         if (*n >= cap) aa_panic(a, "prologue too large");
         words[(*n)++] = aa64_add_imm(1, AA_TMP0, AA_FP, imm12, sh);
       }
       base = AA_TMP0;
     }
   }
   for (u32 i = 0; i < a->frame.ncallee_saves;) {
     const AACalleeSave* cs = &a->frame.callee_saves[i];
     i32 off = aa_fp_off_slot(a, aa_slot(a, cs->slot)->off) - adj;
     if (i + 1u < a->frame.ncallee_saves && cs->cls == (u8)NATIVE_REG_INT &&
         a->frame.callee_saves[i + 1u].cls == (u8)NATIVE_REG_INT) {
       const AACalleeSave* cs2 = &a->frame.callee_saves[i + 1u];
       i32 off2 = aa_fp_off_slot(a, aa_slot(a, cs2->slot)->off) - adj;
       /* cs2 is reserved after cs (larger slot.off), so it is the lower address
        * in both layouts (off2 = off - 8): stp's Rt = cs2, Rt2 = cs, base off2.
        * stp/ldp's signed-7-bit scaled immediate reaches ±504. */
       if (off2 < -512 || off2 > 504)
         aa_panic(a, "callee-save pair offset out of prologue range");
       if (*n >= cap) aa_panic(a, "prologue too large");
       words[(*n)++] = save
                           ? aa64_stp64_soff(cs2->reg, cs->reg, base, off2 / 8)
                           : aa64_ldp64_soff(cs2->reg, cs->reg, base, off2 / 8);
       i += 2u;
     } else {
       u32 v = cs->cls == (u8)NATIVE_REG_FP ? 1u : 0u;
       if (*n >= cap) aa_panic(a, "prologue too large");
       if (a->fp_at_bottom) {
         /* Positive, 8-aligned offset (after the optional base adjustment): the
          * unscaled stur (±256) can't reach it, so use the scaled unsigned-imm
          * str/ldr (reach 32 KB; adj keeps offsets in range for huge frames). */
         if (off < 0 || (u32)off > 0x7ff8u)
           aa_panic(a, "callee-save offset out of prologue range");
         words[(*n)++] = save ? aa_str_uimm_v(3, v, cs->reg, base, (u32)off)
                              : aa_ldr_uimm_v(3, v, cs->reg, base, (u32)off);
       } else {
         if (off < -256 || off > 255)
           aa_panic(a, "callee-save offset out of prologue range");
         words[(*n)++] = save ? aa_stur_v(3, v, cs->reg, base, off)
                              : aa_ldur_v(3, v, cs->reg, base, off);
       }
       i += 1u;
     }
   }
 }
 
 /* Build the prologue instruction words for `L` into `words` (capacity `cap`),
  * returning the count. Shared by the NativeDirectTarget patch path (reserves
  * a fixed worst-case region, then patches it here) and the optimizer path
  * (aa_func_begin_known_frame emits exactly these words up front).
  *
  * All variants establish a post-prologue state defined by L: saved x29/x30 at
  * [fp]/[fp+8], callee-saves at aa_fp_off_slot of each. The top-record variants
  * leave fp = sp + aa_sp_off_saved_pair(L) (saved-pair near the top); the
  * bottom-record variant leaves fp = sp (saved-pair at the bottom). */
 static u32 aa_build_prologue_words(AANativeTarget* a, const AAFrameLayout* L,
                                    u32* words, u32 cap) {
   u32 n = 0;
   /* L10: frameless leaf — no record, no sp adjustment. The body uses only
    * allocated registers + incoming args; sp is untouched and LR holds the live
    * return address (no call clobbered it). */
   if (a->frameless) return 0;
   if (!L->frame_size) return 0;
   if (a->slim_prologue) {
     if (cap < 2u) aa_panic(a, "prologue too large");
     /* `stp x29, x30, [sp, #-16]!; add x29, sp, #0` — the pre-decrement stp
      * moves sp down to the saved-pair address, so a no-op add anchors fp
      * there directly. AAPCS64 frame record. */
     words[n++] = aa64_stp64_pre(AA_FP, AA_LR, AA_SP, -2);
     words[n++] = aa64_add_imm(1, AA_FP, AA_SP, 0, 0);
     return n;
   }
   if (a->fp_at_bottom) {
     /* W1.1 uniform x29-at-bottom: anchor x29 = sp + out_stack, saved pair AT
      * x29, slots above it. Two encodings: */
     u32 os = L->out_stack;
     u32 imm12, sh;
     if (os == 0u && L->frame_size <= 504u) {
       /* Folded fast path (the original fp_at_bottom): the saved pair is at the
        * very bottom, so `stp x29,x30,[sp,#-N]!` decrements sp by the whole frame
        * AND saves the pair in one insn; `mov x29,sp` anchors fp there. The
        * post-index ldp on exit needs N <= 504. */
       if (n + 2u > cap) aa_panic(a, "prologue too large");
       words[n++] =
           aa64_stp64_pre(AA_FP, AA_LR, AA_SP, -(i32)(L->frame_size / 8u));
       words[n++] = aa64_add_imm(1, AA_FP, AA_SP, 0, 0);
       aa_words_callee_saves(a, 1, words, cap, &n);
       return n;
     }
     /* General bottom-record. `sub sp,sp,#fs` (robust / Windows-probed); save the
      * pair at [sp,#os] and anchor x29 = sp + os; callee-saves at positive x29
      * offsets. Outgoing args occupy [sp, sp+os); the body addresses them
      * sp-relative, so anchoring x29 above them keeps every slot positive while
      * leaving the arg area reachable from the (possibly alloca-floated) sp. */
     {
       u32 interval = abi_stack_probe_interval(a->base.c->abi);
       if (interval && L->frame_size > interval)
         aa_words_stack_probe(a, words, cap, &n, L->frame_size, interval);
     }
     aa_words_sub_sp_frame(a, words, cap, &n, L->frame_size);
     if (os <= 504u) {
       /* `stp x29,x30,[sp,#os]` — os fits stp's signed-7-bit scaled imm. */
       if (n >= cap) aa_panic(a, "prologue too large");
       words[n++] = aa64_stp64_soff(AA_FP, AA_LR, AA_SP, (i32)(os / 8u));
     } else {
       /* Far outgoing area: build the saved-pair address (sp + os) into x17. The
        * 3-register add cannot read SP (Rn=31 means XZR), so copy sp into x17 via
        * the immediate `add x17, sp, #0` form first, then add os. */
       if (aa64_addsub_imm_fits(os, &imm12, &sh)) {
         if (n >= cap) aa_panic(a, "prologue too large");
         words[n++] = aa64_add_imm(1, AA_TMP1, AA_SP, imm12, sh);
       } else {
         aa_words_load_imm(a, words, cap, &n, AA_TMP0, os);
         if (n + 2u > cap) aa_panic(a, "prologue too large");
         words[n++] = aa64_add_imm(1, AA_TMP1, AA_SP, 0, 0);
         words[n++] = aa64_add(1, AA_TMP1, AA_TMP1, AA_TMP0);
       }
       if (n >= cap) aa_panic(a, "prologue too large");
       words[n++] = aa64_stp64_soff(AA_FP, AA_LR, AA_TMP1, 0);
     }
     /* x29 = sp + os (the anchor). Same SP-as-Rn restriction: build via immediate
      * copy + register add for a far os. */
     if (aa64_addsub_imm_fits(os, &imm12, &sh)) {
       if (n >= cap) aa_panic(a, "prologue too large");
       words[n++] = aa64_add_imm(1, AA_FP, AA_SP, imm12, sh);
     } else {
       aa_words_load_imm(a, words, cap, &n, AA_TMP0, os);
       if (n + 2u > cap) aa_panic(a, "prologue too large");
       words[n++] = aa64_add_imm(1, AA_FP, AA_SP, 0, 0);
       words[n++] = aa64_add(1, AA_FP, AA_FP, AA_TMP0);
     }
     aa_words_callee_saves(a, 1, words, cap, &n);
     return n;
   }
   /* On targets that don't auto-grow the stack (Windows), probe each page the
    * frame spans before the single large `sub sp` jumps past the guard page.
    * slim_prologue/fp_at_bottom returned above — their frames are bounded to
    * one page (≤16 / ≤504 bytes), so only this path can exceed `interval`. */
   {
     u32 interval = abi_stack_probe_interval(a->base.c->abi);
     if (interval && L->frame_size > interval)
       aa_words_stack_probe(a, words, cap, &n, L->frame_size, interval);
   }
   aa_words_sub_sp_frame(a, words, cap, &n, L->frame_size);
   /* Fat top-record prologue: build the saved-pair address (sp + saved_pair_off)
    * into x17, store the pair there, then anchor fp. Reached only by the
    * single-pass (-O0) path and the Windows-variadic top_home case — the known-
    * frame -O1 path takes slim_prologue or the fp_at_bottom layout above. */
   aa_words_saved_pair_addr(a, words, cap, &n, L);
   if (n >= cap) aa_panic(a, "prologue too large");
   words[n++] = aa64_stp64_soff(AA_FP, AA_LR, AA_TMP1, 0); /* fp,lr @ [x17] */
   aa_words_frame_ptr_from_sp(a, words, cap, &n, L);
   /* Save callee-saved registers the allocator used (fp-relative; their slots
    * were reserved first by aa_reserve_callee_saves so offsets fit stur). */
   aa_words_callee_saves(a, 1, words, cap, &n);
   return n;
 }
 
 /* Build the deferred portion of the single-pass (-O0) prologue: the optional
  * Windows page probe and the `sub sp` that grows the frame below the fp/lr pair
  * the live entry (aa_func_begin) already saved and anchored fp at. Only
  * `frame_size - 16 - top_home` (= aa_sp_off_saved_pair) remains to subtract. */
 static u32 aa_build_ndt_sub_words(AANativeTarget* a, const AAFrameLayout* L,
                                   u32* words, u32 cap) {
   u32 n = 0;
   u32 sub_bytes = aa_sp_off_saved_pair(L);
   if (!sub_bytes) return 0;
   {
     u32 interval = abi_stack_probe_interval(a->base.c->abi);
     if (interval && sub_bytes > interval)
       aa_words_stack_probe(a, words, cap, &n, sub_bytes, interval);
   }
   aa_words_sub_sp_frame(a, words, cap, &n, sub_bytes);
   /* Set up the frame-base anchor for alloca functions: sp now points at the
    * frame base, so save the caller's AA_FRAME_BASE into its slot (positive
    * scaled, valid before the move) and anchor AA_FRAME_BASE = sp. The body
    * never touches AA_FRAME_BASE (callee-saved; the single-pass path allocates
    * caller-saved only), so it stays the frame base across any alloca. */
   if (a->uses_frame_base) {
     u32 off = aa_frame_base_save_off(a, L);
     if (n + 2u > cap) aa_panic(a, "prologue too small for frame-base setup");
     words[n++] = aa_str_uimm_v(3, 0, AA_FRAME_BASE, AA_SP, off);
     words[n++] = aa64_add_imm(1, AA_FRAME_BASE, AA_SP, 0, 0);
   }
   return n;
 }
 
 /* Patch the reserved deferred-`sub` region (prologue_region_words at
  * prologue_pos) once the frame is final. Single-pass (NativeDirectTarget) path
  * only — the optimizer path emits its prologue final and never patches. The
  * fixed entry (pair save + fp anchor) was already emitted live, so this writes
  * just the probe + `sub sp` and branches over any unused tail. */
 static void aa_patch_prologue(AANativeTarget* a, const AAFrameLayout* L) {
   u32 words[AA_NDT_SUB_WORDS];
   u32 region = a->prologue_region_words;
   u32 n;
   ObjSecId sec = a->func->text_section_id;
   if (region > AA_NDT_SUB_WORDS) aa_panic(a, "prologue region too large");
   memset(words, 0, sizeof words);
   n = aa_build_ndt_sub_words(a, L, words, region);
   /* If the deferred prologue is shorter than the reserved region, branch
    * straight to the entry saves rather than executing the trailing NOPs. */
   if (n < region) {
     words[n] = aa64_b(region - n);
     for (u32 i = n + 1u; i < region; ++i) words[i] = 0xd503201fu;
   }
   {
     u8 pbytes[AA_NDT_SUB_WORDS * 4u];
     for (u32 i = 0; i < region; ++i) wr_u32_le(pbytes + i * 4u, words[i]);
     obj_patch(a->base.obj, sec, a->prologue_pos, pbytes, (size_t)region * 4u);
   }
 }
 
 static void aa_emit_restore_frame(AANativeTarget* a, const AAFrameLayout* L) {
   MCEmitter* mc = a->base.mc;
   u32 words[AA_PROLOGUE_WORDS];
   u32 n = 0;
   if (!L->frame_size) return;
   aa_words_restore_frame(a, words, AA_PROLOGUE_WORDS, &n, L);
   for (u32 i = 0; i < n; ++i) aa_emit32(mc, words[i]);
 }
 
 /* Reserve one entry in the deferred-patch list, growing (arena-doubling) as
  * needed. The returned pointer is stable until the next aa_patch_alloc. */
 static AAPatch* aa_patch_alloc(AANativeTarget* a) {
   if (a->npatches == a->patches_cap) {
     u32 cap = a->patches_cap ? a->patches_cap * 2u : 8u;
     AAPatch* nb = arena_zarray(a->base.c->tu, AAPatch, cap);
     if (a->patches) memcpy(nb, a->patches, sizeof(*nb) * a->npatches);
     a->patches = nb;
     a->patches_cap = cap;
   }
   return &a->patches[a->npatches++];
 }
 
 /* Append FP-relative loads that restore the saved callee registers (stp/ldp
  * paired, same as the prologue saves). Shared by the tail-call patch; the
  * function epilogue uses aa_emit_callee_restores. */
 static void aa_words_callee_restores(AANativeTarget* a, u32* words, u32 cap,
                                      u32* n) {
   aa_words_callee_saves(a, 0, words, cap, n);
 }
 
 /* Drain the deferred-patch list. Each entry targets a disjoint, fixed code
  * position, so insertion order does not affect output. */
 static void aa_apply_patches(AANativeTarget* a, const AAFrameLayout* L) {
   ObjSecId sec = a->func->text_section_id;
   for (u32 i = 0; i < a->npatches; ++i) {
     AAPatch* p = &a->patches[i];
     if (p->kind == AA_PATCH_ALLOCA) {
       u32 imm12, sh;
       if (!aa64_addsub_imm_fits(a->frame.max_outgoing, &imm12, &sh))
         aa_panic(a, "outgoing area too large for alloca result");
       aa_patch32(a->base.obj, sec, p->pos,
                  aa64_add_imm(1, p->u.dst_reg, AA_SP, imm12, sh));
     } else if (p->kind == AA_PATCH_SLOT) {
       /* Resolve a far fixed-slot access to a positive scaled load/store off the
        * stable bottom anchor. The slot sits at fp_off below fp (top-record);
        * adding the bytes the prologue subtracted to reach the frame base
        * (aa_sp_off_saved_pair) gives its offset from sp / AA_FRAME_BASE. */
       const AASlotPatch* sl = &p->u.slot;
       i32 fp_off = aa_fp_off_slot(a, sl->slot_off) + sl->extra;
       i32 base_off = (i32)aa_sp_off_saved_pair(L) + fp_off;
       u32 base = a->uses_frame_base ? AA_FRAME_BASE : AA_SP;
       u32 ubo = (u32)base_off;
       /* Natural alignment is guaranteed by the slot allocator (and the byte
        * case is unconditionally aligned); a misalignment would mean a layout
        * bug. */
       if (base_off < 0 || (ubo & ((1u << sl->sz) - 1u)) != 0u)
         aa_panic(a, "far slot offset negative or misaligned");
       if ((ubo >> sl->sz) <= 0xfffu) {
         /* Scaled immediate fits: one-word ldr/str; the reserved second
          * placeholder word stays the nop emitted at body time. */
         aa_patch32(a->base.obj, sec, p->pos,
                    sl->load
                        ? aa_ldr_uimm_v(sl->sz, sl->vbit, sl->rt, base, ubo)
                        : aa_str_uimm_v(sl->sz, sl->vbit, sl->rt, base, ubo));
       } else {
         /* The scaled reach (4 KB/8 KB/16 KB/32 KB by size) is below the largest
          * frames, so build the address into the reserved second word instead:
          * `add x17,base,#(off & ~0xfff) ; ldr/str rt,[x17,#(off & 0xfff)]`. The
          * high part is a multiple of 4096 (fits the shift-12 addsub imm for any
          * frame < 16 MB); the low part is <4096 and, being naturally aligned to
          * the access size (base_off is, and hi is 4096-aligned), scale-divides
          * cleanly. */
         u32 hi = ubo & ~0xfffu;
         u32 lo = ubo & 0xfffu;
         if ((hi >> 12) > 0xfffu)
           aa_panic(a, "far slot offset out of address-build range");
         aa_patch32(a->base.obj, sec, p->pos,
                    aa64_add_imm(1, AA_TMP1, base, hi >> 12, 1));
         aa_patch32(a->base.obj, sec, p->pos + 4u,
                    sl->load
                        ? aa_ldr_uimm_v(sl->sz, sl->vbit, sl->rt, AA_TMP1, lo)
                        : aa_str_uimm_v(sl->sz, sl->vbit, sl->rt, AA_TMP1, lo));
       }
     } else { /* AA_PATCH_TAIL */
       NativeLoc callee = p->u.callee;
       u32 words[AA_TAIL_WORDS];
       u32 n = 0;
       memset(words, 0, sizeof words);
       aa_words_callee_restores(a, words, AA_TAIL_WORDS, &n);
       aa_words_restore_frame(a, words, AA_TAIL_WORDS, &n, L);
       if (n >= AA_TAIL_WORDS) aa_panic(a, "tail patch too small");
       if (callee.kind == NATIVE_LOC_REG) {
         words[n++] = aa64_br(loc_reg(callee));
       } else if (callee.kind == NATIVE_LOC_GLOBAL) {
         while (n + 1u < AA_TAIL_WORDS) words[n++] = 0xd503201fu;
         words[n++] = aa64_b(0);
       } else {
         aa_panic(a, "unsupported tail target");
       }
       while (n < AA_TAIL_WORDS) words[n++] = 0xd503201fu;
       for (u32 w = 0; w < AA_TAIL_WORDS; ++w)
         aa_patch32(a->base.obj, sec, p->pos + w * 4u, words[w]);
     }
   }
 }
 
 static void aa_func_end(NativeTarget* t) {
   AANativeTarget* a = aa_of(t);
   MCEmitter* mc = t->mc;
   AAFrameLayout L = aa_build_layout(a->frame.cum_off, a->frame.max_outgoing,
                                     a->top_home_bytes);
   /* known_frame (optimizer): prologue, allocas, and tail epilogues were emitted
    * final and slim eligibility was settled in aa_func_begin_known_frame — there
    * is nothing to patch. Single-pass (NDT): the frame-independent entry was
    * emitted live and only the deferred `sub` region + patches remain; resolve
    * them now that the frame is final. The advance past the prologue (for CFI)
    * is the live fixed entry plus the reserved deferred region. */
   u32 prologue_advance_words =
       a->frame.known_frame
           ? a->minimal_prologue_words
           : (AA_NDT_FIXED_ENTRY_WORDS + a->prologue_region_words);
   mc_label_place(mc, a->epilogue_label);
   aa_emit_callee_restores(a);
   aa_emit_restore_frame(a, &L);
   aa_emit32(mc, aa64_ret(AA_LR));
   if (a->frame.known_frame) {
     /* The frame-planning pre-pass plus final prologue/alloca/tail emission must
      * leave nothing deferred; a stray patch would mean a body-time frame change
      * the final prologue never saw. */
     if (a->npatches != 0) aa_panic(a, "known-frame path left deferred patches");
   } else {
     aa_patch_prologue(a, &L);
     aa_apply_patches(a, &L);
   }
   if (a->frameless) {
     /* L10: frameless leaf — CFA = sp (unchanged from entry) and the return
      * address stays live in LR (the aa64 CIE default), so no saved-register
      * rules. The state holds from the first instruction (offset 0). */
     mc_cfi_set_next_pc_offset(mc, 0);
     mc_cfi_def_cfa(mc, AA_SP, 0);
   } else {
     i32 cfa = aa_cfa_off(a);
     mc_cfi_set_next_pc_offset(mc, prologue_advance_words * 4u);
     /* CFA = caller's sp, an fp-relative offset that depends on the layout:
      * fp+16 (top-record) or fp+frame_size (bottom-record). saved fp/lr live at
      * [fp]/[fp+8] in both, hence at CFA-cfa / CFA-cfa+8. */
     mc_cfi_def_cfa(mc, AA_FP, cfa);
     mc_cfi_offset(mc, AA_FP, aa_fp_off_saved_fp() - cfa);
     mc_cfi_offset(mc, AA_LR, aa_fp_off_saved_lr() - cfa);
   }
   obj_symbol_define(t->obj, a->func->sym, a->func->text_section_id,
                     a->func_start, mc_pos(mc) - a->func_start);
   if (a->func->atomize) {
     obj_atom_define(t->obj, a->func->text_section_id, a->func_start,
                     mc_pos(mc) - a->func_start, a->func->sym, 0);
   }
   /* Hand the function's PC range to the Debug producer so its line program
    * (and DW_AT_low_pc/high_pc) cover this function — emit_section_line skips
    * functions without a recorded range. */
   if (mc->debug)
     debug_func_pc_range(mc->debug, a->func->text_section_id, a->func_start,
                         mc_pos(mc));
   mc_cfi_endproc(mc);
   mc_end_function(mc);
   a->func = NULL;
 }
 
 static NativeFrameSlot aa_frame_slot(NativeTarget* t,
                                      const NativeFrameSlotDesc* d) {
   return native_frame_slot_alloc(&aa_of(t)->frame, d);
 }
 
 static void aa_release_frame_slot(NativeTarget* t, NativeFrameSlot slot) {
   native_frame_release_slot(&aa_of(t)->frame, slot);
 }
 
 static int aa_frame_slot_debug_loc(NativeTarget* t, NativeFrameSlot slot,
                                    CGDebugLoc* out) {
   AANativeTarget* a = aa_of(t);
   AANativeSlot* s;
   i32 fp_off;
   if (!out) return 0;
   memset(out, 0, sizeof *out);
   if (slot == NATIVE_FRAME_SLOT_NONE || slot > a->frame.nslots) return 0;
   s = aa_slot(a, slot);
   fp_off = aa_fp_off_slot(a, s->off);
   out->kind = CG_DEBUG_LOC_FRAME;
   /* The hosted dbg stop snapshot currently carries x29/fp as the frame base
    * for variable materialization, so report the same FP-relative slot offset
    * used by native memory operands. */
   out->v.frame_ofs = fp_off;
   return 1;
 }
 
 static void aa_asm_clobber_masks(Compiler* c, SrcLoc loc, const Sym* clobbers,
                                  u32 nclob, u32* int_mask, u32* fp_mask);
 /* abi_clobber_masks is shared as native_asm_abi_clobber_masks
  * (cg/native_asm.h); it reads the masks from t->regs->classes. */
 
 /* Build the callee-saved set the prologue must preserve: the allocator-assigned
  * callee-saved registers (frame->callee_saved_used) plus any an inline-asm
  * block clobbers. The latter are opaque to the optimizer's operand scan, so it
  * forwards the raw clobber names (frame->asm_clobbers) and the arch-neutral
  * clobber-ABI sets (frame->asm_clobber_abi_sets); we resolve both into masks
  * and keep only the callee-saved ones, per AAPCS64: x19..x28 and the low 64
  * bits of v8..v15 (x29/x30 are the frame pointer and link register, handled by
  * the prologue head, not as ordinary callee-saves). This is the same register
  * selection the per-block spill used, hoisted into the prologue. */
 static int aa_asm_reg_is_callee_saved(NativeTarget* t, NativeAllocClass cls,
                                       Reg r) {
   (void)t;
   return (cls == NATIVE_REG_INT && r >= 19u && r <= 28u) ||
          (cls == NATIVE_REG_FP && r >= 8u && r <= 15u);
 }
 
 static u32 aa_known_callee_saves(NativeTarget* t,
                                  const NativeKnownFrameDesc* frame, u32* out,
                                  u32 cap) {
   AANativeTarget* a = aa_of(t);
   SrcLoc loc = a->func ? a->func->loc : (SrcLoc){0, 0, 0};
   return native_asm_known_callee_saves(
       t, loc, frame, out, cap, aa_asm_clobber_masks,
       aa_asm_reg_is_callee_saved);
 }
 
 static u32 aa_signature_stack_bytes(NativeTarget* t, KitCgTypeId fn_type,
                                     int* variadic, u32* nparams);
 
 /* Optimizer entry point: the full frame is supplied up front, so the prologue,
  * entry saves, slim-form eligibility, allocas, and tail epilogues are all final
  * the moment they are emitted — no back-patching (aa_func_end skips the patch
  * passes when a->frame.known_frame). Slot creation order matches the
  * single-pass path (callee-saves first for stur range, then the static slots,
  * then sret/variadic entry saves) so offsets are identical to what the patch
  * path would produce. */
 static void aa_func_begin_known_frame(NativeTarget* t, const CGFuncDesc* fd,
                                       const NativeKnownFrameDesc* frame,
                                       NativeFrameSlot* out_slots) {
   AANativeTarget* a = aa_of(t);
   AAFrameLayout L;
   u32 words[AA_PROLOGUE_WORDS];
   u32 n;
   aa_func_begin_common(t, fd);
   a->frame.known_frame = 1;
   if (frame) {
     u32 cs[NATIVE_REG_CLASS_COUNT];
     u32 ncs = aa_known_callee_saves(t, frame, cs, NATIVE_REG_CLASS_COUNT);
     a->frame.has_alloca = frame->has_alloca;
     if (ncs) aa_reserve_callee_saves(t, cs, ncs);
     for (u32 i = 0; i < frame->nslots; ++i) {
       NativeFrameSlot slot = aa_frame_slot(t, &frame->slots[i]);
       if (out_slots) out_slots[i] = slot;
     }
     aa_reserve_entry_saves(a);
     /* Reserve the atomic-RMW scratch spill last (matching its lazy position in
      * the single-pass path), so aa_saved_tmp_spill reuses it instead of growing
      * the frame mid-body. */
     if (frame->needs_scratch_spill) {
       NativeFrameSlotDesc sd;
       memset(&sd, 0, sizeof sd);
       sd.type = builtin_id(KIT_CG_BUILTIN_I64);
       sd.size = 8;
       sd.align = 8;
       sd.kind = NATIVE_FRAME_SLOT_SPILL;
       a->saved_tmp_slot = a->base.frame_slot(&a->base, &sd);
     }
     if (frame->max_outgoing > a->frame.max_outgoing)
       a->frame.max_outgoing = frame->max_outgoing;
   }
   /* Frame is final: slot_bytes (cum_off) and out_stack (max_outgoing) are both
    * known, so the prologue immediates and slim-form choice are settled here.
    * frame_size_final + out_stack_final must be set before aa_build_prologue_words
    * / entry saves, since the bottom-record offset helpers read both. */
   L = aa_build_layout(a->frame.cum_off, a->frame.max_outgoing,
                       a->top_home_bytes);
   a->frame_size_final = L.frame_size;
   a->out_stack_final = L.out_stack;
   /* Layout choice (W1.1). Slim Tier A: no callee-saves, no alloca, no body
    * slots, no outgoing stack args — the whole frame is the 16-byte record, with
    * its own folded `stp [sp,#-16]!` encoding. Otherwise the uniform x29-at-bottom
    * (fp_at_bottom) layout: x29 anchored out_stack bytes above sp, every slot a
    * positive [x29,#k], for ALL remaining known frames (out_stack>0, alloca, and
    * arbitrarily large frames included). The only exception is a Windows-variadic
    * GP home area, which must sit ABOVE the saved pair (the plain-pointer va_list
    * walks register then stack varargs as one contiguous block) — bottom-record
    * has no room for it, so that case keeps the fat top-record layout.
    * (See aa_func_end for the single-pass path, which never takes any slim/bottom
    * form.) */
   a->slim_prologue = a->frame.ncallee_saves == 0 && !a->frame.has_alloca &&
                      L.slot_bytes == 0 && L.out_stack == 0 &&
                      !a->top_home_bytes;
   /* L10: a slim Tier A function that is additionally a leaf (no call clobbers
    * x30/LR), has no inline asm (which could clobber LR / call opaquely), and
    * never reads its own frame chain (__builtin_frame_address /
    * __builtin_return_address) needs no frame record at all. Emit no prologue /
    * epilogue / saved-pair CFI; the return address rides LR and the CFA stays at
    * sp. A frameless function has frame_size 0 in this layout (slim's whole frame
    * is the 16-byte record we are eliding), so it takes neither slim nor
    * fp_at_bottom encoding. Additionally exclude any function with incoming
    * stack arguments: those are addressed fp-relative (aa_fp_off_in_arg, off
    * x29), so a function that reads them needs the x29 anchor a frameless
    * prologue never sets up. (Stack args do not show up in L.slot_bytes /
    * L.out_stack — they sit above the saved pair — so this guard is separate.) */
   a->frameless = a->slim_prologue && frame && frame->is_leaf &&
                  !frame->has_asm && !frame->reads_frame &&
                  aa_signature_stack_bytes(t, fd->fn_type, NULL, NULL) == 0u;
   if (a->frameless) a->slim_prologue = 0;
   a->fp_at_bottom = !a->slim_prologue && !a->frameless && !a->top_home_bytes;
   n = aa_build_prologue_words(a, &L, words, AA_PROLOGUE_WORDS);
   for (u32 i = 0; i < n; ++i) aa_emit32(t->mc, words[i]);
   a->minimal_prologue_words = n;
   a->frame.frame_final = 1;
   aa_emit_entry_save_stores(a);
 }
 
 static void aa_spill(NativeTarget* t, NativeLoc src, NativeFrameSlot slot,
                      MemAccess mem) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = slot;
   addr.base_type = src.type;
   aa_emit_mem(aa_of(t), 0, src, addr, mem);
 }
 
 static void aa_reload(NativeTarget* t, NativeLoc dst, NativeFrameSlot slot,
                       MemAccess mem) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = slot;
   addr.base_type = dst.type;
   aa_emit_mem(aa_of(t), 1, dst, addr, mem);
 }
 
 static MCLabel aa_label_new(NativeTarget* t) { return mc_label_new(t->mc); }
 
 static void aa_label_place(NativeTarget* t, MCLabel label) {
   mc_label_place(t->mc, label);
 }
 
 static void aa_jump(NativeTarget* t, MCLabel label) {
   aa_emit32(t->mc, aa64_b(0));
   mc_emit_label_ref(t->mc, label, R_AARCH64_JUMP26, 4, 0);
 }
 
 static void aa_cmp_branch(NativeTarget* t, CmpOp op, NativeLoc lhs,
                           NativeLoc rhs, MCLabel label) {
   /* Fuse `<reg> == 0` / `!= 0` tests into a single cbz/cbnz, dropping the
    * separate `cmp <reg>,#0`. Only the GPR integer-equality case qualifies: the
    * rhs must be an immediate zero (the cg layer passes IMM 0 for null/zero
    * tests) and lhs a register (always true on the -O0 path). FP compares route
    * through FCMP and keep cmp+b.cond. */
   if ((op == CMP_EQ || op == CMP_NE) && !native_loc_is_fp(lhs) &&
       lhs.kind == NATIVE_LOC_REG && rhs.kind == NATIVE_LOC_IMM &&
       rhs.v.imm == 0) {
     u32 sf = loc_is_64(t, lhs) ? 1u : 0u;
     aa_emit32(t->mc, op == CMP_EQ ? aa64_cbz(sf, loc_reg(lhs), 0)
                                   : aa64_cbnz_imm(sf, loc_reg(lhs), 0));
     mc_emit_label_ref(t->mc, label, R_AARCH64_CONDBR19, 4, 0);
     return;
   }
   aa_emit_cmp_to_flags(t, lhs, rhs);
   /* CMP_ONE_F / CMP_UEQ_F have no single FP condition: take the branch from a
    * pair of conditional branches to the same label (no scratch register). */
   if (op == CMP_ONE_F) {
     /* ordered & !=: branch if a<b (MI) or a>b (GT). */
     aa_emit32(t->mc, aa64_brcond_pack((AA64BrCond){.cond = 0x4u})); /* MI */
     mc_emit_label_ref(t->mc, label, R_AARCH64_CONDBR19, 4, 0);
     aa_emit32(t->mc, aa64_brcond_pack((AA64BrCond){.cond = 0xcu})); /* GT */
     mc_emit_label_ref(t->mc, label, R_AARCH64_CONDBR19, 4, 0);
     return;
   }
   if (op == CMP_UEQ_F) {
     /* unordered | ==: branch if a==b (EQ) or unordered (VS). */
     aa_emit32(t->mc, aa64_brcond_pack((AA64BrCond){.cond = 0x0u})); /* EQ */
     mc_emit_label_ref(t->mc, label, R_AARCH64_CONDBR19, 4, 0);
     aa_emit32(t->mc, aa64_brcond_pack((AA64BrCond){.cond = 0x6u})); /* VS */
     mc_emit_label_ref(t->mc, label, R_AARCH64_CONDBR19, 4, 0);
     return;
   }
   aa_emit32(t->mc, aa64_brcond_pack((AA64BrCond){.cond = cmp_cond(op)}));
   mc_emit_label_ref(t->mc, label, R_AARCH64_CONDBR19, 4, 0);
 }
 
 static void aa_indirect_branch(NativeTarget* t, NativeLoc addr,
                                const MCLabel* valid_targets, u32 ntargets) {
   (void)valid_targets;
   (void)ntargets;
   aa_emit32(t->mc, aa64_br(loc_reg(addr)));
 }
 
 static void aa_load_label_addr(NativeTarget* t, NativeLoc dst, MCLabel target) {
   /* `&&label` address-take: adrp/add with the ADR_PREL_PG_HI21 +
    * ADD_ABS_LO12_NC relocation pair against the label's per-block local symbol
    * — the same form used to address a global — so the reference is genuinely
    * relocatable (reaches ±4 GiB) and any assembler resolves it from the symbol.
    * Replaces the old 16-byte INTRA-label sequence with a baked offset. */
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst);
   ObjSymId sym = mc_label_symbol(mc, target);
   u32 pos = mc_pos(mc);
   aa_emit32(mc, aa64_adrp(rd, 0, 0));
   mc_emit_reloc_at(mc, mc->section_id, pos, R_AARCH64_ADR_PREL_PG_HI21, sym, 0,
                    0, 0);
   pos = mc_pos(mc);
   aa_emit32(mc, aa64_add_imm(1, rd, rd, 0, 0));
   mc_emit_reloc_at(mc, mc->section_id, pos, R_AARCH64_ADD_ABS_LO12_NC, sym, 0,
                    0, 0);
 }
 
 static void aa_move(NativeTarget* t, NativeLoc dst, NativeLoc src) {
   /* Identity move elision: same-class same-reg is a no-op on aarch64
    * regardless of width (mov xN,xN and mov wN,wN both leave the low bits
    * untouched). Catches no-op IR_CONVERT (BITCAST, ZEXT/SEXT with
    * src_bits>=dst_bits, FEXT/FTRUNC across-class) when the allocator put
    * dst and src in the same hard reg — common post #2.5 return-reg
    * coalescing, e.g. `convert opnds=[v0,v0]` after a pointer-returning call
    * was emitting `mov x0,x0`. Cross-class (fp<->gpr) bitcasts are not
    * elided here even when the reg numbers match — the register files are
    * disjoint. */
   if (dst.kind == NATIVE_LOC_REG && src.kind == NATIVE_LOC_REG &&
       native_loc_is_fp(dst) == native_loc_is_fp(src) && dst.v.reg == src.v.reg)
     return;
   if (native_loc_is_fp(dst) && native_loc_is_fp(src)) {
     if (loc_size32(t, dst) == 16u)
       aa_emit32(t->mc, aa_mov_vec16(loc_reg(dst), loc_reg(src)));
     else
       aa_emit32(t->mc, aa_fmov_fp(loc_size32(t, dst) == 8u, loc_reg(dst),
                                   loc_reg(src)));
   } else if (native_loc_is_fp(dst)) {
     aa_emit32(t->mc,
               aa_fmov_gpr_to_fp(loc_is_64(t, src), loc_reg(dst), loc_reg(src)));
   } else if (native_loc_is_fp(src)) {
     aa_emit32(t->mc,
               aa_fmov_fp_to_gpr(loc_is_64(t, dst), loc_reg(dst), loc_reg(src)));
   } else {
     aa_emit32(t->mc,
               aa64_mov_reg(loc_is_64(t, dst), loc_reg(dst), loc_reg(src)));
   }
 }
 
 static NativeLoc aa_tmp_loc(KitCgTypeId type, Reg reg);
 
 static void aa_load_imm_native(NativeTarget* t, NativeLoc dst, i64 imm) {
   aa_emit_load_imm(t->mc, loc_is_64(t, dst), loc_reg(dst), imm);
 }
 
 static void aa_load_const(NativeTarget* t, NativeLoc dst, ConstBytes cbytes) {
   u64 v = 0;
   if (cbytes.size > 8u)
     compiler_panic(t->c, ((AANativeTarget*)t)->loc,
                    "aarch64 native target: byte constant too large");
   for (u32 i = 0; i < cbytes.size; ++i) v |= (u64)cbytes.bytes[i] << (i * 8u);
   if (native_loc_is_fp(dst)) {
     NativeLoc tmp = aa_tmp_loc(cbytes.type, AA_TMP0);
     aa_emit_load_imm(t->mc, cbytes.size == 8u, AA_TMP0, (i64)v);
     aa_move(t, dst, tmp);
   } else {
     aa_emit_load_imm(t->mc, loc_is_64(t, dst), loc_reg(dst), (i64)v);
   }
 }
 
 static void aa_load_addr(NativeTarget* t, NativeLoc dst, NativeAddr addr) {
   AANativeTarget* a = aa_of(t);
   u32 rd = loc_reg(dst);
   aa_materialize_frame_index(a, &addr, rd);
   switch ((NativeAddrBaseKind)addr.base_kind) {
     case NATIVE_ADDR_BASE_FRAME: {
       AANativeSlot* s = aa_slot(a, addr.base.frame);
       aa_load_addr_from_base(a, rd, AA_FP,
                              aa_fp_off_slot(a, s->off) + addr.offset, &addr);
       return;
     }
     case NATIVE_ADDR_BASE_FRAME_VALUE: {
       NativeAddr load;
       MemAccess mem;
       u32 base_reg = aa_addr_index_is_reg(&addr, rd) ? aa_tmp_avoiding(rd) : rd;
       memset(&load, 0, sizeof load);
       load.base_kind = NATIVE_ADDR_BASE_FRAME;
       load.base.frame = addr.base.frame;
       load.base_type =
           addr.base_type ? addr.base_type : builtin_id(KIT_CG_BUILTIN_I64);
       memset(&mem, 0, sizeof mem);
       mem.type = load.base_type;
       mem.size = type_size32(t, load.base_type);
       mem.align = type_align32(t, load.base_type);
       aa_emit_mem(a, 1, native_loc_reg(dst.type, NATIVE_REG_INT, base_reg),
                   load, mem);
       aa_load_addr_from_base(a, rd, base_reg, addr.offset, &addr);
       return;
     }
     case NATIVE_ADDR_BASE_REG:
       aa_load_addr_from_base(a, rd, addr.base.reg, addr.offset, &addr);
       return;
     case NATIVE_ADDR_BASE_GLOBAL: {
       i64 addend = addr.base.global.addend + (i64)addr.offset;
       u32 base_reg = aa_addr_index_is_reg(&addr, rd) ? aa_tmp_avoiding(rd) : rd;
       u32 pos = mc_pos(t->mc);
       if (aa_use_got_for_sym(t, addr.base.global.sym)) {
         aa_emit32(t->mc, aa64_adrp(base_reg, 0, 0));
         mc_emit_reloc_at(t->mc, t->mc->section_id, pos, R_AARCH64_ADR_GOT_PAGE,
                          addr.base.global.sym, 0, 0, 0);
         pos = mc_pos(t->mc);
         aa_emit32(t->mc, aa_ldr_uimm(3, base_reg, base_reg, 0));
         mc_emit_reloc_at(t->mc, t->mc->section_id, pos,
                          R_AARCH64_LD64_GOT_LO12_NC, addr.base.global.sym, 0, 0,
                          0);
         if (addend) aa_emit_add_i64(a, base_reg, base_reg, addend);
         aa_load_addr_from_base(a, rd, base_reg, 0, &addr);
         return;
       }
       aa_emit32(t->mc, aa64_adrp(base_reg, 0, 0));
       mc_emit_reloc_at(t->mc, t->mc->section_id, pos,
                        R_AARCH64_ADR_PREL_PG_HI21, addr.base.global.sym, addend,
                        0, 0);
       pos = mc_pos(t->mc);
       aa_emit32(t->mc, aa64_add_imm(1, base_reg, base_reg, 0, 0));
       mc_emit_reloc_at(t->mc, t->mc->section_id, pos, R_AARCH64_ADD_ABS_LO12_NC,
                        addr.base.global.sym, addend, 0, 0);
       aa_load_addr_from_base(a, rd, base_reg, 0, &addr);
       return;
     }
     default:
       aa_panic(a, "unsupported load_addr");
   }
 }
 
 static void aa_load_native(NativeTarget* t, NativeLoc dst, NativeAddr addr,
                            MemAccess mem) {
   aa_emit_mem(aa_of(t), 1, dst, addr, mem);
 }
 
 static void aa_store_native(NativeTarget* t, NativeAddr addr, NativeLoc src,
                             MemAccess mem) {
   aa_emit_mem(aa_of(t), 0, src, addr, mem);
 }
 
 /* Windows/AArch64 TLS Local-Exec (PE-COFF). Mirrors x64_tls_addr_of_win64:
  *   ldr  rd,  [x18, #0x58]            ; TEB.ThreadLocalStoragePointer
  *   adrp x16, _tls_index              ; PAGEBASE_REL21
  *   add  x16, x16, :lo12:_tls_index   ; PAGEOFFSET_12A
  *   ldr  w16, [x16]                   ; module's TLS index
  *   ldr  rd,  [rd, x16, lsl #3]       ; this module's TLS block base
  *   add  rd,  rd, #:secrel_hi12:sym   ; SECREL_HIGH12A (sh=1)
  *   add  rd,  rd, #:secrel_lo12:sym   ; SECREL_LOW12A  (sh=0)
  * We materialize &_tls_index via ADRP+ADD (not LDR :lo12:) on purpose: the
  * COFF reader collapses LDST32→LDST64 width, so an LDR :lo12: form would be
  * mis-scaled at link time; ADD_ABS_LO12_NC carries no width and round-trips
  * cleanly. AA_TMP0 (x16) is the reserved scratch; rd is an allocated reg
  * distinct from x16/x17/x18. */
 static void aa_tls_addr_of_win(NativeTarget* t, NativeLoc dst, ObjSymId sym,
                                i64 addend) {
   MCEmitter* mc = t->mc;
   u32 sec = mc->section_id;
   u32 rd = loc_reg(dst);
   u32 pos;
   Sym idx_name = pool_intern_slice(t->c->global, SLICE_LIT("_tls_index"));
   ObjSymId idx_sym = obj_symbol_find(t->obj, idx_name);
   if (idx_sym == 0)
     idx_sym =
         obj_symbol(t->obj, idx_name, SB_GLOBAL, SK_UNDEF, OBJ_SEC_NONE, 0, 0);
   /* (1) rd = TEB.ThreadLocalStoragePointer. */
   aa_emit32(mc, aa_ldr_uimm(3, rd, AA_WIN_TEB_REG, AA_WIN_TEB_TLS_PTR_OFF));
   /* (2)+(3) x16 = &_tls_index via ADRP + ADD. */
   pos = mc_pos(mc);
   aa_emit32(mc, aa64_adrp(AA_TMP0, 0, 0));
   mc_emit_reloc_at(mc, sec, pos, R_AARCH64_ADR_PREL_PG_HI21, idx_sym, 0, 0, 0);
   pos = mc_pos(mc);
   aa_emit32(mc, aa64_add_imm(1, AA_TMP0, AA_TMP0, 0, 0));
   mc_emit_reloc_at(mc, sec, pos, R_AARCH64_ADD_ABS_LO12_NC, idx_sym, 0, 0, 0);
   /* (4) w16 = _tls_index (the loaded value). */
   aa_emit32(mc, aa_ldr_uimm(2, AA_TMP0, AA_TMP0, 0));
   /* (5) rd = TLS array slot for this module: ldr rd, [rd, x16, lsl #3]. */
   aa_emit32(mc, aa_ldst_regoff_v(3, 0, 1, rd, rd, AA_TMP0, 1));
   /* (6) rd += :secrel_hi12:sym  (ADD with sh=1; linker patches imm12). */
   pos = mc_pos(mc);
   aa_emit32(mc, aa64_add_imm(1, rd, rd, 0, 1));
   mc_emit_reloc_at(mc, sec, pos, R_COFF_AARCH64_SECREL_HIGH12A, sym, addend, 1,
                    0);
   /* (7) rd += :secrel_lo12:sym  (ADD with sh=0). */
   pos = mc_pos(mc);
   aa_emit32(mc, aa64_add_imm(1, rd, rd, 0, 0));
   mc_emit_reloc_at(mc, sec, pos, R_COFF_AARCH64_SECREL_LOW12A, sym, addend, 1,
                    0);
 }
 
 static void aa_tls_addr_of(NativeTarget* t, NativeLoc dst, ObjSymId sym,
                            i64 addend) {
   AANativeTarget* a = aa_of(t);
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst);
   u32 pos;
   if (obj_format_tls_via_descriptor(t->c)) {
     aa_emit32(mc, aa64_adrp(0, 0, 0));
     pos = mc_pos(mc) - 4u;
     mc_emit_reloc_at(mc, mc->section_id, pos, R_AARCH64_TLVP_LOAD_PAGE21, sym,
                      0, 0, 0);
     aa_emit32(mc, aa_ldr_uimm(3, 0, 0, 0));
     pos = mc_pos(mc) - 4u;
     mc_emit_reloc_at(mc, mc->section_id, pos, R_AARCH64_TLVP_LOAD_PAGEOFF12,
                      sym, 0, 0, 0);
     aa_emit32(mc, aa_ldr_uimm(3, AA_TMP0, 0, 0));
     aa_emit32(mc, aa64_blr(AA_TMP0));
     if (addend) aa_emit_add_i64(a, 0, 0, addend);
     if (rd != 0) aa_emit32(mc, aa64_mov_reg(1, rd, 0));
     return;
   }
   if (obj_format_tls_model(t->c) == OBJ_TLS_WINDOWS_TEB) {
     aa_tls_addr_of_win(t, dst, sym, addend);
     return;
   }
   if (t->c->target.obj != KIT_OBJ_ELF) {
     aa_panic(a, "unsupported TLS object format");
   }
   aa_emit32(mc, aa_mrs_tpidr_el0(rd));
   pos = mc_pos(mc);
   aa_emit32(mc, aa64_add_imm(1, rd, rd, 0, 1));
   mc_emit_reloc_at(mc, mc->section_id, pos, R_AARCH64_TLSLE_ADD_TPREL_HI12, sym,
                    addend, 0, 0);
   pos = mc_pos(mc);
   aa_emit32(mc, aa64_add_imm(1, rd, rd, 0, 0));
   mc_emit_reloc_at(mc, mc->section_id, pos, R_AARCH64_TLSLE_ADD_TPREL_LO12_NC,
                    sym, addend, 0, 0);
 }
 
 static NativeLoc aa_tmp_loc(KitCgTypeId type, Reg reg) {
   NativeLoc loc;
   memset(&loc, 0, sizeof loc);
   loc.kind = NATIVE_LOC_REG;
   loc.cls = NATIVE_REG_INT;
   loc.type = type;
   loc.v.reg = reg;
   return loc;
 }
 
 static NativeAddr aa_addr_plus(NativeAddr addr, u32 off) {
   addr.offset += (i32)off;
   return addr;
 }
 
 static void aa_copy_bytes_dir(NativeTarget* t, NativeAddr dst, NativeAddr src,
                               AggregateAccess access, int backward) {
   KitCgTypeId i64 = builtin_id(KIT_CG_BUILTIN_I64);
   KitCgTypeId i32 = builtin_id(KIT_CG_BUILTIN_I32);
   KitCgTypeId i16 = builtin_id(KIT_CG_BUILTIN_I16);
   KitCgTypeId i8 = builtin_id(KIT_CG_BUILTIN_I8);
   NativeLoc tmp = aa_tmp_loc(i64, AA_TMP0);
   u32 off = 0;
   while (off < access.size) {
     u32 rem = access.size - off;
     u32 pos;
     MemAccess mem = access.mem;
     if (rem >= 8u) {
       mem.type = i64;
       mem.size = 8u;
     } else if (rem >= 4u) {
       mem.type = i32;
       mem.size = 4u;
       tmp.type = i32;
     } else if (rem >= 2u) {
       mem.type = i16;
       mem.size = 2u;
       tmp.type = i16;
     } else {
       mem.type = i8;
       mem.size = 1u;
       tmp.type = i8;
     }
     mem.align = mem.size;
     pos = backward ? access.size - off - mem.size : off;
     aa_load_native(t, tmp, aa_addr_plus(src, pos), mem);
     aa_store_native(t, aa_addr_plus(dst, pos), tmp, mem);
     off += mem.size;
     tmp.type = i64;
   }
 }
 
 static void aa_copy_bytes(NativeTarget* t, NativeAddr dst, NativeAddr src,
                           AggregateAccess access) {
   aa_copy_bytes_dir(t, dst, src, access, 0);
 }
 
 /* L3: widen the AGG_SET (whole-struct/array zero-init, memset) expander. The
  * old form emitted one `strb` per byte even for an aligned whole-struct zero
  * (cjson had a 64-long strb run). Instead, splat the fill byte across a 64-bit
  * register once and store the widest aligned chunk covering the remaining run
  * (`str x` -> `str w` -> `strh` -> `strb` tail), exactly mirroring the
  * aa_copy_bytes_dir width ladder. This is a pure per-call expansion (it only
  * removes emitted instructions); correctness holds for any fill byte (the splat
  * replicates it to all 8 bytes), for unaligned bases/sizes (the ladder narrows
  * the chunk to the remaining run, which the store offset never over-runs), and
  * for the exact byte count.
  *
  * The splat costs a few up-front instructions, so for tiny runs the per-byte
  * loop is still cheaper; gate the widening on a size threshold. */
 #define AA_SET_BYTES_WIDEN_MIN 8u
 
 static void aa_set_bytes(NativeTarget* t, NativeAddr dst, NativeLoc byte_value,
                          AggregateAccess access) {
   KitCgTypeId i8 = builtin_id(KIT_CG_BUILTIN_I8);
   NativeLoc byte = byte_value;
   MemAccess mem = access.mem;
   mem.type = i8;
   mem.size = 1u;
   mem.align = 1u;
   byte.type = i8;
   if (access.size < AA_SET_BYTES_WIDEN_MIN || native_loc_is_fp(byte_value)) {
     /* Small run (or a non-integer fill value): the original per-byte loop. */
     for (u32 off = 0; off < access.size; ++off)
       aa_store_native(t, aa_addr_plus(dst, off), byte, mem);
     return;
   }
   /* Splat the fill byte to every byte of a 64-bit scratch (AA_TMP0), distinct
    * from the address base/index (IR_AGG_SET materializes the value clear of
    * them). `and w,b,#0xff` clears any high bits, then ORR-shift doublings build
    * bb -> bbbb -> bbbbbbbb. For a zero fill these collapse to zeros, but stay
    * correct; the win is replacing N byte stores with N/8 dword stores. */
   {
     MCEmitter* mc = t->mc;
     KitCgTypeId i64 = builtin_id(KIT_CG_BUILTIN_I64);
     KitCgTypeId i32 = builtin_id(KIT_CG_BUILTIN_I32);
     KitCgTypeId i16 = builtin_id(KIT_CG_BUILTIN_I16);
     u32 b = loc_reg(byte_value);
     NativeLoc val = aa_tmp_loc(i64, AA_TMP0);
     u32 immr = 0, imms = 0, N = 0;
     u32 off = 0;
     (void)aa64_logimm_encode(0xffu, 0u, &N, &immr, &imms);
     aa_emit32(mc, aa64_and_imm(0u, AA_TMP0, b, N, immr, imms)); /* w16 = b&0xff */
     aa_emit32(mc, aa64_logsr_pack((AA64LogSR){.sf = 0u,
                                               .opc = AA64_LOG_ORR_OPC,
                                               .Rm = AA_TMP0,
                                               .imm6 = 8u,
                                               .Rn = AA_TMP0,
                                               .Rd = AA_TMP0})); /* orr w,w,w<<8 */
     aa_emit32(mc, aa64_logsr_pack((AA64LogSR){
                       .sf = 0u,
                       .opc = AA64_LOG_ORR_OPC,
                       .Rm = AA_TMP0,
                       .imm6 = 16u,
                       .Rn = AA_TMP0,
                       .Rd = AA_TMP0})); /* orr w,w,w<<16 -> low 32 = bbbb */
     aa_emit32(mc, aa64_logsr_pack((AA64LogSR){
                       .sf = 1u,
                       .opc = AA64_LOG_ORR_OPC,
                       .Rm = AA_TMP0,
                       .imm6 = 32u,
                       .Rn = AA_TMP0,
                       .Rd = AA_TMP0})); /* orr x,x,x<<32 -> all 8 bytes = b */
     while (off < access.size) {
       u32 rem = access.size - off;
       if (rem >= 8u) {
         mem.type = i64;
         mem.size = 8u;
         val.type = i64;
       } else if (rem >= 4u) {
         mem.type = i32;
         mem.size = 4u;
         val.type = i32;
       } else if (rem >= 2u) {
         mem.type = i16;
         mem.size = 2u;
         val.type = i16;
       } else {
         mem.type = i8;
         mem.size = 1u;
         val.type = i8;
       }
       mem.align = mem.size;
       aa_store_native(t, aa_addr_plus(dst, off), val, mem);
       off += mem.size;
     }
   }
 }
 
 static void aa_lsl_imm(NativeTarget* t, u32 sf, u32 rd, u32 rn, u32 sh);
 static void aa_lsr_imm(NativeTarget* t, u32 sf, u32 rd, u32 rn, u32 sh);
 static void aa_asr_imm(NativeTarget* t, u32 sf, u32 rd, u32 rn, u32 sh);
 
 /* Strength-reduce `mul rd, rn, #imm` for the constants accepted by
  * aa64_imul_strength_reducible into a single non-mul instruction. Callers
  * must gate on aa64_imul_strength_reducible(sf, imm) — this routine panics
  * on unhandled constants. */
 static void aa_emit_mul_const_imm(NativeTarget* t, u32 sf, u32 rd, u32 rn,
                                   i64 imm) {
   u64 a;
   if (imm == 0) {
     aa_emit32(t->mc, aa64_mov_reg(sf, rd, AA64_ZR));
     return;
   }
   if (imm == 1) {
     if (rd != rn) aa_emit32(t->mc, aa64_mov_reg(sf, rd, rn));
     return;
   }
   if (imm == -1) {
     aa_emit32(t->mc, aa64_neg(sf, rd, rn));
     return;
   }
   /* +2^k: lsl rd, rn, #k */
   a = (u64)imm;
   if (imm > 0 && (a & (a - 1u)) == 0u) {
     u32 k = (u32)__builtin_ctzll(a);
     aa_lsl_imm(t, sf, rd, rn, k);
     return;
   }
   /* -2^k: sub rd, xzr, rn, lsl #k */
   if (imm < 0) {
     a = (u64)(-imm);
     if (a && (a & (a - 1u)) == 0u) {
       u32 k = (u32)__builtin_ctzll(a);
       aa_emit32(t->mc, aa64_addsubsr_pack((AA64AddSubSR){.sf = sf,
                                                          .op = 1u,
                                                          .S = 0u,
                                                          .shift = 0u,
                                                          .Rm = rn,
                                                          .imm6 = k,
                                                          .Rn = AA64_ZR,
                                                          .Rd = rd}));
       return;
     }
   }
   /* 2^k + 1: add rd, rn, rn, lsl #k */
   if (imm >= 3) {
     u64 m = (u64)(imm - 1);
     if ((m & (m - 1u)) == 0u) {
       u32 k = (u32)__builtin_ctzll(m);
       aa_emit32(t->mc, aa64_addsubsr_pack((AA64AddSubSR){.sf = sf,
                                                          .op = 0u,
                                                          .S = 0u,
                                                          .shift = 0u,
                                                          .Rm = rn,
                                                          .imm6 = k,
                                                          .Rn = rn,
                                                          .Rd = rd}));
       return;
     }
   }
   /* 1 - 2^k: sub rd, rn, rn, lsl #k */
   if (imm <= -1) {
     u64 m = (u64)(1 - imm);
     if (m && (m & (m - 1u)) == 0u) {
       u32 k = (u32)__builtin_ctzll(m);
       aa_emit32(t->mc, aa64_addsubsr_pack((AA64AddSubSR){.sf = sf,
                                                          .op = 1u,
                                                          .S = 0u,
                                                          .shift = 0u,
                                                          .Rm = rn,
                                                          .imm6 = k,
                                                          .Rn = rn,
                                                          .Rd = rd}));
       return;
     }
   }
   aa_panic(aa_of(t), "aa_emit_mul_const_imm: unhandled constant");
 }
 
 static void aa_binop(NativeTarget* t, BinOp op, NativeLoc dst, NativeLoc lhs,
                      NativeLoc rhs) {
   u32 sf = loc_is_64(t, dst) ? 1u : 0u;
   u32 rd = loc_reg(dst), rn = loc_reg(lhs), rm = loc_reg(rhs);
   if (native_loc_is_fp(dst)) {
     u32 d = loc_size32(t, dst) == 8u;
     switch (op) {
       case BO_FADD:
         aa_emit32(t->mc, aa_fp_bin(0x002800u, d, rd, rn, rm));
         return;
       case BO_FSUB:
         aa_emit32(t->mc, aa_fp_bin(0x003800u, d, rd, rn, rm));
         return;
       case BO_FMUL:
         aa_emit32(t->mc, aa_fp_bin(0x000800u, d, rd, rn, rm));
         return;
       case BO_FDIV:
         aa_emit32(t->mc, aa_fp_bin(0x001800u, d, rd, rn, rm));
         return;
       default:
         aa_panic(aa_of(t), "unsupported floating binary op");
     }
   }
   if (rhs.kind == NATIVE_LOC_IMM && (op == BO_IADD || op == BO_ISUB)) {
     i64 imm = rhs.v.imm;
     int is_add = (op == BO_IADD);
     u32 imm12, sh;
     if (imm < 0) {
       is_add = !is_add;
       imm = -imm;
     }
     if (!aa64_addsub_imm_fits(imm, &imm12, &sh))
       aa_panic(aa_of(t), "binop immediate not encodable");
     aa_emit32(t->mc, is_add ? aa64_add_imm(sf, rd, rn, imm12, sh)
                             : aa64_sub_imm(sf, rd, rn, imm12, sh));
     return;
   }
   if (rhs.kind == NATIVE_LOC_IMM && op == BO_IMUL) {
     aa_emit_mul_const_imm(t, sf, rd, rn, rhs.v.imm);
     return;
   }
   if (rhs.kind == NATIVE_LOC_IMM &&
       (op == BO_SHL || op == BO_SHR_U || op == BO_SHR_S)) {
     u32 shamt = (u32)rhs.v.imm; /* imm_legal guarantees 0 <= imm < datasize */
     if (op == BO_SHL)
       aa_lsl_imm(t, sf, rd, rn, shamt);
     else if (op == BO_SHR_U)
       aa_lsr_imm(t, sf, rd, rn, shamt);
     else
       aa_asr_imm(t, sf, rd, rn, shamt);
     return;
   }
   if (rhs.kind == NATIVE_LOC_IMM &&
       (op == BO_AND || op == BO_OR || op == BO_XOR)) {
     u32 N, immr, imms;
     if (!aa64_logimm_encode((u64)rhs.v.imm, sf, &N, &immr, &imms))
       aa_panic(aa_of(t), "logical immediate not encodable");
     if (op == BO_AND)
       aa_emit32(t->mc, aa64_and_imm(sf, rd, rn, N, immr, imms));
     else if (op == BO_OR)
       aa_emit32(t->mc, aa64_orr_imm(sf, rd, rn, N, immr, imms));
     else
       aa_emit32(t->mc, aa64_eor_imm(sf, rd, rn, N, immr, imms));
     return;
   }
   /* L7 shifted-register ALU: a single-use `lsl rm,#k` (k in 1..4) folded into
    * this op's second source. Emit `<op> rd,rn,rm,lsl #k` in one instruction
    * instead of `lsl rT,rm,#k; <op> rd,rn,rT`. Only an integer register rhs can
    * carry a shift rider (immediates never do, and the recognition pass restricts
    * it to add/sub/and/orr/eor). */
   if (rhs.kind == NATIVE_LOC_REG && rhs.shift) {
     u32 k = rhs.shift;
     switch (op) {
       case BO_IADD:
         aa_emit32(t->mc, aa_addsub_lsl(sf, /*op=add*/ 0u, rd, rn, rm, k));
         return;
       case BO_ISUB:
         aa_emit32(t->mc, aa_addsub_lsl(sf, /*op=sub*/ 1u, rd, rn, rm, k));
         return;
       case BO_AND:
         aa_emit32(t->mc, aa_logsr_lsl(sf, AA64_LOG_AND_OPC, rd, rn, rm, k));
         return;
       case BO_OR:
         aa_emit32(t->mc, aa_logsr_lsl(sf, AA64_LOG_ORR_OPC, rd, rn, rm, k));
         return;
       case BO_XOR:
         aa_emit32(t->mc, aa_logsr_lsl(sf, AA64_LOG_EOR_OPC, rd, rn, rm, k));
         return;
       default:
         aa_panic(aa_of(t), "shift rider on unsupported binop");
     }
   }
   switch (op) {
     case BO_IADD:
       aa_emit32(t->mc, aa64_add(sf, rd, rn, rm));
       return;
     case BO_ISUB:
       aa_emit32(t->mc, aa64_sub(sf, rd, rn, rm));
       return;
     case BO_IMUL:
       aa_emit32(t->mc, aa64_mul(sf, rd, rn, rm));
       return;
     case BO_SDIV:
       aa_emit32(t->mc, aa64_sdiv(sf, rd, rn, rm));
       return;
     case BO_UDIV:
       aa_emit32(t->mc, aa64_udiv(sf, rd, rn, rm));
       return;
     case BO_SREM:
       aa_emit32(t->mc, aa64_sdiv(sf, AA_TMP0, rn, rm));
       aa_emit32(t->mc, aa64_mul(sf, AA_TMP0, AA_TMP0, rm));
       aa_emit32(t->mc, aa64_sub(sf, rd, rn, AA_TMP0));
       return;
     case BO_UREM:
       aa_emit32(t->mc, aa64_udiv(sf, AA_TMP0, rn, rm));
       aa_emit32(t->mc, aa64_mul(sf, AA_TMP0, AA_TMP0, rm));
       aa_emit32(t->mc, aa64_sub(sf, rd, rn, AA_TMP0));
       return;
     case BO_AND:
       aa_emit32(t->mc, aa64_and(sf, rd, rn, rm));
       return;
     case BO_OR:
       aa_emit32(t->mc, aa64_orr(sf, rd, rn, rm));
       return;
     case BO_XOR:
       aa_emit32(t->mc, aa64_eor(sf, rd, rn, rm));
       return;
     case BO_SHL:
       aa_emit32(t->mc, aa64_lslv(sf, rd, rn, rm));
       return;
     case BO_SHR_U:
       aa_emit32(t->mc, aa64_lsrv(sf, rd, rn, rm));
       return;
     case BO_SHR_S:
       aa_emit32(t->mc, aa64_asrv(sf, rd, rn, rm));
       return;
     default:
       aa_panic(aa_of(t), "unsupported binary op");
   }
 }
 
 static void aa_unop(NativeTarget* t, UnOp op, NativeLoc dst, NativeLoc src) {
   u32 sf = loc_is_64(t, dst) ? 1u : 0u;
   if (native_loc_is_fp(dst)) {
     switch (op) {
       case UO_FNEG:
       case UO_NEG:
         aa_emit32(t->mc, aa_fneg(loc_size32(t, dst) == 8u, loc_reg(dst),
                                  loc_reg(src)));
         return;
       default:
         aa_panic(aa_of(t), "unsupported floating unary op");
     }
   }
   switch (op) {
     case UO_NEG:
       aa_emit32(t->mc, aa64_neg(sf, loc_reg(dst), loc_reg(src)));
       return;
     case UO_BNOT:
       aa_emit32(t->mc, aa64_mvn(sf, loc_reg(dst), loc_reg(src)));
       return;
     case UO_NOT:
       aa_emit32(t->mc, aa64_subs_imm12(sf, AA64_ZR, loc_reg(src), 0, 0));
       aa_emit32(t->mc, aa_cset(sf, loc_reg(dst), 0x0u));
       return;
     default:
       aa_panic(aa_of(t), "unsupported unary op");
   }
 }
 
 static void aa_emit_cmp_to_flags(NativeTarget* t, NativeLoc lhs,
                                  NativeLoc rhs) {
   if (native_loc_is_fp(lhs)) {
     aa_emit32(t->mc,
               aa_fcmp(loc_size32(t, lhs) == 8u, loc_reg(lhs), loc_reg(rhs)));
     return;
   }
   {
     u32 sf = loc_is_64(t, lhs) ? 1u : 0u;
     if (rhs.kind == NATIVE_LOC_IMM) {
       u32 imm12 = 0, sh = 0;
       if (rhs.v.imm < 0 || !aa64_addsub_imm_fits(rhs.v.imm, &imm12, &sh))
         aa_panic(aa_of(t), "cmp immediate not encodable");
       aa_emit32(t->mc, aa64_subs_imm12(sf, AA64_ZR, loc_reg(lhs), imm12, sh));
       return;
     }
     aa_emit32(t->mc, aa_subs_reg(sf, AA64_ZR, loc_reg(lhs), loc_reg(rhs)));
   }
 }
 
 static void aa_cmp(NativeTarget* t, CmpOp op, NativeLoc dst, NativeLoc lhs,
                    NativeLoc rhs) {
   u32 sf = loc_is_64(t, dst);
   u32 rd = loc_reg(dst);
   aa_emit_cmp_to_flags(t, lhs, rhs);
   /* CMP_ONE_F (ordered & !=) and CMP_UEQ_F (unordered | ==) have no single
    * AArch64 FP condition. After FCMP, unordered sets V (and Z=0), so VC
    * (V==0) selects "ordered". */
   if (op == CMP_ONE_F) {
     /* ordered & not-equal: NE masked to the ordered case. */
     aa_emit32(t->mc, aa_cset(sf, rd, 0x1u)); /* cset  rd, NE */
     aa_emit32(t->mc,
               aa64_csel_enc(sf, rd, rd, AA64_ZR, 0x7u)); /* csel  rd,rd,zr,VC */
     return;
   }
   if (op == CMP_UEQ_F) {
     /* equal, or forced to 1 when unordered. */
     aa_emit32(t->mc, aa_cset(sf, rd, 0x0u)); /* cset  rd, EQ */
     aa_emit32(t->mc, aa64_csinc_enc(sf, rd, rd, AA64_ZR,
                                     0x7u)); /* csinc rd,rd,zr,VC */
     return;
   }
   aa_emit32(t->mc, aa_cset(sf, rd, cmp_cond(op)));
 }
 
 static void aa_convert(NativeTarget* t, ConvKind op, NativeLoc dst,
                        NativeLoc src) {
   int dst_fp = native_loc_is_fp(dst);
   int src_fp = native_loc_is_fp(src);
   switch (op) {
     case CV_TRUNC:
     case CV_BITCAST:
       aa_move(t, dst, src);
       return;
     case CV_ZEXT: {
       u32 src_bits = loc_size32(t, src) * 8u;
       u32 dst_bits = loc_size32(t, dst) * 8u;
       u32 sf = dst_bits > 32u;
       if (src_bits >= dst_bits) {
         aa_move(t, dst, src);
       } else if (src_bits >= 32u) {
         /* w-reg mov zero-extends into the x-reg. When dst and src are the same
          * hard register (the -O0 convert-coalescing rename targets the source's
          * own register) it is a no-op: the value already arrived there from a
          * 32-bit-writing op, which itself cleared the upper 32 bits. */
         if (!(dst.kind == NATIVE_LOC_REG && src.kind == NATIVE_LOC_REG &&
               dst.v.reg == src.v.reg))
           aa_emit32(t->mc, aa64_mov_reg(0, loc_reg(dst), loc_reg(src)));
       } else {
         aa_emit32(t->mc,
                   aa_ubfm(sf, loc_reg(dst), loc_reg(src), 0, src_bits - 1u));
       }
       return;
     }
     case CV_SEXT: {
       u32 src_bits = loc_size32(t, src) * 8u;
       u32 dst_bits = loc_size32(t, dst) * 8u;
       u32 sf = dst_bits > 32u;
       if (src_bits >= dst_bits) {
         aa_move(t, dst, src);
       } else {
         aa_emit32(t->mc,
                   aa_sbfm(sf, loc_reg(dst), loc_reg(src), 0, src_bits - 1u));
       }
       return;
     }
     case CV_ITOF_S:
       aa_emit32(t->mc, aa_scvtf(loc_size32(t, dst) == 8u, loc_is_64(t, src),
                                 loc_reg(dst), loc_reg(src)));
       return;
     case CV_ITOF_U:
       aa_emit32(t->mc, aa_ucvtf(loc_size32(t, dst) == 8u, loc_is_64(t, src),
                                 loc_reg(dst), loc_reg(src)));
       return;
     case CV_FTOI_S:
       aa_emit32(t->mc, aa_fcvtzs(loc_is_64(t, dst), loc_size32(t, src) == 8u,
                                  loc_reg(dst), loc_reg(src)));
       return;
     case CV_FTOI_U:
       aa_emit32(t->mc, aa_fcvtzu(loc_is_64(t, dst), loc_size32(t, src) == 8u,
                                  loc_reg(dst), loc_reg(src)));
       return;
     case CV_FEXT:
       if (dst_fp && src_fp)
         aa_emit32(t->mc, aa_fcvt_d_s(loc_reg(dst), loc_reg(src)));
       else
         aa_move(t, dst, src);
       return;
     case CV_FTRUNC:
       if (dst_fp && src_fp)
         aa_emit32(t->mc, aa_fcvt_s_d(loc_reg(dst), loc_reg(src)));
       else
         aa_move(t, dst, src);
       return;
     default:
       aa_panic(aa_of(t), "unsupported conversion");
   }
 }
 
 /* §E.3 narrow register-only entry points. The NDT crosses 16 B NativeRegLoc
  * here; these reconstruct the NativeLoc the fat hook expects and delegate, so
  * the emitted bytes are identical. Used only on the -O0 NDT path; the opt
  * replay path calls aa_binop/aa_move/aa_cmp/aa_convert directly. */
 static void aa_binop_rr(NativeTarget* t, BinOp op, NativeRegLoc dst,
                         NativeRegLoc a, NativeRegLoc b) {
   aa_binop(t, op, native_loc_from_reg(dst), native_loc_from_reg(a),
            native_loc_from_reg(b));
 }
 
 static void aa_move_rr(NativeTarget* t, NativeRegLoc dst, NativeRegLoc src) {
   aa_move(t, native_loc_from_reg(dst), native_loc_from_reg(src));
 }
 
 static void aa_cmp_rr(NativeTarget* t, CmpOp op, NativeRegLoc dst,
                       NativeRegLoc a, NativeRegLoc b) {
   aa_cmp(t, op, native_loc_from_reg(dst), native_loc_from_reg(a),
          native_loc_from_reg(b));
 }
 
 static void aa_convert_rr(NativeTarget* t, ConvKind op, NativeRegLoc dst,
                           NativeRegLoc src) {
   aa_convert(t, op, native_loc_from_reg(dst), native_loc_from_reg(src));
 }
 
 static void aa_alloca(NativeTarget* t, NativeLoc dst, NativeLoc size,
                       u32 align) {
   AANativeTarget* a = aa_of(t);
   u32 use_align = align < 16u ? 16u : align;
   if (use_align & (use_align - 1u)) aa_panic(a, "alloca alignment not pow2");
   aa_emit_add_imm(a, AA_TMP0, loc_reg(size), (i32)(use_align - 1u));
   aa_emit_load_imm(t->mc, 1, AA_TMP1, -(i64)use_align);
   aa_emit32(t->mc, aa64_and(1, AA_TMP0, AA_TMP0, AA_TMP1));
   aa_emit32(t->mc, aa64_add_imm(1, AA_TMP1, AA_SP, 0, 0));
   aa_emit32(t->mc, aa64_sub(1, AA_TMP1, AA_TMP1, AA_TMP0));
   aa_emit32(t->mc, aa64_add_imm(1, AA_SP, AA_TMP1, 0, 0));
   /* The alloca result is sp + outgoing-area bytes. On the known-frame path
    * max_outgoing is already final, so emit the final `add dst, sp, #N` here; on
    * the single-pass path it is not known yet, so record a patch. */
   if (a->frame.known_frame) {
     u32 imm12, sh;
     if (!aa64_addsub_imm_fits(a->frame.max_outgoing, &imm12, &sh))
       aa_panic(a, "outgoing area too large for alloca result");
     aa_emit32(t->mc, aa64_add_imm(1, loc_reg(dst), AA_SP, imm12, sh));
   } else {
     AAPatch* p;
     /* First alloca arms the frame-base anchor. The far-slot fast path can no
      * longer key off sp (this `sub sp` floats it), so deferred AA_PATCH_SLOTs —
      * those already emitted and any still to come — resolve against
      * AA_FRAME_BASE instead. Reserve a static-frame home for the caller's
      * value; the prologue saves it and sets AA_FRAME_BASE = frame base (before
      * the body runs, so every slot access sees a valid anchor), the epilogue
      * restores it. Only needed when slot_sp_base armed the fast path in the
      * first place. */
     if (a->slot_sp_base && !a->uses_frame_base) {
       NativeFrameSlotDesc sd;
       memset(&sd, 0, sizeof sd);
       sd.type = builtin_id(KIT_CG_BUILTIN_I64);
       sd.size = 8;
       sd.align = 8;
       sd.kind = NATIVE_FRAME_SLOT_SAVE;
       a->frame_base_slot = t->frame_slot(t, &sd);
       a->uses_frame_base = 1;
     }
     p = aa_patch_alloc(a);
     p->kind = AA_PATCH_ALLOCA;
     p->pos = mc_pos(t->mc);
     p->u.dst_reg = loc_reg(dst);
     a->nalloca++;
     aa_emit32(t->mc, aa64_add_imm(1, loc_reg(dst), AA_SP, 0, 0));
   }
 }
 
 static MemAccess aa_mem_for_type(NativeTarget* t, KitCgTypeId type, u32 size) {
   MemAccess mem;
   memset(&mem, 0, sizeof mem);
   mem.type = type;
   mem.size = size ? size : type_size32(t, type);
   mem.align = type_align32(t, type);
   if (mem.align > mem.size && mem.size) mem.align = mem.size;
   return mem;
 }
 
 /* native_loc_reg / native_loc_stack are shared in native_target.h. */
 
 static NativeAddr aa_storage_addr(AANativeTarget* a, NativeLoc loc,
                                   u32 offset) {
   NativeAddr addr;
   if (!native_loc_storage_addr(loc, (i32)offset, &addr))
     aa_panic(a, "location is not storage");
   return addr;
 }
 
 static void aa_addr_of_loc(NativeTarget* t, NativeLoc dst, NativeLoc src) {
   AANativeTarget* a = aa_of(t);
   NativeAddr addr = aa_storage_addr(a, src, 0);
   aa_load_addr(t, dst, addr);
 }
 
 static void aa_load_part(NativeTarget* t, NativeLoc dst, NativeLoc src,
                          u32 offset, u32 size) {
   AANativeTarget* a = aa_of(t);
   MemAccess mem = aa_mem_for_type(t, dst.type, size);
   if (src.kind == NATIVE_LOC_REG) {
     aa_move(t, dst, src);
     return;
   }
   if (native_loc_addr_role(src) == NATIVE_LOC_ADDR_ROLE_VALUE) {
     NativeAddr addr;
     if (offset != 0u || size != t->c->target.ptr_size ||
         !native_loc_address_value(src, &addr))
       aa_panic(a, "split or invalid address-value argument");
     aa_load_addr(t, dst, addr);
     return;
   }
   if (native_loc_addr_role(src) == NATIVE_LOC_ADDR_ROLE_STORAGE) {
     NativeAddr addr = aa_storage_addr(a, src, offset);
     addr.base_type = dst.type;
     aa_emit_mem(a, 1, dst, addr, mem);
     return;
   }
   if (src.kind == NATIVE_LOC_IMM) {
     i64 part;
     if (!native_loc_imm_part(src, offset, size, &part))
       aa_panic(a, "invalid immediate argument part");
     aa_emit_load_imm(t->mc, loc_is_64(t, dst), loc_reg(dst), part);
     return;
   }
   aa_panic(a, "unsupported call argument source");
 }
 
 static void aa_store_part(NativeTarget* t, NativeLoc dst, NativeLoc src,
                           u32 offset, u32 size) {
   AANativeTarget* a = aa_of(t);
   MemAccess mem = aa_mem_for_type(t, src.type, size);
   if (native_loc_addr_role(dst) == NATIVE_LOC_ADDR_ROLE_STORAGE) {
     NativeAddr addr = aa_storage_addr(a, dst, offset);
     addr.base_type = src.type;
     aa_emit_mem(a, 0, src, addr, mem);
     return;
   }
   if (dst.kind == NATIVE_LOC_REG) {
     aa_move(t, dst, src);
     return;
   }
   aa_panic(a, "unsupported call return destination");
 }
 
 static void aa_store_outgoing_part(NativeTarget* t, int tail_call,
                                    u32 stack_off, NativeLoc src, u32 size) {
   NativeAddr addr;
   MemAccess mem = aa_mem_for_type(t, src.type, size);
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_REG;
   addr.base.reg = tail_call ? AA_FP : AA_SP;
   addr.base_type = src.type;
   /* Tail calls write outgoing args into the caller's incoming-args window
    * (= [fp + 16 + off], same address the tail-callee will read via
    * aa_fp_off_in_arg). Non-tail calls write to the sp-anchored outgoing
    * area at the bottom of the caller's frame. */
   addr.offset = tail_call ? aa_fp_off_tail_out_arg(aa_of(t), stack_off)
                           : aa_sp_off_out_arg(stack_off);
   aa_emit_mem(aa_of(t), 0, src, addr, mem);
 }
 
 /* Copy only the bytes that belong to a stack-routed value. The caller owns the
  * separately rounded ABI slot and advances its physical stack cursor itself;
  * this helper never treats that carrier/padding size as readable source data. */
 static void aa_store_outgoing_value_bytes(NativeTarget* t, int tail_call,
                                           u32 stack_off, NativeLoc src,
                                           u32 value_size) {
   NativeLoc tmp =
       native_loc_reg(src.type, NATIVE_REG_INT, AA_TMP0);
   u32 off = 0;
   while (off < value_size) {
     u32 chunk = value_size - off;
     if (chunk > 8u) chunk = 8u;
     aa_load_part(t, tmp, src, off, chunk);
     aa_store_outgoing_part(t, tail_call, stack_off + off, tmp, chunk);
     off += chunk;
   }
 }
 
 static const ABIArgInfo* aa_param_abi(NativeTarget* t, const ABIFuncInfo* abi,
                                       const NativeCallDesc* desc, u32 i,
                                       ABIArgInfo* scratch) {
   if (abi && i < abi->nparams) return &abi->params[i];
   memset(scratch, 0, sizeof *scratch);
   scratch->kind = ABI_ARG_DIRECT;
   scratch->flags = ABI_AF_NONE;
   scratch->nparts = 1;
   scratch->parts = arena_zarray(t->c->tu, ABIArgPart, 1);
   /* Windows ARM64 routes variadic floating-point arguments through the integer
    * registers/stack (the classifier's remap_fp_parts_to_int does the same for
    * the *named* params of a variadic function); the value's bit pattern moves
    * via fmov x,d. Every other ABI keeps the `...` FP args in v registers. */
   ((ABIArgPart*)scratch->parts)[0].cls =
       (cg_type_is_float(t->c, desc->args[i].type) &&
        !(abi && abi->vararg_fp_via_int))
           ? ABI_CLASS_FP
           : ABI_CLASS_INT;
   ((ABIArgPart*)scratch->parts)[0].loc = ABI_LOC_REG;
   ((ABIArgPart*)scratch->parts)[0].size = type_size32(t, desc->args[i].type);
   ((ABIArgPart*)scratch->parts)[0].align = type_align32(t, desc->args[i].type);
   ((ABIArgPart*)scratch->parts)[0].src_offset = 0;
   return scratch;
 }
 
 /* Stack footprint of a single argument part. AAPCS64 uses 8-byte slots. Apple
  * ARM64 uses compact 4-byte slots for fixed stack-passed int32-sized values,
  * but its forced stack variadics still use 8-byte slots. */
 static u32 aa_stack_arg_min_align(const ABIFuncInfo* abi) {
   return (abi && abi->stack_arg_min_align) ? abi->stack_arg_min_align : 8u;
 }
 
 static u32 aa_vararg_stack_arg_min_align(const ABIFuncInfo* abi) {
   if (abi && abi->vararg_stack_arg_min_align)
     return abi->vararg_stack_arg_min_align;
   return aa_stack_arg_min_align(abi);
 }
 
 static u32 aa_vararg_stack_start(const ABIFuncInfo* abi, u32 cursor) {
   return align_up_u32(cursor, aa_vararg_stack_arg_min_align(abi));
 }
 
 /* Natural stack alignment of a part, capped at 16 (binary128). */
 static u32 aa_part_stack_align_min(u32 min_align, const ABIArgPart* part) {
   u32 al = part->align ? part->align : 8u;
   if (al < min_align) al = min_align;
   if (al > 16u) al = 16u;
   return al;
 }
 
 static u32 aa_part_stack_align(const ABIFuncInfo* abi, const ABIArgPart* part) {
   return aa_part_stack_align_min(aa_stack_arg_min_align(abi), part);
 }
 
 static u32 aa_part_vararg_stack_align(const ABIFuncInfo* abi,
                                       const ABIArgPart* part) {
   return aa_part_stack_align_min(aa_vararg_stack_arg_min_align(abi), part);
 }
 
 static u32 aa_part_stack_size(const ABIFuncInfo* abi, const ABIArgPart* part) {
   return align_up_u32(part->size ? part->size : 8u,
                       aa_part_stack_align(abi, part));
 }
 
 static u32 aa_part_vararg_stack_size(const ABIFuncInfo* abi,
                                      const ABIArgPart* part) {
   return align_up_u32(part->size ? part->size : 8u,
                       aa_part_vararg_stack_align(abi, part));
 }
 
 /* The scalar type used to move one ABI part through a register. Aggregate
  * args/results are split into parts; each part must move at its own width, not
  * the (possibly >8-byte) aggregate width. */
 static KitCgTypeId aa_part_scalar_type(const ABIArgPart* part) {
   if (part->cls == ABI_CLASS_FP) {
     if (part->size <= 4u) return builtin_id(KIT_CG_BUILTIN_F32);
     if (part->size <= 8u) return builtin_id(KIT_CG_BUILTIN_F64);
     return builtin_id(KIT_CG_BUILTIN_F128);
   }
   switch (part->size) {
     case 1u:
       return builtin_id(KIT_CG_BUILTIN_I8);
     case 2u:
       return builtin_id(KIT_CG_BUILTIN_I16);
     case 4u:
       return builtin_id(KIT_CG_BUILTIN_I32);
     default:
       return builtin_id(KIT_CG_BUILTIN_I64);
   }
 }
 
 static u32 aa_class_vararg_stack_size(const ABIFuncInfo* abi,
                                       const ABIArgInfo* ai) {
   u32 total = 0;
   u32 min_align = aa_vararg_stack_arg_min_align(abi);
   if (!ai || ai->kind == ABI_ARG_IGNORE) return 0;
   if (ai->kind == ABI_ARG_INDIRECT) return 8u;
   for (u32 p = 0; p < ai->nparts; ++p) {
     total = align_up_u32(total, aa_part_vararg_stack_align(abi, &ai->parts[p]));
     total += aa_part_vararg_stack_size(abi, &ai->parts[p]);
   }
   return align_up_u32(total ? total : min_align, min_align);
 }
 
 static u32 aa_call_stack_size(NativeTarget* t, const NativeCallDesc* desc) {
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, desc->fn_type);
   u32 next_int = 0, next_fp = 0, stack = 0;
   for (u32 i = 0; i < desc->nargs; ++i) {
     ABIArgInfo tmp;
     const ABIArgInfo* ai = aa_param_abi(t, abi, desc, i, &tmp);
     int force_stack =
         abi && abi->variadic && abi->vararg_on_stack && i >= abi->nparams;
     if (ai->kind == ABI_ARG_IGNORE) continue;
     if (force_stack) {
       stack = aa_vararg_stack_start(abi, stack);
       stack += aa_class_vararg_stack_size(abi, ai);
       continue;
     }
     if (ai->kind == ABI_ARG_INDIRECT) {
       if (next_int < 8u)
         next_int++;
       else
         stack += 8u;
       continue;
     }
     for (u32 p = 0; p < ai->nparts; ++p) {
       const ABIArgPart* part = &ai->parts[p];
       if (part->cls == ABI_CLASS_FP) {
         if (next_fp < 8u)
           next_fp++;
         else {
           stack = align_up_u32(stack, aa_part_stack_align(abi, part));
           stack += aa_part_stack_size(abi, part);
         }
       } else {
         if (next_int < 8u)
           next_int++;
         else {
           stack = align_up_u32(stack, aa_part_stack_align(abi, part));
           stack += aa_part_stack_size(abi, part);
         }
       }
     }
   }
   return align_up_u32(stack, 16u);
 }
 
 /* Stack-argument bytes a call with `fn_type`'s fixed parameters uses. Reuses
  * aa_call_stack_size by routing the declared params through it (their ABI
  * classification is independent of the actual operand locations, which
  * aa_call_stack_size ignores for register/stack placement). */
 static u32 aa_signature_stack_bytes(NativeTarget* t, KitCgTypeId fn_type,
                                     int* variadic, u32* nparams) {
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, fn_type);
   NativeCallDesc d;
   if (variadic) *variadic = abi ? (int)abi->variadic : 0;
   if (nparams) *nparams = abi ? abi->nparams : 0u;
   memset(&d, 0, sizeof d);
   d.fn_type = fn_type;
   d.nargs = abi ? abi->nparams : 0u;
   if (d.nargs) d.args = arena_zarray(t->c->tu, NativeLoc, d.nargs);
   return aa_call_stack_size(t, &d);
 }
 
 /* Pure NativeTarget.call_stack_bytes: outgoing stack bytes for a full call
  * descriptor (handles variadic stack args, unlike signature_stack_bytes which
  * sees only the fixed params). aa_call_stack_size reads only fn_type and each
  * args[i].type, so the frame-planning pre-pass can call this before emitting.
  */
 static u32 aa_call_stack_bytes(NativeTarget* t, const NativeCallDesc* desc) {
   return aa_call_stack_size(t, desc);
 }
 
 /* One register-passed call argument: write `src` (or its address) into the
  * argument register `dst`. Collected during planning and emitted as a batch so
  * the backend can order them as a parallel copy (see aa_emit_reg_arg_moves). */
 typedef NativeArgMove AAArgMove;
 
 /* AAPCS64/Apple permit at most 8 GP + 8 FP register-passed argument slots. */
 #define AA_MAX_REG_ARG_MOVES 16u
 
 static void aa_emit_one_arg_move(NativeTarget* t, const NativeArgMove* m) {
   if (m->is_addr)
     aa_addr_of_loc(t, m->dst, m->src);
   else
     aa_load_part(t, m->dst, m->src, m->src_offset, m->size);
 }
 
 /* Emit register-argument moves as a parallel copy via the shared scheduler:
  * every register is read by all moves that source it before any move overwrites
  * it; a true cycle is broken through a scratch. The allocator usually arranges
  * a conflict-free order, but not always (notably variadic args, where it can
  * leave a prior call's result in x0 even though x0 is this call's first arg
  * register), so the backend must not assume a safe order. Cycle scratch is
  * AA_TMP1 (x17) for int and v16 for fp — distinct from x16 (AA_TMP0), which may
  * hold a stashed indirect callee. */
 static void aa_emit_reg_arg_moves(NativeTarget* t, NativeArgMove* moves,
                                   u32 n) {
   NativeArgShuffle s;
   if (n > AA_MAX_REG_ARG_MOVES)
     aa_panic(aa_of(t), "too many register arguments");
   memset(&s, 0, sizeof s);
   s.t = t;
   s.emit_one = aa_emit_one_arg_move;
   s.reg_move = aa_move;
   s.scratch[NATIVE_REG_INT] = AA_TMP1;
   s.scratch[NATIVE_REG_FP] = 16u;
   s.scratch_class_mask = (1u << NATIVE_REG_INT) | (1u << NATIVE_REG_FP);
   native_arg_shuffle(&s, moves, n);
 }
 
 static void aa_marshal_call(NativeTarget* t, const NativeCallDesc* desc,
                             NativeCallPhase* plan) {
   NativeCallPhaseRet* rets;
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, desc->fn_type);
   NativeLoc saved_indirect_callee;
   int preserve_indirect_callee = 0;
   /* Right-size the result scratch to the exact number of entries the ret loops
    * below write: nparts on a DIRECT register return, 1 on the !abi fallback,
    * 0 (NULL) otherwise (IGNORE / sret / no results). */
   u32 nrets_cap = (abi && abi->ret.kind == ABI_ARG_DIRECT && desc->nresults)
                       ? abi->ret.nparts
                       : ((!abi && desc->nresults) ? 1u : 0u);
   memset(plan, 0, sizeof *plan);
   rets =
       nrets_cap ? arena_zarray(t->c->tu, NativeCallPhaseRet, nrets_cap) : NULL;
   plan->callee = desc->callee;
   plan->rets = rets;
   plan->flags = desc->flags;
   plan->has_sret = abi && abi->has_sret;
   plan->is_variadic = abi && abi->variadic;
   plan->stack_arg_size = aa_call_stack_size(t, desc);
   native_frame_note_outgoing(&aa_of(t)->frame, plan->stack_arg_size);
   /* An indirect target in an argument/result or backend-private register can
    * be overwritten while this hook fills x0..x7, x8 (sret), or uses x16/x17
    * for exact-width stack arguments. Preserve it in LR, whose incoming value
    * is already frame-saved for a non-tail call and is dead on a tail path.
    * After all argument work, move it to the final call/branch register. */
   if (plan->callee.kind == NATIVE_LOC_REG &&
       (NativeAllocClass)plan->callee.cls == NATIVE_REG_INT &&
       (plan->callee.v.reg <= AA_X8 || plan->callee.v.reg == AA_TMP0 ||
        plan->callee.v.reg == AA_TMP1)) {
     saved_indirect_callee =
         native_loc_reg(plan->callee.type, NATIVE_REG_INT, AA_LR);
     aa_move(t, saved_indirect_callee, plan->callee);
     preserve_indirect_callee = 1;
   }
   {
     u32 next_int = 0, next_fp = 0, stack = 0, nmoves = 0;
     int tail_call = (desc->flags & CG_CALL_TAIL) != 0;
     AAArgMove moves[AA_MAX_REG_ARG_MOVES];
     /* Stack-passed arguments are stored inline as we walk, *before* any
      * argument register is written, so a stack-arg source that the allocator
      * left in an arg register (e.g. a prior call's result still in x0, consumed
      * as a variadic stack arg) is read while it is still live. Stack stores
      * only touch memory and the AA_TMP0/v16 scratch, never an arg-register
      * source, so emitting them first cannot clobber a register-arg source.
      * Register-passed arguments are collected and emitted afterward as a
      * parallel copy (aa_emit_reg_arg_moves) so they likewise never overwrite a
      * register another argument still needs to read. */
     for (u32 i = 0; i < desc->nargs; ++i) {
       ABIArgInfo tmp;
       const ABIArgInfo* ai = aa_param_abi(t, abi, desc, i, &tmp);
       int force_stack =
           abi && abi->variadic && abi->vararg_on_stack && i >= abi->nparams;
       if (ai->kind == ABI_ARG_IGNORE) continue;
       if (force_stack) {
         u32 n = aa_class_vararg_stack_size(abi, ai);
         stack = aa_vararg_stack_start(abi, stack);
         if (ai->kind == ABI_ARG_DIRECT && ai->nparts == 1 && n <= 8u) {
           /* Scalar variadic argument in one <=8-byte slot. Load the value at
            * its OWN width — a sub-slot integer load zero-extends into the
            * 64-bit temp — then store the whole slot. Loading the slot width
            * from the value's home would pull garbage high bits, and storing
            * only the value width would leave them undefined; either way a
            * callee that va_arg's a wider type than was passed (the common
            * `(int)0` read back as a null pointer) would see garbage. Zero-
            * extend-then-store-the-slot is what clang emits and makes it
            * well-defined. */
           const ABIArgPart* part = &ai->parts[0];
           NativeAllocClass cls =
               part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
           NativeLoc tmpreg = native_loc_reg(
               desc->args[i].type, cls, cls == NATIVE_REG_FP ? 16u : AA_TMP0);
           aa_load_part(t, tmpreg, desc->args[i], part->src_offset, part->size);
           aa_store_outgoing_part(t, tail_call, stack, tmpreg, n);
           stack += n;
           continue;
         }
         {
           /* Aggregate / multi-word variadic argument: `n` is the physical ABI
            * carrier (Apple rounds a 12-byte unnamed aggregate to 16), not a
            * source access width. Copy the value's exact object bytes, leaving
            * carrier padding untouched, then advance by the full slot. */
           u32 value_size = type_size32(t, desc->args[i].type);
           aa_store_outgoing_value_bytes(t, tail_call, stack, desc->args[i],
                                         value_size);
           stack += n;
         }
         continue;
       }
       if (ai->kind == ABI_ARG_INDIRECT) {
         if (next_int < 8u) {
           AAArgMove* m = &moves[nmoves++];
           m->dst = native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64),
                                   NATIVE_REG_INT, next_int++);
           m->src = desc->args[i];
           m->src_offset = 0;
           m->size = 8;
           m->is_addr = 1;
         } else {
           NativeLoc ptr = native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64),
                                          NATIVE_REG_INT, AA_TMP0);
           aa_addr_of_loc(t, ptr, desc->args[i]);
           aa_store_outgoing_part(t, tail_call, stack, ptr, 8);
           stack += 8u;
         }
         continue;
       }
       for (u32 p = 0; p < ai->nparts; ++p) {
         const ABIArgPart* part = &ai->parts[p];
         NativeAllocClass cls =
             part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
         if ((cls == NATIVE_REG_FP && next_fp < 8u) ||
             (cls == NATIVE_REG_INT && next_int < 8u)) {
           AAArgMove* m = &moves[nmoves++];
           m->dst =
               native_loc_reg(desc->args[i].type, cls,
                              cls == NATIVE_REG_FP ? next_fp++ : next_int++);
           m->src = desc->args[i];
           m->src_offset = part->src_offset;
           m->size = part->size;
           m->is_addr = 0;
         } else {
           NativeLoc tmpreg = native_loc_reg(
               desc->args[i].type, cls, cls == NATIVE_REG_FP ? 16u : AA_TMP0);
           aa_load_part(t, tmpreg, desc->args[i], part->src_offset, part->size);
           stack = align_up_u32(stack, aa_part_stack_align(abi, part));
           aa_store_outgoing_part(t, tail_call, stack, tmpreg, part->size);
           stack += aa_part_stack_size(abi, part);
         }
       }
     }
     aa_emit_reg_arg_moves(t, moves, nmoves);
     /* Set the indirect-result register (x8) *after* the argument loads: an
      * argument source may have been allocated to x8, and the sret pointer load
      * would otherwise clobber it before it is moved into its argument
      * register. */
     if (abi && abi->has_sret) {
       NativeLoc x8 =
           native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT, 8u);
       if (desc->flags & CG_CALL_TAIL) {
         AANativeTarget* a = aa_of(t);
         NativeLoc saved = native_loc_stack(x8.type, a->sret_ptr_slot, 0);
         aa_load_part(t, x8, saved, 0, 8);
       } else if (desc->nresults) {
         aa_addr_of_loc(t, x8, desc->results[0]);
       }
     }
     if (preserve_indirect_callee) {
       NativeLoc final_callee = native_loc_reg(
           saved_indirect_callee.type, NATIVE_REG_INT,
           tail_call ? AA_TAIL_TARGET : AA_TMP0);
       aa_move(t, final_callee, saved_indirect_callee);
       plan->callee = final_callee;
     }
   }
   if (abi && abi->ret.kind == ABI_ARG_DIRECT && desc->nresults) {
     u32 nr = 0, ni = 0, nf = 0;
     for (u32 p = 0; p < abi->ret.nparts; ++p) {
       const ABIArgPart* part = &abi->ret.parts[p];
       NativeAllocClass cls =
           part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
       KitCgTypeId pty = aa_part_scalar_type(part);
       rets[nr].src =
           native_loc_reg(pty, cls, cls == NATIVE_REG_FP ? nf++ : ni++);
       rets[nr].dst = desc->results[0];
       if (rets[nr].dst.kind == NATIVE_LOC_FRAME)
         rets[nr].dst = native_loc_stack(pty, desc->results[0].v.frame,
                                         (i32)part->src_offset);
       else if (rets[nr].dst.kind == NATIVE_LOC_STACK) {
         rets[nr].dst.v.stack.offset += (i32)part->src_offset;
         rets[nr].dst.type = pty;
       } else if (rets[nr].dst.kind == NATIVE_LOC_ADDR) {
         rets[nr].dst.v.addr.offset += (i32)part->src_offset;
         rets[nr].dst.type = pty;
       }
       rets[nr].mem = aa_mem_for_type(t, pty, part->size);
       nr++;
     }
     plan->nrets = nr;
   } else if (abi && abi->ret.kind == ABI_ARG_IGNORE) {
     plan->nrets = 0;
   } else if (!abi && desc->nresults) {
     rets[0].src = native_loc_reg(desc->results[0].type, NATIVE_REG_INT, 0);
     rets[0].dst = desc->results[0];
     rets[0].mem = aa_mem_for_type(t, desc->results[0].type, 0);
     plan->nrets = 1;
   }
 }
 
 static void aa_ret(NativeTarget* t);
 
 static void aa_emit_tail_site(NativeTarget* t, NativeLoc callee) {
   AANativeTarget* a = aa_of(t);
   if (a->frame.known_frame) {
     /* Frame is final: emit the tail epilogue (callee restores + frame restore +
      * branch) directly, exactly the words aa_apply_patches would patch in but
      * without the reserved NOP padding. */
     AAFrameLayout L = aa_build_layout(a->frame.cum_off, a->frame.max_outgoing,
                                       a->top_home_bytes);
     u32 words[AA_TAIL_WORDS];
     u32 n = 0;
     aa_words_callee_restores(a, words, AA_TAIL_WORDS, &n);
     aa_words_restore_frame(a, words, AA_TAIL_WORDS, &n, &L);
     if (n >= AA_TAIL_WORDS) aa_panic(a, "tail epilogue too large");
     for (u32 i = 0; i < n; ++i) aa_emit32(t->mc, words[i]);
     if (callee.kind == NATIVE_LOC_REG) {
       aa_emit32(t->mc, aa64_br(loc_reg(callee)));
     } else if (callee.kind == NATIVE_LOC_GLOBAL) {
       u32 pos = mc_pos(t->mc);
       aa_emit32(t->mc, aa64_b(0));
       mc_emit_reloc_at(t->mc, t->mc->section_id, pos, R_AARCH64_JUMP26,
                        callee.v.global.sym, callee.v.global.addend, 0, 0);
     } else {
       aa_panic(a, "unsupported tail target");
     }
     return;
   }
   /* Single-pass: reserve a worst-case region and record a patch; the callee
    * restores and frame restore depend on the not-yet-final frame layout. */
   AAPatch* p = aa_patch_alloc(a);
   p->kind = AA_PATCH_TAIL;
   p->pos = mc_pos(t->mc);
   p->u.callee = callee;
   for (u32 i = 0; i < AA_TAIL_WORDS; ++i) aa_emit32(t->mc, 0xd503201fu);
   if (callee.kind == NATIVE_LOC_GLOBAL) {
     mc_emit_reloc_at(t->mc, t->mc->section_id,
                      p->pos + (AA_TAIL_WORDS - 1u) * 4u, R_AARCH64_JUMP26,
                      callee.v.global.sym, callee.v.global.addend, 0, 0);
   }
 }
 
 static void aa_emit_call(NativeTarget* t, const NativeCallPhase* plan) {
   int is_tail = (plan->flags & CG_CALL_TAIL) != 0;
   if (is_tail) {
     if (plan->callee.kind != NATIVE_LOC_GLOBAL &&
         plan->callee.kind != NATIVE_LOC_REG)
       aa_panic(aa_of(t), "unsupported tail target");
     aa_emit_tail_site(t, plan->callee);
     return;
   }
   if (plan->callee.kind == NATIVE_LOC_GLOBAL) {
     aa_emit32(t->mc, aa64_bl(0));
     mc_emit_reloc_at(t->mc, t->mc->section_id, mc_pos(t->mc) - 4u,
                      R_AARCH64_CALL26, plan->callee.v.global.sym,
                      plan->callee.v.global.addend, 0, 0);
     return;
   }
   if (plan->callee.kind == NATIVE_LOC_REG) {
     aa_emit32(t->mc, aa64_blr(loc_reg(plan->callee)));
     return;
   }
   aa_panic(aa_of(t), "unsupported call target");
 }
 
 static void aa_marshal_ret(NativeTarget* t, const CGFuncDesc* fd,
                            const NativeLoc* value,
                            NativeCallPhaseRet** out_rets, u32* out_nrets) {
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, fd->fn_type);
   NativeCallPhaseRet* rets = NULL;
   u32 nr = 0;
   if (value && abi && abi->ret.kind == ABI_ARG_INDIRECT) {
     AANativeTarget* a = aa_of(t);
     /* Hold the sret destination pointer in x8, not AA_TMP1: aa_copy_bytes
      * materializes out-of-range source/dest frame offsets into AA_TMP1, which
      * would clobber the destination base mid-copy (only triggered once a frame
      * is large enough that the source offset escapes stur's signed-9 range). */
     NativeLoc dstp =
         native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT, AA_X8);
     NativeLoc saved = native_loc_stack(dstp.type, a->sret_ptr_slot, 0);
     NativeAddr dst_addr, src_addr;
     AggregateAccess access;
     aa_load_part(t, dstp, saved, 0, 8);
     memset(&dst_addr, 0, sizeof dst_addr);
     dst_addr.base_kind = NATIVE_ADDR_BASE_REG;
     dst_addr.base.reg = AA_X8;
     dst_addr.base_type = value->type;
     src_addr = aa_storage_addr(a, *value, 0);
     src_addr.base_type = value->type;
     memset(&access, 0, sizeof access);
     access.type = value->type;
     access.size = (u32)cg_type_size(t->c, value->type);
     access.align = type_align32(t, value->type);
     aa_copy_bytes(t, dst_addr, src_addr, access);
     *out_rets = NULL;
     *out_nrets = 0;
     return;
   }
   if (value && abi && abi->ret.kind == ABI_ARG_DIRECT) {
     u32 ni = 0, nf = 0;
     int exact_parts = 0;
     for (u32 p = 0; p < abi->ret.nparts; ++p) {
       const ABIArgPart* part = &abi->ret.parts[p];
       if (part->cls == ABI_CLASS_INT && part->size != 1u &&
           part->size != 2u && part->size != 4u && part->size != 8u) {
         exact_parts = 1;
         break;
       }
     }
 
     /* The generic return write phase materializes a memory source using the
      * scalar carrier type, so an I64 carrier for a 3/5/6/7-byte aggregate part
      * becomes an eight-byte load before NativeCallPhaseRet.mem can narrow the
      * final move. Once one exact-width part is present, marshal every return
      * part here in ABI order. aa_load_part retains the exact byte count and
      * uses only backend-private temporaries, so already-filled x0/x1 or v0..v3
      * return registers remain live while later parts are loaded. */
     if (exact_parts) {
       for (u32 p = 0; p < abi->ret.nparts; ++p) {
         const ABIArgPart* part = &abi->ret.parts[p];
         NativeAllocClass cls =
             part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
         KitCgTypeId pty = aa_part_scalar_type(part);
         Reg rreg = cls == NATIVE_REG_FP ? nf++ : ni++;
         NativeLoc dst = native_loc_reg(pty, cls, rreg);
         aa_load_part(t, dst, *value, part->src_offset, part->size);
       }
       *out_rets = NULL;
       *out_nrets = 0;
       return;
     }
 
     rets = arena_zarray(t->c->tu, NativeCallPhaseRet, abi->ret.nparts);
     for (u32 p = 0; p < abi->ret.nparts; ++p) {
       const ABIArgPart* part = &abi->ret.parts[p];
       NativeAllocClass cls =
           part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
       KitCgTypeId pty = aa_part_scalar_type(part);
       rets[nr].src = *value;
       if (rets[nr].src.kind == NATIVE_LOC_FRAME)
         rets[nr].src =
             native_loc_stack(pty, value->v.frame, (i32)part->src_offset);
       else if (rets[nr].src.kind == NATIVE_LOC_STACK) {
         rets[nr].src.v.stack.offset += (i32)part->src_offset;
         rets[nr].src.type = pty;
       } else if (rets[nr].src.kind == NATIVE_LOC_ADDR) {
         rets[nr].src.v.addr.offset += (i32)part->src_offset;
         rets[nr].src.type = pty;
       }
       rets[nr].dst =
           native_loc_reg(pty, cls, cls == NATIVE_REG_FP ? nf++ : ni++);
       rets[nr].mem = aa_mem_for_type(t, pty, part->size);
       nr++;
     }
   } else if (value) {
     rets = arena_zarray(t->c->tu, NativeCallPhaseRet, 1);
     rets[0].src = *value;
     rets[0].dst = native_loc_reg(value->type, NATIVE_REG_INT, 0);
     rets[0].mem = aa_mem_for_type(t, value->type, 0);
     nr = 1;
   }
   *out_rets = rets;
   *out_nrets = nr;
 }
 
 static void aa_ret(NativeTarget* t) {
   AANativeTarget* a = aa_of(t);
   aa_jump(t, a->epilogue_label);
 }
 
 static u32 aa_bit_storage_reg_bits(u32 storage_bytes) {
   return storage_bytes == 8u ? 64u : 32u;
 }
 
 static void aa_lsl_imm(NativeTarget* t, u32 sf, u32 rd, u32 rn, u32 sh) {
   u32 bits = sf ? 64u : 32u;
   if (!sh) {
     if (rd != rn) aa_emit32(t->mc, aa64_mov_reg(sf, rd, rn));
     return;
   }
   aa_emit32(t->mc, aa_ubfm(sf, rd, rn, bits - sh, bits - 1u - sh));
 }
 
 static void aa_lsr_imm(NativeTarget* t, u32 sf, u32 rd, u32 rn, u32 sh) {
   if (!sh) {
     if (rd != rn) aa_emit32(t->mc, aa64_mov_reg(sf, rd, rn));
     return;
   }
   aa_emit32(t->mc, aa_ubfm(sf, rd, rn, sh, sf ? 63u : 31u));
 }
 
 static void aa_asr_imm(NativeTarget* t, u32 sf, u32 rd, u32 rn, u32 sh) {
   if (!sh) {
     if (rd != rn) aa_emit32(t->mc, aa64_mov_reg(sf, rd, rn));
     return;
   }
   aa_emit32(t->mc, aa_sbfm(sf, rd, rn, sh, sf ? 63u : 31u));
 }
 
 static void aa_bitfield_load(NativeTarget* t, NativeLoc dst, NativeAddr addr,
                              BitFieldAccess bf) {
   u32 storage = bf.storage.size ? bf.storage.size : 4u;
   u32 bits = aa_bit_storage_reg_bits(storage);
   u32 width = bf.bit_width ? bf.bit_width : 1u;
   u32 sf = bits == 64u;
   NativeAddr saddr = aa_addr_plus(addr, bf.storage_offset);
   NativeLoc tmp = dst;
   tmp.type = bf.storage.type ? bf.storage.type : dst.type;
   aa_load_native(t, tmp, saddr, bf.storage);
   aa_lsl_imm(t, sf, loc_reg(dst), loc_reg(dst),
              bits - (u32)bf.bit_offset - width);
   if (bf.signed_)
     aa_asr_imm(t, sf, loc_reg(dst), loc_reg(dst), bits - width);
   else
     aa_lsr_imm(t, sf, loc_reg(dst), loc_reg(dst), bits - width);
 }
 
 static void aa_bitfield_store(NativeTarget* t, NativeAddr addr, NativeLoc src,
                               BitFieldAccess bf) {
   u32 storage = bf.storage.size ? bf.storage.size : 4u;
   u32 bits = aa_bit_storage_reg_bits(storage);
   u32 width = bf.bit_width ? bf.bit_width : 1u;
   u32 sf = bits == 64u;
   u64 ones = width >= 64u ? ~(u64)0 : ((1ull << width) - 1ull);
   u64 field_mask = ones << bf.bit_offset;
   NativeAddr saddr = aa_addr_plus(addr, bf.storage_offset);
   NativeLoc word =
       aa_tmp_loc(bf.storage.type ? bf.storage.type : src.type, AA_TMP0);
   aa_load_native(t, word, saddr, bf.storage);
   aa_emit_load_imm(t->mc, sf, AA_TMP1, (i64)~field_mask);
   aa_emit32(t->mc, aa64_and(sf, AA_TMP0, AA_TMP0, AA_TMP1));
   aa_emit32(t->mc, aa_ubfm(sf, AA_TMP1, loc_reg(src), 0, width - 1u));
   aa_lsl_imm(t, sf, AA_TMP1, AA_TMP1, bf.bit_offset);
   aa_emit32(t->mc, aa64_orr(sf, AA_TMP0, AA_TMP0, AA_TMP1));
   aa_store_native(t, saddr, word, bf.storage);
 }
 
 static void aa_trap(NativeTarget* t);
 
 static int aa_order_acquire(KitCgMemOrder order) {
   return order == KIT_CG_MO_CONSUME || order == KIT_CG_MO_ACQUIRE ||
          order == KIT_CG_MO_ACQ_REL || order == KIT_CG_MO_SEQ_CST;
 }
 
 static int aa_order_release(KitCgMemOrder order) {
   return order == KIT_CG_MO_RELEASE || order == KIT_CG_MO_ACQ_REL ||
          order == KIT_CG_MO_SEQ_CST;
 }
 
 static NativeLoc aa_i64_reg_loc(u32 reg) {
   return native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT, reg);
 }
 
 static void aa_atomic_addr_reg(NativeTarget* t, NativeAddr addr, u32 reg) {
   NativeLoc dst = aa_i64_reg_loc(reg);
   t->load_addr(t, dst, addr);
 }
 
 static u32 aa_saved_tmp_pick(u32 a, u32 b, u32 c) {
   static const u32 regs[] = {11u, 12u, 13u, 14u, 15u};
   for (u32 i = 0; i < sizeof regs / sizeof regs[0]; ++i) {
     if (regs[i] != a && regs[i] != b && regs[i] != c) return regs[i];
   }
   return 15u;
 }
 
 static void aa_saved_tmp_spill(AANativeTarget* a, u32 reg) {
   NativeFrameSlotDesc sd;
   NativeAddr addr;
   MemAccess mem;
   memset(&sd, 0, sizeof sd);
   if (a->saved_tmp_slot == NATIVE_FRAME_SLOT_NONE) {
     sd.type = builtin_id(KIT_CG_BUILTIN_I64);
     sd.size = 8;
     sd.align = 8;
     sd.kind = NATIVE_FRAME_SLOT_SPILL;
     a->saved_tmp_slot = a->base.frame_slot(&a->base, &sd);
   }
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = a->saved_tmp_slot;
   addr.base_type = builtin_id(KIT_CG_BUILTIN_I64);
   mem = aa_mem_for_type(&a->base, addr.base_type, 8);
   aa_store_native(&a->base, addr, aa_i64_reg_loc(reg), mem);
 }
 
 static void aa_saved_tmp_restore(AANativeTarget* a, u32 reg) {
   NativeAddr addr;
   MemAccess mem;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = a->saved_tmp_slot;
   addr.base_type = builtin_id(KIT_CG_BUILTIN_I64);
   mem = aa_mem_for_type(&a->base, addr.base_type, 8);
   aa_load_native(&a->base, aa_i64_reg_loc(reg), addr, mem);
 }
 
 static void aa_atomic_load(NativeTarget* t, NativeLoc dst, NativeAddr addr,
                            MemAccess mem, KitCgMemOrder order) {
   u32 base = AA_TMP0;
   u32 sz = size_idx(mem.size ? mem.size : loc_size32(t, dst));
   aa_atomic_addr_reg(t, addr, base);
   aa_emit32(t->mc, aa_order_acquire(order)
                        ? aa_ldar(sz, loc_reg(dst), base)
                        : aa_ldr_uimm(sz, loc_reg(dst), base, 0));
   if (order == KIT_CG_MO_SEQ_CST)
     aa_emit32(t->mc, aa64_dmb(AA64_BARRIER_OPT_ISH));
 }
 
 static void aa_atomic_store(NativeTarget* t, NativeAddr addr, NativeLoc src,
                             MemAccess mem, KitCgMemOrder order) {
   u32 base = AA_TMP0;
   u32 sz = size_idx(mem.size ? mem.size : loc_size32(t, src));
   if (order == KIT_CG_MO_SEQ_CST)
     aa_emit32(t->mc, aa64_dmb(AA64_BARRIER_OPT_ISH));
   aa_atomic_addr_reg(t, addr, base);
   aa_emit32(t->mc, aa_order_release(order)
                        ? aa_stlr(sz, loc_reg(src), base)
                        : aa_str_uimm(sz, loc_reg(src), base, 0));
   if (order == KIT_CG_MO_SEQ_CST)
     aa_emit32(t->mc, aa64_dmb(AA64_BARRIER_OPT_ISH));
 }
 
 static void aa_atomic_rmw(NativeTarget* t, KitCgAtomicOp op, NativeLoc dst,
                           NativeAddr addr, NativeLoc val, MemAccess mem,
                           KitCgMemOrder order) {
   AANativeTarget* a = aa_of(t);
   u32 base = AA_TMP0;
   u32 next_reg = AA_TMP1;
   u32 status = aa_saved_tmp_pick(loc_reg(dst), loc_reg(val), base);
   NativeLoc next = aa_tmp_loc(dst.type, next_reg);
   MCLabel retry = mc_label_new(t->mc);
   u32 sz = size_idx(mem.size ? mem.size : loc_size32(t, dst));
   if (order == KIT_CG_MO_SEQ_CST)
     aa_emit32(t->mc, aa64_dmb(AA64_BARRIER_OPT_ISH));
   aa_saved_tmp_spill(a, status);
   aa_atomic_addr_reg(t, addr, base);
   mc_label_place(t->mc, retry);
   aa_emit32(t->mc, aa_order_acquire(order) ? aa_ldaxr(sz, loc_reg(dst), base)
                                            : aa_ldxr(sz, loc_reg(dst), base));
   switch (op) {
     case KIT_CG_ATOMIC_XCHG:
       aa_move(t, next, val);
       break;
     case KIT_CG_ATOMIC_ADD:
       aa_binop(t, BO_IADD, next, dst, val);
       break;
     case KIT_CG_ATOMIC_SUB:
       aa_binop(t, BO_ISUB, next, dst, val);
       break;
     case KIT_CG_ATOMIC_AND:
       aa_binop(t, BO_AND, next, dst, val);
       break;
     case KIT_CG_ATOMIC_OR:
       aa_binop(t, BO_OR, next, dst, val);
       break;
     case KIT_CG_ATOMIC_XOR:
       aa_binop(t, BO_XOR, next, dst, val);
       break;
     case KIT_CG_ATOMIC_NAND:
       aa_binop(t, BO_AND, next, dst, val);
       aa_unop(t, UO_BNOT, next, next);
       break;
     default:
       aa_panic(a, "unsupported atomic rmw op");
   }
   aa_emit32(t->mc, aa_order_release(order)
                        ? aa_stlxr(sz, status, next_reg, base)
                        : aa_stxr(sz, status, next_reg, base));
   aa_emit32(t->mc, aa64_cbnz_imm(0, status, 0));
   mc_emit_label_ref(t->mc, retry, R_AARCH64_CONDBR19, 4, 0);
   aa_saved_tmp_restore(a, status);
   if (order == KIT_CG_MO_SEQ_CST)
     aa_emit32(t->mc, aa64_dmb(AA64_BARRIER_OPT_ISH));
 }
 
 static void aa_atomic_cas(NativeTarget* t, NativeLoc prior, NativeLoc ok,
                           NativeAddr addr, NativeLoc expected,
                           NativeLoc desired, MemAccess mem,
                           KitCgMemOrder success, KitCgMemOrder failure) {
   u32 base = AA_TMP0;
   u32 status = AA_TMP1;
   u32 sz = size_idx(mem.size ? mem.size : loc_size32(t, prior));
   u32 sf = sz == 3u;
   int acquire = aa_order_acquire(success) || aa_order_acquire(failure);
   int release = aa_order_release(success);
   MCLabel retry = mc_label_new(t->mc);
   MCLabel fail = mc_label_new(t->mc);
   MCLabel done = mc_label_new(t->mc);
   if (success == KIT_CG_MO_SEQ_CST || failure == KIT_CG_MO_SEQ_CST)
     aa_emit32(t->mc, aa64_dmb(AA64_BARRIER_OPT_ISH));
   aa_atomic_addr_reg(t, addr, base);
   mc_label_place(t->mc, retry);
   aa_emit32(t->mc, acquire ? aa_ldaxr(sz, loc_reg(prior), base)
                            : aa_ldxr(sz, loc_reg(prior), base));
   aa_emit32(t->mc, aa_subs_reg(sf, AA64_ZR, loc_reg(prior), loc_reg(expected)));
   aa_emit32(t->mc, aa64_brcond_pack((AA64BrCond){.cond = cmp_cond(CMP_NE)}));
   mc_emit_label_ref(t->mc, fail, R_AARCH64_CONDBR19, 4, 0);
   aa_emit32(t->mc, release ? aa_stlxr(sz, status, loc_reg(desired), base)
                            : aa_stxr(sz, status, loc_reg(desired), base));
   aa_emit32(t->mc, aa64_cbnz_imm(0, status, 0));
   mc_emit_label_ref(t->mc, retry, R_AARCH64_CONDBR19, 4, 0);
   aa_emit_load_imm(t->mc, loc_is_64(t, ok), loc_reg(ok), 1);
   aa_jump(t, done);
   mc_label_place(t->mc, fail);
   aa_emit32(t->mc, aa64_clrex(AA64_BARRIER_OPT_SY));
   aa_emit_load_imm(t->mc, loc_is_64(t, ok), loc_reg(ok), 0);
   mc_label_place(t->mc, done);
   if (success == KIT_CG_MO_SEQ_CST || failure == KIT_CG_MO_SEQ_CST)
     aa_emit32(t->mc, aa64_dmb(AA64_BARRIER_OPT_ISH));
 }
 
 static void aa_fence(NativeTarget* t, KitCgMemOrder order) {
   if (order != KIT_CG_MO_RELAXED)
     aa_emit32(t->mc, aa64_dmb(AA64_BARRIER_OPT_ISH));
 }
 
 /* Map a KitCgBarrierScope (passed as an immediate arg to DMB/DSB) onto an
  * AArch64 barrier domain option. Defaults to full-system (SY) when the scope
  * is absent or unrecognized. */
 static u32 aa_barrier_opt(const NativeLoc* args, u32 narg) {
   if (narg < 1u || args[0].kind != NATIVE_LOC_IMM) return AA64_BARRIER_OPT_SY;
   switch ((KitCgBarrierScope)args[0].v.imm) {
     case KIT_CG_BARRIER_FULL:
       return AA64_BARRIER_OPT_SY;
     case KIT_CG_BARRIER_INNER:
       return AA64_BARRIER_OPT_ISH;
     case KIT_CG_BARRIER_INNER_STORE:
       return AA64_BARRIER_OPT_ISHST;
     case KIT_CG_BARRIER_OUTER:
       return AA64_BARRIER_OPT_OSH;
     case KIT_CG_BARRIER_OUTER_STORE:
       return AA64_BARRIER_OPT_OSHST;
     case KIT_CG_BARRIER_NON_SHARE:
       return AA64_BARRIER_OPT_NSH;
   }
   return AA64_BARRIER_OPT_SY;
 }
 
 static void aa_intrinsic(NativeTarget* t, IntrinKind kind,
                          const NativeLoc* dsts, u32 ndst, const NativeLoc* args,
                          u32 narg) {
   AggregateAccess access;
   NativeAddr dst_addr;
   NativeAddr src_addr;
   memset(&access, 0, sizeof access);
   memset(&dst_addr, 0, sizeof dst_addr);
   memset(&src_addr, 0, sizeof src_addr);
   switch (kind) {
     case INTRIN_NONE:
       if (ndst == 1u && narg == 3u && native_loc_is_fp(dsts[0])) {
         u32 d = type_size32(t, dsts[0].type) == 8u;
         aa_emit32(t->mc, aa_fp_bin(0x000800u, d, loc_reg(dsts[0]),
                                    loc_reg(args[0]), loc_reg(args[1])));
         aa_emit32(t->mc, aa_fp_bin(0x002800u, d, loc_reg(dsts[0]),
                                    loc_reg(dsts[0]), loc_reg(args[2])));
         return;
       }
       break;
     case INTRIN_CLZ:
       if (ndst == 1u && narg == 1u) {
         aa_emit32(t->mc, aa_clz(loc_is_64(t, args[0]), loc_reg(dsts[0]),
                                 loc_reg(args[0])));
         return;
       }
       break;
     case INTRIN_CTZ:
       if (ndst == 1u && narg == 1u) {
         u32 sf = loc_is_64(t, args[0]);
         aa_emit32(t->mc, aa_rbit(sf, loc_reg(dsts[0]), loc_reg(args[0])));
         aa_emit32(t->mc, aa_clz(sf, loc_reg(dsts[0]), loc_reg(dsts[0])));
         return;
       }
       break;
     case INTRIN_POPCOUNT:
       if (ndst == 1u && narg == 1u) {
         u32 sf = loc_is_64(t, args[0]);
         u32 rd = loc_reg(dsts[0]);
         u32 rn = loc_reg(args[0]);
         MCLabel loop = mc_label_new(t->mc);
         MCLabel done = mc_label_new(t->mc);
         aa_emit_load_imm(t->mc, sf, rd, 0);
         aa_emit32(t->mc, aa64_mov_reg(sf, AA_TMP0, rn));
         mc_label_place(t->mc, loop);
         aa_emit32(t->mc, aa64_cbz(sf, AA_TMP0, 0));
         mc_emit_label_ref(t->mc, done, R_AARCH64_CONDBR19, 4, 0);
         aa_emit_load_imm(t->mc, sf, AA_TMP1, 1);
         aa_emit32(t->mc, aa64_and(sf, AA_TMP1, AA_TMP0, AA_TMP1));
         aa_emit32(t->mc, aa64_add(sf, rd, rd, AA_TMP1));
         aa_emit_load_imm(t->mc, sf, AA_TMP1, 1);
         aa_emit32(t->mc, aa64_lsrv(sf, AA_TMP0, AA_TMP0, AA_TMP1));
         aa_jump(t, loop);
         mc_label_place(t->mc, done);
         return;
       }
       break;
     case INTRIN_BSWAP:
       if (ndst == 1u && narg == 1u) {
         u32 width = abi_cg_sizeof(t->c->abi, dsts[0].type);
         switch (width) {
           case 2: {
             u32 sf = 0;
             aa_emit32(t->mc, aa_rev(sf, loc_reg(dsts[0]), loc_reg(args[0])));
             aa_emit_load_imm(t->mc, 0, AA_TMP0, 16);
             aa_emit32(t->mc, aa64_lsrv(0, loc_reg(dsts[0]), loc_reg(dsts[0]),
                                        AA_TMP0));
             return;
           }
           case 4: {
             u32 sf = 0;
             aa_emit32(t->mc, aa_rev(sf, loc_reg(dsts[0]), loc_reg(args[0])));
             return;
           }
           case 8: {
             u32 sf = 1;
             aa_emit32(t->mc, aa_rev(sf, loc_reg(dsts[0]), loc_reg(args[0])));
             return;
           }
           default:
             break;
         }
       }
       break;
     case INTRIN_SADD_OVERFLOW:
     case INTRIN_UADD_OVERFLOW:
     case INTRIN_SSUB_OVERFLOW:
     case INTRIN_USUB_OVERFLOW:
       if (ndst == 2u && narg == 2u) {
         u32 sf = loc_is_64(t, dsts[0]);
         u32 rd = loc_reg(dsts[0]);
         if (kind == INTRIN_SADD_OVERFLOW || kind == INTRIN_UADD_OVERFLOW)
           aa_emit32(t->mc,
                     aa64_addsubsr_pack((AA64AddSubSR){.sf = sf,
                                                       .op = 0,
                                                       .S = 1,
                                                       .Rm = loc_reg(args[1]),
                                                       .Rn = loc_reg(args[0]),
                                                       .Rd = rd}));
         else
           aa_emit32(t->mc,
                     aa64_addsubsr_pack((AA64AddSubSR){.sf = sf,
                                                       .op = 1,
                                                       .S = 1,
                                                       .Rm = loc_reg(args[1]),
                                                       .Rn = loc_reg(args[0]),
                                                       .Rd = rd}));
         aa_emit32(t->mc,
                   aa_cset(loc_is_64(t, dsts[1]), loc_reg(dsts[1]),
                           (kind == INTRIN_SADD_OVERFLOW ||
                            kind == INTRIN_SSUB_OVERFLOW)
                               ? 0x6u
                               : (kind == INTRIN_UADD_OVERFLOW ? 0x2u : 0x3u)));
         return;
       }
       break;
     case INTRIN_SMUL_OVERFLOW:
     case INTRIN_UMUL_OVERFLOW:
       if (ndst == 2u && narg == 2u) {
         u32 sf = loc_is_64(t, dsts[0]);
         if (sf) {
           if (kind == INTRIN_SMUL_OVERFLOW) {
             aa_emit32(t->mc,
                       aa_smulh(AA_TMP0, loc_reg(args[0]), loc_reg(args[1])));
             aa_emit32(t->mc, aa64_mul(1, loc_reg(dsts[0]), loc_reg(args[0]),
                                       loc_reg(args[1])));
             aa_emit32(t->mc, aa_sbfm(1, AA_TMP1, loc_reg(dsts[0]), 63, 63));
             aa_emit32(t->mc, aa_subs_reg(1, AA64_ZR, AA_TMP0, AA_TMP1));
             aa_emit32(t->mc, aa_cset(0, loc_reg(dsts[1]), cmp_cond(CMP_NE)));
           } else {
             aa_emit32(t->mc,
                       aa_umulh(AA_TMP0, loc_reg(args[0]), loc_reg(args[1])));
             aa_emit32(t->mc, aa64_mul(1, loc_reg(dsts[0]), loc_reg(args[0]),
                                       loc_reg(args[1])));
             aa_emit32(t->mc, aa_subs_reg(1, AA64_ZR, AA_TMP0, AA64_ZR));
             aa_emit32(t->mc, aa_cset(0, loc_reg(dsts[1]), cmp_cond(CMP_NE)));
           }
         } else if (kind == INTRIN_SMUL_OVERFLOW) {
           aa_emit32(t->mc, aa_smaddl(AA_TMP0, loc_reg(args[0]),
                                      loc_reg(args[1]), AA64_ZR));
           aa_emit32(t->mc, aa64_mov_reg(0, loc_reg(dsts[0]), AA_TMP0));
           aa_emit32(t->mc, aa_sbfm(1, AA_TMP1, loc_reg(dsts[0]), 0, 31));
           aa_emit32(t->mc, aa_subs_reg(1, AA64_ZR, AA_TMP0, AA_TMP1));
           aa_emit32(t->mc, aa_cset(0, loc_reg(dsts[1]), cmp_cond(CMP_NE)));
         } else {
           aa_emit32(t->mc, aa_umaddl(AA_TMP0, loc_reg(args[0]),
                                      loc_reg(args[1]), AA64_ZR));
           aa_emit32(t->mc, aa64_mov_reg(0, loc_reg(dsts[0]), AA_TMP0));
           aa_emit_load_imm(t->mc, 1, AA_TMP1, 32);
           aa_emit32(t->mc, aa64_lsrv(1, AA_TMP1, AA_TMP0, AA_TMP1));
           aa_emit32(t->mc, aa_subs_reg(1, AA64_ZR, AA_TMP1, AA64_ZR));
           aa_emit32(t->mc, aa_cset(0, loc_reg(dsts[1]), cmp_cond(CMP_NE)));
         }
         return;
       }
       break;
     case INTRIN_SMUL_HIGH:
     case INTRIN_UMUL_HIGH:
       if (ndst == 1u && narg == 2u) {
         u32 sf = loc_is_64(t, dsts[0]);
         u32 rd = loc_reg(dsts[0]);
         u32 rn = loc_reg(args[0]);
         u32 rm = loc_reg(args[1]);
         if (sf) {
           aa_emit32(t->mc, kind == INTRIN_SMUL_HIGH
                                ? aa_smulh(rd, rn, rm)
                                : aa_umulh(rd, rn, rm));
         } else {
           aa_emit32(t->mc, kind == INTRIN_SMUL_HIGH
                                ? aa_smaddl(AA_TMP0, rn, rm, AA64_ZR)
                                : aa_umaddl(AA_TMP0, rn, rm, AA64_ZR));
           if (kind == INTRIN_SMUL_HIGH)
             aa_asr_imm(t, 1, rd, AA_TMP0, 32);
           else
             aa_lsr_imm(t, 1, rd, AA_TMP0, 32);
         }
         return;
       }
       break;
     case INTRIN_MEMMOVE: {
       MCLabel forward = mc_label_new(t->mc);
       MCLabel done = mc_label_new(t->mc);
       if (narg != 3u || args[0].kind != NATIVE_LOC_REG ||
           args[1].kind != NATIVE_LOC_REG || args[2].kind != NATIVE_LOC_IMM)
         aa_panic(aa_of(t), "unsupported memory intrinsic operands");
       if (args[2].v.imm < 0 || args[2].v.imm > 0xffffffffll)
         aa_panic(aa_of(t), "unsupported memory intrinsic size");
       access.size = (u32)args[2].v.imm;
       access.align = 1u;
       dst_addr.base_kind = NATIVE_ADDR_BASE_REG;
       dst_addr.base.reg = args[0].v.reg;
       src_addr.base_kind = NATIVE_ADDR_BASE_REG;
       src_addr.base.reg = args[1].v.reg;
       aa_emit32(t->mc, aa_subs_reg(1, AA64_ZR, args[0].v.reg, args[1].v.reg));
       aa_emit32(t->mc,
                 aa64_brcond_pack((AA64BrCond){.cond = cmp_cond(CMP_LT_U)}));
       mc_emit_label_ref(t->mc, forward, R_AARCH64_CONDBR19, 4, 0);
       aa_copy_bytes_dir(t, dst_addr, src_addr, access, 1);
       aa_jump(t, done);
       mc_label_place(t->mc, forward);
       aa_copy_bytes_dir(t, dst_addr, src_addr, access, 0);
       mc_label_place(t->mc, done);
       return;
     }
     case INTRIN_EXPECT:
     case INTRIN_ASSUME_ALIGNED:
       if (ndst == 1u && narg >= 1u) {
         if (args[0].kind == NATIVE_LOC_IMM)
           aa_load_imm_native(t, dsts[0], args[0].v.imm);
         else
           aa_move(t, dsts[0], args[0]);
       }
       return;
     case INTRIN_PREFETCH:
       if (narg >= 1u && args[0].kind == NATIVE_LOC_REG) {
         /* GCC locality 0..3 maps to streaming L1, then L3/L2/L1 keep. PST
          * differs from PLD by bit 3 of prfop; both are baseline A64 hints. */
         static const u8 op[4] = {1u, 4u, 2u, 0u};
         u32 rw = 0u;
         u32 locality = 3u;
         if (narg >= 2u && args[1].kind == NATIVE_LOC_IMM)
           rw = (u32)args[1].v.imm;
         if (narg >= 3u && args[2].kind == NATIVE_LOC_IMM)
           locality = (u32)args[2].v.imm;
         if (locality > 3u) locality = 3u;
         aa_emit32(t->mc, aa_prfm(op[locality] | (rw ? 8u : 0u),
                                 loc_reg(args[0])));
       }
       return;
     case INTRIN_TRAP:
       aa_trap(t);
       return;
     case INTRIN_SYSCALL:
       if (ndst == 1u && narg >= 1u && narg <= 7u) {
         static const u32 syscall_regs[7] = {AA_X8, 0u, 1u, 2u, 3u, 4u, 5u};
         AAArgMove moves[7];
         for (u32 i = 0; i < narg; ++i) {
           AAArgMove* m = &moves[i];
           memset(m, 0, sizeof *m);
           m->dst =
               native_loc_reg(dsts[0].type, NATIVE_REG_INT, syscall_regs[i]);
           m->src = args[i];
           m->size = t->c->target.ptr_size;
         }
         aa_emit_reg_arg_moves(t, moves, narg);
         aa_emit32(t->mc, aa64_svc(0));
         aa_move(t, dsts[0], native_loc_reg(dsts[0].type, NATIVE_REG_INT, 0));
       }
       return;
     case INTRIN_READCYCLECOUNTER:
       /* MRS Xd, CNTVCT_EL0 (op0=3 op1=3 CRn=14 CRm=0 op2=2): the EL0-readable
        * virtual counter. Prefer it over PMCCNTR_EL0, which traps at EL0 unless
        * the PMU is explicitly enabled for userspace. */
       if (ndst == 1u)
         aa_emit32(t->mc, aa64_sysreg_move(1, 3, 3, 14, 0, 2, loc_reg(dsts[0])));
       return;
     case INTRIN_CPU_NOP:
       aa_emit32(t->mc, aa64_hint(AA64_HINT_OP_NOP));
       return;
     case INTRIN_CPU_YIELD:
       aa_emit32(t->mc, aa64_hint(AA64_HINT_OP_YIELD));
       return;
     case INTRIN_WFI:
       aa_emit32(t->mc, aa64_hint(AA64_HINT_OP_WFI));
       return;
     case INTRIN_WFE:
       aa_emit32(t->mc, aa64_hint(AA64_HINT_OP_WFE));
       return;
     case INTRIN_SEV:
       aa_emit32(t->mc, aa64_hint(AA64_HINT_OP_SEV));
       return;
     case INTRIN_ISB:
       aa_emit32(t->mc, aa64_isb(AA64_BARRIER_OPT_SY));
       return;
     case INTRIN_DMB:
       aa_emit32(t->mc, aa64_dmb(aa_barrier_opt(args, narg)));
       return;
     case INTRIN_DSB:
       aa_emit32(t->mc, aa64_dsb(aa_barrier_opt(args, narg)));
       return;
     case INTRIN_IRQ_SAVE:
       /* Read the interrupt-mask state, then mask D,A,I,F. */
       if (ndst == 1u) {
         aa_emit32(t->mc, aa64_mrs_daif(loc_reg(dsts[0])));
         aa_emit32(t->mc, aa64_msr_daifset(AA64_DAIF_ALL));
       }
       return;
     case INTRIN_IRQ_RESTORE:
       if (narg == 1u) aa_emit32(t->mc, aa64_msr_daif(loc_reg(args[0])));
       return;
     case INTRIN_IRQ_DISABLE:
       aa_emit32(t->mc, aa64_msr_daifset(AA64_DAIF_ALL));
       return;
     case INTRIN_IRQ_ENABLE:
       aa_emit32(t->mc, aa64_msr_daifclr(AA64_DAIF_ALL));
       return;
     case INTRIN_FRAME_ADDRESS:
     case INTRIN_RETURN_ADDRESS:
       /* Walk the AAPCS64 frame-record chain. Every kit prologue stores
        * {x29, x30} and anchors x29 at the record: [x29] = caller's x29,
        * [x29 + 8] = saved x30 (this frame's return address). The level is a
        * compile-time constant, so the walk unrolls to `level` dependent loads.
        */
       if (ndst == 1u) {
         u32 level = (narg >= 1u && args[0].kind == NATIVE_LOC_IMM)
                         ? (u32)args[0].v.imm
                         : 0u;
         u32 rd = loc_reg(dsts[0]);
         aa_emit32(t->mc, aa64_mov_reg(1, rd, AA_FP));
         for (u32 i = 0; i < level; ++i)
           aa_emit32(t->mc, aa64_ldr64_uimm12(rd, rd, 0)); /* rd = *(rd) */
         if (kind == INTRIN_RETURN_ADDRESS)
           aa_emit32(t->mc, aa64_ldr64_uimm12(rd, rd, 1)); /* rd = *(rd + 8) */
       }
       return;
     default:
       aa_panic(aa_of(t), "unsupported compiler intrinsic");
   }
 }
 
 static void aa_trap(NativeTarget* t) { aa_emit32(t->mc, aa64_brk(0)); }
 
 /* file_scope_asm + finalize are shared (cg/native_asm.h). */
 
 static int aa_machine_op_clobbers(NativeTarget* t, const NativeMachineOp* op,
                                   u32 mask[NATIVE_REG_CLASS_COUNT]) {
   mask[0] = mask[1] = mask[2] = 0;
   switch ((NativeMachineOpKind)op->kind) {
     case NATIVE_MOP_TLS_ADDR:
       /* ELF Local-Exec materializes the address using only the destination
        * register. Mach-O TLV calls its resolver through private x16/x17/LR;
        * only x0 can hold an optimizer-owned value at this boundary. */
       if (!obj_format_tls_via_descriptor(t->c)) return 0;
       mask[NATIVE_REG_INT] = 1u << 0;
       return 1;
     case NATIVE_MOP_INTRINSIC:
       if ((IntrinKind)op->intrin != INTRIN_SYSCALL) return 0;
       mask[NATIVE_REG_INT] = (1u << 0) | (1u << 1) | (1u << 2) | (1u << 3) |
                              (1u << 4) | (1u << 5) | (1u << AA_X8);
       return 1;
     case NATIVE_MOP_BINOP:
     case NATIVE_MOP_BITFIELD_LOAD:
     case NATIVE_MOP_BITFIELD_STORE:
     case NATIVE_MOP_VA_START:
     case NATIVE_MOP_VA_ARG:
     case NATIVE_MOP_ATOMIC_CAS:
     case NATIVE_MOP_ATOMIC_RMW:
       return 0;
     case NATIVE_MOP_COUNT:
       break;
   }
   aa_panic(aa_of(t), "invalid machine-effect operation");
   return 0;
 }
 
 static void aa_set_loc(NativeTarget* t, SrcLoc loc) {
   AANativeTarget* a = aa_of(t);
   a->loc = loc;
   if (t->mc) mc_set_loc(t->mc, loc);
 }
 
 static void aa_bind_native_param(NativeTarget* t, const CGParamDesc* p,
                                  NativeLoc dst);
 
 /* ABI argument/return registers (x0..x7) come FIRST so the -O0 value cache
  * prefers them: an argument expression then computes straight into its ABI arg
  * register and the per-call parallel-copy (native_arg_shuffle) collapses to
  * no-ops — the tcc get_reg(0..N) trick (mirrors RV's NDT a0..a7 bank and
  * x64's leading rsi/rdi). They are caller-saved, so the deferred prologue stays
  * callee-save-free. Incoming params are spilled to frame homes at entry before
  * any body op (aa_bind_native_param), so caching in x0..x7 cannot clobber a
  * live parameter. Next come the remaining caller-saved temps x8/x12..x15;
  * callee-saved x19..x28 are appended and only chosen under register pressure,
  * after which the prologue saves/restores them. x9/x10/x11 are the explicit O1
  * operand-temp bank (also the O0 scratch policy) and stay out of this pool. */
 static const Reg aa_ndt_int_allocable[] = {
     0u,  1u,  2u,  3u,  4u,  5u,  6u,  7u,  8u,  12u, 13u, 14u,
     15u, 19u, 20u, 21u, 22u, 23u, 24u, 25u, 26u, 27u, 28u};
 /* Three O1 operand temporaries, not two: a 3-operand op (for example
  * `binop dst, a, b` or `store [base+index], value`) can require three distinct
  * materializations in one instruction scope. Location MIR leaves spilled
  * operands in their frame homes until this point; the scope leases x9/x10/x11
  * without inventing persistent MIR register values. All three are caller-saved
  * and reserved out of the allocable set above. The O0 direct target happens to
  * use the same bank through its independent `scratch` policy. */
 static const Reg aa_int_scratch[] = {9u, 10u, 11u};
 static const Reg aa_int_asm_temps[] = {AA_TMP0, AA_TMP1};
 static const Reg aa_direct_asm_int[] = {0u, 1u, 2u,  3u,  4u,  5u,  6u,
                                         7u, 8u, 11u, 12u, 13u, 14u, 15u};
 /* ABI fp argument/return registers (v0..v7) come FIRST (same rationale as the
  * int pool: fp args land in place, the shuffle is a no-op). Then the caller-
  * saved fp temps v18/v19, v22..v31 — the rest of the -O0 value cache under
  * ndt_caller_saved_only. v16 is a backend-private/arg-shuffle cycle temporary;
  * v20/v21 are the O1 operand-temp bank and the O0 per-op scratch policy, so all
  * three stay out of the pool. Callee-saved v8..v15 (AAPCS64) are appended
  * like the int set; the caller-saved-only NDT never selects them. */
 static const Reg aa_ndt_fp_allocable[] = {
     0u,  1u,  2u,  3u,  4u,  5u,  6u, 7u, 18u, 19u, 22u, 23u, 24u, 25u,
     26u, 27u, 28u, 29u, 30u, 31u, 8u, 9u, 10u, 11u, 12u, 13u, 14u, 15u};
 static const Reg aa_fp_scratch[] = {20u, 21u};
 static const Reg aa_direct_asm_fp[] = {
     0u,  1u,  2u,  3u,  4u,  5u,  6u,  7u,  16u, 17u, 18u,
     19u, 22u, 23u, 24u, 25u, 26u, 27u, 28u, 29u, 30u, 31u};
 
 #define AA_PHYS_INT_ALLOC(r)                                \
   {.reg = (r),                                              \
    .cls = NATIVE_REG_INT,                                   \
    .abi_index = 0xffu,                                      \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLER_SAVED, \
    .spill_cost = 1u,                                        \
    .copy_cost = 1u}
 #define AA_PHYS_INT_CALLER(r)        \
   {.reg = (r),                       \
    .cls = NATIVE_REG_INT,            \
    .abi_index = 0xffu,               \
    .flags = NATIVE_REG_CALLER_SAVED, \
    .spill_cost = 1u,                 \
    .copy_cost = 1u}
 #define AA_PHYS_INT_ARG(r)                             \
   {.reg = (r),                                         \
    .cls = NATIVE_REG_INT,                              \
    .abi_index = (r),                                   \
    .flags = NATIVE_REG_CALLER_SAVED | NATIVE_REG_ARG | \
             ((r) < 2u ? NATIVE_REG_RET : 0),           \
    .spill_cost = 1u,                                   \
    .copy_cost = 1u}
 #define AA_PHYS_INT_CALLEE(r)                               \
   {.reg = (r),                                              \
    .cls = NATIVE_REG_INT,                                   \
    .abi_index = 0xffu,                                      \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLEE_SAVED, \
    .spill_cost = 4u,                                        \
    .copy_cost = 1u}
 #define AA_PHYS_INT_RESERVED_ROLE(r, role) \
   {.reg = (r),                              \
    .cls = NATIVE_REG_INT,                   \
    .abi_index = 0xffu,                      \
    .flags = NATIVE_REG_RESERVED | (role),   \
    .spill_cost = 0u,                        \
    .copy_cost = 0u}
 #define AA_PHYS_INT_RESERVED(r) \
   AA_PHYS_INT_RESERVED_ROLE((r), NATIVE_REG_NONE)
 #define AA_PHYS_INT_RESERVED_CALLER(r) \
   AA_PHYS_INT_RESERVED_ROLE((r), NATIVE_REG_CALLER_SAVED)
 
 static const NativePhysRegInfo aa_int_phys[] = {
     AA_PHYS_INT_ARG(0u),       AA_PHYS_INT_ARG(1u),
     AA_PHYS_INT_ARG(2u),       AA_PHYS_INT_ARG(3u),
     AA_PHYS_INT_ARG(4u),       AA_PHYS_INT_ARG(5u),
     AA_PHYS_INT_ARG(6u),       AA_PHYS_INT_ARG(7u),
     AA_PHYS_INT_ALLOC(8u),     AA_PHYS_INT_RESERVED_CALLER(9u),
     AA_PHYS_INT_RESERVED_CALLER(10u), AA_PHYS_INT_RESERVED_CALLER(11u),
     AA_PHYS_INT_ALLOC(12u),    AA_PHYS_INT_ALLOC(13u),
     AA_PHYS_INT_ALLOC(14u),    AA_PHYS_INT_ALLOC(15u),
     AA_PHYS_INT_RESERVED_CALLER(16u), AA_PHYS_INT_RESERVED_CALLER(17u),
     AA_PHYS_INT_RESERVED_CALLER(18u), AA_PHYS_INT_CALLEE(19u),
     AA_PHYS_INT_CALLEE(20u),   AA_PHYS_INT_CALLEE(21u),
     AA_PHYS_INT_CALLEE(22u),   AA_PHYS_INT_CALLEE(23u),
     AA_PHYS_INT_CALLEE(24u),   AA_PHYS_INT_CALLEE(25u),
     AA_PHYS_INT_CALLEE(26u),   AA_PHYS_INT_CALLEE(27u),
     AA_PHYS_INT_CALLEE(28u),   AA_PHYS_INT_RESERVED(29u),
     AA_PHYS_INT_RESERVED(30u), AA_PHYS_INT_RESERVED(31u),
 };
 
 #define AA_PHYS_FP_ALLOC(r)                                 \
   {.reg = (r),                                              \
    .cls = NATIVE_REG_FP,                                    \
    .abi_index = 0xffu,                                      \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLER_SAVED, \
    .spill_cost = 1u,                                        \
    .copy_cost = 1u}
 #define AA_PHYS_FP_CALLER(r)         \
   {.reg = (r),                       \
    .cls = NATIVE_REG_FP,             \
    .abi_index = 0xffu,               \
    .flags = NATIVE_REG_CALLER_SAVED, \
    .spill_cost = 1u,                 \
    .copy_cost = 1u}
 #define AA_PHYS_FP_ARG(r)                              \
   {.reg = (r),                                         \
    .cls = NATIVE_REG_FP,                               \
    .abi_index = (r),                                   \
    .flags = NATIVE_REG_CALLER_SAVED | NATIVE_REG_ARG | \
             ((r) < 4u ? NATIVE_REG_RET : 0),           \
    .spill_cost = 1u,                                   \
    .copy_cost = 1u}
 #define AA_PHYS_FP_CALLEE(r)                                \
   {.reg = (r),                                              \
    .cls = NATIVE_REG_FP,                                    \
    .abi_index = 0xffu,                                      \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLEE_SAVED, \
    .spill_cost = 4u,                                        \
    .copy_cost = 1u}
 #define AA_PHYS_FP_RESERVED(r)                   \
   {.reg = (r),                                   \
    .cls = NATIVE_REG_FP,                         \
    .abi_index = 0xffu,                           \
    .flags = NATIVE_REG_RESERVED |                \
             NATIVE_REG_CALLER_SAVED,             \
    .spill_cost = 0u,                             \
    .copy_cost = 0u}
 
 static const NativePhysRegInfo aa_fp_phys[] = {
     AA_PHYS_FP_ARG(0u),       AA_PHYS_FP_ARG(1u),     AA_PHYS_FP_ARG(2u),
     AA_PHYS_FP_ARG(3u),       AA_PHYS_FP_ARG(4u),     AA_PHYS_FP_ARG(5u),
     AA_PHYS_FP_ARG(6u),       AA_PHYS_FP_ARG(7u),     AA_PHYS_FP_CALLEE(8u),
     AA_PHYS_FP_CALLEE(9u),    AA_PHYS_FP_CALLEE(10u), AA_PHYS_FP_CALLEE(11u),
     AA_PHYS_FP_CALLEE(12u),   AA_PHYS_FP_CALLEE(13u), AA_PHYS_FP_CALLEE(14u),
     AA_PHYS_FP_CALLEE(15u),   AA_PHYS_FP_CALLER(16u), AA_PHYS_FP_CALLER(17u),
     AA_PHYS_FP_ALLOC(18u),    AA_PHYS_FP_ALLOC(19u),  AA_PHYS_FP_RESERVED(20u),
     AA_PHYS_FP_RESERVED(21u), AA_PHYS_FP_CALLER(22u), AA_PHYS_FP_CALLER(23u),
     AA_PHYS_FP_CALLER(24u),   AA_PHYS_FP_CALLER(25u), AA_PHYS_FP_CALLER(26u),
     AA_PHYS_FP_CALLER(27u),   AA_PHYS_FP_CALLER(28u), AA_PHYS_FP_CALLER(29u),
     AA_PHYS_FP_CALLER(30u),   AA_PHYS_FP_CALLER(31u),
 };
 
 static const NativeAllocClassInfo aa_classes[] = {
     {.cls = NATIVE_REG_INT,
      .ndt_allocable = aa_ndt_int_allocable,
      .ndt_allocable_count =
          sizeof aa_ndt_int_allocable / sizeof aa_ndt_int_allocable[0],
      .scratch = aa_int_scratch,
      .nscratch = sizeof aa_int_scratch / sizeof aa_int_scratch[0],
      .emit_temps = aa_int_scratch,
      .nemit_temps = sizeof aa_int_scratch / sizeof aa_int_scratch[0],
      .asm_temps = aa_int_asm_temps,
      .nasm_temps = sizeof aa_int_asm_temps / sizeof aa_int_asm_temps[0],
      .direct_asm_allocable = aa_direct_asm_int,
      .ndirect_asm_allocable =
          sizeof aa_direct_asm_int / sizeof aa_direct_asm_int[0],
      .emit_cache_mask = (1u << 9u) | (1u << 10u) | (1u << 11u),
      .phys = aa_int_phys,
      .nphys = sizeof aa_int_phys / sizeof aa_int_phys[0]},
     {.cls = NATIVE_REG_FP,
      .ndt_allocable = aa_ndt_fp_allocable,
      .ndt_allocable_count =
          sizeof aa_ndt_fp_allocable / sizeof aa_ndt_fp_allocable[0],
      .scratch = aa_fp_scratch,
      .nscratch = sizeof aa_fp_scratch / sizeof aa_fp_scratch[0],
      .emit_temps = aa_fp_scratch,
      .nemit_temps = sizeof aa_fp_scratch / sizeof aa_fp_scratch[0],
      .direct_asm_allocable = aa_direct_asm_fp,
      .ndirect_asm_allocable =
          sizeof aa_direct_asm_fp / sizeof aa_direct_asm_fp[0],
      .emit_cache_mask = (1u << 20u) | (1u << 21u),
      .phys = aa_fp_phys,
      .nphys = sizeof aa_fp_phys / sizeof aa_fp_phys[0]},
 };
 
 /* Resolve a register name ("x8", "v3", ...) to its (class, Reg). Powers the
  * optimizer's inline-asm clobber masks and explicit hard-register operands
  * ("{x8}" from a GNU local register variable). x0..x30 are DWARF 0..30; the
  * SIMD/FP bank v0..v31 is DWARF 64..95. Returns non-zero for a non-register
  * name (cc/memory/unknown), which the caller skips. */
 static int aa_resolve_name(const NativeRegInfo* ri, Slice name, Reg* out,
                            NativeAllocClass* cls_out) {
   char buf[16];
   uint32_t dwarf;
   (void)ri;
   if (!name.s || !name.len || name.len >= sizeof buf) return 1;
   memcpy(buf, name.s, name.len);
   buf[name.len] = '\0';
   if (aa64_register_index(buf, &dwarf) != 0) return 1;
   if (dwarf <= 30u) {
     *cls_out = NATIVE_REG_INT;
     *out = (Reg)dwarf;
     return 0;
   }
   if (dwarf >= 64u && dwarf <= 95u) {
     *cls_out = NATIVE_REG_FP;
     *out = (Reg)(dwarf - 64u);
     return 0;
   }
   return 1;
 }
 
 static int aa_asm_operand_reg_ok(const NativeRegInfo* ri, NativeAllocClass cls,
                                  Reg reg) {
   (void)ri;
   if (cls == NATIVE_REG_INT) {
     if (reg <= 8u) return 1;
     if (reg >= 12u && reg <= 15u) return 1;
     if (reg >= 19u && reg <= 28u) return 1;
     return 0;
   }
   if (cls == NATIVE_REG_FP) {
     if (reg <= 19u) return 1;
     if (reg >= 22u && reg <= 31u) return 1;
   }
   return 0;
 }
 
 static int aa_asm_constraint_reg(const NativeRegInfo* ri, const char* body,
                                  NativeAllocClass* cls_out, Reg* fixed_out,
                                  u32* allowed_mask_out) {
   (void)ri;
   if (!body || !body[0] || body[1]) return 0;
   if (fixed_out) *fixed_out = REG_NONE;
   if (allowed_mask_out) *allowed_mask_out = 0;
   switch (body[0]) {
     case 'r':
       if (cls_out) *cls_out = NATIVE_REG_INT;
       return 1;
     case 'w':
       if (cls_out) *cls_out = NATIVE_REG_FP;
       return 1;
     case 'x':
       if (cls_out) *cls_out = NATIVE_REG_FP;
       if (allowed_mask_out) *allowed_mask_out = 0x0000ffffu; /* v0..v15 */
       return 1;
     case 'y':
       if (cls_out) *cls_out = NATIVE_REG_FP;
       if (allowed_mask_out) *allowed_mask_out = 0x000000ffu; /* v0..v7 */
       return 1;
     default:
       return 0;
   }
 }
 
 static const NativeRegInfo aa_reg_info = {
     .classes = aa_classes,
     .nclasses = sizeof aa_classes / sizeof aa_classes[0],
     /* x8..x15 give NDT a rich caller-saved scratch pool, so the -O0 path never
      * needs a callee-saved register — enabling the tcc-style deferred-`sub`
      * prologue (see aa_func_begin / AA_NDT_SUB_WORDS). */
     .ndt_caller_saved_only = 1u,
     /* x0 is clobbered only by calls (div/mul take explicit operands), so a
      * scalar call result can stay cached in it — see L4 Phase 1 in nd_call. */
     .ndt_result_reg_stable = 1u,
     /* aa_emit_mem emits ldrsb/ldrsh (opc=10, X form) for an MF_SEXT_LOAD narrow
      * integer load, filling the whole register — so the -O0 NDT may drop the
      * following CV_SEXT (Lever 4). */
     .load_sext = 1u,
     .resolve_name = aa_resolve_name,
     .asm_operand_reg_ok = aa_asm_operand_reg_ok,
     .asm_constraint_reg = aa_asm_constraint_reg,
 };
 
 static void aa_va_start_native(NativeTarget* t, NativeLoc ap_ptr);
 static void aa_va_arg_native(NativeTarget* t, NativeLoc dst, NativeLoc ap_ptr,
                              KitCgTypeId type);
 static void aa_va_end_native(NativeTarget* t, NativeLoc ap_ptr);
 static void aa_va_copy_native(NativeTarget* t, NativeLoc dst_ap_ptr,
                               NativeLoc src_ap_ptr);
 static void aa_asm_block_native(NativeTarget* t, const char* tmpl,
                                 const AsmConstraint* outs, u32 nout,
                                 NativeLoc* out_locs, const AsmConstraint* ins,
                                 u32 nin, const NativeLoc* in_locs,
                                 const Sym* clobbers, u32 nclob);
 
 NativeTarget* aa64_native_target_new(Compiler* c, ObjBuilder* obj,
                                      MCEmitter* mc) {
   AANativeTarget* a = arena_znew(c->tu, AANativeTarget);
   NativeTarget* t;
   if (!a) return NULL;
   t = &a->base;
   t->c = c;
   t->obj = obj;
   t->mc = mc;
   native_frame_init(&a->frame, c);
   t->regs = &aa_reg_info;
   t->class_for_type = aa_class_for_type;
   t->imm_legal = aa_imm_legal;
   t->addr_legal = aa_addr_legal;
   t->can_fold_shift_into_alu = aa_can_fold_shift_into_alu;
   t->can_fold_extend_into_addr = aa_can_fold_extend_into_addr;
   t->machine_op_clobbers = aa_machine_op_clobbers;
   t->func_begin = aa_func_begin;
   t->func_begin_known_frame = aa_func_begin_known_frame;
   t->note_frame_state = aa_note_frame_state;
   t->reserve_callee_saves = aa_reserve_callee_saves;
   t->signature_stack_bytes = aa_signature_stack_bytes;
   t->call_stack_bytes = aa_call_stack_bytes;
   t->has_store_zero_reg = 1;
   t->store_zero_reg = 31u; /* wzr/xzr in the Rt position of a store */
   t->func_end = aa_func_end;
   t->frame_slot = aa_frame_slot;
   t->release_frame_slot = aa_release_frame_slot;
   t->frame_slot_debug_loc = aa_frame_slot_debug_loc;
   t->bind_param = aa_bind_native_param;
   t->label_new = aa_label_new;
   t->label_place = aa_label_place;
   t->jump = aa_jump;
   t->cmp_branch = aa_cmp_branch;
   t->indirect_branch = aa_indirect_branch;
   t->load_label_addr = aa_load_label_addr;
   t->move = aa_move;
   t->load_imm = aa_load_imm_native;
   t->load_const = aa_load_const;
   t->load_addr = aa_load_addr;
   t->load = aa_load_native;
   t->store = aa_store_native;
   t->tls_addr_of = aa_tls_addr_of;
   t->copy_bytes = aa_copy_bytes;
   t->set_bytes = aa_set_bytes;
   t->bitfield_load = aa_bitfield_load;
   t->bitfield_store = aa_bitfield_store;
   t->binop = aa_binop;
   t->unop = aa_unop;
   t->cmp = aa_cmp;
   t->convert = aa_convert;
   t->binop_rr = aa_binop_rr;
   t->move_rr = aa_move_rr;
   t->cmp_rr = aa_cmp_rr;
   t->convert_rr = aa_convert_rr;
   t->alloca_ = aa_alloca;
   t->spill = aa_spill;
   t->reload = aa_reload;
   t->marshal_call = aa_marshal_call;
   t->emit_call = aa_emit_call;
   t->marshal_ret = aa_marshal_ret;
   t->ret = aa_ret;
   t->atomic_load = aa_atomic_load;
   t->atomic_store = aa_atomic_store;
   t->atomic_rmw = aa_atomic_rmw;
   t->atomic_cas = aa_atomic_cas;
   t->fence = aa_fence;
   t->va_start_ = aa_va_start_native;
   t->va_arg_ = aa_va_arg_native;
   t->va_end_ = aa_va_end_native;
   t->va_copy_ = aa_va_copy_native;
   t->intrinsic = aa_intrinsic;
   t->asm_block = aa_asm_block_native;
   t->file_scope_asm = native_file_scope_asm;
   t->trap = aa_trap;
   t->set_loc = aa_set_loc;
   t->finalize = native_finalize;
   return t;
 }
 
 /* Place the incoming parameter into `dst`: a hard register (the common
  * register-allocated scalar case -> a single arg-reg move, or a stack load
  * straight into the register), a frame slot (address-taken / aggregate /
  * spilled), or nowhere (unused). Incoming arg registers are never allocable,
  * so a register dst never aliases an incoming arg register. */
 static void aa_bind_native_param(NativeTarget* t, const CGParamDesc* p,
                                  NativeLoc dst) {
   AANativeTarget* a = aa_of(t);
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, a->func->fn_type);
   const ABIArgInfo* ai =
       p->index < abi->nparams ? &abi->params[p->index] : NULL;
   int to_reg = dst.kind == NATIVE_LOC_REG;
   if (!ai || ai->kind == ABI_ARG_IGNORE) return;
   if (ai->kind == ABI_ARG_INDIRECT) {
     NativeAddr d_addr, from;
     AggregateAccess access;
     NativeLoc src =
         native_loc_reg(p->type, NATIVE_REG_INT,
                        a->next_param_int < 8u ? a->next_param_int++ : AA_TMP0);
     if (src.v.reg == AA_TMP0) {
       NativeAddr saddr;
       memset(&saddr, 0, sizeof saddr);
       saddr.base_kind = NATIVE_ADDR_BASE_REG;
       saddr.base.reg = AA_FP;
       saddr.offset = aa_fp_off_in_arg(a, a->next_param_stack);
       aa_emit_mem(a, 1, src, saddr, aa_mem_for_type(t, p->type, 8));
       a->next_param_stack += 8u;
     }
     if (dst.kind != NATIVE_LOC_FRAME)
       aa_panic(a, "indirect parameter requires a frame destination");
     memset(&d_addr, 0, sizeof d_addr);
     d_addr.base_kind = NATIVE_ADDR_BASE_FRAME;
     d_addr.base.frame = dst.v.frame;
     d_addr.base_type = p->type;
     memset(&from, 0, sizeof from);
     from.base_kind = NATIVE_ADDR_BASE_REG;
     from.base.reg = src.v.reg;
     from.base_type = p->type;
     memset(&access, 0, sizeof access);
     access.type = p->type;
     access.size = p->size ? p->size : (u32)cg_type_size(t->c, p->type);
     access.align = p->align ? p->align : type_align32(t, p->type);
     aa_copy_bytes(t, d_addr, from, access);
     return;
   }
   for (u32 i = 0; i < ai->nparts; ++i) {
     const ABIArgPart* part = &ai->parts[i];
     NativeAllocClass cls =
         part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
     int reg_dst = to_reg && (NativeAllocClass)dst.cls == cls;
     NativeLoc src;
     if (cls == NATIVE_REG_FP && a->next_param_fp < 8u) {
       src = native_loc_reg(p->type, cls, a->next_param_fp++);
     } else if (cls == NATIVE_REG_INT && a->next_param_int < 8u) {
       src = native_loc_reg(p->type, cls, a->next_param_int++);
     } else {
       /* Stack-passed part: load straight into the dst register when possible,
        * otherwise use a backend-private transfer register for the frame store. */
       Reg tmp =
           reg_dst ? (Reg)dst.v.reg : (cls == NATIVE_REG_FP ? 16u : AA_TMP0);
       NativeAddr saddr;
       src = native_loc_reg(p->type, cls, tmp);
       a->next_param_stack =
           align_up_u32(a->next_param_stack, aa_part_stack_align(abi, part));
       memset(&saddr, 0, sizeof saddr);
       saddr.base_kind = NATIVE_ADDR_BASE_REG;
       saddr.base.reg = AA_FP;
       saddr.base_type = p->type;
       saddr.offset = aa_fp_off_in_arg(a, a->next_param_stack);
       aa_emit_mem(a, 1, src, saddr, aa_mem_for_type(t, p->type, part->size));
       a->next_param_stack += aa_part_stack_size(abi, part);
     }
     if (dst.kind == NATIVE_LOC_NONE) {
       /* Unused parameter: only the ABI cursor advances. */
     } else if (to_reg) {
       NativeLoc d = native_loc_reg(dst.type ? dst.type : p->type,
                                    (NativeAllocClass)dst.cls, (Reg)dst.v.reg);
       if (!(src.kind == NATIVE_LOC_REG && src.v.reg == d.v.reg &&
             (NativeAllocClass)src.cls == (NativeAllocClass)d.cls))
         aa_move(t, d, src);
     } else {
       aa_store_part(
           t, native_loc_stack(p->type, dst.v.frame, (i32)part->src_offset), src,
           0, part->size);
     }
   }
   a->incoming_stack_size = align_up_u32(a->next_param_stack, 16u);
 }
 
 static void aa_bind_param(NativeDirectTarget* d, const CGParamDesc* p,
                           CGLocal local, NativeDirectLocal* l) {
   NativeLoc dst;
   (void)local;
   memset(&dst, 0, sizeof dst);
   dst.kind = NATIVE_LOC_FRAME;
   dst.type = p->type;
   dst.v.frame = l->home;
   aa_bind_native_param(d->native, p, dst);
 }
 
 static const char* aa_no_tail(NativeDirectTarget* d, const CGCallDesc* call) {
   NativeCallDesc nd;
   u32 stack;
   native_direct_project_tail_call_desc(d, call, &nd);
   stack = aa_call_stack_size(d->native, &nd);
   if (stack > aa_of(d->native)->incoming_stack_size)
     return "aarch64 tail call: stack argument area too small";
   return NULL;
 }
 
 static NativeAddr aa_direct_addr(NativeDirectTarget* d, Operand op) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   switch ((OpKind)op.kind) {
     case OPK_LOCAL:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = d->locals[op.v.local - 1u].home;
       addr.base_type = op.type;
       return addr;
     case OPK_INDIRECT:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME_VALUE;
       addr.base.frame = d->locals[op.v.ind.base - 1u].home;
       addr.cls = d->locals[op.v.ind.base - 1u].cls;
       addr.base_type = d->locals[op.v.ind.base - 1u].type;
       addr.offset = op.v.ind.ofs;
       return addr;
     default:
       compiler_panic(d->base.c, d->loc,
                      "aarch64 native target: operand is not addressable");
   }
 }
 
 static NativeAddr aa_direct_materialize_addr(NativeDirectTarget* d,
                                              Operand op) {
   NativeAddr addr = aa_direct_addr(d, op);
   if (addr.base_kind == NATIVE_ADDR_BASE_FRAME_VALUE) {
     NativeLoc base = native_loc_reg(addr.base_type, NATIVE_REG_INT, AA_TMP1);
     NativeAddr load;
     memset(&load, 0, sizeof load);
     load.base_kind = NATIVE_ADDR_BASE_FRAME;
     load.base.frame = addr.base.frame;
     load.base_type = addr.base_type;
     aa_emit_mem(aa_of(d->native), 1, base, load,
                 aa_mem_for_type(d->native, addr.base_type, 8));
     addr.base_kind = NATIVE_ADDR_BASE_REG;
     addr.base.reg = AA_TMP1;
   }
   return addr;
 }
 
 static NativeAddr aa_direct_pointer_addr(NativeDirectTarget* d, Operand op) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   if (op.kind == OPK_LOCAL) {
     NativeLoc base = native_loc_reg(op.type, NATIVE_REG_INT, AA_TMP1);
     NativeAddr load;
     memset(&load, 0, sizeof load);
     load.base_kind = NATIVE_ADDR_BASE_FRAME;
     load.base.frame = d->locals[op.v.local - 1u].home;
     load.base_type = op.type;
     aa_emit_mem(aa_of(d->native), 1, base, load,
                 aa_mem_for_type(d->native, op.type, 8));
     addr.base_kind = NATIVE_ADDR_BASE_REG;
     addr.base.reg = AA_TMP1;
     addr.base_type = op.type;
     return addr;
   }
   return aa_direct_materialize_addr(d, op);
 }
 
 static NativeAddr aa_reg_addr(KitCgTypeId type, u32 reg, i32 offset) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_REG;
   addr.base.reg = reg;
   addr.base_type = type;
   addr.offset = offset;
   return addr;
 }
 
 static void aa_load_ap_addr(NativeDirectTarget* d, Operand ap_addr,
                             u32 dst_reg) {
   NativeLoc dst =
       native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT, dst_reg);
   NativeAddr ap = aa_direct_pointer_addr(d, ap_addr);
   d->native->load_addr(d->native, dst, ap);
 }
 
 /* The va cores use caller-provided operand registers plus backend-private
  * x16/x17 and v16. Direct wrappers choose x9/x10 from the O0 scratch policy;
  * the optimized emitter supplies an instruction-scoped operand location. None
  * can alias a persistently allocated live value. */
 static u32 aa_va_base_reg(AANativeTarget* a, NativeAddr ap) {
   if (ap.base_kind != NATIVE_ADDR_BASE_REG)
     compiler_panic(a->base.c, a->func ? a->func->loc : (SrcLoc){0, 0, 0},
                    "aarch64 native target: va_list pointer not in register");
   return ap.base.reg;
 }
 
 /* va_list layout is queried from the ABI; the optimizer/direct callers pass the
  * va_list pointer opaquely. `ap` addresses the va_list object itself. */
 static void aa_va_start_core(AANativeTarget* a, NativeAddr ap) {
   NativeTarget* t = &a->base;
   const ABIFuncInfo* abi =
       a->func ? abi_cg_func_info(t->c->abi, a->func->fn_type) : NULL;
   ABIVaListInfo vai = abi_va_list_layout(t->c->abi);
   NativeLoc ptr =
       native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT, AA_TMP0);
   if (vai.kind == ABI_VA_LIST_POINTER) {
     if (a->top_home_bytes) {
       /* Windows: `va_list = &<first vararg>` inside the contiguous
        * [GP home area | incoming stack args] block. Named args consume the
        * leading slots; next_param_int (FP params remapped to GP included) plus
        * next_param_stack locate the first unnamed slot. Home slot
        * gp_reg_count coincides with incoming-arg byte 0, so a single formula
        * spans both regions. */
       i32 off =
           aa_fp_off_home_slot(a->next_param_int) + (i32)a->next_param_stack;
       aa_emit_add_imm(a, AA_TMP0, AA_FP, off);
       aa_emit_mem(a, 0, ptr, ap, aa_mem_for_type(t, ptr.type, 8));
       return;
     }
     /* `va_list = &<first vararg>`. Variadic stack args follow the fixed
      * incoming params in the same caller window. Apple ARM64 compact fixed
      * stack args may leave this cursor at +4, while the first variadic slot
      * starts at the next 8-byte boundary. */
     u32 stack = aa_vararg_stack_start(abi, a->next_param_stack);
     aa_emit_add_imm(a, AA_TMP0, AA_FP, aa_fp_off_in_arg(a, stack));
     aa_emit_mem(a, 0, ptr, ap, aa_mem_for_type(t, ptr.type, 8));
     return;
   }
   if (vai.kind == ABI_VA_LIST_AAPCS64) {
     KitCgTypeId i32_ty = builtin_id(KIT_CG_BUILTIN_I32);
     NativeLoc i32tmp = native_loc_reg(i32_ty, NATIVE_REG_INT, AA_TMP1);
     MemAccess ptr_mem = aa_mem_for_type(t, ptr.type, 8);
     MemAccess i32_mem = aa_mem_for_type(t, i32_ty, 4);
     AANativeSlot* gr = aa_slot(a, a->va_gr_slot);
     AANativeSlot* vr = aa_slot(a, a->va_vr_slot);
     u32 base = aa_va_base_reg(a, ap);
     u32 used_gr = a->next_param_int < vai.gp_reg_count ? a->next_param_int
                                                        : vai.gp_reg_count;
     u32 used_vr = a->next_param_fp < vai.fp_reg_count ? a->next_param_fp
                                                       : vai.fp_reg_count;
     /* __stack points at the incoming stack args, which sit above the saved
      * fp/lr pair — the same address bind_param uses (aa_fp_off_in_arg), not the
      * raw next_param_stack cursor. */
     aa_emit_add_imm(a, AA_TMP0, AA_FP,
                     aa_fp_off_in_arg(a, a->next_param_stack));
     aa_emit_mem(a, 0, ptr, aa_reg_addr(ptr.type, base, (i32)vai.stack_offset),
                 ptr_mem);
     aa_emit_add_imm(a, AA_TMP0, AA_FP,
                     aa_fp_off_slot(a, gr->off) +
                         (i32)(vai.gp_reg_count * vai.gp_slot_size));
     aa_emit_mem(a, 0, ptr, aa_reg_addr(ptr.type, base, (i32)vai.gr_top_offset),
                 ptr_mem);
     aa_emit_add_imm(a, AA_TMP0, AA_FP,
                     aa_fp_off_slot(a, vr->off) +
                         (i32)(vai.fp_reg_count * vai.fp_slot_size));
     aa_emit_mem(a, 0, ptr, aa_reg_addr(ptr.type, base, (i32)vai.vr_top_offset),
                 ptr_mem);
     aa_emit_load_imm(t->mc, 0, AA_TMP1,
                      -(i32)((vai.gp_reg_count - used_gr) * vai.gp_slot_size));
     aa_emit_mem(a, 0, i32tmp,
                 aa_reg_addr(i32_ty, base, (i32)vai.gr_offs_offset), i32_mem);
     aa_emit_load_imm(t->mc, 0, AA_TMP1,
                      -(i32)((vai.fp_reg_count - used_vr) * vai.fp_slot_size));
     aa_emit_mem(a, 0, i32tmp,
                 aa_reg_addr(i32_ty, base, (i32)vai.vr_offs_offset), i32_mem);
     return;
   }
   compiler_panic(t->c, a->func ? a->func->loc : (SrcLoc){0, 0, 0},
                  "aarch64 native target: unsupported va_list layout");
 }
 
 static void aa_va_arg_core(AANativeTarget* a, NativeLoc dst, NativeAddr ap,
                            KitCgTypeId type) {
   NativeTarget* t = &a->base;
   ABIVaListInfo vai = abi_va_list_layout(t->c->abi);
   NativeLoc cur =
       native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT, AA_TMP0);
   /* The fetched value is written directly into caller-provided `dst`, which is
    * distinct from the va_list base. Only TMP0/TMP1 are backend-private. */
   NativeLoc val = dst;
   NativeAddr src;
   MemAccess ptr_mem = aa_mem_for_type(t, cur.type, 8);
   MemAccess val_mem = aa_mem_for_type(t, type, type_size32(t, type));
   if (dst.kind != NATIVE_LOC_REG)
     compiler_panic(t->c, a->func ? a->func->loc : (SrcLoc){0, 0, 0},
                    "aarch64 native target: va_arg destination must be a "
                    "register");
   if (vai.kind == ABI_VA_LIST_POINTER) {
     aa_emit_mem(a, 1, cur, ap, ptr_mem);
     src = aa_reg_addr(type, AA_TMP0, 0);
     {
       const ABIFuncInfo* abi =
           a->func ? abi_cg_func_info(t->c->abi, a->func->fn_type) : NULL;
       ABIArgPart part;
       memset(&part, 0, sizeof part);
       part.cls = cg_type_is_float(t->c, type) ? ABI_CLASS_FP : ABI_CLASS_INT;
       part.size = type_size32(t, type);
       part.align = type_align32(t, type);
       aa_emit_add_imm(a, AA_TMP1, AA_TMP0,
                       (i32)aa_part_vararg_stack_size(abi, &part));
     }
     aa_emit_mem(a, 0, native_loc_reg(cur.type, NATIVE_REG_INT, AA_TMP1), ap,
                 ptr_mem);
     aa_emit_mem(a, 1, val, src, val_mem);
     return;
   }
   if (vai.kind == ABI_VA_LIST_AAPCS64) {
     KitCgTypeId i32_ty = builtin_id(KIT_CG_BUILTIN_I32);
     NativeLoc off = native_loc_reg(i32_ty, NATIVE_REG_INT, AA_TMP1);
     MemAccess i32_mem = aa_mem_for_type(t, i32_ty, 4);
     int is_fp = cg_type_is_float(t->c, type);
     u32 base = aa_va_base_reg(a, ap);
     u32 offs_field = is_fp ? vai.vr_offs_offset : vai.gr_offs_offset;
     u32 top_field = is_fp ? vai.vr_top_offset : vai.gr_top_offset;
     u32 slot_size = is_fp ? vai.fp_slot_size : vai.gp_slot_size;
     MCLabel stack_label = mc_label_new(t->mc);
     MCLabel done_label = mc_label_new(t->mc);
     aa_emit_mem(a, 1, off, aa_reg_addr(i32_ty, base, (i32)offs_field), i32_mem);
     aa_emit32(t->mc, aa64_subs_imm12(0, AA64_ZR, AA_TMP1, 0, 0));
     aa_emit32(t->mc,
               aa64_brcond_pack((AA64BrCond){.cond = cmp_cond(CMP_GE_S)}));
     mc_emit_label_ref(t->mc, stack_label, R_AARCH64_CONDBR19, 4, 0);
     aa_emit_mem(a, 1, cur, aa_reg_addr(cur.type, base, (i32)top_field),
                 ptr_mem);
     aa_emit32(t->mc, aa_sbfm(1, AA_TMP1, AA_TMP1, 0, 31));
     aa_emit32(t->mc, aa64_add(1, AA_TMP0, AA_TMP0, AA_TMP1));
     aa_emit_mem(a, 1, val, aa_reg_addr(type, AA_TMP0, 0), val_mem);
     aa_emit_add_imm(a, AA_TMP1, AA_TMP1, (i32)slot_size);
     aa_emit_mem(a, 0, off, aa_reg_addr(i32_ty, base, (i32)offs_field), i32_mem);
     aa_emit32(t->mc, aa64_b(0));
     mc_emit_label_ref(t->mc, done_label, R_AARCH64_JUMP26, 4, 0);
     mc_label_place(t->mc, stack_label);
     aa_emit_mem(a, 1, cur, aa_reg_addr(cur.type, base, (i32)vai.stack_offset),
                 ptr_mem);
     aa_emit_mem(a, 1, val, aa_reg_addr(type, AA_TMP0, 0), val_mem);
     aa_emit_add_imm(a, AA_TMP0, AA_TMP0, 8);
     aa_emit_mem(a, 0, cur, aa_reg_addr(cur.type, base, (i32)vai.stack_offset),
                 ptr_mem);
     mc_label_place(t->mc, done_label);
     return;
   }
   compiler_panic(t->c, a->func ? a->func->loc : (SrcLoc){0, 0, 0},
                  "aarch64 native target: unsupported va_list layout");
 }
 
 static void aa_va_copy_core(AANativeTarget* a, NativeAddr dst_ap,
                             NativeAddr src_ap) {
   NativeTarget* t = &a->base;
   ABIVaListInfo vai = abi_va_list_layout(t->c->abi);
   NativeLoc tmp =
       native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT, AA_TMP0);
   MemAccess mem = aa_mem_for_type(t, tmp.type, 8);
   if (vai.kind == ABI_VA_LIST_POINTER) {
     aa_emit_mem(a, 1, tmp, src_ap, mem);
     aa_emit_mem(a, 0, tmp, dst_ap, mem);
     return;
   }
   if (vai.kind == ABI_VA_LIST_AAPCS64) {
     u32 sb = aa_va_base_reg(a, src_ap);
     u32 db = aa_va_base_reg(a, dst_ap);
     for (u32 off = 0; off < vai.type.size; off += 8u) {
       aa_emit_mem(a, 1, tmp, aa_reg_addr(tmp.type, sb, (i32)off), mem);
       aa_emit_mem(a, 0, tmp, aa_reg_addr(tmp.type, db, (i32)off), mem);
     }
     return;
   }
   compiler_panic(t->c, a->func ? a->func->loc : (SrcLoc){0, 0, 0},
                  "aarch64 native target: unsupported va_list layout");
 }
 
 /* ---- Direct-path (NativeDirectTarget) wrappers: convert semantic operands to
  * NativeAddr/NativeLoc, then call the shared cores above. ---- */
 
 /* The cores reserve x16/x17 (TMP0/TMP1) as backend-private temporaries and
  * require the va_list base register(s) to be distinct. aa_direct_pointer_addr
  * returns the pointer in TMP1, so the direct wrappers first relocate it into
  * x9/x10 before calling the cores. */
 static NativeAddr aa_direct_va_base(NativeDirectTarget* d, Operand ap_addr,
                                     u32 reg) {
   aa_load_ap_addr(d, ap_addr, reg);
   return aa_reg_addr(builtin_id(KIT_CG_BUILTIN_I64), reg, 0);
 }
 
 static void aa_va_start_(NativeDirectTarget* d, Operand ap_addr) {
   aa_va_start_core(aa_of(d->native), aa_direct_va_base(d, ap_addr, 10u));
 }
 
 static void aa_va_arg_(NativeDirectTarget* d, Operand dst_op, Operand ap_addr,
                        KitCgTypeId type) {
   AANativeTarget* a = aa_of(d->native);
   int is_fp = cg_type_is_float(d->base.c, type);
   NativeLoc res = native_loc_reg(type, is_fp ? NATIVE_REG_FP : NATIVE_REG_INT,
                                  is_fp ? 16u : 9u);
   MemAccess val_mem =
       aa_mem_for_type(d->native, type, type_size32(d->native, type));
   NativeAddr dst;
   aa_va_arg_core(a, res, aa_direct_va_base(d, ap_addr, 10u), type);
   dst = aa_direct_materialize_addr(d, dst_op);
   aa_emit_mem(a, 0, res, dst, val_mem);
 }
 
 static void aa_va_end_(NativeDirectTarget* d, Operand ap_addr) {
   (void)d;
   (void)ap_addr;
 }
 
 static void aa_va_copy_(NativeDirectTarget* d, Operand dst_ap_addr,
                         Operand src_ap_addr) {
   AANativeTarget* a = aa_of(d->native);
   NativeAddr src = aa_direct_va_base(d, src_ap_addr, 9u);
   NativeAddr dst = aa_direct_va_base(d, dst_ap_addr, 10u);
   aa_va_copy_core(a, dst, src);
 }
 
 /* ---- NativeTarget (optimizer) hooks: the optimizer passes the va_list
  * pointer as a materialized register; layout is resolved inside the cores. ----
  */
 
 static NativeAddr aa_va_addr_from_ptr(NativeLoc ap_ptr) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_REG;
   addr.cls = NATIVE_REG_INT;
   addr.base.reg = ap_ptr.v.reg;
   addr.base_type = ap_ptr.type;
   return addr;
 }
 
 static void aa_va_start_native(NativeTarget* t, NativeLoc ap_ptr) {
   aa_va_start_core(aa_of(t), aa_va_addr_from_ptr(ap_ptr));
 }
 
 static void aa_va_arg_native(NativeTarget* t, NativeLoc dst, NativeLoc ap_ptr,
                              KitCgTypeId type) {
   aa_va_arg_core(aa_of(t), dst, aa_va_addr_from_ptr(ap_ptr), type);
 }
 
 static void aa_va_end_native(NativeTarget* t, NativeLoc ap_ptr) {
   (void)t;
   (void)ap_ptr;
 }
 
 static void aa_va_copy_native(NativeTarget* t, NativeLoc dst_ap_ptr,
                               NativeLoc src_ap_ptr) {
   aa_va_copy_core(aa_of(t), aa_va_addr_from_ptr(dst_ap_ptr),
                   aa_va_addr_from_ptr(src_ap_ptr));
 }
 
 /* constraint_body / constraint_early / match_index are shared
  * (cg/native_asm.h). */
 
 _Noreturn static void aa_asm_panic_at(Compiler* c, SrcLoc loc,
                                       const char* msg) {
   compiler_panic(c, loc, "aarch64 inline asm: %s", msg);
 }
 
 _Noreturn static void aa_asm_panic(NativeDirectTarget* d, const char* msg) {
   aa_asm_panic_at(d->base.c, d->loc, msg);
 }
 
 AA_UNUSED_FN static void aa_asm_bound_reg(Operand* out, KitCgTypeId type,
                                           NativeAllocClass cls, Reg reg) {
   memset(out, 0, sizeof *out);
   out->kind = AA64_INLINE_OPK_REG;
   out->pad[0] =
       (cls == NATIVE_REG_FP) ? AA64_INLINE_OPCLS_FP : AA64_INLINE_OPCLS_INT;
   out->type = type;
   out->v.local = (CGLocal)reg;
 }
 
 AA_UNUSED_FN static void aa_asm_bound_mem(Operand* out, KitCgTypeId type,
                                           Reg base) {
   memset(out, 0, sizeof *out);
   out->kind = OPK_INDIRECT;
   out->type = type;
   out->v.ind.base = (CGLocal)base;
   out->v.ind.index = CG_LOCAL_NONE;
 }
 
 static int aa_asm_parse_reg_clobber(Compiler* c, SrcLoc loc, Sym name,
                                     NativeAllocClass* cls_out, Reg* reg_out) {
   Slice s = pool_slice(c->global, name);
   char buf[16];
   uint32_t dwarf;
   if (!s.s || !s.len) return 0;
   if (s.len == 2 && s.s[0] == 'c' && s.s[1] == 'c') return 0;
   if (s.len == 6 && memcmp(s.s, "memory", 6) == 0) return 0;
   if (s.len >= sizeof buf) aa_asm_panic_at(c, loc, "clobber name is too long");
   memcpy(buf, s.s, s.len);
   buf[s.len] = '\0';
   if (aa64_register_index(buf, &dwarf) != 0)
     aa_asm_panic_at(c, loc, "unknown clobber register");
   if (dwarf <= 30u) {
     *cls_out = NATIVE_REG_INT;
     *reg_out = (Reg)dwarf;
     return 1;
   }
   if (dwarf >= 64u && dwarf <= 95u) {
     *cls_out = NATIVE_REG_FP;
     *reg_out = (Reg)(dwarf - 64u);
     return 1;
   }
   aa_asm_panic_at(c, loc, "unsupported clobber register");
   return 0;
 }
 
 static void aa_asm_clobber_masks(Compiler* c, SrcLoc loc, const Sym* clobbers,
                                  u32 nclob, u32* int_mask, u32* fp_mask) {
   *int_mask = 0;
   *fp_mask = 0;
   for (u32 i = 0; i < nclob; ++i) {
     NativeAllocClass cls;
     Reg reg;
     if (!aa_asm_parse_reg_clobber(c, loc, clobbers[i], &cls, &reg)) continue;
     if (cls == NATIVE_REG_INT)
       *int_mask |= 1u << reg;
     else if (cls == NATIVE_REG_FP)
       *fp_mask |= 1u << reg;
   }
 }
 
 /* Pin resolution + panic is the shared native_asm_bind_direct_operands path. */
 
 AA_UNUSED_FN static void aa_direct_load_operand_to_reg(NativeDirectTarget* d,
                                                        Operand op,
                                                        NativeLoc dst) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   switch ((OpKind)op.kind) {
     case OPK_IMM:
       if ((NativeAllocClass)dst.cls != NATIVE_REG_INT)
         aa_asm_panic(d, "floating-point immediate asm input is unsupported");
       d->native->load_imm(d->native, dst, op.v.imm);
       return;
     case OPK_LOCAL:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = d->locals[op.v.local - 1u].home;
       addr.base_type = op.type;
       aa_emit_mem(aa_of(d->native), 1, dst, addr,
                   aa_mem_for_type(d->native, op.type, 0));
       return;
     case OPK_GLOBAL:
       addr.base_kind = NATIVE_ADDR_BASE_GLOBAL;
       addr.base.global.sym = op.v.global.sym;
       addr.base.global.addend = op.v.global.addend;
       addr.base_type = op.type;
       d->native->load_addr(d->native, dst, addr);
       return;
     case OPK_INDIRECT:
       addr = aa_direct_materialize_addr(d, op);
       aa_emit_mem(aa_of(d->native), 1, dst, addr,
                   aa_mem_for_type(d->native, op.type, 0));
       return;
   }
   aa_asm_panic(d, "unsupported asm input operand");
 }
 
 AA_UNUSED_FN static void aa_direct_load_address_to_reg(NativeDirectTarget* d,
                                                        Operand op,
                                                        NativeLoc dst) {
   NativeAddr addr = aa_direct_addr(d, op);
   d->native->load_addr(d->native, dst, addr);
 }
 
 AA_UNUSED_FN static void aa_direct_store_reg_to_operand(NativeDirectTarget* d,
                                                         Operand op,
                                                         NativeLoc src) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   if (op.kind == OPK_LOCAL) {
     addr.base_kind = NATIVE_ADDR_BASE_FRAME;
     addr.base.frame = d->locals[op.v.local - 1u].home;
     addr.base_type = op.type;
   } else {
     addr = aa_direct_materialize_addr(d, op);
   }
   aa_emit_mem(aa_of(d->native), 0, src, addr,
               aa_mem_for_type(d->native, op.type, 0));
 }
 
 typedef struct AAAsmSavedClobber {
   NativeFrameSlot slot;
   NativeAllocClass cls;
   Reg reg;
   KitCgTypeId type;
 } AAAsmSavedClobber;
 
 static void aa_asm_save_one(AANativeTarget* a, AAAsmSavedClobber* s) {
   NativeFrameSlotDesc desc;
   NativeAddr addr;
   NativeLoc reg;
   memset(&desc, 0, sizeof desc);
   desc.type = s->type;
   desc.size = 8;
   desc.align = 8;
   desc.kind = NATIVE_FRAME_SLOT_SAVE;
   s->slot = a->base.frame_slot(&a->base, &desc);
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = s->slot;
   addr.base_type = s->type;
   reg = native_loc_reg(s->type, s->cls, s->reg);
   aa_emit_mem(a, 0, reg, addr, aa_mem_for_type(&a->base, s->type, 8));
 }
 
 AA_UNUSED_FN static void aa_asm_restore_one(AANativeTarget* a,
                                             const AAAsmSavedClobber* s) {
   NativeAddr addr;
   NativeLoc reg = native_loc_reg(s->type, s->cls, s->reg);
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = s->slot;
   addr.base_type = s->type;
   aa_emit_mem(a, 1, reg, addr, aa_mem_for_type(&a->base, s->type, 8));
 }
 
 AA_UNUSED_FN static AAAsmSavedClobber* aa_asm_save_callee_clobbers(
     AANativeTarget* a, u32 int_mask, u32 fp_mask, u32* nsaved_out) {
   AAAsmSavedClobber* saved =
       arena_zarray(a->base.c->tu, AAAsmSavedClobber, 20u);
   u32 n = 0;
   KitCgTypeId i64 = builtin_id(KIT_CG_BUILTIN_I64);
   KitCgTypeId f64 = builtin_id(KIT_CG_BUILTIN_F64);
   for (Reg r = 19u; r <= 28u; ++r) {
     if ((int_mask & (1u << r)) == 0) continue;
     saved[n].cls = NATIVE_REG_INT;
     saved[n].reg = r;
     saved[n].type = i64;
     aa_asm_save_one(a, &saved[n++]);
   }
   for (Reg r = 8u; r <= 15u; ++r) {
     if ((fp_mask & (1u << r)) == 0) continue;
     saved[n].cls = NATIVE_REG_FP;
     saved[n].reg = r;
     saved[n].type = f64;
     aa_asm_save_one(a, &saved[n++]);
   }
   *nsaved_out = n;
   return saved;
 }
 
 /* Hook adapters that bridge the arch-typed save/restore + assembler entry to
  * the shared NativeAsmDirectHooks signatures (which speak NativeDirectTarget +
  * an opaque save record). */
 static void* aa_asm_hook_save_callee_clobbers(NativeDirectTarget* d,
                                               u32 int_mask, u32 fp_mask,
                                               u32* nsaved_out) {
   return aa_asm_save_callee_clobbers(aa_of(d->native), int_mask, fp_mask,
                                      nsaved_out);
 }
 static void aa_asm_hook_restore_one(NativeDirectTarget* d, void* saved,
                                     u32 idx) {
   aa_asm_restore_one(aa_of(d->native), &((AAAsmSavedClobber*)saved)[idx]);
 }
 static void aa_asm_hook_run_template(NativeDirectTarget* d, const char* tmpl,
                                      const AsmConstraint* outs, u32 nout,
                                      Operand* bound_outs,
                                      const AsmConstraint* ins, u32 nin,
                                      Operand* bound_ins, const Sym* clobbers,
                                      u32 nclob) {
   AA64Asm* a = aa64_asm_open(d->base.c);
   aa64_inline_bind(a, outs, nout, bound_outs, ins, nin, bound_ins, clobbers,
                    nclob);
   aa64_asm_run_template(a, d->native->mc, tmpl);
   aa64_asm_close(a);
 }
 
 static void aa_direct_asm_block(NativeDirectTarget* d, const char* tmpl,
                                 const AsmConstraint* outs, u32 nout,
                                 Operand* out_ops, const AsmConstraint* ins,
                                 u32 nin, const Operand* in_ops,
                                 const Sym* clobbers, u32 nclob,
                                 u32 clobber_abi_sets) {
   static const NativeAsmDirectHooks hooks = {
       .opk_reg = AA64_INLINE_OPK_REG,
       .opcls_fp = AA64_INLINE_OPCLS_FP,
       .panic = aa_asm_panic,
       .bound_reg = aa_asm_bound_reg,
       .bound_mem = aa_asm_bound_mem,
       .clobber_masks = aa_asm_clobber_masks,
       .save_callee_clobbers = aa_asm_hook_save_callee_clobbers,
       .restore_one = aa_asm_hook_restore_one,
       .load_operand_to_reg = aa_direct_load_operand_to_reg,
       .load_address_to_reg = aa_direct_load_address_to_reg,
       .store_reg_to_operand = aa_direct_store_reg_to_operand,
       .run_template = aa_asm_hook_run_template,
   };
   native_asm_bind_direct_operands(d, tmpl, outs, nout, out_ops, ins, nin,
                                   in_ops, clobbers, nclob, clobber_abi_sets,
                                   &hooks);
 }
 
 /* ---- NativeTarget (optimizer) asm hook ----
  *
  * The optimized emitter owns register placement, staging, and output
  * writeback. This hook binds its concrete locations and only materializes
  * memory-constraint bases in backend-private registers. */
 
 static NativeAddr aa_asm_loc_to_addr(AANativeTarget* a, SrcLoc loc,
                                      NativeLoc src) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_type = src.type;
   switch ((NativeLocKind)src.kind) {
     case NATIVE_LOC_FRAME:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = src.v.frame;
       return addr;
     case NATIVE_LOC_ADDR:
       return src.v.addr;
     case NATIVE_LOC_GLOBAL:
       addr.base_kind = NATIVE_ADDR_BASE_GLOBAL;
       addr.base.global.sym = src.v.global.sym;
       addr.base.global.addend = src.v.global.addend;
       return addr;
     case NATIVE_LOC_REG:
       addr.base_kind = NATIVE_ADDR_BASE_REG;
       addr.cls = NATIVE_REG_INT;
       addr.base.reg = src.v.reg;
       return addr;
     default:
       aa_asm_panic_at(a->base.c, loc, "unsupported memory asm operand");
   }
 }
 
 /* Resolve a memory-constraint operand to a single base register with zero
  * offset, folding any frame/global/offset into a backend-private register. At
  * most x16/x17 are used across one asm block. */
 static Reg aa_asm_native_mem_base(AANativeTarget* a, SrcLoc loc, NativeLoc src,
                                   u32* ntmp) {
   NativeAddr addr = aa_asm_loc_to_addr(a, loc, src);
   u32 base;
   i32 off;
   Reg dst;
   if (addr.index_kind != NATIVE_ADDR_INDEX_NONE)
     aa_asm_panic_at(a->base.c, loc, "indexed memory asm operand unsupported");
   aa_addr_base(a, addr, 0u, &base, &off);
   if (off == 0) return (Reg)base;
   if (*ntmp >= 2u)
     aa_asm_panic_at(a->base.c, loc, "too many memory asm operands");
   dst = (*ntmp == 0u) ? AA_TMP0 : AA_TMP1;
   (*ntmp)++;
   aa_emit_add_imm(a, dst, base, off);
   return dst;
 }
 
 static void aa_asm_native_panic(NativeTarget* t, SrcLoc loc, const char* msg) {
   aa_asm_panic_at(t->c, loc, msg);
 }
 
 static Reg aa_asm_native_mem_base_hook(NativeTarget* t, SrcLoc loc,
                                        NativeLoc src, u32* ntmp) {
   return aa_asm_native_mem_base(aa_of(t), loc, src, ntmp);
 }
 
 static void aa_asm_native_run_template_hook(
     NativeTarget* t, const char* tmpl, const AsmConstraint* outs, u32 nout,
     Operand* bound_outs, const AsmConstraint* ins, u32 nin, Operand* bound_ins,
     const Sym* clobbers, u32 nclob) {
   AA64Asm* asmh = aa64_asm_open(t->c);
   aa64_inline_bind(asmh, outs, nout, bound_outs, ins, nin, bound_ins, clobbers,
                    nclob);
   aa64_asm_run_template(asmh, t->mc, tmpl);
   aa64_asm_close(asmh);
 }
 
 static void aa_asm_block_native(NativeTarget* t, const char* tmpl,
                                 const AsmConstraint* outs, u32 nout,
                                 NativeLoc* out_locs, const AsmConstraint* ins,
                                 u32 nin, const NativeLoc* in_locs,
                                 const Sym* clobbers, u32 nclob) {
   AANativeTarget* a = aa_of(t);
   SrcLoc loc = a->func ? a->func->loc : (SrcLoc){0, 0, 0};
   static const NativeAsmNativeHooks hooks = {
       .panic = aa_asm_native_panic,
       .bound_reg = aa_asm_bound_reg,
       .bound_mem = aa_asm_bound_mem,
       .mem_base = aa_asm_native_mem_base_hook,
       .run_template = aa_asm_native_run_template_hook,
   };
   native_asm_bind_native_operands(t, loc, tmpl, outs, nout, out_locs, ins, nin,
                                   in_locs, clobbers, nclob, &hooks);
 }
 
 static const NativeOps aa_direct_ops = {
     .bind_param = aa_bind_param,
     .tail_call_unrealizable_reason = aa_no_tail,
     .va_start_ = aa_va_start_,
     .va_arg_ = aa_va_arg_,
     .va_end_ = aa_va_end_,
     .va_copy_ = aa_va_copy_,
     .asm_block = aa_direct_asm_block,
 };
 
 const NativeOps* aa64_native_direct_ops(void) { return &aa_direct_ops; }