kit

native.c (171311B)

---

 /* ARM32 (ARMv7-M Thumb-2, soft-float) NativeTarget backend.
  *
  * Built on the shared NativeDirectTarget/MCEmitter/NativeFrame substrate. This
  * is the Phase-1 "walking skeleton" backend: it lowers the smoke set (return,
  * ALU, compare + if/else, and a <=4-int-arg call) correctly; the op-group
  * expansion (i64 lanes, bitfields, atomics, varargs, inline asm, the disasm
  * tables, code-density tiers) is Phase 2.
  *
  * Frame model (single-pass, FP-anchored):
  *   prologue:  PUSH {r7, lr}     ; save the frame pointer + return address
  *              MOV  r7, sp       ; r7 = frame anchor (just below the saved pair)
  *              SUB  sp, sp, #N    ; reserve locals + outgoing args (patched at end)
  *   slots:     addressed [r7, #-off]  (off fixed at allocation, anchor fixed)
  *   epilogue:  MOV  sp, r7        ; drop the frame (size-independent)
  *              POP  {r7, pc}      ; restore r7 + return (Thumb bit from stacked lr)
  *
  * The NDT value-cache pool is the caller-saved arg registers r0..r3 (Lever 1:
  * args materialize directly into their ABI registers); r12 (IP) is its O0
  * scratch and a backend-private hook temporary; r7 is the frame pointer.
  * Callee-saved r4..r11 are reserved from the -O0 pool (the optimizer allocates
  * over them at -O1). */
 #include <string.h>
 
 #include "abi/abi.h"
 #include "arch/arm32/arm32.h"
 #include "arch/arm32/asm.h"
 #include "arch/arm32/isa.h"
 #include "arch/arm32/regs.h"
 #include "arch/arm32/variant.h"
 #include "arch/mc.h"
 #include "arch/native_target.h"
 #include "cg/cgir.h"
 #include "cg/native_argmove.h"
 #include "cg/native_asm.h"
 #include "cg/native_direct_target.h"
 #include "cg/native_frame.h"
 #include "cg/type.h"
 #include "core/bytes.h"
 #include "core/core.h"
 #include "core/pool.h"
 #include "obj/obj.h"
 
 extern void debug_emit_row(Debug*, ObjSecId text_section, u32 offset, SrcLoc);
 extern void debug_func_pc_range(Debug*, ObjSecId text_section, u32 begin_ofs,
                                 u32 end_ofs);
 
 #define ARM_FP 7u  /* frame pointer (Thumb convention) */
 #define ARM_SCRATCH 12u /* IP: NDT operand-materialization / cycle-break scratch */
 /* Backend emit-internal temp for multi-step sequences (div/mod). lr is RESERVED
  * (never holds an NDT operand) and dead in the body — its live value is saved on
  * the stack by the prologue and restored via `pop {..,pc}`, so clobbering the lr
  * register between calls is free and cannot collide with an NDT operand in IP. */
 #define ARM_TMP 14u
 #define ARM_MAX_REG_ARG_MOVES 16u
 /* Worst-case T32 instruction slots the single-pass deferred-`SUB sp` region must
  * hold: an IP-staged large frame is MOVW + MOVT + SUB sp,sp,ip = 3 words. A
  * small frame patches the first slot with one SUBW and NOP-fills the rest. */
 #define ARM_NDT_SUB_WORDS 3u
 
 /* Incoming-arg window geometry (r7-relative). The prologue saves {r7, lr} (8
  * bytes), so r7 sits 8 bytes below the caller's outgoing-arg block: incoming
  * stack args begin at [r7 + 8]. A variadic callee additionally spills the GP
  * argument registers r0..r3 (16 bytes) into a save area pushed ABOVE the saved
  * pair, making them contiguous-below the named stack args; that shifts the
  * stack-arg window down by 16 and locates the variadic register-save area at
  * [r7 + 8]. */
 #define ARM_SAVED_PAIR_BYTES 8u
 #define ARM_VA_GP_SAVE_BYTES 16u /* r0..r3 spilled for variadics */
 
 /* ============================ state ============================ */
 
 typedef struct Arm32NativeTarget {
   NativeTarget base; /* MUST be first member (arm_of is a reinterpret cast) */
   const Arm32Variant* variant;
   SrcLoc loc;
   const CGFuncDesc* func;
   NativeFrame frame;
   u32 next_param_int;  /* incoming param cursor: r0..r3 */
   u32 next_param_stack;
   u8 has_sret;
   u8 is_variadic;
   NativeFrameSlot sret_ptr_slot;
   /* Lazily-allocated 4-byte home for an indirect call target when the call has
    * outgoing stack args. At -O0 the two backend staging registers (IP for the
    * value, LR for a large-offset address) may both be consumed, so the callee
    * cannot remain in either register across staging: it is spilled here and
    * reloaded just before BLX. NATIVE_FRAME_SLOT_NONE until first needed. */
   NativeFrameSlot callee_spill_slot;
   u32 func_start;
   u32 prologue_sub_pos; /* file offset of the deferred SUB sp (patched at end) */
   MCLabel epilogue_label;
   /* Known-frame (-O1) state. On the known-frame path the optimizer hands us the
    * exact frame up front, so the prologue is emitted final (no deferred SUB to
    * patch in arm_func_end). saved_reglist is the PUSH/POP register set (callee-
    * saves + r7 + lr); saved_block_bytes is its size in bytes, which incoming
    * stack args sit above (so bind_param adds it instead of the bare 8). */
   u8 known_frame;
   u8 slim_prologue;   /* leaf, no frame: emit BX lr, no PUSH/anchor */
   u16 saved_reglist;  /* registers PUSHed by the prologue (POP restores via pc) */
   u32 saved_block_bytes;
   /* Bytes of incoming stack-argument window this function owns (its own params
    * beyond r0..r3, 8-byte rounded). A tail/sibling call is realizable only if the
    * callee's outgoing stack args fit here — its outgoing stack args reuse this
    * window in place. */
   u32 incoming_stack_size;
   /* alloca: each dynamic allocation moves sp down and returns sp + max_outgoing
    * (the block sits just above the call outgoing-arg area at the bottom of the
    * frame). max_outgoing is only final at func_end, so the `ADD dst, sp, #off`
    * is emitted with a placeholder and patched here. */
   u32 alloca_patch_pos[16];
   u8 alloca_patch_rd[16];
   u8 n_alloca_patch;
 } Arm32NativeTarget;
 
 static Arm32NativeTarget* arm_of(NativeTarget* t) {
   return (Arm32NativeTarget*)t;
 }
 
 static _Noreturn void arm_panic(Arm32NativeTarget* a, const char* msg) {
   compiler_panic(a->base.c, a->loc, "arm32 native target: %s", msg);
 }
 
 static u32 loc_reg(NativeLoc loc) { return loc.v.reg & 0xfu; }
 
 /* Forward declarations for helpers referenced before their definitions (the
  * aggregate copy is shared by INDIRECT param binding, INDIRECT returns, and the
  * wide va_arg path). */
 static void arm_copy_bytes(NativeTarget* t, NativeAddr dst, NativeAddr src,
                            AggregateAccess acc);
 
 /* ============================ emit ============================ */
 
 static void arm_emit_t16(MCEmitter* mc, u16 hw) {
   u8 b[2];
   u32 ofs = obj_pos(mc->obj, mc->section_id);
   b[0] = (u8)(hw & 0xffu);
   b[1] = (u8)((hw >> 8) & 0xffu);
   mc_emit_bytes(mc, b, sizeof b);
   if (mc->debug) debug_emit_row(mc->debug, mc->section_id, ofs, mc->loc);
 }
 
 /* A 32-bit Thumb-2 instruction (passed as (hw1<<16)|hw2) is two LE half-words,
  * hw1 first in memory. NOT a single LE word — never route through mc_emit32. */
 static void arm_emit_t32(MCEmitter* mc, u32 instr) {
   u8 b[4];
   u32 ofs = obj_pos(mc->obj, mc->section_id);
   u32 hw1 = (instr >> 16) & 0xffffu, hw2 = instr & 0xffffu;
   b[0] = (u8)(hw1 & 0xffu);
   b[1] = (u8)((hw1 >> 8) & 0xffu);
   b[2] = (u8)(hw2 & 0xffu);
   b[3] = (u8)((hw2 >> 8) & 0xffu);
   mc_emit_bytes(mc, b, sizeof b);
   if (mc->debug) debug_emit_row(mc->debug, mc->section_id, ofs, mc->loc);
 }
 
 static void arm_patch_t32(Arm32NativeTarget* a, u32 pos, u32 instr) {
   u8 b[4];
   u32 hw1 = (instr >> 16) & 0xffffu, hw2 = instr & 0xffffu;
   b[0] = (u8)(hw1 & 0xffu);
   b[1] = (u8)((hw1 >> 8) & 0xffu);
   b[2] = (u8)(hw2 & 0xffu);
   b[3] = (u8)((hw2 >> 8) & 0xffu);
   obj_patch(a->base.obj, a->base.mc->section_id, pos, b, 4);
 }
 
 /* Emit PUSH/POP, picking the 16-bit Thumb-1 form (STMDB sp!/LDMIA sp!) when the
  * register list fits it (no flags). The 16-bit PUSH allows {r0..r7, lr}; POP
  * allows {r0..r7, pc}; any other high register (r8..r12, or lr in a POP / pc in
  * a PUSH) forces the 32-bit .W form. The common frame pairs {r7,lr}/{r7,pc} and
  * a low-reg callee-save set narrow; mixed high callee-saves stay .W. */
 static void arm_emit_push(MCEmitter* mc, u32 reglist) {
   u32 low = reglist & 0xffu, hi = reglist & ~0xffu;
   if ((hi & ~(1u << 14u)) == 0u) /* only r0..r7 and optionally lr */
     arm_emit_t16(mc, arm_push16(low, (hi >> 14) & 1u));
   else
     arm_emit_t32(mc, arm_push_w(reglist));
 }
 static void arm_emit_pop(MCEmitter* mc, u32 reglist) {
   u32 low = reglist & 0xffu, hi = reglist & ~0xffu;
   if ((hi & ~(1u << 15u)) == 0u) /* only r0..r7 and optionally pc */
     arm_emit_t16(mc, arm_pop16(low, (hi >> 15) & 1u));
   else
     arm_emit_t32(mc, arm_pop_w(reglist));
 }
 
 /* ============================ register tables ============================ */
 
 #define ARM_PHYS_ARG(r, idx)                           \
   {.reg = (r),                                         \
    .cls = NATIVE_REG_INT,                              \
    .abi_index = (idx),                                 \
    .flags = NATIVE_REG_CALLER_SAVED | NATIVE_REG_ARG | \
             ((idx) < 2u ? NATIVE_REG_RET : 0),         \
    .spill_cost = 1u,                                   \
    .copy_cost = 1u}
 #define ARM_PHYS_CALLEE(r)                                  \
   {.reg = (r),                                              \
    .cls = NATIVE_REG_INT,                                   \
    .abi_index = 0xffu,                                      \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLEE_SAVED, \
    .spill_cost = 4u,                                        \
    .copy_cost = 1u}
 #define ARM_PHYS_RESERVED_ROLE(r, role) \
   {.reg = (r),                             \
    .cls = NATIVE_REG_INT,                  \
    .abi_index = 0xffu,                     \
    .flags = NATIVE_REG_RESERVED | (role),  \
    .spill_cost = 0u,                       \
    .copy_cost = 0u}
 #define ARM_PHYS_RESERVED(r) \
   ARM_PHYS_RESERVED_ROLE((r), NATIVE_REG_NONE)
 
 /* NDT (-O0) value-cache pool: the caller-saved arg registers r0..r3, fronted so
  * a producer materializes a call's args directly into their ABI registers
  * (Lever 1 via api_pack_call_args_in_order). The -O0 cache flushes at every
  * call/branch/return, so caller-saved suffices and the prologue needs no
  * callee-save spills. */
 static const Reg arm_ndt_int_allocable[] = {0u, 1u, 2u, 3u};
 static const Reg arm_int_scratch[] = {ARM_SCRATCH};
 /* O1 operand temporaries are the caller-saved low registers. They are excluded
  * from persistent allocation and every Thumb2 operand hook accepts them. IP
  * and LR remain backend-private: many hooks use those two internally, so
  * exposing either as an operand location would couple independent lifetimes. */
 static const Reg arm_int_emit_temps[] = {0u, 1u, 2u, 3u};
 static const Reg arm_int_asm_temps[] = {ARM_TMP};
 static const Reg arm_direct_asm_int[] = {0u, 1u, 2u, 3u, 4u, 5u,
                                          6u, 8u, 9u, 10u, 11u};
 
 static const NativePhysRegInfo arm_int_phys[] = {
     ARM_PHYS_ARG(0u, 0u),  ARM_PHYS_ARG(1u, 1u),
     ARM_PHYS_ARG(2u, 2u),  ARM_PHYS_ARG(3u, 3u),
     ARM_PHYS_CALLEE(4u),   ARM_PHYS_CALLEE(5u),
     ARM_PHYS_CALLEE(6u),
     ARM_PHYS_RESERVED_ROLE(7u, NATIVE_REG_CALLEE_SAVED), /* fp */
     ARM_PHYS_CALLEE(8u),   ARM_PHYS_CALLEE(9u),
     ARM_PHYS_CALLEE(10u),  ARM_PHYS_CALLEE(11u),
     ARM_PHYS_RESERVED_ROLE(12u, NATIVE_REG_CALLER_SAVED), /* ip/temp */
     ARM_PHYS_RESERVED(13u),                        /* sp */
     ARM_PHYS_RESERVED(14u),                        /* lr */
     ARM_PHYS_RESERVED(15u),                        /* pc */
 };
 
 static int arm_resolve_name(const NativeRegInfo* ri, Slice name, Reg* out,
                             NativeAllocClass* cls_out) {
   char buf[16];
   uint32_t dwarf;
   (void)ri;
   if (!name.s || !name.len || name.len >= sizeof buf) return 1;
   memcpy(buf, name.s, name.len);
   buf[name.len] = '\0';
   if (arm32_register_index(buf, &dwarf) != 0) return 1;
   if (dwarf <= 15u) {
     *cls_out = NATIVE_REG_INT;
     *out = (Reg)dwarf;
     return 0;
   }
   return 1;
 }
 
 static int arm_asm_operand_reg_ok(const NativeRegInfo* ri, NativeAllocClass cls,
                                   Reg reg) {
   (void)ri;
   if (cls != NATIVE_REG_INT) return 0;
   if (reg <= 3u) return 1;            /* r0..r3 */
   if (reg >= 4u && reg <= 11u && reg != 7u) return 1; /* r4..r11 except fp */
   return 0;
 }
 
 static int arm_asm_constraint_reg(const NativeRegInfo* ri, const char* body,
                                   NativeAllocClass* cls_out, Reg* fixed_out,
                                   u32* allowed_mask_out) {
   (void)ri;
   if (!body || !body[0]) return 0;
   if (fixed_out) *fixed_out = REG_NONE;
   if (allowed_mask_out) *allowed_mask_out = 0;
   if ((body[0] == 'r' || body[0] == 'l') && body[1] == '\0') {
     if (cls_out) *cls_out = NATIVE_REG_INT;
     if (body[0] == 'l' && allowed_mask_out)
       *allowed_mask_out = 0x000000ffu; /* r0..r7, filtered by reg_ok. */
     return 1;
   }
   return 0;
 }
 
 static const NativeAllocClassInfo arm_classes[] = {
     {.cls = NATIVE_REG_INT,
      .ndt_allocable = arm_ndt_int_allocable,
      .ndt_allocable_count =
          sizeof arm_ndt_int_allocable / sizeof arm_ndt_int_allocable[0],
      .scratch = arm_int_scratch,
      .nscratch = sizeof arm_int_scratch / sizeof arm_int_scratch[0],
      .emit_temps = arm_int_emit_temps,
      .nemit_temps = sizeof arm_int_emit_temps / sizeof arm_int_emit_temps[0],
      .asm_temps = arm_int_asm_temps,
      .nasm_temps = sizeof arm_int_asm_temps / sizeof arm_int_asm_temps[0],
      .direct_asm_allocable = arm_direct_asm_int,
      .ndirect_asm_allocable =
          sizeof arm_direct_asm_int / sizeof arm_direct_asm_int[0],
      .emit_cache_mask = 0x0000000fu, /* r0..r3; hooks use private ip/lr */
      .phys = arm_int_phys,
      .nphys = sizeof arm_int_phys / sizeof arm_int_phys[0]},
 };
 
 static const NativeRegInfo arm_reg_info = {
     .classes = arm_classes,
     .nclasses = sizeof arm_classes / sizeof arm_classes[0],
     .ndt_caller_saved_only = 1u,
     .ndt_result_reg_stable = 1u,
     .resolve_name = arm_resolve_name,
     .asm_operand_reg_ok = arm_asm_operand_reg_ok,
     .asm_constraint_reg = arm_asm_constraint_reg,
 };
 
 /* ============================ legality ============================ */
 
 /* Resolve `rd = ra <ADD|SUB> imm` into a concrete non-negative (is_sub, mag)
  * immediate. Adding `imm` equals subtracting `-imm` and vice versa (modulo
  * 2^32), so a small negative constant folds into the opposite operation: SUB #3
  * for `+ -3`, ADD #3 for `- -3`. Returns 1 (and fills is_sub/mag) when the
  * magnitude is a Thumb modified-immediate OR fits the 12-bit ADDW/SUBW field;
  * 0 when the constant must be materialized into a register first. The emit path
  * (arm_emit_alu_imm) and the legality predicate (arm_imm_legal) BOTH route
  * through this, so a value can never be accepted as legal yet mis-encoded —
  * the bug where `SUB #-3` legalized via the -imm trick but emitted SUBW with a
  * 12-bit-truncated 0xFFFFFFFD. */
 static int arm_resolve_addsub_imm(BinOp op, i64 imm, int* is_sub, u32* mag) {
   /* k = the value actually added to ra (subtracting imm adds -imm). */
   u32 k = (u32)(op == BO_ISUB ? (u64)0 - (u64)imm : (u64)imm);
   u32 nk = (u32)((u64)0 - (u64)k); /* the magnitude of the SUB alternative */
   u32 enc;
   if (thumb_expand_imm_encode(k, &enc) || k <= 0xfffu) {
     *is_sub = 0;
     *mag = k;
     return 1;
   }
   if (thumb_expand_imm_encode(nk, &enc) || nk <= 0xfffu) {
     *is_sub = 1;
     *mag = nk;
     return 1;
   }
   return 0;
 }
 
 static int arm_imm_legal(NativeTarget* t, NativeImmUse use, u32 op,
                          KitCgTypeId type, i64 imm) {
   u32 enc;
   (void)t;
   (void)type;
   switch (use) {
     case NATIVE_IMM_MOVE:
       return 1; /* MOVW/MOVT materializes any 32-bit value */
     case NATIVE_IMM_CMP:
       return thumb_expand_imm_encode((u32)imm, &enc);
     case NATIVE_IMM_BINOP:
       switch ((BinOp)op) {
         case BO_IADD:
         case BO_ISUB: {
           int is_sub;
           u32 mag;
           return arm_resolve_addsub_imm((BinOp)op, imm, &is_sub, &mag);
         }
         case BO_AND:
         case BO_OR:
         case BO_XOR:
           return thumb_expand_imm_encode((u32)imm, &enc);
         case BO_SHL:
         case BO_SHR_S:
         case BO_SHR_U:
           return imm >= 0 && imm < 32;
         default:
           return 0;
       }
     case NATIVE_IMM_ADDR_OFFSET:
       return imm >= -255 && imm <= 4095;
   }
   return 0;
 }
 
 static int arm_addr_legal(NativeTarget* t, const NativeAddr* addr,
                           MemAccess mem) {
   (void)t;
   (void)mem;
   if (!addr) return 0;
   /* Base + 12-bit positive or 8-bit signed offset, no scaled index in v1. */
   if (addr->base_kind == NATIVE_ADDR_BASE_FRAME_VALUE) return 0;
   if (addr->index_kind != NATIVE_ADDR_INDEX_NONE) return 0;
   return addr->offset >= -255 && addr->offset <= 4095;
 }
 
 /* Capability hook for the arch-neutral L7 shift-into-ALU fold
  * (src/opt/pass_combine.c). arm32 has the 32-bit shifted-register forms wired up
  * (arm_binop's register path emits ADD.W/SUB.W/AND.W/ORR.W/EOR.W rd,rn,rm,LSL #k
  * via the arm_*_reg_lsl encoders), but the capability is left OFF here.
  *
  * WHY OFF: opting in surfaces a latent miscompile in the shared combine pass for
  * the `base + (i << k)` address pattern when k exceeds the address index-scale
  * range (k>=4, e.g. a 16-byte struct stride). The order is: addr-synth's index
  * fold (pass_combine.c rule (b)) only absorbs a shift of 1..3 into the EA's
  * log2_scale; for k=4 it declines, leaving the `i<<4` as an explicit IADD that
  * computes the element address. L7 then stamps a shift rider on that IADD's rhs.
  * On a later combine iteration, addr-synth rule (a) (reg+reg) folds that IADD
  * into the indirect's index but copies only rhs.v.reg, DROPPING rhs.shift — so
  * the `*16` silently becomes `*1` and every element aliases element 0
  * (test/parse/cases/opt_03_addr_index_alias_store regresses 42 -> 1 under qemu).
  * aa64 avoids this because its addr_legal accepts the scaled-index EA, so
  * addr-synth claims the address shift before L7 ever sees it; arm32's addr_legal
  * rejects every indexed address (no scaled-index load/store in v1), so the
  * shift stays in the ALU lane and the rider-drop bites. Re-enable once the
  * shared pass preserves rhs.shift across the rule-(a) index fold; the encoders +
  * arm_binop branch + decode round-trip test are already in place. */
 static int arm_can_fold_shift_into_alu(NativeTarget* t) {
   (void)t;
   return 0;
 }
 
 /* ============================ 16-bit narrow helpers ============================
  * The 16-bit Thumb-1 data-processing forms used by the density pass always set
  * the condition flags (there is no S=0 narrow DP form outside an IT block). kit
  * keeps flags live only across CMP→IT (and comparisons always re-emit their
  * CMP), so binop/move/shift/ldst results are FLAG-DEAD — narrowing them to a
  * flag-setting 16-bit encoding is observationally identical. The MOV.W sites
  * that sit between a CMP and an IT-predicated consumer (arm_cmp, arm_unop's
  * UO_NOT, arm_emit_setcc) are NOT routed through here and stay 32-bit. */
 static inline int arm_low_reg(u32 r) { return r <= 7u; }
 /* True when an unsigned immediate fits an n-bit field. */
 static inline int arm_uimm_fits(u32 v, u32 nbits) { return v < (1u << nbits); }
 
 /* ============================ memory ============================ */
 
 static void arm_emit_load_u32(Arm32NativeTarget* a, u32 rd, u32 v);
 static u32 arm_pick_scratch(Arm32NativeTarget* a, u32 rt, u32 pinned);
 
 /* Add a signed compile-time constant to rd in place (rd += k), using a reserved
  * scratch only for a magnitude exceeding the inline immediate forms. */
 static void arm_emit_add_const(Arm32NativeTarget* a, u32 rd, i64 k) {
   MCEmitter* mc = a->base.mc;
   u32 enc, mag;
   if (k == 0) return;
   mag = (u32)(k < 0 ? -k : k);
   /* 16-bit `ADD/SUB sp,sp,#imm7*4` (no flags) for the SP-adjust forms (e.g. the
    * variadic GP-save reclaim). Requires rd==sp, a non-zero multiple of 4, and
    * imm7 in range. */
   if (rd == 13u && (mag & 3u) == 0u && (mag >> 2) <= 0x7fu) {
     arm_emit_t16(mc, k < 0 ? arm_sub_sp_sp_imm16(mag >> 2)
                            : arm_add_sp_sp_imm16(mag >> 2));
     return;
   }
   if (mag <= 0xfffu) {
     arm_emit_t32(mc, k < 0 ? arm_sub_imm12(rd, rd, mag)
                            : arm_add_imm12(rd, rd, mag));
   } else if (thumb_expand_imm_encode(mag, &enc)) {
     arm_emit_t32(mc, arm_dp_imm(k < 0 ? 13u : 8u, 0u, rd, rd, enc));
   } else {
     u32 s = arm_pick_scratch(a, rd, 0u);
     arm_emit_load_u32(a, s, mag);
     arm_emit_t32(mc, k < 0 ? arm_sub_reg(rd, rd, s) : arm_add_reg(rd, rd, s));
   }
 }
 
 /* Materialize the runtime address of `sym` (+ addend) into register `rd`.
  *
  * arm32's house strategy is MOVW/MOVT absolute (Cortex-M has no literal pools
  * and the bare-metal lane is non-PIC):
  *     MOVW rd, #:lower16:sym    ; R_ARM_THM_MOVW_ABS_NC -> S[15:0]
  *     MOVT rd, #:upper16:sym    ; R_ARM_THM_MOVT_ABS    -> S[31:16]
  * A constant byte `addend` is applied as a SEPARATE add, NOT folded into the
  * reloc: ARM uses REL relocations (the addend rides in the relocated field), and
  * the MOVW/MOVT pair splits a 32-bit value across two 16-bit fields whose
  * per-half (S+A) computation is carry-lossy across the 16-bit boundary. Keeping
  * the reloc addend 0 and adding the offset here is both always-correct and the
  * contract elf_arm_reloc_field_addend documents (and ld.lld-compatible). */
 static void arm_emit_global_addr(Arm32NativeTarget* a, u32 rd, ObjSymId sym,
                                  i64 addend) {
   MCEmitter* mc = a->base.mc;
   u32 pos = mc_pos(mc);
   arm_emit_t32(mc, arm_movw(rd, 0u));
   mc_emit_reloc_at(mc, mc->section_id, pos, R_ARM_THM_MOVW_ABS_NC, sym, 0, 1, 0);
   pos = mc_pos(mc);
   arm_emit_t32(mc, arm_movt(rd, 0u));
   mc_emit_reloc_at(mc, mc->section_id, pos, R_ARM_THM_MOVT_ABS, sym, 0, 1, 0);
   arm_emit_add_const(a, rd, addend);
 }
 
 /* The unified memory-access address layer (ARM32.md §1) lives here. Every
  * load/store/copy resolves a NativeAddr to a legal (base register, in-range
  * displacement) through ONE function, arm_resolve_mem, with ONE temporary-
  * reservation invariant asserted in one place.
  *
  * Backend budget: IP (r12, ARM_SCRATCH) and LR (r14, ARM_TMP) are reserved for
  * hook-internal staging; they are deliberately not O1 operand locations. r7
  * (FP) holds the frame anchor and is never temporary storage. A single access
  * needs at most one address-staging register, distinct from transfer register
  * `rt`. The picker
  * draws it from {IP,LR} minus {rt} minus a caller `pinned` mask (held bases the
  * access must preserve) and panics if none is free, so the invariant is explicit
  * rather than relying on incidental liveness. */
 static void arm_emit_base_off(Arm32NativeTarget* a, u32 rd, u32 base, i32 off);
 static void arm_load_addr(NativeTarget* t, NativeLoc dst, NativeAddr addr);
 
 /* True if every byte of a `span`-byte access at signed displacement `off` fits
  * the Thumb-2 single-granule immediate forms (T4 negative -255..0, T3 positive
  * 0..4095): off >= -255 and off+span <= 4096 implies every granule offset in
  * [off, off+span-1] lands in [-255, 4095]. */
 static int arm_span_inline(i32 off, u32 span) {
   return off >= -255 && off + (i32)span <= 4096;
 }
 
 /* Pick a reserved staging register from {LR, IP} that is neither the transfer
  * register `rt` (0xff = none) nor in `pinned` (held base registers the access
  * must preserve). LR (ARM_TMP) is preferred because IP (ARM_SCRATCH) is the NDT's
  * operand-materialization scratch and can hold a live value across the access
  * (e.g. a store whose value the NDT homed in IP); LR is only ever a transient
  * backend temp within a single emit, so staging an address there is free. The
  * single point that enforces the scratch invariant. */
 static u32 arm_pick_scratch(Arm32NativeTarget* a, u32 rt, u32 pinned) {
   u32 avoid = (rt < 16u ? (1u << rt) : 0u) | pinned;
   if (!(avoid & (1u << ARM_TMP))) return ARM_TMP;
   if (!(avoid & (1u << ARM_SCRATCH))) return ARM_SCRATCH;
   arm_panic(a, "memory access: no free address-staging scratch");
 }
 
 typedef struct ArmMemAddr {
   u32 base; /* base register to address through */
   i32 off;  /* signed displacement; access [base, #off (+lane/granule)] */
 } ArmMemAddr;
 
 /* Materialize the unsigned 32-bit constant `v` into `rd` (MOVW + optional MOVT).
  * Used to stage large frame/memory offsets and SUB-sp amounts that exceed the
  * inline immediate forms. */
 static void arm_emit_load_u32(Arm32NativeTarget* a, u32 rd, u32 v) {
   MCEmitter* mc = a->base.mc;
   u32 enc;
   if (thumb_expand_imm_encode(v, &enc)) {
     arm_emit_t32(mc, arm_mov_imm(rd, enc));
     return;
   }
   if (thumb_expand_imm_encode(~v, &enc)) {
     arm_emit_t32(mc, arm_mvn_imm(rd, enc));
     return;
   }
   arm_emit_t32(mc, arm_movw(rd, v & 0xffffu));
   if ((v >> 16) != 0u) arm_emit_t32(mc, arm_movt(rd, (v >> 16) & 0xffffu));
 }
 
 /* Resolve `addr` for a `span`-byte access whose transfer register is `rt`
  * (0xff = none; the wide path pins both lane registers via `pinned` instead),
  * returning a legal (base, off). A FRAME/REG base already in range is used
  * directly; otherwise the full effective address is materialized once into a
  * picked scratch (off := 0). A GLOBAL base is always materialized via MOVW/MOVT,
  * folding addr.offset into the reloc addend so off := 0 — this is the single
  * point that guarantees a store's global address never lands in the value
  * register (the i64-store-to-global miscompile). */
 static ArmMemAddr arm_resolve_mem(Arm32NativeTarget* a, const NativeAddr* addr,
                                   u32 rt, u32 span, u32 pinned) {
   ArmMemAddr r;
   /* Scaled-index address (base + offset + index<<scale): materialize the whole
    * effective address once into a reserved scratch and return (scratch, 0).
    * arm32 has no LDRD/STRD indexed form, so a wide (8-byte) indexed access must
    * share ONE computed base across both lanes; doing it here keeps every base
    * kind (frame / reg / global) uniform and is the §1 single-base guarantee for
    * the high lane. The scratch aliases neither the transfer register(s) — rt and
    * `pinned` — nor the index register. */
   if (addr->index_kind == NATIVE_ADDR_INDEX_REG) {
     MCEmitter* mc = a->base.mc;
     u32 idx = addr->index.reg & 0xfu;
     u32 s = arm_pick_scratch(a, rt, pinned | (1u << idx));
     switch (addr->base_kind) {
       case NATIVE_ADDR_BASE_FRAME: {
         NativeFrameSlotEntry* e =
             native_frame_slot_at(&a->frame, addr->base.frame);
         arm_emit_base_off(a, s, ARM_FP, -(i32)e->off + addr->offset);
         break;
       }
       case NATIVE_ADDR_BASE_REG:
         arm_emit_base_off(a, s, addr->base.reg & 0xfu, addr->offset);
         break;
       case NATIVE_ADDR_BASE_GLOBAL:
         arm_emit_global_addr(a, s, addr->base.global.sym,
                              addr->base.global.addend + addr->offset);
         break;
       default:
         arm_panic(a, "unsupported indexed addressing mode");
     }
     arm_emit_t32(mc, arm_add_reg_lsl(s, s, idx, addr->log2_scale));
     r.base = s;
     r.off = 0;
     return r;
   }
   switch (addr->base_kind) {
     case NATIVE_ADDR_BASE_FRAME: {
       NativeFrameSlotEntry* s =
           native_frame_slot_at(&a->frame, addr->base.frame);
       i32 off = -(i32)s->off + addr->offset;
       if (arm_span_inline(off, span)) {
         r.base = ARM_FP;
         r.off = off;
         return r;
       }
       r.base = arm_pick_scratch(a, rt, pinned | (1u << ARM_FP));
       arm_emit_base_off(a, r.base, ARM_FP, off);
       r.off = 0;
       return r;
     }
     case NATIVE_ADDR_BASE_REG: {
       u32 base = addr->base.reg & 0xfu;
       if (arm_span_inline(addr->offset, span)) {
         r.base = base;
         r.off = addr->offset;
         return r;
       }
       r.base = arm_pick_scratch(a, rt, pinned | (1u << base));
       arm_emit_base_off(a, r.base, base, addr->offset);
       r.off = 0;
       return r;
     }
     case NATIVE_ADDR_BASE_GLOBAL:
       r.base = arm_pick_scratch(a, rt, pinned);
       arm_emit_global_addr(a, r.base, addr->base.global.sym,
                            addr->base.global.addend + addr->offset);
       r.off = 0;
       return r;
     default:
       arm_panic(a, "unsupported addressing mode");
   }
 }
 
 /* Emit one in-range granule access: LDR/STR (+B/H) of `rt` at [base, #off],
  * dispatching width. The caller (via arm_resolve_mem) guarantees `off` is in the
  * T3/T4 immediate range for this width. Loads zero-extend; signed narrowing is a
  * separate convert. */
 static void arm_emit_mem_one(Arm32NativeTarget* a, int is_load, u32 rt, u32 base,
                              i32 off, u32 size) {
   MCEmitter* mc = a->base.mc;
   u32 t3hw1, t4hw1;
   /* 16-bit narrow forms (no flags, biggest -O0 win since loads/stores dominate):
    *   T2 [sp, #imm8*4]: word only, rt low reg, base == sp (NOT r7 — the frame is
    *     r7-anchored, so this applies only to genuine sp-based slots).
    *   T1 [rn, #imm5*scale]: rt + rn both low regs (r7/ARM_FP qualifies), positive
    *     offset that is an exact multiple of the access size and fits 5 bits.
    * Signed loads (LDRSB/LDRSH) have no 16-bit immediate-offset form, so the
    * width dispatch below only narrows the unsigned word/byte/half loads + stores;
    * the resolver never asks this function for a signed-narrowing load (that is a
    * separate convert). */
   if (size == 4u && base == ARM_SP && arm_low_reg(rt) && off >= 0 &&
       (off & 3) == 0 && arm_uimm_fits((u32)off >> 2, 8u)) {
     arm_emit_t16(mc, arm_ldst_sp_16(is_load ? 0x9800u : 0x9000u, rt,
                                     (u32)off >> 2));
     return;
   }
   if (arm_low_reg(rt) && arm_low_reg(base) && off >= 0) {
     u32 uoff = (u32)off;
     if (size == 4u && (uoff & 3) == 0 && arm_uimm_fits(uoff >> 2, 5u)) {
       arm_emit_t16(mc, arm_ldst_i5_16(is_load ? 0x6800u : 0x6000u, rt, base,
                                       uoff >> 2));
       return;
     }
     if (size == 2u && (uoff & 1) == 0 && arm_uimm_fits(uoff >> 1, 5u)) {
       arm_emit_t16(mc, arm_ldst_i5_16(is_load ? 0x8800u : 0x8000u, rt, base,
                                       uoff >> 1));
       return;
     }
     if (size == 1u && arm_uimm_fits(uoff, 5u)) {
       arm_emit_t16(mc, arm_ldst_i5_16(is_load ? 0x7800u : 0x7000u, rt, base,
                                       uoff));
       return;
     }
   }
   if (size == 1u) {
     t3hw1 = is_load ? 0xf890u : 0xf880u;
     t4hw1 = is_load ? 0xf810u : 0xf800u;
   } else if (size == 2u) {
     t3hw1 = is_load ? 0xf8b0u : 0xf8a0u;
     t4hw1 = is_load ? 0xf830u : 0xf820u;
   } else {
     t3hw1 = is_load ? 0xf8d0u : 0xf8c0u;
     t4hw1 = is_load ? 0xf850u : 0xf840u;
   }
   if (off >= 0 && off <= 4095) {
     arm_emit_t32(mc, arm_t32(t3hw1 | base, (rt << 12) | (u32)off));
   } else if (off < 0 && off >= -255) {
     arm_emit_t32(mc, arm_ldst_t4(t4hw1, rt, base, (u32)(-off), 0u));
   } else {
     arm_panic(a, "arm_emit_mem_one: offset out of range (resolver bug)");
   }
 }
 
 /* Emit a load (is_load=1) or store of `reg` at `addr`, dispatching width from
  * mem.size. A wide (8-byte i64/double) access splits into two 4-byte lanes over
  * the register pair rt:rt+1 sharing ONE resolved base — the high lane's address
  * is thus computed correctly for frame/reg/global uniformly. */
 static void arm_emit_mem(Arm32NativeTarget* a, int is_load, NativeLoc reg,
                          NativeAddr addr, MemAccess mem) {
   u32 rt = loc_reg(reg);
   u32 size = mem.size ? mem.size : native_type_size(&a->base, reg.type);
   if (size == 0u) arm_panic(a, "zero-width memory access");
   if (size > 8u) arm_panic(a, "memory access wider than 8 bytes");
   if (size == 8u) {
     u32 hi = (rt + 1u) & 0xfu;
     ArmMemAddr r =
         arm_resolve_mem(a, &addr, 0xffu, 8u, (1u << rt) | (1u << hi));
     arm_emit_mem_one(a, is_load, rt, r.base, r.off, 4u);
     arm_emit_mem_one(a, is_load, hi, r.base, r.off + 4, 4u);
     return;
   }
   if (size != 1u && size != 2u && size != 4u) {
     NativePartChunkIter it = native_part_chunks(size, 4u);
     KitCgTypeId lane_type = builtin_id(KIT_CG_BUILTIN_I32);
     u32 carrier_mask = 1u << rt;
     u32 addr_inputs = 0u;
     u32 chunk_off, chunk_size;
     u32 tmp = REG_NONE;
     ArmMemAddr r;
     if (size > 4u) carrier_mask |= 1u << ((rt + 1u) & 0xfu);
     if (addr.base_kind == NATIVE_ADDR_BASE_REG)
       addr_inputs |= 1u << (addr.base.reg & 0xfu);
     if (addr.index_kind == NATIVE_ADDR_INDEX_REG)
       addr_inputs |= 1u << (addr.index.reg & 0xfu);
     if (is_load && (carrier_mask & addr_inputs)) {
       /* A single LDR may use the destination as its base (`ldr r0, [r0]`).
        * An exact part is a sequence, so doing that would destroy the address
        * before its later chunks. Retain the complete effective address in a
        * private register first; avoid every address input because load_addr
        * still needs them while it constructs the indexed form. */
       u32 addr_reg = arm_pick_scratch(a, 0xffu, carrier_mask | addr_inputs);
       NativeLoc addr_loc =
           native_loc_reg(lane_type, NATIVE_REG_INT, (Reg)addr_reg);
       arm_load_addr(&a->base, addr_loc, addr);
       r.base = addr_reg;
       r.off = 0;
     } else {
       /* Resolve once for the full span. Besides avoiding repeated global/far
        * address formation, this gives every chunk one stable base and makes
        * scratch ownership explicit: a later pack temporary must avoid r.base. */
       r = arm_resolve_mem(a, &addr, 0xffu, size, carrier_mask);
     }
     /* ABIArgPart.size is exact even when the carrier is a 32-bit register.
      * Split 3/5/6/7-byte parts into bounded 4/2/1-byte accesses and pack each
      * chunk into the appropriate low/high carrier lane. This boundary is also
      * used by generic call-result moves, so no caller can silently widen a
      * semantic part into an adjacent frame object. */
     while (native_part_chunk_next(&it, &chunk_off, &chunk_size)) {
       u32 lane_reg = (rt + chunk_off / 4u) & 0xfu;
       u32 lane_shift = (chunk_off & 3u) * 8u;
       u32 transfer_reg = lane_reg;
       if (lane_shift) {
         if (tmp == REG_NONE)
           tmp = arm_pick_scratch(a, 0xffu,
                                  carrier_mask | (1u << (r.base & 0xfu)));
         transfer_reg = tmp;
         if (!is_load)
           arm_emit_t32(a->base.mc,
                        arm_shift_imm(1u, tmp, lane_reg, lane_shift));
       }
       arm_emit_mem_one(a, is_load, transfer_reg, r.base,
                        r.off + (i32)chunk_off, chunk_size);
       if (is_load && lane_shift)
         arm_emit_t32(a->base.mc,
                      arm_orr_reg_lsl(lane_reg, lane_reg, tmp, lane_shift));
     }
     return;
   }
   {
     ArmMemAddr r = arm_resolve_mem(a, &addr, rt, size, 0u);
     arm_emit_mem_one(a, is_load, rt, r.base, r.off, size);
   }
 }
 
 /* ============================ moves / immediates ============================ */
 
 static void arm_load_imm(NativeTarget* t, NativeLoc dst, i64 imm) {
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst);
   u32 v = (u32)imm;
   u32 enc;
   /* MOVS rd,#imm8 (low reg, 0..255) — flag-dead 16-bit immediate move. This is
    * the general NDT load_imm hook, never the CMP→IT flag-preserving MOV.W (those
    * sites call arm_mov_imm directly), so setting flags here is observationally
    * inert. */
   if (arm_low_reg(rd) && v <= 0xffu) {
     arm_emit_t16(mc, arm_movs_imm8(rd, v));
     return;
   }
   if (thumb_expand_imm_encode(v, &enc)) {
     arm_emit_t32(mc, arm_mov_imm(rd, enc));
     return;
   }
   if (thumb_expand_imm_encode(~v, &enc)) {
     arm_emit_t32(mc, arm_mvn_imm(rd, enc));
     return;
   }
   arm_emit_t32(mc, arm_movw(rd, v & 0xffffu));
   if ((v >> 16) != 0u) arm_emit_t32(mc, arm_movt(rd, (v >> 16) & 0xffffu));
 }
 
 static void arm_move(NativeTarget* t, NativeLoc dst, NativeLoc src) {
   u32 rd = loc_reg(dst), rs = loc_reg(src);
   if (rd == rs) return;
   arm_emit_t16(t->mc, arm_mov_hi(rd, rs));
 }
 
 static void arm_load(NativeTarget* t, NativeLoc dst, NativeAddr addr,
                      MemAccess mem) {
   arm_emit_mem(arm_of(t), 1, dst, addr, mem);
 }
 static void arm_store(NativeTarget* t, NativeAddr addr, NativeLoc src,
                       MemAccess mem) {
   arm_emit_mem(arm_of(t), 0, src, addr, mem);
 }
 static void arm_spill(NativeTarget* t, NativeLoc src, NativeFrameSlot slot,
                       MemAccess mem) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = slot;
   addr.base_type = src.type;
   arm_emit_mem(arm_of(t), 0, src, addr, mem);
 }
 static void arm_reload(NativeTarget* t, NativeLoc dst, NativeFrameSlot slot,
                        MemAccess mem) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = slot;
   addr.base_type = dst.type;
   arm_emit_mem(arm_of(t), 1, dst, addr, mem);
 }
 
 /* Materialize a full address into `rd`: base + signed byte offset. */
 static void arm_emit_base_off(Arm32NativeTarget* a, u32 rd, u32 base, i32 off) {
   MCEmitter* mc = a->base.mc;
   u32 enc;
   if (off == 0) {
     if (rd != base) arm_emit_t16(mc, arm_mov_hi(rd, base));
   } else if (off > 0 && off <= 0xfff) {
     arm_emit_t32(mc, arm_add_imm12(rd, base, (u32)off));
   } else if (off < 0 && thumb_expand_imm_encode((u32)(-off), &enc)) {
     arm_emit_t32(mc, arm_dp_imm(13u, 0u, rd, base, enc)); /* SUB rd, base, #-off */
   } else {
     /* Large frame/array offset: stage |off| in a temp then add/sub into rd. The
      * temp must not alias `base` (read after write). Use rd itself when rd !=
      * base (the resolver guarantees a staged base reg differs from its source);
      * only when rd aliases base (load_addr's address-of-an-indexed-REG case)
      * fall back to the reserved ARM_TMP, which is dead in that context. */
     u32 tmp = (rd != base) ? rd : arm_pick_scratch(a, 0xffu, 1u << base);
     arm_emit_load_u32(a, tmp, (u32)(off < 0 ? -off : off));
     arm_emit_t32(mc, off < 0 ? arm_sub_reg(rd, base, tmp)
                              : arm_add_reg(rd, base, tmp));
   }
 }
 
 /* Materialize the address of a NativeAddr into the destination register.
  *
  * The NDT routes a base+index (or any addr the addr_legal hook rejects)
  * through here and then treats `rd` as a plain BASE_REG, so this MUST fold any
  * scaled index: rd = base + offset + (index << log2_scale). A GLOBAL base's
  * byte offset folds into the MOVW/MOVT reloc addend; FRAME/REG bases add the
  * offset explicitly. `rd` is a fresh NDT scratch, distinct from the pinned
  * index register, so no aliasing post-add is needed. */
 static void arm_load_addr(NativeTarget* t, NativeLoc dst, NativeAddr addr) {
   Arm32NativeTarget* a = arm_of(t);
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst);
   if (addr.base_kind == NATIVE_ADDR_BASE_GLOBAL) {
     arm_emit_global_addr(a, rd, addr.base.global.sym,
                          addr.base.global.addend + addr.offset);
   } else if (addr.base_kind == NATIVE_ADDR_BASE_FRAME) {
     NativeFrameSlotEntry* s = native_frame_slot_at(&a->frame, addr.base.frame);
     arm_emit_base_off(a, rd, ARM_FP, -(i32)s->off + addr.offset);
   } else if (addr.base_kind == NATIVE_ADDR_BASE_FRAME_VALUE) {
     NativeAddr load;
     NativeLoc value;
     MemAccess mem;
     if (!addr.base_type)
       arm_panic(a, "frame-value base has no exact type");
     memset(&load, 0, sizeof load);
     load.base_kind = NATIVE_ADDR_BASE_FRAME;
     load.base.frame = addr.base.frame;
     load.base_type = addr.base_type;
     memset(&value, 0, sizeof value);
     value.kind = NATIVE_LOC_REG;
     value.cls = NATIVE_REG_INT;
     value.type = addr.base_type;
     value.v.reg = rd;
     memset(&mem, 0, sizeof mem);
     mem.type = value.type;
     mem = native_mem_for_type(t, value.type, 0);
     arm_emit_mem(a, 1, value, load, mem);
     arm_emit_base_off(a, rd, rd, addr.offset);
   } else if (addr.base_kind == NATIVE_ADDR_BASE_REG) {
     arm_emit_base_off(a, rd, addr.base.reg & 0xfu, addr.offset);
   } else {
     arm_panic(a, "unsupported addressing mode in load_addr");
   }
   if (addr.index_kind == NATIVE_ADDR_INDEX_REG) {
     u32 idx = addr.index.reg & 0xfu;
     arm_emit_t32(mc, arm_add_reg_lsl(rd, rd, idx, addr.log2_scale));
   } else if (addr.index_kind == NATIVE_ADDR_INDEX_FRAME_VALUE) {
     NativeAddr load;
     NativeLoc value;
     MemAccess mem;
     u32 idx = arm_pick_scratch(a, rd, 1u << rd);
     if (!addr.index_type)
       arm_panic(a, "frame-value index has no exact type");
     memset(&load, 0, sizeof load);
     load.base_kind = NATIVE_ADDR_BASE_FRAME;
     load.base.frame = addr.index.frame;
     load.base_type = addr.index_type;
     memset(&value, 0, sizeof value);
     value.kind = NATIVE_LOC_REG;
     value.cls = NATIVE_REG_INT;
     value.type = addr.index_type;
     value.v.reg = idx;
     memset(&mem, 0, sizeof mem);
     mem.type = value.type;
     mem = native_mem_for_type(t, value.type, 0);
     arm_emit_mem(a, 1, value, load, mem);
     arm_emit_t32(mc, arm_add_reg_lsl(rd, rd, idx, addr.log2_scale));
   } else if (addr.index_kind != NATIVE_ADDR_INDEX_NONE) {
     arm_panic(a, "unsupported address index kind in load_addr");
   }
 }
 
 /* ============================ ALU ============================ */
 
 /* Emit `rd = ra (+|-) mag` in the densest legal Thumb-2 encoding: the 16-bit
  * ADDS/SUBS imm3/imm8 (flag-dead) when the magnitude is small and both registers
  * are low, else the 32-bit modified-immediate form, else the 12-bit ADDW/SUBW
  * raw form. `mag` is guaranteed by arm_resolve_addsub_imm to fit a modified
  * immediate or the 12-bit field, so the raw fallback never truncates. */
 static void arm_emit_alu_imm(MCEmitter* mc, int is_sub, u32 rd, u32 ra,
                              u32 mag) {
   u32 enc;
   if (!is_sub) {
     if (arm_low_reg(rd) && arm_low_reg(ra) && arm_uimm_fits(mag, 3u))
       arm_emit_t16(mc, arm_adds_imm3(rd, ra, mag));
     else if (rd == ra && arm_low_reg(rd) && arm_uimm_fits(mag, 8u))
       arm_emit_t16(mc, arm_adds_imm8(rd, mag));
     else if (thumb_expand_imm_encode(mag, &enc))
       arm_emit_t32(mc, arm_dp_imm(8u, 0u, rd, ra, enc));
     else
       arm_emit_t32(mc, arm_add_imm12(rd, ra, mag & 0xfffu));
   } else {
     if (arm_low_reg(rd) && arm_low_reg(ra) && arm_uimm_fits(mag, 3u))
       arm_emit_t16(mc, arm_subs_imm3(rd, ra, mag));
     else if (rd == ra && arm_low_reg(rd) && arm_uimm_fits(mag, 8u))
       arm_emit_t16(mc, arm_subs_imm8(rd, mag));
     else if (thumb_expand_imm_encode(mag, &enc))
       arm_emit_t32(mc, arm_dp_imm(13u, 0u, rd, ra, enc));
     else
       arm_emit_t32(mc, arm_sub_imm12(rd, ra, mag & 0xfffu));
   }
 }
 
 static void arm_binop(NativeTarget* t, BinOp op, NativeLoc dst, NativeLoc a_loc,
                       NativeLoc b) {
   Arm32NativeTarget* a = arm_of(t);
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst), ra = loc_reg(a_loc);
   int b_imm = b.kind == NATIVE_LOC_IMM;
   u32 rb = b_imm ? 0u : loc_reg(b);
   u32 enc;
   /* L7 shifted-register fold (O1-only; the rider is set by pass_combine when the
    * target advertised can_fold_shift_into_alu). `b` is a register pre-shifted
    * left by `b.shift` (1..4): emit the one-instruction 32-bit shifted form
    * `<op>.W rd,ra,rb,LSL #k` instead of a separate LSL + op. Only the five DP
    * ops binop_takes_shifted_rhs admits ever carry a rider. The 16-bit forms have
    * no shifted operand, so this forces a .W encoding — still a net -1 insn.
    * NOTE: arm_can_fold_shift_into_alu currently returns 0 (see its comment for
    * the shared-pass rider-drop miscompile), so b.shift is always 0 here today —
    * this branch is the ready-to-go emit path for when the fold is re-enabled. */
   if (!b_imm && b.shift) {
     u32 k = b.shift;
     switch (op) {
       case BO_IADD: arm_emit_t32(mc, arm_add_reg_lsl(rd, ra, rb, k)); return;
       case BO_ISUB: arm_emit_t32(mc, arm_sub_reg_lsl(rd, ra, rb, k)); return;
       case BO_AND:  arm_emit_t32(mc, arm_and_reg_lsl(rd, ra, rb, k)); return;
       case BO_OR:   arm_emit_t32(mc, arm_orr_reg_lsl(rd, ra, rb, k)); return;
       case BO_XOR:  arm_emit_t32(mc, arm_eor_reg_lsl(rd, ra, rb, k)); return;
       default:      arm_panic(a, "shift rider on unsupported binop");
     }
   }
   switch (op) {
     case BO_IADD:
       if (b_imm) {
         /* Fold the constant into ADD/SUB (a negative addend becomes SUB) via the
          * shared resolver; arm_emit_alu_imm picks the densest legal encoding. */
         int is_sub;
         u32 mag;
         if (!arm_resolve_addsub_imm(BO_IADD, b.v.imm, &is_sub, &mag))
           arm_panic(a, "add immediate not encodable");
         arm_emit_alu_imm(mc, is_sub, rd, ra, mag);
       } else if (arm_low_reg(rd) && arm_low_reg(ra) && arm_low_reg(rb)) {
         arm_emit_t16(mc, arm_adds_reg16(rd, ra, rb)); /* flag-dead */
       } else {
         arm_emit_t32(mc, arm_add_reg(rd, ra, rb));
       }
       return;
     case BO_ISUB:
       if (b_imm) {
         int is_sub;
         u32 mag;
         if (!arm_resolve_addsub_imm(BO_ISUB, b.v.imm, &is_sub, &mag))
           arm_panic(a, "sub immediate not encodable");
         arm_emit_alu_imm(mc, is_sub, rd, ra, mag);
       } else if (arm_low_reg(rd) && arm_low_reg(ra) && arm_low_reg(rb)) {
         arm_emit_t16(mc, arm_subs_reg16(rd, ra, rb)); /* flag-dead */
       } else {
         arm_emit_t32(mc, arm_sub_reg(rd, ra, rb));
       }
       return;
     case BO_IMUL:
       arm_emit_t32(mc, arm_mul(rd, ra, rb));
       return;
     case BO_SDIV:
       arm_emit_t32(mc, arm_sdiv(rd, ra, rb));
       return;
     case BO_UDIV:
       arm_emit_t32(mc, arm_udiv(rd, ra, rb));
       return;
     case BO_SREM:
     case BO_UREM: {
       /* rd = ra - (ra/rb)*rb via SDIV/UDIV into a quotient temp, then MLS. The
        * temp must survive ra/rb (read by MLS), so it must differ from ra and rb;
        * it may be rd when rd aliases neither operand (the common case — no extra
        * register). Otherwise pick a reserved scratch (LR, else IP): when rd does
        * alias an operand, rd is an allocated register (≠ IP/LR), so at most one of
        * {IP,LR} can be an operand and the other is always free. */
       u32 q = (rd != ra && rd != rb)
                   ? rd
                   : ((ra != ARM_TMP && rb != ARM_TMP) ? ARM_TMP : ARM_SCRATCH);
       arm_emit_t32(mc, op == BO_SREM ? arm_sdiv(q, ra, rb) : arm_udiv(q, ra, rb));
       arm_emit_t32(mc, arm_mls(rd, q, rb, ra));
       return;
     }
     case BO_AND:
       if (b_imm && thumb_expand_imm_encode((u32)b.v.imm, &enc))
         arm_emit_t32(mc, arm_dp_imm(0u, 0u, rd, ra, enc));
       else
         arm_emit_t32(mc, arm_and_reg(rd, ra, rb));
       return;
     case BO_OR:
       if (b_imm && thumb_expand_imm_encode((u32)b.v.imm, &enc))
         arm_emit_t32(mc, arm_dp_imm(2u, 0u, rd, ra, enc));
       else
         arm_emit_t32(mc, arm_orr_reg(rd, ra, rb));
       return;
     case BO_XOR:
       if (b_imm && thumb_expand_imm_encode((u32)b.v.imm, &enc))
         arm_emit_t32(mc, arm_dp_imm(4u, 0u, rd, ra, enc));
       else
         arm_emit_t32(mc, arm_eor_reg(rd, ra, rb));
       return;
     case BO_SHL: {
       /* LSLS rd,rm,#imm5 (low regs, flag-dead). LSL #0 is a plain move in both
        * widths, so narrowing the whole 0..31 range is safe for LSL. */
       u32 sh = (u32)b.v.imm & 31u;
       if (b_imm && arm_low_reg(rd) && arm_low_reg(ra))
         arm_emit_t16(mc, arm_lsls_imm16(rd, ra, sh));
       else if (b_imm)
         arm_emit_t32(mc, arm_shift_imm(0u, rd, ra, sh));
       else
         arm_emit_t32(mc, arm_shift_reg(0u, rd, ra, rb));
       return;
     }
     case BO_SHR_U: {
       /* LSRS: the 16-bit form encodes #32 as imm5=0, so only narrow #1..#31;
        * #0 (no shift) keeps the .W form whose imm5=0 means #0 (a move). */
       u32 sh = (u32)b.v.imm & 31u;
       if (b_imm && sh != 0u && arm_low_reg(rd) && arm_low_reg(ra))
         arm_emit_t16(mc, arm_lsrs_imm16(rd, ra, sh));
       else if (b_imm)
         arm_emit_t32(mc, arm_shift_imm(1u, rd, ra, sh));
       else
         arm_emit_t32(mc, arm_shift_reg(1u, rd, ra, rb));
       return;
     }
     case BO_SHR_S: {
       u32 sh = (u32)b.v.imm & 31u; /* ASRS: same #32-as-0 caveat as LSRS. */
       if (b_imm && sh != 0u && arm_low_reg(rd) && arm_low_reg(ra))
         arm_emit_t16(mc, arm_asrs_imm16(rd, ra, sh));
       else if (b_imm)
         arm_emit_t32(mc, arm_shift_imm(2u, rd, ra, sh));
       else
         arm_emit_t32(mc, arm_shift_reg(2u, rd, ra, rb));
       return;
     }
     default:
       arm_panic(a, "binop not lowered (FP / wide8 is Phase 2)");
   }
 }
 
 static void arm_unop(NativeTarget* t, UnOp op, NativeLoc dst, NativeLoc src) {
   Arm32NativeTarget* a = arm_of(t);
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst), rs = loc_reg(src);
   u32 enc;
   switch (op) {
     case UO_NEG:
       thumb_expand_imm_encode(0u, &enc);
       arm_emit_t32(mc, arm_dp_imm(14u, 0u, rd, rs, enc)); /* RSB rd, rs, #0 */
       return;
     case UO_BNOT:
       arm_emit_t32(mc, arm_mvn_reg(rd, rs));
       return;
     case UO_NOT: {
       /* Logical not -> (rs == 0) materialized as 0/1: CMP rs,#0; MOV rd,#0;
        * IT EQ; MOV<eq> rd,#1. CMP precedes the MOV (rd may alias rs). */
       u32 enc0, enc1;
       thumb_expand_imm_encode(0u, &enc0);
       thumb_expand_imm_encode(1u, &enc1);
       arm_emit_t32(mc, arm_cmp_imm(rs, enc0));
       arm_emit_t32(mc, arm_mov_imm(rd, enc0));
       arm_emit_t16(mc, (u16)(0xbf08u | (ARM_CC_EQ << 4))); /* IT EQ */
       arm_emit_t32(mc, arm_mov_imm(rd, enc1));
       return;
     }
     default:
       arm_panic(a, "unop not lowered (FNEG is a soft-float follow-on)");
   }
 }
 
 /* Map a CmpOp to its ARM condition code (taken-when-true). */
 static u32 arm_cond_for(Arm32NativeTarget* a, CmpOp op) {
   switch (op) {
     case CMP_EQ:   return ARM_CC_EQ;
     case CMP_NE:   return ARM_CC_NE;
     case CMP_LT_S: return ARM_CC_LT;
     case CMP_LE_S: return ARM_CC_LE;
     case CMP_GT_S: return ARM_CC_GT;
     case CMP_GE_S: return ARM_CC_GE;
     case CMP_LT_U: return ARM_CC_CC;
     case CMP_LE_U: return ARM_CC_LS;
     case CMP_GT_U: return ARM_CC_HI;
     case CMP_GE_U: return ARM_CC_CS;
     default:
       arm_panic(a, "FP compare not lowered (Phase 2)");
   }
 }
 
 static void arm_emit_cmp_operands(Arm32NativeTarget* a, NativeLoc x,
                                   NativeLoc y) {
   MCEmitter* mc = a->base.mc;
   u32 enc;
   if (y.kind == NATIVE_LOC_IMM && thumb_expand_imm_encode((u32)y.v.imm, &enc))
     arm_emit_t32(mc, arm_cmp_imm(loc_reg(x), enc));
   else
     arm_emit_t32(mc, arm_cmp_reg(loc_reg(x), loc_reg(y)));
 }
 
 /* Materialize the boolean (a OP b) into dst: CMP; MOV dst,#0; IT cc; MOV dst,#1.
  * The CMP must precede the `MOV dst,#0`: dst can ALIAS a compare operand (e.g. a
  * soft-float compare leaves its libcall result in r0 and the bool destination is
  * also r0), and a leading `MOV dst,#0` would clobber that operand before CMP
  * reads it. MOV.W #imm (T2, S=0) does not set flags, so emitting it after CMP
  * leaves the compare flags intact for the IT-predicated `MOV dst,#1`. */
 static void arm_cmp(NativeTarget* t, CmpOp op, NativeLoc dst, NativeLoc a_loc,
                     NativeLoc b) {
   Arm32NativeTarget* a = arm_of(t);
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst);
   u32 cc = arm_cond_for(a, op);
   u32 enc0, enc1;
   thumb_expand_imm_encode(0u, &enc0);
   thumb_expand_imm_encode(1u, &enc1);
   arm_emit_cmp_operands(a, a_loc, b);          /* CMP a, b (sets flags) */
   arm_emit_t32(mc, arm_mov_imm(rd, enc0));    /* dst = 0 (MOV.W, no flag set) */
   arm_emit_t16(mc, (u16)(0xbf08u | (cc << 4))); /* IT cc (1 insn, mask 0b1000) */
   arm_emit_t32(mc, arm_mov_imm(rd, enc1));    /* MOV<cc> dst, #1 */
 }
 
 static void arm_convert(NativeTarget* t, ConvKind op, NativeLoc dst,
                         NativeLoc src) {
   Arm32NativeTarget* a = arm_of(t);
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst), rs = loc_reg(src);
   u32 dsz = native_type_size(t, dst.type);
   u32 ssz = native_type_size(t, src.type);
   switch (op) {
     case CV_SEXT:
       if (ssz == 1u) arm_emit_t32(mc, arm_sxtb(rd, rs));
       else if (ssz == 2u) arm_emit_t32(mc, arm_sxth(rd, rs));
       else if (rd != rs) arm_emit_t16(mc, arm_mov_hi(rd, rs));
       return;
     case CV_ZEXT:
       if (ssz == 1u) arm_emit_t32(mc, arm_uxtb(rd, rs));
       else if (ssz == 2u) arm_emit_t32(mc, arm_uxth(rd, rs));
       else if (rd != rs) arm_emit_t16(mc, arm_mov_hi(rd, rs));
       return;
     case CV_TRUNC:
       if (dsz == 1u) arm_emit_t32(mc, arm_uxtb(rd, rs));
       else if (dsz == 2u) arm_emit_t32(mc, arm_uxth(rd, rs));
       else if (rd != rs) arm_emit_t16(mc, arm_mov_hi(rd, rs));
       return;
     case CV_BITCAST:
       if (rd != rs) arm_emit_t16(mc, arm_mov_hi(rd, rs));
       return;
     default:
       arm_panic(a, "convert not lowered (FP conversions are Phase 2)");
   }
 }
 
 /* ============================ _rr narrow hooks ============================ */
 
 static void arm_binop_rr(NativeTarget* t, BinOp op, NativeRegLoc dst,
                          NativeRegLoc a, NativeRegLoc b) {
   arm_binop(t, op, native_loc_from_reg(dst), native_loc_from_reg(a),
             native_loc_from_reg(b));
 }
 static void arm_move_rr(NativeTarget* t, NativeRegLoc dst, NativeRegLoc src) {
   arm_move(t, native_loc_from_reg(dst), native_loc_from_reg(src));
 }
 static void arm_cmp_rr(NativeTarget* t, CmpOp op, NativeRegLoc dst,
                        NativeRegLoc a, NativeRegLoc b) {
   arm_cmp(t, op, native_loc_from_reg(dst), native_loc_from_reg(a),
           native_loc_from_reg(b));
 }
 static void arm_convert_rr(NativeTarget* t, ConvKind op, NativeRegLoc dst,
                            NativeRegLoc src) {
   arm_convert(t, op, native_loc_from_reg(dst), native_loc_from_reg(src));
 }
 
 /* ============================ control flow ============================ */
 
 static MCLabel arm_label_new(NativeTarget* t) { return mc_label_new(t->mc); }
 static void arm_label_place(NativeTarget* t, MCLabel l) {
   mc_label_place(t->mc, l);
 }
 static void arm_jump(NativeTarget* t, MCLabel l) {
   arm_emit_t32(t->mc, arm_b_w());
   mc_emit_label_ref(t->mc, l, R_ARM_THM_JUMP24, 4, 0);
 }
 static void arm_cmp_branch(NativeTarget* t, CmpOp op, NativeLoc a_loc,
                            NativeLoc b, MCLabel target) {
   Arm32NativeTarget* a = arm_of(t);
   u32 cc = arm_cond_for(a, op);
   arm_emit_cmp_operands(a, a_loc, b);
   arm_emit_t32(t->mc, arm_b_cond_w(cc));
   mc_emit_label_ref(t->mc, target, R_ARM_THM_JUMP19, 4, 0);
 }
 
 /* ============================ frame lifecycle ============================ */
 
 static u32 arm_signature_stack_bytes(NativeTarget* t, KitCgTypeId fn_type,
                                      int* variadic, u32* nparams);
 
 static NativeFrameSlot arm_frame_slot(NativeTarget* t,
                                       const NativeFrameSlotDesc* d) {
   return native_frame_slot_alloc(&arm_of(t)->frame, d);
 }
 static void arm_release_frame_slot(NativeTarget* t, NativeFrameSlot slot) {
   native_frame_release_slot(&arm_of(t)->frame, slot);
 }
 static int arm_frame_slot_debug_loc(NativeTarget* t, NativeFrameSlot slot,
                                     CGDebugLoc* out) {
   Arm32NativeTarget* a = arm_of(t);
   NativeFrameSlotEntry* s;
   if (!out) return 0;
   memset(out, 0, sizeof *out);
   if (slot == NATIVE_FRAME_SLOT_NONE || slot > a->frame.nslots) return 0;
   s = native_frame_slot_at(&a->frame, slot);
   out->kind = CG_DEBUG_LOC_FRAME;
   out->v.frame_ofs = -(i32)s->off; /* r7-relative */
   return 1;
 }
 
 static u32 arm_frame_size(const Arm32NativeTarget* a) {
   u32 f = align_up_u32(a->frame.cum_off + a->frame.max_outgoing, 8u);
   /* AAPCS: sp must be 8-byte aligned at public interfaces (calls). The total
    * adjustment from entry sp is saved_block_bytes + (variadic ? 16 : 0) + f; the
    * 16-byte GP-save area is 8-aligned, so only an odd-word saved-register block
    * (e.g. {r4,r7,lr} = 12) breaks alignment — pad the frame by a word then. */
   if (a->saved_block_bytes & 4u) f += 4u;
   return f;
 }
 
 /* r7-relative offset where the incoming stack-argument window begins. The saved
  * {r7, lr} pair is 8 bytes; a variadic callee also spilled r0..r3 (16 bytes) in
  * a save area above the pair, pushing the named stack args 16 bytes higher. The
  * spilled r0..r3 occupy [r7 + ARM_SAVED_PAIR_BYTES, r7 + arg_window_base). */
 static u32 arm_arg_window_base(const Arm32NativeTarget* a) {
   /* saved_block_bytes covers the saved-register block ({r7,lr} = 8 at -O0, plus
    * any callee-saves the -O1 known-frame prologue PUSHes); a variadic callee
    * also spilled r0..r3 (16 bytes) just above it. Both push the named incoming
    * stack args higher, so the window base accounts for them together. */
   return a->saved_block_bytes + (a->is_variadic ? ARM_VA_GP_SAVE_BYTES : 0u);
 }
 
 static void arm_reserve_callee_saves(NativeTarget* t, const u32* used_by_class,
                                      u32 nclasses) {
   native_frame_set_callee_saves(&arm_of(t)->frame, used_by_class, nclasses,
                                 NULL, 0, 0);
 }
 
 /* AAPCS callee-saved integer registers this backend lets the allocator use:
  * r4..r11 except r7 (the frame pointer, preserved by the saved-pair head, not as
  * an ordinary callee-save). Matches arm_int_phys's ARM_PHYS_CALLEE set. */
 static int arm_reg_is_callee_int(Reg r) {
   return r >= 4u && r <= 11u && r != ARM_FP;
 }
 
 static void arm_asm_clobber_masks(Compiler* c, SrcLoc loc,
                                   const Sym* clobbers, u32 nclob,
                                   u32* int_mask, u32* fp_mask);
 
 /* Build the callee-saved set the prologue must preserve, from the allocator's
  * per-class masks plus opaque inline-asm clobbers. Only the integer class
  * exists in this soft-float backend. r7 is structural frame state and is
  * excluded here (and rejected when named explicitly by the clobber parser). */
 static int arm_asm_reg_is_callee_saved(NativeTarget* t, NativeAllocClass cls,
                                        Reg r) {
   (void)t;
   return cls == NATIVE_REG_INT && arm_reg_is_callee_int(r);
 }
 
 static u32 arm_known_callee_saves(NativeTarget* t,
                                   const NativeKnownFrameDesc* frame) {
   Arm32NativeTarget* a = arm_of(t);
   u32 masks[NATIVE_REG_CLASS_COUNT] = {0, 0, 0};
   SrcLoc loc = a->func ? a->func->loc : (SrcLoc){0, 0, 0};
   native_asm_known_callee_saves(t, loc, frame, masks,
                                 NATIVE_REG_CLASS_COUNT,
                                 arm_asm_clobber_masks,
                                 arm_asm_reg_is_callee_saved);
   return masks[NATIVE_REG_INT];
 }
 
 /* True (and sets *imm7) when `n` fits the 16-bit `SUB sp,sp,#imm7*4` form:
  * a non-zero multiple of 4 with imm7 in 0..127 (n <= 508). The frame size is a
  * multiple of 8, so the multiple-of-4 requirement is always met. */
 static int arm_sp_adjust_narrow(u32 n, u32* imm7) {
   if (n == 0 || (n & 3u) != 0u || (n >> 2) > 0x7fu) return 0;
   *imm7 = n >> 2;
   return 1;
 }
 
 /* Build the `SUB sp, sp, #n` instruction sequence (n a multiple of 8) into
  * words[], returning the count (<= ARM_NDT_SUB_WORDS). A small frame fits the
  * 16-bit `SUB sp,sp,#imm7*4`, packed (SUB16 in hw1, NOP16 in hw2) into one T32
  * slot so the single-pass deferred patch still overwrites exactly one reserved
  * word; medium amounts use one SUBW/SUB.W modimm; large amounts (n > 4095, big
  * frames) stage n into IP via MOVW(+MOVT) and `SUB sp, sp, ip`. IP is the emit
  * scratch, dead at the prologue. Shared by the inline known-frame emit and the
  * single-pass patch. */
 static u32 arm_build_sub_sp(u32 n, u32* words) {
   u32 enc, imm7, wi = 0;
   if (n == 0) return 0;
   if (arm_sp_adjust_narrow(n, &imm7)) {
     /* hw1 = SUB sp (16-bit), hw2 = NOP (16-bit). arm_patch_t32 writes hw1 first
      * in memory, so SUB executes then NOP. */
     words[wi++] = arm_t32(arm_sub_sp_sp_imm16(imm7), arm_nop16());
     return wi;
   }
   if (n <= 0xfffu) {
     words[wi++] = arm_sub_imm12(13u, 13u, n);
     return wi;
   }
   if (thumb_expand_imm_encode(n, &enc)) {
     words[wi++] = arm_dp_imm(13u, 0u, 13u, 13u, enc); /* SUB.W sp,sp,#modimm */
     return wi;
   }
   words[wi++] = arm_movw(ARM_SCRATCH, n & 0xffffu);
   if ((n >> 16) != 0u) words[wi++] = arm_movt(ARM_SCRATCH, (n >> 16) & 0xffffu);
   words[wi++] = arm_sub_reg(13u, 13u, ARM_SCRATCH);
   return wi;
 }
 
 /* Emit `SUB sp, sp, #n` inline (known-frame prologue, where the size is final).
  * The inline path is not slot-constrained, so a small frame emits a bare 16-bit
  * SUB (no NOP padding); larger frames reuse arm_build_sub_sp's word list. */
 static void arm_emit_sub_sp(Arm32NativeTarget* a, u32 n) {
   u32 words[ARM_NDT_SUB_WORDS];
   u32 imm7, nwords, i;
   if (arm_sp_adjust_narrow(n, &imm7)) {
     arm_emit_t16(a->base.mc, arm_sub_sp_sp_imm16(imm7));
     return;
   }
   nwords = arm_build_sub_sp(n, words);
   for (i = 0; i < nwords; ++i) arm_emit_t32(a->base.mc, words[i]);
 }
 
 /* sret: reserve a hidden frame slot for the incoming destination pointer (r0)
  * and advance the param cursor past it. Shared by the single-pass and
  * known-frame paths (slot creation must happen at the same point in both). */
 static void arm_reserve_sret_slot(Arm32NativeTarget* a) {
   NativeTarget* t = &a->base;
   if (a->has_sret) {
     NativeFrameSlotDesc sd;
     memset(&sd, 0, sizeof sd);
     sd.type = builtin_id(KIT_CG_BUILTIN_I32);
     sd.size = 4;
     sd.align = 4;
     sd.kind = NATIVE_FRAME_SLOT_SAVE;
     a->sret_ptr_slot = arm_frame_slot(t, &sd);
     a->next_param_int = 1; /* r0 consumed by the sret pointer */
   }
 }
 
 /* Lazily reserve the per-function indirect-callee spill home (4 bytes, r7-
  * anchored). Allocated on first need during body codegen — at -O0 the frame is
  * deferred (settled in arm_func_end), so a body-time slot just grows cum_off;
  * one slot is reused across every indirect call in the function. */
 static NativeFrameSlot arm_callee_spill_slot(Arm32NativeTarget* a) {
   if (a->callee_spill_slot == NATIVE_FRAME_SLOT_NONE) {
     NativeFrameSlotDesc sd;
     memset(&sd, 0, sizeof sd);
     sd.type = builtin_id(KIT_CG_BUILTIN_I32);
     sd.size = 4;
     sd.align = 4;
     sd.kind = NATIVE_FRAME_SLOT_SAVE;
     a->callee_spill_slot = arm_frame_slot(&a->base, &sd);
   }
   return a->callee_spill_slot;
 }
 
 /* Spill the incoming sret pointer (r0) to its hidden home. The home is
  * r7-anchored, so this runs correctly after the frame is set up. */
 static void arm_emit_sret_store(Arm32NativeTarget* a) {
   NativeTarget* t = &a->base;
   if (a->has_sret && a->sret_ptr_slot != NATIVE_FRAME_SLOT_NONE) {
     KitCgTypeId i32t = builtin_id(KIT_CG_BUILTIN_I32);
     NativeAddr addr;
     memset(&addr, 0, sizeof addr);
     addr.base_kind = NATIVE_ADDR_BASE_FRAME;
     addr.base.frame = a->sret_ptr_slot;
     addr.base_type = i32t;
     arm_emit_mem(a, 0, native_loc_reg(i32t, NATIVE_REG_INT, ARM_R0), addr,
                  native_mem_for_type(t, i32t, 4));
   }
 }
 
 static void arm_func_begin(NativeTarget* t, const CGFuncDesc* fd) {
   Arm32NativeTarget* a = arm_of(t);
   MCEmitter* mc = t->mc;
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, fd->fn_type);
   a->func = fd;
   a->loc = fd->loc;
   native_frame_reset(&a->frame);
   a->next_param_int = 0;
   a->next_param_stack = 0;
   a->has_sret = (abi && abi->has_sret) ? 1u : 0u;
   a->is_variadic = (abi && abi->variadic) ? 1u : 0u;
   a->sret_ptr_slot = NATIVE_FRAME_SLOT_NONE;
   a->callee_spill_slot = NATIVE_FRAME_SLOT_NONE;
   a->known_frame = 0;
   a->slim_prologue = 0;
   a->saved_reglist = (u16)((1u << ARM_FP) | (1u << 14u)); /* {r7, lr} */
   a->saved_block_bytes = 8u;                              /* two words */
   a->n_alloca_patch = 0;
   a->incoming_stack_size = arm_signature_stack_bytes(t, fd->fn_type, NULL, NULL);
 
   mc_set_section(mc, fd->text_section_id);
   mc_emit_align(mc, 4, 0);
   a->func_start = mc_pos(mc);
   mc_begin_function(mc, fd->sym, fd->text_section_id, a->func_start);
   mc_cfi_startproc(mc);
   a->epilogue_label = mc_label_new(mc);
 
   /* Prologue: save fp+lr, set the frame anchor, reserve the (deferred) frame.
    * The deferred `SUB sp` is patched in arm_func_end once cum_off/max_outgoing
    * are final. Reserve ARM_NDT_SUB_WORDS T32 slots so a large frame (> 4095) can
    * be patched as an IP-staged MOVW/MOVT/SUB sequence; a small frame patches the
    * first slot with SUBW and fills the rest with NOPs.
    * A variadic callee first spills the GP arg registers r0..r3 into a save area
    * just above the saved pair (so they are contiguous-below the named incoming
    * stack args at [r7 + 8 + 16]); va_start/va_arg walk forward across both. */
   if (a->is_variadic)
     arm_emit_push(mc, 0xfu);                        /* PUSH {r0-r3} (16-bit) */
   arm_emit_push(mc, (1u << ARM_FP) | (1u << 14u)); /* PUSH {r7, lr} (16-bit) */
   arm_emit_t16(mc, arm_mov_hi(ARM_FP, 13u));                   /* MOV r7, sp */
   a->prologue_sub_pos = mc_pos(mc);
   for (u32 i = 0; i < ARM_NDT_SUB_WORDS; ++i)
     arm_emit_t32(mc, arm_nop32()); /* placeholder, patched in arm_func_end */
 
   /* sret: spill the incoming destination pointer (r0) to a hidden home. */
   arm_reserve_sret_slot(a);
   arm_emit_sret_store(a);
 }
 
 /* Optimizer (-O1) entry point: the optimizer has run register allocation and
  * supplies the exact frame up front, so the prologue is emitted final the moment
  * it is built — no deferred `SUB sp, #0` placeholder, no arm_func_end patch
  * (arm_func_end skips patching when known_frame). The callee-saved registers the
  * allocator used are folded directly into the PUSH/POP register list rather than
  * given their own frame slots: they ride above the frame anchor r7 in the
  * STMDB-saved block, which keeps slot offsets ([r7, #-off]) byte-identical to
  * the single-pass layout and lets one LDMIA-via-pc restore-and-return.
  *
  * Slot creation order matches the single-pass path: static slots first, then the
  * sret entry-save slot. (Callee-saves take no slots here, so order vs. them is
  * moot.) */
 static void arm_func_begin_known_frame(NativeTarget* t, const CGFuncDesc* fd,
                                        const NativeKnownFrameDesc* kf,
                                        NativeFrameSlot* out_slots) {
   Arm32NativeTarget* a = arm_of(t);
   MCEmitter* mc = t->mc;
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, fd->fn_type);
   u32 cs_mask;
   u32 reglist, nsaved, frame, i;
 
   a->func = fd;
   a->loc = fd->loc;
   cs_mask = arm_known_callee_saves(t, kf);
   native_frame_reset(&a->frame);
   a->next_param_int = 0;
   a->next_param_stack = 0;
   a->has_sret = (abi && abi->has_sret) ? 1u : 0u;
   a->is_variadic = (abi && abi->variadic) ? 1u : 0u;
   a->sret_ptr_slot = NATIVE_FRAME_SLOT_NONE;
   a->callee_spill_slot = NATIVE_FRAME_SLOT_NONE;
   a->known_frame = 1;
   a->slim_prologue = 0;
   a->n_alloca_patch = 0;
   a->incoming_stack_size = arm_signature_stack_bytes(t, fd->fn_type, NULL, NULL);
   a->frame.known_frame = 1;
   a->frame.has_alloca = kf ? kf->has_alloca : 0u;
 
   mc_set_section(mc, fd->text_section_id);
   mc_emit_align(mc, 4, 0);
   a->func_start = mc_pos(mc);
   mc_begin_function(mc, fd->sym, fd->text_section_id, a->func_start);
   mc_cfi_startproc(mc);
   a->epilogue_label = mc_label_new(mc);
 
   /* Record the callee-save set (for parity / debug) and build the body's slots
    * and outgoing-area reservation, then settle the exact frame. */
   if (cs_mask) {
     u32 used[NATIVE_REG_CLASS_COUNT] = {0, 0, 0};
     used[NATIVE_REG_INT] = cs_mask;
     arm_reserve_callee_saves(t, used, NATIVE_REG_CLASS_COUNT);
   }
   if (kf) {
     for (i = 0; i < kf->nslots; ++i) {
       NativeFrameSlot slot = arm_frame_slot(t, &kf->slots[i]);
       if (out_slots) out_slots[i] = slot;
     }
     arm_reserve_sret_slot(a);
     native_frame_note_outgoing(&a->frame, kf->max_outgoing);
   } else {
     arm_reserve_sret_slot(a);
   }
 
   /* The PUSH/POP register list: callee-saves (low to high) + r7 + lr. STMDB
    * stores in ascending register order regardless of the bit order, so r7 lands
    * in the block correctly and `MOV r7, sp` anchors just below the whole block.
    * The popped lr is restored into pc, returning. */
   reglist = cs_mask | (1u << ARM_FP) | (1u << 14u);
   nsaved = 0;
   for (i = 0; i < 16u; ++i)
     if (reglist & (1u << i)) nsaved++;
   a->saved_reglist = (u16)reglist;
   a->saved_block_bytes = nsaved * 4u;
 
   /* Leaf minimal-frame tier (the register-starved analog of the aa64/rv64
    * frameless leaf): a leaf with no callee-saves, no body slots, no outgoing
    * args, no sret/variadic and only register params needs no frame record, no r7
    * anchor and no SUB sp. It cannot, however, return through a bare `BX lr`:
    * arm32 is register-starved and uses LR (ARM_TMP) pervasively as a
    * backend-private temporary — global-address staging in arm_resolve_mem,
    * `__builtin_*_overflow` sequences, popcount, bitfield insert, and the
    * byte-copy granule engine. A leaf body may hit any of these (e.g.
    * `static int g; return g;` stages &g into LR,
    * after which a bare `BX lr` would branch to &g). So the tier still saves and
    * restores the return address with a single-register `PUSH {lr}` / `POP {pc}`,
    * leaving the body free to clobber lr; it just skips the r7 frame record and
    * the stack reservation. (aa64/rv64 keep their return-address register out of
    * the backend temporary bank, so their frameless leaves skip even this push; arm32
    * cannot.) Inline asm cannot occur here (asm_block is not installed). */
   a->slim_prologue = kf && kf->is_leaf && !kf->reads_frame && cs_mask == 0 &&
                      !a->frame.has_alloca && a->frame.cum_off == 0 &&
                      a->frame.max_outgoing == 0 && !a->has_sret &&
                      !a->is_variadic &&
                      arm_signature_stack_bytes(t, fd->fn_type, NULL, NULL) == 0;
   if (a->slim_prologue) {
     a->saved_reglist = (u16)(1u << 14u); /* {lr} */
     a->saved_block_bytes = 4u;
     arm_emit_push(mc, 1u << 14u); /* PUSH {lr} (16-bit, 0xb500) */
     native_frame_set_final(&a->frame);
     return;
   }
 
   /* Emit the final prologue as a fp-at-pair split push: the callee-saves ride
    * ABOVE a tight {r7, lr} frame-record pair so r7 anchors exactly that pair —
    * [r7]=caller's r7, [r7+4]=return address — which @frame_address/@return_address
    * and any frame-pointer backtrace walk. (A single combined PUSH would leave r7
    * pointing at the lowest callee-save, with the return address an arbitrary
    * distance above, breaking the chain whenever r8..r11 sit between r7 and lr.)
    * Push order, high address to low: variadic GP-save (r0..r3) first, then the
    * callee-saves, then the pair; the total block size is unchanged, so
    * saved_block_bytes and every [r7, #-off] body slot keep their offsets, and
    * va_start still reaches the GP-save at r7 + saved_block_bytes.
    * A variadic callee's r0..r3 save area sits just above the saved block (below
    * the named incoming stack args); va_start/va_arg walk forward across both, and
    * the teardown reclaims it (arm_emit_frame_restore). */
   if (a->is_variadic) arm_emit_push(mc, 0xfu); /* PUSH {r0-r3} */
   if (cs_mask) arm_emit_push(mc, cs_mask);     /* PUSH {callee-saves} */
   arm_emit_push(mc, (1u << ARM_FP) | (1u << 14u)); /* PUSH {r7, lr} */
   arm_emit_t16(mc, arm_mov_hi(ARM_FP, 13u));                  /* MOV r7, sp */
   frame = arm_frame_size(a);
   arm_emit_sub_sp(a, frame);
 
   arm_emit_sret_store(a);
   native_frame_set_final(&a->frame);
 }
 
 /* Emit the frame teardown, mirroring the fp-at-pair split prologue: r7 anchors
  * the {r7, lr} pair at the bottom of the saved block, the callee-saves sit above
  * it, and (for a variadic callee) the r0..r3 GP-save above those.
  *
  * The common case — no callee-saves, no GP-save, returning — peels the pair with
  * a single `POP {r7, pc}` (the saved lr lands straight in pc). Otherwise restore
  * the pair with `POP {r7, lr}`, then peel the callee-saves and reclaim the
  * GP-save, leaving lr = the caller's return address — popped into pc with a final
  * `BX lr` when returning, or kept for a tail-call branch when not. */
 static void arm_emit_frame_restore(Arm32NativeTarget* a, int to_pc) {
   MCEmitter* mc = a->base.mc;
   u32 va = a->is_variadic ? ARM_VA_GP_SAVE_BYTES : 0u;
   u32 cs_mask = a->saved_reglist & ~((1u << ARM_FP) | (1u << 14u));
   arm_emit_t16(mc, arm_mov_hi(13u, ARM_FP)); /* MOV sp, r7 (sp -> {r7,lr} pair) */
   if (to_pc && va == 0u && cs_mask == 0u) {
     arm_emit_pop(mc, (1u << ARM_FP) | (1u << 15u)); /* POP {r7, pc} (16-bit) */
     return;
   }
   arm_emit_pop(mc, (1u << ARM_FP) | (1u << 14u)); /* POP {r7, lr} (16-bit) */
   if (cs_mask) arm_emit_pop(mc, cs_mask);         /* POP {callee-saves} */
   if (va) arm_emit_add_const(a, 13u, (i64)va);              /* reclaim r0..r3 save */
   if (to_pc) arm_emit_t16(mc, arm_bx(14u));                 /* BX lr (return) */
 }
 
 static void arm_func_end(NativeTarget* t) {
   Arm32NativeTarget* a = arm_of(t);
   MCEmitter* mc = t->mc;
   u32 frame;
   /* Place the epilogue and emit the teardown. */
   mc_label_place(mc, a->epilogue_label);
   if (a->slim_prologue)
     /* Minimal leaf: only lr was saved (PUSH {lr}); pop it straight into pc. This
      * restores the return address the body may have clobbered using LR as a
      * backend-private temporary — see the func_begin slim_prologue note. */
     arm_emit_pop(mc, 1u << 15u); /* POP {pc} (16-bit, 0xbd00) */
   else
     arm_emit_frame_restore(a, /*to_pc=*/1);
 
   /* Single-pass path: settle the frame and patch the deferred `SUB sp` region.
    * The known-frame path emitted its prologue final in
    * arm_func_begin_known_frame (frame already settled there), so it never
    * patches. The reserved region is ARM_NDT_SUB_WORDS NOP.W slots; patch the
    * leading slots with the built SUB sequence (one SUBW for a small frame, an
    * IP-staged MOVW/MOVT/SUB for a large one) and leave the trailing NOP.W slots,
    * which execute harmlessly. */
   if (!a->known_frame) {
     u32 words[ARM_NDT_SUB_WORDS];
     u32 nwords, i;
     native_frame_set_final(&a->frame);
     frame = arm_frame_size(a);
     nwords = arm_build_sub_sp(frame, words);
     for (i = 0; i < nwords; ++i)
       arm_patch_t32(a, a->prologue_sub_pos + i * 4u, words[i]);
   }
 
   /* Patch each alloca's `ADD dst, sp, #max_outgoing` now that the outgoing-arg
    * area size is final (the alloca block sits just above it, so a later call's
    * sp-relative outgoing stores never clobber it). */
   {
     u32 i, mo = a->frame.max_outgoing;
     if (a->n_alloca_patch && mo > 0xfffu)
       arm_panic(a, "alloca with a large outgoing-arg area (needs IP staging)");
     for (i = 0; i < a->n_alloca_patch; ++i)
       arm_patch_t32(a, a->alloca_patch_pos[i],
                     arm_add_imm12(a->alloca_patch_rd[i], 13u, mo));
   }
 
   /* Publish the function symbol. ARM marks Thumb STT_FUNC symbols with the
    * low bit set (the Thumb bit), so &fn and indirect BLX reach Thumb state;
    * direct-branch relocs mask it off (S & ~1). */
   {
     u32 end = mc_pos(mc);
     ObjSecId sec = a->func->text_section_id;
     obj_symbol_define(t->obj, a->func->sym, sec, (u64)a->func_start | 1u,
                       (u64)(end - a->func_start));
     if (a->func->atomize)
       obj_atom_define(t->obj, sec, a->func_start, end - a->func_start,
                       a->func->sym, 0);
     if (mc->debug) debug_func_pc_range(mc->debug, sec, a->func_start, end);
   }
 
   mc_cfi_endproc(mc);
   mc_end_function(mc);
   a->func = NULL;
 }
 
 /* ============================ param binding ============================ */
 
 #define ARM32_GPR_BYTES 4u
 
 /* Lane scalar type for an ABI part: an 8-byte i64/double DIRECT argument is two
  * 4-byte INT lanes, and an aggregate uses word/sub-word lanes. The NDT sizes
  * every load/store/move it emits for return/call marshalling or a param bind
  * from the lane location's TYPE (nd_type_mem -> cg_type_size), so each lane must
  * carry a type whose natural size equals the lane's byte width — NOT the full
  * 8-byte scalar type (which would drive an 8-byte access the 32-bit backend has
  * no single-instruction form for). Mirrors riscv's rv_part_scalar_type. */
 static KitCgTypeId arm_part_scalar_type(const ABIArgPart* part) {
   switch (part->size) {
     case 1u: return builtin_id(KIT_CG_BUILTIN_I8);
     case 2u: return builtin_id(KIT_CG_BUILTIN_I16);
     case 8u: return builtin_id(KIT_CG_BUILTIN_I64);
     default: return builtin_id(KIT_CG_BUILTIN_I32);
   }
 }
 
 /* AAPCS even/8-align rule: an argument whose natural alignment is 8 bytes
  * rounds NCRN up to an even core register (r0:r1 or r2:r3) and 8-byte-aligns its
  * stack slot. This covers both an i64/double scalar pair AND an 8-byte-aligned
  * aggregate (e.g. struct{double;long;}). The ABI classifier surfaces that
  * alignment on the first part (scalar pair lanes carry align=8; an aggregate's
  * parts[0].align is the record's natural alignment; the synthesized variadic
  * form below mirrors this), so parts[0].align is the single discriminator —
  * uniform for named and variadic args and independent of part count.
  *
  * This predicate is ONLY the even/8-align rule. Whether an argument may split
  * across the reg/stack boundary is not a separate flag: round-to-even makes an
  * 8-byte scalar pair fit-or-fully-spill (its 2 parts never straddle), while a
  * larger aggregate straddles per AAPCS C.5 — the naive per-part walks below
  * realize both with no special case. */
 static int arm_arg_needs_even(const ABIArgInfo* ai) {
   return ai->kind == ABI_ARG_DIRECT && ai->nparts > 0u && ai->parts &&
          ai->parts[0].align >= 2u * ARM32_GPR_BYTES;
 }
 
 static void arm_bind_native_param(NativeTarget* t, const CGParamDesc* p,
                                   NativeLoc dst) {
   Arm32NativeTarget* a = arm_of(t);
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, a->func->fn_type);
   const ABIArgInfo* ai = p->index < abi->nparams ? &abi->params[p->index] : NULL;
   int to_reg = dst.kind == NATIVE_LOC_REG;
   int even = ai && arm_arg_needs_even(ai);
   u32 i;
   if (!ai || ai->kind == ABI_ARG_IGNORE) return;
   if (ai->kind == ABI_ARG_INDIRECT) {
     /* An ABI-classified indirect argument is a pointer to caller-owned storage.
      * Copy it into the parameter's frame home so the body owns a private copy
      * (matching the by-value semantics). */
     KitCgTypeId i32t = builtin_id(KIT_CG_BUILTIN_I32);
     NativeLoc src = native_loc_reg(i32t, NATIVE_REG_INT,
                                    a->next_param_int < 4u
                                        ? (Reg)a->next_param_int
                                        : ARM_SCRATCH);
     NativeAddr d_addr, from;
     AggregateAccess access;
     if (a->next_param_int < 4u) {
       a->next_param_int++;
     } else {
       NativeAddr sa;
       memset(&sa, 0, sizeof sa);
       sa.base_kind = NATIVE_ADDR_BASE_REG;
       sa.base.reg = ARM_FP;
       sa.base_type = i32t;
       sa.offset = (i32)(arm_arg_window_base(a) + a->next_param_stack);
       arm_emit_mem(a, 1, src, sa, native_mem_for_type(t, i32t, 4));
       a->next_param_stack += 4u;
     }
     if (dst.kind != NATIVE_LOC_FRAME)
       arm_panic(a, "indirect parameter requires a frame destination");
     memset(&d_addr, 0, sizeof d_addr);
     d_addr.base_kind = NATIVE_ADDR_BASE_FRAME;
     d_addr.base.frame = dst.v.frame;
     d_addr.base_type = p->type;
     memset(&from, 0, sizeof from);
     from.base_kind = NATIVE_ADDR_BASE_REG;
     from.base.reg = loc_reg(src);
     from.base_type = p->type;
     memset(&access, 0, sizeof access);
     access.type = p->type;
     access.size = p->size ? p->size : (u32)cg_type_size(t->c, p->type);
     access.align = p->align ? p->align : native_type_align(t, p->type);
     arm_copy_bytes(t, d_addr, from, access);
     return;
   }
   /* AAPCS: round NCRN up to even before an 8-byte-aligned argument. The parts
    * then fill core registers r0..r3; any that do not fit spill to the (8-byte-
    * aligned for part 0) incoming stack window. An 8-byte scalar pair fully fits
    * or fully spills (round-to-even guarantees it); a larger aggregate straddles
    * the reg/stack boundary (AAPCS C.5). */
   if (even) a->next_param_int = align_up_u32(a->next_param_int, 2u);
   for (i = 0; i < ai->nparts; ++i) {
     const ABIArgPart* part = &ai->parts[i];
     KitCgTypeId lty = arm_part_scalar_type(part);
     NativeLoc src;
     if (a->next_param_int < 4u) {
       src = native_loc_reg(lty, NATIVE_REG_INT, (Reg)(a->next_param_int++));
     } else {
       /* Incoming stack arg: above the saved register block = [r7 + N + k],
        * where N = saved_block_bytes (8 for the bare {r7,lr} -O0 prologue; larger
        * when the -O1 known-frame prologue also PUSHes callee-saves). The first
        * stack part of an 8-byte-aligned arg lands on an 8-byte boundary. */
       NativeAddr sa;
       if (i == 0 && even)
         a->next_param_stack =
             align_up_u32(a->next_param_stack, 2u * ARM32_GPR_BYTES);
       src = native_loc_reg(lty, NATIVE_REG_INT, ARM_SCRATCH);
       memset(&sa, 0, sizeof sa);
       sa.base_kind = NATIVE_ADDR_BASE_REG;
       sa.base.reg = ARM_FP;
       sa.base_type = lty;
       sa.offset = (i32)(arm_arg_window_base(a) + a->next_param_stack);
       arm_emit_mem(a, 1, src, sa, native_mem_for_type(t, lty, part->size));
       a->next_param_stack += 4u;
     }
     if (dst.kind == NATIVE_LOC_NONE) {
       /* unused parameter; cursor already advanced */
     } else if (to_reg) {
       NativeLoc d = native_loc_reg(dst.type ? dst.type : p->type,
                                    (NativeAllocClass)dst.cls, (Reg)dst.v.reg);
       if (!(src.kind == NATIVE_LOC_REG && loc_reg(src) == loc_reg(d)))
         arm_move(t, d, src);
     } else {
       NativeAddr home;
       memset(&home, 0, sizeof home);
       home.base_kind = NATIVE_ADDR_BASE_FRAME;
       home.base.frame = dst.v.frame;
       home.base_type = lty;
       home.offset = (i32)part->src_offset;
       arm_emit_mem(a, 0, src, home, native_mem_for_type(t, lty, part->size));
     }
   }
 }
 
 /* ============================ calls / returns ============================ */
 
 /* Advance the AAPCS argument-placement cursor across ONE argument `ai`, updating
  * the core-register count *next_int (r0..r3) and the outgoing-stack byte count
  * *stack. The single authority for the pair rule, shared by the signature
  * stack-size accounting so it cannot drift from emission:
  *   - an 8-byte aligned pair rounds NCRN up to even and, if it does not fully
  *     fit in r0..r3, passes ENTIRELY on the 8-byte-aligned stack (no straddle);
  *   - every other part takes the next core register, else a 4-byte stack slot. */
 static void arm_arg_advance(const ABIArgInfo* ai, u32* next_int, u32* stack) {
   int even = arm_arg_needs_even(ai);
   u32 p;
   if (ai->kind == ABI_ARG_IGNORE) return;
   if (ai->kind == ABI_ARG_INDIRECT) {
     if (*next_int < 4u) (*next_int)++;
     else *stack += 4u;
     return;
   }
   if (even) *next_int = align_up_u32(*next_int, 2u);
   for (p = 0; p < ai->nparts; ++p) {
     if (p == 0 && even && *next_int >= 4u)
       *stack = align_up_u32(*stack, 2u * ARM32_GPR_BYTES);
     if (*next_int < 4u) (*next_int)++;
     else *stack += 4u;
   }
 }
 
 /* Resolve the ABI classification of call argument `i`. Named params come from
  * the callee's ABIFuncInfo; unnamed (variadic) args are synthesized here. A
  * scalar wider than one GPR (8-byte i64 / soft double) becomes two 4-byte INT
  * parts (AAPCS32 passes them in an even/odd register pair or 8-byte-aligned on
  * the stack — the even/8-byte alignment is applied by the caller below). */
 static const ABIArgInfo* arm_param_abi(NativeTarget* t, const ABIFuncInfo* abi,
                                        const NativeCallDesc* desc, u32 i,
                                        ABIArgInfo* scratch) {
   u32 sz, align;
   if (abi && i < abi->nparams) return &abi->params[i];
   sz = native_type_size(t, desc->args[i].type);
   align = native_type_align(t, desc->args[i].type);
   memset(scratch, 0, sizeof *scratch);
   scratch->kind = ABI_ARG_DIRECT;
   if (sz > 4u) {
     u32 nparts = (sz + 3u) / 4u, p;
     ABIArgPart* parts = arena_zarray(t->c->tu, ABIArgPart, nparts);
     for (p = 0; p < nparts; ++p) {
       u32 off = p * 4u;
       parts[p].cls = ABI_CLASS_INT;
       parts[p].loc = ABI_LOC_REG;
       parts[p].size = (sz - off) < 4u ? (sz - off) : 4u;
       parts[p].align = 4u;
       parts[p].src_offset = off;
     }
     /* Mirror the named classifier: parts[0].align carries the argument's natural
      * alignment so the AAPCS even/8-align walk treats a synthesized variadic
      * i64/double (or 8-aligned aggregate) the same as a named one. */
     parts[0].align = align;
     scratch->nparts = nparts;
     scratch->parts = parts;
     return scratch;
   }
   scratch->nparts = 1;
   {
     ABIArgPart* part = arena_zarray(t->c->tu, ABIArgPart, 1);
     part->cls = ABI_CLASS_INT;
     part->loc = ABI_LOC_REG;
     part->size = sz;
     part->align = align;
     scratch->parts = part;
   }
   return scratch;
 }
 
 /* Outgoing stack-argument bytes for a call: int parts beyond r0..r3. */
 static u32 arm_call_stack_size(NativeTarget* t, const NativeCallDesc* desc) {
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, desc->fn_type);
   u32 next_int = (abi && abi->has_sret) ? 1u : 0u;
   u32 stack = 0, i, p;
   ABIArgInfo scratch;
   if (!abi) return 0;
   for (i = 0; i < desc->nargs; ++i) {
     const ABIArgInfo* ai = arm_param_abi(t, abi, desc, i, &scratch);
     int even = arm_arg_needs_even(ai);
     if (ai->kind == ABI_ARG_IGNORE) continue;
     if (ai->kind == ABI_ARG_INDIRECT) {
       if (next_int < 4u) next_int++;
       else {
         stack = align_up_u32(stack, 4u);
         stack += 4u;
       }
       continue;
     }
     if (even) next_int = align_up_u32(next_int, 2u); /* even-pair */
     for (p = 0; p < ai->nparts; ++p) {
       if (p == 0 && even && next_int >= 4u) stack = align_up_u32(stack, 8u);
       if (next_int < 4u) next_int++;
       else stack += 4u;
     }
   }
   return align_up_u32(stack, 8u);
 }
 
 static u32 arm_signature_stack_bytes(NativeTarget* t, KitCgTypeId fn_type,
                                      int* variadic, u32* nparams) {
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, fn_type);
   u32 next_int = (abi && abi->has_sret) ? 1u : 0u;
   u32 stack = 0, i;
   if (variadic) *variadic = abi && abi->variadic;
   if (nparams) *nparams = abi ? abi->nparams : 0u;
   if (!abi) return 0;
   for (i = 0; i < abi->nparams; ++i)
     arm_arg_advance(&abi->params[i], &next_int, &stack);
   return align_up_u32(stack, 8u);
 }
 
 static u32 arm_call_stack_bytes(NativeTarget* t, const NativeCallDesc* desc) {
   return arm_call_stack_size(t, desc);
 }
 
 static void arm_load_part(NativeTarget* t, NativeLoc dst, NativeLoc src,
                           u32 src_offset, u32 size) {
   NativeAddr addr;
   /* Load `size` bytes at src_offset of `src` into register `dst`. */
   if (src.kind == NATIVE_LOC_REG) {
     arm_move(t, dst, src);
     return;
   }
   if (src.kind == NATIVE_LOC_IMM) {
     i64 part;
     if (!native_loc_imm_part(src, src_offset, size, &part))
       arm_panic(arm_of(t), "invalid immediate argument part");
     arm_load_imm(t, dst, part);
     return;
   }
   if (native_loc_addr_role(src) == NATIVE_LOC_ADDR_ROLE_VALUE) {
     if (src_offset != 0u || size != t->c->target.ptr_size ||
         !native_loc_address_value(src, &addr))
       arm_panic(arm_of(t), "split or invalid address-value argument");
     arm_load_addr(t, dst, addr);
     return;
   }
   {
     MemAccess mem;
     if (!native_loc_storage_addr(src, (i32)src_offset, &addr))
       arm_panic(arm_of(t), "unsupported arg source location");
     addr.base_type = dst.type;
     memset(&mem, 0, sizeof mem);
     mem.type = dst.type;
     mem.size = size;
     arm_emit_mem(arm_of(t), 1, dst, addr, mem);
   }
 }
 
 /* Store an outgoing stack argument. A normal call writes the sp-anchored
  * outgoing-arg area ([sp, #stack_off]). A tail/sibling call instead writes the
  * caller's incoming-arg window ([r7 + arg_window_base + stack_off]) — the exact
  * address the tail-callee will read after the epilogue restores sp to the
  * caller's entry sp and branches. */
 static void arm_store_outgoing(NativeTarget* t, u32 stack_off, NativeLoc src,
                                u32 size, int tail) {
   Arm32NativeTarget* a = arm_of(t);
   NativeAddr addr;
   MemAccess mem;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_REG;
   addr.base.reg = tail ? ARM_FP : 13u;
   addr.offset = (i32)(tail ? arm_arg_window_base(a) + stack_off : stack_off);
   addr.base_type = src.type;
   memset(&mem, 0, sizeof mem);
   mem.type = src.type;
   mem.size = size;
   arm_emit_mem(a, 0, src, addr, mem);
 }
 
 /* Materialize the address of a frame/stack-resident NativeLoc into `dst` (used
  * for arguments that the ABI explicitly classifies as indirect). */
 static void arm_addr_of_loc(NativeTarget* t, NativeLoc dst, NativeLoc src) {
   NativeAddr addr;
   if (!native_loc_storage_addr(src, 0, &addr))
     arm_panic(arm_of(t), "address-of non-memory location");
   addr.base_type = dst.type;
   arm_load_addr(t, dst, addr);
 }
 
 static void arm_emit_one_arg_move(NativeTarget* t, const NativeArgMove* m) {
   if (m->is_addr)
     arm_addr_of_loc(t, m->dst, m->src);
   else
     arm_load_part(t, m->dst, m->src, m->src_offset, m->size);
 }
 
 static void arm_emit_reg_arg_moves(NativeTarget* t, NativeArgMove* moves,
                                    u32 n) {
   NativeArgShuffle s;
   if (n > ARM_MAX_REG_ARG_MOVES) arm_panic(arm_of(t), "too many register args");
   memset(&s, 0, sizeof s);
   s.t = t;
   s.emit_one = arm_emit_one_arg_move;
   s.reg_move = arm_move;
   s.scratch[NATIVE_REG_INT] = ARM_SCRATCH;
   s.scratch_class_mask = 1u << NATIVE_REG_INT;
   native_arg_shuffle(&s, moves, n);
 }
 
 static void arm_marshal_call(NativeTarget* t, const NativeCallDesc* desc,
                              NativeCallPhase* plan) {
   Arm32NativeTarget* a = arm_of(t);
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, desc->fn_type);
   NativeCallPhaseRet* rets;
   u32 nrets_cap = (abi && abi->ret.kind == ABI_ARG_DIRECT && desc->nresults)
                       ? abi->ret.nparts
                       : ((!abi && desc->nresults) ? 1u : 0u);
   int tail = (desc->flags & CG_CALL_TAIL) != 0;
   int tail_callee_saved = 0;
   memset(plan, 0, sizeof *plan);
   rets = nrets_cap ? arena_zarray(t->c->tu, NativeCallPhaseRet, nrets_cap) : NULL;
   plan->callee = desc->callee;
   plan->rets = rets;
   plan->flags = desc->flags;
   plan->has_sret = abi && abi->has_sret;
   plan->is_variadic = abi && abi->variadic;
   plan->stack_arg_size = arm_call_stack_size(t, desc);
   /* A tail call's outgoing stack args reuse the incoming-arg window in place, so
    * they do not enlarge this frame's outgoing-arg area. */
   if (!tail && plan->stack_arg_size > a->frame.max_outgoing)
     a->frame.max_outgoing = plan->stack_arg_size;
 
   /* Outgoing stack args stage through IP normally; a tail call holds the callee
    * in IP, so it stages through LR instead (LR is restored by the epilogue
    * afterward; the in-window store address needs no scratch). */
   u32 arg_stage = tail ? ARM_TMP : ARM_SCRATCH;
 
   /* Stage an indirect callee into a register that survives the arg-register
    * setup, the outgoing stack-arg staging, AND (for a tail call) the epilogue's
    * POP of r0..r11. A tail call holds it in IP (never popped) and stages args via
    * LR; a normal call stages args via IP, so an IP- or arg-register-resident
    * callee moves to LR (BLX lr is valid). A callee already in a preserved
    * register (r4..r11, or the chosen staging reg) needs no move.
    *
    * The exception is the -O0 (single-pass) call with outgoing stack args: staging
    * those args may consume BOTH backend temporaries (IP for the value, LR for a
    * large-offset store address), so the callee cannot survive in LR. Spill it
    * to its frame home now — before any arg move clobbers its register — and
    * reload just before BLX (arm_emit_call). The known-frame (-O1) path keeps
    * the register form: its optimizer-allocated callee already survives, and
    * its frame is final (no body-time slot allocation). */
   if (plan->callee.kind == NATIVE_LOC_REG &&
       (NativeAllocClass)plan->callee.cls == NATIVE_REG_INT) {
     u32 cr = plan->callee.v.reg & 0xfu;
     if (!tail && !a->known_frame && plan->stack_arg_size > 0u) {
       NativeFrameSlot slot = arm_callee_spill_slot(a);
       NativeAddr home;
       memset(&home, 0, sizeof home);
       home.base_kind = NATIVE_ADDR_BASE_FRAME;
       home.base.frame = slot;
       home.base_type = plan->callee.type;
       arm_emit_mem(a, 0, plan->callee, home,
                    native_mem_for_type(t, plan->callee.type, 4));
       plan->callee = native_loc_stack(plan->callee.type, slot, 0);
     } else {
       u32 dst = tail ? ARM_SCRATCH : ARM_TMP;
       int needs = tail ? 1 : (cr <= 3u || cr == arg_stage);
       if (needs && cr != dst) {
         NativeLoc scratch =
             native_loc_reg(plan->callee.type, NATIVE_REG_INT, dst);
         arm_move(t, scratch, plan->callee);
         plan->callee = scratch;
       }
     }
   }
 
   /* An indirect sibling call lives in IP across argument marshalling because
    * r0-r11 are restored by the tail epilogue and LR is the stack-part carrier.
    * A three-byte part needs both private registers: LR carries the part while
    * IP packs its final byte (and a far outgoing address can need IP as its
    * resolved base). Preserve the callee with one balanced temporary stack save
    * for the entire no-call marshalling interval, making both private registers
    * available to the exact-width boundary. Tail stack arguments are addressed
    * through FP into the incoming window, never through the temporarily shifted
    * SP; CFA is FP-anchored. There are no compiler-time exits between this PUSH
    * and its matching POP below. */
   if (tail && plan->callee.kind == NATIVE_LOC_REG &&
       loc_reg(plan->callee) == ARM_SCRATCH) {
     arm_emit_push(t->mc, 1u << ARM_SCRATCH);
     tail_callee_saved = 1;
   }
   {
     u32 next_int = (abi && abi->has_sret) ? 1u : 0u;
     u32 stack = 0, nmoves = 0, i, p;
     NativeArgMove moves[ARM_MAX_REG_ARG_MOVES];
     KitCgTypeId i32t = builtin_id(KIT_CG_BUILTIN_I32);
     ABIArgInfo scratch;
     for (i = 0; i < desc->nargs; ++i) {
       const ABIArgInfo* ai = arm_param_abi(t, abi, desc, i, &scratch);
       int even = arm_arg_needs_even(ai);
       if (ai->kind == ABI_ARG_IGNORE) continue;
       if (ai->kind == ABI_ARG_INDIRECT) {
         /* Pass an ABI-classified indirect argument as a pointer to the caller's
          * copy (the callee makes its own copy in bind_param). The pointer rides
          * a core reg when one is free, else the outgoing stack. */
         if (next_int < 4u) {
           NativeArgMove* m = &moves[nmoves++];
           m->dst = native_loc_reg(i32t, NATIVE_REG_INT, (Reg)(next_int++));
           m->src = desc->args[i];
           m->src_offset = 0;
           m->size = 4u;
           m->is_addr = 1;
         } else {
           NativeLoc ptr = native_loc_reg(i32t, NATIVE_REG_INT, arg_stage);
           arm_addr_of_loc(t, ptr, desc->args[i]);
           stack = align_up_u32(stack, 4u);
           arm_store_outgoing(t, stack, ptr, 4u, tail);
           stack += 4u;
         }
         continue;
       }
       /* AAPCS32: round NCRN up to even for an 8-byte-aligned arg, then fill core
        * registers r0..r3; parts that do not fit spill to the (8-byte-aligned for
        * part 0) outgoing stack. An 8-byte scalar pair fits-or-fully-spills;
        * a larger aggregate straddles the reg/stack boundary (AAPCS C.5). */
       if (even) next_int = align_up_u32(next_int, 2u);
       for (p = 0; p < ai->nparts; ++p) {
         const ABIArgPart* part = &ai->parts[p];
         KitCgTypeId lty = arm_part_scalar_type(part);
         if (next_int < 4u) {
           NativeArgMove* m = &moves[nmoves++];
           m->dst = native_loc_reg(lty, NATIVE_REG_INT, (Reg)(next_int++));
           m->src = desc->args[i];
           m->src_offset = part->src_offset;
           m->size = part->size;
           m->is_addr = 0;
         } else {
           if (p == 0 && even) stack = align_up_u32(stack, 8u);
           else stack = align_up_u32(stack, 4u);
           if (desc->args[i].kind == NATIVE_LOC_REG) {
             /* Source already in a register: store it straight to the outgoing
              * slot. Routing it through backend-private IP is unnecessary and
              * would add an independent lifetime while other call operands are
              * still leased. Keeping the source location intact also preserves
              * the instruction-scoped ownership contract. */
             arm_store_outgoing(t, stack, desc->args[i], part->size, tail);
           } else {
             NativeLoc tmp = native_loc_reg(lty, NATIVE_REG_INT, arg_stage);
             arm_load_part(t, tmp, desc->args[i], part->src_offset, part->size);
             arm_store_outgoing(t, stack, tmp, part->size, tail);
           }
           stack += 4u;
         }
       }
     }
     arm_emit_reg_arg_moves(t, moves, nmoves);
     if (abi && abi->has_sret) {
       /* sret destination pointer in r0. A tail call forwards OUR own incoming
        * sret pointer (spilled at entry) so the tail-callee writes the result
        * straight into our caller's buffer; a normal call passes &result. */
       NativeLoc r0 = native_loc_reg(i32t, NATIVE_REG_INT, ARM_R0);
       if (tail && a->has_sret) {
         NativeLoc saved = native_loc_stack(i32t, a->sret_ptr_slot, 0);
         arm_load_part(t, r0, saved, 0, 4u);
       } else if (desc->nresults) {
         arm_load_addr(t, r0, (NativeAddr){.base_kind = NATIVE_ADDR_BASE_FRAME,
                                           .base.frame = desc->results[0].v.frame,
                                           .base_type = i32t});
       }
     }
   }
   if (tail_callee_saved) arm_emit_pop(t->mc, 1u << ARM_SCRATCH);
   if (abi && abi->ret.kind == ABI_ARG_DIRECT && desc->nresults) {
     u32 nr = 0, ni = 0, p;
     for (p = 0; p < abi->ret.nparts; ++p) {
       const ABIArgPart* part = &abi->ret.parts[p];
       /* Per-lane type so each NDT-emitted result move is sized to the lane (4),
        * not the full 8-byte scalar (which the 32-bit backend cannot single-step
        * load/store). The ret pair is always r0:r1 (no even-rounding needed). */
       KitCgTypeId pty = arm_part_scalar_type(part);
       rets[nr].src = native_loc_reg(pty, NATIVE_REG_INT, (Reg)(ni++));
       rets[nr].dst = desc->results[0];
       if (rets[nr].dst.kind == NATIVE_LOC_FRAME)
         rets[nr].dst = native_loc_stack(pty, desc->results[0].v.frame,
                                         (i32)part->src_offset);
       rets[nr].mem = native_mem_for_type(t, pty, part->size);
       nr++;
     }
     plan->nrets = nr;
   } else if (!abi && desc->nresults) {
     rets[0].src = native_loc_reg(desc->results[0].type, NATIVE_REG_INT, ARM_R0);
     rets[0].dst = desc->results[0];
     rets[0].mem = native_mem_for_type(t, desc->results[0].type, 0);
     plan->nrets = 1;
   }
 }
 
 static void arm_emit_call(NativeTarget* t, const NativeCallPhase* plan) {
   Arm32NativeTarget* a = arm_of(t);
   MCEmitter* mc = t->mc;
   if (plan->flags & CG_CALL_TAIL) {
     /* Sibling call: the outgoing args are already placed (register args live;
      * stack args written into the incoming-arg window). Tear the frame down with
      * lr restored to OUR caller's return address, then BRANCH to the callee — it
      * runs on our caller's frame and returns straight to our caller. The arm32
      * epilogue is frame-size-independent (MOV sp,r7), so no patch is needed even
      * on the single-pass path. The callee is direct (B.W) or held in IP (BX). */
     if (plan->callee.kind != NATIVE_LOC_GLOBAL &&
         plan->callee.kind != NATIVE_LOC_REG)
       arm_panic(a, "unsupported tail target");
     arm_emit_frame_restore(a, /*to_pc=*/0);
     if (plan->callee.kind == NATIVE_LOC_GLOBAL) {
       u32 pos = mc_pos(mc);
       arm_emit_t32(mc, arm_b_w());
       mc_emit_reloc_at(mc, mc->section_id, pos, R_ARM_THM_JUMP24,
                        plan->callee.v.global.sym, plan->callee.v.global.addend, 0,
                        0);
     } else {
       arm_emit_t16(mc, arm_bx(loc_reg(plan->callee)));
     }
     return;
   }
   if (plan->callee.kind == NATIVE_LOC_GLOBAL) {
     u32 pos = mc_pos(mc);
     arm_emit_t32(mc, arm_bl());
     mc_emit_reloc_at(mc, mc->section_id, pos, R_ARM_THM_CALL,
                      plan->callee.v.global.sym, plan->callee.v.global.addend, 0,
                      0);
     return;
   }
   if (plan->callee.kind == NATIVE_LOC_REG) {
     arm_emit_t16(mc, arm_blx_reg(loc_reg(plan->callee)));
     return;
   }
   if (plan->callee.kind == NATIVE_LOC_STACK) {
     /* -O0 indirect call with stack args: the callee was spilled to a frame home
      * (arm_marshal_call) to free the backend staging registers. Arg staging is
      * done now, so IP is free; reload the target and BLX it. */
     NativeLoc tmp = native_loc_reg(plan->callee.type, NATIVE_REG_INT, ARM_SCRATCH);
     NativeAddr home;
     memset(&home, 0, sizeof home);
     home.base_kind = NATIVE_ADDR_BASE_FRAME;
     home.base.frame = plan->callee.v.stack.slot;
     home.base_type = plan->callee.type;
     arm_emit_mem(a, 1, tmp, home, native_mem_for_type(t, plan->callee.type, 4));
     arm_emit_t16(mc, arm_blx_reg(ARM_SCRATCH));
     return;
   }
   arm_panic(a, "unsupported call target");
 }
 
 static void arm_marshal_ret(NativeTarget* t, const CGFuncDesc* fd,
                             const NativeLoc* value,
                             NativeCallPhaseRet** out_rets, u32* out_nrets) {
   Arm32NativeTarget* a = arm_of(t);
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, fd->fn_type);
   NativeCallPhaseRet* rets = NULL;
   u32 nr = 0;
   if (value) rets = arena_zarray(t->c->tu, NativeCallPhaseRet, 4);
   if (value && abi && abi->ret.kind == ABI_ARG_INDIRECT) {
     /* Large aggregate return: copy the value into the caller-provided
      * destination via the hidden sret pointer spilled at entry. The body's
      * return slot is *value; load the saved pointer into ip, then copy. No
      * register return parts. */
     KitCgTypeId i32t = builtin_id(KIT_CG_BUILTIN_I32);
     NativeLoc dstp = native_loc_reg(i32t, NATIVE_REG_INT, ARM_SCRATCH);
     NativeLoc saved = native_loc_stack(i32t, a->sret_ptr_slot, 0);
     NativeAddr dst_addr, src_addr;
     AggregateAccess access;
     arm_load_part(t, dstp, saved, 0, 4u);
     memset(&dst_addr, 0, sizeof dst_addr);
     dst_addr.base_kind = NATIVE_ADDR_BASE_REG;
     dst_addr.base.reg = ARM_SCRATCH;
     dst_addr.base_type = value->type;
     memset(&src_addr, 0, sizeof src_addr);
     if (value->kind == NATIVE_LOC_FRAME) {
       src_addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       src_addr.base.frame = value->v.frame;
     } else if (value->kind == NATIVE_LOC_STACK) {
       src_addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       src_addr.base.frame = value->v.stack.slot;
       src_addr.offset = value->v.stack.offset;
     } else {
       arm_panic(a, "indirect return value must be in memory");
     }
     src_addr.base_type = value->type;
     memset(&access, 0, sizeof access);
     access.type = value->type;
     access.size = (u32)cg_type_size(t->c, value->type);
     access.align = native_type_align(t, value->type);
     arm_copy_bytes(t, dst_addr, src_addr, access);
     *out_rets = NULL;
     *out_nrets = 0;
     return;
   }
   if (value && abi && abi->ret.kind == ABI_ARG_DIRECT) {
     u32 ni = 0, p;
     int exact_parts = 0;
     for (p = 0; p < abi->ret.nparts; ++p) {
       const ABIArgPart* part = &abi->ret.parts[p];
       if (part->size != 1u && part->size != 2u && part->size != 4u &&
           part->size != 8u) {
         exact_parts = 1;
         break;
       }
     }
 
     /* A return-plan register destination makes the generic write phase
      * materialize its source using the scalar carrier type. For a three-byte
      * AAPCS aggregate tail that carrier is I32, so the exact mem.size=3 on the
      * final move arrives too late: materialization has already read four bytes.
      * Future 5/6/7-byte parts have the same mismatch. Once any such part is
      * present, fill every return register here in ABI order through the shared
      * exact-width argument boundary. arm_load_part uses only backend-private
      * temporaries, preserving return registers populated by earlier parts. */
     if (exact_parts) {
       for (p = 0; p < abi->ret.nparts; ++p) {
         const ABIArgPart* part = &abi->ret.parts[p];
         KitCgTypeId pty = arm_part_scalar_type(part);
         NativeLoc dst =
             native_loc_reg(pty, NATIVE_REG_INT, (Reg)(ni++));
         arm_load_part(t, dst, *value, part->src_offset, part->size);
       }
       *out_rets = NULL;
       *out_nrets = 0;
       return;
     }
 
     for (p = 0; p < abi->ret.nparts; ++p) {
       const ABIArgPart* part = &abi->ret.parts[p];
       /* Per-lane type: an i64/double return is two i32 lanes in r0:r1; sizing
        * each move from the lane type keeps the 32-bit backend off the 8-byte
        * memory path. (See arm_part_scalar_type.) */
       KitCgTypeId pty = arm_part_scalar_type(part);
       rets[nr].src = *value;
       if (rets[nr].src.kind == NATIVE_LOC_FRAME)
         rets[nr].src =
             native_loc_stack(pty, value->v.frame, (i32)part->src_offset);
       rets[nr].dst = native_loc_reg(pty, NATIVE_REG_INT, (Reg)(ni++));
       rets[nr].mem = native_mem_for_type(t, pty, part->size);
       nr++;
     }
   } else if (value) {
     rets[0].src = *value;
     rets[0].dst = native_loc_reg(value->type, NATIVE_REG_INT, ARM_R0);
     rets[0].mem = native_mem_for_type(t, value->type, 0);
     nr = 1;
   }
   *out_rets = rets;
   *out_nrets = nr;
 }
 
 static void arm_ret(NativeTarget* t) {
   Arm32NativeTarget* a = arm_of(t);
   arm_jump(t, a->epilogue_label);
 }
 
 /* ============================ misc / stubs ============================ */
 
 static void arm_trap(NativeTarget* t) { arm_emit_t16(t->mc, arm_bkpt(0u)); }
 static void arm_set_loc(NativeTarget* t, SrcLoc loc) {
   arm_of(t)->loc = loc;
   mc_set_loc(t->mc, loc);
 }
 
 #define ARM_UNIMPL(name) arm_panic(arm_of(t), name " not implemented in Phase 1")
 
 /* Assemble up to 4 little-endian bytes of `c` starting at byte `off` into a u32. */
 static u32 arm_const_lane(ConstBytes c, u32 off) {
   u32 v = 0, i;
   for (i = 0; i < 4u && off + i < c.size; ++i)
     v |= (u32)c.bytes[off + i] << (i * 8u);
   return v;
 }
 
 static void arm_load_const(NativeTarget* t, NativeLoc dst, ConstBytes c) {
   Arm32NativeTarget* a = arm_of(t);
   /* Scalar constant <=4 bytes: reinterpret the ABI bytes (little-endian on
    * arm32) as a u32 and materialize via arm_load_imm (MOV.W/MVN.W/MOVW+MOVT). */
   if (c.size <= 4u) {
     arm_load_imm(t, dst, (i64)(i32)arm_const_lane(c, 0u));
     return;
   }
   /* 8-byte (i64/double) backstop. At -O0 the NDT lowers wide8 constants into two
    * 32-bit lanes before they reach the backend (nd_load_const guards it), so
    * this is only exercised by paths that hand the full 8-byte constant straight
    * to a destination — materialize the low lane then the high lane, little-end
    * first. A memory destination (frame/stack home) carries both lanes 4 bytes
    * apart; a single 32-bit register can only hold the low lane (the high lane
    * has no home in one NativeLoc), which matches a 32-bit reinterpret. */
   if (c.size > 8u) arm_panic(a, "load_const wider than 8 bytes");
   if (dst.kind == NATIVE_LOC_REG) {
     arm_load_imm(t, dst, (i64)(i32)arm_const_lane(c, 0u));
     return;
   }
   {
     KitCgTypeId i32t = builtin_id(KIT_CG_BUILTIN_I32);
     NativeLoc lane = native_loc_reg(i32t, NATIVE_REG_INT, ARM_TMP);
     NativeAddr home;
     MemAccess mem = native_mem_for_type(t, i32t, 4u);
     u32 off;
     memset(&home, 0, sizeof home);
     home.base_type = i32t;
     if (dst.kind == NATIVE_LOC_FRAME) {
       home.base_kind = NATIVE_ADDR_BASE_FRAME;
       home.base.frame = dst.v.frame;
     } else if (dst.kind == NATIVE_LOC_STACK) {
       home.base_kind = NATIVE_ADDR_BASE_FRAME;
       home.base.frame = dst.v.stack.slot;
       home.offset = dst.v.stack.offset;
     } else {
       arm_panic(a, "8-byte load_const destination not lowered");
     }
     for (off = 0; off < c.size; off += 4u) {
       NativeAddr d = home;
       d.offset += (i32)off;
       arm_load_imm(t, lane, (i64)(i32)arm_const_lane(c, off));
       arm_emit_mem(a, 0, lane, d, mem);
     }
   }
 }
 static void arm_load_label_addr(NativeTarget* t, NativeLoc dst, MCLabel l) {
   /* `&&label` address-take (computed goto / jump-table base): materialize the
    * label's address via MOVW/MOVT ABS against its per-block local symbol — the
    * same form arm_emit_global_addr uses for a global, so an encoding-divergent
    * assembler recomputes the split immediate. (Setting the Thumb bit for a
    * BX/BLX through this address is the indirect-branch consumer's job.) */
   Arm32NativeTarget* a = arm_of(t);
   ObjSymId sym = mc_label_symbol(t->mc, l);
   arm_emit_global_addr(a, loc_reg(dst), sym, 0);
 }
 static void arm_indirect_branch(NativeTarget* t, NativeLoc addr,
                                 const MCLabel* targets, u32 n) {
   /* The cg layer materialized the dense switch's target code address into
    * `addr` (a load of table[idx] from the .Lkit_jt rodata table). The table
    * entries are R_ARM_ABS32 relocations against per-block local label symbols,
    * which carry NO Thumb bit (only STT_FUNC symbols get it in arm_func_end), so
    * the loaded address is even. BX to an even address would switch to ARM
    * state (UNPREDICTABLE on M-profile); force the Thumb bit with ORR #1 first.
    * `addr` is released by the caller right after this, so clobbering it is free.
    */
   MCEmitter* mc = t->mc;
   u32 ra = loc_reg(addr);
   u32 enc;
   (void)targets;
   (void)n;
   thumb_expand_imm_encode(1u, &enc);
   arm_emit_t32(mc, arm_dp_imm(2u, 0u, ra, ra, enc)); /* ORR ra, ra, #1 */
   arm_emit_t16(mc, arm_bx(ra));                       /* BX ra */
 }
 static void arm_tls_addr_of(NativeTarget* t, NativeLoc dst, ObjSymId sym,
                             i64 addend) {
   /* The M-profile (Cortex-M) has no CP15 thread-ID register and the bare-metal
    * arm32 lane is single-threaded, so the local-exec model degenerates: each
    * `__thread` object has exactly one instance, addressed absolutely like a
    * regular static. Materialize its address with the same MOVW/MOVT-absolute
    * sequence as a global; the freestanding image places the TLS image as part
    * of its data, so &tlsvar is a link-time constant. (A true variant-I TLS path
    * with a software thread pointer is a follow-on for a multi-threaded lane.) */
   arm_emit_global_addr(arm_of(t), loc_reg(dst), sym, addend);
 }
 /* Aggregate / wide byte copy (struct/array, INDIRECT param/return copies, the
  * wide va_arg copy, and memmove's loops). Each side is resolved ONCE to a stable
  * (base register, displacement): a FRAME/REG side in range for the whole span is
  * addressed directly; a GLOBAL or out-of-range side has its effective address
  * materialized into a held scratch. A 4|2|1 granule ladder then transfers each
  * chunk through a register that aliases neither base.
  *
  * Register budget, allocated explicitly up front: two reserved scratch (IP, LR);
  * one callee-save borrowed (pushed/popped) only for the rare case where BOTH
  * sides must be materialized AND the two bases plus the transfer exceed {IP,LR}.
  * The borrow is taken before any frame-relative materialization, so r7-relative
  * offsets (and already-held REG bases) are unaffected by the SP change. */
 
 /* The natural (base register, displacement) of a FRAME/REG addr, no range check. */
 static void arm_addr_natural(Arm32NativeTarget* a, const NativeAddr* addr,
                              u32* base, i32* off) {
   if (addr->base_kind == NATIVE_ADDR_BASE_FRAME) {
     NativeFrameSlotEntry* s = native_frame_slot_at(&a->frame, addr->base.frame);
     *base = ARM_FP;
     *off = -(i32)s->off + addr->offset;
   } else {
     *base = addr->base.reg & 0xfu;
     *off = addr->offset;
   }
 }
 
 /* Whether `addr` can be addressed directly for a `span`-byte access (a real base
  * register + in-range displacement for the whole span). GLOBAL is never direct.
  * On success sets base+off to the natural (base register, displacement). */
 static int arm_addr_direct(Arm32NativeTarget* a, const NativeAddr* addr,
                            u32 span, u32* base, i32* off) {
   if (addr->base_kind != NATIVE_ADDR_BASE_FRAME &&
       addr->base_kind != NATIVE_ADDR_BASE_REG)
     return 0;
   arm_addr_natural(a, addr, base, off);
   return arm_span_inline(*off, span);
 }
 
 /* Take a transfer/held-base register for copy_bytes: a free reserved scratch
  * (IP then LR) not already in `*used`, else a borrowed callee-save (r4..r6 not in
  * `*used`, pushed; recorded in `*borrowed`, which starts 0xff and is set once). */
 static u32 arm_copy_take_reg(Arm32NativeTarget* a, u32* used, u32* borrowed) {
   u32 r;
   if (!(*used & (1u << ARM_SCRATCH))) r = ARM_SCRATCH;
   else if (!(*used & (1u << ARM_TMP))) r = ARM_TMP;
   else {
     r = (*used & (1u << 4)) ? ((*used & (1u << 5)) ? 6u : 5u) : 4u;
     *borrowed = r;
     arm_emit_push(a->base.mc, 1u << r);
   }
   *used |= 1u << r;
   return r;
 }
 
 static void arm_copy_bytes(NativeTarget* t, NativeAddr dst, NativeAddr src,
                            AggregateAccess acc) {
   Arm32NativeTarget* a = arm_of(t);
   u32 size = acc.size, off;
   u32 sbase, dbase, xfer, used = 0u, borrowed = 0xffu;
   i32 soff = 0, doff = 0;
   int sdir, ddir;
   if (!size) return;
   if (size > 4096u) arm_panic(a, "aggregate copy > 4096 bytes (needs memcpy)");
 
   sdir = arm_addr_direct(a, &src, size, &sbase, &soff);
   ddir = arm_addr_direct(a, &dst, size, &dbase, &doff);
   if (sdir) used |= 1u << sbase;
   if (ddir) used |= 1u << dbase;
 
   /* Allocate held scratch for the non-direct sides, then the transfer reg. */
   if (!sdir) sbase = arm_copy_take_reg(a, &used, &borrowed);
   if (!ddir) dbase = arm_copy_take_reg(a, &used, &borrowed);
   xfer = arm_copy_take_reg(a, &used, &borrowed);
 
   /* Materialize the non-direct sides (after any borrow-push so a frame-relative
    * address still reads the unchanged r7). */
   if (!sdir) {
     if (src.base_kind == NATIVE_ADDR_BASE_GLOBAL)
       arm_emit_global_addr(a, sbase, src.base.global.sym,
                            src.base.global.addend + src.offset);
     else {
       u32 nb;
       i32 no;
       arm_addr_natural(a, &src, &nb, &no);
       arm_emit_base_off(a, sbase, nb, no);
     }
     soff = 0;
   }
   if (!ddir) {
     if (dst.base_kind == NATIVE_ADDR_BASE_GLOBAL)
       arm_emit_global_addr(a, dbase, dst.base.global.sym,
                            dst.base.global.addend + dst.offset);
     else {
       u32 nb;
       i32 no;
       arm_addr_natural(a, &dst, &nb, &no);
       arm_emit_base_off(a, dbase, nb, no);
     }
     doff = 0;
   }
 
   /* Granule ladder: word, then trailing halfword/byte. Each chunk stays in range
    * (the resolver guaranteed the span fits, and a materialized base has off 0). */
   for (off = 0; off < size;) {
     u32 rem = size - off;
     u32 sz = rem >= 4u ? 4u : rem >= 2u ? 2u : 1u;
     arm_emit_mem_one(a, 1, xfer, sbase, soff + (i32)off, sz);
     arm_emit_mem_one(a, 0, xfer, dbase, doff + (i32)off, sz);
     off += sz;
   }
   if (borrowed != 0xffu) arm_emit_pop(a->base.mc, 1u << borrowed);
 }
 
 /* Copy `size` bytes between two register-based addresses with the same 4|2|1
  * granule ladder as copy_bytes, but in a chosen direction: forward (low→high)
  * or backward (high→low). memmove picks the overlap-safe direction at runtime.
  * Transfers through `xfer`, which must alias neither base. */
 static void arm_copy_granules_dir(Arm32NativeTarget* a, u32 dbase, u32 sbase,
                                   u32 size, u32 xfer, int backward) {
   if (backward) {
     u32 off = size;
     while (off > 0u) {
       u32 sz = off >= 4u ? 4u : off >= 2u ? 2u : 1u;
       off -= sz;
       arm_emit_mem_one(a, 1, xfer, sbase, (i32)off, sz);
       arm_emit_mem_one(a, 0, xfer, dbase, (i32)off, sz);
     }
   } else {
     u32 off = 0;
     while (off < size) {
       u32 rem = size - off;
       u32 sz = rem >= 4u ? 4u : rem >= 2u ? 2u : 1u;
       arm_emit_mem_one(a, 1, xfer, sbase, (i32)off, sz);
       arm_emit_mem_one(a, 0, xfer, dbase, (i32)off, sz);
       off += sz;
     }
   }
 }
 
 /* set_bytes: store the materialized fill byte `v` across acc.size bytes. The
  * common case is struct/array zero-init (v == 0); any fill byte is correct. One
  * STRB per byte — correctness-first; word-splat widening is deferred (it would
  * need a second scratch for the 0x01010101 multiplier or a shifted-ORR encoder,
  * and the dst base may itself live in ip). */
 static void arm_set_bytes(NativeTarget* t, NativeAddr dst, NativeLoc v,
                           AggregateAccess acc) {
   Arm32NativeTarget* a = arm_of(t);
   MemAccess mem = acc.mem;
   mem.size = 1u;
   mem.align = 1u;
   for (u32 off = 0; off < acc.size; ++off) {
     NativeAddr d = dst;
     d.offset += (i32)off;
     arm_emit_mem(a, 0, v, d, mem); /* STRB v, [dst+off] */
   }
 }
 /* C bit-fields. The storage unit {bf.storage.type, bf.storage.size} lives at
  * bf.storage_offset from `addr`; the field is `bf.bit_width` bits at bit
  * bf.bit_offset within it. Load = load the unit (zero-extended) then UBFX/SBFX;
  * store = load the unit, BFI the value in, store the unit back. Sub-word storage
  * units (1/2 bytes) are loaded/stored at their natural width, so the BFI/UBFX
  * geometry (offset+width <= storage_bits) stays in range. */
 static void arm_bitfield_load(NativeTarget* t, NativeLoc dst, NativeAddr addr,
                               BitFieldAccess bf) {
   Arm32NativeTarget* a = arm_of(t);
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst);
   u32 width = bf.bit_width ? bf.bit_width : 1u;
   NativeAddr saddr = addr;
   NativeLoc unit = dst;
   saddr.offset += (i32)bf.storage_offset;
   unit.type = bf.storage.type ? bf.storage.type : dst.type;
   arm_emit_mem(a, 1, unit, saddr, bf.storage); /* LDR(B/H) rd, [addr+soff] */
   if (bf.signed_)
     arm_emit_t32(mc, arm_sbfx(rd, rd, bf.bit_offset, width));
   else
     arm_emit_t32(mc, arm_ubfx(rd, rd, bf.bit_offset, width));
 }
 static void arm_bitfield_store(NativeTarget* t, NativeAddr addr, NativeLoc v,
                                BitFieldAccess bf) {
   Arm32NativeTarget* a = arm_of(t);
   MCEmitter* mc = t->mc;
   u32 rv = loc_reg(v);
   u32 width = bf.bit_width ? bf.bit_width : 1u;
   KitCgTypeId unit_ty = bf.storage.type ? bf.storage.type : v.type;
   NativeLoc word = native_loc_reg(unit_ty, NATIVE_REG_INT, ARM_TMP);
   NativeAddr saddr = addr;
   saddr.offset += (i32)bf.storage_offset;
   arm_emit_mem(a, 1, word, saddr, bf.storage);              /* LDR tmp, [addr] */
   arm_emit_t32(mc, arm_bfi(ARM_TMP, rv, bf.bit_offset, width)); /* insert v */
   arm_emit_mem(a, 0, word, saddr, bf.storage);              /* STR tmp, [addr] */
 }
 static void arm_alloca(NativeTarget* t, NativeLoc dst, NativeLoc size,
                        u32 align) {
   Arm32NativeTarget* a = arm_of(t);
   MCEmitter* mc = t->mc;
   u32 rsz = loc_reg(size), rd = loc_reg(dst);
   u32 enc7;
   (void)align; /* AAPCS keeps sp 8-byte aligned; round the request up to 8. */
   thumb_expand_imm_encode(7u, &enc7);
   /* ip = (size + 7) & ~7; sp -= ip. */
   arm_emit_t32(mc, arm_add_imm12(ARM_SCRATCH, rsz, 7u));
   arm_emit_t32(mc, arm_dp_imm(1u, 0u, ARM_SCRATCH, ARM_SCRATCH, enc7)); /* BIC */
   arm_emit_t32(mc, arm_sub_reg(13u, 13u, ARM_SCRATCH));
   /* dst = sp + max_outgoing (the block sits above the outgoing-arg area). The
    * offset is finalized in arm_func_end; emit a placeholder ADDW to patch. */
   if (a->n_alloca_patch >= 16u) arm_panic(a, "too many alloca sites");
   a->alloca_patch_rd[a->n_alloca_patch] = (u8)rd;
   a->alloca_patch_pos[a->n_alloca_patch++] = mc_pos(mc);
   arm_emit_t32(mc, arm_add_imm12(rd, 13u, 0u));
   a->frame.has_alloca = 1;
 }
 /* ============================ atomics ============================ */
 /* ARMv7-M atomics over LDREX/STREX (word/halfword/byte) + DMB. The cg layer
  * routes 8-byte _Atomic to the spinlock libcall (atomic_lock_free_max=4), so
  * only <=4-byte ops reach here.
  *
  * Register budget: LR (ARM_TMP) holds the access address. On the shared O0
  * path, a value operand may occupy IP (ARM_SCRATCH); on O1, operands arrive in
  * allocated r4..r11 or instruction-scoped r0..r3 locations and IP/LR remain
  * backend-private. The core must therefore keep IP unavailable for its own
  * extra state so it is safe on both paths. STREX additionally needs its status,
  * value and base registers mutually distinct.
  *
  * The rmw/cas loops therefore borrow callee-saved registers for the private
  * temporaries (new value + STREX status). At -O1 the optimizer freely allocates
  * the operands (prior/expected/desired/ok, or rmw dst/val) and the address base
  * across the whole callee-saved set, so a fixed r4/r5 would alias an operand. The
  * borrow is therefore OPERAND-AWARE (arm_atomic_borrow): it picks free callee-
  * saved registers clear of the live operand+base set, guaranteeing base + all
    * operands + the STREX temporaries are mutually distinct. The picked
    * registers are push/pop-balanced so the
  * caller's callee-saved values are preserved (free at -O0, safe at -O1). */
 
 /* Pick `n` distinct callee-saved temporaries (r4,r5,r6,r8,r9,r10,r11 — fp/r7 is
  * reserved) clear of the `used` register set, for the atomic LDREX/STREX private
  * temporaries. Fills regs[0..n) and returns their push/pop mask in *mask. The
  * pool has 7 registers; the largest live set (rmw: dst+val+base = 3, cas:
  * 4 operands + base = 5) always leaves enough free for n (2 / 1 respectively). */
 static void arm_atomic_borrow(Arm32NativeTarget* a, u32 used, u32 n, u32* regs,
                               u32* mask) {
   static const u32 pool[] = {4u, 5u, 6u, 8u, 9u, 10u, 11u};
   u32 got = 0u, m = 0u, i;
   for (i = 0; i < sizeof pool / sizeof pool[0] && got < n; ++i) {
     if (used & (1u << pool[i])) continue;
     regs[got++] = pool[i];
     m |= 1u << pool[i];
   }
   if (got < n) arm_panic(a, "atomic: no free scratch register");
   *mask = m;
 }
 
 static int arm_order_acquire(KitCgMemOrder o) {
   return o == KIT_CG_MO_CONSUME || o == KIT_CG_MO_ACQUIRE ||
          o == KIT_CG_MO_ACQ_REL || o == KIT_CG_MO_SEQ_CST;
 }
 static int arm_order_release(KitCgMemOrder o) {
   return o == KIT_CG_MO_RELEASE || o == KIT_CG_MO_ACQ_REL ||
          o == KIT_CG_MO_SEQ_CST;
 }
 
 /* Materialize the access address into lr (ARM_TMP) and return it. lr is never an
  * NDT operand and is dead in the body, so it never collides with the value
  * operands (which the value-cache keeps in r0..r3 / ip). The incoming NativeAddr
  * may be a frame slot or a register; load_addr lands the pointer in lr, which
  * the LDREX/STREX sequence then uses as the base. */
 static u32 arm_atomic_addr_reg(Arm32NativeTarget* a, NativeAddr addr) {
   /* Use the pointer's own register when the address already is one (the -O1
    * case: location MIR keeps the allocated address component explicit).
    * Otherwise (a FRAME/GLOBAL address, e.g. -O0) materialize into
    * backend-private LR, which holds no operand there. */
   if (addr.base_kind == NATIVE_ADDR_BASE_REG && addr.offset == 0 &&
       addr.index_kind == NATIVE_ADDR_INDEX_NONE)
     return addr.base.reg & 0xfu;
   arm_load_addr(&a->base,
                 native_loc_reg(builtin_id(KIT_CG_BUILTIN_I32), NATIVE_REG_INT,
                                ARM_TMP),
                 addr);
   return ARM_TMP;
 }
 
 /* LDREX-family load for the access width (word / halfword / byte). */
 static void arm_emit_ldrex(Arm32NativeTarget* a, u32 sz, u32 rt, u32 base) {
   MCEmitter* mc = a->base.mc;
   if (sz >= 4u) arm_emit_t32(mc, arm_ldrex(rt, base, 0u));
   else if (sz == 2u) arm_emit_t32(mc, arm_ldrexh(rt, base));
   else arm_emit_t32(mc, arm_ldrexb(rt, base));
 }
 /* STREX-family store; writes the 0/1 success status into `rd`. */
 static void arm_emit_strex(Arm32NativeTarget* a, u32 sz, u32 rd, u32 rt,
                            u32 base) {
   MCEmitter* mc = a->base.mc;
   if (sz >= 4u) arm_emit_t32(mc, arm_strex(rd, rt, base, 0u));
   else if (sz == 2u) arm_emit_t32(mc, arm_strexh(rd, rt, base));
   else arm_emit_t32(mc, arm_strexb(rd, rt, base));
 }
 
 static void arm_atomic_load(NativeTarget* t, NativeLoc dst, NativeAddr addr,
                             MemAccess mem, KitCgMemOrder order) {
   Arm32NativeTarget* a = arm_of(t);
   MCEmitter* mc = t->mc;
   u32 sz = mem.size ? mem.size : native_type_size(t, dst.type);
   u32 base = arm_atomic_addr_reg(a, addr);
   NativeAddr m;
   if (sz > 4u) arm_panic(a, "8-byte atomic load not lowered (spinlock libcall)");
   memset(&m, 0, sizeof m);
   m.base_kind = NATIVE_ADDR_BASE_REG;
   m.base.reg = base;
   m.base_type = dst.type;
   arm_emit_mem(a, 1, dst, m, mem); /* plain LDR/LDRH/LDRB is atomic for <=4B */
   if (arm_order_acquire(order)) arm_emit_t32(mc, arm_dmb(0xfu));
 }
 
 static void arm_atomic_store(NativeTarget* t, NativeAddr addr, NativeLoc v,
                              MemAccess mem, KitCgMemOrder order) {
   Arm32NativeTarget* a = arm_of(t);
   MCEmitter* mc = t->mc;
   u32 sz = mem.size ? mem.size : native_type_size(t, v.type);
   u32 base = arm_atomic_addr_reg(a, addr);
   NativeAddr m;
   if (sz > 4u) arm_panic(a, "8-byte atomic store not lowered (spinlock libcall)");
   if (arm_order_release(order)) arm_emit_t32(mc, arm_dmb(0xfu));
   memset(&m, 0, sizeof m);
   m.base_kind = NATIVE_ADDR_BASE_REG;
   m.base.reg = base;
   m.base_type = v.type;
   arm_emit_mem(a, 0, v, m, mem); /* plain STR/STRH/STRB is atomic for <=4B */
   if (order == KIT_CG_MO_SEQ_CST) arm_emit_t32(mc, arm_dmb(0xfu));
 }
 /* The ARMv7-M barrier option for a full-system barrier ("sy"). 64-bit clz/ctz/
  * bswap on a 32-bit target are already routed to __*di2 libcalls by cg, so the
  * backend only ever sees the 16/32-bit forms here. Each lowered intrinsic must
  * stay in lockstep with arm32_supports_intrinsic (arch.c). */
 #define ARM_BARRIER_SY 0xfu
 
 static void arm_atomic_rmw(NativeTarget* t, KitCgAtomicOp op, NativeLoc dst,
                            NativeAddr addr, NativeLoc val, MemAccess mem,
                            KitCgMemOrder order) {
   Arm32NativeTarget* a = arm_of(t);
   MCEmitter* mc = t->mc;
   u32 sz = mem.size ? mem.size : native_type_size(t, dst.type);
   u32 rd = loc_reg(dst);
   u32 rv = loc_reg(val);
   u32 base, newv, status, temps[2], pushmask;
   MCLabel retry = mc_label_new(mc);
   if (sz > 4u) arm_panic(a, "8-byte atomic rmw not lowered (spinlock libcall)");
   /* Borrow a new-value temp + a STREX-status temp clear of {addr base, rd, rv}
    * across a balanced push/pop, so the sequence touches no live operand. ip is
    * left alone (an operand may live there) and the operand-aware pick avoids the
    * base/operand registers the optimizer chose at -O1. */
   base = arm_atomic_addr_reg(a, addr); /* lr (-O0) or the pointer's own reg */
   arm_atomic_borrow(a, (1u << rd) | (1u << rv) | (1u << base), 2u, temps,
                     &pushmask);
   newv = temps[0];
   status = temps[1];
   arm_emit_push(mc, pushmask);
   if (arm_order_release(order)) arm_emit_t32(mc, arm_dmb(0xfu));
   /* Retry loop: rd = *base (LDREX); r4 = rd OP val; STREX r5,r4,[base]; retry
    * while r5 != 0. */
   mc_label_place(mc, retry);
   arm_emit_ldrex(a, sz, rd, base);
   switch (op) {
     case KIT_CG_ATOMIC_XCHG:
       arm_emit_t16(mc, arm_mov_hi(newv, rv));
       break;
     case KIT_CG_ATOMIC_ADD:
       arm_emit_t32(mc, arm_add_reg(newv, rd, rv));
       break;
     case KIT_CG_ATOMIC_SUB:
       arm_emit_t32(mc, arm_sub_reg(newv, rd, rv));
       break;
     case KIT_CG_ATOMIC_AND:
       arm_emit_t32(mc, arm_and_reg(newv, rd, rv));
       break;
     case KIT_CG_ATOMIC_OR:
       arm_emit_t32(mc, arm_orr_reg(newv, rd, rv));
       break;
     case KIT_CG_ATOMIC_XOR:
       arm_emit_t32(mc, arm_eor_reg(newv, rd, rv));
       break;
     case KIT_CG_ATOMIC_NAND:
       arm_emit_t32(mc, arm_and_reg(newv, rd, rv));
       arm_emit_t32(mc, arm_mvn_reg(newv, newv));
       break;
     default:
       arm_panic(a, "unsupported atomic rmw op");
   }
   arm_emit_strex(a, sz, status, newv, base); /* STREX r5, r4, [lr] */
   arm_emit_t32(mc, arm_cmp_imm(status, 0u)); /* CMP r5, #0 */
   arm_emit_t32(mc, arm_b_cond_w(ARM_CC_NE)); /* BNE retry */
   mc_emit_label_ref(mc, retry, R_ARM_THM_JUMP19, 4, 0);
   if (arm_order_acquire(order)) arm_emit_t32(mc, arm_dmb(0xfu));
   arm_emit_pop(mc, pushmask);
 }
 
 static void arm_atomic_cas(NativeTarget* t, NativeLoc prior, NativeLoc ok,
                            NativeAddr addr, NativeLoc expected,
                            NativeLoc desired, MemAccess mem,
                            KitCgMemOrder success, KitCgMemOrder failure) {
   Arm32NativeTarget* a = arm_of(t);
   MCEmitter* mc = t->mc;
   u32 sz = mem.size ? mem.size : native_type_size(t, prior.type);
   u32 rprior = loc_reg(prior);
   u32 rexp = loc_reg(expected);
   u32 rdes = loc_reg(desired);
   u32 rok = loc_reg(ok);
   u32 base, status, statusv[1], pushmask;
   u32 enc0, enc1;
   MCLabel retry = mc_label_new(mc);
   MCLabel fail = mc_label_new(mc);
   MCLabel done = mc_label_new(mc);
   (void)failure;
   if (sz > 4u) arm_panic(a, "8-byte atomic cas not lowered (spinlock libcall)");
   thumb_expand_imm_encode(0u, &enc0);
   thumb_expand_imm_encode(1u, &enc1);
   /* STREX status temp clear of {addr base, prior, expected, desired, ok} —
    * never ip (an operand may live there), and operand-aware so the optimizer's
    * -O1 register choices for the five live values never alias it. */
   base = arm_atomic_addr_reg(a, addr); /* lr (-O0) or the pointer's own reg */
   arm_atomic_borrow(a,
                     (1u << rprior) | (1u << rexp) | (1u << rdes) | (1u << rok) |
                         (1u << base),
                     1u, statusv, &pushmask);
   status = statusv[0];
   arm_emit_push(mc, pushmask);
   if (arm_order_release(success)) arm_emit_t32(mc, arm_dmb(0xfu));
   /* retry: prior = *base (LDREX); if prior != expected goto fail; STREX
    * r4,desired,[base]; retry on failure; ok = 1; goto done. fail: clear the
    * monitor (CLREX) and ok = 0. */
   mc_label_place(mc, retry);
   arm_emit_ldrex(a, sz, rprior, base);
   arm_emit_t32(mc, arm_cmp_reg(rprior, rexp));
   arm_emit_t32(mc, arm_b_cond_w(ARM_CC_NE)); /* BNE fail */
   mc_emit_label_ref(mc, fail, R_ARM_THM_JUMP19, 4, 0);
   arm_emit_strex(a, sz, status, rdes, base);
   arm_emit_t32(mc, arm_cmp_imm(status, 0u));
   arm_emit_t32(mc, arm_b_cond_w(ARM_CC_NE)); /* BNE retry */
   mc_emit_label_ref(mc, retry, R_ARM_THM_JUMP19, 4, 0);
   arm_emit_t32(mc, arm_mov_imm(rok, enc1)); /* ok = 1 */
   arm_emit_t32(mc, arm_b_w());              /* B done */
   mc_emit_label_ref(mc, done, R_ARM_THM_JUMP24, 4, 0);
   mc_label_place(mc, fail);
   arm_emit_t32(mc, arm_clrex());            /* drop the exclusive reservation */
   arm_emit_t32(mc, arm_mov_imm(rok, enc0)); /* ok = 0 */
   mc_label_place(mc, done);
   if (arm_order_acquire(success)) arm_emit_t32(mc, arm_dmb(0xfu));
   arm_emit_pop(mc, pushmask);
 }
 
 static void arm_fence(NativeTarget* t, KitCgMemOrder order) {
   if (order == KIT_CG_MO_RELAXED) return;
   arm_emit_t32(t->mc, arm_dmb(0xfu));
 }
 
 /* ============================ varargs ============================ */
 /* AAPCS32 va_list is a plain 4-byte pointer to the next argument slot. The
  * prologue spilled the unconsumed GP arg registers r0..r3 into a save area just
  * above the saved {r7, lr} pair (at [r7 + ARM_SAVED_PAIR_BYTES]); the named
  * incoming stack args follow contiguously at [r7 + arg_window_base], so a
  * uniform 4-byte stride (8-byte-aligned for i64/double) walks both regions.
  * `ap` is a NativeAddr addressing the va_list object itself. */
 
 static void arm_va_start_core(Arm32NativeTarget* a, NativeAddr ap) {
   NativeTarget* t = &a->base;
   MCEmitter* mc = t->mc;
   KitCgTypeId i32t = builtin_id(KIT_CG_BUILTIN_I32);
   NativeLoc ptr = native_loc_reg(i32t, NATIVE_REG_INT, ARM_TMP);
   ABIVaListInfo vai = abi_va_list_layout(t->c->abi);
   if (vai.kind != ABI_VA_LIST_POINTER) arm_panic(a, "unsupported va_list layout");
   if (!a->is_variadic) arm_panic(a, "va_start: function not variadic");
   /* *ap = r7 + saved_block_bytes + next_param_int*4: the GP save area (r0..r3)
    * sits just above the saved register block, so its base is saved_block_bytes
    * (8 for the bare {r7,lr} single-pass prologue; larger when the -O1 known-frame
    * prologue also PUSHes callee-saves), and next_param_int*4 skips the GP slots
    * the fixed params already consumed. lr (ARM_TMP) is the staging temp. */
   arm_emit_t32(mc, arm_add_imm12(ARM_TMP, ARM_FP,
                                  a->saved_block_bytes + a->next_param_int * 4u));
   arm_emit_mem(a, 0, ptr, ap, native_mem_for_type(t, i32t, 4));
 }
 
 /* Whether a va_arg value is too wide to move through one core register (an
  * 8-byte i64 / soft-double): it occupies two GP slots and is copied straight
  * from the save area into its destination memory. */
 static int arm_va_arg_is_wide(NativeTarget* t, KitCgTypeId type) {
   return native_type_size(t, type) > 4u;
 }
 
 /* Wide / aggregate va_arg: read the cursor, 8-byte-align it (i64/double slots
  * are 8-byte aligned on the stack per AAPCS32), advance past the whole span,
  * then byte-copy from the (aligned) cursor into the destination memory. ip holds
  * the cursor across the copy; arm_copy_bytes uses lr internally. */
 static void arm_va_arg_wide(Arm32NativeTarget* a, NativeAddr dst, NativeAddr ap,
                             u32 sz) {
   NativeTarget* t = &a->base;
   MCEmitter* mc = t->mc;
   KitCgTypeId i32t = builtin_id(KIT_CG_BUILTIN_I32);
   u32 span = align_up_u32(sz, 4u);
   u32 enc7;
   NativeLoc cur = native_loc_reg(i32t, NATIVE_REG_INT, ARM_SCRATCH);
   NativeLoc nxt = native_loc_reg(i32t, NATIVE_REG_INT, ARM_TMP);
   NativeAddr src;
   AggregateAccess acc;
   /* cur = *ap. */
   arm_emit_mem(a, 1, cur, ap, native_mem_for_type(t, i32t, 4));
   /* 8-byte align the cursor: cur = (cur + 7) & ~7. ~7 (0xFFFFFFF8) is not a
    * Thumb modified immediate, so clear the low 3 bits with BIC #7 (op4=1; #7 IS
    * encodable). */
   arm_emit_t32(mc, arm_add_imm12(ARM_SCRATCH, ARM_SCRATCH, 7u));
   thumb_expand_imm_encode(7u, &enc7);
   arm_emit_t32(mc, arm_dp_imm(1u, 0u, ARM_SCRATCH, ARM_SCRATCH, enc7)); /* BIC */
   /* *ap = cur + span. */
   arm_emit_t32(mc, arm_add_imm12(ARM_TMP, ARM_SCRATCH, span));
   arm_emit_mem(a, 0, nxt, ap, native_mem_for_type(t, i32t, 4));
   /* Copy sz bytes from [cur] to the destination. */
   memset(&src, 0, sizeof src);
   src.base_kind = NATIVE_ADDR_BASE_REG;
   src.base.reg = ARM_SCRATCH;
   src.base_type = i32t;
   memset(&acc, 0, sizeof acc);
   acc.type = i32t;
   acc.size = sz;
   acc.align = 4u;
   arm_copy_bytes(t, dst, src, acc);
 }
 
 static void arm_va_arg_core(Arm32NativeTarget* a, NativeLoc dst, NativeAddr ap,
                             KitCgTypeId type) {
   NativeTarget* t = &a->base;
   MCEmitter* mc = t->mc;
   KitCgTypeId i32t = builtin_id(KIT_CG_BUILTIN_I32);
   u32 sz = native_type_size(t, type);
   /* The cursor must survive the value load (dst = [cursor]) and the +4 update,
    * so it cannot alias `dst`. O1 destinations come from the scoped r0..r3 bank;
    * the O0 direct path may use IP, so choose LR only for that case and IP
    * otherwise. */
   u32 cur_reg = (dst.kind == NATIVE_LOC_REG && loc_reg(dst) == ARM_SCRATCH)
                     ? ARM_TMP
                     : ARM_SCRATCH;
   NativeLoc cur = native_loc_reg(i32t, NATIVE_REG_INT, cur_reg);
   NativeAddr from;
   ABIVaListInfo vai = abi_va_list_layout(t->c->abi);
   if (vai.kind != ABI_VA_LIST_POINTER) arm_panic(a, "unsupported va_list layout");
   if (dst.kind != NATIVE_LOC_REG) arm_panic(a, "va_arg destination must be reg");
   /* cur = *ap; load value from [cur]; *ap = cur + 4 (one GP-slot stride). */
   arm_emit_mem(a, 1, cur, ap, native_mem_for_type(t, i32t, 4));
   memset(&from, 0, sizeof from);
   from.base_kind = NATIVE_ADDR_BASE_REG;
   from.base.reg = cur_reg;
   from.base_type = type;
   arm_emit_mem(a, 1, dst, from, native_mem_for_type(t, type, sz));
   arm_emit_t32(mc, arm_add_imm12(cur_reg, cur_reg, 4u));
   arm_emit_mem(a, 0, cur, ap, native_mem_for_type(t, i32t, 4));
 }
 
 static void arm_va_copy_core(Arm32NativeTarget* a, NativeAddr dst_ap,
                              NativeAddr src_ap) {
   NativeTarget* t = &a->base;
   KitCgTypeId i32t = builtin_id(KIT_CG_BUILTIN_I32);
   NativeLoc tmp = native_loc_reg(i32t, NATIVE_REG_INT, ARM_SCRATCH);
   arm_emit_mem(a, 1, tmp, src_ap, native_mem_for_type(t, i32t, 4));
   arm_emit_mem(a, 0, tmp, dst_ap, native_mem_for_type(t, i32t, 4));
 }
 
 /* ---- NativeTarget (optimizer) hooks: ap is a materialized register ---- */
 static NativeAddr arm_va_addr_from_ptr(NativeLoc ap_ptr) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_REG;
   addr.cls = NATIVE_REG_INT;
   addr.base.reg = ap_ptr.v.reg;
   addr.base_type = ap_ptr.type;
   return addr;
 }
 /* The destination memory of a value-location: a wide (i64/double) va_arg result
  * is memory-resident, so the optimizer hands its storage (a frame/stack slot, or
  * an explicit address), NOT a pointer in a register. Convert it to a NativeAddr
  * the byte-copy can write through. (Mirrors RISC-V's storage-address
  * resolver.) */
 static NativeAddr arm_loc_addr(Arm32NativeTarget* a, NativeLoc loc) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   switch ((NativeLocKind)loc.kind) {
     case NATIVE_LOC_FRAME:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = loc.v.frame;
       addr.base_type = loc.type;
       return addr;
     case NATIVE_LOC_STACK:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = loc.v.stack.slot;
       addr.base_type = loc.type;
       addr.offset = loc.v.stack.offset;
       return addr;
     case NATIVE_LOC_ADDR:
       return loc.v.addr;
     default:
       arm_panic(a, "va_arg destination is not addressable");
   }
 }
 static void arm_va_start_native(NativeTarget* t, NativeLoc ap_ptr) {
   arm_va_start_core(arm_of(t), arm_va_addr_from_ptr(ap_ptr));
 }
 static void arm_va_arg_native(NativeTarget* t, NativeLoc dst, NativeLoc ap_ptr,
                               KitCgTypeId type) {
   Arm32NativeTarget* a = arm_of(t);
   if (arm_va_arg_is_wide(t, type)) {
     arm_va_arg_wide(a, arm_loc_addr(a, dst), arm_va_addr_from_ptr(ap_ptr),
                     native_type_size(t, type));
     return;
   }
   arm_va_arg_core(a, dst, arm_va_addr_from_ptr(ap_ptr), type);
 }
 static void arm_va_end_native(NativeTarget* t, NativeLoc ap_ptr) {
   (void)t;
   (void)ap_ptr;
 }
 static void arm_va_copy_native(NativeTarget* t, NativeLoc dst_ap_ptr,
                                NativeLoc src_ap_ptr) {
   arm_va_copy_core(arm_of(t), arm_va_addr_from_ptr(dst_ap_ptr),
                    arm_va_addr_from_ptr(src_ap_ptr));
 }
 
 /* Materialize the boolean of ARM condition `cc` into rd: MOV rd,#0; IT cc;
  * MOV<cc> rd,#1. The flag state must already be set by a preceding CMP; the
  * leading MOV.W (no S) does not disturb it. */
 static void arm_emit_setcc(MCEmitter* mc, u32 rd, u32 cc) {
   u32 enc0, enc1;
   thumb_expand_imm_encode(0u, &enc0);
   thumb_expand_imm_encode(1u, &enc1);
   arm_emit_t32(mc, arm_mov_imm(rd, enc0));
   arm_emit_t16(mc, (u16)(0xbf08u | (cc << 4))); /* IT cc (mask 0b1000) */
   arm_emit_t32(mc, arm_mov_imm(rd, enc1));
 }
 
 /* The 32-bit `__builtin_*_overflow` family. dsts = [value, overflow]; args =
  * [a, b]. i64 overflow is lowered to lane ops by the cg layer (arith.c), so the
  * backend only sees the 32-bit forms. LR/IP are backend-private temporaries on
  * O1; on O0, IP's operand value has already been consumed before these internal
  * sequences use it. */
 /* nd_intrinsic forwards an immediate argument as NATIVE_LOC_IMM (not in a
  * register). A value-consuming intrinsic needs it in a register: materialize it
  * into `into` (an output register, free to reuse as an input) and return the reg
  * loc; pass a register arg through unchanged. Mirrors riscv's intrinsic path. */
 static NativeLoc arm_intrin_arg(Arm32NativeTarget* a, NativeLoc arg, u32 into) {
   NativeLoc r;
   if (arg.kind != NATIVE_LOC_IMM) return arg;
   r = native_loc_reg(arg.type ? arg.type : builtin_id(KIT_CG_BUILTIN_I32),
                       NATIVE_REG_INT, into);
   arm_load_imm(&a->base, r, arg.v.imm);
   return r;
 }
 
 static void arm_overflow(Arm32NativeTarget* a, IntrinKind kind,
                          const NativeLoc* dsts, const NativeLoc* args) {
   MCEmitter* mc = a->base.mc;
   u32 ra = loc_reg(args[0]), rb = loc_reg(args[1]);
   u32 rd = loc_reg(dsts[0]), rovf = loc_reg(dsts[1]);
   u32 enc0;
   switch (kind) {
     case INTRIN_SADD_OVERFLOW:
     case INTRIN_SSUB_OVERFLOW:
       /* r = a +/- b; signed ovf is the sign bit of ((a^r)&(b^r)) [ADD] or
        * ((a^b)&(a^r)) [SUB]. */
       if (kind == INTRIN_SADD_OVERFLOW)
         arm_emit_t32(mc, arm_add_reg(ARM_TMP, ra, rb));
       else
         arm_emit_t32(mc, arm_sub_reg(ARM_TMP, ra, rb));
       arm_emit_t32(mc, arm_eor_reg(ARM_SCRATCH, ra, ARM_TMP)); /* a ^ r */
       if (kind == INTRIN_SADD_OVERFLOW)
         arm_emit_t32(mc, arm_eor_reg(rovf, rb, ARM_TMP)); /* b ^ r */
       else
         arm_emit_t32(mc, arm_eor_reg(rovf, ra, rb)); /* a ^ b */
       arm_emit_t32(mc, arm_and_reg(rovf, rovf, ARM_SCRATCH));
       arm_emit_t32(mc, arm_shift_imm(1u, rovf, rovf, 31u)); /* LSR #31 -> 0/1 */
       arm_emit_t16(mc, arm_mov_hi(rd, ARM_TMP));
       return;
     case INTRIN_UADD_OVERFLOW:
       arm_emit_t32(mc, arm_add_reg(ARM_TMP, ra, rb)); /* r = a + b */
       arm_emit_t32(mc, arm_cmp_reg(ARM_TMP, ra));     /* CMP r, a */
       arm_emit_setcc(mc, rovf, ARM_CC_CC);            /* ovf = (r < a) unsigned */
       arm_emit_t16(mc, arm_mov_hi(rd, ARM_TMP));
       return;
     case INTRIN_USUB_OVERFLOW:
       arm_emit_t32(mc, arm_cmp_reg(ra, rb));          /* CMP a, b */
       arm_emit_t32(mc, arm_sub_reg(ARM_TMP, ra, rb)); /* r = a - b (no flags) */
       arm_emit_setcc(mc, rovf, ARM_CC_CC);            /* ovf = (a < b) unsigned */
       arm_emit_t16(mc, arm_mov_hi(rd, ARM_TMP));
       return;
     case INTRIN_UMUL_OVERFLOW:
       /* UMULL lr:ip = a*b; ovf = (high != 0); value = low. */
       arm_emit_t32(mc, arm_umull(ARM_TMP, ARM_SCRATCH, ra, rb));
       thumb_expand_imm_encode(0u, &enc0);
       arm_emit_t32(mc, arm_cmp_imm(ARM_SCRATCH, enc0));
       arm_emit_setcc(mc, rovf, ARM_CC_NE);
       arm_emit_t16(mc, arm_mov_hi(rd, ARM_TMP));
       return;
     case INTRIN_SMUL_OVERFLOW:
       /* SMULL lr:ip = a*b; ovf = (high != (low >>s 31)); value = low. */
       arm_emit_t32(mc, arm_smull(ARM_TMP, ARM_SCRATCH, ra, rb));
       arm_emit_t32(mc, arm_shift_imm(2u, rovf, ARM_TMP, 31u)); /* rovf = low ASR 31 */
       arm_emit_t32(mc, arm_cmp_reg(ARM_SCRATCH, rovf));
       arm_emit_setcc(mc, rovf, ARM_CC_NE);
       arm_emit_t16(mc, arm_mov_hi(rd, ARM_TMP));
       return;
     default:
       arm_panic(a, "arm_overflow: not an overflow intrinsic");
   }
 }
 
 static void arm_intrinsic(NativeTarget* t, IntrinKind kind,
                           const NativeLoc* dsts, u32 ndst, const NativeLoc* args,
                           u32 narg) {
   Arm32NativeTarget* a = arm_of(t);
   MCEmitter* mc = t->mc;
   switch (kind) {
     case INTRIN_TRAP:
       arm_trap(t);
       return;
     case INTRIN_MEMMOVE: {
       /* Overlap-safe move: copy forward when dst < src (unsigned), else backward,
        * so each granule's load precedes any store that could clobber it. dst/src
        * are pointer registers, size a constant; the transfer reg aliases neither
        * base. Shares the granule ladder with copy_bytes. */
       MCLabel forward = arm_label_new(t), done = arm_label_new(t);
       u32 dst, src, size, xfer;
       if (narg != 3u || args[0].kind != NATIVE_LOC_REG ||
           args[1].kind != NATIVE_LOC_REG || args[2].kind != NATIVE_LOC_IMM)
         arm_panic(a, "memmove: expected (reg dst, reg src, imm size)");
       if (args[2].v.imm < 0 || args[2].v.imm > 4096)
         arm_panic(a, "memmove: size out of range (needs a runtime loop)");
       dst = loc_reg(args[0]);
       src = loc_reg(args[1]);
       size = (u32)args[2].v.imm;
       xfer = (dst != ARM_TMP && src != ARM_TMP) ? ARM_TMP : ARM_SCRATCH;
       if (xfer == dst || xfer == src)
         arm_panic(a, "memmove: no free transfer register");
       arm_cmp_branch(t, CMP_LT_U, args[0], args[1], forward);
       arm_copy_granules_dir(a, dst, src, size, xfer, /*backward=*/1);
       arm_jump(t, done);
       arm_label_place(t, forward);
       arm_copy_granules_dir(a, dst, src, size, xfer, /*backward=*/0);
       arm_label_place(t, done);
       return;
     }
     case INTRIN_CLZ:
       if (ndst == 1u && narg == 1u) {
         NativeLoc a0 = arm_intrin_arg(a, args[0], loc_reg(dsts[0]));
         arm_emit_t32(mc, arm_clz(loc_reg(dsts[0]), loc_reg(a0)));
         return;
       }
       break;
     case INTRIN_CTZ:
       /* RBIT reverses the bit order, so the trailing-zero count becomes a
        * leading-zero count of the reversed word. */
       if (ndst == 1u && narg == 1u) {
         NativeLoc a0 = arm_intrin_arg(a, args[0], loc_reg(dsts[0]));
         arm_emit_t32(mc, arm_rbit(loc_reg(dsts[0]), loc_reg(a0)));
         arm_emit_t32(mc, arm_clz(loc_reg(dsts[0]), loc_reg(dsts[0])));
         return;
       }
       break;
     case INTRIN_BSWAP:
       if (ndst == 1u && narg == 1u) {
         u32 width = native_type_size(t, dsts[0].type);
         NativeLoc a0 = arm_intrin_arg(a, args[0], loc_reg(dsts[0]));
         /* Use the 32-bit REV.W/REV16.W forms: the dst may be a high register
          * (IP at -O0 or r4..r11 when allocated at -O1) that the 16-bit T1
          * encoding cannot represent. */
         if (width == 4u) {
           arm_emit_t32(mc, arm_rev_w(loc_reg(dsts[0]), loc_reg(a0)));
           return;
         }
         if (width == 2u) {
           /* REV16 byte-swaps within each halfword: the low halfword of the
            * input becomes the swapped 16-bit result in the low halfword. */
           arm_emit_t32(mc, arm_rev16_w(loc_reg(dsts[0]), loc_reg(a0)));
           return;
         }
       }
       break;
     case INTRIN_POPCOUNT:
       /* 32-bit SWAR popcount (no hardware popcount on ARMv7-M). All masks are
        * ThumbExpandImm-encodable, so no MOVW/MOVT is needed. lr/ip are the
        * scratch. 64-bit popcount stays in the cg lane path / a follow-on. */
       if (ndst == 1u && narg == 1u && native_type_size(t, dsts[0].type) <= 4u) {
         u32 rd = loc_reg(dsts[0]);
         u32 rs = loc_reg(arm_intrin_arg(a, args[0], rd));
         u32 e55, e33, e0f, e01;
         thumb_expand_imm_encode(0x55555555u, &e55);
         thumb_expand_imm_encode(0x33333333u, &e33);
         thumb_expand_imm_encode(0x0f0f0f0fu, &e0f);
         thumb_expand_imm_encode(0x01010101u, &e01);
         arm_emit_t32(mc, arm_shift_imm(1u, ARM_SCRATCH, rs, 1u));            /* ip = x>>1 */
         arm_emit_t32(mc, arm_dp_imm(0u, 0u, ARM_SCRATCH, ARM_SCRATCH, e55)); /* ip &= 0x55.. */
         arm_emit_t32(mc, arm_sub_reg(ARM_TMP, rs, ARM_SCRATCH));            /* lr = x - ip */
         arm_emit_t32(mc, arm_dp_imm(0u, 0u, ARM_SCRATCH, ARM_TMP, e33));    /* ip = lr & 0x33.. */
         arm_emit_t32(mc, arm_shift_imm(1u, ARM_TMP, ARM_TMP, 2u));          /* lr >>= 2 */
         arm_emit_t32(mc, arm_dp_imm(0u, 0u, ARM_TMP, ARM_TMP, e33));        /* lr &= 0x33.. */
         arm_emit_t32(mc, arm_add_reg(ARM_TMP, ARM_TMP, ARM_SCRATCH));       /* lr += ip */
         arm_emit_t32(mc, arm_shift_imm(1u, ARM_SCRATCH, ARM_TMP, 4u));      /* ip = lr>>4 */
         arm_emit_t32(mc, arm_add_reg(ARM_TMP, ARM_TMP, ARM_SCRATCH));       /* lr += ip */
         arm_emit_t32(mc, arm_dp_imm(0u, 0u, ARM_TMP, ARM_TMP, e0f));        /* lr &= 0x0f.. */
         arm_emit_t32(mc, arm_mov_imm(ARM_SCRATCH, e01));                    /* ip = 0x01010101 */
         arm_emit_t32(mc, arm_mul(ARM_TMP, ARM_TMP, ARM_SCRATCH));           /* lr *= ip */
         arm_emit_t32(mc, arm_shift_imm(1u, rd, ARM_TMP, 24u));              /* rd = lr>>24 */
         return;
       }
       break;
     case INTRIN_EXPECT:
     case INTRIN_ASSUME_ALIGNED:
       /* Branch/alignment hints: the result is the first argument, unchanged. An
        * immediate value materializes straight into dst; an 8-byte value carries
        * two lanes, so move each (a single arm_move would drop the high lane). */
       if (ndst == 1u && narg >= 1u) {
         if (args[0].kind == NATIVE_LOC_IMM) {
           arm_load_imm(t, dsts[0], args[0].v.imm);
         } else if (native_type_size(t, dsts[0].type) > 4u &&
                    dsts[0].kind == NATIVE_LOC_REG &&
                    args[0].kind == NATIVE_LOC_REG) {
           u32 dlo = loc_reg(dsts[0]), slo = loc_reg(args[0]);
           if (dlo != slo) arm_emit_t16(mc, arm_mov_hi(dlo, slo));
           arm_emit_t16(mc, arm_mov_hi((dlo + 1u) & 0xfu, (slo + 1u) & 0xfu));
         } else {
           arm_move(t, dsts[0], args[0]);
         }
         return;
       }
       break;
     case INTRIN_PREFETCH:
       return; /* no-op: M-profile has no PLD */
     case INTRIN_CPU_NOP:
       arm_emit_t16(mc, arm_nop16());
       return;
     case INTRIN_CPU_YIELD:
       arm_emit_t16(mc, arm_yield16());
       return;
     case INTRIN_WFI:
       arm_emit_t16(mc, arm_wfi16());
       return;
     case INTRIN_WFE:
       arm_emit_t16(mc, arm_wfe16());
       return;
     case INTRIN_SEV:
       arm_emit_t16(mc, arm_sev16());
       return;
     case INTRIN_DMB:
       arm_emit_t32(mc, arm_dmb(ARM_BARRIER_SY));
       return;
     case INTRIN_DSB:
       arm_emit_t32(mc, arm_dsb(ARM_BARRIER_SY));
       return;
     case INTRIN_ISB:
       arm_emit_t32(mc, arm_isb(ARM_BARRIER_SY));
       return;
     case INTRIN_SADD_OVERFLOW:
     case INTRIN_SSUB_OVERFLOW:
     case INTRIN_UADD_OVERFLOW:
     case INTRIN_USUB_OVERFLOW:
     case INTRIN_SMUL_OVERFLOW:
     case INTRIN_UMUL_OVERFLOW:
       if (ndst == 2u && narg == 2u) {
         /* Materialize immediate operands into the output registers (free to use
          * as inputs: arm_overflow reads both operands before writing dsts). */
         NativeLoc av[2];
         av[0] = arm_intrin_arg(a, args[0], loc_reg(dsts[0]));
         av[1] = arm_intrin_arg(a, args[1], loc_reg(dsts[1]));
         arm_overflow(a, kind, dsts, av);
         return;
       }
       break;
     case INTRIN_SMUL_HIGH:
     case INTRIN_UMUL_HIGH:
       if (ndst == 1u && narg == 2u) {
         u32 rd = loc_reg(dsts[0]);
         u32 ra = loc_reg(args[0]);
         u32 rb = loc_reg(args[1]);
         arm_emit_t32(mc, kind == INTRIN_SMUL_HIGH
                              ? arm_smull(ARM_TMP, ARM_SCRATCH, ra, rb)
                              : arm_umull(ARM_TMP, ARM_SCRATCH, ra, rb));
         arm_emit_t16(mc, arm_mov_hi(rd, ARM_SCRATCH));
         return;
       }
       break;
     case INTRIN_FRAME_ADDRESS:
     case INTRIN_RETURN_ADDRESS:
       /* kit's prologue anchors r7 at the saved pair: [r7]=caller r7,
        * [r7+4]=this frame's saved lr (return address). The level is constant
        * (unrolls to `level` dependent loads up the fp chain). A frame-reading
        * function is kept off the frameless-leaf tier (reads_frame), so r7 is
        * always valid here. */
       if (ndst == 1u) {
         u32 level = (narg >= 1u && args[0].kind == NATIVE_LOC_IMM)
                         ? (u32)args[0].v.imm
                         : 0u;
         u32 rd = loc_reg(dsts[0]), i;
         arm_emit_t16(mc, arm_mov_hi(rd, ARM_FP)); /* rd = r7 */
         for (i = 0; i < level; ++i)
           arm_emit_t32(mc, arm_ldr_imm(rd, rd, 0)); /* rd = *(rd) */
         if (kind == INTRIN_RETURN_ADDRESS)
           arm_emit_t32(mc, arm_ldr_imm(rd, rd, 4u)); /* rd = *(rd+4) = saved lr */
         return;
       }
       break;
     default:
       break;
   }
   arm_panic(a, "intrinsic not lowered (FMA/syscall/setjmp/etc. are follow-ons)");
 }
 
 /* ============================ inline asm ============================ */
 
 _Noreturn static void arm_asm_panic_at(Compiler* c, SrcLoc loc,
                                        const char* msg) {
   compiler_panic(c, loc, "arm32 inline asm: %s", msg);
 }
 _Noreturn static void arm_asm_panic(NativeDirectTarget* d, const char* msg) {
   arm_asm_panic_at(d->base.c, d->loc, msg);
 }
 
 static void arm_asm_bound_reg(Operand* out, KitCgTypeId type,
                               NativeAllocClass cls, Reg reg) {
   memset(out, 0, sizeof *out);
   out->kind = ARM32_INLINE_OPK_REG;
   out->pad[0] =
       (cls == NATIVE_REG_FP) ? ARM32_INLINE_OPCLS_FP : ARM32_INLINE_OPCLS_INT;
   out->type = type;
   out->v.local = (CGLocal)reg;
 }
 static void arm_asm_bound_mem(Operand* out, KitCgTypeId type, Reg base) {
   memset(out, 0, sizeof *out);
   out->kind = OPK_INDIRECT;
   out->type = type;
   out->v.ind.base = (CGLocal)base;
   out->v.ind.index = CG_LOCAL_NONE;
 }
 
 /* Parse one clobber name into a core-register mask bit. r7 is the live frame
  * anchor and sp/pc are structural state, so an inline block may not declare
  * them clobbered; cc/memory contribute no register. */
 static void arm_asm_clobber_masks(Compiler* c, SrcLoc loc, const Sym* clobbers,
                                   u32 nclob, u32* int_mask, u32* fp_mask) {
   *int_mask = 0;
   *fp_mask = 0;
   for (u32 i = 0; i < nclob; ++i) {
     Slice s = pool_slice(c->global, clobbers[i]);
     char buf[16];
     uint32_t dwarf;
     if (!s.s || !s.len || s.len >= sizeof buf) continue;
     memcpy(buf, s.s, s.len);
     buf[s.len] = '\0';
     if (!strcmp(buf, "cc") || !strcmp(buf, "memory")) continue;
     if (arm32_register_index(buf, &dwarf) != 0 || dwarf > 15u)
       arm_asm_panic_at(c, loc, "unsupported clobber register");
     if (dwarf == ARM_FP || dwarf == 13u || dwarf == 15u)
       arm_asm_panic_at(c, loc, "structural register cannot be clobbered");
     if (dwarf <= 15u)
       *int_mask |= 1u << dwarf;
   }
 }
 
 /* Operand-address resolution for the direct (-O0) path (mirrors aa64). */
 static NativeAddr arm_direct_addr(NativeDirectTarget* d, Operand op) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   switch ((OpKind)op.kind) {
     case OPK_LOCAL:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = d->locals[op.v.local - 1u].home;
       addr.base_type = op.type;
       return addr;
     case OPK_INDIRECT:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME_VALUE;
       addr.base.frame = d->locals[op.v.ind.base - 1u].home;
       addr.cls = d->locals[op.v.ind.base - 1u].cls;
       addr.base_type = d->locals[op.v.ind.base - 1u].type;
       addr.offset = op.v.ind.ofs;
       return addr;
     default:
       arm_asm_panic(d, "operand is not addressable");
   }
 }
 static NativeAddr arm_direct_materialize_addr(NativeDirectTarget* d,
                                               Operand op) {
   NativeAddr addr = arm_direct_addr(d, op);
   if (addr.base_kind == NATIVE_ADDR_BASE_FRAME_VALUE) {
     NativeLoc base = native_loc_reg(addr.base_type, NATIVE_REG_INT, ARM_SCRATCH);
     NativeAddr load;
     memset(&load, 0, sizeof load);
     load.base_kind = NATIVE_ADDR_BASE_FRAME;
     load.base.frame = addr.base.frame;
     load.base_type = addr.base_type;
     arm_emit_mem(arm_of(d->native), 1, base, load,
                  native_mem_for_type(d->native, addr.base_type, 4));
     addr.base_kind = NATIVE_ADDR_BASE_REG;
     addr.base.reg = ARM_SCRATCH;
   }
   return addr;
 }
 
 static void arm_direct_load_operand_to_reg(NativeDirectTarget* d, Operand op,
                                            NativeLoc dst) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   switch ((OpKind)op.kind) {
     case OPK_IMM:
       d->native->load_imm(d->native, dst, op.v.imm);
       return;
     case OPK_LOCAL:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = d->locals[op.v.local - 1u].home;
       addr.base_type = op.type;
       arm_emit_mem(arm_of(d->native), 1, dst, addr,
                    native_mem_for_type(d->native, op.type, 0));
       return;
     case OPK_GLOBAL:
       addr.base_kind = NATIVE_ADDR_BASE_GLOBAL;
       addr.base.global.sym = op.v.global.sym;
       addr.base.global.addend = op.v.global.addend;
       addr.base_type = op.type;
       d->native->load_addr(d->native, dst, addr);
       return;
     case OPK_INDIRECT:
       addr = arm_direct_materialize_addr(d, op);
       arm_emit_mem(arm_of(d->native), 1, dst, addr,
                    native_mem_for_type(d->native, op.type, 0));
       return;
   }
   arm_asm_panic(d, "unsupported asm input operand");
 }
 static void arm_direct_load_address_to_reg(NativeDirectTarget* d, Operand op,
                                            NativeLoc dst) {
   /* For an "m" memory operand: a LOCAL's address is its frame home; an INDIRECT
    * (*(base+ofs)) names an address held in a frame slot; load_addr materializes
    * that typed FRAME_VALUE through backend-owned temporaries. */
   d->native->load_addr(d->native, dst, arm_direct_materialize_addr(d, op));
 }
 static void arm_direct_store_reg_to_operand(NativeDirectTarget* d, Operand op,
                                             NativeLoc src) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   if (op.kind == OPK_LOCAL) {
     addr.base_kind = NATIVE_ADDR_BASE_FRAME;
     addr.base.frame = d->locals[op.v.local - 1u].home;
     addr.base_type = op.type;
   } else {
     addr = arm_direct_materialize_addr(d, op);
   }
   arm_emit_mem(arm_of(d->native), 0, src, addr,
                native_mem_for_type(d->native, op.type, 0));
 }
 
 /* Callee-saved spill/restore for asm clobbers (r4..r11, excluding r7=fp). */
 typedef struct ArmAsmSavedClobber {
   NativeFrameSlot slot;
   Reg reg;
   KitCgTypeId type;
 } ArmAsmSavedClobber;
 
 static ArmAsmSavedClobber* arm_asm_save_callee_clobbers(Arm32NativeTarget* a,
                                                         u32 int_mask,
                                                         u32 fp_mask,
                                                         u32* nsaved_out) {
   ArmAsmSavedClobber* saved =
       arena_zarray(a->base.c->tu, ArmAsmSavedClobber, 12u);
   KitCgTypeId i32t = builtin_id(KIT_CG_BUILTIN_I32);
   u32 n = 0;
   (void)fp_mask;
   for (Reg r = 4u; r <= 11u; ++r) {
     NativeFrameSlotDesc sd;
     NativeAddr addr;
     if (r == ARM_FP || (int_mask & (1u << r)) == 0) continue;
     memset(&sd, 0, sizeof sd);
     sd.type = i32t;
     sd.size = 4;
     sd.align = 4;
     sd.kind = NATIVE_FRAME_SLOT_SAVE;
     saved[n].slot = a->base.frame_slot(&a->base, &sd);
     saved[n].reg = r;
     saved[n].type = i32t;
     memset(&addr, 0, sizeof addr);
     addr.base_kind = NATIVE_ADDR_BASE_FRAME;
     addr.base.frame = saved[n].slot;
     addr.base_type = i32t;
     arm_emit_mem(a, 0, native_loc_reg(i32t, NATIVE_REG_INT, r), addr,
                  native_mem_for_type(&a->base, i32t, 4));
     n++;
   }
   *nsaved_out = n;
   return saved;
 }
 static void arm_asm_restore_one_rec(Arm32NativeTarget* a,
                                     const ArmAsmSavedClobber* s) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = s->slot;
   addr.base_type = s->type;
   arm_emit_mem(a, 1, native_loc_reg(s->type, NATIVE_REG_INT, s->reg), addr,
                native_mem_for_type(&a->base, s->type, 4));
 }
 static void* arm_asm_hook_save_callee_clobbers(NativeDirectTarget* d,
                                                u32 int_mask, u32 fp_mask,
                                                u32* nsaved_out) {
   return arm_asm_save_callee_clobbers(arm_of(d->native), int_mask, fp_mask,
                                       nsaved_out);
 }
 static void arm_asm_hook_restore_one(NativeDirectTarget* d, void* saved,
                                      u32 idx) {
   arm_asm_restore_one_rec(arm_of(d->native),
                           &((ArmAsmSavedClobber*)saved)[idx]);
 }
 
 static void arm_asm_hook_run_template(NativeDirectTarget* d, const char* tmpl,
                                       const AsmConstraint* outs, u32 nout,
                                       Operand* bound_outs,
                                       const AsmConstraint* ins, u32 nin,
                                       Operand* bound_ins, const Sym* clobbers,
                                       u32 nclob) {
   Arm32Asm* asmh = arm32_asm_open(d->base.c);
   arm32_inline_bind(asmh, outs, nout, bound_outs, ins, nin, bound_ins, clobbers,
                     nclob);
   arm32_asm_run_template(asmh, d->native->mc, tmpl);
   arm32_asm_close(asmh);
 }
 
 static void arm_direct_asm_block(NativeDirectTarget* d, const char* tmpl,
                                  const AsmConstraint* outs, u32 nout,
                                  Operand* out_ops, const AsmConstraint* ins,
                                  u32 nin, const Operand* in_ops,
                                  const Sym* clobbers, u32 nclob,
                                  u32 clobber_abi_sets) {
   static const NativeAsmDirectHooks hooks = {
       .opk_reg = ARM32_INLINE_OPK_REG,
       .opcls_fp = ARM32_INLINE_OPCLS_FP,
       .panic = arm_asm_panic,
       .bound_reg = arm_asm_bound_reg,
       .bound_mem = arm_asm_bound_mem,
       .clobber_masks = arm_asm_clobber_masks,
       .save_callee_clobbers = arm_asm_hook_save_callee_clobbers,
       .restore_one = arm_asm_hook_restore_one,
       .load_operand_to_reg = arm_direct_load_operand_to_reg,
       .load_address_to_reg = arm_direct_load_address_to_reg,
       .store_reg_to_operand = arm_direct_store_reg_to_operand,
       .run_template = arm_asm_hook_run_template,
   };
   native_asm_bind_direct_operands(d, tmpl, outs, nout, out_ops, ins, nin,
                                   in_ops, clobbers, nclob, clobber_abi_sets,
                                   &hooks);
 }
 
 /* Convert a concrete memory-constraint location to a NativeAddr. */
 static NativeAddr arm_asm_in_loc_addr(NativeTarget* t, NativeLoc loc,
                                       KitCgTypeId type) {
   NativeAddr ma;
   memset(&ma, 0, sizeof ma);
   if (loc.kind == NATIVE_LOC_ADDR) {
     ma = loc.v.addr;
   } else if (loc.kind == NATIVE_LOC_FRAME || loc.kind == NATIVE_LOC_STACK) {
     ma.base_kind = NATIVE_ADDR_BASE_FRAME;
     ma.base.frame =
         loc.kind == NATIVE_LOC_FRAME ? loc.v.frame : loc.v.stack.slot;
     ma.offset = loc.kind == NATIVE_LOC_STACK ? loc.v.stack.offset : 0;
     ma.base_type = type;
   } else {
     arm_asm_panic_at(t->c, arm_of(t)->loc, "unsupported memory asm input");
   }
   return ma;
 }
 
 /* Reserve the next backend-private memory-base register: IP first, then LR.
  * The optimized emitter never presents IP/LR as asm operands, so both are free
  * here. */
 static Reg arm_asm_mem_base_reg(NativeTarget* t, u32* nbase) {
   Reg r;
   if (*nbase == 0u)
     r = (Reg)ARM_SCRATCH;
   else if (*nbase == 1u)
     r = (Reg)ARM_TMP;
   else
     arm_asm_panic_at(t->c, arm_of(t)->loc,
                      "too many memory inline-asm operands");
   (*nbase)++;
   return r;
 }
 
 static void arm_asm_native_panic(NativeTarget* t, SrcLoc loc,
                                  const char* msg) {
   arm_asm_panic_at(t->c, loc, msg);
 }
 
 static Reg arm_asm_native_mem_base_hook(NativeTarget* t, SrcLoc loc,
                                         NativeLoc src, u32* ntmp) {
   Reg base = arm_asm_mem_base_reg(t, ntmp);
   NativeAddr ma = arm_asm_in_loc_addr(t, src, src.type);
   (void)loc;
   arm_load_addr(
       t, native_loc_reg(builtin_id(KIT_CG_BUILTIN_I32), NATIVE_REG_INT, base),
       ma);
   return base;
 }
 
 static void arm_asm_native_run_template_hook(
     NativeTarget* t, const char* tmpl, const AsmConstraint* outs, u32 nout,
     Operand* bound_outs, const AsmConstraint* ins, u32 nin, Operand* bound_ins,
     const Sym* clobbers, u32 nclob) {
   Arm32Asm* asmh = arm32_asm_open(t->c);
   arm32_inline_bind(asmh, outs, nout, bound_outs, ins, nin, bound_ins, clobbers,
                     nclob);
   arm32_asm_run_template(asmh, t->mc, tmpl);
   arm32_asm_close(asmh);
 }
 
 static void arm_asm_block_native(NativeTarget* t, const char* tmpl,
                                  const AsmConstraint* outs, u32 nout,
                                  NativeLoc* out_locs, const AsmConstraint* ins,
                                  u32 nin, const NativeLoc* in_locs,
                                  const Sym* clobbers, u32 nclob) {
   SrcLoc loc = arm_of(t)->loc;
   static const NativeAsmNativeHooks hooks = {
       .panic = arm_asm_native_panic,
       .bound_reg = arm_asm_bound_reg,
       .bound_mem = arm_asm_bound_mem,
       .mem_base = arm_asm_native_mem_base_hook,
       .run_template = arm_asm_native_run_template_hook,
   };
   native_asm_bind_native_operands(t, loc, tmpl, outs, nout, out_locs, ins, nin,
                                   in_locs, clobbers, nclob, &hooks);
 }
 
 /* ============================ construction ============================ */
 
 NativeTarget* arm32_native_target_new(Compiler* c, ObjBuilder* obj,
                                       MCEmitter* mc) {
   Arm32NativeTarget* a = arena_znew(c->tu, Arm32NativeTarget);
   NativeTarget* t;
   if (!a) return NULL;
   t = &a->base;
   t->c = c;
   t->obj = obj;
   t->mc = mc;
   a->variant = arm32_variant_for_kind(c->target.arch);
   native_frame_init(&a->frame, c);
   t->regs = &arm_reg_info;
   t->class_for_type = native_class_for_type_fp_le8;
   t->imm_legal = arm_imm_legal;
   t->addr_legal = arm_addr_legal;
   t->can_fold_shift_into_alu = arm_can_fold_shift_into_alu;
   t->func_begin = arm_func_begin;
   t->func_begin_known_frame = arm_func_begin_known_frame;
   t->reserve_callee_saves = arm_reserve_callee_saves;
   t->signature_stack_bytes = arm_signature_stack_bytes;
   t->call_stack_bytes = arm_call_stack_bytes;
   t->has_store_zero_reg = 0; /* ARM has no zero register */
   t->func_end = arm_func_end;
   t->frame_slot = arm_frame_slot;
   t->release_frame_slot = arm_release_frame_slot;
   t->frame_slot_debug_loc = arm_frame_slot_debug_loc;
   t->bind_param = arm_bind_native_param;
   t->label_new = arm_label_new;
   t->label_place = arm_label_place;
   t->jump = arm_jump;
   t->cmp_branch = arm_cmp_branch;
   t->indirect_branch = arm_indirect_branch;
   t->load_label_addr = arm_load_label_addr;
   t->move = arm_move;
   t->load_imm = arm_load_imm;
   t->load_const = arm_load_const;
   t->load_addr = arm_load_addr;
   t->load = arm_load;
   t->store = arm_store;
   t->tls_addr_of = arm_tls_addr_of;
   t->copy_bytes = arm_copy_bytes;
   t->set_bytes = arm_set_bytes;
   t->bitfield_load = arm_bitfield_load;
   t->bitfield_store = arm_bitfield_store;
   t->binop = arm_binop;
   t->unop = arm_unop;
   t->cmp = arm_cmp;
   t->convert = arm_convert;
   t->binop_rr = arm_binop_rr;
   t->move_rr = arm_move_rr;
   t->cmp_rr = arm_cmp_rr;
   t->convert_rr = arm_convert_rr;
   t->alloca_ = arm_alloca;
   t->spill = arm_spill;
   t->reload = arm_reload;
   t->marshal_call = arm_marshal_call;
   t->emit_call = arm_emit_call;
   t->marshal_ret = arm_marshal_ret;
   t->ret = arm_ret;
   t->atomic_load = arm_atomic_load;
   t->atomic_store = arm_atomic_store;
   t->atomic_rmw = arm_atomic_rmw;
   t->atomic_cas = arm_atomic_cas;
   t->fence = arm_fence;
   t->va_start_ = arm_va_start_native;
   t->va_arg_ = arm_va_arg_native;
   t->va_end_ = arm_va_end_native;
   t->va_copy_ = arm_va_copy_native;
   t->intrinsic = arm_intrinsic;
   t->asm_block = arm_asm_block_native;
   t->file_scope_asm = native_file_scope_asm;
   t->trap = arm_trap;
   t->set_loc = arm_set_loc;
   t->finalize = native_finalize;
   return t;
 }
 
 /* ============================ NativeOps (-O0) ============================ */
 
 static void arm_bind_param(NativeDirectTarget* d, const CGParamDesc* p,
                            CGLocal local, NativeDirectLocal* l) {
   NativeLoc dst;
   (void)local;
   memset(&dst, 0, sizeof dst);
   dst.kind = NATIVE_LOC_FRAME;
   dst.type = p->type;
   dst.v.frame = l->home;
   arm_bind_native_param(d->native, p, dst);
 }
 
 /* A sibling call's outgoing stack args reuse the caller's incoming-arg window in
  * place; it is realizable only if they fit. (Register-only callees always fit.) */
 static const char* arm_no_tail(NativeDirectTarget* d, const CGCallDesc* call) {
   NativeCallDesc nd;
   native_direct_project_tail_call_desc(d, call, &nd);
   if (arm_call_stack_size(d->native, &nd) > arm_of(d->native)->incoming_stack_size)
     return "arm32 tail call: callee stack args exceed the caller's window";
   return NULL;
 }
 
 /* ---- Direct (-O0) varargs wrappers: resolve a va_list operand's address into
  * a register, then call the shared cores. The va cores use ip (cursor/temp) and
  * lr (copy scratch); the va_list base register must be distinct, so the direct
  * wrappers stage it into a callee-saved scratch (r4) before calling. ---- */
 static NativeAddr arm_direct_pointer_addr(NativeDirectTarget* d, Operand op,
                                           Reg reg) {
   NativeAddr addr;
   KitCgTypeId i32t = builtin_id(KIT_CG_BUILTIN_I32);
   memset(&addr, 0, sizeof addr);
   if (op.kind == OPK_LOCAL) {
     /* The local HOLDS the pointer (&ap): load its home value. */
     NativeLoc base = native_loc_reg(i32t, NATIVE_REG_INT, reg);
     NativeAddr load;
     memset(&load, 0, sizeof load);
     load.base_kind = NATIVE_ADDR_BASE_FRAME;
     load.base.frame = d->locals[op.v.local - 1u].home;
     load.base_type = op.type;
     arm_emit_mem(arm_of(d->native), 1, base, load,
                  native_mem_for_type(d->native, op.type, 4));
     addr.base_kind = NATIVE_ADDR_BASE_REG;
     addr.base.reg = reg;
     addr.base_type = i32t;
     return addr;
   }
   /* OPK_INDIRECT names *(base+ofs): its address base+ofs IS the pointer. */
   {
     NativeAddr m = arm_direct_materialize_addr(d, op);
     NativeLoc dst = native_loc_reg(i32t, NATIVE_REG_INT, reg);
     d->native->load_addr(d->native, dst, m);
     addr.base_kind = NATIVE_ADDR_BASE_REG;
     addr.base.reg = reg;
     addr.base_type = i32t;
     return addr;
   }
 }
 
 static void arm_va_start_(NativeDirectTarget* d, Operand ap_addr) {
   arm_va_start_core(arm_of(d->native), arm_direct_pointer_addr(d, ap_addr, 4u));
 }
 static void arm_va_arg_(NativeDirectTarget* d, Operand dst_op, Operand ap_addr,
                         KitCgTypeId type) {
   Arm32NativeTarget* a = arm_of(d->native);
   NativeAddr ap = arm_direct_pointer_addr(d, ap_addr, 4u);
   if (arm_va_arg_is_wide(d->native, type)) {
     /* Wide (i64/double) va_arg copies straight from the save area into the
      * destination memory. arm_va_arg_wide uses ip (cursor) + lr (copy scratch),
      * so an OPK_INDIRECT destination pointer must live in a distinct reg (r5),
      * not ip — materialize it there. A FRAME (OPK_LOCAL) destination needs no
      * register. */
     NativeAddr dst = arm_direct_addr(d, dst_op);
     if (dst.base_kind == NATIVE_ADDR_BASE_FRAME_VALUE) {
       NativeLoc base = native_loc_reg(dst.base_type, NATIVE_REG_INT, 5u);
       NativeAddr load;
       memset(&load, 0, sizeof load);
       load.base_kind = NATIVE_ADDR_BASE_FRAME;
       load.base.frame = dst.base.frame;
       load.base_type = dst.base_type;
       arm_emit_mem(a, 1, base, load,
                    native_mem_for_type(d->native, dst.base_type, 4));
       dst.base_kind = NATIVE_ADDR_BASE_REG;
       dst.base.reg = 5u;
     }
     arm_va_arg_wide(a, dst, ap, native_type_size(d->native, type));
     return;
   }
   {
     NativeLoc res = native_loc_reg(type, NATIVE_REG_INT, 5u);
     NativeAddr dst;
     arm_va_arg_core(a, res, ap, type);
     dst = arm_direct_materialize_addr(d, dst_op);
     arm_emit_mem(a, 0, res, dst,
                  native_mem_for_type(d->native, type,
                                      native_type_size(d->native, type)));
   }
 }
 static void arm_va_end_(NativeDirectTarget* d, Operand ap_addr) {
   (void)d;
   (void)ap_addr;
 }
 static void arm_va_copy_(NativeDirectTarget* d, Operand dst_op, Operand src_op) {
   Arm32NativeTarget* a = arm_of(d->native);
   NativeAddr src = arm_direct_pointer_addr(d, src_op, 4u);
   NativeAddr dst = arm_direct_pointer_addr(d, dst_op, 5u);
   arm_va_copy_core(a, dst, src);
 }
 
 static const NativeOps arm_direct_ops = {
     .bind_param = arm_bind_param,
     .tail_call_unrealizable_reason = arm_no_tail,
     .va_start_ = arm_va_start_,
     .va_arg_ = arm_va_arg_,
     .va_end_ = arm_va_end_,
     .va_copy_ = arm_va_copy_,
     .asm_block = arm_direct_asm_block,
 };
 
 const NativeOps* arm32_native_direct_ops(void) { return &arm_direct_ops; }