kit

pass_native_emit.c (121889B)

---

 #include <string.h>
 
 #include "cg/native_asm.h"
 #include "cg/type.h"
 #include "core/metrics.h"
 #include "core/pool.h"
 #include "opt/opt_internal.h"
 
 #undef Operand
 #undef CGParamDesc
 #undef CGCallDesc
 #undef CGFuncDesc
 #undef CGLocalStorage
 #undef CGABIValue
 #undef CGABIPart
 #undef CGCallPlan
 #undef CGCallPlanMove
 #undef CGCallPlanRet
 #undef CGScopeDesc
 
 typedef struct NativeEmitTempScope NativeEmitTempScope;
 
 typedef struct NativeEmitCtx {
   Compiler* c;
   Func* f;
   NativeTarget* target;
   NativeFrameSlot* slot_map;
   MCLabel* labels;
   u8* label_placed;
   ObjSecId local_static_sec;
   ObjSymId local_static_sym;
   u32 local_static_base;
   u32 local_static_size;
   u8 local_static_active;
   /* Set by emit_block for the IR_RET that is the last inst of the last block
    * in emit_order. emit_ret consults it to skip the trailing
    * branch-to-epilogue: func_end places the epilogue label at the very next
    * position, so the branch would just jump to the next 4 bytes. */
   u8 emitting_terminal_ret;
   /* Instruction-scoped materialization state. NULL outside emit_inst (notably
    * during frame planning/parameter binding). */
   NativeEmitTempScope* temps;
   /* Physical registers live after each MIR instruction, indexed by InstId.
    * Used only to scavenge dead caller-saved registers after the dedicated temp
    * bank is exhausted. */
   OptHardRegSet* live_after_by_inst;
   u32 live_after_cap;
   u8 live_after_ready;
   /* Clean frame-value forwarding cache, indexed by physical register. A
    * nonzero slot means the register holds the exact value already stored in
    * that private FS_SPILL slot. Memory is always authoritative. */
   FrameSlot frame_cache_slot[OPT_REG_CLASSES][OPT_MAX_HARD_REGS];
   KitCgTypeId frame_cache_type[OPT_REG_CLASSES][OPT_MAX_HARD_REGS];
   u32 frame_cache_held[OPT_REG_CLASSES];
   /* Callee-saved registers selected by the pre-frame asm staging plan.  The
    * actual instruction-local allocator is deterministic and chooses the same
    * registers during emission; these masks make their ABI preservation part of
    * the known frame instead of allocating save slots mid-body. */
   u32 asm_stage_callee_used[OPT_REG_CLASSES];
   /* Callee-saved registers already represented in the known frame, from MIR
    * allocation or asm staging. A dead register in this set may safely retain a
    * clean spill value without creating a new ABI preservation obligation. */
   u32 callee_saved_used[OPT_REG_CLASSES];
 } NativeEmitCtx;
 
 static void frame_cache_clear(NativeEmitCtx* e);
 static void frame_cache_drop_reg(NativeEmitCtx* e, u32 cls, Reg reg);
 static void frame_cache_invalidate_masks(NativeEmitCtx* e,
                                          const u32* masks);
 static NativeLoc frame_cache_take_slot(NativeEmitCtx* e, FrameSlot slot,
                                        NativeLoc source,
                                        NativeAllocClass cls,
                                        KitCgTypeId type, Reg avoid_a,
                                        Reg avoid_b);
 static void compute_emit_live_after(NativeEmitCtx* e);
 
 static _Noreturn void emit_panic(NativeEmitCtx* e, SrcLoc loc,
                                  const char* msg) {
   compiler_panic(e->c, loc, "opt native emit: %s", msg);
 }
 
 static void emit_local_static_begin(NativeEmitCtx* e,
                                     const CGLocalStaticDataDesc* desc,
                                     SrcLoc loc) {
   Sym name;
   SecKind kind;
   u16 flags;
   u32 align;
   if (!desc) emit_panic(e, loc, "missing local static data descriptor");
   if (e->local_static_active) emit_panic(e, loc, "nested local static data");
   if (desc->attrs.section) {
     name = (Sym)desc->attrs.section;
     kind =
         (desc->attrs.flags & KIT_CG_DATADEF_READONLY) ? SEC_RODATA : SEC_DATA;
     flags = (desc->attrs.flags & KIT_CG_DATADEF_READONLY)
                 ? SF_ALLOC
                 : (SF_ALLOC | SF_WRITE);
   } else if (desc->attrs.flags & KIT_CG_DATADEF_READONLY) {
     name = pool_intern_slice(e->c->global, SLICE_LIT(".rodata"));
     kind = SEC_RODATA;
     flags = SF_ALLOC;
   } else {
     name = pool_intern_slice(e->c->global, SLICE_LIT(".data"));
     kind = SEC_DATA;
     flags = SF_ALLOC | SF_WRITE;
   }
   align = desc->align ? desc->align : 1u;
   e->local_static_sec = obj_section(e->target->obj, name, kind, flags, align);
   e->local_static_base =
       obj_align_to(e->target->obj, e->local_static_sec, align);
   e->local_static_size = 0;
   e->local_static_sym = desc->sym;
   e->local_static_active = 1;
 }
 
 static void emit_local_static_write(NativeEmitCtx* e, const u8* data, u64 len,
                                     SrcLoc loc) {
   u8 zero[64];
   u64 orig_len = len;
   if (!e->local_static_active) emit_panic(e, loc, "local static data inactive");
   if (!len) return;
   if (data) {
     obj_write(e->target->obj, e->local_static_sec, data, (size_t)len);
   } else {
     memset(zero, 0, sizeof zero);
     while (len >= sizeof zero) {
       obj_write(e->target->obj, e->local_static_sec, zero, sizeof zero);
       len -= sizeof zero;
     }
     if (len) obj_write(e->target->obj, e->local_static_sec, zero, (size_t)len);
   }
   e->local_static_size += (u32)orig_len;
 }
 
 static void emit_local_static_label_addr(NativeEmitCtx* e, MCLabel target,
                                          i64 addend, u32 width, SrcLoc loc) {
   u8 zero[8];
   u32 off;
   RelocKind kind;
   if (!e->local_static_active) emit_panic(e, loc, "local static data inactive");
   /* A jump-table / label-address slot is one target pointer wide: 8 bytes
    * (R_ABS64) on a 64-bit target, 4 bytes (R_ABS32) on rv32/ELFCLASS32. */
   if (width == 8u)
     kind = R_ABS64;
   else if (width == 4u)
     kind = R_ABS32;
   else {
     emit_panic(e, loc, "unsupported local static label width");
     return;
   }
   memset(zero, 0, sizeof zero);
   off = e->local_static_base + e->local_static_size;
   obj_write(e->target->obj, e->local_static_sec, zero, width);
   mc_emit_label_data_reloc(e->target->mc, e->local_static_sec, off,
                                        target, kind, width, addend);
   e->local_static_size += width;
 }
 
 static void emit_local_static_end(NativeEmitCtx* e, SrcLoc loc) {
   if (!e->local_static_active) emit_panic(e, loc, "local static data inactive");
   obj_symbol_define_live(e->target->obj, e->local_static_sym,
                          e->local_static_sec, e->local_static_base,
                          e->local_static_size);
   e->local_static_active = 0;
   e->local_static_sec = OBJ_SEC_NONE;
   e->local_static_sym = OBJ_SYM_NONE;
   e->local_static_base = 0;
   e->local_static_size = 0;
 }
 
 static u32 type_size_or(Compiler* c, KitCgTypeId type, u32 fallback) {
   u64 n = type ? cg_type_size(c, type) : 0u;
   if (!n || n > 0xffffffffull) return fallback;
   return (u32)n;
 }
 
 static u32 type_align_or(Compiler* c, KitCgTypeId type, u32 fallback) {
   u64 n = type ? cg_type_align(c, type) : 0u;
   if (!n || n > 0xffffffffull) return fallback;
   return (u32)n;
 }
 
 static MemAccess mem_for_type(Compiler* c, KitCgTypeId type) {
   MemAccess mem;
   memset(&mem, 0, sizeof mem);
   mem.type = type;
   mem.size = type_size_or(c, type, 8u);
   mem.align = type_align_or(c, type, mem.size >= 8u ? 8u : mem.size);
   return mem;
 }
 
 static NativeAllocClass class_for_type(NativeEmitCtx* e, KitCgTypeId type) {
   if (e->target->class_for_type)
     return e->target->class_for_type(e->target, type);
   return cg_type_is_float(e->c, type) ? NATIVE_REG_FP : NATIVE_REG_INT;
 }
 
 static NativeLoc loc_none(void) {
   NativeLoc loc;
   memset(&loc, 0, sizeof loc);
   return loc;
 }
 
 static NativeLoc loc_reg(KitCgTypeId type, NativeAllocClass cls, Reg reg) {
   NativeLoc loc;
   memset(&loc, 0, sizeof loc);
   loc.kind = NATIVE_LOC_REG;
   loc.cls = (u8)cls;
   loc.type = type;
   loc.v.reg = reg;
   return loc;
 }
 
 static NativeLoc loc_frame(KitCgTypeId type, NativeAllocClass cls,
                            NativeFrameSlot slot) {
   NativeLoc loc;
   memset(&loc, 0, sizeof loc);
   loc.kind = NATIVE_LOC_FRAME;
   loc.cls = (u8)cls;
   loc.type = type;
   loc.v.frame = slot;
   return loc;
 }
 
 static NativeLoc loc_frame_addr(KitCgTypeId type, NativeFrameSlot slot) {
   NativeLoc loc;
   memset(&loc, 0, sizeof loc);
   loc.kind = NATIVE_LOC_FRAME_ADDR;
   loc.cls = NATIVE_REG_INT;
   loc.type = type;
   loc.v.frame = slot;
   return loc;
 }
 
 static NativeLoc loc_imm(KitCgTypeId type, i64 imm) {
   NativeLoc loc;
   memset(&loc, 0, sizeof loc);
   loc.kind = NATIVE_LOC_IMM;
   loc.cls = NATIVE_REG_INT;
   loc.type = type;
   loc.v.imm = imm;
   return loc;
 }
 
 static NativeLoc loc_global(KitCgTypeId type, ObjSymId sym, i64 addend) {
   NativeLoc loc;
   memset(&loc, 0, sizeof loc);
   loc.kind = NATIVE_LOC_GLOBAL;
   loc.cls = NATIVE_REG_INT;
   loc.type = type;
   loc.v.global.sym = sym;
   loc.v.global.addend = addend;
   return loc;
 }
 
 static int loc_same_frame(NativeLoc a, NativeLoc b) {
   return a.kind == NATIVE_LOC_FRAME && b.kind == NATIVE_LOC_FRAME &&
          a.v.frame == b.v.frame;
 }
 
 #define NATIVE_EMIT_TEMP_CACHE 16u
 
 typedef struct NativeEmitTempCacheEntry {
   NativeLoc source;
   NativeLoc result;
 } NativeEmitTempCacheEntry;
 
 struct NativeEmitTempScope {
   NativeEmitCtx* emit;
   const Inst* inst;
   u8 allow_asm_temps;
   u8 effects_ready;
   u8 live_after_applied;
   u8 pad;
   u32 leased[OPT_REG_CLASSES];
   u32 unavailable[OPT_REG_CLASSES];
   /* Definition registers that currently hold a cached stack source consumed
    * by this same instruction. They may be used in place as source+destination
    * even though the normal avoidance set contains the destination register. */
   u32 cache_def_reuse[OPT_REG_CLASSES];
   NativeEmitTempCacheEntry cache[NATIVE_EMIT_TEMP_CACHE];
   u32 ncache;
 };
 
 typedef struct NativeEmitTempMark {
   u32 leased[OPT_REG_CLASSES];
   u32 ncache;
 } NativeEmitTempMark;
 
 typedef struct FrameCacheNeedCtx {
   NativeEmitCtx* emit;
   u32 regs[OPT_REG_CLASSES];
 } FrameCacheNeedCtx;
 
 static void frame_cache_collect_needed(const Inst* in,
                                        FrameCacheNeedCtx* need) {
   NativeEmitCtx* e = need->emit;
   for (u32 i = 0; in && i < in->nopnds; ++i) {
     const OptOperand* op = &in->opnds[i];
     if (opt_inst_operand_is_def(in, i) || op->kind != OPK_STACK) continue;
     for (u32 c = 0; c < OPT_REG_CLASSES; ++c)
       for (Reg r = 0; r < OPT_MAX_HARD_REGS; ++r)
         if ((e->frame_cache_held[c] & (1u << r)) &&
             e->frame_cache_slot[c][r] == op->v.frame_slot &&
             e->frame_cache_type[c][r] == op->type)
           need->regs[c] |= 1u << r;
   }
 }
 
 static int temp_loc_equal(NativeLoc a, NativeLoc b) {
   if (a.kind != b.kind || a.cls != b.cls || a.type != b.type) return 0;
   switch ((NativeLocKind)a.kind) {
     case NATIVE_LOC_NONE:
       return 1;
     case NATIVE_LOC_REG:
       return a.v.reg == b.v.reg;
     case NATIVE_LOC_FRAME:
     case NATIVE_LOC_FRAME_ADDR:
       return a.v.frame == b.v.frame;
     case NATIVE_LOC_STACK:
       return a.v.stack.slot == b.v.stack.slot &&
              a.v.stack.offset == b.v.stack.offset;
     case NATIVE_LOC_IMM:
       return a.v.imm == b.v.imm;
     case NATIVE_LOC_GLOBAL:
       return a.v.global.sym == b.v.global.sym &&
              a.v.global.addend == b.v.global.addend;
     case NATIVE_LOC_ADDR:
       return memcmp(&a.v.addr, &b.v.addr, sizeof a.v.addr) == 0;
   }
   return 0;
 }
 
 static void temp_scope_apply_effects(NativeEmitCtx* e,
                                      NativeEmitTempScope* scope) {
   OptRegEffectMasks effects;
   FrameCacheNeedCtx needed;
   u32 invalidate[OPT_REG_CLASSES];
   if (!e || !scope || scope->effects_ready) return;
   memset(&needed, 0, sizeof needed);
   opt_inst_reg_effect_masks(e->f, scope->inst, &effects);
   needed.emit = e;
   frame_cache_collect_needed(scope->inst, &needed);
   for (u32 c = 0; c < OPT_REG_CLASSES; ++c) {
     u32 clobbers = effects.clobbers.cls[c];
     /* Call clobbers happen after argument/callee staging. A call-local temp is
      * consumed by the parallel-copy plan before emit_call and may therefore
      * live in a caller-saved register; forbidding the entire ABI clobber set
      * makes high-pressure calls impossible precisely when every callee-saved
      * allocation register is live across them. Calls clear the block cache
      * below, so this exception cannot preserve a stale value past the call. */
     if (scope->inst && (IROp)scope->inst->op == IR_CALL) clobbers = 0u;
     scope->unavailable[c] |=
         effects.uses.cls[c] | effects.defs.cls[c] | clobbers;
     scope->cache_def_reuse[c] =
         needed.regs[c] & effects.defs.cls[c] &
         ~(effects.uses.cls[c] | effects.clobbers.cls[c]);
     invalidate[c] = scope->unavailable[c] & ~scope->cache_def_reuse[c];
   }
   if (scope->inst && ((IROp)scope->inst->op == IR_CALL ||
                       (IROp)scope->inst->op == IR_ASM_BLOCK))
     frame_cache_clear(e);
   else
     frame_cache_invalidate_masks(e, invalidate);
   scope->effects_ready = 1u;
 }
 
 static void temp_scope_begin(NativeEmitCtx* e, NativeEmitTempScope* scope,
                              const Inst* in) {
   const u32* machine_clobbers = NULL;
   for (u32 c = 0; c < OPT_REG_CLASSES; ++c) {
     scope->leased[c] = 0u;
     scope->unavailable[c] = 0u;
     scope->cache_def_reuse[c] = 0u;
   }
   scope->emit = e;
   scope->inst = in;
   scope->allow_asm_temps = 0u;
   scope->effects_ready = 0u;
   scope->live_after_applied = 0u;
   /* ncache is the sole validity boundary for the materialization entries.
    * Clearing the 16-entry payload on every MIR instruction only burns memory
    * bandwidth; each entry is fully assigned before ncache exposes it. */
   scope->ncache = 0u;
   e->temps = scope;
 
   if (e->live_after_ready) {
     if (in && in->id != INST_ID_NONE && in->id < e->live_after_cap)
       for (u32 c = 0; c < OPT_REG_CLASSES; ++c)
         scope->unavailable[c] |= e->live_after_by_inst[in->id].cls[c];
     scope->live_after_applied = 1u;
     temp_scope_apply_effects(e, scope);
     return;
   }
 
   /* Before the allocation-register scavenger is enabled, MIR's ownership
    * invariant keeps every explicit use/def out of the reserved emission-temp
    * bank. Only an exceptional backend-declared machine clobber can collide
    * with those temps, so handle that table directly and defer the full
    * use/def analysis alongside hard liveness. */
   machine_clobbers = opt_inst_machine_clobber_masks(e->f, in);
   if (in && ((IROp)in->op == IR_CALL || (IROp)in->op == IR_ASM_BLOCK)) {
     frame_cache_clear(e);
   } else if (machine_clobbers) {
     for (u32 c = 0; c < OPT_REG_CLASSES; ++c)
       scope->unavailable[c] |= machine_clobbers[c];
     frame_cache_invalidate_masks(e, machine_clobbers);
   }
 }
 
 /* Physical live-after is needed only for the allocation-register scavenger.
  * Most instructions (and most functions) fit entirely in the target's
  * dedicated emission-temp bank, so defer the full hard-liveness dataflow and
  * per-instruction table until a scope actually exhausts that bank. */
 static void temp_scope_require_live_after(NativeEmitCtx* e) {
   u32 live[OPT_REG_CLASSES];
   const Inst* in;
   if (!e) return;
   if (!e->live_after_ready) compute_emit_live_after(e);
   if (!e->temps) return;
   if (e->temps->live_after_applied) return;
   in = e->temps->inst;
   memset(live, 0, sizeof live);
   if (in && in->id != INST_ID_NONE && in->id < e->live_after_cap)
     for (u32 c = 0; c < OPT_REG_CLASSES; ++c) {
       live[c] = e->live_after_by_inst[in->id].cls[c];
       e->temps->unavailable[c] |= live[c];
     }
   e->temps->live_after_applied = 1u;
   temp_scope_apply_effects(e, e->temps);
   for (u32 c = 0; c < OPT_REG_CLASSES; ++c) {
     live[c] &= ~e->temps->cache_def_reuse[c];
   }
   frame_cache_invalidate_masks(e, live);
 }
 
 static NativeEmitTempMark temp_scope_mark(NativeEmitCtx* e) {
   NativeEmitTempMark mark;
   memset(&mark, 0, sizeof mark);
   if (!e->temps) return mark;
   memcpy(mark.leased, e->temps->leased, sizeof mark.leased);
   mark.ncache = e->temps->ncache;
   return mark;
 }
 
 static void temp_scope_rewind(NativeEmitCtx* e, NativeEmitTempMark mark) {
   if (!e->temps) return;
   memcpy(e->temps->leased, mark.leased, sizeof mark.leased);
   e->temps->ncache = mark.ncache;
 }
 
 /* Forget every instruction-local materialization fact whose value lived in
  * `reg`. This does not release the register: a caller that overwrote a cached
  * value in place still owns the new result through the existing lease. */
 static int temp_scope_drop_cached_reg(NativeEmitCtx* e,
                                       NativeAllocClass cls, Reg reg) {
   u32 c = (u32)cls;
   u32 out = 0;
   int owned = 0;
   if (!e->temps || c >= OPT_REG_CLASSES || reg >= OPT_MAX_HARD_REGS) return 0;
   for (u32 i = 0; i < e->temps->ncache; ++i) {
     NativeEmitTempCacheEntry entry = e->temps->cache[i];
     if (entry.result.kind == NATIVE_LOC_REG && entry.result.cls == (u8)cls &&
         entry.result.v.reg == reg) {
       owned = 1;
       continue;
     }
     if (out != i) e->temps->cache[out] = entry;
     ++out;
   }
   if (!owned) return 0;
   e->temps->ncache = out;
   e->temps->cache_def_reuse[c] &= ~(1u << reg);
   return 1;
 }
 
 /* Relinquish a register only when this instruction acquired it as a cached
  * materialization. Earlier ABI-phase destinations are leased without a cache
  * entry and therefore remain protected. */
 static void temp_scope_release_cached_reg(NativeEmitCtx* e,
                                           NativeAllocClass cls, Reg reg) {
   u32 c = (u32)cls;
   if (!temp_scope_drop_cached_reg(e, cls, reg)) return;
   e->temps->leased[c] &= ~(1u << reg);
 }
 
 /* End one ordering phase inside a compound native operation. A call consumes
  * all argument/callee staging before it clobbers caller-saved registers; none
  * of those leases or materialization facts may be observed by return
  * writeback. The instruction scope (and its hard-reg avoidance set) remains
  * active for the post-call phase. */
 static void temp_scope_phase_barrier(NativeEmitCtx* e) {
   if (!e->temps) return;
   memset(e->temps->leased, 0, sizeof e->temps->leased);
   e->temps->ncache = 0;
 }
 
 /* Keep a completed register destination owned until the current compound
  * operation reaches its next ordering barrier.  write_loc deliberately
  * rewinds the temporary leases used to perform one move; without this separate
  * phase lease, a later move could reuse (and overwrite) an earlier ABI
  * argument or return register while materializing its own source. */
 static void temp_scope_hold_loc(NativeEmitCtx* e, NativeLoc loc) {
   u32 c;
   Reg r;
   if (!e->temps || loc.kind != NATIVE_LOC_REG) return;
   c = (u32)loc.cls;
   r = loc.v.reg;
   if (c >= OPT_REG_CLASSES || r >= OPT_MAX_HARD_REGS)
     emit_panic(e, e->temps->inst ? e->temps->inst->loc : (SrcLoc){0, 0, 0},
                "invalid phase-owned register");
   frame_cache_drop_reg(e, c, r);
   e->temps->leased[c] |= 1u << r;
 }
 
 static void temp_scope_end(NativeEmitCtx* e, NativeEmitTempScope* scope) {
   if (e->temps != scope)
     emit_panic(e, scope->inst ? scope->inst->loc : (SrcLoc){0, 0, 0},
                "temporary scope mismatch");
 #ifndef NDEBUG
   for (u32 c = 0; c < OPT_REG_CLASSES; ++c)
     if (scope->leased[c] & e->frame_cache_held[c])
       emit_panic(e, scope->inst ? scope->inst->loc : (SrcLoc){0, 0, 0},
                  "temporary lease escaped into frame cache");
 #endif
   memset(scope->leased, 0, sizeof scope->leased);
   scope->ncache = 0;
   e->temps = NULL;
 }
 
 static int temp_reg_available(NativeEmitCtx* e, NativeAllocClass cls, Reg r,
                               Reg a, Reg b) {
   u32 c = (u32)cls;
   u32 bit;
   if (r == a || r == b || r >= 32u || c >= OPT_REG_CLASSES) return 0;
   bit = 1u << r;
   if (!e->temps) return 1;
   return !(e->temps->leased[c] & bit) &&
          !(e->temps->unavailable[c] & bit) &&
          !(e->frame_cache_held[c] & bit);
 }
 
 static Reg temp_acquire(NativeEmitCtx* e, NativeAllocClass cls, Reg a, Reg b,
                         SrcLoc loc) {
   u32 c = (u32)cls;
   if (c < OPT_REG_CLASSES) {
     /* Prefer the target-declared O1 operand-temp bank. These registers are
      * excluded from allocation, but still respect explicit/implicit instruction
      * clobbers and other leases in this scope. */
     for (u32 i = 0; i < e->f->emit_temp_reg_count[c]; ++i) {
       Reg r = e->f->emit_temp_regs[c][i];
       if (!temp_reg_available(e, cls, r, a, b)) continue;
       if (e->temps) e->temps->leased[c] |= 1u << r;
       return r;
     }
     /* A dead caller-saved allocation register is an equally local temporary:
      * it needs no prologue save and liveness proves its value is dead before
      * we overwrite it. This is the pressure escape hatch for two-temp targets
      * and for ARM32, whose backend-internal LR must not be leased. */
     if (e->temps) {
       temp_scope_require_live_after(e);
       for (u32 i = 0; i < e->f->opt_hard_reg_count[c]; ++i) {
         Reg r = e->f->opt_hard_regs[c][i];
         if (r >= 32u || !(e->f->opt_caller_saved[c] & (1u << r))) continue;
         if (!temp_reg_available(e, cls, r, a, b)) continue;
         e->temps->leased[c] |= 1u << r;
         metrics_count(e->c, "opt.native_emit.scavenges", 1);
         return r;
       }
     }
     /* A clean cache entry is always evictable: its frame store is already
      * authoritative. Prefer losing the forwarding opportunity over failing a
      * real instruction's temporary demand. */
     if (e->temps) {
       for (u32 i = 0; i < e->f->emit_temp_reg_count[c]; ++i) {
         Reg r = e->f->emit_temp_regs[c][i];
         u32 bit = r < 32u ? 1u << r : 0u;
         if (!bit || !(e->frame_cache_held[c] & bit) || r == a || r == b ||
             (e->temps->leased[c] & bit) ||
             (e->temps->unavailable[c] & bit))
           continue;
         frame_cache_drop_reg(e, c, r);
         e->temps->leased[c] |= bit;
         return r;
       }
     }
   }
   compiler_panic(
       e->c, loc,
       "opt native emit: no instruction-local native temporary "
       "(class=%u op=%u inst=%u leased=0x%08x unavailable=0x%08x "
       "cached=0x%08x avoid=%u/%u)",
       (unsigned)c,
       e->temps && e->temps->inst ? (unsigned)e->temps->inst->op : 0u,
       e->temps && e->temps->inst ? (unsigned)e->temps->inst->id : 0u,
       e->temps && c < OPT_REG_CLASSES ? (unsigned)e->temps->leased[c] : 0u,
       e->temps && c < OPT_REG_CLASSES ? (unsigned)e->temps->unavailable[c]
                                       : 0u,
       c < OPT_REG_CLASSES ? (unsigned)e->frame_cache_held[c] : 0u,
       (unsigned)a, (unsigned)b);
 }
 
 static int asm_temp_reg_ok(NativeEmitCtx* e, NativeAllocClass cls, Reg r,
                            u32 allowed_mask) {
   if (r >= 32u || (allowed_mask && !(allowed_mask & (1u << r)))) return 0;
   return !e->target->regs || !e->target->regs->asm_operand_reg_ok ||
          e->target->regs->asm_operand_reg_ok(e->target->regs, cls, r);
 }
 
 static const NativeAllocClassInfo* emit_class_info(NativeEmitCtx* e,
                                                     NativeAllocClass cls) {
   if (!e->target->regs) return NULL;
   for (u32 i = 0; i < e->target->regs->nclasses; ++i)
     if (e->target->regs->classes[i].cls == (u8)cls)
       return &e->target->regs->classes[i];
   return NULL;
 }
 
 static void frame_cache_drop_reg(NativeEmitCtx* e, u32 cls, Reg reg) {
   if (cls >= OPT_REG_CLASSES || reg >= OPT_MAX_HARD_REGS) return;
   e->frame_cache_slot[cls][reg] = FRAME_SLOT_NONE;
   e->frame_cache_type[cls][reg] = 0;
   e->frame_cache_held[cls] &= ~(1u << reg);
 }
 
 static void frame_cache_clear(NativeEmitCtx* e) {
   memset(e->frame_cache_slot, 0, sizeof e->frame_cache_slot);
   memset(e->frame_cache_type, 0, sizeof e->frame_cache_type);
   memset(e->frame_cache_held, 0, sizeof e->frame_cache_held);
 }
 
 static void frame_cache_invalidate_masks(NativeEmitCtx* e,
                                          const u32* masks) {
   if (!masks) return;
   for (u32 c = 0; c < OPT_REG_CLASSES; ++c) {
     u32 killed = e->frame_cache_held[c] & masks[c];
     for (Reg r = 0; killed && r < 32u; ++r)
       if (killed & (1u << r)) {
         frame_cache_drop_reg(e, c, r);
         killed &= ~(1u << r);
       }
   }
 }
 
 static u32 frame_cache_retainable_mask(NativeEmitCtx* e,
                                        NativeAllocClass cls) {
   const NativeAllocClassInfo* ci = emit_class_info(e, cls);
   u32 c = (u32)cls;
   u32 mask;
   u32 preserved_allocable;
   if (!ci || c >= OPT_REG_CLASSES) return 0u;
   mask = ci->emit_cache_mask;
   /* Without hard live-after data, only target-reserved cache registers can be
    * retained across instructions. An ordinary allocation register reaching a
    * spill store may still carry a live MIR value; treating it as a cache entry
    * would let a later reload lease and overwrite that value. */
   if (!e->live_after_ready) return mask;
   preserved_allocable = native_target_caller_saved_mask(e->target, cls);
   preserved_allocable |= e->callee_saved_used[c];
   /* An allocable register is a legal persistent cache only when acquisition
    * proved it dead and its ABI preservation is already established:
    * caller-saved until the next call boundary, or a callee-save already present
    * in the known frame because MIR/asm uses it. The NativeTarget contract
    * requires ordinary hooks to preserve such registers unless the current
    * instruction's exhaustive machine_op_clobbers effect says otherwise;
    * calls/asm clear the cache at their phase boundary. */
   for (u32 i = 0; i < e->f->opt_hard_reg_count[c]; ++i) {
     Reg r = e->f->opt_hard_regs[c][i];
     if (r < 32u && (preserved_allocable & (1u << r))) mask |= 1u << r;
   }
   return mask;
 }
 
 /* Acquire a lease satisfying an inline-asm register constraint.  Unlike the
  * general temporary picker this may name a fixed ABI register (for example
  * x86 "a").  Regalloc has already kept values live across the asm out of that
  * register; the availability check enforces that contract at the boundary. */
 static Reg temp_acquire_asm(NativeEmitCtx* e, NativeAllocClass cls, Reg fixed,
                             u32 allowed_mask, SrcLoc loc) {
   u32 c = (u32)cls;
   if (!e->temps || c >= OPT_REG_CLASSES)
     emit_panic(e, loc, "asm register staging outside temporary scope");
   temp_scope_require_live_after(e);
   if (fixed != (Reg)REG_NONE) {
     if (!asm_temp_reg_ok(e, cls, fixed, allowed_mask) ||
         !temp_reg_available(e, cls, fixed, REG_NONE, REG_NONE))
       emit_panic(e, loc, "fixed asm staging register is unavailable");
     e->temps->leased[c] |= 1u << fixed;
     return fixed;
   }
   for (u32 i = 0; i < e->f->emit_temp_reg_count[c]; ++i) {
     Reg r = e->f->emit_temp_regs[c][i];
     /* The O1 operand-temp bank is deliberately hidden from user
      * named-register requirements and allocator homes, but it is valid as an
      * internally substituted unrestricted asm operand. Restricted masks still
      * apply. */
     if (r >= 32u || (allowed_mask && !(allowed_mask & (1u << r))) ||
         !temp_reg_available(e, cls, r, REG_NONE, REG_NONE))
       continue;
     e->temps->leased[c] |= 1u << r;
     return r;
   }
   if (e->temps->allow_asm_temps) {
     const NativeAllocClassInfo* ci = emit_class_info(e, cls);
     for (u32 i = 0; ci && i < ci->nasm_temps; ++i) {
       Reg r = ci->asm_temps[i];
       if (r >= 32u || (allowed_mask && !(allowed_mask & (1u << r))) ||
           !temp_reg_available(e, cls, r, REG_NONE, REG_NONE))
         continue;
       e->temps->leased[c] |= 1u << r;
       return r;
     }
   }
   /* An asm boundary may use every target-declared asm register, not just the
    * allocator's general-purpose subset. This is what makes high-arity blocks
    * possible without assigning a fixed register to an entire PReg live range.
    * A setup prepass records any selected callee-saved registers in the known
    * frame, so choosing one here remains ABI-safe. */
   for (u32 i = 0; i < e->f->opt_phys_reg_count[c]; ++i) {
     Reg r = e->f->opt_phys_regs[c][i].reg;
     if (!asm_temp_reg_ok(e, cls, r, allowed_mask) ||
         !temp_reg_available(e, cls, r, REG_NONE, REG_NONE))
       continue;
     e->temps->leased[c] |= 1u << r;
     return r;
   }
   compiler_panic(e->c, loc,
                  "opt native emit: no register available for asm operand "
                  "staging (class=%u leased=0x%08x unavailable=0x%08x "
                  "allowed=0x%08x)",
                  (unsigned)c, (unsigned)e->temps->leased[c],
                  (unsigned)e->temps->unavailable[c],
                  (unsigned)allowed_mask);
 }
 
 /* A leased register is owned by the current emission scope and may therefore
  * be overwritten in place.  An ordinary OPK_REG may still be live after the
  * instruction even when the result itself is frame-resident, so reusing it as
  * the result would silently clobber an allocator-owned value. */
 static int temp_reg_is_leased(NativeEmitCtx* e, NativeAllocClass cls, Reg r) {
   u32 c = (u32)cls;
   return e->temps && c < OPT_REG_CLASSES && r < 32u &&
          (e->temps->leased[c] & (1u << r)) != 0;
 }
 
 static int temp_available(NativeEmitCtx* e, NativeAllocClass cls, Reg a,
                           Reg b) {
   u32 c = (u32)cls;
   if (c < OPT_REG_CLASSES) {
     for (u32 i = 0; i < e->f->emit_temp_reg_count[c]; ++i) {
       Reg r = e->f->emit_temp_regs[c][i];
       if (temp_reg_available(e, cls, r, a, b)) return 1;
     }
     if (e->temps) {
       temp_scope_require_live_after(e);
       for (u32 i = 0; i < e->f->opt_hard_reg_count[c]; ++i) {
         Reg r = e->f->opt_hard_regs[c][i];
         if (r < 32u && (e->f->opt_caller_saved[c] & (1u << r)) &&
             temp_reg_available(e, cls, r, a, b))
           return 1;
       }
     }
   }
   return 0;
 }
 
 static NativeLoc temp_loc_acquire(NativeEmitCtx* e, KitCgTypeId type,
                                   NativeAllocClass cls, Reg a, Reg b,
                                   SrcLoc loc) {
   return loc_reg(type, cls, temp_acquire(e, cls, a, b, loc));
 }
 
 static NativeFrameSlot map_slot(NativeEmitCtx* e, NativeFrameSlot slot,
                                 SrcLoc loc) {
   if (slot == NATIVE_FRAME_SLOT_NONE) return NATIVE_FRAME_SLOT_NONE;
   if (slot > e->f->nframe_slots) emit_panic(e, loc, "bad frame slot");
   if (!e->slot_map[slot]) emit_panic(e, loc, "unmapped frame slot");
   return e->slot_map[slot];
 }
 
 static MCLabel ensure_label(NativeEmitCtx* e, u32 block, SrcLoc loc) {
   if (block >= e->f->nblocks) emit_panic(e, loc, "bad block label");
   if (e->labels[block] == MC_LABEL_NONE)
     e->labels[block] = e->target->label_new(e->target);
   return e->labels[block];
 }
 
 static NativeAddr addr_from_loc(NativeEmitCtx* e, NativeLoc loc,
                                 SrcLoc src_loc) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_type = loc.type;
   switch ((NativeLocKind)loc.kind) {
     case NATIVE_LOC_FRAME:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = loc.v.frame;
       return addr;
     case NATIVE_LOC_FRAME_ADDR:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = loc.v.frame;
       return addr;
     case NATIVE_LOC_STACK:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = loc.v.stack.slot;
       addr.offset = loc.v.stack.offset;
       return addr;
     case NATIVE_LOC_GLOBAL:
       addr.base_kind = NATIVE_ADDR_BASE_GLOBAL;
       addr.base.global.sym = loc.v.global.sym;
       addr.base.global.addend = loc.v.global.addend;
       return addr;
     case NATIVE_LOC_REG:
       addr.base_kind = NATIVE_ADDR_BASE_REG;
       addr.cls = loc.cls;
       addr.base.reg = loc.v.reg;
       return addr;
     case NATIVE_LOC_ADDR:
       return loc.v.addr;
     default:
       emit_panic(e, src_loc, "location is not addressable");
   }
 }
 
 static NativeAddr addr_from_operand(NativeEmitCtx* e, const OptOperand* op,
                                     SrcLoc loc) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   if (!op) emit_panic(e, loc, "missing address operand");
   addr.base_type = op->type;
   switch ((OptOperandKind)op->kind) {
     case OPT_OPK_LOCAL:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = map_slot(e, op->v.frame_slot, loc);
       return addr;
     case OPT_OPK_FRAME_ADDR:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = map_slot(e, op->v.frame_slot, loc);
       return addr;
     case OPT_OPK_STACK:
       addr.base_type = op->type;
       {
         NativeFrameSlot mapped = map_slot(e, op->v.frame_slot, loc);
         NativeAllocClass cls = class_for_type(e, addr.base_type);
         NativeLoc held = frame_cache_take_slot(
             e, op->v.frame_slot, loc_frame(addr.base_type, cls, mapped), cls,
             addr.base_type, REG_NONE, REG_NONE);
         if (held.kind == NATIVE_LOC_REG) {
           addr.base_kind = NATIVE_ADDR_BASE_REG;
           addr.cls = held.cls;
           addr.base.reg = held.v.reg;
         } else {
           addr.base_kind = NATIVE_ADDR_BASE_FRAME_VALUE;
           addr.base.frame = mapped;
         }
       }
       return addr;
     case OPT_OPK_GLOBAL:
       addr.base_kind = NATIVE_ADDR_BASE_GLOBAL;
       addr.base.global.sym = op->v.global.sym;
       addr.base.global.addend = op->v.global.addend;
       return addr;
     case OPT_OPK_INDIRECT:
       addr.cls = NATIVE_REG_INT;
       addr.base_type = op->v.ind.base_type;
       if (!addr.base_type)
         emit_panic(e, loc, "indirect base has no location type");
       if (op->v.ind.base_kind == OPT_INDIRECT_FRAME) {
         {
           NativeFrameSlot mapped = map_slot(e, op->v.ind.base, loc);
           NativeLoc held = frame_cache_take_slot(
               e, op->v.ind.base,
               loc_frame(addr.base_type, NATIVE_REG_INT, mapped),
               NATIVE_REG_INT, addr.base_type, REG_NONE, REG_NONE);
           if (held.kind == NATIVE_LOC_REG) {
             addr.base_kind = NATIVE_ADDR_BASE_REG;
             addr.base.reg = held.v.reg;
           } else {
             addr.base_kind = NATIVE_ADDR_BASE_FRAME_VALUE;
             addr.base.frame = mapped;
           }
         }
       } else if (op->v.ind.base_kind == OPT_INDIRECT_FRAME_ADDR) {
         addr.base_kind = NATIVE_ADDR_BASE_FRAME;
         addr.base.frame = map_slot(e, op->v.ind.base, loc);
       } else {
         addr.base_kind = NATIVE_ADDR_BASE_REG;
         addr.base.reg = op->v.ind.base;
       }
       addr.index_cls = NATIVE_REG_INT;
       if (op->v.ind.index_kind == OPT_INDIRECT_FRAME) {
         addr.index_type = op->v.ind.index_type;
         if (!addr.index_type)
           emit_panic(e, loc, "indirect index has no location type");
         {
           NativeFrameSlot mapped = map_slot(e, op->v.ind.index, loc);
           NativeLoc held = frame_cache_take_slot(
               e, op->v.ind.index,
               loc_frame(addr.index_type, NATIVE_REG_INT, mapped),
               NATIVE_REG_INT, addr.index_type, REG_NONE, REG_NONE);
           if (held.kind == NATIVE_LOC_REG) {
             addr.index_kind = NATIVE_ADDR_INDEX_REG;
             addr.index.reg = held.v.reg;
           } else {
             addr.index_kind = NATIVE_ADDR_INDEX_FRAME_VALUE;
             addr.index.frame = mapped;
           }
         }
       } else if (op->v.ind.index_kind == OPT_INDIRECT_FRAME_ADDR) {
         emit_panic(e, loc, "frame address cannot be an indirect index");
       } else if (op->v.ind.index == (Reg)REG_NONE) {
         addr.index_kind = NATIVE_ADDR_INDEX_NONE;
       } else {
         addr.index_type = op->v.ind.index_type;
         if (!addr.index_type)
           emit_panic(e, loc, "indirect index has no location type");
         addr.index_kind = NATIVE_ADDR_INDEX_REG;
         addr.index.reg = op->v.ind.index;
       }
       addr.log2_scale = op->v.ind.log2_scale;
       /* L8 extend rider: OPT_IDX_EXT_* and NATIVE_ADDR_IDX_EXT_* share the
        * 0=NONE/1=SXTW/2=UXTW encoding. Only set by the addr-fold recognition
        * when the target advertised can_fold_extend_into_addr, so a backend
        * without the capability always sees NONE. */
       addr.index_ext = op->v.ind.index_ext;
       addr.offset = op->v.ind.ofs;
       return addr;
     case OPT_OPK_REG:
       addr.base_kind = NATIVE_ADDR_BASE_REG;
       addr.cls = op->cls;
       addr.base.reg = op->v.reg;
       return addr;
     default:
       emit_panic(e, loc, "operand is not addressable");
   }
 }
 
 static NativeAddr pointer_addr_from_operand(NativeEmitCtx* e,
                                             const OptOperand* op, SrcLoc loc,
                                             Reg avoid_a, Reg avoid_b) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   if (!op) emit_panic(e, loc, "missing pointer operand");
   addr.base_type = op->type;
   switch ((OptOperandKind)op->kind) {
     case OPT_OPK_LOCAL: {
       NativeAddr frame;
       NativeLoc dst;
       NativeAllocClass cls;
       Reg r;
       /* An OPK_LOCAL in a pointer-address position is ambiguous. When the
        * operand's type is a pointer, the local *holds* the pointer value and
        * must be loaded to get the address. Otherwise the local *is* the
        * aggregate storage and its frame home is the address directly — loading
        * it would dereference the aggregate's first 8 bytes as a pointer (e.g.
        * an `__int128` call result copied by `agg_copy`). Mirrors the
        * single-pass path's nd_addr_pointer. */
       if (!cg_type_is_ptr(e->c, op->type)) {
         addr.base_kind = NATIVE_ADDR_BASE_FRAME;
         addr.base.frame = map_slot(e, op->v.frame_slot, loc);
         return addr;
       }
       cls = class_for_type(e, op->type);
       r = temp_acquire(e, cls, avoid_a, avoid_b, loc);
       memset(&frame, 0, sizeof frame);
       frame.base_kind = NATIVE_ADDR_BASE_FRAME;
       frame.base.frame = map_slot(e, op->v.frame_slot, loc);
       frame.base_type = op->type;
       dst = loc_reg(op->type, cls, r);
       e->target->load(e->target, dst, frame, mem_for_type(e->c, op->type));
       addr.base_kind = NATIVE_ADDR_BASE_REG;
       addr.cls = (u8)cls;
       addr.base.reg = r;
       return addr;
     }
     case OPT_OPK_FRAME_ADDR:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = map_slot(e, op->v.frame_slot, loc);
       return addr;
     case OPT_OPK_STACK:
       addr.cls = class_for_type(e, op->type);
       addr.base_type = op->type;
       {
         NativeFrameSlot mapped = map_slot(e, op->v.frame_slot, loc);
         NativeAllocClass cls = (NativeAllocClass)addr.cls;
         NativeLoc held = frame_cache_take_slot(
             e, op->v.frame_slot, loc_frame(addr.base_type, cls, mapped), cls,
             addr.base_type, avoid_a, avoid_b);
         if (held.kind == NATIVE_LOC_REG) {
           addr.base_kind = NATIVE_ADDR_BASE_REG;
           addr.base.reg = held.v.reg;
         } else {
           addr.base_kind = NATIVE_ADDR_BASE_FRAME_VALUE;
           addr.base.frame = mapped;
         }
       }
       return addr;
     case OPT_OPK_GLOBAL:
       addr.base_kind = NATIVE_ADDR_BASE_GLOBAL;
       addr.base.global.sym = op->v.global.sym;
       addr.base.global.addend = op->v.global.addend;
       return addr;
     case OPT_OPK_INDIRECT:
       return addr_from_operand(e, op, loc);
     case OPT_OPK_REG:
       addr.base_kind = NATIVE_ADDR_BASE_REG;
       addr.cls = op->cls;
       addr.base.reg = op->v.reg;
       return addr;
     default:
       emit_panic(e, loc, "operand is not a pointer address");
   }
 }
 
 static Reg addr_base_reg(const NativeAddr* addr) {
   return addr && addr->base_kind == NATIVE_ADDR_BASE_REG ? addr->base.reg
                                                          : REG_NONE;
 }
 
 static Reg addr_index_reg(const NativeAddr* addr) {
   return addr && addr->index_kind == NATIVE_ADDR_INDEX_REG ? addr->index.reg
                                                            : REG_NONE;
 }
 
 static void collapse_addr_to_reg(NativeEmitCtx* e, NativeAddr* addr,
                                  SrcLoc loc) {
   /* Materialize the full address into an instruction-local temporary. We
    * normally must not reuse the base register as the destination: physical MIR
    * may keep that value live past this memory op (e.g. a pointer stored into
    * several of its own fields and then returned), so an in-place `add base,
    * base, #off` would corrupt it. Avoid both base and index so load_addr can
    * still read them.
    *
    * If no third temporary is available and the base is itself a scoped
    * temporary, that base is transient and may be updated in place. load_addr
    * reads the base before adding the scaled index, so dst==base is sound;
    * dst==index would clobber the index before use and is never selected. If the
    * base is caller-owned, temp_acquire reports exhaustion instead of silently
    * changing its value. */
   Reg base = addr_base_reg(addr);
   Reg index = addr_index_reg(addr);
   Reg r;
   if (temp_available(e, NATIVE_REG_INT, base, index))
     r = temp_acquire(e, NATIVE_REG_INT, base, index, loc);
   else if (base != index && temp_reg_is_leased(e, NATIVE_REG_INT, base))
     r = base;
   else
     r = temp_acquire(e, NATIVE_REG_INT, base, index, loc); /* diagnoses cleanly */
   NativeLoc dst = loc_reg(addr->base_type, NATIVE_REG_INT, r);
   e->target->load_addr(e->target, dst, *addr);
   /* load_addr replaced any prior value fact attached to its destination. The
    * result remains leased; only the old cache identity is invalid now. */
   temp_scope_drop_cached_reg(e, NATIVE_REG_INT, r);
   if (base != r)
     temp_scope_release_cached_reg(e, NATIVE_REG_INT, base);
   if (index != r)
     temp_scope_release_cached_reg(e, NATIVE_REG_INT, index);
   memset(addr, 0, sizeof *addr);
   addr->base_kind = NATIVE_ADDR_BASE_REG;
   addr->cls = NATIVE_REG_INT;
   addr->base.reg = r;
   addr->base_type = dst.type;
 }
 
 /* Collapse an address the target cannot encode for this access (e.g. an
  * index scale aarch64 cannot fold into a load/store) into a single base
  * register via load_addr. Mirrors NativeDirectTarget's nd_addr_materialize so
  * the O1 emit path legalizes the same address shapes as direct -O0 emission. */
 static void legalize_addr(NativeEmitCtx* e, NativeAddr* addr, MemAccess mem,
                           SrcLoc loc) {
   if (e->target->addr_legal && !e->target->addr_legal(e->target, addr, mem))
     collapse_addr_to_reg(e, addr, loc);
 }
 
 static NativeLoc loc_from_operand(NativeEmitCtx* e, const OptOperand* op,
                                   SrcLoc loc) {
   if (!op) return loc_none();
   switch ((OptOperandKind)op->kind) {
     case OPT_OPK_REG:
       return loc_reg(op->type, (NativeAllocClass)op->cls, op->v.reg);
     case OPT_OPK_IMM:
       return loc_imm(op->type, op->v.imm);
     case OPT_OPK_GLOBAL:
       return loc_global(op->type, op->v.global.sym, op->v.global.addend);
     case OPT_OPK_LOCAL:
       return loc_frame(op->type, class_for_type(e, op->type),
                        map_slot(e, op->v.frame_slot, loc));
     case OPT_OPK_STACK:
       return loc_frame(op->type, class_for_type(e, op->type),
                        map_slot(e, op->v.frame_slot, loc));
     case OPT_OPK_FRAME_ADDR:
       return loc_frame_addr(op->type, map_slot(e, op->v.frame_slot, loc));
     case OPT_OPK_INDIRECT: {
       NativeLoc out = loc_none();
       out.kind = NATIVE_LOC_ADDR;
       out.cls = op->cls;
       out.type = op->type;
       out.v.addr = addr_from_operand(e, op, loc);
       return out;
     }
   }
   emit_panic(e, loc, "bad operand kind");
 }
 
 static NativeLoc materialize(NativeEmitCtx* e, NativeLoc src,
                              NativeAllocClass cls, KitCgTypeId type,
                              Reg avoid_a, Reg avoid_b, SrcLoc loc) {
   NativeLoc dst;
   NativeAddr addr;
   MemAccess mem;
   if (src.kind == NATIVE_LOC_REG) return src;
   if (e->temps) {
     KitCgTypeId want_type = type ? type : src.type;
     for (u32 i = 0; i < e->temps->ncache; ++i) {
       NativeEmitTempCacheEntry* entry = &e->temps->cache[i];
       Reg cached_reg = entry->result.v.reg;
       u32 cached_bit = cached_reg < 32u ? 1u << cached_reg : 0u;
       int avoided = cached_reg == avoid_a || cached_reg == avoid_b;
       if (entry->result.cls != (u8)cls || entry->result.type != want_type ||
           (avoided &&
            !(e->temps->cache_def_reuse[(u32)cls] & cached_bit)))
         continue;
       if (temp_loc_equal(entry->source, src)) return entry->result;
     }
   }
   dst = temp_loc_acquire(e, type ? type : src.type, cls, avoid_a, avoid_b,
                          loc);
   switch ((NativeLocKind)src.kind) {
     case NATIVE_LOC_IMM:
       e->target->load_imm(e->target, dst, src.v.imm);
       break;
     case NATIVE_LOC_GLOBAL:
       addr = addr_from_loc(e, src, loc);
       e->target->load_addr(e->target, dst, addr);
       break;
     case NATIVE_LOC_FRAME_ADDR:
       addr = addr_from_loc(e, src, loc);
       e->target->load_addr(e->target, dst, addr);
       break;
     case NATIVE_LOC_FRAME:
     case NATIVE_LOC_STACK:
     case NATIVE_LOC_ADDR:
       addr = addr_from_loc(e, src, loc);
       mem = mem_for_type(e->c, dst.type);
       e->target->load(e->target, dst, addr, mem);
       break;
     default:
       emit_panic(e, loc, "cannot materialize location");
   }
   if (e->temps && e->temps->ncache < NATIVE_EMIT_TEMP_CACHE) {
     NativeEmitTempCacheEntry* entry = &e->temps->cache[e->temps->ncache++];
     entry->source = src;
     entry->result = dst;
   }
   return dst;
 }
 
 static int frame_cache_slot_is_exact_spill(NativeEmitCtx* e,
                                            const OptOperand* op,
                                            MemAccess mem) {
   const IRFrameSlot* slot;
   if (!op || op->kind != OPK_STACK || op->v.frame_slot == FRAME_SLOT_NONE ||
       op->v.frame_slot > e->f->nframe_slots)
     return 0;
   slot = &e->f->frame_slots[op->v.frame_slot - 1u];
   /* Spill-slot coloring may reuse one slot for different, non-overlapping type
    * IDs. Exactness is the stored bit width; the cache entry separately records
    * the current value's type/class and only answers an identical use. */
   return slot->kind == FS_SPILL && slot->size >= mem.size &&
          mem.size == type_size_or(e->c, op->type, 0u);
 }
 
 static void frame_cache_invalidate_slot(NativeEmitCtx* e, FrameSlot slot) {
   if (slot == FRAME_SLOT_NONE) return;
   for (u32 c = 0; c < OPT_REG_CLASSES; ++c)
     for (Reg r = 0; r < OPT_MAX_HARD_REGS; ++r)
       if (e->frame_cache_slot[c][r] == slot)
         frame_cache_drop_reg(e, c, r);
 }
 
 static int frame_cache_types_compatible(NativeEmitCtx* e,
                                         KitCgTypeId cached,
                                         KitCgTypeId requested) {
   (void)e;
   return cached == requested;
 }
 
 static NativeLoc frame_cache_take_slot(NativeEmitCtx* e, FrameSlot slot,
                                        NativeLoc source,
                                        NativeAllocClass cls,
                                        KitCgTypeId type, Reg avoid_a,
                                        Reg avoid_b) {
   u32 c = (u32)cls;
   if (!e->temps || slot == FRAME_SLOT_NONE || c >= OPT_REG_CLASSES)
     return loc_none();
   for (Reg r = 0; r < OPT_MAX_HARD_REGS; ++r) {
     u32 bit = 1u << r;
     NativeLoc result;
     int avoided = r == avoid_a || r == avoid_b;
     if (!(e->frame_cache_held[c] & bit) ||
         (avoided && !(e->temps->cache_def_reuse[c] & bit)) ||
         e->frame_cache_slot[c][r] != slot ||
         !frame_cache_types_compatible(e, e->frame_cache_type[c][r], type) ||
         ((e->temps->unavailable[c] & bit) &&
          !(e->temps->cache_def_reuse[c] & bit)))
       continue;
     frame_cache_drop_reg(e, c, r);
     e->temps->leased[c] |= bit;
     result = loc_reg(type, cls, r);
     if (e->temps->ncache < NATIVE_EMIT_TEMP_CACHE) {
       NativeEmitTempCacheEntry* entry =
           &e->temps->cache[e->temps->ncache++];
       entry->source = source;
       entry->result = result;
     }
     return result;
   }
   return loc_none();
 }
 
 static NativeLoc frame_cache_take(NativeEmitCtx* e, const OptOperand* op,
                                   NativeLoc source, NativeAllocClass cls,
                                   KitCgTypeId type, Reg avoid_a, Reg avoid_b) {
   if (!op || op->kind != OPK_STACK) return loc_none();
   return frame_cache_take_slot(e, op->v.frame_slot, source, cls, type,
                                avoid_a, avoid_b);
 }
 
 static void frame_cache_note_store(NativeEmitCtx* e, const OptOperand* op,
                                    NativeLoc src, MemAccess mem) {
   NativeAllocClass cls;
   u32 c, bit;
   Reg r;
   int retainable;
   if (!op || op->kind != OPK_STACK) return;
   frame_cache_invalidate_slot(e, op->v.frame_slot);
   if (!e->temps || src.kind != NATIVE_LOC_REG ||
       !frame_cache_slot_is_exact_spill(e, op, mem))
     return;
   cls = (NativeAllocClass)src.cls;
   c = (u32)cls;
   r = src.v.reg;
   if (c >= OPT_REG_CLASSES || r >= OPT_MAX_HARD_REGS) return;
   bit = 1u << r;
   /* Cross-instruction retention follows the explicit NativeTarget preservation
    * contract: declared cache-enabled emission temps plus liveness-proven dead
    * O1 allocation registers
    * whose caller/callee-save preservation is established, with every
    * exceptional fixed clobber represented by the current instruction's
    * exhaustive machine effect. */
   retainable = (frame_cache_retainable_mask(e, cls) & bit) != 0;
   if (!retainable) return;
   frame_cache_drop_reg(e, c, r);
   e->temps->leased[c] &= ~bit; /* transfer scope ownership to block cache */
   e->frame_cache_slot[c][r] = op->v.frame_slot;
   e->frame_cache_type[c][r] = op->type;
   e->frame_cache_held[c] |= bit;
 }
 
 static NativeLoc materialize_operand(NativeEmitCtx* e, const OptOperand* op,
                                      NativeAllocClass cls, KitCgTypeId type,
                                      Reg avoid_a, Reg avoid_b, SrcLoc loc) {
   NativeLoc source = loc_from_operand(e, op, loc);
   NativeLoc cached = frame_cache_take(e, op, source, cls,
                                       type ? type : source.type, avoid_a,
                                       avoid_b);
   if (cached.kind != NATIVE_LOC_NONE) return cached;
   return materialize(e, source, cls, type, avoid_a, avoid_b, loc);
 }
 
 static void write_loc(NativeEmitCtx* e, NativeLoc dst, NativeLoc src,
                       MemAccess mem, SrcLoc loc) {
   NativeAddr addr;
   NativeEmitTempMark mark = temp_scope_mark(e);
   if (dst.kind == NATIVE_LOC_NONE) goto done;
   if (loc_same_frame(dst, src)) goto done;
   if (dst.kind == NATIVE_LOC_REG) {
     if (src.kind == NATIVE_LOC_REG) {
       if (dst.v.reg != src.v.reg || dst.cls != src.cls)
         e->target->move(e->target, dst, src);
       goto done;
     }
     /* An immediate goes straight into the destination register; routing it
      * through an instruction-local temp and then moving would cost an extra
      * instruction. */
     if (src.kind == NATIVE_LOC_IMM) {
       e->target->load_imm(e->target, dst, src.v.imm);
       goto done;
     }
     /* A register destination is itself the best materialization register.
      * Loading through an emitter temp first adds a move to every spill reload
      * and ABI/frame copy. Native loads consume their address before defining
      * the destination, so even an address based on dst is safe here. */
     if (src.kind == NATIVE_LOC_GLOBAL ||
         src.kind == NATIVE_LOC_FRAME_ADDR) {
       addr = addr_from_loc(e, src, loc);
       e->target->load_addr(e->target, dst, addr);
       goto done;
     }
     if (src.kind == NATIVE_LOC_FRAME || src.kind == NATIVE_LOC_STACK ||
         src.kind == NATIVE_LOC_ADDR) {
       addr = addr_from_loc(e, src, loc);
       e->target->load(e->target, dst, addr, mem);
       goto done;
     }
     emit_panic(e, loc, "cannot write location to register");
   }
   addr = addr_from_loc(e, dst, loc);
   if (src.kind != NATIVE_LOC_REG)
     src = materialize(e, src, (NativeAllocClass)dst.cls, dst.type, REG_NONE,
                       REG_NONE, loc);
   e->target->store(e->target, addr, src, mem);
 done:
   temp_scope_rewind(e, mark);
 }
 
 /* Execute one move in a multi-move ABI phase and retain its register
  * destination. The backend call/return marshallers promise that moves are
  * source-safe in the order returned; this helper supplies the complementary
  * destination-lifetime guarantee for emitter-side materialization. */
 static void write_phase_loc(NativeEmitCtx* e, NativeLoc dst, NativeLoc src,
                             MemAccess mem, SrcLoc loc) {
   write_loc(e, dst, src, mem, loc);
   temp_scope_hold_loc(e, dst);
 }
 
 static void write_operand(NativeEmitCtx* e, const OptOperand* dst,
                           NativeLoc src, MemAccess mem, SrcLoc loc) {
   NativeLoc real_dst = loc_from_operand(e, dst, loc);
   /* Keep the register that actually reaches a spill store. Besides avoiding a
    * second materialization in write_loc, this lets ownership transfer from the
    * instruction lease to the clean block cache. Constants and copies into a
    * spill home are common producers and otherwise lose their forwarding fact
    * at this abstraction boundary. */
   if (dst && dst->kind == OPK_STACK && src.kind != NATIVE_LOC_REG &&
       !loc_same_frame(real_dst, src)) {
     /* Keep integer zero in the target's architectural zero register when it
      * can be stored directly. Materializing it into an emitter temp costs an
      * otherwise redundant movz/li on every zero-initialized spill. */
     if (src.kind == NATIVE_LOC_IMM && src.v.imm == 0 &&
         e->target->has_store_zero_reg &&
         class_for_type(e, real_dst.type) == NATIVE_REG_INT)
       src = loc_reg(real_dst.type, NATIVE_REG_INT,
                     e->target->store_zero_reg);
     else
       src = materialize(e, src, (NativeAllocClass)real_dst.cls, real_dst.type,
                         REG_NONE, REG_NONE, loc);
   }
   write_loc(e, real_dst, src, mem, loc);
   frame_cache_note_store(e, dst, src, mem);
 }
 
 /* Place a value directly in an already-leased register.  This is the boundary
  * operation used for fixed/constrained asm operands: going through materialize
  * would acquire an unrelated temporary and then copy, which both wastes a
  * register and can make a satisfiable high-arity asm look impossible. */
 static void load_loc_into_reg(NativeEmitCtx* e, NativeLoc dst, NativeLoc src,
                               SrcLoc loc) {
   NativeAddr addr;
   if (dst.kind != NATIVE_LOC_REG)
     emit_panic(e, loc, "staging destination is not a register");
   switch ((NativeLocKind)src.kind) {
     case NATIVE_LOC_REG:
       if (src.v.reg != dst.v.reg || src.cls != dst.cls)
         e->target->move(e->target, dst, src);
       return;
     case NATIVE_LOC_IMM:
       e->target->load_imm(e->target, dst, src.v.imm);
       return;
     case NATIVE_LOC_GLOBAL:
     case NATIVE_LOC_FRAME_ADDR:
       addr = addr_from_loc(e, src, loc);
       e->target->load_addr(e->target, dst, addr);
       return;
     case NATIVE_LOC_FRAME:
     case NATIVE_LOC_STACK:
     case NATIVE_LOC_ADDR:
       addr = addr_from_loc(e, src, loc);
       e->target->load(e->target, dst, addr, mem_for_type(e->c, dst.type));
       return;
     default:
       emit_panic(e, loc, "cannot stage asm operand in a register");
   }
 }
 
 static int asm_reg_loc_satisfies(NativeEmitCtx* e, NativeLoc loc,
                                  NativeAllocClass cls, Reg fixed,
                                  u32 allowed_mask) {
   if (loc.kind != NATIVE_LOC_REG || loc.cls != (u8)cls) return 0;
   if (fixed != (Reg)REG_NONE && loc.v.reg != fixed) return 0;
   return asm_temp_reg_ok(e, cls, loc.v.reg, allowed_mask);
 }
 
 static int asm_reg_requirement(NativeEmitCtx* e,
                                const IRAsmRegRequirement* req,
                                NativeAllocClass* cls, Reg* fixed,
                                u32* allowed, SrcLoc loc) {
   if (!req || !req->present) return 0;
   if (req->cls >= OPT_REG_CLASSES || req->fixed_reg < -1 ||
       req->fixed_reg >= (i32)OPT_MAX_HARD_REGS)
     emit_panic(e, loc, "invalid pre-resolved asm register requirement");
   *cls = (NativeAllocClass)req->cls;
   *fixed = req->fixed_reg < 0 ? REG_NONE : (Reg)req->fixed_reg;
   *allowed = req->allowed_mask;
   return 1;
 }
 
 /* Localize an asm register constraint at the instruction boundary. An
  * unrestricted allocated register may already satisfy it; fixed/subset
  * requirements are normally fulfilled by scoped staging. No emitter-owned
  * register becomes a MIR home. */
 static NativeLoc asm_stage_reg_operand(NativeEmitCtx* e,
                                        const AsmConstraint* constraint,
                                        const IRAsmRegRequirement* req,
                                        NativeLoc real, int load_value,
                                        int* staged, SrcLoc loc) {
   NativeAllocClass cls;
   KitCgTypeId type;
   Reg fixed;
   u32 allowed;
   Reg r;
   NativeLoc out;
   if (staged) *staged = 0;
   if (!asm_reg_requirement(e, req, &cls, &fixed, &allowed, loc))
     return real;
   type = constraint->type ? constraint->type : real.type;
   if (asm_reg_loc_satisfies(e, real, cls, fixed, allowed)) return real;
   r = temp_acquire_asm(e, cls, fixed, allowed, loc);
   out = loc_reg(type, cls, r);
   if (load_value) load_loc_into_reg(e, out, real, loc);
   if (staged) *staged = 1;
   return out;
 }
 
 /* For an arithmetic / compare source operand: keep it as an immediate when it
  * is a constant the target can encode for `use` (so no register is wasted
  * materializing it); otherwise materialize into a register. */
 static NativeLoc operand_imm_or_reg(NativeEmitCtx* e, const OptOperand* op,
                                     NativeImmUse use, u32 sub, Reg avoid_a,
                                     Reg avoid_b, SrcLoc loc) {
   if (op->kind == OPK_IMM && e->target->imm_legal &&
       e->target->imm_legal(e->target, use, sub, op->type, op->v.imm))
     return loc_imm(op->type, op->v.imm);
   return materialize_operand(e, op, class_for_type(e, op->type), op->type,
                              avoid_a, avoid_b, loc);
 }
 
 static Reg loc_avoid_reg(NativeLoc l) {
   return l.kind == NATIVE_LOC_REG ? l.v.reg : REG_NONE;
 }
 
 static int type_is_aggregate_or_large(NativeEmitCtx* e, KitCgTypeId type) {
   /* "Large" = wider than one machine word (ptr_size): such a value cannot move
    * through a single register, so IR_COPY/IR_LOAD/IR_STORE of it must go
    * through copy_bytes. 8 on rv64/x64/aa64, 4 on rv32 (so an 8-byte i64/double
    * is large there and is copied as two words rather than truncated into one
    * register). */
   return type && (cg_type_is_aggregate(e->c, type) ||
                   type_size_or(e->c, type, 8u) > e->c->target.ptr_size);
 }
 
 /* Acquire the register that receives a scalar memory load whose MIR result has
  * a frame home. If the address already consumes every available value temp,
  * first collapse it to one plain register. A leased integer base is then dead
  * after address evaluation and may receive the loaded scalar in place: every
  * NativeTarget load hook consumes a simple effective address before writing
  * its destination. Never apply this to an allocator-owned base, an FP result,
  * or a value wider than one machine register. */
 static NativeLoc acquire_frame_load_result(NativeEmitCtx* e,
                                            KitCgTypeId type,
                                            NativeAddr* addr, SrcLoc loc) {
   NativeAllocClass cls = class_for_type(e, type);
   Reg base = addr_base_reg(addr);
   Reg index = addr_index_reg(addr);
   if (temp_available(e, cls, base, index))
     return temp_loc_acquire(e, type, cls, base, index, loc);
 
   collapse_addr_to_reg(e, addr, loc);
   base = addr_base_reg(addr);
   if (cls == NATIVE_REG_INT &&
       type_size_or(e->c, type, e->c->target.ptr_size) <=
           e->c->target.ptr_size &&
       addr->base_kind == NATIVE_ADDR_BASE_REG &&
       addr->index_kind == NATIVE_ADDR_INDEX_NONE && addr->offset == 0 &&
       temp_reg_is_leased(e, NATIVE_REG_INT, base))
     return loc_reg(type, cls, base);
 
   return temp_loc_acquire(e, type, cls, base, addr_index_reg(addr), loc);
 }
 
 /* Materialize an address whose MIR destination has a frame home. A normal
  * free temp receives the address directly. Under full pressure, collapse the
  * expression into its leased base and return that already-materialized value;
  * collapse_addr_to_reg refuses to overwrite an allocator-owned base. */
 static NativeLoc emit_frame_address_result(NativeEmitCtx* e,
                                            KitCgTypeId type,
                                            NativeAddr* addr, SrcLoc loc) {
   NativeAllocClass cls = class_for_type(e, type);
   Reg base = addr_base_reg(addr);
   Reg index = addr_index_reg(addr);
   NativeLoc dst;
   if (temp_available(e, cls, base, index)) {
     dst = temp_loc_acquire(e, type, cls, base, index, loc);
     e->target->load_addr(e->target, dst, *addr);
     return dst;
   }
   collapse_addr_to_reg(e, addr, loc);
   base = addr_base_reg(addr);
   if (cls != NATIVE_REG_INT ||
       !temp_reg_is_leased(e, NATIVE_REG_INT, base))
     emit_panic(e, loc, "materialized address has no scoped integer owner");
   return loc_reg(type, cls, base);
 }
 
 /* Materialize a scalar value while an effective address remains live.  If
  * the address consumes the whole value-temp bank, reduce it to one scoped
  * base first; collapse releases cache-owned components that are dead after
  * address evaluation.  The value then explicitly avoids the surviving base
  * and index, keeping the backend hook's address and data roles disjoint. */
 static NativeLoc materialize_operand_away_from_addr(
     NativeEmitCtx* e, const OptOperand* op, NativeAddr* addr, SrcLoc loc) {
   NativeAllocClass cls = class_for_type(e, op->type);
   if (!temp_available(e, cls, addr_base_reg(addr), addr_index_reg(addr)))
     collapse_addr_to_reg(e, addr, loc);
   return materialize_operand(e, op, cls, op->type, addr_base_reg(addr),
                              addr_index_reg(addr), loc);
 }
 
 /* Copy an aggregate / oversized value between two memory locations. dst and
  * src must be addressable (frame/global/indirect/reg-as-pointer); used for
  * IR_COPY/IR_LOAD/IR_STORE whose value type cannot move through one register.
  */
 static void emit_agg_move(NativeEmitCtx* e, NativeAddr da, NativeAddr sa,
                           KitCgTypeId type, SrcLoc loc) {
   AggregateAccess acc;
   memset(&acc, 0, sizeof acc);
   acc.type = type;
   acc.size = type_size_or(e->c, type, 8u);
   acc.align = type_align_or(e->c, type, 8u);
   acc.mem = mem_for_type(e->c, type);
   /* Legalize both addresses exactly as the scalar load/store path does: a
    * copy_bytes granule ladder addresses base+offset only, so any addressing mode
    * the target cannot encode — notably a scaled index, which an 8-byte i64/double
    * load on a 32-bit target arrives with (it is a "large" move here, the array
    * index folded into the address) — must collapse into a plain base register
    * first. Without this the index is silently dropped and every element aliases
    * element 0. */
   legalize_addr(e, &da, acc.mem, loc);
   legalize_addr(e, &sa, acc.mem, loc);
   e->target->copy_bytes(e->target, da, sa, acc);
 }
 
 static CGFuncDesc semantic_func_desc(NativeEmitCtx* e) {
   OptCGFuncDesc* in = &e->f->desc;
   CGFuncDesc out;
   memset(&out, 0, sizeof out);
   out.sym = in->sym;
   out.text_section_id = in->text_section_id;
   out.group_id = in->group_id;
   out.fn_type = in->fn_type;
   out.result_type = in->result_type;
   out.nparams = in->nparams;
   out.loc = in->loc;
   out.flags = in->flags;
   out.inline_policy = in->inline_policy;
   out.atomize = in->atomize;
   if (in->nparams && in->params) {
     CGParamDesc* params = arena_zarray(e->f->arena, CGParamDesc, in->nparams);
     for (u32 i = 0; i < in->nparams; ++i) {
       params[i].index = in->params[i].index;
       params[i].name = in->params[i].name;
       params[i].type = in->params[i].type;
       params[i].size = in->params[i].size;
       params[i].align = in->params[i].align;
       params[i].flags = in->params[i].flags;
       params[i].loc = in->params[i].loc;
     }
     out.params = params;
   }
   return out;
 }
 
 static CGParamDesc semantic_param_desc(const IRParam* p) {
   CGParamDesc out;
   memset(&out, 0, sizeof out);
   out.index = p->index;
   out.name = p->name;
   out.type = p->type;
   out.size = p->size;
   out.align = p->align;
   out.flags = p->flags;
   out.loc = p->loc;
   return out;
 }
 
 static NativeLoc loc_for_preg(NativeEmitCtx* e, PReg preg, KitCgTypeId type,
                               SrcLoc loc) {
   u8 kind = opt_preg_alloc_kind(e->f, preg);
   if (kind == OPT_ALLOC_HARD)
     return loc_reg(type, (NativeAllocClass)opt_preg_loc_cls(e->f, preg),
                    opt_preg_hard_reg(e->f, preg));
   if (kind == OPT_ALLOC_SPILL)
     return loc_frame(type, class_for_type(e, type),
                      map_slot(e, opt_preg_spill_slot(e->f, preg), loc));
   return loc_none();
 }
 
 static void bind_params(NativeEmitCtx* e) {
   for (u32 i = 0; i < e->f->nparams; ++i) {
     IRParam* p = &e->f->params[i];
     CGParamDesc sd = semantic_param_desc(p);
     NativeLoc dst;
     if (p->storage.kind == CG_LOCAL_STORAGE_REG)
       dst = loc_for_preg(e, (PReg)p->storage.v.reg, p->type, p->loc);
     else
       dst = loc_frame(p->type, class_for_type(e, p->type),
                       map_slot(e, p->storage.v.frame_slot, p->loc));
     if (e->target->bind_param) e->target->bind_param(e->target, &sd, dst);
   }
   /* Let a backend that defers register-destination binds resolve them now (as a
    * parallel copy), once every param's incoming location has been read. */
   if (e->target->bind_params_end) e->target->bind_params_end(e->target);
 }
 
 /* The parameter value is placed into its allocated location by bind_param at
  * function entry; the IR_PARAM_DECL marker emits nothing. */
 static void emit_param_decl(NativeEmitCtx* e, Inst* in) {
   (void)e;
   (void)in;
 }
 
 static NativeFrameSlot temp_slot(NativeEmitCtx* e, KitCgTypeId type, SrcLoc loc,
                                  NativeFrameSlotKind kind) {
   NativeFrameSlotDesc d;
   memset(&d, 0, sizeof d);
   d.type = type;
   d.loc = loc;
   d.size = type_size_or(e->c, type, 8u);
   d.align = type_align_or(e->c, type, d.size >= 8u ? 8u : d.size);
   d.kind = kind;
   return e->target->frame_slot(e->target, &d);
 }
 
 static NativeLoc abi_storage_loc(NativeEmitCtx* e, const OptCGABIValue* v,
                                  SrcLoc loc) {
   if (!v) return loc_none();
   return loc_from_operand(e, &v->storage, loc);
 }
 
 static void emit_call(NativeEmitCtx* e, Inst* in) {
   IRCallAux* aux = (IRCallAux*)in->extra.aux;
   NativeCallDesc d;
   NativeCallPhase plan;
   NativeLoc* args = NULL;
   NativeLoc* results = NULL;
   NativeLoc final_result = loc_none();
   NativeFrameSlot result_slot = NATIVE_FRAME_SLOT_NONE;
   MemAccess result_mem;
   if (!aux) return;
   memset(&d, 0, sizeof d);
   memset(&plan, 0, sizeof plan);
   if (aux->desc.nargs)
     args = arena_zarray(e->f->arena, NativeLoc, aux->desc.nargs);
   for (u32 i = 0; i < aux->desc.nargs; ++i)
     args[i] = abi_storage_loc(e, &aux->desc.args[i], in->loc);
   if (aux->desc.ret.storage.kind) {
     KitCgTypeId rty = aux->desc.ret.type;
     results = arena_zarray(e->f->arena, NativeLoc, 1);
     final_result = abi_storage_loc(e, &aux->desc.ret, in->loc);
     /* Hand marshal_call the value's real destination directly whenever it is a
      * register or a frame slot: a scalar result is a single move out of the ABI
      * result register, and an aggregate / oversized result — which marshal_call or
      * the callee writes in parts and so must land in memory — lands straight in
      * its frame home. Routing either through a fresh temp slot (store then
      * reload / copy_bytes) was a pure round trip on every call. The temp slot
      * is a fallback for the rare result whose storage is neither a register nor
      * a frame slot (e.g. written into a global); lowering hoists aggregates to
      * a frame home (opt_lower_to_mir), so this branch is scalar-only in
      * practice. */
     if (final_result.kind == NATIVE_LOC_REG ||
         final_result.kind == NATIVE_LOC_FRAME) {
       results[0] = final_result;
     } else {
       result_slot = temp_slot(e, rty, in->loc, NATIVE_FRAME_SLOT_SPILL);
       results[0] = loc_frame(rty, class_for_type(e, rty), result_slot);
     }
   }
   d.fn_type = aux->desc.fn_type;
   d.callee = loc_from_operand(e, &aux->desc.callee, in->loc);
   d.args = args;
   d.results = results;
   d.nargs = aux->desc.nargs;
   d.nresults = results ? 1u : 0u;
   d.flags = aux->desc.flags;
   d.tail_policy = aux->desc.tail_policy;
   d.inline_policy = aux->desc.inline_policy;
   /* Materialize a frame-resident indirect callee before marshal_call starts its
    * argument phase.  Real backends marshal register arguments inside
    * marshal_call; doing this afterward allowed the emitter to reuse an already
    * populated ABI argument register for the callee.  The acquired lease also
    * keeps generic phase argument moves from selecting the callee register. */
   if (d.callee.kind != NATIVE_LOC_REG &&
       d.callee.kind != NATIVE_LOC_GLOBAL)
     d.callee = materialize(e, d.callee, NATIVE_REG_INT, d.callee.type,
                            REG_NONE, REG_NONE, in->loc);
   e->target->marshal_call(e->target, &d, &plan);
   for (u32 i = 0; i < plan.nargs; ++i)
     write_phase_loc(e, plan.args[i].dst, plan.args[i].src, plan.args[i].mem,
                     in->loc);
   e->target->emit_call(e->target, &plan);
   temp_scope_phase_barrier(e);
   for (u32 i = 0; i < plan.nrets; ++i)
     write_phase_loc(e, plan.rets[i].dst, plan.rets[i].src, plan.rets[i].mem,
                     in->loc);
   if (result_slot && final_result.kind != NATIVE_LOC_NONE) {
     KitCgTypeId rty = aux->desc.ret.type;
     NativeLoc tmp = loc_frame(rty, class_for_type(e, rty), result_slot);
     result_mem = mem_for_type(e->c, rty);
     if (final_result.kind != NATIVE_LOC_REG &&
         (cg_type_is_aggregate(e->c, rty) ||
          type_size_or(e->c, rty, 8u) > e->c->target.ptr_size)) {
       /* Aggregate / oversized result: move bytes rather than a scalar copy
        * (which would exceed the single-register width). The result was either
        * written in parts by marshal_call's rets, or by the callee via the sret
        * pointer; either way it now lives in the temp slot. */
       AggregateAccess acc;
       NativeAddr da = addr_from_loc(e, final_result, in->loc);
       NativeAddr sa = addr_from_loc(e, tmp, in->loc);
       memset(&acc, 0, sizeof acc);
       acc.type = rty;
       acc.size = type_size_or(e->c, rty, 8u);
       acc.align = type_align_or(e->c, rty, 8u);
       acc.mem = result_mem;
       e->target->copy_bytes(e->target, da, sa, acc);
     } else {
       write_loc(e, final_result, tmp, result_mem, in->loc);
     }
   }
 }
 
 static void emit_ret(NativeEmitCtx* e, Inst* in, const CGFuncDesc* fd) {
   IRRetAux* aux = (IRRetAux*)in->extra.aux;
   NativeLoc value = loc_none();
   const NativeLoc* values = NULL;
   NativeCallPhaseRet* rets = NULL;
   u32 nrets = 0;
   if (aux && aux->present) {
     /* Hand marshal_ret the value's location directly. For an aggregate / oversized
      * result it is a memory location (marshal_ret copies to the sret pointer or
      * reads parts into the return registers); for a scalar it is the value's
      * register or slot, which marshal_ret moves into the return register. The old
      * code spilled scalars to a fresh slot and reloaded them, a pure round
      * trip on every return. */
     value = abi_storage_loc(e, &aux->val, in->loc);
     values = &value;
   }
   e->target->marshal_ret(e->target, fd, values, &rets, &nrets);
   for (u32 i = 0; i < nrets; ++i)
     write_phase_loc(e, rets[i].dst, rets[i].src, rets[i].mem, in->loc);
   /* Skip the trailing branch-to-epilogue when this IR_RET is the very last
    * inst emitted: func_end will place the epilogue label at mc_pos right
    * after this, so the branch would jump to the next 4 bytes. The actual
    * `ret` instruction lives in func_end's restore-frame sequence and is
    * unaffected. */
   if (!e->emitting_terminal_ret) e->target->ret(e->target);
 }
 
 static void emit_inst_body(NativeEmitCtx* e, u32 block, u32 order_index,
                            Inst* in, const CGFuncDesc* fd) {
   NativeLoc dst, a, b, src, tmp;
   NativeAddr addr, addr2;
   Reg dst_reg;
   (void)block;
   if (e->target->set_loc) e->target->set_loc(e->target, in->loc);
   switch ((IROp)in->op) {
     case IR_NOP:
     case IR_CONST_I:
     case IR_CONST_BYTES:
     case IR_PHI:
     case IR_SCOPE_BEGIN:
     case IR_SCOPE_END:
       return;
     case IR_PARAM_DECL:
       emit_param_decl(e, in);
       return;
     case IR_LOAD_IMM:
       dst = loc_from_operand(e, &in->opnds[0], in->loc);
       write_operand(e, &in->opnds[0],
                     loc_imm(in->opnds[0].type, in->extra.imm),
                     mem_for_type(e->c, in->opnds[0].type), in->loc);
       return;
     case IR_LOAD_CONST: {
       NativeLoc real = loc_from_operand(e, &in->opnds[0], in->loc);
       dst = real;
       if (dst.kind != NATIVE_LOC_REG)
         dst = temp_loc_acquire(e, in->opnds[0].type,
                           class_for_type(e, in->opnds[0].type), REG_NONE,
                           REG_NONE, in->loc);
       e->target->load_const(e->target, dst, in->extra.cbytes);
       if (real.kind != NATIVE_LOC_REG)
         write_operand(e, &in->opnds[0], dst,
                       mem_for_type(e->c, in->opnds[0].type), in->loc);
       return;
     }
     case IR_COPY:
       if (type_is_aggregate_or_large(e, in->opnds[0].type)) {
         emit_agg_move(e, addr_from_operand(e, &in->opnds[0], in->loc),
                       addr_from_operand(e, &in->opnds[1], in->loc),
                       in->opnds[0].type, in->loc);
         return;
       }
       dst = loc_from_operand(e, &in->opnds[0], in->loc);
       src = loc_from_operand(e, &in->opnds[1], in->loc);
       if (in->opnds[1].kind == OPK_STACK) {
         NativeLoc cached = frame_cache_take(
             e, &in->opnds[1], src,
             class_for_type(e, in->opnds[1].type), in->opnds[1].type,
             REG_NONE, REG_NONE);
         if (cached.kind != NATIVE_LOC_NONE) src = cached;
       }
       write_operand(e, &in->opnds[0], src,
                     mem_for_type(e->c, in->opnds[0].type), in->loc);
       return;
     case IR_LOAD:
       if (type_is_aggregate_or_large(e, in->opnds[0].type)) {
         addr = addr_from_operand(e, &in->opnds[1], in->loc);
         emit_agg_move(e, addr_from_operand(e, &in->opnds[0], in->loc), addr,
                       in->opnds[0].type, in->loc);
         return;
       }
       dst = loc_from_operand(e, &in->opnds[0], in->loc);
       addr = addr_from_operand(e, &in->opnds[1], in->loc);
       legalize_addr(e, &addr, in->extra.mem, in->loc);
       if (dst.kind == NATIVE_LOC_REG) {
         e->target->load(e->target, dst, addr, in->extra.mem);
       } else {
         tmp = acquire_frame_load_result(e, in->opnds[0].type, &addr, in->loc);
         e->target->load(e->target, tmp, addr, in->extra.mem);
         write_operand(e, &in->opnds[0], tmp, in->extra.mem, in->loc);
       }
       return;
     case IR_STORE:
       if (type_is_aggregate_or_large(e, in->opnds[1].type)) {
         emit_agg_move(e, addr_from_operand(e, &in->opnds[0], in->loc),
                       addr_from_operand(e, &in->opnds[1], in->loc),
                       in->opnds[1].type, in->loc);
         return;
       }
       addr = addr_from_operand(e, &in->opnds[0], in->loc);
       legalize_addr(e, &addr, in->extra.mem, in->loc);
       src = loc_from_operand(e, &in->opnds[1], in->loc);
       /* Storing a constant 0 from the hardware zero register avoids
        * materializing 0 into an operand temp first (e.g. `strb wzr, [..]`
        * rather than `movz w9,0; strb w9, [..]`). */
       if (src.kind == NATIVE_LOC_IMM && src.v.imm == 0 &&
           e->target->has_store_zero_reg &&
           class_for_type(e, in->opnds[1].type) == NATIVE_REG_INT)
         src = loc_reg(in->opnds[1].type, NATIVE_REG_INT,
                       e->target->store_zero_reg);
       /* Source register aliases the address base/index (e.g. `*p = (T)p`).
        * Collapse the address into an instruction-local temp:
        * collapse_addr_to_reg selects one distinct from both base and index —
        * hence distinct from `src` — so the store reads `src` and writes through
        * the fresh temp with no alias. This stays entirely in registers; the
        * frame is fully planned before emission, so emit never allocates a slot
        * here. */
       if (src.kind == NATIVE_LOC_REG && (src.v.reg == addr_base_reg(&addr) ||
                                          src.v.reg == addr_index_reg(&addr)))
         collapse_addr_to_reg(e, &addr, in->loc);
       if (src.kind != NATIVE_LOC_REG) {
         src = materialize_operand_away_from_addr(e, &in->opnds[1], &addr,
                                                  in->loc);
       }
       e->target->store(e->target, addr, src, in->extra.mem);
       return;
     case IR_ADDR_OF: {
       NativeLoc real = loc_from_operand(e, &in->opnds[0], in->loc);
       addr = addr_from_operand(e, &in->opnds[1], in->loc);
       dst = real;
       if (dst.kind != NATIVE_LOC_REG ||
           (addr.index_kind == NATIVE_ADDR_INDEX_REG &&
            dst.v.reg == addr.index.reg)) {
         dst = emit_frame_address_result(e, in->opnds[0].type, &addr, in->loc);
       } else {
         e->target->load_addr(e->target, dst, addr);
       }
       if (!temp_loc_equal(real, dst))
         write_operand(e, &in->opnds[0], dst,
                       mem_for_type(e->c, in->opnds[0].type), in->loc);
       return;
     }
     case IR_TLS_ADDR_OF: {
       IRTlsAux* aux = (IRTlsAux*)in->extra.aux;
       NativeLoc real = loc_from_operand(e, &in->opnds[0], in->loc);
       dst = real;
       if (dst.kind != NATIVE_LOC_REG)
         dst = temp_loc_acquire(e, in->opnds[0].type, NATIVE_REG_INT, REG_NONE,
                           REG_NONE, in->loc);
       e->target->tls_addr_of(e->target, dst, aux->sym, aux->addend);
       if (real.kind != NATIVE_LOC_REG)
         write_operand(e, &in->opnds[0], dst,
                       mem_for_type(e->c, in->opnds[0].type), in->loc);
       return;
     }
     case IR_AGG_COPY: {
       IRAggAux* aux = (IRAggAux*)in->extra.aux;
       addr = pointer_addr_from_operand(e, &in->opnds[0], in->loc, REG_NONE,
                                        REG_NONE);
       addr2 = pointer_addr_from_operand(
           e, &in->opnds[1], in->loc,
           addr.base_kind == NATIVE_ADDR_BASE_REG ? addr.base.reg : REG_NONE,
           REG_NONE);
       legalize_addr(e, &addr, aux->access.mem, in->loc);
       legalize_addr(e, &addr2, aux->access.mem, in->loc);
       e->target->copy_bytes(e->target, addr, addr2, aux->access);
       return;
     }
     case IR_AGG_SET: {
       IRAggAux* aux = (IRAggAux*)in->extra.aux;
       addr = pointer_addr_from_operand(e, &in->opnds[0], in->loc, REG_NONE,
                                        REG_NONE);
       legalize_addr(e, &addr, aux->access.mem, in->loc);
       src = loc_from_operand(e, &in->opnds[1], in->loc);
       if (src.kind != NATIVE_LOC_REG) {
         src = materialize_operand_away_from_addr(e, &in->opnds[1], &addr,
                                                  in->loc);
       }
       e->target->set_bytes(e->target, addr, src, aux->access);
       return;
     }
     case IR_BITFIELD_LOAD: {
       IRBitFieldAux* aux = (IRBitFieldAux*)in->extra.aux;
       NativeLoc real = loc_from_operand(e, &in->opnds[0], in->loc);
       dst = real;
       addr = addr_from_operand(e, &in->opnds[1], in->loc);
       legalize_addr(e, &addr, aux->access.storage, in->loc);
       if (dst.kind != NATIVE_LOC_REG)
         dst = acquire_frame_load_result(e, in->opnds[0].type, &addr,
                                         in->loc);
       e->target->bitfield_load(e->target, dst, addr, aux->access);
       if (real.kind != NATIVE_LOC_REG)
         write_operand(e, &in->opnds[0], dst,
                       mem_for_type(e->c, in->opnds[0].type), in->loc);
       return;
     }
     case IR_BITFIELD_STORE: {
       IRBitFieldAux* aux = (IRBitFieldAux*)in->extra.aux;
       addr = addr_from_operand(e, &in->opnds[0], in->loc);
       legalize_addr(e, &addr, aux->access.storage, in->loc);
       src = loc_from_operand(e, &in->opnds[1], in->loc);
       /* The field store is a read-modify-write that keeps the address live
        * across the value placement, so the value must not be materialized into
        * the address's base/index register. Mirror IR_STORE/IR_AGG_SET: when no
        * other operand temp is free, collapse the address to one temp first,
        * then materialize the value avoiding the address regs. Without this, a
        * spilled pointer reloaded into the first temp and a value reusing that
        * temp collide — the value `mov`s over the address, nulling it (SIGSEGV
        * in optimized SQLite). */
       if (src.kind != NATIVE_LOC_REG) {
         src = materialize_operand_away_from_addr(e, &in->opnds[1], &addr,
                                                  in->loc);
       }
       e->target->bitfield_store(e->target, addr, src, aux->access);
       return;
     }
     case IR_BINOP:
       dst = loc_from_operand(e, &in->opnds[0], in->loc);
       dst_reg = dst.kind == NATIVE_LOC_REG ? dst.v.reg : REG_NONE;
       b = loc_from_operand(e, &in->opnds[2], in->loc);
       a = materialize_operand(e, &in->opnds[1],
                               class_for_type(e, in->opnds[1].type),
                               in->opnds[1].type, dst_reg, loc_avoid_reg(b),
                               in->loc);
       b = operand_imm_or_reg(e, &in->opnds[2], NATIVE_IMM_BINOP,
                              (u32)in->extra.imm, a.v.reg, dst_reg, in->loc);
       /* L7 shift rider: a single-use SHL folded into this binop's second
        * source. The rider only survives onto a register `b` (a shifted-reg
        * ALU operand); an immediate `b` never carries one. The recognition
        * pass only stamps it for targets advertising can_fold_shift_into_alu,
        * so a backend ignoring NativeLoc.shift never receives a nonzero rider. */
       if (b.kind == NATIVE_LOC_REG) b.shift = in->opnds[2].shift;
       if (dst.kind != NATIVE_LOC_REG) {
         NativeAllocClass dst_cls = class_for_type(e, in->opnds[0].type);
         if (a.kind == NATIVE_LOC_REG && a.cls == (u8)dst_cls &&
             temp_reg_is_leased(e, dst_cls, a.v.reg))
           dst = loc_reg(in->opnds[0].type, dst_cls, a.v.reg);
         else
           dst = temp_loc_acquire(e, in->opnds[0].type, dst_cls, a.v.reg,
                             loc_avoid_reg(b), in->loc);
       }
       e->target->binop(e->target, (BinOp)in->extra.imm, dst, a, b);
       if (in->opnds[0].kind != OPK_REG)
         write_operand(e, &in->opnds[0], dst,
                       mem_for_type(e->c, in->opnds[0].type), in->loc);
       return;
     case IR_UNOP:
       dst = loc_from_operand(e, &in->opnds[0], in->loc);
       dst_reg = dst.kind == NATIVE_LOC_REG ? dst.v.reg : REG_NONE;
       a = materialize_operand(e, &in->opnds[1],
                               class_for_type(e, in->opnds[1].type),
                               in->opnds[1].type, dst_reg, REG_NONE, in->loc);
       if (dst.kind != NATIVE_LOC_REG) {
         NativeAllocClass dst_cls = class_for_type(e, in->opnds[0].type);
         if (a.kind == NATIVE_LOC_REG && a.cls == (u8)dst_cls &&
             temp_reg_is_leased(e, dst_cls, a.v.reg))
           dst = loc_reg(in->opnds[0].type, dst_cls, a.v.reg);
         else
           dst = temp_loc_acquire(e, in->opnds[0].type, dst_cls, a.v.reg,
                                  REG_NONE,
                             in->loc);
       }
       e->target->unop(e->target, (UnOp)in->extra.imm, dst, a);
       if (in->opnds[0].kind != OPK_REG)
         write_operand(e, &in->opnds[0], dst,
                       mem_for_type(e->c, in->opnds[0].type), in->loc);
       return;
     case IR_CMP:
       dst = loc_from_operand(e, &in->opnds[0], in->loc);
       dst_reg = dst.kind == NATIVE_LOC_REG ? dst.v.reg : REG_NONE;
       b = loc_from_operand(e, &in->opnds[2], in->loc);
       a = materialize_operand(e, &in->opnds[1],
                               class_for_type(e, in->opnds[1].type),
                               in->opnds[1].type, dst_reg, loc_avoid_reg(b),
                               in->loc);
       b = operand_imm_or_reg(e, &in->opnds[2], NATIVE_IMM_CMP,
                              (u32)in->extra.imm, a.v.reg, dst_reg, in->loc);
       if (dst.kind != NATIVE_LOC_REG) {
         NativeAllocClass dst_cls = class_for_type(e, in->opnds[0].type);
         if (a.kind == NATIVE_LOC_REG && a.cls == (u8)dst_cls &&
             temp_reg_is_leased(e, dst_cls, a.v.reg))
           dst = loc_reg(in->opnds[0].type, dst_cls, a.v.reg);
         else
           dst = temp_loc_acquire(e, in->opnds[0].type, dst_cls, a.v.reg,
                             loc_avoid_reg(b), in->loc);
       }
       e->target->cmp(e->target, (CmpOp)in->extra.imm, dst, a, b);
       if (in->opnds[0].kind != OPK_REG)
         write_operand(e, &in->opnds[0], dst,
                       mem_for_type(e->c, in->opnds[0].type), in->loc);
       return;
     case IR_CONVERT:
       dst = loc_from_operand(e, &in->opnds[0], in->loc);
       dst_reg = dst.kind == NATIVE_LOC_REG ? dst.v.reg : REG_NONE;
       src = materialize_operand(e, &in->opnds[1],
                                 class_for_type(e, in->opnds[1].type),
                                 in->opnds[1].type, dst_reg, REG_NONE, in->loc);
       if (dst.kind != NATIVE_LOC_REG) {
         NativeAllocClass dst_cls = class_for_type(e, in->opnds[0].type);
         if (src.kind == NATIVE_LOC_REG && src.cls == (u8)dst_cls &&
             temp_reg_is_leased(e, dst_cls, src.v.reg))
           dst = loc_reg(in->opnds[0].type, dst_cls, src.v.reg);
         else
           dst = temp_loc_acquire(e, in->opnds[0].type, dst_cls, src.v.reg,
                                  REG_NONE,
                             in->loc);
       }
       e->target->convert(e->target, (ConvKind)in->extra.imm, dst, src);
       if (in->opnds[0].kind != OPK_REG)
         write_operand(e, &in->opnds[0], dst,
                       mem_for_type(e->c, in->opnds[0].type), in->loc);
       return;
     case IR_CALL:
       emit_call(e, in);
       return;
     case IR_BR:
       e->target->jump(e->target,
                       ensure_label(e, e->f->blocks[block].succ[0], in->loc));
       return;
     case IR_CMP_BRANCH: {
       u32 next = order_index + 1u < e->f->emit_order_n
                      ? e->f->emit_order[order_index + 1u]
                      : UINT32_MAX;
       b = loc_from_operand(e, &in->opnds[1], in->loc);
       a = materialize_operand(e, &in->opnds[0],
                               class_for_type(e, in->opnds[0].type),
                               in->opnds[0].type, REG_NONE, loc_avoid_reg(b),
                               in->loc);
       b = operand_imm_or_reg(e, &in->opnds[1], NATIVE_IMM_CMP,
                              (u32)in->extra.imm, a.v.reg, REG_NONE, in->loc);
       e->target->cmp_branch(
           e->target, (CmpOp)in->extra.imm, a, b,
           ensure_label(e, e->f->blocks[block].succ[0], in->loc));
       if (e->f->blocks[block].nsucc > 1u && e->f->blocks[block].succ[1] != next)
         e->target->jump(e->target,
                         ensure_label(e, e->f->blocks[block].succ[1], in->loc));
       return;
     }
     case IR_SWITCH: {
       IRSwitchAux* aux = (IRSwitchAux*)in->extra.aux;
       /* Lease the selector in one register for the whole chain. */
       NativeLoc sel = materialize_operand(
           e, &in->opnds[0], class_for_type(e, in->opnds[0].type),
           in->opnds[0].type, REG_NONE, REG_NONE, in->loc);
       /* Each case value is interpreted at the selector type (matches the CG
        * fallback chain in cg/native_direct_target.c). Route each through the
        * same NATIVE_IMM_CMP immediate-or-register logic IR_CMP_BRANCH uses, so
        * a legal compare immediate becomes `cmp sel, #k` with no per-case
        * materialization; an illegal immediate (e.g. too large for the target's
        * compare encoding) falls back to load_imm into a temp that avoids the
        * leased selector register. */
       KitCgTypeId case_type =
           aux && aux->selector_type ? aux->selector_type : in->opnds[0].type;
       NativeEmitTempMark case_mark = temp_scope_mark(e);
       for (u32 i = 0; aux && i < aux->ncases; ++i) {
         OptOperand case_op;
         NativeLoc imm;
         memset(&case_op, 0, sizeof case_op);
         case_op.kind = OPK_IMM;
         case_op.type = case_type;
         case_op.v.imm = (i64)aux->cases[i].value;
         imm = operand_imm_or_reg(e, &case_op, NATIVE_IMM_CMP, (u32)CMP_EQ,
                                  sel.v.reg, REG_NONE, in->loc);
         e->target->cmp_branch(e->target, CMP_EQ, sel, imm,
                               ensure_label(e, aux->cases[i].block, in->loc));
         temp_scope_rewind(e, case_mark);
       }
       if (aux)
         e->target->jump(e->target,
                         ensure_label(e, aux->default_block, in->loc));
       return;
     }
     case IR_INDIRECT_BRANCH: {
       IRIndirectAux* aux = (IRIndirectAux*)in->extra.aux;
       MCLabel* labels = aux && aux->ntargets
                             ? arena_array(e->f->arena, MCLabel, aux->ntargets)
                             : NULL;
       for (u32 i = 0; aux && i < aux->ntargets; ++i)
         labels[i] = ensure_label(e, aux->targets[i], in->loc);
       src = materialize_operand(e, &in->opnds[0], NATIVE_REG_INT,
                                 in->opnds[0].type, REG_NONE, REG_NONE,
                                 in->loc);
       e->target->indirect_branch(e->target, src, labels,
                                  aux ? aux->ntargets : 0u);
       return;
     }
     case IR_LOAD_LABEL_ADDR: {
       NativeLoc real = loc_from_operand(e, &in->opnds[0], in->loc);
       dst = real;
       if (dst.kind != NATIVE_LOC_REG)
         dst = temp_loc_acquire(e, in->opnds[0].type, NATIVE_REG_INT, REG_NONE,
                           REG_NONE, in->loc);
       e->target->load_label_addr(e->target, dst,
                                  ensure_label(e, (u32)in->extra.imm, in->loc));
       if (real.kind != NATIVE_LOC_REG)
         write_operand(e, &in->opnds[0], dst,
                       mem_for_type(e->c, in->opnds[0].type), in->loc);
       return;
     }
     case IR_LOCAL_STATIC_DATA_BEGIN: {
       CgIrLocalStaticBeginAux* aux = (CgIrLocalStaticBeginAux*)in->extra.aux;
       emit_local_static_begin(e, aux ? &aux->desc : NULL, in->loc);
       return;
     }
     case IR_LOCAL_STATIC_DATA_WRITE: {
       CgIrLocalStaticWriteAux* aux = (CgIrLocalStaticWriteAux*)in->extra.aux;
       if (!aux) emit_panic(e, in->loc, "missing local static data write");
       emit_local_static_write(e, aux->has_data ? aux->data : NULL, aux->len,
                               in->loc);
       return;
     }
     case IR_LOCAL_STATIC_DATA_LABEL_ADDR: {
       CgIrLocalStaticLabelAux* aux = (CgIrLocalStaticLabelAux*)in->extra.aux;
       if (!aux) emit_panic(e, in->loc, "missing local static label data");
       (void)aux->address_space;
       emit_local_static_label_addr(e,
                                    ensure_label(e, (u32)aux->target, in->loc),
                                    aux->addend, aux->width, in->loc);
       return;
     }
     case IR_LOCAL_STATIC_DATA_END:
       emit_local_static_end(e, in->loc);
       return;
     case IR_RET:
       emit_ret(e, in, fd);
       return;
     case IR_ALLOCA: {
       NativeLoc real = loc_from_operand(e, &in->opnds[0], in->loc);
       dst = real;
       src = materialize_operand(e, &in->opnds[1], NATIVE_REG_INT,
                                 in->opnds[1].type, REG_NONE, REG_NONE,
                                 in->loc);
       if (dst.kind != NATIVE_LOC_REG)
         dst = temp_loc_acquire(e, in->opnds[0].type, NATIVE_REG_INT, src.v.reg,
                           REG_NONE, in->loc);
       e->target->alloca_(e->target, dst, src, (u32)in->extra.imm);
       if (real.kind != NATIVE_LOC_REG)
         write_operand(e, &in->opnds[0], dst,
                       mem_for_type(e->c, in->opnds[0].type), in->loc);
       return;
     }
     case IR_ATOMIC_LOAD: {
       IRAtomicAux* aux = (IRAtomicAux*)in->extra.aux;
       dst = loc_from_operand(e, &in->opnds[0], in->loc);
       addr = pointer_addr_from_operand(e, &in->opnds[1], in->loc, REG_NONE,
                                        REG_NONE);
       legalize_addr(e, &addr, aux->mem, in->loc);
       if (dst.kind != NATIVE_LOC_REG)
         dst = acquire_frame_load_result(e, in->opnds[0].type, &addr,
                                         in->loc);
       e->target->atomic_load(e->target, dst, addr, aux->mem, aux->mo);
       if (in->opnds[0].kind != OPK_REG)
         write_operand(e, &in->opnds[0], dst, aux->mem, in->loc);
       return;
     }
     case IR_ATOMIC_STORE: {
       IRAtomicAux* aux = (IRAtomicAux*)in->extra.aux;
       addr = pointer_addr_from_operand(e, &in->opnds[0], in->loc, REG_NONE,
                                        REG_NONE);
       legalize_addr(e, &addr, aux->mem, in->loc);
       src = loc_from_operand(e, &in->opnds[1], in->loc);
       if (src.kind != NATIVE_LOC_REG)
         src = materialize_operand_away_from_addr(e, &in->opnds[1], &addr,
                                                  in->loc);
       e->target->atomic_store(e->target, addr, src, aux->mem, aux->mo);
       return;
     }
     case IR_ATOMIC_RMW: {
       IRAtomicAux* aux = (IRAtomicAux*)in->extra.aux;
       dst = loc_from_operand(e, &in->opnds[0], in->loc);
       addr = pointer_addr_from_operand(e, &in->opnds[1], in->loc, REG_NONE,
                                        REG_NONE);
       legalize_addr(e, &addr, aux->mem, in->loc);
       src = materialize_operand(e, &in->opnds[2],
                                 class_for_type(e, in->opnds[2].type),
                                 in->opnds[2].type, REG_NONE, REG_NONE, in->loc);
       if (dst.kind != NATIVE_LOC_REG)
         dst = temp_loc_acquire(e, in->opnds[0].type,
                           class_for_type(e, in->opnds[0].type), src.v.reg,
                           REG_NONE, in->loc);
       e->target->atomic_rmw(e->target, (KitCgAtomicOp)aux->op, dst, addr, src,
                             aux->mem, aux->mo);
       if (in->opnds[0].kind != OPK_REG)
         write_operand(e, &in->opnds[0], dst, aux->mem, in->loc);
       return;
     }
     case IR_ATOMIC_CAS: {
       IRCasAux* aux = (IRCasAux*)in->extra.aux;
       NativeLoc ok;
       NativeLoc expected;
       NativeLoc desired;
       dst = loc_from_operand(e, &in->opnds[0], in->loc);
       ok = loc_from_operand(e, &in->opnds[1], in->loc);
       addr = pointer_addr_from_operand(e, &in->opnds[2], in->loc, REG_NONE,
                                        REG_NONE);
       legalize_addr(e, &addr, aux->mem, in->loc);
       expected = materialize_operand(
           e, &in->opnds[3], class_for_type(e, in->opnds[3].type),
           in->opnds[3].type, REG_NONE, REG_NONE, in->loc);
       desired = materialize_operand(
           e, &in->opnds[4], class_for_type(e, in->opnds[4].type),
           in->opnds[4].type, expected.v.reg, REG_NONE, in->loc);
       if (dst.kind != NATIVE_LOC_REG)
         dst = temp_loc_acquire(e, in->opnds[0].type,
                           class_for_type(e, in->opnds[0].type), expected.v.reg,
                           desired.v.reg, in->loc);
       if (ok.kind != NATIVE_LOC_REG)
         ok = temp_loc_acquire(e, in->opnds[1].type,
                          class_for_type(e, in->opnds[1].type), dst.v.reg,
                          expected.v.reg, in->loc);
       e->target->atomic_cas(e->target, dst, ok, addr, expected, desired,
                             aux->mem, aux->success, aux->failure);
       if (in->opnds[0].kind != OPK_REG)
         write_operand(e, &in->opnds[0], dst, aux->mem, in->loc);
       if (in->opnds[1].kind != OPK_REG)
         write_operand(e, &in->opnds[1], ok,
                       mem_for_type(e->c, in->opnds[1].type), in->loc);
       return;
     }
     case IR_VA_START: {
       NativeLoc ap = materialize_operand(e, &in->opnds[0], NATIVE_REG_INT,
                                          in->opnds[0].type, REG_NONE, REG_NONE,
                                          in->loc);
       e->target->va_start_(e->target, ap);
       return;
     }
     case IR_VA_END: {
       NativeLoc ap = materialize_operand(e, &in->opnds[0], NATIVE_REG_INT,
                                          in->opnds[0].type, REG_NONE, REG_NONE,
                                          in->loc);
       e->target->va_end_(e->target, ap);
       return;
     }
     case IR_VA_COPY: {
       NativeLoc d = materialize_operand(e, &in->opnds[0], NATIVE_REG_INT,
                                         in->opnds[0].type, REG_NONE, REG_NONE,
                                         in->loc);
       NativeLoc s = materialize_operand(e, &in->opnds[1], NATIVE_REG_INT,
                                         in->opnds[1].type, d.v.reg, REG_NONE,
                                         in->loc);
       e->target->va_copy_(e->target, d, s);
       return;
     }
     case IR_VA_ARG: {
       KitCgTypeId ty = in->opnds[0].type;
       NativeLoc ap = materialize_operand(e, &in->opnds[1], NATIVE_REG_INT,
                                          in->opnds[1].type, REG_NONE, REG_NONE,
                                          in->loc);
       NativeLoc res;
       if (type_is_aggregate_or_large(e, ty)) {
         /* A value too wide for one register (an 8-byte i64/double on a 32-bit
          * target, or an aggregate) cannot pass through one instruction-local
          * register; hand the target its memory destination so it can copy the
          * value directly. */
         e->target->va_arg_(e->target,
                            loc_from_operand(e, &in->opnds[0], in->loc), ap, ty);
         return;
       }
       /* The result must land in a register distinct from the va_list pointer;
        * fetch into an instruction-local temp, then write to the real
        * destination. */
       res = temp_loc_acquire(e, ty, class_for_type(e, ty), ap.v.reg, REG_NONE,
                         in->loc);
       e->target->va_arg_(e->target, res, ap, ty);
       write_operand(e, &in->opnds[0], res, mem_for_type(e->c, ty), in->loc);
       return;
     }
     case IR_ASM_BLOCK: {
       IRAsmAux* aux = (IRAsmAux*)in->extra.aux;
       NativeLoc* out_locs = aux && aux->nout
                                 ? arena_array(e->f->arena, NativeLoc, aux->nout)
                                 : NULL;
       NativeLoc* real_out_locs =
           aux && aux->nout
               ? arena_array(e->f->arena, NativeLoc, aux->nout)
               : NULL;
       NativeLoc* in_locs = aux && aux->nin
                                ? arena_array(e->f->arena, NativeLoc, aux->nin)
                                : NULL;
       int* staged_outs =
           aux && aux->nout ? arena_zarray(e->f->arena, int, aux->nout) : NULL;
       if (e->temps)
         e->temps->allow_asm_temps =
             (u8)!(aux && aux->has_memory_constraint);
       /* Register constraints are an instruction-local placement requirement,
        * not a property of the value's whole live range. Keep regalloc's chosen
        * hard register when it satisfies the constraint; otherwise stage the
        * value through a scoped lease. The backend asm binder validates and
        * binds concrete register operands; only backend-private materialization
        * for memory constraints remains on its side of the boundary. */
       for (u32 i = 0; aux && i < aux->nout; ++i) {
         real_out_locs[i] =
             loc_from_operand(e, &aux->out_ops[i], in->loc);
         out_locs[i] = asm_stage_reg_operand(
             e, &aux->outs[i],
             aux->out_reg_reqs ? &aux->out_reg_reqs[i] : NULL,
             real_out_locs[i],
             aux->outs[i].dir == KIT_CG_ASM_INOUT, &staged_outs[i], in->loc);
       }
       for (u32 i = 0; aux && i < aux->nin; ++i) {
         NativeLoc real = loc_from_operand(e, &aux->in_ops[i], in->loc);
         int matched = native_asm_match_index(aux->ins[i].str);
         if (matched >= 0) {
           if ((u32)matched >= aux->nout ||
               out_locs[matched].kind != NATIVE_LOC_REG)
             emit_panic(e, in->loc, "invalid matching asm operand");
           if (native_asm_constraint_early(aux->outs[matched].str))
             emit_panic(e, in->loc,
                        "matching input names early-clobber output");
           load_loc_into_reg(e, out_locs[matched], real, in->loc);
           in_locs[i] = out_locs[matched];
         } else {
           in_locs[i] = asm_stage_reg_operand(
               e, &aux->ins[i],
               aux->in_reg_reqs ? &aux->in_reg_reqs[i] : NULL, real, 1, NULL,
               in->loc);
         }
       }
       e->target->asm_block(e->target, aux ? aux->tmpl : "",
                            aux ? aux->outs : NULL, aux ? aux->nout : 0,
                            out_locs, aux ? aux->ins : NULL, aux ? aux->nin : 0,
                            in_locs, aux ? aux->clobbers : NULL,
                            aux ? aux->nclob : 0);
       for (u32 i = 0; aux && i < aux->nout; ++i)
         if (staged_outs[i])
           write_operand(e, &aux->out_ops[i], out_locs[i],
                         mem_for_type(e->c, out_locs[i].type), in->loc);
       return;
     }
     case IR_BREAK_TO:
     case IR_CONTINUE_TO:
       emit_panic(e, in->loc, "operation is not wired to NativeTarget yet");
     case IR_FENCE:
       e->target->fence(e->target, (KitCgMemOrder)in->extra.imm);
       return;
     case IR_UNREACHABLE:
       e->target->trap(e->target);
       return;
     case IR_INTRINSIC: {
       IRIntrinAux* aux = (IRIntrinAux*)in->extra.aux;
       NativeLoc* dsts = aux && aux->ndst
                             ? arena_array(e->f->arena, NativeLoc, aux->ndst)
                             : NULL;
       NativeLoc* real_dsts =
           aux && aux->ndst
               ? arena_array(e->f->arena, NativeLoc, aux->ndst)
               : NULL;
       NativeLoc* args = aux && aux->narg
                             ? arena_array(e->f->arena, NativeLoc, aux->narg)
                             : NULL;
       for (u32 i = 0; aux && i < aux->narg; ++i) {
         if (aux->args[i].kind == OPK_IMM &&
             native_intrinsic_arg_accepts_imm(e->target, aux->kind, i)) {
           args[i] = loc_from_operand(e, &aux->args[i], in->loc);
         } else {
           args[i] = materialize_operand(
               e, &aux->args[i], class_for_type(e, aux->args[i].type),
               aux->args[i].type, REG_NONE, REG_NONE, in->loc);
         }
       }
       for (u32 i = 0; aux && i < aux->ndst; ++i) {
         NativeAllocClass cls = class_for_type(e, aux->dsts[i].type);
         real_dsts[i] = loc_from_operand(e, &aux->dsts[i], in->loc);
         dsts[i] = real_dsts[i];
         if (dsts[i].kind != NATIVE_LOC_NONE &&
             dsts[i].kind != NATIVE_LOC_REG)
           dsts[i] = temp_loc_acquire(e, aux->dsts[i].type, cls, REG_NONE,
                                      REG_NONE,
                                 in->loc);
       }
       e->target->intrinsic(e->target, aux->kind, dsts, aux->ndst, args,
                            aux->narg);
       for (u32 i = 0; aux && i < aux->ndst; ++i) {
         if (real_dsts[i].kind == NATIVE_LOC_NONE ||
             real_dsts[i].kind == NATIVE_LOC_REG)
           continue;
         write_operand(e, &aux->dsts[i], dsts[i],
                       mem_for_type(e->c, aux->dsts[i].type), in->loc);
       }
       return;
     }
     default:
       emit_panic(e, in->loc, "unknown IR op");
   }
 }
 
 static void emit_inst(NativeEmitCtx* e, u32 block, u32 order_index, Inst* in,
                       const CGFuncDesc* fd) {
   NativeEmitTempScope scope;
   temp_scope_begin(e, &scope, in);
   emit_inst_body(e, block, order_index, in, fd);
   temp_scope_end(e, &scope);
 }
 
 static int native_emit_terminates(const Inst* in) {
   if (!in) return 0;
   switch ((IROp)in->op) {
     case IR_BR:
     case IR_CONDBR:
     case IR_CMP_BRANCH:
     case IR_SWITCH:
     case IR_INDIRECT_BRANCH:
     case IR_RET:
     case IR_UNREACHABLE:
     case IR_BREAK_TO:
     case IR_CONTINUE_TO:
       return 1;
     case IR_INTRINSIC: {
       IRIntrinAux* aux = (IRIntrinAux*)in->extra.aux;
       return aux && (aux->kind == INTRIN_LONGJMP || aux->kind == INTRIN_TRAP);
     }
     default:
       return 0;
   }
 }
 
 static void emit_block(NativeEmitCtx* e, u32 block, u32 order_index,
                        const CGFuncDesc* fd) {
   if (block >= e->f->nblocks) return;
   /* No cache fact crosses a control-flow edge. Even a single-predecessor block
    * may be entered through a layout-independent branch, so block entry is the
    * simple, explicit ownership boundary. */
   frame_cache_clear(e);
   if (!e->label_placed[block]) {
     e->label_placed[block] = 1u;
     e->target->label_place(e->target,
                            ensure_label(e, block, (SrcLoc){0, 0, 0}));
   }
   Block* bl = &e->f->blocks[block];
   int is_last_block = order_index + 1u == e->f->emit_order_n;
   for (u32 i = 0; i < bl->ninsts; ++i) {
     e->emitting_terminal_ret = is_last_block && i + 1u == bl->ninsts &&
                                (IROp)bl->insts[i].op == IR_RET;
     emit_inst(e, block, order_index, &bl->insts[i], fd);
   }
   e->emitting_terminal_ret = 0;
   if (bl->nsucc == 1u &&
       (bl->ninsts == 0 ||
        !native_emit_terminates(&bl->insts[bl->ninsts - 1u]))) {
     u32 next = order_index + 1u < e->f->emit_order_n
                    ? e->f->emit_order[order_index + 1u]
                    : UINT32_MAX;
     if (bl->succ[0] != next)
       e->target->jump(e->target,
                       ensure_label(e, bl->succ[0], (SrcLoc){0, 0, 0}));
   }
 }
 
 #define EMIT_MAX_REG_CLASSES 4u
 
 static void collect_used_reg(Func* f, Inst* in, OptOperand* op, int is_def,
                              void* ctx) {
   u32* used = (u32*)ctx;
   (void)f;
   (void)in;
   (void)is_def;
   if (op && op->kind == OPT_OPK_REG && op->cls < EMIT_MAX_REG_CLASSES &&
       op->v.reg < 32u)
     used[op->cls] |= 1u << op->v.reg;
 }
 
 static void compute_emit_live_after(NativeEmitCtx* e) {
   OptHardBlockLive* blocks = opt_maybe_build_hard_live(e->f);
   e->live_after_ready = 1u;
   e->live_after_cap = e->f->next_inst_id;
   if (!e->live_after_cap) return;
   e->live_after_by_inst =
       arena_zarray(e->f->arena, OptHardRegSet, e->live_after_cap);
   for (u32 b = 0; b < e->f->nblocks; ++b) {
     Block* bl = &e->f->blocks[b];
     OptHardRegSet live =
         opt_hard_live_out_for_block(blocks ? &blocks[b] : NULL);
     for (u32 ri = bl->ninsts; ri > 0; --ri) {
       Inst* in = &bl->insts[ri - 1u];
       OptRegEffectMasks effects;
       OptHardRegSet kills;
       if (in->id != INST_ID_NONE && in->id < e->live_after_cap)
         e->live_after_by_inst[in->id] = live;
       opt_inst_reg_effect_masks(e->f, in, &effects);
       opt_reg_effect_mask_kills(&effects, &kills);
       opt_hard_live_step(&live, &effects.uses, &kills);
     }
   }
 }
 
 static NativeLoc asm_planning_loc(const OptOperand* op) {
   NativeLoc loc = loc_none();
   if (!op) return loc;
   loc.type = op->type;
   if (op->kind == OPK_REG)
     loc = loc_reg(op->type, (NativeAllocClass)op->cls, op->v.reg);
   return loc;
 }
 
 /* Dry-run one instruction's asm allocator before frame finalization. The same
  * candidate order and unavailable masks are used by final emission, so retain
  * only the ABI-relevant result: callee-saved staging registers overwritten. */
 static void plan_one_asm_staging(NativeEmitCtx* e, Inst* in, IRAsmAux* aux) {
   NativeEmitTempScope scope;
   if (!aux) return;
   temp_scope_begin(e, &scope, in);
   scope.allow_asm_temps = (u8)!aux->has_memory_constraint;
   for (u32 k = 0; k < aux->nout; ++k) {
     (void)asm_stage_reg_operand(
         e, &aux->outs[k],
         aux->out_reg_reqs ? &aux->out_reg_reqs[k] : NULL,
         asm_planning_loc(&aux->out_ops[k]), 0, NULL, in->loc);
   }
   for (u32 k = 0; k < aux->nin; ++k) {
     if (native_asm_match_index(aux->ins[k].str) >= 0) continue;
     (void)asm_stage_reg_operand(
         e, &aux->ins[k],
         aux->in_reg_reqs ? &aux->in_reg_reqs[k] : NULL,
         asm_planning_loc(&aux->in_ops[k]), 0, NULL, in->loc);
   }
   for (u32 c = 0; c < OPT_REG_CLASSES; ++c)
     e->asm_stage_callee_used[c] |=
         scope.leased[c] & e->f->opt_callee_saved[c];
   temp_scope_end(e, &scope);
 }
 
 static NativeLoc* marshal_call_args_reserve(NativeEmitCtx* e, NativeLoc* args,
                                          u32* cap, u32 want) {
   u32 grown;
   if (want <= *cap) return args;
   grown = *cap ? *cap : want;
   while (grown < want && grown <= 0x7fffffffu) grown *= 2u;
   if (grown < want) grown = want;
   *cap = grown;
   return arena_zarray(e->f->arena, NativeLoc, grown);
 }
 
 static void plan_asm_clobber_push(NativeEmitCtx* e, Sym** clobbers, u32* count,
                                   u32* cap, Sym clobber) {
   if (*count == *cap) {
     u32 grown = *cap ? *cap * 2u : 8u;
     Sym* next;
     if (grown < *cap) grown = *count + 1u;
     next = arena_array(e->f->arena, Sym, grown);
     if (*count) memcpy(next, *clobbers, sizeof *next * *count);
     *clobbers = next;
     *cap = grown;
   }
   (*clobbers)[(*count)++] = clobber;
 }
 
 /* Plan the complete call frame before any code is emitted, then hand it to the
  * backend via func_begin_known_frame so the prologue is emitted final. The
  * optimizer knows everything the frame needs after register allocation and MIR
  * lowering: the callee-saved set (scanned from the MIR), every static frame
  * slot (f->frame_slots), and the outgoing-arg area (the max over all calls of
  * the pure call_stack_bytes query). The body therefore allocates no slots, so
  * the frame is final up front and nothing is back-patched. Populates
  * e->slot_map from the backend-assigned slot handles for the body to use. */
 static void plan_frame(NativeEmitCtx* e, const CGFuncDesc* fd) {
   NativeTarget* t = e->target;
   NativeKnownFrameDesc frame;
   NativeFrameSlotDesc* slots = NULL;
   NativeFrameSlot* out_slots = NULL;
   u32 used[EMIT_MAX_REG_CLASSES];
   u32 nclasses;
   NativeLoc* call_args = NULL;
   u32 call_args_cap = 0;
   u32 max_outgoing = 0;
   u8 has_alloca = 0;
   u8 needs_scratch_spill = 0;
   u8 has_call = 0;
   u8 has_asm = 0;
   u8 reads_frame = 0;
   u32 nasm_clob = 0;
   u32 asm_clob_cap = 0;
   u32 asm_clobber_abi_sets = 0;
   Sym* asm_clobbers = NULL;
   memset(&frame, 0, sizeof frame);
   memset(used, 0, sizeof used);
   memset(e->asm_stage_callee_used, 0, sizeof e->asm_stage_callee_used);
   memset(e->callee_saved_used, 0, sizeof e->callee_saved_used);
   /* One primary MIR scan collects allocated hard registers, asm staging, and
    * all frame-shape facts. This used to be three independent full walks. */
   for (u32 b = 0; b < e->f->nblocks; ++b) {
     Block* bl = &e->f->blocks[b];
     for (u32 i = 0; i < bl->ninsts; ++i) {
       Inst* in = &bl->insts[i];
       if (t->reserve_callee_saves)
         opt_walk_inst_operands(e->f, in, collect_used_reg, used);
       if ((IROp)in->op == IR_ALLOCA) {
         has_alloca = 1;
       } else if ((IROp)in->op == IR_ATOMIC_RMW) {
         needs_scratch_spill = 1;
       } else if ((IROp)in->op == IR_CALL) {
         IRCallAux* aux = (IRCallAux*)in->extra.aux;
         /* Any call (regular or sibling/tail) means the function is not a leaf:
          * it clobbers the return-address register and the stack below sp. */
         has_call = 1;
         if (aux && t->call_stack_bytes) {
           NativeCallDesc d;
           u32 sb;
           call_args = marshal_call_args_reserve(
               e, call_args, &call_args_cap, aux->desc.nargs);
           memset(&d, 0, sizeof d);
           d.fn_type = aux->desc.fn_type;
           d.flags = aux->desc.flags;
           d.nargs = aux->desc.nargs;
           for (u32 k = 0; k < aux->desc.nargs; ++k) {
             memset(&call_args[k], 0, sizeof call_args[k]);
             call_args[k].type = aux->desc.args[k].type;
           }
           d.args = call_args;
           sb = t->call_stack_bytes(t, &d);
           if (sb > max_outgoing) max_outgoing = sb;
         }
       } else if ((IROp)in->op == IR_ASM_BLOCK) {
         /* Inline asm may clobber the return-address register or the red zone
          * opaquely; disqualifies the frame-eliding tiers (see has_asm). Its
          * callee-saved register clobbers and named-register operand
          * requirements are equally opaque to the operand scan below; collect
          * them now so the backend can fold them into the saved set. */
         IRAsmAux* aux = (IRAsmAux*)in->extra.aux;
         has_asm = 1;
         if (aux) {
           plan_one_asm_staging(e, in, aux);
           for (u32 k = 0; k < aux->nclob; ++k)
             plan_asm_clobber_push(e, &asm_clobbers, &nasm_clob,
                                   &asm_clob_cap, aux->clobbers[k]);
           for (u32 k = 0; k < aux->nout; ++k)
             if (aux->outs[k].reg)
               plan_asm_clobber_push(e, &asm_clobbers, &nasm_clob,
                                     &asm_clob_cap, aux->outs[k].reg);
           for (u32 k = 0; k < aux->nin; ++k)
             if (aux->ins[k].reg)
               plan_asm_clobber_push(e, &asm_clobbers, &nasm_clob,
                                     &asm_clob_cap, aux->ins[k].reg);
           asm_clobber_abi_sets |= aux->clobber_abi_sets;
         }
       } else if ((IROp)in->op == IR_INTRINSIC) {
         /* __builtin_frame_address / __builtin_return_address read the frame
          * record, so the function must keep one (disables the rv64 frameless
          * leaf tier; see NativeKnownFrameDesc.reads_frame). */
         IRIntrinAux* aux = (IRIntrinAux*)in->extra.aux;
         if (aux && (aux->kind == INTRIN_FRAME_ADDRESS ||
                     aux->kind == INTRIN_RETURN_ADDRESS))
           reads_frame = 1;
       } else if ((IROp)in->op == IR_TLS_ADDR_OF) {
         /* The Mach-O TLV descriptor access calls the resolver thunk — a hidden
          * call the leaf analysis must see, or the frame-eliding leaf tiers (the
          * x64 SysV red zone, which keeps spills below sp; any return-address
          * elision) miscompile: the thunk's `call`/`blr` clobbers the return
          * address and the bytes below sp. ELF Local-Exec and Windows TEB
          * accesses make no call, so only the descriptor model disqualifies. */
         if (obj_format_tls_via_descriptor(t->c)) has_call = 1;
       }
     }
   }
   nclasses = t->reserve_callee_saves && t->regs
                  ? (t->regs->nclasses < EMIT_MAX_REG_CLASSES
                         ? t->regs->nclasses
                         : EMIT_MAX_REG_CLASSES)
                  : 0u;
   for (u32 i = 0; t->regs && i < t->regs->nclasses; ++i) {
     const NativeAllocClassInfo* ci = &t->regs->classes[i];
     if (ci->cls >= EMIT_MAX_REG_CLASSES) continue;
     used[ci->cls] &=
         native_target_callee_saved_mask(t, (NativeAllocClass)ci->cls);
     used[ci->cls] |= e->asm_stage_callee_used[ci->cls];
   }
   for (u32 c = 0; c < nclasses && c < OPT_REG_CLASSES; ++c)
     e->callee_saved_used[c] = used[c];
   e->slot_map =
       arena_zarray(e->f->arena, NativeFrameSlot, e->f->nframe_slots + 1u);
   if (e->f->nframe_slots) {
     slots = arena_zarray(e->f->arena, NativeFrameSlotDesc, e->f->nframe_slots);
     out_slots = arena_zarray(e->f->arena, NativeFrameSlot, e->f->nframe_slots);
     for (u32 i = 0; i < e->f->nframe_slots; ++i) {
       IRFrameSlot* s = &e->f->frame_slots[i];
       NativeFrameSlotDesc* d = &slots[i];
       memset(d, 0, sizeof *d);
       d->type = s->type;
       d->name = s->name;
       d->loc = s->loc;
       d->size = s->size;
       d->align = s->align;
       d->priority = s->priority;
       d->kind = s->kind;
       d->flags = s->flags;
     }
   }
   /* W1.0 hot-slot-low ordering. The backend's known-frame path bump-allocates
    * each body slot in the order it is presented (cum_off is monotonic, see
    * native_frame_slot_alloc), so slots presented first get the smallest final
    * frame displacement. Present the body slots in DESCENDING priority order so
    * the hottest spills land low: disp8 on x64 (-4 bytes/access), inside the
    * scaled reach on aa64, inside the +/-2KB imm12 window on rv64.
    *
    * `order[k]` is the original slot index now presented at position k. Fixed-
    * offset slots keep their original positions (their displacement is not
    * order-derived); the remaining slots are stable-sorted by descending priority
    * with an original-index tie-break (so the sort is deterministic and, among
    * equal priorities, byte-stable). The native slot the backend returns for
    * position k therefore belongs to original IR slot `order[k]`, and the slot_map
    * is written back through `order` so e->slot_map[ir_id] stays exact -- a
    * transposed map silently miscompiles every spill, so it is asserted below. */
   u32 nfs = e->f->nframe_slots;
   u32* slot_order = NULL; /* slot_order[k] = original IR slot index at position k */
   if (nfs > 1u) {
     u32* order = arena_array(e->f->arena, u32, nfs);
     u32* movable = arena_array(e->f->arena, u32, nfs);
     u32 nmovable = 0;
     for (u32 i = 0; i < nfs; ++i) {
       order[i] = i; /* fixed slots stay pinned at their original index */
       if (!(slots[i].flags & NATIVE_FRAME_SLOT_FIXED_OFFSET))
         movable[nmovable++] = i;
     }
     /* Stable descending-priority sort of the movable indices (insertion sort:
      * nslots is small per function and this is off the per-instruction path; if
      * a profile shows it material, swap in a radix over the integer priority --
      * see O1.md W1.0 "Linear?"). Insertion sort is naturally stable, preserving
      * original order among equal priorities. */
     for (u32 i = 1; i < nmovable; ++i) {
       u32 cur = movable[i];
       u32 cur_pri = slots[cur].priority;
       u32 j = i;
       while (j > 0 && slots[movable[j - 1]].priority < cur_pri) {
         movable[j] = movable[j - 1];
         --j;
       }
       movable[j] = cur;
     }
     /* Drop the sorted movable indices back into the non-fixed positions. */
     u32 mi = 0;
     for (u32 k = 0; k < nfs; ++k)
       if (!(slots[k].flags & NATIVE_FRAME_SLOT_FIXED_OFFSET))
         order[k] = movable[mi++];
     /* Reorder the desc array in place via a scratch copy so frame.slots is the
      * contiguous, permuted view the backend iterates. */
     NativeFrameSlotDesc* ordered =
         arena_array(e->f->arena, NativeFrameSlotDesc, nfs);
     for (u32 k = 0; k < nfs; ++k) ordered[k] = slots[order[k]];
     slot_order = order;
     slots = ordered;
   }
   frame.slots = slots;
   frame.nslots = e->f->nframe_slots;
   frame.max_outgoing = max_outgoing;
   frame.callee_saved_used = nclasses ? used : NULL;
   frame.ncallee_classes = nclasses;
   frame.has_alloca = has_alloca;
   frame.needs_scratch_spill = needs_scratch_spill;
   frame.is_leaf = !has_call;
   frame.has_asm = has_asm;
   frame.reads_frame = reads_frame;
   frame.asm_clobbers = asm_clobbers;
   frame.nasm_clobbers = nasm_clob;
   frame.asm_clobber_abi_sets = asm_clobber_abi_sets;
   t->func_begin_known_frame(t, fd, &frame, out_slots);
   /* Map each backend native slot (returned for presentation position k) back to
    * its original IR frame slot id. With ordering, position k holds IR slot
    * order[k]; without it (nfs<=1 or no reorder), position k == IR slot k. */
   for (u32 k = 0; k < e->f->nframe_slots; ++k) {
     u32 ir_idx = slot_order ? slot_order[k] : k;
     e->slot_map[e->f->frame_slots[ir_idx].id] = out_slots[k];
   }
 #ifndef NDEBUG
   /* The slot_map must stay an exact 1:1 of IR frame slots to the native slots
    * the backend returned -- a transposed/dropped map silently miscompiles every
    * spill. The actual native slot ids are NOT bounded by nframe_slots (the
    * backend reserves callee-saves / entry-saves / scratch around the body slots,
    * so body native ids are an arbitrary distinct block), so verify the real
    * correctness condition instead: `order` is a permutation of [0, nfs) -- i.e.
    * every IR frame slot is presented exactly once -- and each presented position
    * received a non-NONE native slot. Both are O(nslots) (seen array sized by the
    * IR index domain), off the per-instruction path. With identity ordering this
    * is vacuously the original code's invariant. */
   if (nfs) {
     u8* seen = arena_zarray(e->f->arena, u8, nfs);
     for (u32 k = 0; k < nfs; ++k) {
       u32 ir_idx = slot_order ? slot_order[k] : k;
       if (ir_idx >= nfs || seen[ir_idx])
         compiler_panic(e->c, (SrcLoc){0, 0, 0},
                        "opt W1.0: slot order is not a permutation at pos %u",
                        (unsigned)k);
       seen[ir_idx] = 1u;
       if (out_slots[k] == NATIVE_FRAME_SLOT_NONE)
         compiler_panic(e->c, (SrcLoc){0, 0, 0},
                        "opt W1.0: backend returned no native slot for pos %u",
                        (unsigned)k);
     }
   }
 #endif
 }
 
 void opt_emit_native(Compiler* c, Func* f, NativeTarget* target) {
   NativeEmitCtx e;
   Func view;
   CGFuncDesc fd;
   if (!f || !target) return;
   memset(&e, 0, sizeof e);
   e.f = opt_mir_view(f, &view) ? &view : f;
   e.c = c;
   e.target = target;
   metrics_scope_begin(c, "opt.native_emit.setup");
   e.labels = arena_array(e.f->arena, MCLabel, e.f->nblocks ? e.f->nblocks : 1u);
   e.label_placed =
       arena_zarray(e.f->arena, u8, e.f->nblocks ? e.f->nblocks : 1u);
   for (u32 i = 0; i < e.f->nblocks; ++i) e.labels[i] = MC_LABEL_NONE;
   fd = semantic_func_desc(&e);
   metrics_scope_end(c, "opt.native_emit.setup");
 
   metrics_scope_begin(c, "opt.native_emit.func_begin");
   /* The optimizer has the whole frame after regalloc + MIR lowering, so it
    * plans it up front (plan_frame) and drives func_begin_known_frame: the
    * backend emits a final prologue with no reserved NOP region and no
    * back-patching. The body allocates no frame slots, so the frame stays final;
    * allocas and tail epilogues are emitted final too. (Contrast the
    * single-pass NativeDirectTarget path, which reserves and patches.) */
   plan_frame(&e, &fd);
   bind_params(&e);
   metrics_scope_end(c, "opt.native_emit.func_begin");
 
   metrics_scope_begin(c, "opt.native_emit.body");
   for (u32 i = 0; i < e.f->emit_order_n; ++i)
     emit_block(&e, e.f->emit_order[i], i, &fd);
   metrics_scope_end(c, "opt.native_emit.body");
 
   metrics_scope_begin(c, "opt.native_emit.func_end");
   target->func_end(target);
   metrics_scope_end(c, "opt.native_emit.func_end");
 }