kit

pass_lower.c (81403B)

---

 #include <stdlib.h>
 #include <string.h>
 
 #include "core/arena.h"
 #include "core/core.h"
 #include "core/metrics.h"
 #include "core/pool.h"
 #include "core/slice.h"
 #include "core/strbuf.h"
 #include "opt/opt_internal.h"
 
 enum {
   OPT_CG_TYPE_SEG_SHIFT = 6,
   OPT_CG_TYPE_SEG_MASK = (1u << OPT_CG_TYPE_SEG_SHIFT) - 1u,
   OPT_CG_TYPE_BUILTIN_SEG = 1u,
 };
 
 static u32 type_size_fallback(const Func* f, KitCgTypeId t) {
   KitCgBuiltinType b;
   if (!t) return f->opt_target.ptr_size ? f->opt_target.ptr_size : 8u;
   if ((t >> OPT_CG_TYPE_SEG_SHIFT) != OPT_CG_TYPE_BUILTIN_SEG) {
     return f->opt_target.ptr_size ? f->opt_target.ptr_size : 8u;
   }
   b = (KitCgBuiltinType)(t & OPT_CG_TYPE_SEG_MASK);
   switch (b) {
     case KIT_CG_BUILTIN_BOOL:
     case KIT_CG_BUILTIN_I8:
       return 1;
     case KIT_CG_BUILTIN_I16:
       return 2;
     case KIT_CG_BUILTIN_I32:
     case KIT_CG_BUILTIN_F32:
       return 4;
     case KIT_CG_BUILTIN_I64:
     case KIT_CG_BUILTIN_F64:
       return 8;
     case KIT_CG_BUILTIN_I128:
       return 16;
     case KIT_CG_BUILTIN_VOID:
       return 0;
     case KIT_CG_BUILTIN_VARARG_STATE:
     case KIT_CG_BUILTIN_COUNT:
     default:
       return f->opt_target.ptr_size ? f->opt_target.ptr_size : 8u;
   }
 }
 
 static u32 bit_words(u32 npregs) { return (npregs + 63u) / 64u; }
 
 static void bit_set(u64* bits, PReg v) {
   u32 w = v / 64u;
   u64 mask = 1ull << (v % 64u);
   u64 old = bits[w];
   bits[w] = old | mask;
 }
 static void bit_clear(u64* bits, PReg v) {
   u32 w = v / 64u;
   u64 mask = 1ull << (v % 64u);
   u64 old = bits[w];
   bits[w] = old & ~mask;
 }
 static int bit_has(const u64* bits, PReg v) {
   u32 w = v / 64u;
   u64 mask = 1ull << (v % 64u);
   u64 old = bits[w];
   return (old & mask) != 0;
 }
 
 typedef struct BitsCtx {
   u64* use;
   u64* def;
 } BitsCtx;
 
 typedef struct InstRefs {
   PReg* uses;
   PReg* defs;
   u32 nuses;
   u32 ndefs;
   u32 use_cap;
   u32 def_cap;
 } InstRefs;
 
 static void collect_bits(Func* f, Inst* in, Operand* op, int is_def,
                          void* arg) {
   (void)in;
   BitsCtx* c = (BitsCtx*)arg;
   if (op->kind != OPK_REG) return;
   PReg v = (PReg)op->v.reg;
   if (v == PREG_NONE || v == 0 || v >= opt_reg_count(f)) return;
   if (is_def)
     bit_set(c->def, v);
   else
     bit_set(c->use, v);
 }
 
 static void refs_reset(InstRefs* refs) {
   refs->nuses = 0;
   refs->ndefs = 0;
 }
 
 static void refs_push(Func* f, PReg** pregs, u32* npregs, u32* cap, PReg v) {
   for (u32 i = 0; i < *npregs; ++i)
     if ((*pregs)[i] == v) return;
   if (*npregs == *cap) {
     u32 ncap = *cap ? *cap * 2u : 8u;
     PReg* nv = arena_array(f->arena, PReg, ncap);
     if (*pregs) memcpy(nv, *pregs, sizeof((*pregs)[0]) * *npregs);
     *pregs = nv;
     *cap = ncap;
   }
   (*pregs)[(*npregs)++] = v;
 }
 
 static void refs_collect(Func* f, Inst* in, Operand* op, int is_def,
                          void* arg) {
   (void)in;
   InstRefs* refs = (InstRefs*)arg;
   if (op->kind != OPK_REG) return;
   PReg v = (PReg)op->v.reg;
   if (v == PREG_NONE || v == 0 || v >= opt_reg_count(f)) return;
   if (is_def)
     refs_push(f, &refs->defs, &refs->ndefs, &refs->def_cap, v);
   else
     refs_push(f, &refs->uses, &refs->nuses, &refs->use_cap, v);
 }
 
 static int refs_has_def(const InstRefs* refs, PReg v) {
   for (u32 i = 0; i < refs->ndefs; ++i)
     if (refs->defs[i] == v) return 1;
   return 0;
 }
 
 static void live_update_refs_before(u64* live, const InstRefs* refs) {
   for (u32 i = 0; i < refs->ndefs; ++i) bit_clear(live, refs->defs[i]);
   for (u32 i = 0; i < refs->nuses; ++i) bit_set(live, refs->uses[i]);
 }
 
 static u32 live_update_refs_before_active(u64* live, u32 active_words,
                                           u32 nwords, const InstRefs* refs) {
   for (u32 i = 0; i < refs->ndefs; ++i) {
     PReg v = refs->defs[i];
     if (v == PREG_NONE || v == 0) continue;
     u32 w = v / 64u;
     if (w < active_words) live[w] &= ~(1ull << (v % 64u));
   }
   while (active_words && live[active_words - 1u] == 0) --active_words;
   for (u32 i = 0; i < refs->nuses; ++i) {
     PReg v = refs->uses[i];
     if (v == PREG_NONE || v == 0) continue;
     u32 w = v / 64u;
     if (w >= nwords) continue;
     live[w] |= 1ull << (v % 64u);
     if (active_words <= w) active_words = w + 1u;
   }
   return active_words;
 }
 
 static void forbid_preg_reg(Func* f, PReg v, u8 cls, Reg r) {
   if (v == PREG_NONE || v == 0 || v >= opt_reg_count(f) ||
       cls >= OPT_REG_CLASSES || r >= 32)
     return;
   if (opt_reg_cls(f, v) != cls) return;
   f->preg_info[v].forbidden_hard_regs |= 1u << r;
 }
 
 static void apply_fixed_asm_operand(Func* f, Operand* op, i32 fixed,
                                     u8 fixed_cls) {
   if (!op || op->kind != OPK_REG || fixed < 0) return;
   PReg v = (PReg)op->v.reg;
   if (v == PREG_NONE || v == 0 || v >= opt_reg_count(f)) return;
   if (fixed_cls >= OPT_REG_CLASSES || opt_reg_cls(f, v) != fixed_cls) {
     SrcLoc loc = {0, 0, 0};
     compiler_panic(f->c, loc, "opt asm: fixed register class mismatch");
   }
   f->preg_info[v].tied_hard_reg = fixed;
 }
 
 static void apply_allowed_asm_operand(Func* f, Operand* op, u32 allowed,
                                       u8 allowed_cls) {
   u32 hard_mask = 0;
   if (!op || op->kind != OPK_REG || !allowed) return;
   PReg v = (PReg)op->v.reg;
   if (v == PREG_NONE || v == 0 || v >= opt_reg_count(f)) return;
   if (allowed_cls >= OPT_REG_CLASSES || opt_reg_cls(f, v) != allowed_cls) {
     SrcLoc loc = {0, 0, 0};
     compiler_panic(f->c, loc, "opt asm: allowed register class mismatch");
   }
   for (u32 i = 0; i < f->opt_hard_reg_count[allowed_cls]; ++i) {
     Reg r = f->opt_hard_regs[allowed_cls][i];
     if (r < 32) hard_mask |= 1u << r;
   }
   if (f->preg_info[v].allowed_hard_regs) {
     u32 both = f->preg_info[v].allowed_hard_regs & allowed;
     if (!both) {
       SrcLoc loc = {0, 0, 0};
       compiler_panic(f->c, loc, "opt asm: conflicting allowed register sets");
     }
     f->preg_info[v].allowed_hard_regs = both;
   } else {
     f->preg_info[v].allowed_hard_regs = allowed;
   }
   f->preg_info[v].forbidden_hard_regs |= hard_mask & ~allowed;
 }
 
 static void apply_asm_register_constraints(Func* f, Inst* in, u64* use,
                                            u64* def, u64* live_after) {
   IRAsmAux* aux = (IRAsmAux*)in->extra.aux;
   if (!aux || !f->preg_info) return;
 
   for (u32 i = 0; i < aux->nout; ++i) {
     i32 fixed = aux->out_fixed_regs ? aux->out_fixed_regs[i] : -1;
     u8 cls = aux->out_fixed_cls ? aux->out_fixed_cls[i] : 0;
     u32 allowed = aux->out_allowed_masks ? aux->out_allowed_masks[i] : 0;
     u8 allowed_cls = aux->out_allowed_cls ? aux->out_allowed_cls[i] : 0;
     apply_allowed_asm_operand(f, &aux->out_ops[i], allowed, allowed_cls);
     apply_fixed_asm_operand(f, &aux->out_ops[i], fixed, cls);
   }
   for (u32 i = 0; i < aux->nin; ++i) {
     i32 fixed = aux->in_fixed_regs ? aux->in_fixed_regs[i] : -1;
     u8 cls = aux->in_fixed_cls ? aux->in_fixed_cls[i] : 0;
     u32 allowed = aux->in_allowed_masks ? aux->in_allowed_masks[i] : 0;
     u8 allowed_cls = aux->in_allowed_cls ? aux->in_allowed_cls[i] : 0;
     apply_allowed_asm_operand(f, &aux->in_ops[i], allowed, allowed_cls);
     apply_fixed_asm_operand(f, &aux->in_ops[i], fixed, cls);
   }
 
   for (u32 cls = 0; cls < OPT_REG_CLASSES; ++cls) {
     u32 mask = aux->clobber_mask[cls];
     if (!mask) continue;
     for (Reg r = 0; r < 32; ++r) {
       if ((mask & (1u << r)) == 0) continue;
       for (PReg v = 1; v < opt_reg_count(f); ++v) {
         if (!bit_has(use, v) && !bit_has(def, v) &&
             !(live_after && bit_has(live_after, v)))
           continue;
         forbid_preg_reg(f, v, (u8)cls, r);
       }
     }
   }
 }
 
 /* Apply the per-instruction fixed-register clobbers recorded in machinization
  * (Func.inst_clobbers). A register the instruction's encoding destroys must not
  * hold any value live AFTER the instruction unless that value is (re)defined
  * here — so forbid each clobbered register for every live-after, non-def value.
  * Values that merely die at the instruction (dying uses) need no constraint:
  * the backend stages them into/out of the fixed registers itself. */
 static void apply_machine_reg_clobbers(Func* f, Inst* in, u64* def,
                                        u64* live_after) {
   if (!f->preg_info || !f->inst_clobbers || in->id == INST_ID_NONE ||
       in->id >= f->inst_clobbers_cap)
     return;
   for (u32 cls = 0; cls < OPT_REG_CLASSES; ++cls) {
     u32 mask = f->inst_clobbers[in->id][cls];
     if (!mask) continue;
     for (Reg r = 0; r < 32; ++r) {
       if ((mask & (1u << r)) == 0) continue;
       for (PReg v = 1; v < opt_reg_count(f); ++v) {
         if (!(live_after && bit_has(live_after, v)) || bit_has(def, v))
           continue;
         if ((u8)opt_reg_cls(f, v) != (u8)cls) continue;
         f->preg_info[v].forbidden_hard_regs |= 1u << r;
         /* Record this as a hard clobber so the return-register hint won't later
          * clear the forbid and place the value in a register the instruction
          * destroys (see set_preg_pref_to_ret_reg). */
         f->preg_info[v].clobbered_hard_regs |= 1u << r;
       }
     }
   }
 }
 
 static int phys_arg_reg_for_index(Func* f, u8 cls, u32 abi_index, Reg* out) {
   if (!f || cls >= OPT_REG_CLASSES) return 0;
   for (u32 i = 0; i < f->opt_phys_reg_count[cls]; ++i) {
     const CGPhysRegInfo* pi = &f->opt_phys_regs[cls][i];
     if ((pi->flags & CG_REG_ARG) && pi->abi_index == abi_index) {
       if (out) *out = pi->reg;
       return 1;
     }
   }
   return 0;
 }
 
 static int hard_available(Func* f, u8 cls, Reg r);
 
 static int is_caller_saved(Func* f, u8 cls, Reg r) {
   if (cls >= OPT_REG_CLASSES || r >= 32) return 0;
   return (f->opt_caller_saved[cls] & (1u << r)) != 0;
 }
 
 static Reg first_ret_reg(Func* f, u8 cls) {
   if (!f || cls >= OPT_REG_CLASSES) return REG_NONE;
   u32 mask = f->opt_ret_regs[cls];
   for (Reg r = 0; r < 32; ++r)
     if (mask & (1u << r)) return r;
   return REG_NONE;
 }
 
 static void set_preg_pref_to_ret_reg(Func* f, const Operand* op) {
   if (!op || op->kind != OPK_REG) return;
   PReg v = (PReg)op->v.reg;
   if (v == PREG_NONE || v == 0 || v >= opt_reg_count(f)) return;
   u8 cls = f->preg_info[v].cls;
   if (cls >= OPT_REG_CLASSES) return;
   Reg hint = first_ret_reg(f, cls);
   if (hint == REG_NONE || hint >= 32) return;
   /* Don't override a real pin. */
   if (f->preg_info[v].tied_hard_reg >= 0) return;
   /* A value live across an instruction that clobbers the ret reg cannot live
    * there; skip the hint so the allocator places it elsewhere and the return
    * copy moves it into the ret reg (e.g. an accumulator returned past a va_arg,
    * which uses rax). */
   if (f->preg_info[v].clobbered_hard_regs & (1u << hint)) return;
   /* The hint reg may not be in opt_hard_regs (e.g. x0 on aa64 is reserved
    * as the ABI ret reg, outside aa_int_allocable); the allocator's
    * preferred-reg branch will still consider it via the unit-overlap
    * precision check. Clear any leftover soft forbid bit so the hint isn't
    * silently blocked. */
   f->preg_info[v].forbidden_hard_regs &= ~(1u << hint);
   f->preg_info[v].preferred_hard_reg = (i8)hint;
 }
 
 static void set_preg_pref_for_abivalue(Func* f, const CGABIValue* v) {
   if (!v) return;
   set_preg_pref_to_ret_reg(f, &v->storage);
   for (u32 i = 0; i < v->nparts; ++i)
     set_preg_pref_to_ret_reg(f, &v->parts[i].op);
 }
 
 /* Soft hint: prefer a specific ABI register for `op`'s PReg. Symmetric to
  * set_preg_pref_to_ret_reg but takes an arbitrary hint reg (the matching
  * arg reg for the i-th call argument). */
 static void set_preg_pref_to_arg_reg(Func* f, const Operand* op, Reg hint) {
   if (!op || op->kind != OPK_REG) return;
   if (hint == REG_NONE || hint >= 32) return;
   PReg v = (PReg)op->v.reg;
   if (v == PREG_NONE || v == 0 || v >= opt_reg_count(f)) return;
   u8 cls = f->preg_info[v].cls;
   if (cls >= OPT_REG_CLASSES) return;
   if (f->preg_info[v].tied_hard_reg >= 0) return;
   if (f->preg_info[v].preferred_hard_reg >= 0) return;
   f->preg_info[v].forbidden_hard_regs &= ~(1u << hint);
   f->preg_info[v].preferred_hard_reg = (i8)hint;
 }
 
 /* Hint each single-PReg-stored param toward its own incoming ABI reg. When
  * the allocator picks the incoming reg, bind_param sees src==dst and emits
  * no entry move (aa_bind_native_param checks at native.c:3227). Live-range
  * conflicts at body use sites still go through the normal allocator check,
  * so cross-call params that need a callee-save get one. */
 /* True iff `f` contains any IR_CALL flagged as a tail call. Tail-call arg
  * routing goes through the backend shuffle which can permute the caller's
  * incoming arg regs into different positions for the callee — pinning each
  * param PReg to its own incoming reg turns those permutations into multi-reg
  * cycles the shuffle can't break. Symmetric to the per-call tail skip in
  * set_preg_pref_for_call_args. */
 static int func_has_tail_call(const Func* f) {
   if (!f) return 0;
   for (u32 b = 0; b < f->nblocks; ++b) {
     const Block* bl = &f->blocks[b];
     for (u32 i = 0; i < bl->ninsts; ++i) {
       const Inst* in = &bl->insts[i];
       if ((IROp)in->op != IR_CALL) continue;
       const IRCallAux* aux = (const IRCallAux*)in->extra.aux;
       if (aux && (aux->desc.flags & CG_CALL_TAIL)) return 1;
     }
   }
   return 0;
 }
 
 static void set_preg_pref_for_params(Func* f) {
   if (!f || !f->preg_info || !f->nparams) return;
   if (func_has_tail_call(f)) return;
   /* Per-class ABI arg cursors. Drives from per-param ABI info rather than
    * f->desc.abi so this fires on paths where only f->params[i].abi is set. */
   u32 next_int = 0;
   u32 next_fp = 0;
   /* An sret pointer passed in the first integer argument register consumes
    * that slot (SysV-x64 rdi, Win64 rcx, RISC-V a0); ABIs that return it in a
    * dedicated register (AArch64 x8) do not. Driven by the ABI descriptor so no
    * arch identity is needed here. */
   if (f->desc.abi && f->desc.abi->sret_consumes_int_arg) next_int = 1;
   for (u32 i = 0; i < f->nparams; ++i) {
     IRParam* p = &f->params[i];
     const ABIArgInfo* ai = p->abi;
     if (!ai || ai->kind == ABI_ARG_IGNORE) continue;
     if (ai->kind == ABI_ARG_INDIRECT) {
       ++next_int;
       continue;
     }
     if (ai->kind != ABI_ARG_DIRECT) continue;
     /* Only hint single-part DIRECT params whose home is a single PReg.
      * Aggregate / split params take the bind_param frame-store path. */
     int single_part_to_preg =
         (ai->nparts == 1) && (p->storage.kind == CG_LOCAL_STORAGE_REG);
     if (single_part_to_preg) {
       const ABIArgPart* part = &ai->parts[0];
       u8 cls = (part->cls == ABI_CLASS_FP) ? RC_FP : RC_INT;
       u32* counter = (cls == RC_FP) ? &next_fp : &next_int;
       Reg hint = REG_NONE;
       if (*counter < 8u && phys_arg_reg_for_index(f, cls, *counter, &hint)) {
         PReg v = (PReg)p->storage.v.reg;
         if (v != PREG_NONE && v != 0 && v < opt_reg_count(f) &&
             f->preg_info[v].cls == cls && f->preg_info[v].tied_hard_reg < 0 &&
             f->preg_info[v].preferred_hard_reg < 0 && hint != REG_NONE &&
             hint < 32) {
           f->preg_info[v].forbidden_hard_regs &= ~(1u << hint);
           f->preg_info[v].preferred_hard_reg = (i8)hint;
         }
       }
     }
     /* Advance the ABI cursors for every part of this param's home, regardless
      * of whether we hinted, so subsequent params see the right slot. */
     for (u16 j = 0; j < ai->nparts; ++j) {
       u32* c = (ai->parts[j].cls == ABI_CLASS_FP) ? &next_fp : &next_int;
       *c += 1u;
     }
   }
 }
 
 /* For each IR_CALL arg whose source storage is a single OPK_REG, hint that
  * PReg to the matching ABI arg register. Sequential int/fp counters mirror
  * the per-class arg slot assignment used by set_preg_pref_for_params. Skips
  * variadic, has_sret, and indirect/aggregate args: they need per-target
  * counter logic that hasn't been factored out of plan_call. */
 static void set_preg_pref_for_call_args(Func* f, const CGCallDesc* desc) {
   if (!f || !desc) return;
   /* Tail calls handle arg routing through the tail-call shuffle in the
    * backend, which can resolve permutations (e.g. swap of caller's incoming
    * x0/x1 into tail-call x1/x0). Hinting the arg source PRegs across that
    * shuffle creates cycles bind_param / the entry-bind moves can't unbreak,
    * miscompiling permute / cycle cases (toy 24, 27, 28, ...). Skip. */
   if (desc->flags & CG_CALL_TAIL) return;
   const ABIFuncInfo* abi = desc->abi;
   if (!abi && f->c && f->c->abi)
     abi = abi_cg_func_info(f->c->abi, desc->fn_type);
   if (!abi || abi->variadic || abi->has_sret) return;
   u32 next_int = 0, next_fp = 0;
   for (u32 i = 0; i < desc->nargs; ++i) {
     if (i >= abi->nparams) break;
     const CGABIValue* a = &desc->args[i];
     const ABIArgInfo* ai = &abi->params[i];
     if (ai->kind == ABI_ARG_IGNORE) continue;
     if (ai->kind == ABI_ARG_INDIRECT) {
       ++next_int;
       continue;
     }
     if (ai->kind != ABI_ARG_DIRECT || ai->nparts == 0) continue;
     const ABIArgPart* part0 = &ai->parts[0];
     u8 cls = part0->cls == ABI_CLASS_FP ? RC_FP : RC_INT;
     u32* counter = cls == RC_FP ? &next_fp : &next_int;
     if (*counter < 8u && ai->nparts == 1) {
       Reg hint = REG_NONE;
       if (phys_arg_reg_for_index(f, cls, *counter, &hint))
         set_preg_pref_to_arg_reg(f, &a->storage, hint);
     }
     for (u16 p = 0; p < ai->nparts; ++p) {
       u32* c = (ai->parts[p].cls == ABI_CLASS_FP) ? &next_fp : &next_int;
       *c += 1u;
     }
   }
 }
 
 /* Propagate preferred_hard_reg backward across IR_COPY chains. When the
  * frontend emits `copy def=v_arg opnds=[v_arg, v_value]` to carry a value
  * into a call's arg slot, set_preg_pref_for_call_args hints v_arg to the
  * ABI dest (e.g. x0). But the underlying producer (v_value, e.g. the load
  * result) is unhinted and lands in a generic caller-save (e.g. x8); the
  * copy then emits `mov x0, x8`. Propagating the hint from v_arg to v_value
  * lets both land at x0 and turns the copy into an identity move that
  * combine elides. Walks insts; for each IR_COPY whose def has a hint and
  * whose single OPK_REG source operand has none, propagate (and clear the
  * source's forbid for the hint reg). Safe because the copy itself dies
  * once both sides share the reg. */
 static void propagate_hint_through_copies(Func* f) {
   if (!f || !f->preg_info) return;
   for (u32 b = 0; b < f->nblocks; ++b) {
     Block* bl = &f->blocks[b];
     for (u32 i = 0; i < bl->ninsts; ++i) {
       Inst* in = &bl->insts[i];
       if ((IROp)in->op != IR_COPY) continue;
       if (in->nopnds < 2 || in->opnds[0].kind != OPK_REG ||
           in->opnds[1].kind != OPK_REG)
         continue;
       PReg dst = (PReg)in->opnds[0].v.reg;
       PReg src = (PReg)in->opnds[1].v.reg;
       if (dst == PREG_NONE || dst == 0 || dst >= opt_reg_count(f)) continue;
       if (src == PREG_NONE || src == 0 || src >= opt_reg_count(f)) continue;
       i8 dst_pref = f->preg_info[dst].preferred_hard_reg;
       if (dst_pref < 0) continue;
       if (f->preg_info[src].tied_hard_reg >= 0) continue;
       if (f->preg_info[src].preferred_hard_reg >= 0) continue;
       if (f->preg_info[dst].cls != f->preg_info[src].cls) continue;
       /* Don't propagate the hint into a value clobbered out of that register by
        * a machine instruction live across it (e.g. a loop accumulator returned
        * past a va_arg/idiv): it cannot live there. Leave it allocated
        * elsewhere; the copy moves it into the hinted register. */
       if (f->preg_info[src].clobbered_hard_regs & (1u << (Reg)dst_pref))
         continue;
       f->preg_info[src].forbidden_hard_regs &= ~(1u << (Reg)dst_pref);
       f->preg_info[src].preferred_hard_reg = dst_pref;
     }
   }
 }
 
 /* Set a soft "prefer the ABI return reg" hint on:
  *   - IR_CALL result PRegs (so emit_call's `mov result, x0` is elided)
  *   - IR_RET value PRegs   (so emit_ret's `mov x0, value` is elided)
  *   - IR_CALL arg source PRegs (so emit_call's `mov x0, src` is elided);
  *     hints are propagated backward through IR_COPY so the actual producer
  *     of the value also prefers the ABI dest reg.
  *
  * The hint is a tie-breaker only — see hard_reg_alloc_score. The allocator's
  * existing conflict checks already exclude regs with real interference (e.g.
  * a result PReg live across another call cannot pick x0). */
 static void apply_abi_aliasing_hints(Func* f) {
   if (!f || !f->preg_info) return;
   set_preg_pref_for_params(f);
   for (u32 b = 0; b < f->nblocks; ++b) {
     Block* bl = &f->blocks[b];
     for (u32 i = 0; i < bl->ninsts; ++i) {
       Inst* in = &bl->insts[i];
       if ((IROp)in->op == IR_CALL) {
         IRCallAux* aux = (IRCallAux*)in->extra.aux;
         if (aux) {
           set_preg_pref_for_abivalue(f, &aux->desc.ret);
           set_preg_pref_for_call_args(f, &aux->desc);
         }
       } else if ((IROp)in->op == IR_RET) {
         IRRetAux* aux = (IRRetAux*)in->extra.aux;
         if (aux && aux->present) set_preg_pref_for_abivalue(f, &aux->val);
       }
     }
   }
   propagate_hint_through_copies(f);
 }
 
 /* ---------------------------------------------------------------------------
  * Register allocator, MIR-shaped.
  *
  * Data structures and assignment algorithm mirror MIR's reg_alloc/assign
  * (mir-gen.c:7551-7728, simplified_p branch). Conflict detection uses a
  * point-indexed bitmap of locations (hard regs + stack slots) instead of
  * a sorted interval vector per hard register.
  *
  *   used_locs[p * loc_words .. p * loc_words + loc_words)  -- one row per
  *                                                             compressed
  *                                                             program point
  *
  *   Bit indices:
  *     0 .. hard_loc_bits-1   -> hard registers (hard_loc_bit(cls, r))
  *     hard_loc_bits + k      -> stack slot index k (k < stack_slot_count)
  *
  * Live-range splitting (`get_hard_reg_with_split`, `lr_gap_t`, `split()`)
  * is deferred per doc/plan/OPTIMIZER.md.
  * ------------------------------------------------------------------------- */
 
 typedef struct OptAllocator OptAllocator;
 
 static int hard_available(Func* f, u8 cls, Reg r) {
   if (cls >= OPT_REG_CLASSES) return 0;
   for (u32 i = 0; i < f->opt_hard_reg_count[cls]; ++i)
     if (f->opt_hard_regs[cls][i] == r) return 1;
   return 0;
 }
 
 static const CGPhysRegInfo* phys_info_for(Func* f, u8 cls, Reg r) {
   if (cls >= OPT_REG_CLASSES) return NULL;
   for (u32 i = 0; i < f->opt_phys_reg_count[cls]; ++i)
     if (f->opt_phys_regs[cls][i].reg == r) return &f->opt_phys_regs[cls][i];
   return NULL;
 }
 
 static FrameSlot spill_slot_for(Func* f, PReg v) {
   FrameSlot existing = opt_preg_spill_slot(f, v);
   if (existing != FRAME_SLOT_NONE) return existing;
   FrameSlotDesc d;
   memset(&d, 0, sizeof d);
   d.type = opt_reg_type(f, v);
   d.size = type_size_fallback(f, opt_reg_type(f, v));
   d.align = d.size >= 8 ? 8 : d.size;
   d.kind = FS_SPILL;
   f->preg_info[v].spill_slot = ir_frame_slot_new(f, &d);
   return f->preg_info[v].spill_slot;
 }
 
 static u32 hard_loc_bit(u8 cls, Reg r) { return ((u32)cls * 32u) + (u32)r; }
 
 typedef struct OptAllocCandidate {
   PReg v; /* coalesce-root PReg with live ranges */
   u32 spill_cost;
   u32 live_length;
   u8 tied;
   u8 pad[3];
 } OptAllocCandidate;
 
 typedef struct OptAllocGroupInfo {
   PReg root;
   u32 spill_cost;
   u32 live_length;
   u32 first;
   u32 last;
   i32 tied_hard_reg;
   u32 forbidden_hard_regs;
   u32 allowed_hard_regs;
   u8 cls;
   u8 pad[3];
 } OptAllocGroupInfo;
 
 typedef struct OptAllocator {
   OptLoc* locs; /* per-PReg result (cls, hard_reg, spill_slot) */
 
   /* Per-point bitmap of locations. used_locs[p * loc_words + w] is word w
    * of the bitmap for compressed program point p. Bit indices:
    *   0 .. hard_loc_bits - 1            -> hard regs (hard_loc_bit)
    *   hard_loc_bits + stack_idx         -> stack slot indices */
   u64* used_locs;
   u32 point_count;
   u32 loc_words;     /* width of one row, in u64 words */
   u32 hard_loc_bits; /* OPT_REG_CLASSES * 32 */
 
   /* Stack slot table (parallel arrays). */
   FrameSlot* stack_slots;
   u32 stack_slot_count;
   u32 stack_slot_cap;
 
   /* hard_open[hard_loc_bit] is 1 if at least one PReg has been assigned to
    * this hard reg in the current function. Drives `hard_reg_alloc_score`'s
    * callee-save bias. */
   u8* hard_open;
 
   /* Scratch bitmap of loc_words. Reused per candidate. */
   u64* conflict_locs;
 
   /* Metrics. */
   u64 hard_point_visits;  /* points scanned during hard-reg conflict probe */
   u64 stack_point_visits; /* points scanned during stack-slot probe */
   u64 hard_word_ors;      /* word-OR operations into conflict_locs */
   u64 stack_word_ors;
   u64 hard_mark_points; /* points marked when assigning a hard reg */
   u64 stack_mark_points;
 } OptAllocator;
 
 /* Bitmap helpers over loc_words-wide rows of used_locs. */
 static u64* used_locs_row(OptAllocator* a, u32 p) {
   return &a->used_locs[(u64)p * a->loc_words];
 }
 
 static int loc_bit_in_conflict(const u64* conflict_locs, u32 bit) {
   return (conflict_locs[bit / 64u] & (1ull << (bit % 64u))) != 0;
 }
 
 static u32 alloc_loc_words_for_bits(u32 bits) { return (bits + 63u) / 64u; }
 
 static void alloc_grow_loc_words(Func* f, OptAllocator* a, u32 need_words) {
   if (need_words <= a->loc_words) return;
   u32 new_words = a->loc_words ? a->loc_words : 1u;
   while (new_words < need_words) new_words *= 2u;
   u64* nb = arena_zarray(f->arena, u64, (u64)a->point_count * new_words);
   if (a->used_locs && a->loc_words) {
     for (u32 p = 0; p < a->point_count; ++p)
       memcpy(&nb[(u64)p * new_words], &a->used_locs[(u64)p * a->loc_words],
              sizeof(u64) * a->loc_words);
   }
   a->used_locs = nb;
   a->loc_words = new_words;
   u64* nc = arena_zarray(f->arena, u64, new_words);
   a->conflict_locs = nc;
 }
 
 static u32 alloc_alloc_stack_slot(Func* f, OptAllocator* a, FrameSlot fs) {
   if (a->stack_slot_count == a->stack_slot_cap) {
     u32 ncap = a->stack_slot_cap ? a->stack_slot_cap * 2u : 16u;
     FrameSlot* ns = arena_array(f->arena, FrameSlot, ncap);
     if (a->stack_slots)
       memcpy(ns, a->stack_slots,
              sizeof(a->stack_slots[0]) * a->stack_slot_count);
     a->stack_slots = ns;
     a->stack_slot_cap = ncap;
   }
   u32 idx = a->stack_slot_count++;
   a->stack_slots[idx] = fs;
   u32 needed_bits = a->hard_loc_bits + a->stack_slot_count;
   u32 needed_words = alloc_loc_words_for_bits(needed_bits);
   alloc_grow_loc_words(f, a, needed_words);
   return idx;
 }
 
 static u32 hard_reg_alloc_score(Func* f, const OptAllocator* a,
                                 const OptPRegInfo* vi, Reg hr) {
   const CGPhysRegInfo* pi = phys_info_for(f, vi->cls, hr);
   u32 score = pi ? pi->spill_cost : 0;
   if (vi->live_across_call_freq) {
     if (is_caller_saved(f, vi->cls, hr))
       score += 1000u + vi->live_across_call_freq;
     else
       score += 20u;
   } else if (!is_caller_saved(f, vi->cls, hr)) {
     u32 bit = hard_loc_bit(vi->cls, hr);
     int already_open =
         a->hard_open && bit < a->hard_loc_bits && a->hard_open[bit];
     if (!already_open) score += pi ? pi->copy_cost : 50u;
   }
   /* Soft hint: tie-break toward a preferred hard reg by adding +1 to every
    * non-hinted choice. Used by apply_abi_aliasing_hints to put IR_CALL
    * results / IR_RET values into the ABI return register so emit_call /
    * emit_ret can elide the materialization move. Stays well below the
    * live-across-call (+1000) penalty and the callee-save (+20) gap so it
    * never overrides real costs. */
   if (vi->preferred_hard_reg >= 0 && (Reg)vi->preferred_hard_reg != hr)
     score += 1u;
   return score;
 }
 
 static int alloc_candidate_higher(const OptAllocCandidate* a,
                                   const OptAllocCandidate* b) {
   if (a->tied != b->tied) return a->tied > b->tied;
   if (a->spill_cost != b->spill_cost) return a->spill_cost > b->spill_cost;
   if (a->live_length != b->live_length) return a->live_length < b->live_length;
   return a->v < b->v;
 }
 
 static int alloc_candidate_cmp(const void* va, const void* vb) {
   const OptAllocCandidate* a = (const OptAllocCandidate*)va;
   const OptAllocCandidate* b = (const OptAllocCandidate*)vb;
   if (alloc_candidate_higher(a, b)) return -1;
   if (alloc_candidate_higher(b, a)) return 1;
   return 0;
 }
 
 static void alloc_sort_candidates(OptAllocCandidate* cands, u32 n) {
   if (n > 1) qsort(cands, n, sizeof(cands[0]), alloc_candidate_cmp);
 }
 
 static PReg alloc_coalesce_root(Func* f, PReg v) {
   if (!f->opt_coalesce_parent || v == PREG_NONE || v >= opt_reg_count(f))
     return v;
   PReg p = (PReg)f->opt_coalesce_parent[v];
   while (p != f->opt_coalesce_parent[p]) p = (PReg)f->opt_coalesce_parent[p];
   while (v != p) {
     PReg n = (PReg)f->opt_coalesce_parent[v];
     f->opt_coalesce_parent[v] = p;
     v = n;
   }
   return p;
 }
 
 static int alloc_group_member(Func* f, PReg root, PReg v) {
   return alloc_coalesce_root(f, v) == root;
 }
 
 static void alloc_group_info(Func* f, const OptLiveRangeSet* ranges, PReg root,
                              OptAllocGroupInfo* out) {
   memset(out, 0, sizeof *out);
   out->root = root;
   out->first = (u32)~0u;
   out->tied_hard_reg = -1;
   out->cls = f->preg_info[root].cls;
   for (PReg v = 1; v < opt_reg_count(f); ++v) {
     if (ranges->first_range_by_preg[v] == OPT_RANGE_NONE) continue;
     if (!alloc_group_member(f, root, v)) continue;
     OptPRegInfo* vi = &f->preg_info[v];
     out->spill_cost += vi->frequency ? vi->frequency : vi->spill_cost;
     out->live_length += vi->live_length;
     u32 first = vi->first_pos ? vi->first_pos - 1u : 0;
     if (first < out->first) out->first = first;
     if (vi->last_pos > out->last) out->last = vi->last_pos;
     out->forbidden_hard_regs |= vi->forbidden_hard_regs;
     if (vi->allowed_hard_regs) {
       if (out->allowed_hard_regs) {
         u32 both = out->allowed_hard_regs & vi->allowed_hard_regs;
         if (!both) {
           SrcLoc loc = {0, 0, 0};
           compiler_panic(f->c, loc,
                          "opt asm: conflicting allowed register sets");
         }
         out->allowed_hard_regs = both;
       } else {
         out->allowed_hard_regs = vi->allowed_hard_regs;
       }
     }
     if (vi->tied_hard_reg >= 0) out->tied_hard_reg = vi->tied_hard_reg;
   }
   if (out->first == (u32)~0u) out->first = 0;
 }
 
 static void opt_init_preg_info_from_ranges(Func* f,
                                            const OptLiveRangeSet* ranges) {
   OptPRegInfo* old = f->preg_info;
   OptPRegInfo* info = arena_zarray(f->arena, OptPRegInfo,
                                    opt_reg_count(f) ? opt_reg_count(f) : 1u);
   for (PReg v = 0; v < opt_reg_count(f); ++v) {
     i32 tied = old ? old[v].tied_hard_reg : -1;
     u32 forbidden = old ? old[v].forbidden_hard_regs : 0;
     u32 allowed = old ? old[v].allowed_hard_regs : 0;
     u32 clobbered = old ? old[v].clobbered_hard_regs : 0;
     u32 old_frequency = old ? old[v].frequency : 0;
     i8 pref = old ? old[v].preferred_hard_reg : (i8)-1;
     OptPRegInfo* vi = &info[v];
     vi->tied_hard_reg = tied;
     vi->preferred_hard_reg = pref;
     vi->hard_reg = REG_NONE;
     vi->spill_slot = FRAME_SLOT_NONE;
     vi->alloc_kind = OPT_ALLOC_NONE;
     vi->cls = opt_reg_cls(f, v);
     vi->forbidden_hard_regs = forbidden;
     vi->allowed_hard_regs = allowed;
     vi->clobbered_hard_regs = clobbered;
     if (!ranges || v == PREG_NONE || v == 0 ||
         ranges->first_range_by_preg[v] == OPT_RANGE_NONE) {
       continue;
     }
     u32 first = (u32)~0u;
     u32 last = 0;
     for (u32 r = ranges->first_range_by_preg[v]; r != OPT_RANGE_NONE;
          r = ranges->ranges[r].next) {
       const OptLiveRange* lr = &ranges->ranges[r];
       if (lr->start < first) first = lr->start;
       if (lr->end > last) last = lr->end;
     }
     vi->first_pos = first == (u32)~0u ? 0 : first + 1u;
     vi->last_pos = last;
     vi->live_length = ranges->live_length_by_preg[v];
     vi->use_freq = ranges->use_freq_by_preg[v];
     vi->def_freq = ranges->def_freq_by_preg[v];
     vi->live_block_freq = ranges->live_block_freq_by_preg[v];
     vi->live_across_call_freq = ranges->live_across_call_freq_by_preg[v];
     vi->spill_cost = ranges->spill_cost_by_preg[v];
     vi->frequency = vi->spill_cost;
     if (old_frequency > vi->frequency) vi->frequency = old_frequency;
   }
   f->preg_info = info;
 }
 
 static void bits_clear(u64* bits, u32 words) {
   for (u32 i = 0; i < words; ++i) bits[i] = 0;
 }
 
 static void live_update_before(u64* live, const u64* use, const u64* def,
                                u32 words) {
   for (u32 w = 0; w < words; ++w) live[w] = (live[w] & ~def[w]) | use[w];
 }
 
 static void live_copy_block_out(Func* f, const OptLiveInfo* live_info, u32 b,
                                 u64* live, u32 words) {
   (void)f;
   bits_clear(live, words);
   if (live_info) {
     const OptBitset* out = &live_info->blocks[b].live_out;
     for (u32 w = 0; w < words && w < out->nwords; ++w) live[w] = out->words[w];
   }
 }
 
 static u32 live_copy_block_out_active(const OptLiveInfo* live_info, u32 b,
                                       u64* live, u32 words,
                                       u32 old_active_words) {
   for (u32 w = 0; w < old_active_words; ++w) live[w] = 0;
   if (!live_info) return 0;
   const OptBitset* out = &live_info->blocks[b].live_out;
   u32 active = words < out->active_words ? words : out->active_words;
   for (u32 w = 0; w < active; ++w) live[w] = out->words[w];
   return active;
 }
 
 static void opt_apply_asm_constraints_from_live(Func* f,
                                                 const OptLiveInfo* live_info) {
   int has_asm = 0;
   for (u32 b = 0; b < f->nblocks && !has_asm; ++b) {
     Block* bl = &f->blocks[b];
     for (u32 i = 0; i < bl->ninsts; ++i) {
       if ((IROp)bl->insts[i].op == IR_ASM_BLOCK) {
         has_asm = 1;
         break;
       }
     }
   }
   /* The live walk drives both inline-asm operand constraints and
    * per-instruction fixed-register clobbers (Func.inst_clobbers); run it if
    * either is present. */
   if (!has_asm && !f->inst_clobbers) return;
 
   u32 words = live_info ? live_info->words : f->opt_live_words;
   if (!words) words = bit_words(opt_reg_count(f));
   f->opt_live_words = (u16)words;
 
   u64* live = arena_zarray(f->arena, u64, words ? words : 1u);
   u64* use = arena_zarray(f->arena, u64, words ? words : 1u);
   u64* def = arena_zarray(f->arena, u64, words ? words : 1u);
   for (u32 b = 0; b < f->nblocks; ++b) {
     Block* bl = &f->blocks[b];
     live_copy_block_out(f, live_info, b, live, words);
     for (u32 ri = bl->ninsts; ri > 0; --ri) {
       u32 i = ri - 1u;
       Inst* in = &bl->insts[i];
       bits_clear(use, words);
       bits_clear(def, words);
       BitsCtx bc = {use, def};
       opt_walk_inst_operands(f, in, collect_bits, &bc);
       if ((IROp)in->op == IR_ASM_BLOCK)
         apply_asm_register_constraints(f, in, use, def, live);
       apply_machine_reg_clobbers(f, in, def, live);
       live_update_before(live, use, def, words);
     }
   }
 }
 
 static int spill_slot_compatible(Func* f, FrameSlot fs, PReg v) {
   if (fs == FRAME_SLOT_NONE || fs > f->nframe_slots) return 0;
   IRFrameSlot* s = &f->frame_slots[fs - 1u];
   u32 size = type_size_fallback(f, opt_reg_type(f, v));
   u32 align = size >= 8 ? 8 : size;
   if (s->kind != FS_SPILL) return 0;
   if (s->size < size) return 0;
   if (s->align < align) return 0;
   return 1;
 }
 
 /* Compute conflict_locs = union of used_locs[j] for j in every live range
  * point of every PReg in `root`'s coalesce group. */
 static void alloc_compute_group_conflicts(Func* f, OptAllocator* a,
                                           const OptLiveRangeSet* ranges,
                                           PReg root) {
   for (u32 w = 0; w < a->loc_words; ++w) a->conflict_locs[w] = 0;
   for (PReg v = 1; v < opt_reg_count(f); ++v) {
     if (ranges->first_range_by_preg[v] == OPT_RANGE_NONE) continue;
     if (!alloc_group_member(f, root, v)) continue;
     for (u32 ri = ranges->first_range_by_preg[v]; ri != OPT_RANGE_NONE;
          ri = ranges->ranges[ri].next) {
       const OptLiveRange* lr = &ranges->ranges[ri];
       u32 end = lr->end < a->point_count ? lr->end : a->point_count;
       for (u32 j = lr->start; j < end; ++j) {
         ++a->hard_point_visits;
         const u64* row = used_locs_row(a, j);
         for (u32 w = 0; w < a->loc_words; ++w) {
           a->conflict_locs[w] |= row[w];
           ++a->hard_word_ors;
         }
       }
     }
   }
 }
 
 /* Mark `loc_bit` as occupied at every point covered by `root`'s group's
  * live ranges. */
 static void alloc_mark_group_loc(Func* f, OptAllocator* a,
                                  const OptLiveRangeSet* ranges, PReg root,
                                  u32 loc_bit) {
   u32 w = loc_bit / 64u;
   u64 mask = 1ull << (loc_bit % 64u);
   for (PReg v = 1; v < opt_reg_count(f); ++v) {
     if (ranges->first_range_by_preg[v] == OPT_RANGE_NONE) continue;
     if (!alloc_group_member(f, root, v)) continue;
     for (u32 ri = ranges->first_range_by_preg[v]; ri != OPT_RANGE_NONE;
          ri = ranges->ranges[ri].next) {
       const OptLiveRange* lr = &ranges->ranges[ri];
       u32 end = lr->end < a->point_count ? lr->end : a->point_count;
       for (u32 j = lr->start; j < end; ++j) {
         ++a->hard_mark_points;
         used_locs_row(a, j)[w] |= mask;
       }
     }
   }
 }
 
 static void alloc_assign_group_hard(Func* f, OptAllocator* a,
                                     const OptLiveRangeSet* ranges, PReg root,
                                     Reg r) {
   u8 cls = f->preg_info[root].cls;
   u32 bit = hard_loc_bit(cls, r);
   for (PReg v = 1; v < opt_reg_count(f); ++v) {
     if (ranges->first_range_by_preg[v] == OPT_RANGE_NONE) continue;
     if (!alloc_group_member(f, root, v)) continue;
     OptPRegInfo* vi = &f->preg_info[v];
     vi->alloc_kind = OPT_ALLOC_HARD;
     vi->hard_reg = r;
     a->locs[v].kind = OPT_LOC_HARD;
     a->locs[v].cls = vi->cls;
     a->locs[v].hard_reg = r;
     a->locs[v].spill_slot = FRAME_SLOT_NONE;
   }
   alloc_mark_group_loc(f, a, ranges, root, bit);
   if (bit < a->hard_loc_bits) a->hard_open[bit] = 1;
 }
 
 static void alloc_assign_group_stack(Func* f, OptAllocator* a,
                                      const OptLiveRangeSet* ranges, PReg root) {
   /* Try to reuse an existing stack slot whose bit is clear in conflict_locs
    * and whose frame slot is compatible. The conflict_locs scratch must
    * already be populated for `root` by the caller. */
   u32 stack_idx = (u32)~0u;
   for (u32 k = 0; k < a->stack_slot_count; ++k) {
     u32 bit = a->hard_loc_bits + k;
     if (loc_bit_in_conflict(a->conflict_locs, bit)) continue;
     if (!spill_slot_compatible(f, a->stack_slots[k], root)) continue;
     stack_idx = k;
     break;
   }
   if (stack_idx == (u32)~0u) {
     FrameSlot fs = spill_slot_for(f, root);
     stack_idx = alloc_alloc_stack_slot(f, a, fs);
     /* alloc_alloc_stack_slot may have widened a->loc_words: refresh
      * conflict_locs (callers don't reuse it after this). */
   }
   FrameSlot slot = a->stack_slots[stack_idx];
   for (PReg v = 1; v < opt_reg_count(f); ++v) {
     if (ranges->first_range_by_preg[v] == OPT_RANGE_NONE) continue;
     if (!alloc_group_member(f, root, v)) continue;
     OptPRegInfo* vi = &f->preg_info[v];
     vi->spill_slot = slot;
     vi->alloc_kind = OPT_ALLOC_SPILL;
     a->locs[v].kind = OPT_LOC_STACK;
     a->locs[v].cls = vi->cls;
     a->locs[v].hard_reg = REG_NONE;
     a->locs[v].spill_slot = slot;
   }
   u32 bit = a->hard_loc_bits + stack_idx;
   u32 w = bit / 64u;
   u64 mask = 1ull << (bit % 64u);
   for (PReg v = 1; v < opt_reg_count(f); ++v) {
     if (ranges->first_range_by_preg[v] == OPT_RANGE_NONE) continue;
     if (!alloc_group_member(f, root, v)) continue;
     for (u32 ri = ranges->first_range_by_preg[v]; ri != OPT_RANGE_NONE;
          ri = ranges->ranges[ri].next) {
       const OptLiveRange* lr = &ranges->ranges[ri];
       u32 end = lr->end < a->point_count ? lr->end : a->point_count;
       for (u32 j = lr->start; j < end; ++j) {
         ++a->stack_mark_points;
         used_locs_row(a, j)[w] |= mask;
       }
     }
   }
 }
 
 static int alloc_group_conflicts_bit(const OptAllocator* a, u32 bit) {
   if (bit / 64u >= a->loc_words) return 1;
   return loc_bit_in_conflict(a->conflict_locs, bit);
 }
 
 static void opt_assign_ranges(Func* f, const OptLiveRangeSet* ranges,
                               OptAllocator* a, int allow_live_range_split) {
   (void)allow_live_range_split; /* live-range splitting deferred per
                                    doc/plan/OPTIMIZER.md; the parameter is
                                    passed through for ABI compatibility. */
   memset(a, 0, sizeof *a);
   a->point_count = ranges->point_count ? ranges->point_count : 1u;
   a->hard_loc_bits = OPT_REG_CLASSES * 32u;
   a->loc_words = alloc_loc_words_for_bits(a->hard_loc_bits);
   a->used_locs =
       arena_zarray(f->arena, u64, (u64)a->point_count * a->loc_words);
   a->conflict_locs = arena_zarray(f->arena, u64, a->loc_words);
   a->locs =
       arena_zarray(f->arena, OptLoc, opt_reg_count(f) ? opt_reg_count(f) : 1u);
   a->hard_open = arena_zarray(f->arena, u8, a->hard_loc_bits);
   a->stack_slots = NULL;
   a->stack_slot_count = 0;
   a->stack_slot_cap = 0;
 
   /* Build candidate list: every coalesce-root PReg that has live ranges. */
   u32 ncands = 0;
   for (PReg v = 1; v < opt_reg_count(f); ++v) {
     if (ranges->first_range_by_preg[v] == OPT_RANGE_NONE) continue;
     if (alloc_coalesce_root(f, v) != v) continue;
     ++ncands;
   }
   OptAllocCandidate* cands =
       arena_array(f->arena, OptAllocCandidate, ncands ? ncands : 1u);
   u32 n = 0;
   for (PReg v = 1; v < opt_reg_count(f); ++v) {
     if (ranges->first_range_by_preg[v] == OPT_RANGE_NONE) continue;
     PReg root = alloc_coalesce_root(f, v);
     if (root != v) continue;
     OptAllocGroupInfo gi;
     alloc_group_info(f, ranges, root, &gi);
     cands[n].v = v;
     cands[n].spill_cost = gi.spill_cost;
     cands[n].live_length = gi.live_length;
     cands[n].tied = gi.tied_hard_reg >= 0;
     memset(cands[n].pad, 0, sizeof cands[n].pad);
     ++n;
   }
   alloc_sort_candidates(cands, n);
 
   for (u32 i = 0; i < n; ++i) {
     PReg v = cands[i].v;
     OptAllocGroupInfo gi;
     alloc_group_info(f, ranges, v, &gi);
     OptPRegInfo* vi = &f->preg_info[v];
     u8 cls = gi.cls;
     alloc_compute_group_conflicts(f, a, ranges, v);
 
     if (gi.tied_hard_reg >= 0) {
       Reg fixed = (Reg)gi.tied_hard_reg;
       /* Machinize has already validated inline-asm hard-register pins against
        * the target's operand-register policy. Some legal pins are ABI registers
        * outside the standard allocable set (aa64 x0, rv64 a7), so the allocator
        * accepts validated physical registers here and relies on the
        * conflict/clobber checks below for placement correctness. */
       if (!hard_available(f, cls, fixed) && !phys_info_for(f, cls, fixed)) {
         SrcLoc loc = {0, 0, 0};
         compiler_panic(
             f->c, loc,
             "opt regalloc: fixed hard reg %u unavailable in class %u",
             (unsigned)fixed, (unsigned)cls);
       }
       if (fixed < 32 && (gi.forbidden_hard_regs & (1u << fixed))) {
         SrcLoc loc = {0, 0, 0};
         compiler_panic(f->c, loc,
                        "opt regalloc: fixed hard reg %u is clobbered",
                        (unsigned)fixed);
       }
       if (fixed >= 32 ||
           (gi.allowed_hard_regs &&
            (gi.allowed_hard_regs & (1u << fixed)) == 0)) {
         SrcLoc loc = {0, 0, 0};
         compiler_panic(f->c, loc,
                        "opt regalloc: fixed hard reg %u violates asm "
                        "constraint",
                        (unsigned)fixed);
       }
       u32 bit = hard_loc_bit(cls, fixed);
       if (fixed >= 32 || alloc_group_conflicts_bit(a, bit)) {
         SrcLoc loc = {0, 0, 0};
         compiler_panic(f->c, loc, "opt regalloc: conflicting fixed hard reg %u",
                        (unsigned)fixed);
       }
       alloc_assign_group_hard(f, a, ranges, v, fixed);
       continue;
     }
 
     int found = 0;
     Reg best = REG_NONE;
     u32 best_score = 0xffffffffu;
     for (u32 r = 0; r < f->opt_hard_reg_count[cls]; ++r) {
       Reg hr = f->opt_hard_regs[cls][r];
       if (hr >= 32) continue;
       if (gi.allowed_hard_regs && (gi.allowed_hard_regs & (1u << hr)) == 0)
         continue;
       if (gi.forbidden_hard_regs & (1u << hr)) continue;
       u32 bit = hard_loc_bit(cls, hr);
       if (alloc_group_conflicts_bit(a, bit)) continue;
       u32 score = hard_reg_alloc_score(f, a, vi, hr);
       if (!found || score < best_score) {
         found = 1;
         best = hr;
         best_score = score;
       }
     }
     if (gi.allowed_hard_regs) {
       for (Reg hr = 0; hr < 32; ++hr) {
         const CGPhysRegInfo* pi;
         if ((gi.allowed_hard_regs & (1u << hr)) == 0) continue;
         if (hard_available(f, cls, hr)) continue;
         if (gi.forbidden_hard_regs & (1u << hr)) continue;
         pi = phys_info_for(f, cls, hr);
         if (!pi || (pi->flags & CG_REG_RESERVED)) continue;
         u32 bit = hard_loc_bit(cls, hr);
         if (alloc_group_conflicts_bit(a, bit)) continue;
         u32 score = hard_reg_alloc_score(f, a, vi, hr);
         if (!found || score < best_score) {
           found = 1;
           best = hr;
           best_score = score;
         }
       }
     }
     /* Also consider the preferred hard reg if it's outside the standard
      * allocable set (e.g. x0 on aa64: reserved as the ABI ret reg, not in
      * aa_int_allocable). Used by apply_abi_aliasing_hints to let an IR_CALL
      * result PReg or IR_RET value PReg land directly in x0, eliding the
      * materialization move emit_call/emit_ret would otherwise emit. Only
      * short-lived PRegs benefit: a value live across a call cannot survive in
      * a caller-saved hint reg, and this is the *only* path that can reach an
      * out-of-allocable-set reg, so guard it explicitly. The +1000 caller-save
      * penalty in hard_reg_alloc_score only deflects the hint when a cheaper
      * reg is found; under high register pressure (found == 0) the fallback
      * below would otherwise take the hint reg regardless of score, parking a
      * cross-call value in x0 where it collides with the next call's result. */
     if (vi->preferred_hard_reg >= 0 &&
         !(vi->live_across_call_freq &&
           is_caller_saved(f, cls, (Reg)vi->preferred_hard_reg))) {
       Reg hint = (Reg)vi->preferred_hard_reg;
       int already_tried = 0;
       if (hint < 32 &&
           (!gi.allowed_hard_regs || (gi.allowed_hard_regs & (1u << hint)))) {
         for (u32 r = 0; r < f->opt_hard_reg_count[cls]; ++r) {
           if (f->opt_hard_regs[cls][r] == hint) {
             already_tried = 1;
             break;
           }
         }
         if (!already_tried && !(gi.forbidden_hard_regs & (1u << hint))) {
           u32 bit = hard_loc_bit(cls, hint);
           int hint_safe = !alloc_group_conflicts_bit(a, bit);
           /* The bitmap conflict can be falsely positive when an
            * already-assigned PReg ends exactly where v begins — the
            * swap-friendly pattern like `sub x0, x21, x0`, where the previous
            * call's result occupies x0 and the sub both reads it and writes
            * the new value. Fall back to a precise per-PReg interference check
            * that allows the unit-length overlap (same rule used by
            * opt_coalesce_ranges for moves). */
           if (!hint_safe) {
             int real_conflict = 0;
             for (PReg u = 1; u < opt_reg_count(f); ++u) {
               if (u == v) continue;
               const OptPRegInfo* ui = &f->preg_info[u];
               if (ui->alloc_kind != OPT_ALLOC_HARD) continue;
               if (ui->hard_reg != hint) continue;
               if (opt_ranges_overlap_kind(ranges, u, v) >= 2) {
                 real_conflict = 1;
                 break;
               }
             }
             if (!real_conflict) hint_safe = 1;
           }
           if (hint_safe) {
             u32 score = hard_reg_alloc_score(f, a, vi, hint);
             if (!found || score < best_score) {
               found = 1;
               best = hint;
               best_score = score;
             }
           }
         }
       }
     }
     if (found) {
       alloc_assign_group_hard(f, a, ranges, v, best);
     } else if (gi.allowed_hard_regs) {
       SrcLoc loc = {0, 0, 0};
       compiler_panic(f->c, loc,
                      "opt regalloc: no hard register satisfies asm constraint");
     } else {
       alloc_assign_group_stack(f, a, ranges, v);
     }
   }
 
   /* Report storage metrics in u64 words (used_locs is the only bitmap). */
   u32 total_words = a->point_count * a->loc_words;
   u32 hard_words = alloc_loc_words_for_bits(a->hard_loc_bits) * a->point_count;
   if (hard_words > total_words) hard_words = total_words;
   f->opt_alloc_hard_loc_words = hard_words;
   f->opt_alloc_stack_loc_words = total_words - hard_words;
   f->opt_alloc_stack_slots = a->stack_slot_count;
   f->opt_used_loc_words = total_words;
   f->opt_alloc_hard_point_visits = a->hard_point_visits;
   f->opt_alloc_stack_point_visits = a->stack_point_visits;
   f->opt_alloc_hard_word_ors = a->hard_word_ors;
   f->opt_alloc_stack_word_ors = a->stack_word_ors;
   f->opt_alloc_hard_mark_points = a->hard_mark_points;
   f->opt_alloc_stack_mark_points = a->stack_mark_points;
   f->preg_locs = a->locs;
 }
 
 typedef struct RewriteList {
   Inst* data;
   u32 n;
   u32 cap;
 } RewriteList;
 
 typedef enum EdgeMatKind {
   EDGE_MAT_STORE,
   EDGE_MAT_RELOAD,
 } EdgeMatKind;
 
 typedef struct EdgeMat {
   u32 pred;
   u32 succ;
   PReg v;
   Reg hard_reg;
   u8 kind;
   u8 pad[3];
 } EdgeMat;
 
 typedef struct EdgeMatPlan {
   EdgeMat* mats;
   u32 n;
   u32 cap;
 } EdgeMatPlan;
 
 typedef struct RewriteOut {
   Inst* data;
   u32 cap;
   u32 start;
 } RewriteOut;
 
 static Inst* list_push(Func* f, RewriteList* l, IROp op) {
   if (l->n == l->cap) {
     u32 ncap = l->cap ? l->cap * 2u : 16u;
     Inst* nb = arena_zarray(f->arena, Inst, ncap);
     if (l->data) memcpy(nb, l->data, sizeof(Inst) * l->n);
     l->data = nb;
     l->cap = ncap;
   }
   Inst* in = &l->data[l->n++];
   memset(in, 0, sizeof *in);
   in->op = (u16)op;
   ir_assign_inst_id(f, in);
   ++f->opt_rewrite_inserted_insts;
   return in;
 }
 
 static void list_reset(RewriteList* l) {
   if (l) l->n = 0;
 }
 
 static void out_init(Func* f, RewriteOut* out, u32 cap) {
   out->cap = cap ? cap : 16u;
   out->data = arena_zarray(f->arena, Inst, out->cap);
   out->start = out->cap;
 }
 
 static Inst* out_push_front(Func* f, RewriteOut* out, IROp op) {
   if (out->start == 0) {
     u32 n = out->cap;
     u32 ncap = out->cap ? out->cap * 2u : 16u;
     Inst* nb = arena_zarray(f->arena, Inst, ncap);
     if (out->data && n)
       memcpy(nb + (ncap - n), out->data + out->start, sizeof(Inst) * n);
     out->data = nb;
     out->cap = ncap;
     out->start = ncap - n;
   }
   Inst* in = &out->data[--out->start];
   memset(in, 0, sizeof *in);
   in->op = (u16)op;
   ir_assign_inst_id(f, in);
   return in;
 }
 
 static void out_prepend_list_reverse(Func* f, RewriteOut* out,
                                      const RewriteList* list) {
   for (u32 i = list->n; i > 0; --i) {
     Inst* dst = out_push_front(f, out, (IROp)list->data[i - 1u].op);
     *dst = list->data[i - 1u];
   }
 }
 
 static void out_prepend_inst(Func* f, RewriteOut* out, const Inst* in) {
   Inst* dst = out_push_front(f, out, (IROp)in->op);
   *dst = *in;
 }
 
 static MemAccess spill_mem(Func* f, PReg v) {
   MemAccess m;
   memset(&m, 0, sizeof m);
   m.type = opt_reg_type(f, v);
   m.size = type_size_fallback(f, opt_reg_type(f, v));
   m.align = m.size >= 8 ? 8 : m.size;
   m.alias.kind = ALIAS_LOCAL;
   m.alias.v.local_id = (i32)spill_slot_for(f, v);
   return m;
 }
 
 static Operand spill_addr(Func* f, PReg v) {
   Operand o;
   memset(&o, 0, sizeof o);
   o.kind = OPK_LOCAL;
   o.cls = RC_INT;
   o.type = opt_reg_type(f, v);
   o.v.frame_slot = spill_slot_for(f, v);
   return o;
 }
 
 static Operand hard_operand(Func* f, PReg v) {
   Operand o;
   memset(&o, 0, sizeof o);
   o.kind = OPK_REG;
   o.cls = opt_preg_loc_cls(f, v);
   o.type = opt_reg_type(f, v);
   o.v.reg = opt_preg_hard_reg(f, v);
   return o;
 }
 
 static Operand hard_operand_reg(Func* f, PReg v, Reg hard_reg) {
   Operand o;
   memset(&o, 0, sizeof o);
   o.kind = OPK_REG;
   o.cls = opt_preg_loc_cls(f, v);
   o.type = opt_reg_type(f, v);
   o.v.reg = hard_reg;
   return o;
 }
 
 static void append_store_preg(Func* f, RewriteList* out, PReg v) {
   Inst* st = list_push(f, out, IR_STORE);
   st->opnds = arena_array(f->arena, Operand, 2);
   st->opnds[0] = spill_addr(f, v);
   st->opnds[1] = hard_operand(f, v);
   st->nopnds = 2;
   st->extra.mem = spill_mem(f, v);
 }
 
 static void append_load_preg(Func* f, RewriteList* out, PReg v) {
   Inst* ld = list_push(f, out, IR_LOAD);
   ld->opnds = arena_array(f->arena, Operand, 2);
   ld->opnds[0] = hard_operand(f, v);
   ld->opnds[1] = spill_addr(f, v);
   ld->nopnds = 2;
   ld->extra.mem = spill_mem(f, v);
 }
 
 static void append_store_preg_hard(Func* f, RewriteList* out, PReg v,
                                    Reg hard_reg) {
   Inst* st = list_push(f, out, IR_STORE);
   st->opnds = arena_array(f->arena, Operand, 2);
   st->opnds[0] = spill_addr(f, v);
   st->opnds[1] = hard_operand_reg(f, v, hard_reg);
   st->nopnds = 2;
   st->extra.mem = spill_mem(f, v);
 }
 
 static void append_load_preg_hard(Func* f, RewriteList* out, PReg v,
                                   Reg hard_reg) {
   Inst* ld = list_push(f, out, IR_LOAD);
   ld->opnds = arena_array(f->arena, Operand, 2);
   ld->opnds[0] = hard_operand_reg(f, v, hard_reg);
   ld->opnds[1] = spill_addr(f, v);
   ld->nopnds = 2;
   ld->extra.mem = spill_mem(f, v);
 }
 
 static Reg scratch_for(Func* f, u8 cls, u32* next) {
   u32 n = f->opt_scratch_reg_count[cls];
   if (!n) return REG_NONE;
   Reg r = f->opt_scratch_regs[cls][*next % n];
   ++*next;
   return r;
 }
 
 typedef struct RewriteCtx {
   RewriteList* before;
   RewriteList* after;
   u32 next_scratch[OPT_REG_CLASSES];
   u32 raw_point;
 } RewriteCtx;
 
 static const OptAllocSegment* split_segment_at(Func* f, PReg v, u32 raw_point) {
   if (!f->opt_first_segment_by_preg || v >= opt_reg_count(f)) return NULL;
   for (u32 si = f->opt_first_segment_by_preg[v]; si != OPT_RANGE_NONE;
        si = f->opt_alloc_segments[si].next) {
     const OptAllocSegment* s = &f->opt_alloc_segments[si];
     if (s->start <= raw_point && raw_point < s->end) return s;
   }
   return NULL;
 }
 
 static void rewrite_one_operand(Func* f, Inst* owner, Operand* op, int is_def,
                                 void* arg) {
   RewriteCtx* c = (RewriteCtx*)arg;
   if (op->kind != OPK_REG) return;
   PReg v = (PReg)op->v.reg;
   if (v == PREG_NONE || v == 0 || v >= opt_reg_count(f)) return;
   u8 alloc_kind = opt_preg_alloc_kind(f, v);
   if (alloc_kind == OPT_ALLOC_HARD) {
     op->v.reg = opt_preg_hard_reg(f, v);
     return;
   }
   if (alloc_kind == OPT_ALLOC_SPLIT) {
     const OptAllocSegment* seg = split_segment_at(f, v, c->raw_point);
     if (seg && seg->loc_kind == OPT_LOC_HARD) {
       op->v.reg = seg->hard_reg;
       return;
     }
   } else if (alloc_kind != OPT_ALLOC_SPILL) {
     return;
   }
   u8 cls = opt_preg_loc_cls(f, v);
   Reg scratch = scratch_for(f, cls, &c->next_scratch[cls]);
   if (scratch == (Reg)REG_NONE) {
     SrcLoc loc = owner ? owner->loc : (SrcLoc){0, 0, 0};
     compiler_panic(f->c, loc,
                    "opt rewrite: no scratch register for spilled class %u",
                    (unsigned)cls);
   }
   op->v.reg = scratch;
   if (!is_def) {
     Inst* ld = list_push(f, c->before, IR_LOAD);
     ld->opnds = arena_array(f->arena, Operand, 2);
     ld->opnds[0] = *op;
     ld->opnds[1] = spill_addr(f, v);
     ld->nopnds = 2;
     ld->extra.mem = spill_mem(f, v);
   } else {
     Inst* st = list_push(f, c->after, IR_STORE);
     st->opnds = arena_array(f->arena, Operand, 2);
     st->opnds[0] = spill_addr(f, v);
     st->opnds[1] = *op;
     st->nopnds = 2;
     st->extra.mem = spill_mem(f, v);
   }
 }
 
 static void rewrite_call_arg_operand(Func* f, Operand* op) {
   if (!op || op->kind != OPK_REG) return;
   PReg v = (PReg)op->v.reg;
   if (v == PREG_NONE || v == 0 || v >= opt_reg_count(f)) return;
   u8 alloc_kind = opt_preg_alloc_kind(f, v);
   if (alloc_kind == OPT_ALLOC_HARD) {
     op->v.reg = opt_preg_hard_reg(f, v);
   } else if (alloc_kind == OPT_ALLOC_SPILL || alloc_kind == OPT_ALLOC_SPLIT) {
     *op = spill_addr(f, v);
   }
 }
 
 static void rewrite_store_value_operand(Func* f, Inst* owner, Operand* op,
                                         RewriteCtx* ctx) {
   PReg v;
   u8 alloc_kind;
   const OptAllocSegment* seg;
   if (!op || op->kind != OPK_REG) return;
   v = (PReg)op->v.reg;
   if (v == PREG_NONE || v == 0 || v >= opt_reg_count(f)) return;
   alloc_kind = opt_preg_alloc_kind(f, v);
   if (alloc_kind == OPT_ALLOC_HARD) {
     op->v.reg = opt_preg_hard_reg(f, v);
     return;
   }
   if (alloc_kind == OPT_ALLOC_SPLIT) {
     seg = split_segment_at(f, v, ctx->raw_point);
     if (seg && seg->loc_kind == OPT_LOC_HARD) {
       op->v.reg = seg->hard_reg;
       return;
     }
     *op = spill_addr(f, v);
     return;
   }
   if (alloc_kind == OPT_ALLOC_SPILL) {
     *op = spill_addr(f, v);
     return;
   }
   rewrite_one_operand(f, owner, op, 0, ctx);
 }
 
 static void rewrite_call_arg_indirect_base(Func* f, Inst* owner, Operand* op,
                                            RewriteCtx* ctx) {
   if (!op || op->kind != OPK_INDIRECT) return;
   Operand base = *op;
   base.kind = OPK_REG;
   base.cls = RC_INT;
   base.v.reg = op->v.ind.base;
   rewrite_one_operand(f, owner, &base, 0, ctx);
   op->v.ind.base = base.v.reg;
 }
 
 static void rewrite_call_arg_value(Func* f, Inst* owner, CGABIValue* v,
                                    RewriteCtx* ctx) {
   if (!v) return;
   rewrite_call_arg_indirect_base(f, owner, &v->storage, ctx);
   rewrite_call_arg_operand(f, &v->storage);
   for (u32 i = 0; i < v->nparts; ++i) {
     Operand* op = (Operand*)&v->parts[i].op;
     rewrite_call_arg_indirect_base(f, owner, op, ctx);
     rewrite_call_arg_operand(f, op);
   }
 }
 
 typedef struct RewriteCallSaveCtx {
   Func* f;
   RewriteList* out;
   const InstRefs* refs;
   const Inst* call;
   int emit_restore;
 } RewriteCallSaveCtx;
 
 static void rewrite_call_save_one(PReg v, void* arg) {
   RewriteCallSaveCtx* c = (RewriteCallSaveCtx*)arg;
   Func* f = c->f;
   if (v == PREG_NONE || v == 0 || v >= opt_reg_count(f)) return;
   if (c->refs && refs_has_def(c->refs, v)) return;
   if (opt_preg_alloc_kind(f, v) != OPT_ALLOC_HARD) return;
   u8 cls = opt_preg_loc_cls(f, v);
   Reg hr = opt_preg_hard_reg(f, v);
   if (cls >= OPT_REG_CLASSES || hr >= 32u) return;
   if ((opt_call_clobber_mask_for(f, c->call, cls) & (1u << hr)) == 0) return;
   if (c->emit_restore)
     append_load_preg(f, c->out, v);
   else
     append_store_preg(f, c->out, v);
 }
 
 static void append_live_call_saves(Func* f, RewriteList* out, const Inst* call,
                                    const u64* live_after, u32 live_active_words,
                                    const InstRefs* refs,
                                    const PReg* call_save_pregs,
                                    u32 ncall_save_pregs, int emit_restore) {
   RewriteCallSaveCtx ctx = {f, out, refs, call, emit_restore};
   f->opt_rewrite_live_words_touched += ncall_save_pregs;
   for (u32 i = 0; i < ncall_save_pregs; ++i) {
     PReg v = call_save_pregs[i];
     u32 w = v / 64u;
     if (w >= live_active_words) continue;
     if (!bit_has(live_after, v)) continue;
     rewrite_call_save_one(v, &ctx);
   }
 }
 
 static PReg* rewrite_collect_call_save_pregs(Func* f, u32* count_out) {
   u32 n = 0;
   for (PReg v = 1; v < opt_reg_count(f); ++v) {
     OptPRegInfo* vi = &f->preg_info[v];
     if (opt_preg_alloc_kind(f, v) != OPT_ALLOC_HARD) continue;
     if (!vi->live_across_call_freq) continue;
     ++n;
   }
   PReg* pregs = arena_array(f->arena, PReg, n ? n : 1u);
   u32 w = 0;
   for (PReg v = 1; v < opt_reg_count(f); ++v) {
     OptPRegInfo* vi = &f->preg_info[v];
     if (opt_preg_alloc_kind(f, v) != OPT_ALLOC_HARD) continue;
     if (!vi->live_across_call_freq) continue;
     pregs[w++] = v;
   }
   *count_out = n;
   return pregs;
 }
 
 static void append_split_reloads_at(Func* f, RewriteList* out, u32 block,
                                     u32 raw_point) {
   if (!f->opt_first_segment_by_preg) return;
   for (PReg v = 1; v < opt_reg_count(f); ++v) {
     if (opt_preg_alloc_kind(f, v) != OPT_ALLOC_SPLIT) continue;
     for (u32 si = f->opt_first_segment_by_preg[v]; si != OPT_RANGE_NONE;
          si = f->opt_alloc_segments[si].next) {
       const OptAllocSegment* s = &f->opt_alloc_segments[si];
       if (s->block == block && s->start == raw_point && s->reload_at_start &&
           s->loc_kind == OPT_LOC_HARD) {
         append_load_preg_hard(f, out, v, s->hard_reg);
       }
     }
   }
 }
 
 static void append_split_stores_at(Func* f, RewriteList* out, u32 block,
                                    u32 raw_point) {
   if (!f->opt_first_segment_by_preg) return;
   for (PReg v = 1; v < opt_reg_count(f); ++v) {
     if (opt_preg_alloc_kind(f, v) != OPT_ALLOC_SPLIT) continue;
     for (u32 si = f->opt_first_segment_by_preg[v]; si != OPT_RANGE_NONE;
          si = f->opt_alloc_segments[si].next) {
       const OptAllocSegment* s = &f->opt_alloc_segments[si];
       if (s->block == block && s->end == raw_point && s->store_at_end &&
           s->loc_kind == OPT_LOC_HARD) {
         append_store_preg_hard(f, out, v, s->hard_reg);
       }
     }
   }
 }
 
 static void edge_mat_push(Func* f, EdgeMatPlan* plan, u32 pred, u32 succ,
                           PReg v, Reg hard_reg, EdgeMatKind kind) {
   if (!plan || pred >= f->nblocks || succ >= f->nblocks) return;
   if (hard_reg == (Reg)REG_NONE) return;
   if (plan->n == plan->cap) {
     u32 ncap = plan->cap ? plan->cap * 2u : 16u;
     EdgeMat* nm = arena_array(f->arena, EdgeMat, ncap);
     if (plan->mats) memcpy(nm, plan->mats, sizeof(plan->mats[0]) * plan->n);
     plan->mats = nm;
     plan->cap = ncap;
   }
   EdgeMat* m = &plan->mats[plan->n++];
   memset(m, 0, sizeof *m);
   m->pred = pred;
   m->succ = succ;
   m->v = v;
   m->hard_reg = hard_reg;
   m->kind = (u8)kind;
 }
 
 static void prepare_split_edge_materialization(Func* f, EdgeMatPlan* plan) {
   if (!f || !plan || !f->opt_first_segment_by_preg) return;
   u32* block_base = arena_array(f->arena, u32, f->nblocks ? f->nblocks : 1u);
   u32 raw = 0;
   for (u32 b = 0; b < f->nblocks; ++b) {
     block_base[b] = raw;
     raw += f->blocks[b].ninsts ? f->blocks[b].ninsts : 1u;
   }
 
   for (PReg v = 1; v < opt_reg_count(f); ++v) {
     if (opt_preg_alloc_kind(f, v) != OPT_ALLOC_SPLIT) continue;
     for (u32 si = f->opt_first_segment_by_preg[v]; si != OPT_RANGE_NONE;
          si = f->opt_alloc_segments[si].next) {
       OptAllocSegment* s = &f->opt_alloc_segments[si];
       if (s->loc_kind != OPT_LOC_HARD || s->block >= f->nblocks) continue;
       Block* bl = &f->blocks[s->block];
       u32 block_start = block_base[s->block];
       u32 block_end = block_start + (bl->ninsts ? bl->ninsts : 1u);
       if (s->reload_at_start && s->start == block_start && bl->npreds) {
         for (u32 p = 0; p < bl->npreds; ++p)
           edge_mat_push(f, plan, bl->preds[p], s->block, v, s->hard_reg,
                         EDGE_MAT_RELOAD);
         s->reload_at_start = 0;
       }
       if (s->store_at_end && s->end == block_end && bl->nsucc) {
         for (u32 e = 0; e < bl->nsucc; ++e)
           edge_mat_push(f, plan, s->block, bl->succ[e], v, s->hard_reg,
                         EDGE_MAT_STORE);
         s->store_at_end = 0;
       }
     }
   }
 }
 
 static int lower_is_terminator(const Inst* in) {
   if (!in) return 0;
   switch ((IROp)in->op) {
     case IR_BR:
     case IR_CONDBR:
     case IR_CMP_BRANCH:
     case IR_SWITCH:
     case IR_INDIRECT_BRANCH:
     case IR_RET:
     case IR_UNREACHABLE:
     case IR_BREAK_TO:
     case IR_CONTINUE_TO:
       return 1;
     case IR_INTRINSIC: {
       IRIntrinAux* aux = (IRIntrinAux*)in->extra.aux;
       return aux && (aux->kind == INTRIN_LONGJMP || aux->kind == INTRIN_TRAP);
     }
     default:
       return 0;
   }
 }
 
 static void block_insert_list(Func* f, u32 block, const RewriteList* list) {
   if (!f || block >= f->nblocks || !list || !list->n) return;
   Block* bl = &f->blocks[block];
   u32 term = bl->ninsts && lower_is_terminator(&bl->insts[bl->ninsts - 1u]);
   u32 insert_at = bl->ninsts - term;
   Inst* insts = arena_zarray(f->arena, Inst, bl->ninsts + list->n);
   if (insert_at) memcpy(insts, bl->insts, sizeof(Inst) * insert_at);
   memcpy(insts + insert_at, list->data, sizeof(Inst) * list->n);
   if (bl->ninsts > insert_at) {
     memcpy(insts + insert_at + list->n, bl->insts + insert_at,
            sizeof(Inst) * (bl->ninsts - insert_at));
   }
   bl->insts = insts;
   bl->ninsts += list->n;
   bl->cap = bl->ninsts;
 }
 
 typedef struct EdgePlace {
   u32 pred;
   u32 succ;
   u32 block;
 } EdgePlace;
 
 static u32 edge_place_for(Func* f, EdgePlace** places, u32* nplaces, u32* cap,
                           u32 pred, u32 succ) {
   for (u32 i = 0; i < *nplaces; ++i)
     if ((*places)[i].pred == pred && (*places)[i].succ == succ)
       return (*places)[i].block;
   if (*nplaces == *cap) {
     u32 ncap = *cap ? *cap * 2u : 16u;
     EdgePlace* np = arena_array(f->arena, EdgePlace, ncap);
     if (*places) memcpy(np, *places, sizeof((*places)[0]) * *nplaces);
     *places = np;
     *cap = ncap;
   }
   u32 block = opt_split_edge(f, pred, succ);
   EdgePlace* p = &(*places)[(*nplaces)++];
   p->pred = pred;
   p->succ = succ;
   p->block = block;
   return block;
 }
 
 static void apply_split_edge_materialization(Func* f, const EdgeMatPlan* plan) {
   if (!f || !plan || !plan->n) return;
   EdgePlace* places = NULL;
   u32 nplaces = 0;
   u32 place_cap = 0;
   for (u32 i = 0; i < plan->n; ++i) {
     const EdgeMat* m = &plan->mats[i];
     if (m->pred < f->nblocks && m->succ < f->nblocks)
       (void)edge_place_for(f, &places, &nplaces, &place_cap, m->pred, m->succ);
   }
   for (u32 pass = 0; pass < 2; ++pass) {
     EdgeMatKind kind = pass == 0 ? EDGE_MAT_STORE : EDGE_MAT_RELOAD;
     for (u32 i = 0; i < plan->n; ++i) {
       const EdgeMat* m = &plan->mats[i];
       if ((EdgeMatKind)m->kind != kind) continue;
       u32 block =
           edge_place_for(f, &places, &nplaces, &place_cap, m->pred, m->succ);
       RewriteList list;
       memset(&list, 0, sizeof list);
       if (kind == EDGE_MAT_STORE)
         append_store_preg_hard(f, &list, m->v, m->hard_reg);
       else
         append_load_preg_hard(f, &list, m->v, m->hard_reg);
       block_insert_list(f, block, &list);
     }
   }
   opt_build_cfg(f);
 }
 
 static void rewrite_func(Func* f, const OptLiveInfo* live_info) {
   u32 words = live_info ? live_info->words : f->opt_live_words;
   if (!words) words = bit_words(opt_reg_count(f));
   f->opt_live_words = (u16)words;
   f->opt_rewrite_inserted_insts = 0;
   f->opt_rewrite_live_words_touched = 0;
 
   u64* live = arena_zarray(f->arena, u64, words ? words : 1u);
   u32 ncall_save_pregs = 0;
   PReg* call_save_pregs = rewrite_collect_call_save_pregs(f, &ncall_save_pregs);
   InstRefs refs;
   memset(&refs, 0, sizeof refs);
   u32 live_active_words = 0;
   u32* block_base = arena_array(f->arena, u32, f->nblocks ? f->nblocks : 1u);
   u32 raw = 0;
   for (u32 b = 0; b < f->nblocks; ++b) {
     block_base[b] = raw;
     raw += f->blocks[b].ninsts ? f->blocks[b].ninsts : 1u;
   }
   for (u32 b = 0; b < f->nblocks; ++b) {
     Block* bl = &f->blocks[b];
     RewriteOut out;
     out_init(f, &out, bl->ninsts + 16u);
     RewriteList before, after, call_saves, call_restores;
     memset(&before, 0, sizeof before);
     memset(&after, 0, sizeof after);
     memset(&call_saves, 0, sizeof call_saves);
     memset(&call_restores, 0, sizeof call_restores);
     live_active_words = live_copy_block_out_active(live_info, b, live, words,
                                                    live_active_words);
     f->opt_rewrite_live_words_touched += live_active_words;
 
     for (u32 ri = bl->ninsts; ri > 0; --ri) {
       u32 i = ri - 1u;
       Inst in = bl->insts[i];
       if ((IROp)in.op == IR_PARAM_DECL) {
         out_prepend_inst(f, &out, &in);
         continue;
       }
       refs_reset(&refs);
       opt_walk_inst_operands(f, &in, refs_collect, &refs);
       list_reset(&before);
       list_reset(&after);
       list_reset(&call_saves);
       list_reset(&call_restores);
       RewriteCtx ctx;
       memset(&ctx, 0, sizeof ctx);
       ctx.before = &before;
       ctx.after = &after;
       ctx.raw_point = block_base[b] + i;
       if ((IROp)in.op == IR_CALL) {
         IRCallAux* aux = (IRCallAux*)in.extra.aux;
         if (aux) {
           if (aux->use_plan_replay) {
             rewrite_one_operand(f, &in, &aux->plan.callee, 0, &ctx);
             for (u32 k = 0; k < aux->plan.nargs; ++k) {
               rewrite_call_arg_indirect_base(f, &in, &aux->plan.args[k].src,
                                              &ctx);
               rewrite_call_arg_operand(f, &aux->plan.args[k].src);
             }
             for (u32 k = 0; k < aux->plan.nrets; ++k)
               rewrite_one_operand(f, &in, &aux->plan.rets[k].dst, 1, &ctx);
           } else {
             rewrite_one_operand(f, &in, &aux->desc.callee, 0, &ctx);
             for (u32 k = 0; k < aux->desc.nargs; ++k)
               rewrite_call_arg_value(f, &in, (CGABIValue*)&aux->desc.args[k],
                                      &ctx);
             opt_walk_abivalue(f, &in, &aux->desc.ret, 1, rewrite_one_operand,
                               &ctx);
           }
         }
       } else if ((IROp)in.op == IR_STORE && in.nopnds >= 2) {
         opt_walk_operand(f, &in, &in.opnds[0], 0, rewrite_one_operand, &ctx);
         rewrite_store_value_operand(f, &in, &in.opnds[1], &ctx);
       } else {
         opt_walk_inst_operands(f, &in, rewrite_one_operand, &ctx);
       }
       if ((IROp)in.op == IR_CALL) {
         append_live_call_saves(f, &call_saves, &in, live, live_active_words,
                                &refs, call_save_pregs, ncall_save_pregs, 0);
         append_live_call_saves(f, &call_restores, &in, live, live_active_words,
                                &refs, call_save_pregs, ncall_save_pregs, 1);
       }
       append_split_reloads_at(f, &before, b, block_base[b] + i);
       append_split_stores_at(f, &after, b, block_base[b] + i + 1u);
 
       out_prepend_list_reverse(f, &out, &after);
       out_prepend_list_reverse(f, &out, &call_restores);
       out_prepend_inst(f, &out, &in);
       out_prepend_list_reverse(f, &out, &call_saves);
       out_prepend_list_reverse(f, &out, &before);
       live_active_words =
           live_update_refs_before_active(live, live_active_words, words, &refs);
       f->opt_rewrite_live_words_touched += refs.nuses + refs.ndefs;
     }
     bl->insts = out.data + out.start;
     bl->ninsts = out.cap - out.start;
     bl->cap = bl->ninsts;
   }
   f->opt_rewritten = 1;
 }
 
 void opt_lower_to_mir(Func* f, const OptLiveInfo* live_info) {
   if (!f) return;
   Func phys = *f;
   phys.blocks = f->blocks;
   phys.opt_rewritten = 0;
   phys.mir = NULL;
 
   rewrite_func(&phys, live_info);
 
   MFunc* m = arena_zarray(f->arena, MFunc, 1);
   m->blocks = phys.blocks;
   m->nblocks = phys.nblocks;
   m->entry = phys.entry;
   m->emit_order_n = phys.emit_order_n;
   m->emit_order_cap = phys.emit_order_n;
   m->emit_order =
       arena_array(f->arena, u32, phys.emit_order_n ? phys.emit_order_n : 1u);
   if (phys.emit_order_n)
     memcpy(m->emit_order, phys.emit_order, sizeof(u32) * phys.emit_order_n);
 
   f->mir = m;
   f->blocks = phys.blocks;
   f->nblocks = phys.nblocks;
   f->blocks_cap = phys.blocks_cap;
   f->frame_slots = phys.frame_slots;
   f->nframe_slots = phys.nframe_slots;
   f->frame_slots_cap = phys.frame_slots_cap;
   f->next_inst_id = phys.next_inst_id;
   f->opt_rewrite_inserted_insts = phys.opt_rewrite_inserted_insts;
   f->opt_rewrite_live_words_touched = phys.opt_rewrite_live_words_touched;
 }
 
 static void rewrite_dump_sb(Writer* w, const StrBuf* sb) {
   kit_writer_write(w, strbuf_cstr(sb), strbuf_len(sb));
 }
 
 void opt_rewrite_dump(Func* f, Writer* w) {
   if (!f || !w) return;
   char buf[96];
   StrBuf sb;
   strbuf_init(&sb, buf, sizeof buf);
   strbuf_puts(&sb, "rewrite blocks=");
   strbuf_put_u64(&sb, (u64)(unsigned)f->nblocks);
   strbuf_puts(&sb, " pregs=");
   strbuf_put_u64(&sb, (u64)(unsigned)opt_reg_count(f));
   strbuf_puts(&sb, " rewritten=");
   strbuf_put_u64(&sb, (u64)(unsigned)f->opt_rewritten);
   strbuf_putc(&sb, '\n');
   rewrite_dump_sb(w, &sb);
   for (u32 b = 0; b < f->nblocks; ++b) {
     Block* bl = &f->blocks[b];
     strbuf_reset(&sb);
     strbuf_puts(&sb, "block ");
     strbuf_put_u64(&sb, (u64)(unsigned)b);
     strbuf_puts(&sb, " insts=");
     strbuf_put_u64(&sb, (u64)(unsigned)bl->ninsts);
     strbuf_putc(&sb, '\n');
     rewrite_dump_sb(w, &sb);
     for (u32 i = 0; i < bl->ninsts; ++i) {
       Inst* in = &bl->insts[i];
       strbuf_reset(&sb);
       strbuf_puts(&sb, "  ");
       strbuf_put_u64(&sb, (u64)(unsigned)i);
       strbuf_puts(&sb, " op=");
       strbuf_put_u64(&sb, (u64)(unsigned)in->op);
       strbuf_puts(&sb, " operands=");
       strbuf_put_u64(&sb, (u64)(unsigned)in->nopnds);
       strbuf_putc(&sb, '\n');
       rewrite_dump_sb(w, &sb);
     }
   }
 }
 
 static int all_defs_dead(Func* f, Inst* in, u64* live) {
   if (in->def != 0 && in->def < opt_reg_count(f) && bit_has(live, in->def))
     return 0;
   for (u32 i = 0; i < in->ndefs; ++i) {
     PReg r = (PReg)in->defs[i];
     if (r != 0 && r < opt_reg_count(f) && bit_has(live, r)) return 0;
   }
   return 1;
 }
 
 void opt_dead_def_elim_with_live(Func* f, const OptLiveInfo* live_info) {
   u32 words = live_info ? live_info->words : f->opt_live_words;
   if (!words) words = bit_words(opt_reg_count(f));
   f->opt_dde_live_words_touched = 0;
   InstRefs refs;
   memset(&refs, 0, sizeof refs);
   for (u32 b = 0; b < f->nblocks; ++b) {
     Block* bl = &f->blocks[b];
     u64* live = arena_zarray(f->arena, u64, words ? words : 1u);
     f->opt_dde_live_words_touched += words;
     live_copy_block_out(f, live_info, b, live, words);
 
     Inst* new_insts = arena_array(f->arena, Inst, bl->ninsts);
     u32 w = 0;
     for (u32 ri = bl->ninsts; ri > 0; --ri) {
       u32 i = ri - 1u;
       Inst* in = &bl->insts[i];
       if (!opt_inst_has_side_effect(f, in) && all_defs_dead(f, in, live)) {
         continue;
       }
       new_insts[w++] = *in;
 
       refs_reset(&refs);
       opt_walk_inst_operands(f, in, refs_collect, &refs);
       live_update_refs_before(live, &refs);
       f->opt_dde_live_words_touched += refs.nuses + refs.ndefs;
     }
 
     for (u32 i = 0; i < w / 2; ++i) {
       Inst tmp = new_insts[i];
       new_insts[i] = new_insts[w - 1 - i];
       new_insts[w - 1 - i] = tmp;
     }
 
     bl->insts = new_insts;
     bl->ninsts = w;
     bl->cap = w;
   }
 }
 
 void opt_dead_def_elim(Func* f) {
   OptLiveInfo live;
   opt_live_blocks(f, &live);
   opt_dead_def_elim_with_live(f, &live);
 }
 
 /* Collect the def and use PRegs of one instruction (operands + aux fan-out). */
 typedef struct AllocVerifyRefs {
   PReg defs[256];
   u32 ndefs;
   PReg uses[256];
   u32 nuses;
 } AllocVerifyRefs;
 
 static void alloc_verify_collect(Func* f, Inst* in, Operand* op, int is_def,
                                  void* ctx) {
   (void)in;
   AllocVerifyRefs* r = (AllocVerifyRefs*)ctx;
   PReg p;
   if (!op || op->kind != OPK_REG) return;
   p = (PReg)op->v.reg;
   if (p == 0 || p >= opt_reg_count(f)) return;
   if (is_def) {
     if (r->ndefs < 256u) r->defs[r->ndefs++] = p;
   } else {
     if (r->nuses < 256u) r->uses[r->nuses++] = p;
   }
 }
 
 /* Post-allocation interference verifier for the O1 (no-coalesce) path. Since
  * O1 never coalesces, two *distinct* PRegs that are live simultaneously must
  * never share a hard register. Re-derive per-instruction liveness from the
  * block live-out sets and panic if any definition collides with a different
  * live value in the same hard reg — turning a silent miscompile into a hard
  * error. */
 static void opt_verify_alloc(Func* f, const OptLiveInfo* live) {
   u32 nregs = opt_reg_count(f);
   u8* cur;
   if (nregs <= 1u || !live) return;
   /* No "left in incoming reg" pre-check: the hint path's
    * opt_ranges_overlap_kind precision check already permits the unit-overlap
    * between a param PReg and its own incoming reg (= "no entry move"), and
    * the standard allocator's bitmap rejects every other overlap. The
    * per-instruction interference scan below is the residual safety net. */
   cur = arena_array(f->arena, u8, nregs);
   for (u32 b = 0; b < f->nblocks; ++b) {
     Block* bl = &f->blocks[b];
     const OptBlockLive* lb = &live->blocks[b];
     memset(cur, 0, nregs);
     for (PReg p = 1; p < nregs; ++p)
       if (opt_bitset_has(&lb->live_out, p)) cur[p] = 1u;
     for (u32 ri = bl->ninsts; ri > 0; --ri) {
       Inst* in = &bl->insts[ri - 1u];
       AllocVerifyRefs refs;
       refs.ndefs = 0;
       refs.nuses = 0;
       opt_walk_inst_operands(f, in, alloc_verify_collect, &refs);
       for (u32 k = 0; k < refs.ndefs; ++k) {
         PReg d = refs.defs[k];
         u8 d_kind = opt_preg_alloc_kind(f, d);
         if (d_kind != OPT_ALLOC_HARD && d_kind != OPT_ALLOC_SPILL) continue;
         for (PReg p = 1; p < nregs; ++p) {
           u8 p_kind;
           if (!cur[p] || p == d) continue;
           p_kind = opt_preg_alloc_kind(f, p);
           if (p_kind == OPT_ALLOC_HARD && d_kind == OPT_ALLOC_HARD &&
               opt_preg_loc_cls(f, p) == opt_preg_loc_cls(f, d) &&
               opt_preg_hard_reg(f, p) == opt_preg_hard_reg(f, d)) {
             SrcLoc loc = {0, 0, 0};
             compiler_panic(f->c, loc,
                            "opt regalloc: O1 interference — pregs %u and %u "
                            "share cls%u reg%u, both live at block %u inst %u "
                            "(op %u)",
                            (unsigned)d, (unsigned)p,
                            (unsigned)opt_preg_loc_cls(f, d),
                            (unsigned)opt_preg_hard_reg(f, d), b, ri - 1u,
                            (unsigned)in->op);
           }
           if (p_kind == OPT_ALLOC_SPILL && d_kind == OPT_ALLOC_SPILL &&
               opt_preg_spill_slot(f, p) == opt_preg_spill_slot(f, d)) {
             SrcLoc loc = {0, 0, 0};
             compiler_panic(f->c, loc,
                            "opt regalloc: O1 interference — pregs %u and %u "
                            "share spill slot %u, both live at block %u inst %u "
                            "(op %u)",
                            (unsigned)d, (unsigned)p,
                            (unsigned)opt_preg_spill_slot(f, d), b, ri - 1u,
                            (unsigned)in->op);
           }
         }
       }
       for (u32 k = 0; k < refs.ndefs; ++k) cur[refs.defs[k]] = 0u;
       for (u32 k = 0; k < refs.nuses; ++k) cur[refs.uses[k]] = 1u;
     }
   }
 }
 
 static void opt_regalloc_place(Func* f, int allow_live_range_split,
                                OptLiveInfo* live_out) {
   metrics_scope_begin(f->c, "opt.live_ranges.regalloc");
   OptLiveInfo live;
   opt_live_blocks(f, &live);
   OptLiveRangeSet ranges;
   opt_live_ranges_build(f, &live, &ranges);
   opt_init_preg_info_from_ranges(f, &ranges);
   opt_apply_asm_constraints_from_live(f, &live);
   apply_abi_aliasing_hints(f);
   /* The ABI hint passes clear forbid bits to steer values toward ABI registers
    * (ret reg, arg regs). Restore the hard machine-clobber forbids afterward so
    * no allocation path (normal, preferred-reg, or two-address group) can place
    * a value in a register that an instruction live across it destroys. */
   if (f->preg_info)
     for (PReg v = 1; v < opt_reg_count(f); ++v)
       f->preg_info[v].forbidden_hard_regs |=
           f->preg_info[v].clobbered_hard_regs;
   /* MIR coalesces only at -O2 (mir-gen.c:9431); match that here. At O1 the
    * point-bitmap allocator emits copies through the natural conflict-free
    * path. IRF_NO_COALESCE protects SSA edge copies inserted at O2. */
   if (allow_live_range_split) opt_coalesce_ranges(f, &ranges);
   metrics_count(f->c, "opt.live_words", f->opt_live_words);
   metrics_count(f->c, "opt.ranges", ranges.nranges);
   metrics_count(f->c, "opt.range_points", ranges.point_count);
   metrics_count(f->c, "opt.range_raw_points", ranges.raw_point_count);
   metrics_count(f->c, "opt.range_max_per_preg", ranges.max_ranges_per_preg);
   metrics_count(f->c, "opt.range_max_length", ranges.max_live_length);
   metrics_count(f->c, "opt.range_whole_block_spans", ranges.whole_block_spans);
   metrics_count(f->c, "opt.live.bitset_words_touched",
                 live.bitset_words_touched);
   metrics_count(f->c, "opt.live.dataflow_iterations", live.dataflow_iterations);
   metrics_count(f->c, "opt.live.dataflow_block_visits",
                 live.dataflow_block_visits);
   metrics_count(f->c, "opt.range.point_visits", ranges.range_point_visits);
   metrics_count(f->c, "opt.range.preg_scans", ranges.preg_scans);
   metrics_count(f->c, "opt.range.live_words_touched",
                 ranges.live_words_touched);
   metrics_count(f->c, "opt.conflict_bytes", 0);
   metrics_scope_end(f->c, "opt.live_ranges.regalloc");
 
   OptAllocator alloc;
   opt_assign_ranges(f, &ranges, &alloc, allow_live_range_split);
   if (!allow_live_range_split) opt_verify_alloc(f, &live);
   if (live_out) *live_out = live;
 }
 
 void opt_regalloc_locations(Func* f, int allow_live_range_split,
                             OptLiveInfo* live_out) {
   opt_regalloc_place(f, allow_live_range_split, live_out);
 }
 
 void opt_regalloc(Func* f, int allow_live_range_split) {
   OptLiveInfo live;
   opt_regalloc_place(f, allow_live_range_split, &live);
   EdgeMatPlan edge_mats;
   memset(&edge_mats, 0, sizeof edge_mats);
   if (allow_live_range_split) prepare_split_edge_materialization(f, &edge_mats);
   rewrite_func(f, &live);
   apply_split_edge_materialization(f, &edge_mats);
 }