kit

native.c (168825B)

---

 /* src/arch/rv64/native.c — RISC-V (RV64GC, LP64D) NativeTarget implementation.
  *
  * Mirrors the aa64 reference (src/arch/aa64/native.c): a physical-emission
  * NativeTarget driven at -O0 by the shared NativeDirectTarget and at -O1+ by
  * the optimizer emit path. ABI decisions go through the abi/ interface; this
  * file owns only ISA emission and the RV64 frame layout.
  *
  * Frame model (single, top-record): s0 (x8) is the frame pointer anchored at
  * the saved s0/ra pair; slots live below s0 at positive byte offsets `off`
  * (address = s0 - off); outgoing args sit at the bottom of the frame (sp+0..).
  *   frame_size  = align16(16 + cum_off + max_outgoing + va_save_sz)
  *   fp_pair_off = frame_size - 16 - va_save_sz   (saved pair, sp-relative)
  *   CFA = s0 + (frame_size - fp_pair_off)
  * RISC-V has no condition flags: comparisons materialize a 0/1 via SLT/SLTU or
  * FLT/FLE; branches compare two registers directly. x0 is a hardware zero. */
 
 #include <string.h>
 
 #include "abi/abi.h"
 #include "arch/riscv/asm.h"
 #include "arch/riscv/isa.h"
 #include "arch/riscv/regs.h"
 #include "arch/riscv/rv64.h"
 #include "arch/riscv/variant.h"
 #include "asm/asm.h"
 #include "asm/asm_lex.h"
 #include "cg/native_argmove.h"
 #include "cg/native_asm.h"
 #include "cg/native_direct_target.h"
 #include "cg/native_frame.h"
 #include "cg/type.h"
 #include "core/arena.h"
 #include "core/bytes.h"
 #include "core/pool.h"
 #include "core/slice.h"
 #include "obj/obj.h"
 
 enum {
   RV_TMP0 = 5u,  /* t0: emit-internal scratch (reserved, never allocable) */
   RV_TMP1 = 6u,  /* t1: emit-internal scratch */
   RV_TMP2 = 7u,  /* t2: emit-internal scratch (reserved in phys table) */
   RV_TMP3 = 28u, /* t3: emit-internal scratch (reserved in phys table) */
   RV_FTMP0 = 0u, /* ft0: emit-internal FP scratch */
   RV_FTMP1 = 1u, /* ft1: emit-internal FP scratch */
   RV_FA0 = 10u,  /* fa0..fa7 = f10..f17 (FP arg/return registers) */
   RV_FA7 = 17u,
   /* Single-pass (-O0) worst-case prologue: sp adjust (3) + far save pair (7)
    * + sret spill (1) + variadic GP spills (8). No callee-saves at -O0. */
   RV_PROLOGUE_WORDS = 32u,
   /* Known-frame (-O1) prologues are emitted directly, not into the fixed -O0
    * NOP region, and additionally save callee-saved registers (up to 11 int + 12
    * fp, each up to 4 words for a far s0-relative offset) on top of the header,
    * sret, and variadic spills. Size the build buffer for the worst case. */
   RV_KNOWN_PROLOGUE_WORDS = 192u,
   RV_FRAME_SAVE_SIZE = 16u,
 };
 
 /* s1..s11 (11) + fs0..fs11 (12); separate int/fp collect arrays use this cap.
  */
 #define RV_MAX_CALLEE_SAVES 16u
 #define RV_MAX_REG_ARG_MOVES 16u
 
 extern void debug_emit_row(Debug*, ObjSecId text_section, u32 offset, SrcLoc);
 extern void debug_func_pc_range(Debug*, ObjSecId text_section, u32 begin_ofs,
                                 u32 end_ofs);
 
 /* ============================ low-level emit ============================ */
 
 void rv64_emit32(MCEmitter* mc, u32 word) {
   u8 b[4];
   u32 ofs = obj_pos(mc->obj, mc->section_id);
   wr_u32_le(b, word);
   mc->emit_bytes(mc, b, sizeof b);
   if (mc->debug) debug_emit_row(mc->debug, mc->section_id, ofs, mc->loc);
 }
 
 void rv64_emit16(MCEmitter* mc, u32 halfword) {
   u8 b[2];
   u32 ofs = obj_pos(mc->obj, mc->section_id);
   b[0] = (u8)(halfword & 0xff);
   b[1] = (u8)((halfword >> 8) & 0xff);
   mc->emit_bytes(mc, b, sizeof b);
   if (mc->debug) debug_emit_row(mc->debug, mc->section_id, ofs, mc->loc);
 }
 
 static void rv_patch32(ObjBuilder* obj, ObjSecId sec, u32 off, u32 word) {
   u8 b[4];
   wr_u32_le(b, word);
   obj_patch(obj, sec, off, b, sizeof b);
 }
 
 static int fits_i12(i64 v) { return v >= -2048 && v <= 2047; }
 static int fits_i32(i64 v) {
   return v >= (i64)(i32)0x80000000 && v <= (i64)(i32)0x7fffffff;
 }
 
 static u32 align_up_u32(u32 v, u32 align) {
   u32 mask = align ? align - 1u : 0u;
   return (v + mask) & ~mask;
 }
 
 static i64 floor_div_4096(i64 v) {
   if (v >= 0) return v / 4096;
   return -((-v + 4095) / 4096);
 }
 
 static void rv_emit_li32(const RiscvVariant* v, MCEmitter* mc, u32 rd,
                          i32 imm) {
   if (imm >= -2048 && imm <= 2047) {
     rv64_emit32(mc, rv_addi(rd, RV_ZERO, imm));
     return;
   }
   {
     i64 hi64 = floor_div_4096((i64)imm + 0x800);
     i32 hi = (i32)hi64;
     i32 lo = (i32)((i64)imm - hi64 * 4096);
     rv64_emit32(mc, rv_lui(rd, (u32)hi & 0xfffffu));
     /* ADDIW is RV64-only; on RV32 the value fits 32 bits so plain ADDI is
      * exact (and identical to ADDIW's low result on RV64). */
     if (lo)
       rv64_emit32(mc, v->has_w_forms ? rv_addiw(rd, rd, lo) : rv_addi(rd, rd, lo));
   }
 }
 
 static i32 sext12(u32 v) {
   v &= 0xfffu;
   return (v & 0x800u) ? (i32)v - 4096 : (i32)v;
 }
 
 /* Builds a full XLEN-wide value. The recursion / slli-12 chain assembles bits
  * above 32 and is only ever reached on rv64 (a single rv32 register cannot hold
  * a value wider than 32 bits — the cg layer legalizes those into pairs). */
 static void rv_emit_li64(const RiscvVariant* v, MCEmitter* mc, u32 rd, u64 imm) {
   if (fits_i32((i64)imm)) {
     rv_emit_li32(v, mc, rd, (i32)(i64)imm);
     return;
   }
   {
     i32 lo = sext12((u32)imm);
     u64 hi = (imm - (u64)(i64)lo) >> 12;
     rv_emit_li64(v, mc, rd, hi);
     rv64_emit32(mc, rv_slli(rd, rd, 12));
     if (lo) rv64_emit32(mc, rv_addi(rd, rd, lo));
   }
 }
 
 /* sf!=0 selects a full native-width materialization; sf==0 a 32-bit value. On
  * rv32 the native width is 32, so the wide branch collapses to the 32-bit
  * path. */
 static void rv_emit_load_imm(const RiscvVariant* v, MCEmitter* mc, u32 sf,
                              u32 rd, i64 imm) {
   if (!sf || v->xlen == 32u) {
     rv_emit_li32(v, mc, rd, (i32)imm);
     return;
   }
   if (fits_i32(imm))
     rv_emit_li32(v, mc, rd, (i32)imm);
   else
     rv_emit_li64(v, mc, rd, (u64)imm);
 }
 
 /* rd = base + off, materializing the offset when it exceeds imm12. Uses RV_TMP1
  * as scratch for the wide path, so callers must keep RV_TMP1 free. */
 static void rv_emit_addr_adjust(const RiscvVariant* v, MCEmitter* mc, u32 rd,
                                 u32 base, i32 off) {
   if (off == 0) {
     if (rd != base) rv64_emit32(mc, rv_addi(rd, base, 0));
     return;
   }
   if (fits_i12(off)) {
     rv64_emit32(mc, rv_addi(rd, base, off));
     return;
   }
   rv_emit_load_imm(v, mc, 1, RV_TMP1, (i64)off);
   rv64_emit32(mc, rv_add(rd, base, RV_TMP1));
 }
 
 static u32 enc_int_store(const RiscvVariant* v, u32 nbytes, u32 src, u32 base,
                          i32 off) {
   switch (nbytes) {
     case 1:
       return rv_sb(src, base, off);
     case 2:
       return rv_sh(src, base, off);
     case 4:
       return rv_sw(src, base, off);
     default:
       /* The widest GPR store is SD on rv64, SW on rv32. */
       return v->ptr_bytes == 8u ? rv_sd(src, base, off) : rv_sw(src, base, off);
   }
 }
 static u32 enc_int_load(const RiscvVariant* v, u32 nbytes, int sign_ext, u32 rd,
                         u32 base, i32 off) {
   switch (nbytes) {
     case 1:
       return sign_ext ? rv_lb(rd, base, off) : rv_lbu(rd, base, off);
     case 2:
       return sign_ext ? rv_lh(rd, base, off) : rv_lhu(rd, base, off);
     case 4:
       /* LWU (zero-extending 32-bit load) is RV64-only; on rv32 a 4-byte load
        * is just LW (no wider container to zero-extend into). */
       return sign_ext || v->xlen == 32u ? rv_lw(rd, base, off)
                                         : rv_lwu(rd, base, off);
     default:
       /* The widest GPR load is LD on rv64, LW on rv32. */
       return v->ptr_bytes == 8u ? rv_ld(rd, base, off) : rv_lw(rd, base, off);
   }
 }
 
 /* Pointer-width GPR load/store (GOT entries, frame-value bases, saved ra/s0,
  * sret/indirect/va_list pointers): LD/SD on rv64, LW/SW on rv32. */
 static u32 rv_ld_ptr(const RiscvVariant* v, u32 rd, u32 base, i32 off) {
   return v->ptr_bytes == 8u ? rv_ld(rd, base, off) : rv_lw(rd, base, off);
 }
 static u32 rv_sd_ptr(const RiscvVariant* v, u32 src, u32 base, i32 off) {
   return v->ptr_bytes == 8u ? rv_sd(src, base, off) : rv_sw(src, base, off);
 }
 
 /* ============================ target state ============================ */
 
 /* Frame slots and callee-save records live in the shared NativeFrame
  * bookkeeping (cg/native_frame.h); these aliases keep the rv64-local spellings.
  */
 typedef NativeFrameSlotEntry RvNativeSlot;
 typedef NativeFrameCalleeSave RvCalleeSave;
 
 typedef enum RvPatchKind { RV_PATCH_ALLOCA } RvPatchKind;
 
 typedef struct RvPatch {
   u8 kind; /* RvPatchKind */
   u32 pos;
   u32 dst_reg;
 } RvPatch;
 
 typedef struct RvNativeTarget {
   NativeTarget base;
   /* Immutable per-XLEN descriptor (rv32 / rv64), set once in the constructor
    * from c->target.arch. Every XLEN-dependent emit site reads it; with the
    * rv64 variant each site reproduces the historical literal exactly. */
   const RiscvVariant* variant;
   SrcLoc loc;
   const CGFuncDesc* func;
 
   /* Shared frame bookkeeping: slot table, cum_off, max_outgoing, callee-save
    * set, and the known_frame / has_alloca / frame_final flags. */
   NativeFrame frame;
   u32 frame_size_final;
   u32 fp_pair_off;
   u32 minimal_prologue_words; /* known-frame path: exact prologue length, else 0
                                */
 
   /* Known-frame (-O1) leaf no-frame tier (aa64's slim_prologue equivalent),
    * settled in rv_func_begin_known_frame; always 0 on the single-pass path. A
    * leaf with no callee-saves, no body slots, no outgoing args, no
    * sret/variadic and register-only params never reads s0 nor clobbers ra, so
    * it emits NO prologue and a bare `ret` — the whole frame setup/teardown is
    * elided. RISC-V has no pre/post-indexed store, so aa64's fp_at_bottom fold
    * would save zero instructions on a kept frame and is intentionally not
    * ported (see doc/plan/ARCH.md §2); this leaf tier is the rv64 win. */
   u8 slim_prologue;
 
   u32 incoming_stack_size; /* fixed-param stack bytes (tail-call check) */
   u32 next_param_int;
   u32 next_param_fp;
   u32 next_param_stack;
   u8 has_sret;
   u8 is_variadic;
   NativeFrameSlot sret_ptr_slot;
 
   RvPatch* patches;
   u32 npatches;
   u32 patches_cap;
   u32 nalloca;
 
   u32 func_start;
   u32 prologue_pos;
   MCLabel epilogue_label;
 } RvNativeTarget;
 
 static RvNativeTarget* rv_of(NativeTarget* t) { return (RvNativeTarget*)t; }
 
 static _Noreturn void rv_panic(RvNativeTarget* a, const char* msg) {
   compiler_panic(a->base.c, a->loc, "rv64 native target: %s", msg);
 }
 
 static RvNativeSlot* rv_slot_get(RvNativeTarget* a, NativeFrameSlot fs) {
   return native_frame_slot_at(&a->frame, fs);
 }
 
 /* s0-relative byte offset of a frame slot's base (address = s0 + ret). */
 static i32 rv_s0_off_slot(const RvNativeSlot* s) { return -(i32)s->off; }
 
 static u32 rv_va_save_sz(const RvNativeTarget* a) {
   /* ABI-derived: the variadic register-save area is gp_reg_count*gp_slot_size
    * (a0..a7 = 64 bytes for LP64D, 32 for ILP32). Only present in variadics. */
   return a->is_variadic ? native_frame_va_save_bytes(a->base.c->abi) : 0u;
 }
 
 /* s0-relative byte offset of incoming stack arg at byte_off. Stack args sit
  * just above the saved pair; the variadic GP save area (when present) is
  * contiguous with them at [s0 + frame_save_size). */
 static i32 rv_s0_off_in_arg(const RvNativeTarget* a, u32 byte_off) {
   u32 base = a->variant->frame_save_size;
   if (a->is_variadic) base += rv_va_save_sz(a);
   return (i32)(base + byte_off);
 }
 
 /* Callee-saved registers are homed just below the locals at rv_save_off() —
  * they are NOT frame slots, so the frame size must reserve their bytes
  * explicitly. Integer saves are ptr_bytes wide (sd on rv64, sw on rv32); FP
  * saves are always 8 bytes (fsd, even on rv32d). On rv64 both are 8 so the sum
  * is identical to the historical ncallee_saves*8. Zero at -O0. */
 static u32 rv_callee_save_bytes(const RvNativeTarget* a) {
   u32 ptr = a->variant->ptr_bytes;
   u32 i, bytes = 0;
   for (i = 0; i < a->frame.ncallee_saves; ++i)
     bytes += a->frame.callee_saves[i].cls == NATIVE_REG_FP ? 8u : ptr;
   return bytes;
 }
 
 static u32 rv_frame_size(const RvNativeTarget* a) {
   u32 raw = a->variant->frame_save_size + a->frame.cum_off +
             rv_callee_save_bytes(a) + a->frame.max_outgoing + rv_va_save_sz(a);
   return align_up_u32(raw, 16u);
 }
 
 static u32 rv_fp_pair_off(const RvNativeTarget* a, u32 frame_size) {
   return frame_size - a->variant->frame_save_size - rv_va_save_sz(a);
 }
 
 /* ============================ type helpers ============================ */
 
 /* Scalar size/align/mem/class/loc constructors are shared in native_target.h
  * (native_type_size, native_type_align, native_mem_for_type,
  * native_class_for_type_fp_le8, native_loc_reg, native_loc_stack,
  * native_loc_is_fp). loc_reg's mask is arch-specific and stays here. */
 
 /* True when a scalar value is WIDER than XLEN's natural single-register width,
  * i.e. it needs the "wide" (rv64 64-bit) ops rather than the base ops. On rv64
  * a pointer is 8 bytes and counts as wide alongside i64/double; on rv32 a
  * pointer is 4 bytes and fits a single 32-bit register, so it is NOT wide and
  * the base (non-W) ops apply. (Kept named rv_is_64 to minimize churn; for the
  * rv64 variant the result is byte-identical to the old predicate.) */
 static int rv_is_64(NativeTarget* t, KitCgTypeId type) {
   const RiscvVariant* v = rv_of(t)->variant;
   return native_type_size(t, type) >= 8u ||
          (v->xlen == 64u && cg_type_is_ptr(t->c, type));
 }
 
 static u32 loc_reg(NativeLoc loc) { return loc.v.reg & 0x1fu; }
 
 /* ============================ register tables ============================ */
 
 #define RV_PHYS_INT_ARG(r, idx)                        \
   {.reg = (r),                                         \
    .cls = NATIVE_REG_INT,                              \
    .abi_index = (idx),                                 \
    .flags = NATIVE_REG_CALLER_SAVED | NATIVE_REG_ARG | \
             ((idx) < 2u ? NATIVE_REG_RET : 0),         \
    .spill_cost = 1u,                                   \
    .copy_cost = 1u}
 #define RV_PHYS_INT_CALLER(r)                               \
   {.reg = (r),                                              \
    .cls = NATIVE_REG_INT,                                   \
    .abi_index = 0xffu,                                      \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLER_SAVED, \
    .spill_cost = 1u,                                        \
    .copy_cost = 1u}
 #define RV_PHYS_INT_CALLEE(r)                               \
   {.reg = (r),                                              \
    .cls = NATIVE_REG_INT,                                   \
    .abi_index = 0xffu,                                      \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLEE_SAVED, \
    .spill_cost = 4u,                                        \
    .copy_cost = 1u}
 #define RV_PHYS_INT_RESERVED(r)  \
   {.reg = (r),                   \
    .cls = NATIVE_REG_INT,        \
    .abi_index = 0xffu,           \
    .flags = NATIVE_REG_RESERVED, \
    .spill_cost = 0u,             \
    .copy_cost = 0u}
 
 /* t0..t3 (x5,x6,x7,x28) are emit-internal scratch (RV_TMP0..RV_TMP3), reserved
  * and never handed to the allocator or driver. t4/t5 are the driver scratch
  * pool (disjoint from the emit temps so a hook can never clobber an operand the
  * driver parked there). t6 is the lone caller-saved allocable (the -O0 cache's
  * only caller-saved home); s1..s11 are appended callee-saved, chosen under
  * pressure (and saved by the optimizer prologue at -O1). */
 static const Reg rv_int_allocable[] = {31u, 9u,  18u, 19u, 20u, 21u,
                                        22u, 23u, 24u, 25u, 26u, 27u};
 static const Reg rv_int_scratch[] = {29u, 30u}; /* t4, t5 */
 
 static const NativePhysRegInfo rv_int_phys[] = {
     RV_PHYS_INT_RESERVED(0u), /* zero */
     RV_PHYS_INT_RESERVED(1u), /* ra */
     RV_PHYS_INT_RESERVED(2u), /* sp */
     RV_PHYS_INT_RESERVED(3u), /* gp */
     RV_PHYS_INT_RESERVED(4u), /* tp */
     RV_PHYS_INT_RESERVED(5u), /* t0 = TMP0 */
     RV_PHYS_INT_RESERVED(6u), /* t1 = TMP1 */
     RV_PHYS_INT_RESERVED(7u), /* t2 = TMP2 (emit) */
     RV_PHYS_INT_RESERVED(8u), /* s0/fp */
     RV_PHYS_INT_CALLEE(9u),   /* s1 */
     RV_PHYS_INT_ARG(10u, 0u),  RV_PHYS_INT_ARG(11u, 1u),
     RV_PHYS_INT_ARG(12u, 2u),  RV_PHYS_INT_ARG(13u, 3u),
     RV_PHYS_INT_ARG(14u, 4u),  RV_PHYS_INT_ARG(15u, 5u),
     RV_PHYS_INT_ARG(16u, 6u),  RV_PHYS_INT_ARG(17u, 7u),
     RV_PHYS_INT_CALLEE(18u),   RV_PHYS_INT_CALLEE(19u),
     RV_PHYS_INT_CALLEE(20u),   RV_PHYS_INT_CALLEE(21u),
     RV_PHYS_INT_CALLEE(22u),   RV_PHYS_INT_CALLEE(23u),
     RV_PHYS_INT_CALLEE(24u),   RV_PHYS_INT_CALLEE(25u),
     RV_PHYS_INT_CALLEE(26u),   RV_PHYS_INT_CALLEE(27u),
     RV_PHYS_INT_RESERVED(28u), /* t3 = TMP3 (emit) */
     RV_PHYS_INT_RESERVED(29u), /* t4 = driver scratch */
     RV_PHYS_INT_RESERVED(30u), /* t5 = driver scratch */
     RV_PHYS_INT_CALLER(31u),   /* t6 = caller-saved allocable */
 };
 
 #define RV_PHYS_FP_ARG(r, idx)                         \
   {.reg = (r),                                         \
    .cls = NATIVE_REG_FP,                               \
    .abi_index = (idx),                                 \
    .flags = NATIVE_REG_CALLER_SAVED | NATIVE_REG_ARG | \
             ((idx) < 2u ? NATIVE_REG_RET : 0),         \
    .spill_cost = 1u,                                   \
    .copy_cost = 1u}
 #define RV_PHYS_FP_CALLER(r)                                \
   {.reg = (r),                                              \
    .cls = NATIVE_REG_FP,                                    \
    .abi_index = 0xffu,                                      \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLER_SAVED, \
    .spill_cost = 1u,                                        \
    .copy_cost = 1u}
 #define RV_PHYS_FP_CALLEE(r)                                \
   {.reg = (r),                                              \
    .cls = NATIVE_REG_FP,                                    \
    .abi_index = 0xffu,                                      \
    .flags = NATIVE_REG_ALLOCABLE | NATIVE_REG_CALLEE_SAVED, \
    .spill_cost = 4u,                                        \
    .copy_cost = 1u}
 #define RV_PHYS_FP_RESERVED(r)   \
   {.reg = (r),                   \
    .cls = NATIVE_REG_FP,         \
    .abi_index = 0xffu,           \
    .flags = NATIVE_REG_RESERVED, \
    .spill_cost = 0u,             \
    .copy_cost = 0u}
 
 /* Caller-saved allocable first (ft4..ft7, ft8..ft11), then callee (fs0..fs11).
  * ft0/ft1 reserved as emit-internal scratch; ft2/ft3 driver scratch. */
 static const Reg rv_fp_allocable[] = {4u,  5u,  6u,  7u,  28u, 29u, 30u,
                                       31u, 8u,  9u,  18u, 19u, 20u, 21u,
                                       22u, 23u, 24u, 25u, 26u, 27u};
 static const Reg rv_fp_scratch[] = {2u, 3u}; /* ft2, ft3 */
 
 static const NativePhysRegInfo rv_fp_phys[] = {
     RV_PHYS_FP_RESERVED(0u), /* ft0 = FTMP0 */
     RV_PHYS_FP_RESERVED(1u), /* ft1 = FTMP1 */
     RV_PHYS_FP_RESERVED(2u), /* ft2 = scratch */
     RV_PHYS_FP_RESERVED(3u), /* ft3 = scratch */
     RV_PHYS_FP_CALLER(4u),   RV_PHYS_FP_CALLER(5u),   RV_PHYS_FP_CALLER(6u),
     RV_PHYS_FP_CALLER(7u),   RV_PHYS_FP_CALLEE(8u),   RV_PHYS_FP_CALLEE(9u),
     RV_PHYS_FP_ARG(10u, 0u), RV_PHYS_FP_ARG(11u, 1u), RV_PHYS_FP_ARG(12u, 2u),
     RV_PHYS_FP_ARG(13u, 3u), RV_PHYS_FP_ARG(14u, 4u), RV_PHYS_FP_ARG(15u, 5u),
     RV_PHYS_FP_ARG(16u, 6u), RV_PHYS_FP_ARG(17u, 7u), RV_PHYS_FP_CALLEE(18u),
     RV_PHYS_FP_CALLEE(19u),  RV_PHYS_FP_CALLEE(20u),  RV_PHYS_FP_CALLEE(21u),
     RV_PHYS_FP_CALLEE(22u),  RV_PHYS_FP_CALLEE(23u),  RV_PHYS_FP_CALLEE(24u),
     RV_PHYS_FP_CALLEE(25u),  RV_PHYS_FP_CALLEE(26u),  RV_PHYS_FP_CALLEE(27u),
     RV_PHYS_FP_CALLER(28u),  RV_PHYS_FP_CALLER(29u),  RV_PHYS_FP_CALLER(30u),
     RV_PHYS_FP_CALLER(31u),
 };
 
 static const NativeAllocClassInfo rv_classes[] = {
     {.cls = NATIVE_REG_INT,
      .allocable = rv_int_allocable,
      .nallocable = sizeof rv_int_allocable / sizeof rv_int_allocable[0],
      .scratch = rv_int_scratch,
      .nscratch = sizeof rv_int_scratch / sizeof rv_int_scratch[0],
      .phys = rv_int_phys,
      .nphys = sizeof rv_int_phys / sizeof rv_int_phys[0],
      /* t0-t6 (5-7,28-31) + a0-a7 (10-17) */
      .caller_saved_mask = 0xf00400e0u | 0x0001fc00u,
      /* s0-s11 (8,9,18-27) */
      .callee_saved_mask = 0x0ffc0300u,
      .arg_mask = 0x0001fc00u,
      .ret_mask = 0x00000c00u,
      /* zero,ra,sp,gp,tp,t0,t1,t2,s0 (bits 0-8) + t3 (bit 28). t4/t5 are the
       * driver scratch pool (reserved-from-alloc but listed in scratch[]). */
      .reserved_mask = 0x000001ffu | (1u << 28)},
     {.cls = NATIVE_REG_FP,
      .allocable = rv_fp_allocable,
      .nallocable = sizeof rv_fp_allocable / sizeof rv_fp_allocable[0],
      .scratch = rv_fp_scratch,
      .nscratch = sizeof rv_fp_scratch / sizeof rv_fp_scratch[0],
      .phys = rv_fp_phys,
      .nphys = sizeof rv_fp_phys / sizeof rv_fp_phys[0],
      /* ft0-ft7 (0-7), fa0-fa7 (10-17), ft8-ft11 (28-31) */
      .caller_saved_mask = 0xf00400ffu | 0x0001fc00u,
      /* fs0-fs11 (8,9,18-27) */
      .callee_saved_mask = 0x0ffc0300u,
      .arg_mask = 0x0001fc00u,
      .ret_mask = 0x00000c00u,
      .reserved_mask = 0x0000000fu /* ft0-ft3 */},
 };
 
 /* Resolve a register name ("a7", "fa0", ...) to its (class, Reg). Powers the
  * optimizer's inline-asm clobber masks and explicit hard-register operands
  * ("{a7}" from a GNU local register variable). x0..x31 are DWARF 0..31; the
  * FP bank f0..f31 is DWARF 32..63. Returns non-zero for a non-register name
  * (cc/memory/unknown), which the caller skips. */
 static int rv_resolve_name(const NativeRegInfo* ri, Slice name, Reg* out,
                            NativeAllocClass* cls_out) {
   char buf[16];
   uint32_t dwarf;
   (void)ri;
   if (!name.s || !name.len || name.len >= sizeof buf) return 1;
   memcpy(buf, name.s, name.len);
   buf[name.len] = '\0';
   if (rv64_register_index(buf, &dwarf) != 0) return 1;
   if (dwarf <= 31u) {
     *cls_out = NATIVE_REG_INT;
     *out = (Reg)dwarf;
     return 0;
   }
   if (dwarf >= 32u && dwarf <= 63u) {
     *cls_out = NATIVE_REG_FP;
     *out = (Reg)(dwarf - 32u);
     return 0;
   }
   return 1;
 }
 
 static int rv_asm_operand_reg_ok(const NativeRegInfo* ri, NativeAllocClass cls,
                                  Reg reg) {
   (void)ri;
   if (cls == NATIVE_REG_INT) {
     if (reg == 9u) return 1;                /* s1 */
     if (reg >= 10u && reg <= 17u) return 1; /* a0..a7 */
     if (reg >= 18u && reg <= 27u) return 1; /* s2..s11 */
     if (reg == 31u) return 1;               /* t6 */
     return 0;
   }
   if (cls == NATIVE_REG_FP) return reg >= 4u && reg <= 31u;
   return 0;
 }
 
 static int rv_asm_constraint_reg(const NativeRegInfo* ri, const char* body,
                                  NativeAllocClass* cls_out, Reg* fixed_out,
                                  u32* allowed_mask_out) {
   (void)ri;
   if (!body || !body[0]) return 0;
   if (fixed_out) *fixed_out = REG_NONE;
   if (allowed_mask_out) *allowed_mask_out = 0;
   if (body[0] == 'r' && body[1] == '\0') {
     if (cls_out) *cls_out = NATIVE_REG_INT;
     return 1;
   }
   if (body[0] == 'f' && body[1] == '\0') {
     if (cls_out) *cls_out = NATIVE_REG_FP;
     return 1;
   }
   if (body[0] == 'c' && body[1] == 'r' && body[2] == '\0') {
     if (cls_out) *cls_out = NATIVE_REG_INT;
     if (allowed_mask_out) *allowed_mask_out = 0x0000ff00u; /* x8..x15 */
     return 1;
   }
   if (body[0] == 'c' && body[1] == 'f' && body[2] == '\0') {
     if (cls_out) *cls_out = NATIVE_REG_FP;
     if (allowed_mask_out) *allowed_mask_out = 0x0000ff00u; /* f8..f15 */
     return 1;
   }
   return 0;
 }
 
 static const NativeRegInfo rv_reg_info = {
     .classes = rv_classes,
     .nclasses = sizeof rv_classes / sizeof rv_classes[0],
     .resolve_name = rv_resolve_name,
     .asm_operand_reg_ok = rv_asm_operand_reg_ok,
     .asm_constraint_reg = rv_asm_constraint_reg,
 };
 
 /* ============================ legality ============================ */
 
 static int rv_imm_legal(NativeTarget* t, NativeImmUse use, u32 op,
                         KitCgTypeId type, i64 imm) {
   /* SLLI/SRLI/SRAI shamt is shamt_bits wide: 6 bits (max 63) on rv64, 5 bits
    * (max 31) on rv32. */
   i64 shamt_max = (i64)((1u << rv_of(t)->variant->shamt_bits) - 1u);
   (void)type;
   switch (use) {
     case NATIVE_IMM_MOVE:
       return 1;
     case NATIVE_IMM_BINOP:
       switch ((BinOp)op) {
         case BO_IADD:
           return fits_i12(imm);
         case BO_ISUB:
           return fits_i12(-imm); /* emitted as ADDI with negated imm */
         case BO_AND:
         case BO_OR:
         case BO_XOR:
           return fits_i12(imm);
         case BO_SHL:
         case BO_SHR_S:
         case BO_SHR_U:
           return imm >= 0 && imm <= shamt_max;
         default:
           return 0;
       }
     case NATIVE_IMM_CMP:
       return imm == 0; /* compares need both ends in registers (SLT/branch) */
     case NATIVE_IMM_ADDR_OFFSET:
       return fits_i12(imm);
   }
   return 0;
 }
 
 static int rv_addr_legal(NativeTarget* t, const NativeAddr* addr,
                          MemAccess mem) {
   (void)t;
   (void)mem;
   if (!addr) return 0;
   if (addr->index_kind != NATIVE_ADDR_INDEX_NONE) return 0;
   if (addr->base_kind != NATIVE_ADDR_BASE_REG &&
       addr->base_kind != NATIVE_ADDR_BASE_FRAME)
     return 0;
   return fits_i12(addr->offset);
 }
 
 /* ============================ memory ============================ */
 
 /* Materialize the runtime address of a global into `dst`, including addend. */
 static void rv_emit_global_addr(RvNativeTarget* a, u32 dst, ObjSymId sym,
                                 i64 addend) {
   NativeTarget* t = &a->base;
   MCEmitter* mc = t->mc;
   u32 sec = mc->section_id;
   if (obj_symbol_extern_via_got(t->c, t->obj, sym)) {
     u32 ap = mc->pos(mc);
     rv64_emit32(mc, rv_auipc(dst, 0));
     mc->emit_reloc_at(mc, sec, ap, R_RV_GOT_HI20, sym, 0, 0, 0);
     {
       Sym an = pool_intern_slice(t->c->global, SLICE_LIT(".LpcrelHi"));
       ObjSymId anchor =
           obj_symbol(t->obj, an, SB_LOCAL, SK_OBJ, sec, (u64)ap, 0);
       u32 lp = mc->pos(mc);
       rv64_emit32(mc, rv_ld_ptr(a->variant, dst, dst, 0));
       mc->emit_reloc_at(mc, sec, lp, R_RV_PCREL_LO12_I, anchor, 0, 0, 0);
     }
   } else {
     u32 ap = mc->pos(mc);
     rv64_emit32(mc, rv_auipc(dst, 0));
     mc->emit_reloc_at(mc, sec, ap, R_RV_PCREL_HI20, sym, 0, 0, 0);
     {
       Sym an = pool_intern_slice(t->c->global, SLICE_LIT(".LpcrelHi"));
       ObjSymId anchor =
           obj_symbol(t->obj, an, SB_LOCAL, SK_OBJ, sec, (u64)ap, 0);
       u32 lp = mc->pos(mc);
       rv64_emit32(mc, rv_addi(dst, dst, 0));
       mc->emit_reloc_at(mc, sec, lp, R_RV_PCREL_LO12_I, anchor, 0, 0, 0);
     }
   }
   if (addend) rv_emit_addr_adjust(a->variant, mc, dst, dst, (i32)addend);
 }
 
 /* Fold (base_reg << 0) + (index << scale) into RV_TMP0 via Zba. */
 static u32 rv_fold_index(RvNativeTarget* a, u32 base, u32 idx, u8 log2_scale) {
   MCEmitter* mc = a->base.mc;
   switch (log2_scale) {
     case 0:
       rv64_emit32(mc, rv_add(RV_TMP0, base, idx));
       break;
     case 1:
       rv64_emit32(mc, rv_sh1add(RV_TMP0, idx, base));
       break;
     case 2:
       rv64_emit32(mc, rv_sh2add(RV_TMP0, idx, base));
       break;
     default:
       rv64_emit32(mc, rv_sh3add(RV_TMP0, idx, base));
       break;
   }
   return RV_TMP0;
 }
 
 /* Resolve any NativeAddr to a base register + imm12 offset. RISC-V has no
  * indexed load/store, so an index is folded into RV_TMP0 via Zba; far offsets
  * and FRAME/FRAME_VALUE/GLOBAL bases are materialized into RV_TMP0/RV_TMP1. */
 static void rv_resolve_mem_addr(RvNativeTarget* a, const NativeAddr* addr,
                                 u32* base_out, i32* off_out) {
   MCEmitter* mc = a->base.mc;
   u32 base;
   i32 off;
   switch (addr->base_kind) {
     case NATIVE_ADDR_BASE_REG:
       base = addr->base.reg & 0x1fu;
       off = addr->offset;
       break;
     case NATIVE_ADDR_BASE_FRAME: {
       RvNativeSlot* s = rv_slot_get(a, addr->base.frame);
       base = RV_S0;
       off = rv_s0_off_slot(s) + addr->offset;
       break;
     }
     case NATIVE_ADDR_BASE_FRAME_VALUE: {
       RvNativeSlot* s = rv_slot_get(a, addr->base.frame);
       rv64_emit32(mc, rv_ld_ptr(a->variant, RV_TMP0, RV_S0, rv_s0_off_slot(s)));
       base = RV_TMP0;
       off = addr->offset;
       break;
     }
     case NATIVE_ADDR_BASE_GLOBAL:
       rv_emit_global_addr(a, RV_TMP0, addr->base.global.sym,
                           addr->base.global.addend);
       base = RV_TMP0;
       off = addr->offset;
       break;
     default:
       rv_panic(a, "unsupported address base");
   }
   if (addr->index_kind == NATIVE_ADDR_INDEX_REG) {
     base = rv_fold_index(a, base, addr->index.reg & 0x1fu, addr->log2_scale);
   } else if (addr->index_kind == NATIVE_ADDR_INDEX_FRAME_VALUE) {
     RvNativeSlot* s = rv_slot_get(a, addr->index.frame);
     rv64_emit32(mc, rv_ld_ptr(a->variant, RV_TMP1, RV_S0, rv_s0_off_slot(s)));
     base = rv_fold_index(a, base, RV_TMP1, addr->log2_scale);
   }
   if (!fits_i12(off)) {
     rv_emit_load_imm(a->variant, mc, 1, RV_TMP1, (i64)off);
     rv64_emit32(mc, rv_add(RV_TMP0, base, RV_TMP1));
     base = RV_TMP0;
     off = 0;
   }
   *base_out = base;
   *off_out = off;
 }
 
 /* Central load/store primitive. is_load: 1 load into reg, 0 store reg to mem.
  */
 static void rv_emit_mem(RvNativeTarget* a, int is_load, NativeLoc reg,
                         NativeAddr addr, MemAccess mem) {
   NativeTarget* t = &a->base;
   MCEmitter* mc = t->mc;
   u32 r = loc_reg(reg);
   int fp = native_loc_is_fp(reg);
   u32 sz = mem.size ? mem.size : native_type_size(t, reg.type);
   u32 base;
   i32 off;
 
   rv_resolve_mem_addr(a, &addr, &base, &off);
   if (fp) {
     rv64_emit32(
         mc, is_load ? (sz == 8u ? rv_fld(r, base, off) : rv_flw(r, base, off))
                     : (sz == 8u ? rv_fsd(r, base, off) : rv_fsw(r, base, off)));
   } else {
     rv64_emit32(mc, is_load ? enc_int_load(a->variant, sz, 0, r, base, off)
                             : enc_int_store(a->variant, sz, r, base, off));
   }
 }
 
 /* ============================ moves / data ============================ */
 
 static void rv_move(NativeTarget* t, NativeLoc dst, NativeLoc src) {
   MCEmitter* mc = t->mc;
   int dfp = native_loc_is_fp(dst), sfp = native_loc_is_fp(src);
   u32 rd = loc_reg(dst), rs = loc_reg(src);
   if (dfp && sfp) {
     u32 fmt = native_type_size(t, dst.type) == 8u ? RV_FMT_D : RV_FMT_S;
     if (rd == rs) return;
     rv64_emit32(mc, rv_fsgnj(fmt, rd, rs, rs));
     return;
   }
   if (!dfp && sfp) {
     u32 sz = native_type_size(t, src.type);
     rv64_emit32(mc, sz == 8u ? rv_fmv_x_d(rd, rs) : rv_fmv_x_w(rd, rs));
     return;
   }
   if (dfp && !sfp) {
     u32 sz = native_type_size(t, dst.type);
     rv64_emit32(mc, sz == 8u ? rv_fmv_d_x(rd, rs) : rv_fmv_w_x(rd, rs));
     return;
   }
   if (rd == rs) return;
   rv64_emit32(mc, rv_addi(rd, rs, 0));
 }
 
 static void rv_load_imm(NativeTarget* t, NativeLoc dst, i64 imm) {
   rv_emit_load_imm(rv_of(t)->variant, t->mc, rv_is_64(t, dst.type) ? 1u : 0u,
                    loc_reg(dst), imm);
 }
 
 static void rv_load_const(NativeTarget* t, NativeLoc dst, ConstBytes cb) {
   RvNativeTarget* a = rv_of(t);
   u64 v = 0;
   u32 i;
   if (!native_loc_is_fp(dst)) {
     for (i = 0; i < cb.size && i < 8u; ++i) v |= (u64)cb.bytes[i] << (i * 8u);
     rv_load_imm(t, dst, (i64)v);
     return;
   }
   /* FP constant: materialize the bit pattern in TMP0, bitcast into the FPR. */
   for (i = 0; i < cb.size && i < 8u; ++i) v |= (u64)cb.bytes[i] << (i * 8u);
   rv_emit_load_imm(a->variant, t->mc, 1, RV_TMP0, (i64)v);
   if (cb.size == 8u)
     rv64_emit32(t->mc, rv_fmv_d_x(loc_reg(dst), RV_TMP0));
   else
     rv64_emit32(t->mc, rv_fmv_w_x(loc_reg(dst), RV_TMP0));
   (void)a;
 }
 
 static void rv_load_addr(NativeTarget* t, NativeLoc dst, NativeAddr addr) {
   RvNativeTarget* a = rv_of(t);
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst);
   u32 base;
   i32 off;
   if (addr.base_kind == NATIVE_ADDR_BASE_GLOBAL) {
     rv_emit_global_addr(a, rd, addr.base.global.sym,
                         addr.base.global.addend + addr.offset);
     base = rd;
     off = 0;
   } else if (addr.base_kind == NATIVE_ADDR_BASE_FRAME_VALUE) {
     /* Load the pointer stored in the frame slot, then add the offset. */
     RvNativeSlot* s = rv_slot_get(a, addr.base.frame);
     rv64_emit32(mc, rv_ld_ptr(a->variant, rd, RV_S0, rv_s0_off_slot(s)));
     base = rd;
     off = addr.offset;
   } else if (addr.base_kind == NATIVE_ADDR_BASE_FRAME) {
     RvNativeSlot* s = rv_slot_get(a, addr.base.frame);
     base = RV_S0;
     off = rv_s0_off_slot(s) + addr.offset;
   } else if (addr.base_kind == NATIVE_ADDR_BASE_REG) {
     base = addr.base.reg & 0x1fu;
     off = addr.offset;
   } else {
     rv_panic(a, "unsupported address base in load_addr");
   }
   /* Fold any index via Zba sh{1,2,3}add (index << scale) + base. */
   if (addr.index_kind == NATIVE_ADDR_INDEX_REG) {
     u32 idx = addr.index.reg & 0x1fu;
     if (off != 0 || base != rd)
       rv_emit_addr_adjust(a->variant, mc, rd, base, off);
     switch (addr.log2_scale) {
       case 0:
         rv64_emit32(mc, rv_add(rd, rd, idx));
         break;
       case 1:
         rv64_emit32(mc, rv_sh1add(rd, idx, rd));
         break;
       case 2:
         rv64_emit32(mc, rv_sh2add(rd, idx, rd));
         break;
       default:
         rv64_emit32(mc, rv_sh3add(rd, idx, rd));
         break;
     }
     return;
   }
   rv_emit_addr_adjust(a->variant, mc, rd, base, off);
 }
 
 static void rv_load(NativeTarget* t, NativeLoc dst, NativeAddr addr,
                     MemAccess mem) {
   rv_emit_mem(rv_of(t), 1, dst, addr, mem);
 }
 static void rv_store(NativeTarget* t, NativeAddr addr, NativeLoc src,
                      MemAccess mem) {
   rv_emit_mem(rv_of(t), 0, src, addr, mem);
 }
 
 /* copy_bytes: resolve dst and src to dedicated pointer regs (RV_TMP3 / RV_TMP0)
  * once, then copy granule-by-granule advancing both pointers. dst is resolved
  * first because its base may itself live in RV_TMP1 (the transfer reg, e.g. the
  * sret pointer from plan_ret); capturing it into RV_TMP3 before src resolution
  * (which may clobber RV_TMP1 for far offsets) keeps it live. Advancing the
  * pointers keeps every load/store at offset 0, so no offset ever exceeds imm12
  * and the transfer reg never aliases a base. */
 static void rv_copy_bytes(NativeTarget* t, NativeAddr dst, NativeAddr src,
                           AggregateAccess access) {
   RvNativeTarget* a = rv_of(t);
   const RiscvVariant* v = a->variant;
   MCEmitter* mc = t->mc;
   KitCgTypeId i64t = builtin_id(KIT_CG_BUILTIN_I64);
   u32 rem = access.size;
   u32 maxg = v->ptr_bytes; /* widest granule: 8 on rv64, 4 on rv32 */
   rv_load_addr(t, native_loc_reg(i64t, NATIVE_REG_INT, RV_TMP3), dst);
   rv_load_addr(t, native_loc_reg(i64t, NATIVE_REG_INT, RV_TMP0), src);
   while (rem) {
     u32 sz = rem >= 8u && maxg >= 8u ? 8u
              : rem >= 4u             ? 4u
              : rem >= 2u             ? 2u
                                      : 1u;
     rv64_emit32(mc, enc_int_load(v, sz, 0, RV_TMP1, RV_TMP0, 0));
     rv64_emit32(mc, enc_int_store(v, sz, RV_TMP1, RV_TMP3, 0));
     rv64_emit32(mc, rv_addi(RV_TMP0, RV_TMP0, (i32)sz));
     rv64_emit32(mc, rv_addi(RV_TMP3, RV_TMP3, (i32)sz));
     rem -= sz;
   }
 }
 
 static void rv_set_bytes(NativeTarget* t, NativeAddr dst, NativeLoc byte_value,
                          AggregateAccess access) {
   MCEmitter* mc = t->mc;
   KitCgTypeId i64t = builtin_id(KIT_CG_BUILTIN_I64);
   u32 bv = loc_reg(byte_value);
   u32 rem = access.size;
   rv_load_addr(t, native_loc_reg(i64t, NATIVE_REG_INT, RV_TMP3), dst);
   while (rem) {
     rv64_emit32(mc, rv_sb(bv, RV_TMP3, 0));
     rv64_emit32(mc, rv_addi(RV_TMP3, RV_TMP3, 1));
     rem -= 1u;
   }
 }
 
 /* ============================ arithmetic ============================ */
 
 static void rv_binop(NativeTarget* t, BinOp op, NativeLoc dst, NativeLoc aop,
                      NativeLoc bop) {
   const RiscvVariant* v = rv_of(t)->variant;
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst);
   u32 ra = loc_reg(aop);
   int sf = rv_is_64(t, dst.type);
   /* The W-form ops (ADDW/SUBW/MULW/SLLW/...) are RV64-only and act on a 32-bit
    * value held in a 64-bit register. They are emitted only for a narrow value
    * on rv64; on rv32 the BASE ops ARE the 32-bit ops, so `w` is always 0 and we
    * fall to the base ops. */
   int w = !sf && v->has_w_forms;
   /* Immediate shamt mask: 5-bit (&31) for a W-form / rv32 op, else shamt_bits
    * (6-bit &63 on rv64) for the native-width op. */
   u32 shmask = w ? 31u : ((1u << v->shamt_bits) - 1u);
   int b_imm = bop.kind == NATIVE_LOC_IMM;
   u32 rb = b_imm ? 0u : loc_reg(bop);
   i64 imm = b_imm ? bop.v.imm : 0;
 
   switch (op) {
     case BO_FADD:
     case BO_FSUB:
     case BO_FMUL:
     case BO_FDIV: {
       u32 fmt = native_type_size(t, dst.type) == 8u ? RV_FMT_D : RV_FMT_S;
       switch (op) {
         case BO_FADD:
           rv64_emit32(mc, rv_fadd(fmt, rd, ra, rb));
           break;
         case BO_FSUB:
           rv64_emit32(mc, rv_fsub(fmt, rd, ra, rb));
           break;
         case BO_FMUL:
           rv64_emit32(mc, rv_fmul(fmt, rd, ra, rb));
           break;
         default:
           rv64_emit32(mc, rv_fdiv(fmt, rd, ra, rb));
           break;
       }
       return;
     }
     case BO_IADD:
       if (b_imm) {
         rv64_emit32(
             mc, w ? rv_addiw(rd, ra, (i32)imm) : rv_addi(rd, ra, (i32)imm));
       } else {
         rv64_emit32(mc, w ? rv_addw(rd, ra, rb) : rv_add(rd, ra, rb));
       }
       return;
     case BO_ISUB:
       if (b_imm) {
         rv64_emit32(
             mc, w ? rv_addiw(rd, ra, (i32)-imm) : rv_addi(rd, ra, (i32)-imm));
       } else {
         rv64_emit32(mc, w ? rv_subw(rd, ra, rb) : rv_sub(rd, ra, rb));
       }
       return;
     case BO_IMUL:
       rv64_emit32(mc, w ? rv_mulw(rd, ra, rb) : rv_mul(rd, ra, rb));
       return;
     case BO_SDIV:
       rv64_emit32(mc, w ? rv_divw(rd, ra, rb) : rv_div(rd, ra, rb));
       return;
     case BO_UDIV:
       rv64_emit32(mc, w ? rv_divuw(rd, ra, rb) : rv_divu(rd, ra, rb));
       return;
     case BO_SREM:
       rv64_emit32(mc, w ? rv_remw(rd, ra, rb) : rv_rem(rd, ra, rb));
       return;
     case BO_UREM:
       rv64_emit32(mc, w ? rv_remuw(rd, ra, rb) : rv_remu(rd, ra, rb));
       return;
     case BO_AND:
       rv64_emit32(mc, b_imm ? rv_andi(rd, ra, (i32)imm) : rv_and(rd, ra, rb));
       return;
     case BO_OR:
       rv64_emit32(mc, b_imm ? rv_ori(rd, ra, (i32)imm) : rv_or(rd, ra, rb));
       return;
     case BO_XOR:
       rv64_emit32(mc, b_imm ? rv_xori(rd, ra, (i32)imm) : rv_xor(rd, ra, rb));
       return;
     case BO_SHL:
       if (b_imm)
         rv64_emit32(mc, w ? rv_slliw(rd, ra, (u32)imm & shmask)
                           : rv_slli(rd, ra, (u32)imm & shmask));
       else
         rv64_emit32(mc, w ? rv_sllw(rd, ra, rb) : rv_sll(rd, ra, rb));
       return;
     case BO_SHR_U:
       if (b_imm)
         rv64_emit32(mc, w ? rv_srliw(rd, ra, (u32)imm & shmask)
                           : rv_srli(rd, ra, (u32)imm & shmask));
       else
         rv64_emit32(mc, w ? rv_srlw(rd, ra, rb) : rv_srl(rd, ra, rb));
       return;
     case BO_SHR_S:
       if (b_imm)
         rv64_emit32(mc, w ? rv_sraiw(rd, ra, (u32)imm & shmask)
                           : rv_srai(rd, ra, (u32)imm & shmask));
       else
         rv64_emit32(mc, w ? rv_sraw(rd, ra, rb) : rv_sra(rd, ra, rb));
       return;
     default:
       rv_panic(rv_of(t), "unsupported binop");
   }
 }
 
 static void rv_unop(NativeTarget* t, UnOp op, NativeLoc dst, NativeLoc src) {
   const RiscvVariant* v = rv_of(t)->variant;
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst), rs = loc_reg(src);
   int sf = rv_is_64(t, dst.type);
   int w = !sf && v->has_w_forms; /* SUBW is RV64-only; base SUB on rv32 */
   switch (op) {
     case UO_NEG:
       rv64_emit32(mc, w ? rv_subw(rd, RV_ZERO, rs) : rv_sub(rd, RV_ZERO, rs));
       return;
     case UO_FNEG: {
       u32 fmt = native_type_size(t, dst.type) == 8u ? RV_FMT_D : RV_FMT_S;
       rv64_emit32(mc, rv_fsgnjn(fmt, rd, rs, rs));
       return;
     }
     case UO_BNOT:
       rv64_emit32(mc, rv_xori(rd, rs, -1));
       return;
     case UO_NOT:
       rv64_emit32(mc, rv_sltiu(rd, rs, 1));
       return;
     default:
       rv_panic(rv_of(t), "unsupported unop");
   }
 }
 
 /* Sign/zero-extend a 32-bit operand into a 64-bit register for comparison.
  * Returns the register to compare. */
 static u32 rv_cmp_ext(NativeTarget* t, int is_signed, NativeLoc op, u32 tmp) {
   const RiscvVariant* v = rv_of(t)->variant;
   MCEmitter* mc = t->mc;
   u32 r = loc_reg(op);
   /* On rv32 a 32-bit operand already fills the whole register — there is no
    * wider container to canonicalize into, so the extension is a no-op. */
   if (v->xlen == 32u) return r;
   if (rv_is_64(t, op.type)) return r;
   if (is_signed) {
     rv64_emit32(mc, rv_addiw(tmp, r, 0)); /* sign-extend low 32 */
   } else {
     rv64_emit32(mc, rv_slli(tmp, r, 32));
     rv64_emit32(mc, rv_srli(tmp, tmp, 32));
   }
   return tmp;
 }
 
 static int cmp_is_signed(CmpOp op) {
   switch (op) {
     case CMP_LT_U:
     case CMP_LE_U:
     case CMP_GT_U:
     case CMP_GE_U:
       return 0;
     default:
       return 1;
   }
 }
 
 /* Emit a 0/1 comparison result into rd from two integer registers. */
 static void rv_emit_icmp(NativeTarget* t, CmpOp op, u32 rd, u32 ra, u32 rb) {
   MCEmitter* mc = t->mc;
   switch (op) {
     case CMP_EQ:
       rv64_emit32(mc, rv_sub(rd, ra, rb));
       rv64_emit32(mc, rv_sltiu(rd, rd, 1));
       return;
     case CMP_NE:
       rv64_emit32(mc, rv_sub(rd, ra, rb));
       rv64_emit32(mc, rv_sltu(rd, RV_ZERO, rd));
       return;
     case CMP_LT_S:
       rv64_emit32(mc, rv_slt(rd, ra, rb));
       return;
     case CMP_LT_U:
       rv64_emit32(mc, rv_sltu(rd, ra, rb));
       return;
     case CMP_GT_S:
       rv64_emit32(mc, rv_slt(rd, rb, ra));
       return;
     case CMP_GT_U:
       rv64_emit32(mc, rv_sltu(rd, rb, ra));
       return;
     case CMP_GE_S:
       rv64_emit32(mc, rv_slt(rd, ra, rb));
       rv64_emit32(mc, rv_xori(rd, rd, 1));
       return;
     case CMP_GE_U:
       rv64_emit32(mc, rv_sltu(rd, ra, rb));
       rv64_emit32(mc, rv_xori(rd, rd, 1));
       return;
     case CMP_LE_S:
       rv64_emit32(mc, rv_slt(rd, rb, ra));
       rv64_emit32(mc, rv_xori(rd, rd, 1));
       return;
     case CMP_LE_U:
       rv64_emit32(mc, rv_sltu(rd, rb, ra));
       rv64_emit32(mc, rv_xori(rd, rd, 1));
       return;
     default:
       rv_panic(rv_of(t), "unsupported integer cmp");
   }
 }
 
 /* Format-dispatching wrappers over the ordered FP compares (feq/flt/fle are
  * ordered: they yield 0 on NaN; flt/fle are signaling, raising NV on NaN —
  * pre-existing for ordered ops, and the boolean result is still correct). */
 static u32 rv_feq_fmt(u32 fmt, u32 rd, u32 ra, u32 rb) {
   return fmt == RV_FMT_D ? rv_feq_d(rd, ra, rb) : rv_feq_s(rd, ra, rb);
 }
 static u32 rv_flt_fmt(u32 fmt, u32 rd, u32 ra, u32 rb) {
   return fmt == RV_FMT_D ? rv_flt_d(rd, ra, rb) : rv_flt_s(rd, ra, rb);
 }
 static u32 rv_fle_fmt(u32 fmt, u32 rd, u32 ra, u32 rb) {
   return fmt == RV_FMT_D ? rv_fle_d(rd, ra, rb) : rv_fle_s(rd, ra, rb);
 }
 
 static void rv_cmp(NativeTarget* t, CmpOp op, NativeLoc dst, NativeLoc aop,
                    NativeLoc bop) {
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst);
   /* FP-ness is self-describing from the opcode (FP block starts at CMP_OEQ_F).
    * Unordered predicates use unordered-R == NOT(ordered-not-R): the ordered
    * compare into rd, then `xori rd,rd,1`. ONE/UEQ have no single ordered
    * primitive and OR the two strict relations (a<b | a>b) via scratch RV_TMP2
    * (x7, reserved & never allocable, so it can't alias rd). */
   if (op >= CMP_OEQ_F) {
     u32 fmt = native_type_size(t, aop.type) == 8u ? RV_FMT_D : RV_FMT_S;
     u32 ra = loc_reg(aop), rb = loc_reg(bop);
     switch (op) {
       case CMP_OEQ_F:
         rv64_emit32(mc, rv_feq_fmt(fmt, rd, ra, rb));
         return;
       case CMP_UNE_F: /* !(OEQ) */
         rv64_emit32(mc, rv_feq_fmt(fmt, rd, ra, rb));
         rv64_emit32(mc, rv_xori(rd, rd, 1));
         return;
       case CMP_OLT_F:
         rv64_emit32(mc, rv_flt_fmt(fmt, rd, ra, rb));
         return;
       case CMP_OLE_F:
         rv64_emit32(mc, rv_fle_fmt(fmt, rd, ra, rb));
         return;
       case CMP_OGT_F:
         rv64_emit32(mc, rv_flt_fmt(fmt, rd, rb, ra));
         return;
       case CMP_OGE_F:
         rv64_emit32(mc, rv_fle_fmt(fmt, rd, rb, ra));
         return;
       case CMP_UGE_F: /* !(OLT) */
         rv64_emit32(mc, rv_flt_fmt(fmt, rd, ra, rb));
         rv64_emit32(mc, rv_xori(rd, rd, 1));
         return;
       case CMP_UGT_F: /* !(OLE) */
         rv64_emit32(mc, rv_fle_fmt(fmt, rd, ra, rb));
         rv64_emit32(mc, rv_xori(rd, rd, 1));
         return;
       case CMP_ULE_F: /* !(OGT) */
         rv64_emit32(mc, rv_flt_fmt(fmt, rd, rb, ra));
         rv64_emit32(mc, rv_xori(rd, rd, 1));
         return;
       case CMP_ULT_F: /* !(OGE) */
         rv64_emit32(mc, rv_fle_fmt(fmt, rd, rb, ra));
         rv64_emit32(mc, rv_xori(rd, rd, 1));
         return;
       case CMP_ONE_F: /* ordered & !=: (a<b) | (a>b) */
         rv64_emit32(mc, rv_flt_fmt(fmt, rd, ra, rb));
         rv64_emit32(mc, rv_flt_fmt(fmt, RV_TMP2, rb, ra));
         rv64_emit32(mc, rv_or(rd, rd, RV_TMP2));
         return;
       case CMP_UEQ_F: /* unordered | ==: !((a<b) | (a>b)) */
         rv64_emit32(mc, rv_flt_fmt(fmt, rd, ra, rb));
         rv64_emit32(mc, rv_flt_fmt(fmt, RV_TMP2, rb, ra));
         rv64_emit32(mc, rv_or(rd, rd, RV_TMP2));
         rv64_emit32(mc, rv_xori(rd, rd, 1));
         return;
       default:
         rv_panic(rv_of(t), "unsupported fp cmp");
     }
   }
   {
     int sg = cmp_is_signed(op);
     u32 ra = rv_cmp_ext(t, sg, aop, RV_TMP0);
     u32 rb = rv_cmp_ext(t, sg, bop, RV_TMP1);
     rv_emit_icmp(t, op, rd, ra, rb);
   }
 }
 
 static void rv_convert(NativeTarget* t, ConvKind op, NativeLoc dst,
                        NativeLoc src) {
   const RiscvVariant* v = rv_of(t)->variant;
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst), rs = loc_reg(src);
   u32 src_sz = native_type_size(t, src.type);
   u32 dst_sz = native_type_size(t, dst.type);
   /* `il` (int-side wide): the 64-bit-integer fcvt L-forms are RV64-only; on
    * rv32 only the w/wu forms exist and a 64-bit int<->fp is legalized to a
    * libcall before reaching here. */
   int il = v->has_w_forms;
   switch (op) {
     case CV_SEXT:
       if (src_sz >= 4u) {
         /* ADDIW sign-extends bits[31:0] into a 64-bit reg (RV64). On rv32 a
          * 4-byte value already spans the whole register, so a plain move (or
          * nothing when rd==rs) is the sign extension. */
         if (v->has_w_forms)
           rv64_emit32(mc, rv_addiw(rd, rs, 0));
         else if (rd != rs)
           rv64_emit32(mc, rv_addi(rd, rs, 0));
       } else {
         u32 sh = v->xlen - src_sz * 8u;
         rv64_emit32(mc, rv_slli(rd, rs, sh));
         rv64_emit32(mc, rv_srai(rd, rd, sh));
       }
       return;
     case CV_ZEXT: {
       u32 sh = v->xlen - src_sz * 8u;
       rv64_emit32(mc, rv_slli(rd, rs, sh));
       rv64_emit32(mc, rv_srli(rd, rd, sh));
       return;
     }
     case CV_TRUNC:
       if (rd != rs || dst_sz <= 4u)
         rv64_emit32(mc, rv_addi(rd, rs, 0)); /* low bits; users re-narrow */
       return;
     case CV_ITOF_S:
       if (native_type_size(t, dst.type) == 8u)
         rv64_emit32(mc, il && src_sz == 8u ? rv_fcvt_d_l(rd, rs)
                                            : rv_fcvt_d_w(rd, rs));
       else
         rv64_emit32(mc, il && src_sz == 8u ? rv_fcvt_s_l(rd, rs)
                                            : rv_fcvt_s_w(rd, rs));
       return;
     case CV_ITOF_U:
       if (native_type_size(t, dst.type) == 8u)
         rv64_emit32(mc, il && src_sz == 8u ? rv_fcvt_d_lu(rd, rs)
                                            : rv_fcvt_d_wu(rd, rs));
       else
         rv64_emit32(mc, il && src_sz == 8u ? rv_fcvt_s_lu(rd, rs)
                                            : rv_fcvt_s_wu(rd, rs));
       return;
     case CV_FTOI_S:
       if (src_sz == 8u)
         rv64_emit32(mc, il && dst_sz == 8u ? rv_fcvt_l_d(rd, rs)
                                            : rv_fcvt_w_d(rd, rs));
       else
         rv64_emit32(mc, il && dst_sz == 8u ? rv_fcvt_l_s(rd, rs)
                                            : rv_fcvt_w_s(rd, rs));
       return;
     case CV_FTOI_U:
       if (src_sz == 8u)
         rv64_emit32(mc, il && dst_sz == 8u ? rv_fcvt_lu_d(rd, rs)
                                            : rv_fcvt_wu_d(rd, rs));
       else
         rv64_emit32(mc, il && dst_sz == 8u ? rv_fcvt_lu_s(rd, rs)
                                            : rv_fcvt_wu_s(rd, rs));
       return;
     case CV_FEXT:
       rv64_emit32(mc, rv_fcvt_d_s(rd, rs));
       return;
     case CV_FTRUNC:
       rv64_emit32(mc, rv_fcvt_s_d(rd, rs));
       return;
     case CV_BITCAST:
       rv_move(t, dst, src);
       return;
     default:
       rv_panic(rv_of(t), "unsupported convert");
   }
 }
 
 /* ============================ spill / reload ============================ */
 
 static void rv_spill(NativeTarget* t, NativeLoc src, NativeFrameSlot slot,
                      MemAccess mem) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = slot;
   addr.base_type = src.type;
   rv_emit_mem(rv_of(t), 0, src, addr, mem);
 }
 static void rv_reload(NativeTarget* t, NativeLoc dst, NativeFrameSlot slot,
                       MemAccess mem) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = slot;
   addr.base_type = dst.type;
   rv_emit_mem(rv_of(t), 1, dst, addr, mem);
 }
 
 /* ============================ control flow ============================ */
 
 static MCLabel rv_label_new(NativeTarget* t) { return t->mc->label_new(t->mc); }
 static void rv_label_place(NativeTarget* t, MCLabel l) {
   t->mc->label_place(t->mc, l);
 }
 static void rv_jump(NativeTarget* t, MCLabel l) {
   rv64_emit32(t->mc, rv_jal(RV_ZERO, 0));
   t->mc->emit_label_ref(t->mc, l, R_RV_JAL, 4, 0);
 }
 
 static void rv_cmp_branch(NativeTarget* t, CmpOp op, NativeLoc aop,
                           NativeLoc bop, MCLabel l) {
   MCEmitter* mc = t->mc;
   /* RISC-V B-type branches reach only ±4 KiB, which a single (especially
    * -O0) function can exceed between a branch and its target. Rather than a
    * lone conditional branch to the label, emit a short *inverted* branch
    * that skips an unconditional `jal` (±1 MiB) to the target. The inverted
    * branch's displacement is the constant SKIP_JAL (skip just the jal) and
    * so is always in range; the jal carries the long reach. See rv_jump. */
   enum { SKIP_JAL = 8 }; /* branch over the 4-byte jal that follows it */
   /* FP compares have no register-register branch form: materialize the 0/1
    * into TMP0 via rv_cmp (handles all 12 predicates), then branch on nonzero.
    */
   if (op >= CMP_OEQ_F) {
     NativeLoc tmp =
         native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT, RV_TMP0);
     rv_cmp(t, op, tmp, aop, bop);
     /* Skip the jal when the result is 0 (condition false). */
     rv64_emit32(mc, rv_beq(RV_TMP0, RV_ZERO, SKIP_JAL));
     rv_jump(t, l);
     return;
   }
   {
     int sg = cmp_is_signed(op);
     u32 ra = rv_cmp_ext(t, sg, aop, RV_TMP0);
     u32 rb = rv_cmp_ext(t, sg, bop, RV_TMP1);
     u32 word;
     /* Encode the *inverse* of `op`, skipping the jal when NOT taken. */
     switch (op) {
       case CMP_EQ:
         word = rv_bne(ra, rb, SKIP_JAL);
         break;
       case CMP_NE:
         word = rv_beq(ra, rb, SKIP_JAL);
         break;
       case CMP_LT_S:
         word = rv_bge(ra, rb, SKIP_JAL);
         break;
       case CMP_GE_S:
         word = rv_blt(ra, rb, SKIP_JAL);
         break;
       case CMP_LT_U:
         word = rv_bgeu(ra, rb, SKIP_JAL);
         break;
       case CMP_GE_U:
         word = rv_bltu(ra, rb, SKIP_JAL);
         break;
       case CMP_GT_S:
         word = rv_bge(rb, ra, SKIP_JAL);
         break;
       case CMP_LE_S:
         word = rv_blt(rb, ra, SKIP_JAL);
         break;
       case CMP_GT_U:
         word = rv_bgeu(rb, ra, SKIP_JAL);
         break;
       case CMP_LE_U:
         word = rv_bltu(rb, ra, SKIP_JAL);
         break;
       default:
         rv_panic(rv_of(t), "unsupported cmp_branch");
     }
     rv64_emit32(mc, word);
     rv_jump(t, l);
   }
 }
 
 static void rv_indirect_branch(NativeTarget* t, NativeLoc addr,
                                const MCLabel* valid_targets, u32 ntargets) {
   (void)valid_targets;
   (void)ntargets;
   rv64_emit32(t->mc, rv_jalr(RV_ZERO, loc_reg(addr), 0));
 }
 
 static void rv_load_label_addr(NativeTarget* t, NativeLoc dst, MCLabel l) {
   /* `&&label` address-take: auipc/addi with a %pcrel_hi/%pcrel_lo relocation
    * pair against the label's per-block local symbol — the same form
    * rv_emit_global_addr uses for a global — so a compressing/re-encoding
    * assembler recomputes the displacement (a baked offset would break under
    * the C extension). */
   MCEmitter* mc = t->mc;
   u32 rd = loc_reg(dst);
   u32 sec = mc->section_id;
   ObjSymId sym = mc_label_symbol(mc, l);
   u32 ap = mc->pos(mc);
   rv64_emit32(mc, rv_auipc(rd, 0));
   mc->emit_reloc_at(mc, sec, ap, R_RV_PCREL_HI20, sym, 0, 0, 0);
   {
     Sym an = pool_intern_slice(t->c->global, SLICE_LIT(".LpcrelHi"));
     ObjSymId anchor = obj_symbol(t->obj, an, SB_LOCAL, SK_OBJ, sec, (u64)ap, 0);
     u32 lp = mc->pos(mc);
     rv64_emit32(mc, rv_addi(rd, rd, 0));
     mc->emit_reloc_at(mc, sec, lp, R_RV_PCREL_LO12_I, anchor, 0, 0, 0);
   }
 }
 
 /* ============================ frame / lifecycle ============================
  */
 
 static NativeFrameSlot rv_frame_slot(NativeTarget* t,
                                      const NativeFrameSlotDesc* d) {
   return native_frame_slot_alloc(&rv_of(t)->frame, d);
 }
 
 static int rv_frame_slot_debug_loc(NativeTarget* t, NativeFrameSlot slot,
                                    CGDebugLoc* out) {
   RvNativeTarget* a = rv_of(t);
   RvNativeSlot* s;
   if (!out) return 0;
   memset(out, 0, sizeof *out);
   if (slot == NATIVE_FRAME_SLOT_NONE || slot > a->frame.nslots) return 0;
   s = rv_slot_get(a, slot);
   out->kind = CG_DEBUG_LOC_FRAME;
   /* rv64 slots are addressed s0/fp-relative (rv_s0_off_slot); the hosted dbg
    * snapshot seeds the frame base with s0, matching aa64's FP-relative
    * convention. */
   out->v.frame_ofs = rv_s0_off_slot(s);
   return 1;
 }
 
 static void rv_func_begin_common(NativeTarget* t, const CGFuncDesc* fd) {
   RvNativeTarget* a = rv_of(t);
   MCEmitter* mc = t->mc;
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, fd->fn_type);
   a->func = fd;
   a->loc = fd->loc;
   /* Shared frame bookkeeping: clears the slot table, cum_off, max_outgoing,
    * callee-save set, and known_frame/has_alloca/frame_final. */
   native_frame_reset(&a->frame);
   a->incoming_stack_size = 0;
   a->next_param_int = 0;
   a->next_param_fp = 0;
   a->next_param_stack = 0;
   a->has_sret = (abi && abi->has_sret) ? 1u : 0u;
   a->is_variadic = (abi && abi->variadic) ? 1u : 0u;
   a->sret_ptr_slot = NATIVE_FRAME_SLOT_NONE;
   a->npatches = 0;
   a->nalloca = 0;
   a->minimal_prologue_words = 0;
   a->slim_prologue = 0;
 
   mc->set_section(mc, fd->text_section_id);
   mc->emit_align(mc, 4, 0);
   a->func_start = mc->pos(mc);
   mc_begin_function(mc, fd->sym, fd->text_section_id, a->func_start);
   if (mc->cfi_startproc) mc->cfi_startproc(mc);
   a->epilogue_label = mc->label_new(mc);
 }
 
 /* sret: reserve a hidden slot for the incoming destination pointer (a0). */
 static void rv_reserve_entry_saves(RvNativeTarget* a) {
   NativeTarget* t = &a->base;
   if (a->has_sret) {
     NativeFrameSlotDesc sd;
     u32 ptr = a->variant->ptr_bytes;
     memset(&sd, 0, sizeof sd);
     sd.type = builtin_id(KIT_CG_BUILTIN_I64);
     sd.size = ptr; /* a pointer slot: 8 on rv64, 4 on rv32 */
     sd.align = ptr;
     sd.kind = NATIVE_FRAME_SLOT_SAVE;
     a->sret_ptr_slot = t->frame_slot(t, &sd);
     a->next_param_int = 1; /* a0 consumed by the sret pointer */
   }
 }
 
 static void rv_emit_entry_save_stores(RvNativeTarget* a) {
   NativeTarget* t = &a->base;
   if (a->has_sret && a->sret_ptr_slot != NATIVE_FRAME_SLOT_NONE) {
     KitCgTypeId i64t = builtin_id(KIT_CG_BUILTIN_I64);
     u32 ptr = a->variant->ptr_bytes;
     NativeAddr addr;
     memset(&addr, 0, sizeof addr);
     addr.base_kind = NATIVE_ADDR_BASE_FRAME;
     addr.base.frame = a->sret_ptr_slot;
     addr.base_type = i64t;
     rv_emit_mem(a, 0, native_loc_reg(i64t, NATIVE_REG_INT, RV_A0), addr,
                 native_mem_for_type(t, i64t, ptr));
   }
 }
 
 /* Collect the callee-saves the body used (none at -O0). */
 static u32 rv_collect_int_saves(RvNativeTarget* a, u32* regs) {
   u32 n = 0, i;
   for (i = 0; i < a->frame.ncallee_saves; ++i)
     if (a->frame.callee_saves[i].cls == NATIVE_REG_INT)
       regs[n++] = a->frame.callee_saves[i].reg;
   return n;
 }
 static u32 rv_collect_fp_saves(RvNativeTarget* a, u32* regs) {
   u32 n = 0, i;
   for (i = 0; i < a->frame.ncallee_saves; ++i)
     if (a->frame.callee_saves[i].cls == NATIVE_REG_FP)
       regs[n++] = a->frame.callee_saves[i].reg;
   return n;
 }
 
 /* s0-relative offset of a saved register, below the locals. The flat index runs
  * 0..n_int-1 over integer saves (each ptr_bytes wide) then n_int..n_int+n_fp-1
  * over fp saves (each 8 bytes wide, fsd). On rv64 ptr_bytes==8 so this reduces
  * to the historical uniform -cum_off-8-8*idx layout, byte-for-byte. */
 static i32 rv_save_off(RvNativeTarget* a, u32 n_int, u32 idx) {
   i32 base = -(i32)(a->frame.cum_off);
   u32 ptr = a->variant->ptr_bytes;
   if (idx < n_int) return base - (i32)ptr * (i32)(idx + 1u);
   return base - (i32)(ptr * n_int) - 8 * (i32)(idx - n_int + 1u);
 }
 
 static void rv_load_s0(const RiscvVariant* v, MCEmitter* mc, int fp, u32 reg,
                        i32 off) {
   if (fits_i12(off)) {
     rv64_emit32(mc, fp ? rv_fld(reg, RV_S0, off)
                        : rv_ld_ptr(v, reg, RV_S0, off));
     return;
   }
   rv_emit_load_imm(v, mc, 1, RV_TMP0, (i64)off);
   rv64_emit32(mc, rv_add(RV_TMP0, RV_S0, RV_TMP0));
   rv64_emit32(mc, fp ? rv_fld(reg, RV_TMP0, 0) : rv_ld_ptr(v, reg, RV_TMP0, 0));
 }
 
 /* Build the prologue instruction sequence into words[]. Returns count. */
 static u32 rv_build_prologue(RvNativeTarget* a, u32* words, u32 cap,
                              u32 frame_size, u32 fp_pair_off,
                              const u32* int_regs, u32 n_int, const u32* fp_regs,
                              u32 n_fp) {
   const RiscvVariant* v = a->variant;
   u32 ptr = v->ptr_bytes;                 /* saved-pair / int-save stride */
   u32 gp_slot = v->gp_slot_bytes;         /* vararg GP-slot stride */
   u32 fsz = v->frame_save_size;           /* saved ra+s0 pair base offset */
   u32 wi = 0;
   /* lui+ADD{I,IW} materializes a 32-bit constant in TMP0; ADDIW is RV64-only so
    * use plain ADDI on rv32 (the value already fits 32 bits). */
 #define ADDI_LO(rd, lo) (v->has_w_forms ? rv_addiw((rd), (rd), (lo)) : rv_addi((rd), (rd), (lo)))
 #define PUSH(w)                                                  \
   do {                                                           \
     if (wi >= cap) rv_panic(a, "prologue placeholder overflow"); \
     words[wi++] = (w);                                           \
   } while (0)
   /* sp -= frame_size */
   if (fits_i12(-(i32)frame_size)) {
     PUSH(rv_addi(RV_SP, RV_SP, -(i32)frame_size));
   } else {
     i32 neg = -(i32)frame_size;
     i32 hi = (i32)(((i64)neg + 0x800) >> 12);
     i32 lo = neg - (i32)((u32)hi << 12);
     PUSH(rv_lui(RV_TMP0, (u32)hi & 0xfffffu));
     if (lo) PUSH(ADDI_LO(RV_TMP0, lo));
     PUSH(rv_add(RV_SP, RV_SP, RV_TMP0));
   }
   /* save s0/ra at [sp + fp_pair_off], set s0 = sp + fp_pair_off. The saved-pair
    * internal stride is ptr_bytes (s0 at +0, ra at +ptr). */
   if (fits_i12((i32)fp_pair_off + (i32)ptr)) {
     PUSH(rv_sd_ptr(v, RV_S0, RV_SP, (i32)fp_pair_off));
     PUSH(rv_sd_ptr(v, RV_RA, RV_SP, (i32)fp_pair_off + (i32)ptr));
     PUSH(rv_addi(RV_S0, RV_SP, (i32)fp_pair_off));
   } else {
     i32 off = (i32)fp_pair_off;
     i32 hi = (i32)(((i64)off + 0x800) >> 12);
     i32 lo = off - (i32)((u32)hi << 12);
     PUSH(rv_lui(RV_TMP0, (u32)hi & 0xfffffu));
     if (lo) PUSH(ADDI_LO(RV_TMP0, lo));
     PUSH(rv_add(RV_TMP0, RV_SP, RV_TMP0));
     PUSH(rv_sd_ptr(v, RV_S0, RV_TMP0, 0));
     PUSH(rv_sd_ptr(v, RV_RA, RV_TMP0, (i32)ptr));
     PUSH(rv_addi(RV_S0, RV_TMP0, 0));
   }
   /* sret a0 spill */
   if (a->has_sret && a->sret_ptr_slot != NATIVE_FRAME_SLOT_NONE) {
     RvNativeSlot* s = rv_slot_get(a, a->sret_ptr_slot);
     PUSH(rv_sd_ptr(v, RV_A0, RV_S0, rv_s0_off_slot(s)));
   }
   /* variadic GP save area: spill unconsumed a-regs at
    * [s0 + frame_save_size + i*gp_slot_bytes] */
   if (a->is_variadic) {
     u32 i;
     for (i = a->next_param_int; i < 8u; ++i)
       PUSH(rv_sd_ptr(v, RV_A0 + i, RV_S0, (i32)fsz + (i32)i * (i32)gp_slot));
   }
   /* callee saves: integer with the pointer-width store (sw/sd), fp with fsd. */
   {
     u32 i;
     for (i = 0; i < n_int; ++i) {
       i32 off = rv_save_off(a, n_int, i);
       if (fits_i12(off)) {
         PUSH(rv_sd_ptr(v, int_regs[i], RV_S0, off));
       } else {
         /* rare; emitted directly is fine in the known-frame path, but the
          * single-pass placeholder must hold these too. Use the wide form. */
         i32 hi = (i32)(((i64)off + 0x800) >> 12);
         i32 lo = off - (i32)((u32)hi << 12);
         PUSH(rv_lui(RV_TMP0, (u32)hi & 0xfffffu));
         if (lo) PUSH(ADDI_LO(RV_TMP0, lo));
         PUSH(rv_add(RV_TMP0, RV_S0, RV_TMP0));
         PUSH(rv_sd_ptr(v, int_regs[i], RV_TMP0, 0));
       }
     }
     for (i = 0; i < n_fp; ++i) {
       i32 off = rv_save_off(a, n_int, n_int + i);
       if (fits_i12(off)) {
         PUSH(rv_fsd(fp_regs[i], RV_S0, off));
       } else {
         i32 hi = (i32)(((i64)off + 0x800) >> 12);
         i32 lo = off - (i32)((u32)hi << 12);
         PUSH(rv_lui(RV_TMP0, (u32)hi & 0xfffffu));
         if (lo) PUSH(ADDI_LO(RV_TMP0, lo));
         PUSH(rv_add(RV_TMP0, RV_S0, RV_TMP0));
         PUSH(rv_fsd(fp_regs[i], RV_TMP0, 0));
       }
     }
   }
 #undef PUSH
 #undef ADDI_LO
   return wi;
 }
 
 static void rv_func_begin(NativeTarget* t, const CGFuncDesc* fd) {
   RvNativeTarget* a = rv_of(t);
   MCEmitter* mc = t->mc;
   u32 i;
   rv_func_begin_common(t, fd);
   a->prologue_pos = mc->pos(mc);
   for (i = 0; i < RV_PROLOGUE_WORDS; ++i) rv64_emit32(mc, RV_NOP);
   rv_reserve_entry_saves(a);
   rv_emit_entry_save_stores(a);
 }
 
 static void rv_func_end(NativeTarget* t) {
   RvNativeTarget* a = rv_of(t);
   MCEmitter* mc = t->mc;
   ObjBuilder* obj = t->obj;
   ObjSecId sec = a->func->text_section_id;
   u32 int_regs[16], fp_regs[16];
   u32 n_int = rv_collect_int_saves(a, int_regs);
   u32 n_fp = rv_collect_fp_saves(a, fp_regs);
   u32 frame_size = rv_frame_size(a);
   u32 fp_pair_off = rv_fp_pair_off(a, frame_size);
   u32 end;
   i32 i;
   a->frame_size_final = frame_size;
   a->fp_pair_off = fp_pair_off;
 
   /* epilogue */
   mc->label_place(mc, a->epilogue_label);
   if (a->slim_prologue) {
     /* Frameless leaf: no callee-saves, no s0/ra to reload, sp untouched. */
     rv64_emit32(mc, rv_jalr(RV_ZERO, RV_RA, 0));
   } else {
     const RiscvVariant* v = a->variant;
     for (i = (i32)n_int - 1; i >= 0; --i)
       rv_load_s0(v, mc, 0, int_regs[i], rv_save_off(a, n_int, (u32)i));
     for (i = (i32)n_fp - 1; i >= 0; --i)
       rv_load_s0(v, mc, 1, fp_regs[i], rv_save_off(a, n_int, n_int + (u32)i));
     if (a->frame.has_alloca)
       rv_emit_addr_adjust(v, mc, RV_SP, RV_S0, -(i32)fp_pair_off);
     /* Reload ra/s0 from the saved pair (s0 at +0, ra at +ptr_bytes), pointer
      * width. */
     rv64_emit32(mc, rv_ld_ptr(v, RV_RA, RV_S0, (i32)v->ptr_bytes));
     rv64_emit32(mc, rv_ld_ptr(v, RV_S0, RV_S0, 0));
     /* sp += frame_size */
     if (fits_i12((i32)frame_size)) {
       rv64_emit32(mc, rv_addi(RV_SP, RV_SP, (i32)frame_size));
     } else {
       rv_emit_load_imm(v, mc, 1, RV_TMP0, (i64)frame_size);
       rv64_emit32(mc, rv_add(RV_SP, RV_SP, RV_TMP0));
     }
     rv64_emit32(mc, rv_jalr(RV_ZERO, RV_RA, 0));
   }
 
   /* patch prologue */
   if (!a->frame.known_frame) {
     u32 words[RV_PROLOGUE_WORDS];
     u32 nwords, k;
     for (k = 0; k < RV_PROLOGUE_WORDS; ++k) words[k] = RV_NOP;
     nwords = rv_build_prologue(a, words, RV_PROLOGUE_WORDS, frame_size,
                                fp_pair_off, int_regs, n_int, fp_regs, n_fp);
     (void)nwords;
     for (k = 0; k < RV_PROLOGUE_WORDS; ++k)
       rv_patch32(obj, sec, a->prologue_pos + k * 4u, words[k]);
   }
   /* patch alloca sites: addi dst, sp, max_outgoing */
   {
     u32 mo = align_up_u32(a->frame.max_outgoing, 16u);
     u32 k;
     if (mo > 2047u) rv_panic(a, "max_outgoing too large for alloca patch");
     for (k = 0; k < a->npatches; ++k)
       rv_patch32(obj, sec, a->patches[k].pos,
                  rv_addi(a->patches[k].dst_reg, RV_SP, (i32)mo));
   }
 
   /* CFI: CFA = s0 + (frame_size - fp_pair_off) */
   if (mc->cfi_set_next_pc_offset && mc->cfi_def_cfa && mc->cfi_offset) {
     if (a->slim_prologue) {
       /* Frameless leaf: CFA = sp (unchanged from entry) and the return address
        * stays live in ra (the CIE default), so no saved-register rules. The
        * state holds from the first instruction (offset 0). */
       mc->cfi_set_next_pc_offset(mc, 0);
       mc->cfi_def_cfa(mc, RV_SP, 0);
     } else {
       i32 cfa = (i32)frame_size - (i32)fp_pair_off;
       u32 post = a->prologue_pos + (a->frame.known_frame
                                         ? a->minimal_prologue_words * 4u
                                         : RV_PROLOGUE_WORDS * 4u);
       u32 k;
       mc->cfi_set_next_pc_offset(mc, post - a->func_start);
       mc->cfi_def_cfa(mc, RV_S0, cfa);
       mc->cfi_offset(mc, RV_S0, -cfa);
       /* ra is saved at the saved-pair stride above s0 (ptr_bytes). */
       mc->cfi_offset(mc, RV_RA, -cfa + (i32)a->variant->ptr_bytes);
       for (k = 0; k < n_int; ++k)
         mc->cfi_offset(mc, int_regs[k], rv_save_off(a, n_int, k) - cfa);
       for (k = 0; k < n_fp; ++k)
         mc->cfi_offset(mc, 32u + fp_regs[k],
                        rv_save_off(a, n_int, n_int + k) - cfa);
     }
   }
 
   end = mc->pos(mc);
   obj_symbol_define(obj, a->func->sym, sec, (u64)a->func_start,
                     (u64)(end - a->func_start));
   if (a->func->atomize)
     obj_atom_define(obj, sec, a->func_start, end - a->func_start, a->func->sym,
                     0);
   if (mc->debug) debug_func_pc_range(mc->debug, sec, a->func_start, end);
   if (mc->cfi_endproc) mc->cfi_endproc(mc);
   mc_end_function(mc);
   a->func = NULL;
 }
 
 /* rv64 homes its callee-saves below the locals at rv_save_off(idx) rather than
  * in frame slots, so alloc_slots=0: native_frame just records the {reg,cls} set
  * derived from the optimizer's per-class used-masks. */
 static void rv_reserve_callee_saves(NativeTarget* t, const u32* used,
                                     u32 nclasses) {
   native_frame_set_callee_saves(&rv_of(t)->frame, used, nclasses, NULL, 0, 0);
 }
 
 static int rv_reg_is_callee_int(Reg r);
 static int rv_reg_is_callee_fp(Reg r);
 static void rv_asm_clobber_masks(Compiler* c, SrcLoc loc, const Sym* clobbers,
                                  u32 nclob, u32* int_mask, u32* fp_mask);
 
 /* Expand the arch-neutral clobber-ABI sets (KitCgAsmClobberAbiSet bits) into
  * this target's per-class caller/callee-saved register masks. */
 /* abi_clobber_masks is shared as native_asm_abi_clobber_masks
  * (cg/native_asm.h); it reads the masks from t->regs->classes. */
 
 /* Build the callee-saved set the prologue must preserve: the allocator-assigned
  * callee-saved registers (frame->callee_saved_used) plus any an inline-asm
  * block clobbers. The latter are opaque to the optimizer's operand scan, so it
  * forwards the raw clobber names (frame->asm_clobbers) and the arch-neutral
  * clobber-ABI sets (frame->asm_clobber_abi_sets); we resolve both into masks
  * and keep only the callee-saved ones — rv_reg_is_callee_int excludes s0 (the
  * frame pointer, preserved by the prologue head, not as an ordinary
  * callee-save). This is the same register selection the per-block spill used,
  * hoisted into the prologue. Writes up to `cap` per-class masks into `out` and
  * returns the class count to reserve. */
 static u32 rv_known_callee_saves(NativeTarget* t,
                                  const NativeKnownFrameDesc* frame, u32* out,
                                  u32 cap) {
   u32 ncls = frame->ncallee_classes;
   u32 clob_int = 0, clob_fp = 0, abi_int, abi_fp;
   if (ncls > cap) ncls = cap;
   for (u32 c = 0; c < ncls; ++c)
     out[c] = frame->callee_saved_used ? frame->callee_saved_used[c] : 0u;
   if (frame->asm_clobbers && frame->nasm_clobbers) {
     RvNativeTarget* a = rv_of(t);
     SrcLoc loc = a->func ? a->func->loc : (SrcLoc){0, 0, 0};
     rv_asm_clobber_masks(t->c, loc, frame->asm_clobbers, frame->nasm_clobbers,
                          &clob_int, &clob_fp);
   }
   native_asm_abi_clobber_masks(t, frame->asm_clobber_abi_sets, &abi_int,
                                &abi_fp);
   clob_int |= abi_int;
   clob_fp |= abi_fp;
   for (Reg r = 0; r < 32u; ++r) {
     if (NATIVE_REG_INT < ncls && (clob_int & (1u << r)) &&
         rv_reg_is_callee_int(r))
       out[NATIVE_REG_INT] |= 1u << r;
     if (NATIVE_REG_FP < ncls && (clob_fp & (1u << r)) && rv_reg_is_callee_fp(r))
       out[NATIVE_REG_FP] |= 1u << r;
   }
   return ncls;
 }
 
 static u32 rv_signature_stack_bytes(NativeTarget* t, KitCgTypeId fn_type,
                                     int* variadic, u32* nparams);
 
 /* Optimizer entry point: the full frame is supplied up front, so the prologue
  * is emitted final the moment it is built — no NOP region, no func_end patch
  * (rv_func_end skips patching when known_frame). rv_build_prologue emits the
  * sret spill and the variadic register-save stores inline, so there is no
  * separate entry-save emission. Slot creation order matches the single-pass
  * path: callee-saves first (only recorded for rv64), then static slots, then
  * the sret entry-save slot. */
 static void rv_func_begin_known_frame(NativeTarget* t, const CGFuncDesc* fd,
                                       const NativeKnownFrameDesc* frame,
                                       NativeFrameSlot* out_slots) {
   RvNativeTarget* a = rv_of(t);
   MCEmitter* mc = t->mc;
   u32 int_regs[RV_MAX_CALLEE_SAVES], fp_regs[RV_MAX_CALLEE_SAVES];
   u32 n_int, n_fp, frame_size, fp_pair_off, nwords, i;
   u32 words[RV_KNOWN_PROLOGUE_WORDS];
   rv_func_begin_common(t, fd);
   a->frame.known_frame = 1;
   if (frame) {
     u32 cs[NATIVE_CALL_PLAN_CLASSES];
     u32 ncs = rv_known_callee_saves(t, frame, cs, NATIVE_CALL_PLAN_CLASSES);
     a->frame.has_alloca = frame->has_alloca;
     if (ncs) rv_reserve_callee_saves(t, cs, ncs);
     for (i = 0; i < frame->nslots; ++i) {
       NativeFrameSlot slot = rv_frame_slot(t, &frame->slots[i]);
       if (out_slots) out_slots[i] = slot;
     }
     rv_reserve_entry_saves(a);
     native_frame_note_outgoing(&a->frame, frame->max_outgoing);
   }
   /* Frame is final: size and offsets are settled, so emit the exact prologue.
    */
   frame_size = rv_frame_size(a);
   fp_pair_off = rv_fp_pair_off(a, frame_size);
   a->frame_size_final = frame_size;
   a->fp_pair_off = fp_pair_off;
   a->prologue_pos = mc->pos(mc);
   /* Leaf no-frame tier (aa64 slim_prologue equivalent): a leaf with no
    * callee-saves, no body slots, no outgoing args, no sret/variadic and
    * register-only params never reads s0 (no frame slots / stack args) nor
    * clobbers ra (no calls). Emit no prologue at all; rv_func_end emits a bare
    * `ret`. cum_off==0 already implies no sret slot and no param spills, but the
    * extra guards keep the intent explicit. Inline asm is excluded: it can
    * clobber ra opaquely, and without the saved record the bare `ret` would
    * return through the destroyed link register. */
   a->slim_prologue = frame && frame->is_leaf && !frame->has_asm &&
                      !frame->reads_frame && a->frame.ncallee_saves == 0 &&
                      !a->frame.has_alloca && a->frame.cum_off == 0 &&
                      a->frame.max_outgoing == 0 && !a->has_sret &&
                      !a->is_variadic &&
                      rv_signature_stack_bytes(t, fd->fn_type, NULL, NULL) == 0;
   if (a->slim_prologue) {
     a->minimal_prologue_words = 0;
     native_frame_set_final(&a->frame);
     return;
   }
   n_int = rv_collect_int_saves(a, int_regs);
   n_fp = rv_collect_fp_saves(a, fp_regs);
   nwords = rv_build_prologue(a, words, RV_KNOWN_PROLOGUE_WORDS, frame_size,
                              fp_pair_off, int_regs, n_int, fp_regs, n_fp);
   for (i = 0; i < nwords; ++i) rv64_emit32(mc, words[i]);
   a->minimal_prologue_words = nwords;
   native_frame_set_final(&a->frame);
 }
 
 /* ============================ params / ABI helpers
  * ============================ */
 
 static const ABIArgInfo* rv_param_abi(NativeTarget* t, const ABIFuncInfo* abi,
                                       const NativeCallDesc* desc, u32 i,
                                       ABIArgInfo* scratch) {
   /* Synthesized for unnamed (variadic) args, or untyped calls. RISC-V LP64D
    * passes variadic FP args in INTEGER registers (as their bit pattern), not
    * the FP pool — so a variadic float part is ABI_CLASS_INT. */
   int variadic = abi && i >= abi->nparams;
   u32 gpr = rv_of(t)->variant->ptr_bytes; /* GPR width: 4 ilp32 / 8 lp64 */
   u32 sz, align;
   int is_fp;
   if (abi && i < abi->nparams) return &abi->params[i];
   sz = native_type_size(t, desc->args[i].type);
   align = native_type_align(t, desc->args[i].type);
   /* A variadic FP arg rides the INTEGER pool as its bit pattern (RISC-V passes
    * unnamed FP args in GPRs), so it is INT-class here. */
   is_fp = !variadic && cg_type_is_float(t->c, desc->args[i].type);
   memset(scratch, 0, sizeof *scratch);
   scratch->kind = ABI_ARG_DIRECT;
   /* A scalar wider than one GPR (an 8-byte i64 / soft-double on ilp32) rides a
    * register pair, matching the named-arg classifier (abi_rv64.c). Synthesize
    * one INT part per GPR-word so the per-part marshaller fills both registers
    * (low word in the lower-numbered reg) instead of dropping the high half into
    * a single register. FP-class args (hardware-float, size<=GPR) stay single. */
   if (!is_fp && sz > gpr) {
     u32 nparts = (sz + gpr - 1u) / gpr, p;
     ABIArgPart* parts = arena_zarray(t->c->tu, ABIArgPart, nparts);
     for (p = 0; p < nparts; ++p) {
       u32 off = p * gpr;
       parts[p].cls = ABI_CLASS_INT;
       parts[p].loc = ABI_LOC_REG;
       parts[p].size = (sz - off) < gpr ? (sz - off) : gpr;
       parts[p].align = gpr;
       parts[p].src_offset = off;
     }
     scratch->nparts = (u16)nparts;
     scratch->parts = parts;
     return scratch;
   }
   scratch->nparts = 1;
   scratch->parts = arena_zarray(t->c->tu, ABIArgPart, 1);
   ((ABIArgPart*)scratch->parts)[0].cls = is_fp ? ABI_CLASS_FP : ABI_CLASS_INT;
   ((ABIArgPart*)scratch->parts)[0].loc = ABI_LOC_REG;
   ((ABIArgPart*)scratch->parts)[0].size = sz;
   ((ABIArgPart*)scratch->parts)[0].align = align;
   return scratch;
 }
 
 /* Outgoing stack-slot size/align: the xlen-word (gp_slot_bytes: 8 lp64d /
  * 4 ilp32) is the natural slot stride; stack ABI alignment caps at 16. */
 static u32 rv_part_stack_size(const RiscvVariant* v, const ABIArgPart* part) {
   u32 slot = v->gp_slot_bytes;
   return align_up_u32(part->size ? part->size : slot, slot);
 }
 static u32 rv_part_stack_align(const RiscvVariant* v, const ABIArgPart* part) {
   u32 slot = v->gp_slot_bytes;
   u32 al = part->align ? part->align : slot;
   if (al < slot) al = slot;
   if (al > 16u) al = 16u;
   return al;
 }
 
 static KitCgTypeId rv_part_scalar_type(const ABIArgPart* part) {
   if (part->cls == ABI_CLASS_FP) {
     if (part->size <= 4u) return builtin_id(KIT_CG_BUILTIN_F32);
     return builtin_id(KIT_CG_BUILTIN_F64);
   }
   switch (part->size) {
     case 1u:
       return builtin_id(KIT_CG_BUILTIN_I8);
     case 2u:
       return builtin_id(KIT_CG_BUILTIN_I16);
     case 4u:
       return builtin_id(KIT_CG_BUILTIN_I32);
     default:
       return builtin_id(KIT_CG_BUILTIN_I64);
   }
 }
 
 static u32 rv_class_stack_size(const RiscvVariant* v, const ABIArgInfo* ai) {
   u32 slot = v->gp_slot_bytes;
   u32 total = 0, p;
   if (!ai || ai->kind == ABI_ARG_IGNORE) return 0;
   if (ai->kind == ABI_ARG_INDIRECT) return v->ptr_bytes;
   for (p = 0; p < ai->nparts; ++p) {
     total = align_up_u32(total, rv_part_stack_align(v, &ai->parts[p]));
     total += rv_part_stack_size(v, &ai->parts[p]);
   }
   return align_up_u32(total ? total : slot, slot);
 }
 
 static u32 rv_call_stack_size(NativeTarget* t, const NativeCallDesc* desc) {
   const RiscvVariant* v = rv_of(t)->variant;
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, desc->fn_type);
   /* sret consumes a0 as the implicit first integer argument. */
   u32 next_int = (abi && abi->has_sret) ? 1u : 0u;
   u32 next_fp = 0, stack = 0, i, p;
   for (i = 0; i < desc->nargs; ++i) {
     ABIArgInfo tmp;
     const ABIArgInfo* ai = rv_param_abi(t, abi, desc, i, &tmp);
     int force_stack =
         abi && abi->variadic && abi->vararg_on_stack && i >= abi->nparams;
     if (ai->kind == ABI_ARG_IGNORE) continue;
     if (force_stack) {
       stack += rv_class_stack_size(v, ai);
       continue;
     }
     if (ai->kind == ABI_ARG_INDIRECT) {
       if (next_int < 8u)
         next_int++;
       else
         stack += v->ptr_bytes;
       continue;
     }
     for (p = 0; p < ai->nparts; ++p) {
       const ABIArgPart* part = &ai->parts[p];
       if (part->cls == ABI_CLASS_FP) {
         if (next_fp < 8u)
           next_fp++;
         else {
           stack = align_up_u32(stack, rv_part_stack_align(v, part));
           stack += rv_part_stack_size(v, part);
         }
       } else {
         if (next_int < 8u)
           next_int++;
         else {
           stack = align_up_u32(stack, rv_part_stack_align(v, part));
           stack += rv_part_stack_size(v, part);
         }
       }
     }
   }
   return align_up_u32(stack, 16u);
 }
 
 static u32 rv_signature_stack_bytes(NativeTarget* t, KitCgTypeId fn_type,
                                     int* variadic, u32* nparams) {
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, fn_type);
   NativeCallDesc d;
   if (variadic) *variadic = abi ? (int)abi->variadic : 0;
   if (nparams) *nparams = abi ? abi->nparams : 0u;
   memset(&d, 0, sizeof d);
   d.fn_type = fn_type;
   d.nargs = abi ? abi->nparams : 0u;
   if (d.nargs) d.args = arena_zarray(t->c->tu, NativeLoc, d.nargs);
   return rv_call_stack_size(t, &d);
 }
 
 static u32 rv_call_stack_bytes(NativeTarget* t, const NativeCallDesc* desc) {
   return rv_call_stack_size(t, desc);
 }
 
 /* Resolve a NativeLoc to an addressable NativeAddr (frame/stack/addr). */
 static NativeAddr rv_loc_addr(RvNativeTarget* a, NativeLoc loc, u32 offset) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   switch ((NativeLocKind)loc.kind) {
     case NATIVE_LOC_FRAME:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = loc.v.frame;
       addr.base_type = loc.type;
       addr.offset = (i32)offset;
       return addr;
     case NATIVE_LOC_STACK:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = loc.v.stack.slot;
       addr.base_type = loc.type;
       addr.offset = loc.v.stack.offset + (i32)offset;
       return addr;
     case NATIVE_LOC_ADDR:
       addr = loc.v.addr;
       addr.offset += (i32)offset;
       return addr;
     default:
       rv_panic(a, "location is not addressable");
   }
   return addr;
 }
 
 static void rv_load_part(NativeTarget* t, NativeLoc dst, NativeLoc src,
                          u32 offset, u32 size) {
   RvNativeTarget* a = rv_of(t);
   if (src.kind == NATIVE_LOC_REG) {
     rv_move(t, dst, src);
     return;
   }
   if (src.kind == NATIVE_LOC_FRAME || src.kind == NATIVE_LOC_STACK ||
       src.kind == NATIVE_LOC_ADDR) {
     NativeAddr addr = rv_loc_addr(a, src, offset);
     addr.base_type = dst.type;
     rv_emit_mem(a, 1, dst, addr, native_mem_for_type(t, dst.type, size));
     return;
   }
   if (src.kind == NATIVE_LOC_IMM) {
     rv_emit_load_imm(a->variant, t->mc, rv_is_64(t, dst.type) ? 1u : 0u,
                      loc_reg(dst), src.v.imm);
     return;
   }
   rv_panic(a, "unsupported part source");
 }
 
 static void rv_store_part(NativeTarget* t, NativeLoc dst, NativeLoc src,
                           u32 offset, u32 size) {
   RvNativeTarget* a = rv_of(t);
   if (dst.kind == NATIVE_LOC_FRAME || dst.kind == NATIVE_LOC_STACK ||
       dst.kind == NATIVE_LOC_ADDR) {
     NativeAddr addr = rv_loc_addr(a, dst, offset);
     addr.base_type = src.type;
     rv_emit_mem(a, 0, src, addr, native_mem_for_type(t, src.type, size));
     return;
   }
   if (dst.kind == NATIVE_LOC_REG) {
     rv_move(t, dst, src);
     return;
   }
   rv_panic(a, "unsupported part destination");
 }
 
 static void rv_addr_of_loc(NativeTarget* t, NativeLoc dst, NativeLoc src) {
   NativeAddr addr = rv_loc_addr(rv_of(t), src, 0);
   rv_load_addr(t, dst, addr);
 }
 
 static void rv_store_outgoing_part(NativeTarget* t, int tail_call,
                                    u32 stack_off, NativeLoc src, u32 size) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_REG;
   addr.base_type = src.type;
   if (tail_call) {
     /* A sibling call reuses the caller's frame: its outgoing stack args land in
      * the caller's incoming-arg window ([s0 + 16 + va_save + off]) — physically
      * the same address the tail-callee will read at [sp+off] once the teardown
      * has restored sp to the caller's entry sp (the CFA). */
     addr.base.reg = RV_S0;
     addr.offset = rv_s0_off_in_arg(rv_of(t), stack_off);
   } else {
     addr.base.reg = RV_SP;
     addr.offset = (i32)stack_off;
   }
   rv_emit_mem(rv_of(t), 0, src, addr, native_mem_for_type(t, src.type, size));
 }
 
 /* NativeTarget bind_param: route incoming param (ABI loc) into dst. */
 static void rv_bind_native_param(NativeTarget* t, const CGParamDesc* p,
                                  NativeLoc dst) {
   RvNativeTarget* a = rv_of(t);
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, a->func->fn_type);
   const ABIArgInfo* ai =
       p->index < abi->nparams ? &abi->params[p->index] : NULL;
   int to_reg = dst.kind == NATIVE_LOC_REG;
   u32 i;
   if (!ai || ai->kind == ABI_ARG_IGNORE) return;
   if (ai->kind == ABI_ARG_INDIRECT) {
     NativeLoc src = native_loc_reg(
         builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT,
         a->next_param_int < 8u ? RV_A0 + a->next_param_int : RV_TMP0);
     NativeAddr d_addr, from;
     AggregateAccess access;
     if (a->next_param_int < 8u) {
       a->next_param_int++;
     } else {
       NativeAddr sa;
       memset(&sa, 0, sizeof sa);
       sa.base_kind = NATIVE_ADDR_BASE_REG;
       sa.base.reg = RV_S0;
       sa.offset = rv_s0_off_in_arg(a, a->next_param_stack);
       sa.base_type = src.type;
       rv_emit_mem(a, 1, src, sa,
                   native_mem_for_type(t, src.type, a->variant->ptr_bytes));
       a->next_param_stack += a->variant->ptr_bytes;
     }
     if (dst.kind != NATIVE_LOC_FRAME)
       rv_panic(a, "indirect parameter requires a frame destination");
     memset(&d_addr, 0, sizeof d_addr);
     d_addr.base_kind = NATIVE_ADDR_BASE_FRAME;
     d_addr.base.frame = dst.v.frame;
     d_addr.base_type = p->type;
     memset(&from, 0, sizeof from);
     from.base_kind = NATIVE_ADDR_BASE_REG;
     from.base.reg = loc_reg(src);
     from.base_type = p->type;
     memset(&access, 0, sizeof access);
     access.type = p->type;
     access.size = p->size ? p->size : (u32)cg_type_size(t->c, p->type);
     access.align = p->align ? p->align : native_type_align(t, p->type);
     rv_copy_bytes(t, d_addr, from, access);
     return;
   }
   for (i = 0; i < ai->nparts; ++i) {
     const ABIArgPart* part = &ai->parts[i];
     NativeAllocClass cls =
         part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
     NativeLoc src;
     if (cls == NATIVE_REG_FP && a->next_param_fp < 8u) {
       src = native_loc_reg(p->type, cls, RV_FA0 + a->next_param_fp++);
     } else if (cls == NATIVE_REG_INT && a->next_param_int < 8u) {
       src = native_loc_reg(p->type, cls, RV_A0 + a->next_param_int++);
     } else {
       Reg tmp = (cls == NATIVE_REG_FP) ? RV_FTMP0 : RV_TMP0;
       NativeAddr sa;
       src = native_loc_reg(p->type, cls, tmp);
       a->next_param_stack = align_up_u32(
           a->next_param_stack, rv_part_stack_align(a->variant, part));
       memset(&sa, 0, sizeof sa);
       sa.base_kind = NATIVE_ADDR_BASE_REG;
       sa.base.reg = RV_S0;
       sa.base_type = p->type;
       sa.offset = rv_s0_off_in_arg(a, a->next_param_stack);
       rv_emit_mem(a, 1, src, sa, native_mem_for_type(t, p->type, part->size));
       a->next_param_stack += rv_part_stack_size(a->variant, part);
     }
     if (dst.kind == NATIVE_LOC_NONE) {
       /* unused parameter; cursors already advanced */
     } else if (to_reg) {
       NativeLoc d = native_loc_reg(dst.type ? dst.type : p->type,
                                    (NativeAllocClass)dst.cls, (Reg)dst.v.reg);
       if (!(src.kind == NATIVE_LOC_REG && loc_reg(src) == loc_reg(d) &&
             (NativeAllocClass)src.cls == (NativeAllocClass)d.cls))
         rv_move(t, d, src);
     } else {
       rv_store_part(
           t, native_loc_stack(p->type, dst.v.frame, (i32)part->src_offset), src,
           0, part->size);
     }
   }
   a->incoming_stack_size = align_up_u32(a->next_param_stack, 16u);
 }
 
 /* ============================ calls / returns ============================ */
 
 typedef NativeArgMove RvArgMove;
 
 static void rv_emit_one_arg_move(NativeTarget* t, const NativeArgMove* m) {
   if (m->is_addr)
     rv_addr_of_loc(t, m->dst, m->src);
   else
     rv_load_part(t, m->dst, m->src, m->src_offset, m->size);
 }
 
 /* Parallel-copy register arg moves via the shared scheduler; cycles break
  * through the int/fp emit scratch (t1 / ft1). */
 static void rv_emit_reg_arg_moves(NativeTarget* t, NativeArgMove* moves,
                                   u32 n) {
   NativeArgShuffle s;
   if (n > RV_MAX_REG_ARG_MOVES) rv_panic(rv_of(t), "too many register args");
   memset(&s, 0, sizeof s);
   s.t = t;
   s.emit_one = rv_emit_one_arg_move;
   s.reg_move = rv_move;
   s.scratch[NATIVE_REG_INT] = RV_TMP1;
   s.scratch[NATIVE_REG_FP] = RV_FTMP1;
   native_arg_shuffle(&s, moves, n);
 }
 
 static void rv_plan_call(NativeTarget* t, const NativeCallDesc* desc,
                          NativeCallPlan* plan) {
   RvNativeTarget* a = rv_of(t);
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, desc->fn_type);
   NativeCallPlanRet* rets;
   KitCgTypeId i64t = builtin_id(KIT_CG_BUILTIN_I64);
   memset(plan, 0, sizeof *plan);
   rets = desc->nresults ? arena_zarray(t->c->tu, NativeCallPlanRet, 4) : NULL;
   plan->callee = desc->callee;
   plan->rets = rets;
   plan->flags = desc->flags;
   plan->has_sret = abi && abi->has_sret;
   plan->is_variadic = abi && abi->variadic;
   plan->stack_arg_size = rv_call_stack_size(t, desc);
   if (plan->stack_arg_size > a->frame.max_outgoing)
     a->frame.max_outgoing = plan->stack_arg_size;
   /* Indirect callee in an arg register would be clobbered by arg loads. */
   if (plan->callee.kind == NATIVE_LOC_REG &&
       (NativeAllocClass)plan->callee.cls == NATIVE_REG_INT &&
       plan->callee.v.reg >= RV_A0 && plan->callee.v.reg <= RV_A7) {
     NativeLoc scratch =
         native_loc_reg(plan->callee.type, NATIVE_REG_INT, RV_TMP0);
     rv_move(t, scratch, plan->callee);
     plan->callee = scratch;
   }
   {
     /* sret returns pass the hidden destination pointer as the implicit first
      * integer argument (a0), so the real args start at a1. */
     u32 next_int = (abi && abi->has_sret) ? 1u : 0u;
     u32 next_fp = 0, stack = 0, nmoves = 0, i, p;
     int tail = (desc->flags & CG_CALL_TAIL) != 0;
     RvArgMove moves[RV_MAX_REG_ARG_MOVES];
     for (i = 0; i < desc->nargs; ++i) {
       ABIArgInfo tmp;
       const ABIArgInfo* ai = rv_param_abi(t, abi, desc, i, &tmp);
       int force_stack =
           abi && abi->variadic && abi->vararg_on_stack && i >= abi->nparams;
       if (ai->kind == ABI_ARG_IGNORE) continue;
       if (force_stack) {
         NativeLoc tmpreg =
             native_loc_reg(desc->args[i].type, NATIVE_REG_INT, RV_TMP0);
         u32 slot = a->variant->gp_slot_bytes; /* xlen-word: 8 lp64d / 4 ilp32 */
         u32 n = rv_class_stack_size(a->variant, ai), off = 0;
         while (off < n) {
           rv_load_part(t, tmpreg, desc->args[i], off, slot);
           rv_store_outgoing_part(t, tail, stack + off, tmpreg, slot);
           off += slot;
         }
         stack += n;
         continue;
       }
       if (ai->kind == ABI_ARG_INDIRECT) {
         u32 ptr_sz = a->variant->ptr_bytes;
         if (next_int < 8u) {
           RvArgMove* m = &moves[nmoves++];
           m->dst = native_loc_reg(i64t, NATIVE_REG_INT, RV_A0 + next_int++);
           m->src = desc->args[i];
           m->src_offset = 0;
           m->size = ptr_sz;
           m->is_addr = 1;
         } else {
           NativeLoc ptr = native_loc_reg(i64t, NATIVE_REG_INT, RV_TMP0);
           rv_addr_of_loc(t, ptr, desc->args[i]);
           rv_store_outgoing_part(t, tail, stack, ptr, ptr_sz);
           stack += ptr_sz;
         }
         continue;
       }
       for (p = 0; p < ai->nparts; ++p) {
         const ABIArgPart* part = &ai->parts[p];
         NativeAllocClass cls =
             part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
         if ((cls == NATIVE_REG_FP && next_fp < 8u) ||
             (cls == NATIVE_REG_INT && next_int < 8u)) {
           RvArgMove* m = &moves[nmoves++];
           Reg areg =
               cls == NATIVE_REG_FP ? RV_FA0 + next_fp++ : RV_A0 + next_int++;
           m->dst = native_loc_reg(desc->args[i].type, cls, areg);
           m->src = desc->args[i];
           m->src_offset = part->src_offset;
           m->size = part->size;
           m->is_addr = 0;
         } else {
           Reg tmp = cls == NATIVE_REG_FP ? RV_FTMP0 : RV_TMP0;
           NativeLoc tmpreg = native_loc_reg(desc->args[i].type, cls, tmp);
           rv_load_part(t, tmpreg, desc->args[i], part->src_offset, part->size);
           stack = align_up_u32(stack, rv_part_stack_align(a->variant, part));
           rv_store_outgoing_part(t, tail, stack, tmpreg, part->size);
           stack += rv_part_stack_size(a->variant, part);
         }
       }
     }
     rv_emit_reg_arg_moves(t, moves, nmoves);
     if (abi && abi->has_sret) {
       /* sret pointer goes in a0; arg loads have completed. A tail call forwards
        * the caller's own incoming sret pointer (spilled at entry) so the
        * sibling writes the result into the caller's caller's destination;
        * otherwise pass the address of this call's result slot. */
       NativeLoc a0 = native_loc_reg(i64t, NATIVE_REG_INT, RV_A0);
       if (tail)
         rv_load_part(t, a0, native_loc_stack(i64t, a->sret_ptr_slot, 0), 0,
                      a->variant->ptr_bytes);
       else if (desc->nresults)
         rv_addr_of_loc(t, a0, desc->results[0]);
     }
   }
   if (abi && abi->ret.kind == ABI_ARG_DIRECT && desc->nresults) {
     u32 nr = 0, ni = 0, nf = 0, p;
     for (p = 0; p < abi->ret.nparts; ++p) {
       const ABIArgPart* part = &abi->ret.parts[p];
       NativeAllocClass cls =
           part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
       KitCgTypeId pty = rv_part_scalar_type(part);
       Reg rreg = cls == NATIVE_REG_FP ? RV_FA0 + nf++ : RV_A0 + ni++;
       rets[nr].src = native_loc_reg(pty, cls, rreg);
       rets[nr].dst = desc->results[0];
       if (rets[nr].dst.kind == NATIVE_LOC_FRAME)
         rets[nr].dst = native_loc_stack(pty, desc->results[0].v.frame,
                                         (i32)part->src_offset);
       else if (rets[nr].dst.kind == NATIVE_LOC_STACK) {
         rets[nr].dst.v.stack.offset += (i32)part->src_offset;
         rets[nr].dst.type = pty;
       }
       rets[nr].mem = native_mem_for_type(t, pty, part->size);
       nr++;
     }
     plan->nrets = nr;
   } else if (abi && abi->ret.kind == ABI_ARG_IGNORE) {
     plan->nrets = 0;
   } else if (!abi && desc->nresults) {
     rets[0].src = native_loc_reg(desc->results[0].type, NATIVE_REG_INT, RV_A0);
     rets[0].dst = desc->results[0];
     rets[0].mem = native_mem_for_type(t, desc->results[0].type, 0);
     plan->nrets = 1;
   }
 }
 
 /* Emit a sibling (tail) call: tear the frame down to the caller's entry state
  * and jump (no link) to the callee. Outgoing args are already in the arg regs /
  * the caller's incoming-arg window. At -O0 there are no callee-saves, and the
  * sp restore uses the CFA offset (s0 + 16 + va_save), which is independent of
  * the not-yet-final frame_size — so no func_end patching is needed. */
 static void rv_emit_tail_site(NativeTarget* t, NativeLoc callee) {
   RvNativeTarget* a = rv_of(t);
   const RiscvVariant* v = a->variant;
   MCEmitter* mc = t->mc;
   i32 cfa = (i32)(v->frame_save_size + rv_va_save_sz(a));
   int indirect = callee.kind == NATIVE_LOC_REG;
   u32 int_regs[RV_MAX_CALLEE_SAVES], fp_regs[RV_MAX_CALLEE_SAVES];
   u32 n_int = rv_collect_int_saves(a, int_regs);
   u32 n_fp = rv_collect_fp_saves(a, fp_regs);
   i32 i;
   /* Stage an indirect callee into a reserved scratch (t1) BEFORE the teardown:
    * regalloc parks the function pointer in a callee-saved register so it
    * survives arg marshalling, and the callee-save / s0 / ra restores below
    * would otherwise overwrite it. t1 is reserved (never allocable) and
    * untouched by the restore loop (which only uses t0 for far offsets). */
   if (indirect) rv64_emit32(mc, rv_addi(RV_TMP1, loc_reg(callee), 0));
   /* Restore callee-saves before tearing the frame down (O1 path; none at -O0).
    * Their save offsets are s0-relative via rv_save_off, so the restore is
    * frame-size- and teardown-order-independent. */
   for (i = (i32)n_int - 1; i >= 0; --i)
     rv_load_s0(v, mc, 0, int_regs[i], rv_save_off(a, n_int, (u32)i));
   for (i = (i32)n_fp - 1; i >= 0; --i)
     rv_load_s0(v, mc, 1, fp_regs[i], rv_save_off(a, n_int, n_int + (u32)i));
   rv64_emit32(mc, rv_ld_ptr(v, RV_RA, RV_S0, (i32)v->ptr_bytes));
   rv64_emit32(mc, rv_addi(RV_SP, RV_S0, cfa));
   rv64_emit32(mc, rv_ld_ptr(v, RV_S0, RV_S0, 0));
   if (callee.kind == NATIVE_LOC_GLOBAL) {
     u32 pos = mc->pos(mc);
     rv64_emit32(mc, rv_auipc(RV_TMP0, 0));
     rv64_emit32(mc, rv_jalr(RV_ZERO, RV_TMP0, 0));
     mc->emit_reloc_at(mc, mc->section_id, pos, R_RV_CALL, callee.v.global.sym,
                       callee.v.global.addend, 0, 0);
   } else if (indirect) {
     rv64_emit32(mc, rv_jalr(RV_ZERO, RV_TMP1, 0));
   } else {
     rv_panic(a, "unsupported tail call target");
   }
 }
 
 static void rv_emit_call(NativeTarget* t, const NativeCallPlan* plan) {
   MCEmitter* mc = t->mc;
   ObjSecId sec = mc->section_id;
   if (plan->flags & CG_CALL_TAIL) {
     rv_emit_tail_site(t, plan->callee);
     return;
   }
   if (plan->callee.kind == NATIVE_LOC_GLOBAL) {
     u32 pos = mc->pos(mc);
     rv64_emit32(mc, rv_auipc(RV_RA, 0));
     rv64_emit32(mc, rv_jalr(RV_RA, RV_RA, 0));
     mc->emit_reloc_at(mc, sec, pos, R_RV_CALL, plan->callee.v.global.sym,
                       plan->callee.v.global.addend, 0, 0);
     return;
   }
   if (plan->callee.kind == NATIVE_LOC_REG) {
     rv64_emit32(mc, rv_jalr(RV_RA, loc_reg(plan->callee), 0));
     return;
   }
   rv_panic(rv_of(t), "unsupported call target");
 }
 
 static void rv_plan_ret(NativeTarget* t, const CGFuncDesc* fd,
                         const NativeLoc* value,
                         NativeCallPlanRet** out_rets, u32* out_nrets) {
   RvNativeTarget* a = rv_of(t);
   const ABIFuncInfo* abi = abi_cg_func_info(t->c->abi, fd->fn_type);
   NativeCallPlanRet* rets = NULL;
   u32 nr = 0;
   if (value) rets = arena_zarray(t->c->tu, NativeCallPlanRet, 4);
   if (value && abi && abi->ret.kind == ABI_ARG_INDIRECT) {
     KitCgTypeId i64t = builtin_id(KIT_CG_BUILTIN_I64);
     NativeLoc dstp = native_loc_reg(i64t, NATIVE_REG_INT, RV_TMP1);
     NativeLoc saved = native_loc_stack(i64t, a->sret_ptr_slot, 0);
     NativeAddr dst_addr, src_addr;
     AggregateAccess access;
     rv_load_part(t, dstp, saved, 0, a->variant->ptr_bytes);
     memset(&dst_addr, 0, sizeof dst_addr);
     dst_addr.base_kind = NATIVE_ADDR_BASE_REG;
     dst_addr.base.reg = RV_TMP1;
     dst_addr.base_type = value->type;
     src_addr = rv_loc_addr(a, *value, 0);
     src_addr.base_type = value->type;
     memset(&access, 0, sizeof access);
     access.type = value->type;
     access.size = (u32)cg_type_size(t->c, value->type);
     access.align = native_type_align(t, value->type);
     rv_copy_bytes(t, dst_addr, src_addr, access);
     *out_rets = NULL;
     *out_nrets = 0;
     return;
   }
   if (value && abi && abi->ret.kind == ABI_ARG_DIRECT) {
     u32 ni = 0, nf = 0, p;
     for (p = 0; p < abi->ret.nparts; ++p) {
       const ABIArgPart* part = &abi->ret.parts[p];
       NativeAllocClass cls =
           part->cls == ABI_CLASS_FP ? NATIVE_REG_FP : NATIVE_REG_INT;
       KitCgTypeId pty = rv_part_scalar_type(part);
       Reg rreg = cls == NATIVE_REG_FP ? RV_FA0 + nf++ : RV_A0 + ni++;
       rets[nr].src = *value;
       if (rets[nr].src.kind == NATIVE_LOC_FRAME)
         rets[nr].src =
             native_loc_stack(pty, value->v.frame, (i32)part->src_offset);
       else if (rets[nr].src.kind == NATIVE_LOC_STACK) {
         rets[nr].src.v.stack.offset += (i32)part->src_offset;
         rets[nr].src.type = pty;
       }
       rets[nr].dst = native_loc_reg(pty, cls, rreg);
       rets[nr].mem = native_mem_for_type(t, pty, part->size);
       nr++;
     }
   } else if (value) {
     rets[0].src = *value;
     rets[0].dst = native_loc_reg(value->type, NATIVE_REG_INT, RV_A0);
     rets[0].mem = native_mem_for_type(t, value->type, 0);
     nr = 1;
   }
   *out_rets = rets;
   *out_nrets = nr;
 }
 
 static void rv_ret(NativeTarget* t) {
   RvNativeTarget* a = rv_of(t);
   rv_jump(t, a->epilogue_label);
 }
 
 /* ============================ alloca ============================ */
 
 static void rv_alloca(NativeTarget* t, NativeLoc dst, NativeLoc size,
                       u32 align) {
   RvNativeTarget* a = rv_of(t);
   MCEmitter* mc = t->mc;
   u32 rsz = loc_reg(size);
   u32 rd = loc_reg(dst);
   u32 al = align ? align : 16u;
   if (al < 16u) al = 16u;
   /* round up: t0 = (size + (al-1)) & ~(al-1) */
   rv64_emit32(mc, rv_addi(RV_TMP0, rsz, (i32)(al - 1u)));
   rv_emit_load_imm(a->variant, mc, 1, RV_TMP1, -(i64)al);
   rv64_emit32(mc, rv_and(RV_TMP0, RV_TMP0, RV_TMP1));
   rv64_emit32(mc, rv_sub(RV_SP, RV_SP, RV_TMP0));
   a->frame.has_alloca = 1;
   /* dst = sp + max_outgoing (patched in func_end) */
   if (a->npatches == a->patches_cap) {
     u32 cap = a->patches_cap ? a->patches_cap * 2u : 8u;
     RvPatch* nb = arena_zarray(t->c->tu, RvPatch, cap);
     if (a->patches) memcpy(nb, a->patches, sizeof(*nb) * a->npatches);
     a->patches = nb;
     a->patches_cap = cap;
   }
   a->patches[a->npatches].kind = RV_PATCH_ALLOCA;
   a->patches[a->npatches].pos = mc->pos(mc);
   a->patches[a->npatches].dst_reg = rd;
   a->npatches++;
   a->nalloca++;
   rv64_emit32(mc, RV_NOP); /* placeholder for addi dst, sp, max_outgoing */
 }
 
 /* ============================ TLS / bitfield / atomics
  * ============================ */
 
 static void rv_tls_addr_of(NativeTarget* t, NativeLoc dst, ObjSymId sym,
                            i64 addend) {
   MCEmitter* mc = t->mc;
   u32 sec = mc->section_id;
   u32 rd = loc_reg(dst);
   /* Local-Exec only, matching aa64 (aa_tls_addr_of) and x64 (x64_tls_addr_of):
    * kit links the whole module statically, so every _Thread_local symbol is
    * resolved within the image and TPREL is always valid. An Initial-Exec GOT
    * path (R_RV_TLS_GOT_HI20) used to be emitted for extern-via-GOT symbols
    * under -fPIE (the hosted default), but the linker has no layout/apply for
    * that reloc, so it produced a hard "unsupported reloc kind" link failure
    * rather than a working binary. */
   /* lui t0, %tprel_hi(sym); add t0, tp, t0; addi dst, t0, %tprel_lo(sym). */
   {
     u32 hp = mc->pos(mc);
     rv64_emit32(mc, rv_lui(RV_TMP0, 0));
     mc->emit_reloc_at(mc, sec, hp, R_RV_TPREL_HI20, sym, addend, 0, 0);
     rv64_emit32(mc, rv_add(RV_TMP0, RV_TP, RV_TMP0));
     {
       u32 lp = mc->pos(mc);
       rv64_emit32(mc, rv_addi(rd, RV_TMP0, 0));
       mc->emit_reloc_at(mc, sec, lp, R_RV_TPREL_LO12_I, sym, addend, 0, 0);
     }
   }
 }
 static void rv_bitfield_load(NativeTarget* t, NativeLoc dst, NativeAddr ra,
                              BitFieldAccess bf) {
   RvNativeTarget* a = rv_of(t);
   const RiscvVariant* v = a->variant;
   MCEmitter* mc = t->mc;
   u32 storage_bytes = bf.storage.size ? bf.storage.size : 4u;
   u32 rd = loc_reg(dst);
   u32 base;
   i32 off;
   u32 lsb = bf.bit_offset;
   u32 width = bf.bit_width ? bf.bit_width : 1u;
   /* Shift left so the field's MSB lands at the register top (XLEN-1), then shift
    * right to sign/zero extend it down. Shifts are XLEN-wide. */
   u32 sh_left = v->xlen - (lsb + width);
   u32 sh_right = v->xlen - width;
   ra.offset += (i32)bf.storage_offset;
   rv_resolve_mem_addr(a, &ra, &base, &off);
   rv64_emit32(mc, enc_int_load(v, storage_bytes, 0, rd, base, off));
   rv64_emit32(mc, rv_slli(rd, rd, sh_left));
   if (bf.signed_)
     rv64_emit32(mc, rv_srai(rd, rd, sh_right));
   else
     rv64_emit32(mc, rv_srli(rd, rd, sh_right));
 }
 static void rv_bitfield_store(NativeTarget* t, NativeAddr ra, NativeLoc src,
                               BitFieldAccess bf) {
   RvNativeTarget* a = rv_of(t);
   const RiscvVariant* v = a->variant;
   MCEmitter* mc = t->mc;
   u32 storage_bytes = bf.storage.size ? bf.storage.size : 4u;
   u32 src_reg = loc_reg(src);
   u32 base;
   i32 off;
   u32 lsb = bf.bit_offset;
   u32 width = bf.bit_width ? bf.bit_width : 1u;
   u64 ones = width >= 64u ? ~(u64)0 : (((u64)1 << width) - 1u);
   u64 mask_in = ones << lsb;
   ra.offset += (i32)bf.storage_offset;
   /* Resolve the field address; rv_resolve_mem_addr may use RV_TMP0/RV_TMP1, so
    * stabilize the base into RV_TMP1 before consuming the scratch temps. */
   rv_resolve_mem_addr(a, &ra, &base, &off);
   if (base != RV_S0 && base != RV_TMP1) {
     rv_emit_addr_adjust(v, mc, RV_TMP1, base, off);
     base = RV_TMP1;
     off = 0;
   } else if (base == RV_TMP1 && off != 0) {
     rv_emit_addr_adjust(v, mc, RV_TMP1, RV_TMP1, off);
     off = 0;
   }
   /* word in RV_TMP2; merged via RV_TMP0 (clear mask, then shifted src). */
   rv64_emit32(mc, enc_int_load(v, storage_bytes, 0, RV_TMP2, base, off));
   rv_emit_load_imm(v, mc, 1, RV_TMP0, (i64)~mask_in);
   rv64_emit32(mc, rv_and(RV_TMP2, RV_TMP2, RV_TMP0));
   rv_emit_load_imm(v, mc, 1, RV_TMP0, (i64)ones);
   rv64_emit32(mc, rv_and(RV_TMP0, src_reg, RV_TMP0));
   if (lsb) rv64_emit32(mc, rv_slli(RV_TMP0, RV_TMP0, lsb));
   rv64_emit32(mc, rv_or(RV_TMP2, RV_TMP2, RV_TMP0));
   rv64_emit32(mc, enc_int_store(v, storage_bytes, RV_TMP2, base, off));
 }
 static int rv_order_acquire(KitCgMemOrder o) {
   return o == KIT_CG_MO_CONSUME || o == KIT_CG_MO_ACQUIRE ||
          o == KIT_CG_MO_ACQ_REL || o == KIT_CG_MO_SEQ_CST;
 }
 static int rv_order_release(KitCgMemOrder o) {
   return o == KIT_CG_MO_RELEASE || o == KIT_CG_MO_ACQ_REL ||
          o == KIT_CG_MO_SEQ_CST;
 }
 
 /* Materialize the atomic operand address into RV_TMP0 (a bare pointer, since
  * LR/SC and AMO take a base register with no offset) and return it. */
 static u32 rv_atomic_addr_reg(RvNativeTarget* a, NativeAddr addr) {
   NativeLoc dst =
       native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT, RV_TMP0);
   rv_load_addr(&a->base, dst, addr);
   return RV_TMP0;
 }
 
 static void rv_atomic_load(NativeTarget* t, NativeLoc dst, NativeAddr addr,
                            MemAccess mem, KitCgMemOrder mo) {
   RvNativeTarget* a = rv_of(t);
   MCEmitter* mc = t->mc;
   u32 sf =
       (mem.size ? mem.size : native_type_size(t, dst.type)) == 8u ? 1u : 0u;
   u32 base = rv_atomic_addr_reg(a, addr);
   if (mo == KIT_CG_MO_SEQ_CST) rv64_emit32(mc, rv_fence_rw_rw());
   if (rv_order_acquire(mo)) {
     /* lr.w/d as an ordered load (aq=1). */
     rv64_emit32(mc, sf ? rv_lr_d(loc_reg(dst), base, 1, 0)
                        : rv_lr_w(loc_reg(dst), base, 1, 0));
   } else {
     rv64_emit32(mc, enc_int_load(a->variant,
                                  mem.size ? mem.size
                                           : native_type_size(t, dst.type),
                                  0, loc_reg(dst), base, 0));
   }
 }
 
 static void rv_atomic_store(NativeTarget* t, NativeAddr addr, NativeLoc src,
                             MemAccess mem, KitCgMemOrder mo) {
   RvNativeTarget* a = rv_of(t);
   MCEmitter* mc = t->mc;
   u32 sz = mem.size ? mem.size : native_type_size(t, src.type);
   /* RV_TMP0 holds the address; never collides with src (an allocable reg). */
   u32 base = rv_atomic_addr_reg(a, addr);
   if (rv_order_release(mo)) rv64_emit32(mc, rv_fence_rw_rw());
   rv64_emit32(mc, enc_int_store(a->variant, sz, loc_reg(src), base, 0));
   if (mo == KIT_CG_MO_SEQ_CST) rv64_emit32(mc, rv_fence_rw_rw());
 }
 
 static void rv_atomic_rmw(NativeTarget* t, KitCgAtomicOp op, NativeLoc dst,
                           NativeAddr addr, NativeLoc val, MemAccess mem,
                           KitCgMemOrder mo) {
   RvNativeTarget* a = rv_of(t);
   const RiscvVariant* v = a->variant;
   MCEmitter* mc = t->mc;
   u32 sf = (mem.size ? mem.size : native_type_size(t, dst.type)) == 8u ? 1u : 0u;
   /* W-form add/sub apply only to a 32-bit value on rv64; on rv32 the base ops
    * are the 32-bit ops. */
   int w = !sf && v->has_w_forms;
   u32 base = rv_atomic_addr_reg(a, addr); /* RV_TMP0 */
   u32 vreg = loc_reg(val);
   u32 rd = loc_reg(dst);
   u32 aq = (u32)rv_order_acquire(mo);
   u32 rl = (u32)rv_order_release(mo);
   MCLabel retry = mc->label_new(mc);
   /* LR/SC loop: dst = *base; new = dst op val; sc new; retry on failure.
    * RV_TMP1 carries the SC status, RV_TMP3 the computed new value. */
   mc->label_place(mc, retry);
   rv64_emit32(mc, sf ? rv_lr_d(rd, base, aq, 0) : rv_lr_w(rd, base, aq, 0));
   switch (op) {
     case KIT_CG_ATOMIC_XCHG:
       rv64_emit32(mc, rv_addi(RV_TMP3, vreg, 0));
       break;
     case KIT_CG_ATOMIC_ADD:
       rv64_emit32(mc,
                   w ? rv_addw(RV_TMP3, rd, vreg) : rv_add(RV_TMP3, rd, vreg));
       break;
     case KIT_CG_ATOMIC_SUB:
       rv64_emit32(mc,
                   w ? rv_subw(RV_TMP3, rd, vreg) : rv_sub(RV_TMP3, rd, vreg));
       break;
     case KIT_CG_ATOMIC_AND:
       rv64_emit32(mc, rv_and(RV_TMP3, rd, vreg));
       break;
     case KIT_CG_ATOMIC_OR:
       rv64_emit32(mc, rv_or(RV_TMP3, rd, vreg));
       break;
     case KIT_CG_ATOMIC_XOR:
       rv64_emit32(mc, rv_xor(RV_TMP3, rd, vreg));
       break;
     case KIT_CG_ATOMIC_NAND:
       rv64_emit32(mc, rv_and(RV_TMP3, rd, vreg));
       rv64_emit32(mc, rv_xori(RV_TMP3, RV_TMP3, -1));
       break;
     default:
       rv_panic(a, "unsupported atomic rmw op");
   }
   rv64_emit32(mc, sf ? rv_sc_d(RV_TMP1, base, RV_TMP3, 0, rl)
                      : rv_sc_w(RV_TMP1, base, RV_TMP3, 0, rl));
   rv64_emit32(mc, rv_bne(RV_TMP1, RV_ZERO, 0));
   mc->emit_label_ref(mc, retry, R_RV_BRANCH, 4, 0);
 }
 
 static void rv_atomic_cas(NativeTarget* t, NativeLoc prior, NativeLoc ok,
                           NativeAddr addr, NativeLoc expected,
                           NativeLoc desired, MemAccess mem,
                           KitCgMemOrder success, KitCgMemOrder failure) {
   RvNativeTarget* a = rv_of(t);
   MCEmitter* mc = t->mc;
   u32 sf =
       (mem.size ? mem.size : native_type_size(t, prior.type)) == 8u ? 1u : 0u;
   u32 base = rv_atomic_addr_reg(a, addr); /* RV_TMP0 */
   u32 rprior = loc_reg(prior);
   u32 rexp = loc_reg(expected);
   u32 rdes = loc_reg(desired);
   u32 rok = loc_reg(ok);
   u32 aq = (u32)rv_order_acquire(success);
   u32 rl = (u32)rv_order_release(success);
   MCLabel retry = mc->label_new(mc);
   MCLabel fail = mc->label_new(mc);
   MCLabel done = mc->label_new(mc);
   (void)failure;
   mc->label_place(mc, retry);
   rv64_emit32(mc,
               sf ? rv_lr_d(rprior, base, aq, 0) : rv_lr_w(rprior, base, aq, 0));
   /* if (prior != expected) -> fail */
   rv64_emit32(mc, rv_bne(rprior, rexp, 0));
   mc->emit_label_ref(mc, fail, R_RV_BRANCH, 4, 0);
   /* sc.w/d status, desired, (base); retry on failure. */
   rv64_emit32(mc, sf ? rv_sc_d(RV_TMP1, base, rdes, 0, rl)
                      : rv_sc_w(RV_TMP1, base, rdes, 0, rl));
   rv64_emit32(mc, rv_bne(RV_TMP1, RV_ZERO, 0));
   mc->emit_label_ref(mc, retry, R_RV_BRANCH, 4, 0);
   /* ok = 1; jump done. */
   rv_emit_load_imm(a->variant, mc, 0, rok, 1);
   rv64_emit32(mc, rv_jal(RV_ZERO, 0));
   mc->emit_label_ref(mc, done, R_RV_JAL, 4, 0);
   mc->label_place(mc, fail);
   rv_emit_load_imm(a->variant, mc, 0, rok, 0);
   mc->label_place(mc, done);
 }
 
 static void rv_fence(NativeTarget* t, KitCgMemOrder mo) {
   if (mo == KIT_CG_MO_RELAXED) return;
   rv64_emit32(t->mc, rv_fence_rw_rw());
 }
 /* ---- variadics (LP64D ABI_VA_LIST_POINTER) ----
  * va_list is a single void* to the next argument slot. The prologue spilled
  * unconsumed a-regs into the 64-byte save area at [s0+16); incoming stack args
  * follow contiguously, so a uniform 8-byte stride covers both. `ap` is a
  * NativeAddr that addresses the va_list object itself. */
 
 static void rv_va_start_core(RvNativeTarget* a, NativeAddr ap) {
   NativeTarget* t = &a->base;
   const RiscvVariant* v = a->variant;
   MCEmitter* mc = t->mc;
   ABIVaListInfo vai = abi_va_list_layout(t->c->abi);
   KitCgTypeId i64t = builtin_id(KIT_CG_BUILTIN_I64);
   u32 slot = vai.gp_slot_size ? vai.gp_slot_size : v->gp_slot_bytes;
   if (vai.kind != ABI_VA_LIST_POINTER)
     rv_panic(a, "unsupported va_list layout");
   if (!a->is_variadic) rv_panic(a, "va_start: function not variadic");
   /* *ap = s0 + frame_save + next_param_int*gp_slot (skip named-int slots). */
   rv64_emit32(mc, rv_addi(RV_TMP1, RV_S0,
                           (i32)v->frame_save_size +
                               (i32)(a->next_param_int * slot)));
   rv_emit_mem(a, 0, native_loc_reg(i64t, NATIVE_REG_INT, RV_TMP1), ap,
               native_mem_for_type(t, i64t, v->ptr_bytes));
 }
 
 /* Wide / aggregate va_arg: a value too large for a single GPR (an 8-byte
  * i64 / soft-double on ilp32) occupies consecutive GP slots in the save area
  * and cannot move through one register. Read the cursor, advance it past the
  * whole span, then byte-copy the value from the (saved) cursor into the
  * destination memory. RV_TMP2 holds the cursor across the rv_copy_bytes call,
  * which itself uses RV_TMP0/RV_TMP1/RV_TMP3. */
 static void rv_va_arg_wide(RvNativeTarget* a, NativeAddr dst, NativeAddr ap,
                            u32 sz) {
   NativeTarget* t = &a->base;
   const RiscvVariant* v = a->variant;
   MCEmitter* mc = t->mc;
   KitCgTypeId i64t = builtin_id(KIT_CG_BUILTIN_I64);
   ABIVaListInfo vai = abi_va_list_layout(t->c->abi);
   u32 slot = vai.gp_slot_size ? vai.gp_slot_size : v->gp_slot_bytes;
   u32 span = align_up_u32(sz, slot);
   NativeLoc cur = native_loc_reg(i64t, NATIVE_REG_INT, RV_TMP2);
   NativeLoc nxt = native_loc_reg(i64t, NATIVE_REG_INT, RV_TMP1);
   NativeAddr src;
   AggregateAccess acc;
   /* cur = *ap; *ap = cur + span. */
   rv_emit_mem(a, 1, cur, ap, native_mem_for_type(t, i64t, v->ptr_bytes));
   rv64_emit32(mc, rv_addi(RV_TMP1, RV_TMP2, (i32)span));
   rv_emit_mem(a, 0, nxt, ap, native_mem_for_type(t, i64t, v->ptr_bytes));
   /* Copy sz bytes from [cur] to the destination. */
   memset(&src, 0, sizeof src);
   src.base_kind = NATIVE_ADDR_BASE_REG;
   src.base.reg = RV_TMP2;
   src.base_type = i64t;
   memset(&acc, 0, sizeof acc);
   acc.type = i64t;
   acc.size = sz;
   acc.align = slot;
   rv_copy_bytes(t, dst, src, acc);
 }
 
 static void rv_va_arg_core(RvNativeTarget* a, NativeLoc dst, NativeAddr ap,
                            KitCgTypeId type) {
   NativeTarget* t = &a->base;
   const RiscvVariant* v = a->variant;
   MCEmitter* mc = t->mc;
   ABIVaListInfo vai = abi_va_list_layout(t->c->abi);
   KitCgTypeId i64t = builtin_id(KIT_CG_BUILTIN_I64);
   u32 sz = native_type_size(t, type);
   u32 slot = vai.gp_slot_size ? vai.gp_slot_size : v->gp_slot_bytes;
   NativeLoc cur = native_loc_reg(i64t, NATIVE_REG_INT, RV_TMP1);
   NativeAddr from;
   if (vai.kind != ABI_VA_LIST_POINTER)
     rv_panic(a, "unsupported va_list layout");
   if (dst.kind != NATIVE_LOC_REG) rv_panic(a, "va_arg destination must be reg");
   /* cur = *ap; load value from [cur]; *ap = cur + slot (one GP-slot stride). */
   rv_emit_mem(a, 1, cur, ap, native_mem_for_type(t, i64t, v->ptr_bytes));
   memset(&from, 0, sizeof from);
   from.base_kind = NATIVE_ADDR_BASE_REG;
   from.base.reg = RV_TMP1;
   from.base_type = type;
   if (native_loc_is_fp(dst)) {
     /* Variadic FP args sit in the integer save area as their bit pattern;
      * load into RV_TMP2 and bitcast into the FPR. The fmv_d_x (double) path is
      * RV64-only — on rv32 doubles are passed soft and never reach here. */
     NativeLoc itmp = native_loc_reg(type, NATIVE_REG_INT, RV_TMP2);
     rv_emit_mem(a, 1, itmp, from, native_mem_for_type(t, type, sz));
     rv64_emit32(mc, sz == 8u ? rv_fmv_d_x(loc_reg(dst), RV_TMP2)
                              : rv_fmv_w_x(loc_reg(dst), RV_TMP2));
   } else {
     rv_emit_mem(a, 1, dst, from, native_mem_for_type(t, type, sz));
   }
   rv64_emit32(mc, rv_addi(RV_TMP1, RV_TMP1, (i32)slot));
   rv_emit_mem(a, 0, cur, ap, native_mem_for_type(t, i64t, v->ptr_bytes));
 }
 
 static void rv_va_copy_core(RvNativeTarget* a, NativeAddr dst_ap,
                             NativeAddr src_ap) {
   NativeTarget* t = &a->base;
   u32 ptr = a->variant->ptr_bytes;
   KitCgTypeId i64t = builtin_id(KIT_CG_BUILTIN_I64);
   NativeLoc tmp = native_loc_reg(i64t, NATIVE_REG_INT, RV_TMP1);
   /* va_list is a single pointer-width slot. */
   rv_emit_mem(a, 1, tmp, src_ap, native_mem_for_type(t, i64t, ptr));
   rv_emit_mem(a, 0, tmp, dst_ap, native_mem_for_type(t, i64t, ptr));
 }
 
 static NativeAddr rv_va_addr_from_ptr(NativeLoc ap_ptr) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_REG;
   addr.cls = NATIVE_REG_INT;
   addr.base.reg = ap_ptr.v.reg;
   addr.base_type = ap_ptr.type;
   return addr;
 }
 
 static void rv_va_start_native(NativeTarget* t, NativeLoc ap_ptr) {
   rv_va_start_core(rv_of(t), rv_va_addr_from_ptr(ap_ptr));
 }
 /* A scalar whose value cannot move through one GPR (size > GPR width, e.g. an
  * 8-byte i64 / soft-double on ilp32). pass_native_emit hands such a va_arg its
  * memory destination directly rather than a scratch register. */
 static int rv_va_arg_is_wide(NativeTarget* t, KitCgTypeId type) {
   return native_type_size(t, type) > rv_of(t)->variant->ptr_bytes;
 }
 
 static void rv_va_arg_native(NativeTarget* t, NativeLoc dst, NativeLoc ap_ptr,
                              KitCgTypeId type) {
   RvNativeTarget* a = rv_of(t);
   if (rv_va_arg_is_wide(t, type)) {
     rv_va_arg_wide(a, rv_loc_addr(a, dst, 0), rv_va_addr_from_ptr(ap_ptr),
                    native_type_size(t, type));
     return;
   }
   rv_va_arg_core(a, dst, rv_va_addr_from_ptr(ap_ptr), type);
 }
 static void rv_va_end_native(NativeTarget* t, NativeLoc ap_ptr) {
   (void)t;
   (void)ap_ptr;
 }
 static void rv_va_copy_native(NativeTarget* t, NativeLoc dst, NativeLoc src) {
   rv_va_copy_core(rv_of(t), rv_va_addr_from_ptr(dst), rv_va_addr_from_ptr(src));
 }
 /* Software popcount of RV_TMP1 (already width-normalized) into rd, using
  * RV_TMP1/RV_TMP2/RV_TMP3 as scratch. Mirrors the legacy bit-twiddling. On rv32
  * only the 32-bit (is64==0) path is reachable for a single register. */
 static void rv_emit_popcount(const RiscvVariant* v, MCEmitter* mc, u32 rd,
                              int is64) {
   rv64_emit32(mc, rv_srli(RV_TMP2, RV_TMP1, 1));
   rv_emit_load_imm(v, mc, 1, RV_TMP3,
                    is64 ? (i64)0x5555555555555555ll : (i64)0x55555555);
   rv64_emit32(mc, rv_and(RV_TMP2, RV_TMP2, RV_TMP3));
   rv64_emit32(mc, rv_sub(RV_TMP1, RV_TMP1, RV_TMP2));
   rv_emit_load_imm(v, mc, 1, RV_TMP3,
                    is64 ? (i64)0x3333333333333333ll : (i64)0x33333333);
   rv64_emit32(mc, rv_and(RV_TMP2, RV_TMP1, RV_TMP3));
   rv64_emit32(mc, rv_srli(RV_TMP1, RV_TMP1, 2));
   rv64_emit32(mc, rv_and(RV_TMP1, RV_TMP1, RV_TMP3));
   rv64_emit32(mc, rv_add(RV_TMP1, RV_TMP1, RV_TMP2));
   rv64_emit32(mc, rv_srli(RV_TMP2, RV_TMP1, 4));
   rv64_emit32(mc, rv_add(RV_TMP1, RV_TMP1, RV_TMP2));
   rv_emit_load_imm(v, mc, 1, RV_TMP3,
                    is64 ? (i64)0x0f0f0f0f0f0f0f0fll : (i64)0x0f0f0f0f);
   rv64_emit32(mc, rv_and(RV_TMP1, RV_TMP1, RV_TMP3));
   rv_emit_load_imm(v, mc, 1, RV_TMP3,
                    is64 ? (i64)0x0101010101010101ll : (i64)0x01010101);
   rv64_emit32(mc, rv_mul(RV_TMP1, RV_TMP1, RV_TMP3));
   rv64_emit32(mc, rv_srli(rd, RV_TMP1, is64 ? 56u : 24u));
   /* The 32-bit SWAR sum lives in product bits [24,32); since the multiply is
    * 64-bit, bits [32,64) survive the >>24 and must be masked off. (The 64-bit
    * path's >>56 already isolates the top byte, so it needs no mask.) */
   if (!is64) rv64_emit32(mc, rv_andi(rd, rd, 0xff));
 }
 
 /* Inline byte-granule copy/set between bare base registers (memcpy/memmove/
  * memset intrinsics). dir<0 copies high-to-low (memmove backward). The 8-byte
  * granule (ld/sd) and zero-extending lwu are RV64-only; on rv32 the widest
  * granule is 4 bytes via lw/sw. */
 static void rv_intrin_copy(const RiscvVariant* v, MCEmitter* mc, u32 dr, u32 sr,
                            u32 n, int backward) {
   int wide = v->ptr_bytes == 8u;
   if (!backward) {
     u32 i = 0;
     while (wide && i + 8u <= n) {
       rv64_emit32(mc, rv_ld(RV_TMP3, sr, (i32)i));
       rv64_emit32(mc, rv_sd(RV_TMP3, dr, (i32)i));
       i += 8u;
     }
     while (i + 4u <= n) {
       rv64_emit32(mc, wide ? rv_lwu(RV_TMP3, sr, (i32)i)
                            : rv_lw(RV_TMP3, sr, (i32)i));
       rv64_emit32(mc, rv_sw(RV_TMP3, dr, (i32)i));
       i += 4u;
     }
     while (i + 2u <= n) {
       rv64_emit32(mc, rv_lhu(RV_TMP3, sr, (i32)i));
       rv64_emit32(mc, rv_sh(RV_TMP3, dr, (i32)i));
       i += 2u;
     }
     while (i < n) {
       rv64_emit32(mc, rv_lbu(RV_TMP3, sr, (i32)i));
       rv64_emit32(mc, rv_sb(RV_TMP3, dr, (i32)i));
       i += 1u;
     }
   } else {
     u32 i = n;
     while (wide && i >= 8u) {
       i -= 8u;
       rv64_emit32(mc, rv_ld(RV_TMP3, sr, (i32)i));
       rv64_emit32(mc, rv_sd(RV_TMP3, dr, (i32)i));
     }
     while (i >= 4u) {
       i -= 4u;
       rv64_emit32(mc, wide ? rv_lwu(RV_TMP3, sr, (i32)i)
                            : rv_lw(RV_TMP3, sr, (i32)i));
       rv64_emit32(mc, rv_sw(RV_TMP3, dr, (i32)i));
     }
     while (i >= 2u) {
       i -= 2u;
       rv64_emit32(mc, rv_lhu(RV_TMP3, sr, (i32)i));
       rv64_emit32(mc, rv_sh(RV_TMP3, dr, (i32)i));
     }
     while (i >= 1u) {
       i -= 1u;
       rv64_emit32(mc, rv_lbu(RV_TMP3, sr, (i32)i));
       rv64_emit32(mc, rv_sb(RV_TMP3, dr, (i32)i));
     }
   }
 }
 
 static void rv_intrinsic(NativeTarget* t, IntrinKind kind,
                          const NativeLoc* dsts, u32 ndst, const NativeLoc* args,
                          u32 narg) {
   RvNativeTarget* a = rv_of(t);
   const RiscvVariant* v = a->variant;
   MCEmitter* mc = t->mc;
   (void)ndst;
   (void)narg;
   switch (kind) {
     case INTRIN_NONE:
       break;
     case INTRIN_EXPECT:
     case INTRIN_ASSUME_ALIGNED: {
       /* dst = val (hint dropped). */
       if (args[0].kind == NATIVE_LOC_IMM)
         rv_emit_load_imm(v, mc, rv_is_64(t, dsts[0].type) ? 1u : 0u,
                          loc_reg(dsts[0]), args[0].v.imm);
       else
         rv_move(t, dsts[0], args[0]);
       return;
     }
     case INTRIN_PREFETCH:
       return;
     case INTRIN_TRAP:
       rv64_emit32(mc, rv_ebreak());
       return;
     case INTRIN_SYSCALL:
       if (ndst == 1u && narg >= 1u && narg <= 7u) {
         static const u32 syscall_regs[7] = {RV_A7, RV_A0, RV_A1, RV_A2,
                                             RV_A3, RV_A4, RV_A5};
         RvArgMove moves[7];
         for (u32 i = 0; i < narg; ++i) {
           RvArgMove* m = &moves[i];
           memset(m, 0, sizeof *m);
           m->dst =
               native_loc_reg(dsts[0].type, NATIVE_REG_INT, syscall_regs[i]);
           m->src = args[i];
           m->size = t->c->target.ptr_size;
         }
         rv_emit_reg_arg_moves(t, moves, narg);
         rv64_emit32(mc, rv_ecall());
         rv_move(t, dsts[0], native_loc_reg(dsts[0].type, NATIVE_REG_INT, RV_A0));
       }
       return;
     case INTRIN_BSWAP: {
       u32 width = abi_cg_sizeof(t->c->abi, dsts[0].type);
       switch (width) {
         case 2: {
           u32 rd = loc_reg(dsts[0]), rs = loc_reg(args[0]);
           /* rd = ((rs & 0xff) << 8) | ((rs >> 8) & 0xff). */
           rv64_emit32(mc, rv_addi(RV_TMP2, RV_ZERO, 0xff));
           rv64_emit32(mc, rv_slli(RV_TMP2, RV_TMP2, 8)); /* 0xff00 */
           rv64_emit32(mc, rv_slli(RV_TMP1, rs, 8));
           rv64_emit32(mc, rv_and(RV_TMP1, RV_TMP1, RV_TMP2));
           rv64_emit32(mc, rv_srli(RV_TMP3, rs, 8));
           rv64_emit32(mc, rv_andi(RV_TMP3, RV_TMP3, 0xff));
           rv64_emit32(mc, rv_or(rd, RV_TMP1, RV_TMP3));
           return;
         }
         case 4: {
           u32 rd = loc_reg(dsts[0]), rs = loc_reg(args[0]);
           /* SRLIW is RV64-only; on rv32 SRLI on a 32-bit reg is equivalent. */
           int w = v->has_w_forms;
           rv64_emit32(mc, w ? rv_srliw(RV_TMP1, rs, 24) : rv_srli(RV_TMP1, rs, 24));
           rv64_emit32(mc, rv_andi(RV_TMP1, RV_TMP1, 0xff));
           rv64_emit32(mc, w ? rv_srliw(RV_TMP2, rs, 16) : rv_srli(RV_TMP2, rs, 16));
           rv64_emit32(mc, rv_andi(RV_TMP2, RV_TMP2, 0xff));
           rv64_emit32(mc, rv_slli(RV_TMP2, RV_TMP2, 8));
           rv64_emit32(mc, rv_or(RV_TMP1, RV_TMP1, RV_TMP2));
           rv64_emit32(mc, w ? rv_srliw(RV_TMP2, rs, 8) : rv_srli(RV_TMP2, rs, 8));
           rv64_emit32(mc, rv_andi(RV_TMP2, RV_TMP2, 0xff));
           rv64_emit32(mc, rv_slli(RV_TMP2, RV_TMP2, 16));
           rv64_emit32(mc, rv_or(RV_TMP1, RV_TMP1, RV_TMP2));
           rv64_emit32(mc, rv_andi(RV_TMP2, rs, 0xff));
           rv64_emit32(mc, rv_slli(RV_TMP2, RV_TMP2, 24));
           rv64_emit32(mc, rv_or(rd, RV_TMP1, RV_TMP2));
           /* Canonicalize to a 32-bit value in a 64-bit reg (RV64 only); on rv32
            * the result already occupies the whole register. */
           if (w) {
             rv64_emit32(mc, rv_slli(rd, rd, 32));
             rv64_emit32(mc, rv_srli(rd, rd, 32));
           }
           return;
         }
         case 8: {
           u32 rd = loc_reg(dsts[0]), rs = loc_reg(args[0]);
           int i;
           rv64_emit32(mc, rv_addi(RV_TMP1, RV_ZERO, 0));
           for (i = 0; i < 8; ++i) {
             int sh = 56 - 8 * i;
             if (i == 0) {
               rv64_emit32(mc, rv_andi(RV_TMP2, rs, 0xff));
             } else {
               rv64_emit32(mc, rv_srli(RV_TMP2, rs, (u32)(8 * i)));
               rv64_emit32(mc, rv_andi(RV_TMP2, RV_TMP2, 0xff));
             }
             if (sh) rv64_emit32(mc, rv_slli(RV_TMP2, RV_TMP2, (u32)sh));
             rv64_emit32(mc, rv_or(RV_TMP1, RV_TMP1, RV_TMP2));
           }
           rv64_emit32(mc, rv_addi(rd, RV_TMP1, 0));
           return;
         }
         default:
           break;
       }
       return;
     }
     case INTRIN_POPCOUNT: {
       u32 rd = loc_reg(dsts[0]), rs = loc_reg(args[0]);
       int is64 = rv_is_64(t, args[0].type);
       /* The narrow-in-wide normalization clears the high 32 bits of a 64-bit
        * reg; on rv32 there are none, so it is skipped. */
       int nrm = !is64 && v->xlen == 64u;
       rv64_emit32(mc, rv_addi(RV_TMP1, rs, 0));
       if (nrm) {
         rv64_emit32(mc, rv_slli(RV_TMP1, RV_TMP1, 32));
         rv64_emit32(mc, rv_srli(RV_TMP1, RV_TMP1, 32));
       }
       rv_emit_popcount(v, mc, rd, is64);
       return;
     }
     case INTRIN_CTZ: {
       /* ctz(x) = popcount((x & -x) - 1) for x != 0. */
       u32 rd = loc_reg(dsts[0]), rs = loc_reg(args[0]);
       int is64 = rv_is_64(t, args[0].type);
       int nrm = !is64 && v->xlen == 64u;
       rv64_emit32(mc, rv_sub(RV_TMP1, RV_ZERO, rs));
       rv64_emit32(mc, rv_and(RV_TMP1, RV_TMP1, rs));
       rv64_emit32(mc, rv_addi(RV_TMP1, RV_TMP1, -1));
       if (nrm) {
         rv64_emit32(mc, rv_slli(RV_TMP1, RV_TMP1, 32));
         rv64_emit32(mc, rv_srli(RV_TMP1, RV_TMP1, 32));
       }
       rv_emit_popcount(v, mc, rd, is64);
       return;
     }
     case INTRIN_CLZ: {
       /* Fold the high bit downward, then clz = popcount(~folded). */
       u32 rd = loc_reg(dsts[0]), rs = loc_reg(args[0]);
       int is64 = rv_is_64(t, args[0].type);
       int nrm = !is64 && v->xlen == 64u;
       u32 shifts[6] = {1, 2, 4, 8, 16, 32};
       u32 ns = is64 ? 6u : 5u, i;
       rv64_emit32(mc, rv_addi(RV_TMP1, rs, 0));
       if (nrm) {
         rv64_emit32(mc, rv_slli(RV_TMP1, RV_TMP1, 32));
         rv64_emit32(mc, rv_srli(RV_TMP1, RV_TMP1, 32));
       }
       for (i = 0; i < ns; ++i) {
         rv64_emit32(mc, rv_srli(RV_TMP2, RV_TMP1, shifts[i]));
         rv64_emit32(mc, rv_or(RV_TMP1, RV_TMP1, RV_TMP2));
       }
       rv64_emit32(mc, rv_xori(RV_TMP1, RV_TMP1, -1));
       if (nrm) {
         rv64_emit32(mc, rv_slli(RV_TMP1, RV_TMP1, 32));
         rv64_emit32(mc, rv_srli(RV_TMP1, RV_TMP1, 32));
       }
       rv_emit_popcount(v, mc, rd, is64);
       return;
     }
     case INTRIN_SADD_OVERFLOW:
     case INTRIN_SSUB_OVERFLOW: {
       /* dsts: [val, ovf]. ADD: ovf=((a^r)&(b^r))>>(w-1);
        * SUB: ovf=((a^b)&(a^r))>>(w-1). */
       int is64 = rv_is_64(t, dsts[0].type);
       int w = !is64 && v->has_w_forms; /* narrow op on rv64 -> W-form */
       u32 ra = loc_reg(args[0]), rb = loc_reg(args[1]);
       u32 rd = loc_reg(dsts[0]), rovf = loc_reg(dsts[1]);
       u32 sh = is64 ? 63u : 31u;
       if (kind == INTRIN_SADD_OVERFLOW)
         rv64_emit32(mc, w ? rv_addw(RV_TMP2, ra, rb) : rv_add(RV_TMP2, ra, rb));
       else
         rv64_emit32(mc, w ? rv_subw(RV_TMP2, ra, rb) : rv_sub(RV_TMP2, ra, rb));
       rv64_emit32(mc, rv_xor(RV_TMP3, ra, RV_TMP2)); /* a ^ r */
       if (kind == INTRIN_SADD_OVERFLOW) {
         rv64_emit32(mc, rv_xor(rovf, rb, RV_TMP2)); /* b ^ r */
         rv64_emit32(mc, rv_and(rovf, rovf, RV_TMP3));
       } else {
         rv64_emit32(mc, rv_xor(rovf, ra, rb)); /* a ^ b */
         rv64_emit32(mc, rv_and(rovf, rovf, RV_TMP3));
       }
       rv64_emit32(mc, w ? rv_srliw(rovf, rovf, sh) : rv_srli(rovf, rovf, sh));
       rv64_emit32(mc, rv_andi(rovf, rovf, 1));
       rv64_emit32(mc, rv_addi(rd, RV_TMP2, 0));
       return;
     }
     case INTRIN_UADD_OVERFLOW:
     case INTRIN_USUB_OVERFLOW: {
       int is64 = rv_is_64(t, dsts[0].type);
       /* `single`: the value fills the whole native register (rv64 i64 or any
        * rv32 value), so the native carry/borrow sequence applies directly; the
        * `!single` branch is the rv64 32-bit-in-64-bit-register implementation
        * (zero-extend + srli-32), reachable only on rv64. */
       int single = is64 || v->xlen == 32u;
       u32 ra = loc_reg(args[0]), rb = loc_reg(args[1]);
       u32 rd = loc_reg(dsts[0]), rovf = loc_reg(dsts[1]);
       if (!single) {
         rv64_emit32(mc, rv_slli(RV_TMP2, ra, 32));
         rv64_emit32(mc, rv_srli(RV_TMP2, RV_TMP2, 32));
         rv64_emit32(mc, rv_slli(RV_TMP3, rb, 32));
         rv64_emit32(mc, rv_srli(RV_TMP3, RV_TMP3, 32));
         ra = RV_TMP2;
         rb = RV_TMP3;
       }
       if (kind == INTRIN_UADD_OVERFLOW) {
         if (single) {
           rv64_emit32(mc, rv_add(RV_TMP2, ra, rb));
           rv64_emit32(mc, rv_sltu(rovf, RV_TMP2, ra));
         } else {
           rv64_emit32(mc, rv_add(RV_TMP2, ra, rb));
           rv64_emit32(mc, rv_srli(rovf, RV_TMP2, 32));
           rv64_emit32(mc, rv_sltu(rovf, RV_ZERO, rovf));
           rv64_emit32(mc, rv_addiw(RV_TMP2, RV_TMP2, 0));
         }
       } else {
         rv64_emit32(mc, rv_sltu(rovf, ra, rb));
         rv64_emit32(mc, single ? rv_sub(RV_TMP2, ra, rb)
                                : rv_subw(RV_TMP2, ra, rb));
       }
       rv64_emit32(mc, rv_addi(rd, RV_TMP2, 0));
       return;
     }
     case INTRIN_SMUL_OVERFLOW: {
       int is64 = rv_is_64(t, dsts[0].type);
       /* `single`: native-width product overflow via MUL + MULH and a sign-bit
        * compare (shift xlen-1). rv64 i64 and any rv32 value take this path; the
        * `!single` branch is the rv64 32-bit-in-64-bit-register sequence. */
       int single = is64 || v->xlen == 32u;
       u32 sh = is64 ? 63u : 31u;
       u32 ra = loc_reg(args[0]), rb = loc_reg(args[1]);
       u32 rd = loc_reg(dsts[0]), rovf = loc_reg(dsts[1]);
       if (single) {
         rv64_emit32(mc, rv_mul(RV_TMP2, ra, rb));
         rv64_emit32(mc, rv_mulh(RV_TMP3, ra, rb));
         rv64_emit32(mc, rv_srai(rovf, RV_TMP2, sh));
         rv64_emit32(mc, rv_xor(rovf, RV_TMP3, rovf));
         rv64_emit32(mc, rv_sltu(rovf, RV_ZERO, rovf));
         rv64_emit32(mc, rv_addi(rd, RV_TMP2, 0));
       } else {
         rv64_emit32(mc, rv_addiw(RV_TMP2, ra, 0));
         rv64_emit32(mc, rv_addiw(RV_TMP3, rb, 0));
         rv64_emit32(mc, rv_mul(RV_TMP2, RV_TMP2, RV_TMP3));
         rv64_emit32(mc, rv_addiw(RV_TMP3, RV_TMP2, 0));
         rv64_emit32(mc, rv_xor(rovf, RV_TMP2, RV_TMP3));
         rv64_emit32(mc, rv_sltu(rovf, RV_ZERO, rovf));
         rv64_emit32(mc, rv_addiw(rd, RV_TMP2, 0));
       }
       return;
     }
     case INTRIN_UMUL_OVERFLOW: {
       int is64 = rv_is_64(t, dsts[0].type);
       /* `single`: native-width product, overflow = (high word != 0) via MULHU.
        * rv64 i64 and any rv32 value take this path; `!single` is the rv64
        * 32-bit-in-64-bit-register sequence. */
       int single = is64 || v->xlen == 32u;
       u32 ra = loc_reg(args[0]), rb = loc_reg(args[1]);
       u32 rd = loc_reg(dsts[0]), rovf = loc_reg(dsts[1]);
       if (single) {
         rv64_emit32(mc, rv_mulhu(rovf, ra, rb));
         rv64_emit32(mc, rv_mul(rd, ra, rb));
         rv64_emit32(mc, rv_sltu(rovf, RV_ZERO, rovf));
       } else {
         rv64_emit32(mc, rv_slli(RV_TMP2, ra, 32));
         rv64_emit32(mc, rv_srli(RV_TMP2, RV_TMP2, 32));
         rv64_emit32(mc, rv_slli(RV_TMP3, rb, 32));
         rv64_emit32(mc, rv_srli(RV_TMP3, RV_TMP3, 32));
         rv64_emit32(mc, rv_mul(RV_TMP2, RV_TMP2, RV_TMP3));
         rv64_emit32(mc, rv_srli(rovf, RV_TMP2, 32));
         rv64_emit32(mc, rv_sltu(rovf, RV_ZERO, rovf));
         rv64_emit32(mc, rv_addiw(rd, RV_TMP2, 0));
       }
       return;
     }
     case INTRIN_MEMMOVE: {
       u32 dr, sr, n;
       if (narg != 3u || args[0].kind != NATIVE_LOC_REG ||
           args[1].kind != NATIVE_LOC_REG || args[2].kind != NATIVE_LOC_IMM)
         rv_panic(a, "unsupported memory intrinsic operands");
       if (args[2].v.imm < 0 || args[2].v.imm > 0xffffffffll)
         rv_panic(a, "unsupported memory intrinsic size");
       dr = loc_reg(args[0]);
       sr = loc_reg(args[1]);
       n = (u32)args[2].v.imm;
       rv_intrin_copy(v, mc, dr, sr, n, /*reverse (overlap-safe)=*/1);
       return;
     }
     case INTRIN_CPU_NOP:
       rv64_emit32(mc, rv_nop());
       return;
     case INTRIN_CPU_YIELD:
       rv64_emit32(mc, rv_pause());
       return;
     case INTRIN_ISB:
       rv64_emit32(mc, rv_fence_i());
       return;
     case INTRIN_DMB:
     case INTRIN_DSB:
       rv64_emit32(mc, rv_fence_rw_rw());
       return;
     case INTRIN_WFI:
       rv64_emit32(mc, rv_wfi());
       return;
     case INTRIN_FRAME_ADDRESS:
     case INTRIN_RETURN_ADDRESS:
       /* Walk the s0 frame-record chain. kit's RISC-V prologue anchors s0 at the
        * saved pair: [s0] = caller's s0, [s0 + ptr_bytes] = saved ra (this
        * frame's return address). NOTE: this differs from the psABI's
        * ra@s0-8 / fp@s0-16 layout — kit stores the pair at and above s0. A
        * function that reads its frame is forced off the frameless-leaf tier
        * (see NativeKnownFrameDesc.reads_frame), so s0 is always valid here. The
        * level is constant, so the walk unrolls to `level` dependent loads. */
       if (ndst == 1u) {
         u32 level = (narg >= 1u && args[0].kind == NATIVE_LOC_IMM)
                         ? (u32)args[0].v.imm
                         : 0u;
         u32 rd = loc_reg(dsts[0]);
         rv64_emit32(mc, rv_addi(rd, RV_S0, 0)); /* rd = s0 */
         for (u32 i = 0; i < level; ++i)
           rv64_emit32(mc, rv_ld_ptr(v, rd, rd, 0)); /* rd = *(rd) */
         if (kind == INTRIN_RETURN_ADDRESS)
           rv64_emit32(mc, rv_ld_ptr(v, rd, rd, (i32)v->ptr_bytes));
       }
       return;
     default:
       break;
   }
   rv_panic(a, "unsupported compiler intrinsic");
 }
 /* ============================ inline asm ============================ */
 
 _Noreturn static void rv_asm_panic_at(Compiler* c, SrcLoc loc,
                                       const char* msg) {
   compiler_panic(c, loc, "rv64 inline asm: %s", msg);
 }
 _Noreturn static void rv_asm_panic(NativeDirectTarget* d, const char* msg) {
   rv_asm_panic_at(d->base.c, d->loc, msg);
 }
 
 /* constraint_body / constraint_early / match_index are shared
  * (cg/native_asm.h). */
 
 /* Build a bound register pseudo-operand in the rv64 inline shape. */
 static void rv_asm_bound_reg(Operand* out, KitCgTypeId type,
                              NativeAllocClass cls, Reg reg) {
   memset(out, 0, sizeof *out);
   out->kind = RV64_INLINE_OPK_REG;
   out->pad[0] =
       (cls == NATIVE_REG_FP) ? RV64_INLINE_OPCLS_FP : RV64_INLINE_OPCLS_INT;
   out->type = type;
   out->v.local = (CGLocal)reg;
 }
 static void rv_asm_bound_mem(Operand* out, KitCgTypeId type, Reg base) {
   memset(out, 0, sizeof *out);
   out->kind = OPK_INDIRECT;
   out->type = type;
   out->v.ind.base = (CGLocal)base;
   out->v.ind.index = CG_LOCAL_NONE;
   out->v.ind.ofs = 0;
 }
 
 /* Parse a clobber register name into (class, reg). Returns 0 for the special
  * "cc"/"memory" clobbers and panics on an unknown register. RV64 dwarf: int
  * x0..x31 = 0..31, fp f0..f31 = 32..63. */
 static int rv_asm_parse_reg_clobber(Compiler* c, SrcLoc loc, Sym name,
                                     NativeAllocClass* cls_out, Reg* reg_out) {
   Slice s = pool_slice(c->global, name);
   char buf[16];
   uint32_t dwarf;
   if (!s.s || !s.len) return 0;
   if (s.len == 2 && s.s[0] == 'c' && s.s[1] == 'c') return 0;
   if (s.len == 6 && memcmp(s.s, "memory", 6) == 0) return 0;
   if (s.len >= sizeof buf) rv_asm_panic_at(c, loc, "clobber name is too long");
   memcpy(buf, s.s, s.len);
   buf[s.len] = '\0';
   if (rv64_register_index(buf, &dwarf) != 0)
     rv_asm_panic_at(c, loc, "unknown clobber register");
   if (dwarf <= 31u) {
     *cls_out = NATIVE_REG_INT;
     *reg_out = (Reg)dwarf;
     return 1;
   }
   if (dwarf >= 32u && dwarf <= 63u) {
     *cls_out = NATIVE_REG_FP;
     *reg_out = (Reg)(dwarf - 32u);
     return 1;
   }
   rv_asm_panic_at(c, loc, "unsupported clobber register");
   return 0;
 }
 
 static void rv_asm_clobber_masks(Compiler* c, SrcLoc loc, const Sym* clobbers,
                                  u32 nclob, u32* int_mask, u32* fp_mask) {
   u32 i;
   *int_mask = 0;
   *fp_mask = 0;
   for (i = 0; i < nclob; ++i) {
     NativeAllocClass cls;
     Reg reg;
     if (!rv_asm_parse_reg_clobber(c, loc, clobbers[i], &cls, &reg)) continue;
     if (cls == NATIVE_REG_INT)
       *int_mask |= 1u << reg;
     else
       *fp_mask |= 1u << reg;
   }
 }
 
 static int rv_asm_resolve_pin_or_panic(NativeDirectTarget* d, Sym reg,
                                        const char* constraint,
                                        NativeAsmRegPin* pin) {
   NativeAsmRegPinStatus st =
       native_asm_resolve_pin(d->native, reg, constraint, pin);
   if (st == NATIVE_ASM_REG_PIN_ABSENT) return 0;
   if (st != NATIVE_ASM_REG_PIN_OK)
     rv_asm_panic(d, native_asm_pin_status_message(st));
   return 1;
 }
 
 /* Pick a free register from the arch's caller-saved allocable pools for an
  * asm operand the direct path must self-allocate. */
 static Reg rv_asm_alloc_reg(NativeDirectTarget* d, NativeAllocClass cls,
                             u32 allowed_mask, u32* used_int, u32* used_fp) {
   /* int: a0..a7 (10..17) then t-temps that aren't emit scratch. */
   static const Reg int_pool[] = {10u, 11u, 12u, 13u, 14u, 15u,
                                  16u, 17u, 29u, 30u, 31u};
   /* fp: fa0..fa7 (10..17) then ft caller-saved. */
   static const Reg fp_pool[] = {10u, 11u, 12u, 13u, 14u, 15u, 16u, 17u,
                                 4u,  5u,  6u,  7u,  28u, 29u, 30u, 31u};
   const Reg* pool = cls == NATIVE_REG_FP ? fp_pool : int_pool;
   u32 n = cls == NATIVE_REG_FP ? (u32)(sizeof fp_pool / sizeof fp_pool[0])
                                : (u32)(sizeof int_pool / sizeof int_pool[0]);
   u32* used = cls == NATIVE_REG_FP ? used_fp : used_int;
   u32 i;
   for (i = 0; i < n; ++i) {
     Reg r = pool[i];
     if (allowed_mask && (allowed_mask & (1u << r)) == 0) continue;
     if ((*used & (1u << r)) != 0) continue;
     *used |= 1u << r;
     return r;
   }
   rv_asm_panic(d, "out of registers for asm operands");
   return REG_NONE;
 }
 
 /* Direct (-O0) path: resolve a semantic Operand to a NativeAddr. */
 static NativeAddr rv_direct_addr(NativeDirectTarget* d, Operand op) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   switch ((OpKind)op.kind) {
     case OPK_LOCAL:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = d->locals[op.v.local - 1u].home;
       addr.base_type = op.type;
       return addr;
     case OPK_INDIRECT:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME_VALUE;
       addr.base.frame = d->locals[op.v.ind.base - 1u].home;
       addr.cls = d->locals[op.v.ind.base - 1u].cls;
       addr.base_type = d->locals[op.v.ind.base - 1u].type;
       addr.offset = op.v.ind.ofs;
       return addr;
     default:
       rv_asm_panic(d, "operand is not addressable");
   }
 }
 
 /* Materialize an OPK_INDIRECT (frame-value) base into a register, returning a
  * plain register-based NativeAddr. */
 static NativeAddr rv_direct_materialize_addr(NativeDirectTarget* d,
                                              Operand op) {
   RvNativeTarget* a = rv_of(d->native);
   NativeAddr addr = rv_direct_addr(d, op);
   if (addr.base_kind == NATIVE_ADDR_BASE_FRAME_VALUE) {
     NativeLoc base = native_loc_reg(addr.base_type, NATIVE_REG_INT, RV_TMP1);
     NativeAddr load;
     memset(&load, 0, sizeof load);
     load.base_kind = NATIVE_ADDR_BASE_FRAME;
     load.base.frame = addr.base.frame;
     load.base_type = addr.base_type;
     rv_emit_mem(a, 1, base, load,
                 native_mem_for_type(d->native, addr.base_type, 8));
     addr.base_kind = NATIVE_ADDR_BASE_REG;
     addr.base.reg = RV_TMP1;
   }
   return addr;
 }
 
 static void rv_direct_load_operand_to_reg(NativeDirectTarget* d, Operand op,
                                           NativeLoc dst) {
   RvNativeTarget* a = rv_of(d->native);
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   switch ((OpKind)op.kind) {
     case OPK_IMM:
       if ((NativeAllocClass)dst.cls != NATIVE_REG_INT)
         rv_asm_panic(d, "floating-point immediate asm input is unsupported");
       d->native->load_imm(d->native, dst, op.v.imm);
       return;
     case OPK_LOCAL:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = d->locals[op.v.local - 1u].home;
       addr.base_type = op.type;
       rv_emit_mem(a, 1, dst, addr, native_mem_for_type(d->native, op.type, 0));
       return;
     case OPK_GLOBAL:
       addr.base_kind = NATIVE_ADDR_BASE_GLOBAL;
       addr.base.global.sym = op.v.global.sym;
       addr.base.global.addend = op.v.global.addend;
       addr.base_type = op.type;
       d->native->load_addr(d->native, dst, addr);
       return;
     case OPK_INDIRECT:
       addr = rv_direct_materialize_addr(d, op);
       rv_emit_mem(a, 1, dst, addr, native_mem_for_type(d->native, op.type, 0));
       return;
   }
   rv_asm_panic(d, "unsupported asm input operand");
 }
 
 static void rv_direct_load_address_to_reg(NativeDirectTarget* d, Operand op,
                                           NativeLoc dst) {
   d->native->load_addr(d->native, dst, rv_direct_addr(d, op));
 }
 
 static void rv_direct_store_reg_to_operand(NativeDirectTarget* d, Operand op,
                                            NativeLoc src) {
   RvNativeTarget* a = rv_of(d->native);
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   if (op.kind == OPK_LOCAL) {
     addr.base_kind = NATIVE_ADDR_BASE_FRAME;
     addr.base.frame = d->locals[op.v.local - 1u].home;
     addr.base_type = op.type;
   } else {
     addr = rv_direct_materialize_addr(d, op);
   }
   rv_emit_mem(a, 0, src, addr, native_mem_for_type(d->native, op.type, 0));
 }
 
 /* Callee-saved registers an asm block clobbers must be spilled/restored around
  * the block (the only ABI duty the allocator cannot discharge itself). */
 typedef struct RvAsmSavedClobber {
   NativeFrameSlot slot;
   NativeAllocClass cls;
   Reg reg;
   KitCgTypeId type;
 } RvAsmSavedClobber;
 
 /* A clobber save slot is register-width: ptr_bytes for an integer reg (4 on
  * rv32, 8 on rv64) but always 8 for an FP reg (fsd, even on rv32d). */
 static u32 rv_asm_save_bytes(const RvNativeTarget* a, const RvAsmSavedClobber* s) {
   return s->cls == NATIVE_REG_FP ? 8u : a->variant->ptr_bytes;
 }
 static void rv_asm_save_one(RvNativeTarget* a, RvAsmSavedClobber* s) {
   NativeFrameSlotDesc desc;
   NativeAddr addr;
   u32 sz = rv_asm_save_bytes(a, s);
   memset(&desc, 0, sizeof desc);
   desc.type = s->type;
   desc.size = sz;
   desc.align = sz;
   desc.kind = NATIVE_FRAME_SLOT_SAVE;
   s->slot = a->base.frame_slot(&a->base, &desc);
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = s->slot;
   addr.base_type = s->type;
   rv_emit_mem(a, 0, native_loc_reg(s->type, s->cls, s->reg), addr,
               native_mem_for_type(&a->base, s->type, sz));
 }
 static void rv_asm_restore_one(RvNativeTarget* a, const RvAsmSavedClobber* s) {
   NativeAddr addr;
   u32 sz = rv_asm_save_bytes(a, s);
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_FRAME;
   addr.base.frame = s->slot;
   addr.base_type = s->type;
   rv_emit_mem(a, 1, native_loc_reg(s->type, s->cls, s->reg), addr,
               native_mem_for_type(&a->base, s->type, sz));
 }
 
 /* psABI callee-saved: integer s0..s11 (x8,x9,x18..x27), fp fs0..fs11
  * (f8,f9,f18..f27). x8 is the frame pointer and never asm-clobbered. */
 static int rv_reg_is_callee_int(Reg r) {
   return r == 9u || (r >= 18u && r <= 27u);
 }
 static int rv_reg_is_callee_fp(Reg r) {
   return r == 8u || r == 9u || (r >= 18u && r <= 27u);
 }
 
 static RvAsmSavedClobber* rv_asm_save_callee_clobbers(RvNativeTarget* a,
                                                       u32 int_mask, u32 fp_mask,
                                                       u32* nsaved_out) {
   RvAsmSavedClobber* saved =
       arena_zarray(a->base.c->tu, RvAsmSavedClobber, 24u);
   KitCgTypeId i64 = builtin_id(KIT_CG_BUILTIN_I64);
   KitCgTypeId f64 = builtin_id(KIT_CG_BUILTIN_F64);
   u32 n = 0;
   Reg r;
   for (r = 0; r <= 31u; ++r) {
     if ((int_mask & (1u << r)) == 0 || !rv_reg_is_callee_int(r)) continue;
     saved[n].cls = NATIVE_REG_INT;
     saved[n].reg = r;
     saved[n].type = i64;
     rv_asm_save_one(a, &saved[n++]);
   }
   for (r = 0; r <= 31u; ++r) {
     if ((fp_mask & (1u << r)) == 0 || !rv_reg_is_callee_fp(r)) continue;
     saved[n].cls = NATIVE_REG_FP;
     saved[n].reg = r;
     saved[n].type = f64;
     rv_asm_save_one(a, &saved[n++]);
   }
   *nsaved_out = n;
   return saved;
 }
 
 /* ---- NativeTarget (optimizer) asm hook ----
  * The optimizer pre-allocated every operand register and arranged surrounding
  * data flow, so this binds pre-allocated registers to the template and only
  * materializes memory-operand bases into the reserved scratch + spills the
  * callee-saved registers the asm clobbers. */
 
 static NativeAddr rv_asm_loc_to_addr(RvNativeTarget* a, SrcLoc loc,
                                      NativeLoc src) {
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   addr.base_type = src.type;
   switch ((NativeLocKind)src.kind) {
     case NATIVE_LOC_FRAME:
       addr.base_kind = NATIVE_ADDR_BASE_FRAME;
       addr.base.frame = src.v.frame;
       return addr;
     case NATIVE_LOC_ADDR:
       return src.v.addr;
     case NATIVE_LOC_GLOBAL:
       addr.base_kind = NATIVE_ADDR_BASE_GLOBAL;
       addr.base.global.sym = src.v.global.sym;
       addr.base.global.addend = src.v.global.addend;
       return addr;
     case NATIVE_LOC_REG:
       addr.base_kind = NATIVE_ADDR_BASE_REG;
       addr.cls = NATIVE_REG_INT;
       addr.base.reg = src.v.reg;
       return addr;
     default:
       rv_asm_panic_at(a->base.c, loc, "unsupported memory asm operand");
   }
 }
 
 /* Resolve a memory-constraint operand to a single base register with zero
  * offset, folding any frame/global/offset into a reserved scratch register. */
 static Reg rv_asm_native_mem_base(RvNativeTarget* a, SrcLoc loc, NativeLoc src,
                                   u32* ntmp) {
   NativeAddr addr = rv_asm_loc_to_addr(a, loc, src);
   u32 base;
   i32 off;
   Reg dst;
   if (addr.index_kind != NATIVE_ADDR_INDEX_NONE)
     rv_asm_panic_at(a->base.c, loc, "indexed memory asm operand unsupported");
   rv_resolve_mem_addr(a, &addr, &base, &off);
   if (off == 0 && base != RV_TMP0 && base != RV_TMP1) return (Reg)base;
   if (*ntmp >= 2u)
     rv_asm_panic_at(a->base.c, loc, "too many memory asm operands");
   dst = (*ntmp == 0u) ? RV_TMP0 : RV_TMP1;
   (*ntmp)++;
   rv_emit_addr_adjust(a->variant, a->base.mc, dst, base, off);
   return dst;
 }
 
 static u32 rv_asm_reg_mem_size(RvNativeTarget* a, NativeAllocClass cls,
                                KitCgTypeId type) {
   u32 sz = native_type_size(&a->base, type);
   if (cls == NATIVE_REG_INT && sz > a->variant->ptr_bytes)
     sz = a->variant->ptr_bytes;
   return sz;
 }
 
 static Reg rv_asm_stage_reg(RvNativeTarget* a, SrcLoc loc, NativeAllocClass cls,
                             u32* nint, u32* nfp) {
   static const Reg int_regs[] = {RV_TMP2, RV_TMP3};
   static const Reg fp_regs[] = {RV_FTMP0, RV_FTMP1};
   if (cls == NATIVE_REG_FP) {
     if (*nfp >= (u32)(sizeof fp_regs / sizeof fp_regs[0]))
       rv_asm_panic_at(a->base.c, loc, "too many staged fp asm operands");
     return fp_regs[(*nfp)++];
   }
   if (*nint >= (u32)(sizeof int_regs / sizeof int_regs[0]))
     rv_asm_panic_at(a->base.c, loc, "too many staged integer asm operands");
   return int_regs[(*nint)++];
 }
 
 static void rv_asm_load_loc_to_reg(RvNativeTarget* a, SrcLoc loc, NativeLoc src,
                                    NativeLoc dst) {
   NativeTarget* t = &a->base;
   NativeAllocClass cls = (NativeAllocClass)dst.cls;
   if (src.kind == NATIVE_LOC_REG) {
     if (src.v.reg != dst.v.reg || src.cls != dst.cls) t->move(t, dst, src);
     return;
   }
   if (src.kind == NATIVE_LOC_IMM) {
     if (cls != NATIVE_REG_INT)
       rv_asm_panic_at(t->c, loc,
                       "floating-point immediate asm input is unsupported");
     t->load_imm(t, dst, src.v.imm);
     return;
   }
   rv_emit_mem(a, 1, dst, rv_asm_loc_to_addr(a, loc, src),
               native_mem_for_type(t, dst.type,
                                   rv_asm_reg_mem_size(a, cls, dst.type)));
 }
 
 static void rv_asm_store_reg_to_loc(RvNativeTarget* a, SrcLoc loc, NativeLoc dst,
                                     NativeLoc src) {
   NativeTarget* t = &a->base;
   NativeAllocClass cls = (NativeAllocClass)src.cls;
   if (dst.kind == NATIVE_LOC_REG) {
     if (dst.v.reg != src.v.reg || dst.cls != src.cls) t->move(t, dst, src);
     return;
   }
   rv_emit_mem(a, 0, src, rv_asm_loc_to_addr(a, loc, dst),
               native_mem_for_type(t, src.type,
                                   rv_asm_reg_mem_size(a, cls, src.type)));
 }
 
 static void rv_asm_bind_native(RvNativeTarget* a, SrcLoc loc, Operand* out,
                                const char* constraint, KitCgTypeId type,
                                NativeLoc src, u32* ntmp) {
   const char* body = native_asm_constraint_body(constraint);
   NativeAsmConstraintInfo info;
   if (native_asm_constraint_reg_info(&a->base, constraint, &info)) {
     if (src.kind != NATIVE_LOC_REG)
       rv_asm_panic_at(a->base.c, loc, "register asm operand not in a register");
     if (info.fixed_reg != REG_NONE && info.fixed_reg != (Reg)src.v.reg)
       rv_asm_panic_at(a->base.c, loc,
                       "fixed-register asm operand in wrong register");
     if (info.allowed_mask &&
         ((Reg)src.v.reg >= 32 ||
          (info.allowed_mask & (1u << (Reg)src.v.reg)) == 0))
       rv_asm_panic_at(a->base.c, loc,
                       "register asm operand violates constraint register set");
     rv_asm_bound_reg(out, type, info.cls, (Reg)src.v.reg);
   } else if (body[0] == 'i') {
     if (src.kind != NATIVE_LOC_IMM)
       rv_asm_panic_at(a->base.c, loc, "immediate asm operand is not immediate");
     memset(out, 0, sizeof *out);
     out->kind = OPK_IMM;
     out->type = type;
     out->v.imm = src.v.imm;
   } else if (body[0] == 'm') {
     rv_asm_bound_mem(out, type, rv_asm_native_mem_base(a, loc, src, ntmp));
   } else {
     rv_asm_panic_at(a->base.c, loc, "unsupported asm constraint");
   }
 }
 
 static void rv_asm_block_native(NativeTarget* t, const char* tmpl,
                                 const AsmConstraint* outs, u32 nout,
                                 NativeLoc* out_locs, const AsmConstraint* ins,
                                 u32 nin, const NativeLoc* in_locs,
                                 const Sym* clobbers, u32 nclob) {
   RvNativeTarget* a = rv_of(t);
   Compiler* c = t->c;
   SrcLoc loc = a->func ? a->func->loc : (SrcLoc){0, 0, 0};
   Operand* bound_outs = nout ? arena_zarray(c->tu, Operand, nout) : NULL;
   Operand* bound_ins = nin ? arena_zarray(c->tu, Operand, nin) : NULL;
   u8* staged_outs = nout ? arena_zarray(c->tu, u8, nout) : NULL;
   u32 ntmp = 0, nstage_int = 0, nstage_fp = 0, i;
   Rv64Asm* asmh;
 
   for (i = 0; i < nout; ++i) {
     KitCgTypeId type = outs[i].type ? outs[i].type : out_locs[i].type;
     NativeLoc outloc = out_locs[i];
     NativeAsmConstraintInfo info;
     NativeAsmPinnedLoc pinned =
         native_asm_prepare_pinned_loc(t, outs[i].reg, outs[i].str, type, outloc);
     if (pinned.has_pin) {
       if (pinned.pin_status != NATIVE_ASM_REG_PIN_OK)
         rv_asm_panic_at(c, loc,
                         native_asm_pin_status_message(pinned.pin_status));
       if (pinned.wrong_reg)
         rv_asm_panic_at(c, loc, "hard-register asm operand in wrong register");
       outloc = pinned.loc;
       if (pinned.needs_stage) {
         staged_outs[i] = 1u;
         if (outs[i].dir == KIT_CG_ASM_INOUT)
           rv_asm_load_loc_to_reg(a, loc, out_locs[i], outloc);
       }
     } else if (native_asm_constraint_reg_info(t, outs[i].str, &info) &&
                info.allowed_mask == 0 && outloc.kind != NATIVE_LOC_REG) {
       Reg r = rv_asm_stage_reg(a, loc, info.cls, &nstage_int, &nstage_fp);
       outloc = native_loc_reg(type, info.cls, r);
       staged_outs[i] = 1u;
       if (outs[i].dir == KIT_CG_ASM_INOUT)
         rv_asm_load_loc_to_reg(a, loc, out_locs[i], outloc);
     }
     rv_asm_bind_native(a, loc, &bound_outs[i], outs[i].str, type, outloc, &ntmp);
   }
   for (i = 0; i < nin; ++i) {
     const char* body = native_asm_constraint_body(ins[i].str);
     int matched = native_asm_match_index(body);
     KitCgTypeId type;
     NativeLoc inloc;
     if (matched >= 0) {
       if ((u32)matched >= nout)
         rv_asm_panic_at(c, loc, "matching constraint out of range");
       bound_ins[i] = bound_outs[matched];
       continue;
     }
     type = ins[i].type ? ins[i].type : in_locs[i].type;
     inloc = in_locs[i];
     {
       NativeAsmConstraintInfo info;
       NativeAsmPinnedLoc pinned =
           native_asm_prepare_pinned_loc(t, ins[i].reg, ins[i].str, type, inloc);
       if (pinned.has_pin) {
         if (pinned.pin_status != NATIVE_ASM_REG_PIN_OK)
           rv_asm_panic_at(c, loc,
                           native_asm_pin_status_message(pinned.pin_status));
         if (pinned.wrong_reg)
           rv_asm_panic_at(c, loc,
                           "hard-register asm operand in wrong register");
         inloc = pinned.loc;
         if (pinned.needs_stage)
           rv_asm_load_loc_to_reg(a, loc, in_locs[i], inloc);
       } else if (native_asm_constraint_reg_info(t, ins[i].str, &info) &&
                  info.allowed_mask == 0 && inloc.kind != NATIVE_LOC_REG) {
         Reg r = rv_asm_stage_reg(a, loc, info.cls, &nstage_int, &nstage_fp);
         inloc = native_loc_reg(type, info.cls, r);
         rv_asm_load_loc_to_reg(a, loc, in_locs[i], inloc);
       }
     }
     rv_asm_bind_native(a, loc, &bound_ins[i], ins[i].str, type, inloc, &ntmp);
   }
 
   /* No per-block callee-saved spill here: plan_frame forwarded the asm clobber
    * masks and rv_known_callee_saves folded the callee-saved ones into the
    * function's saved set, so the prologue/epilogue already preserve them. */
   asmh = rv64_asm_open(c);
   rv64_inline_bind(asmh, outs, nout, bound_outs, ins, nin, bound_ins, clobbers,
                    nclob);
   rv64_asm_run_template(asmh, t->mc, tmpl);
   rv64_asm_close(asmh);
 
   for (i = 0; i < nout; ++i) {
     NativeAllocClass cls;
     NativeLoc src;
     if (!staged_outs || !staged_outs[i]) continue;
     if (bound_outs[i].kind != RV64_INLINE_OPK_REG) continue;
     cls = bound_outs[i].pad[0] == RV64_INLINE_OPCLS_FP ? NATIVE_REG_FP
                                                        : NATIVE_REG_INT;
     src = native_loc_reg(bound_outs[i].type, cls, (Reg)bound_outs[i].v.local);
     rv_asm_store_reg_to_loc(a, loc, out_locs[i], src);
   }
 }
 /* file_scope_asm + finalize are shared (cg/native_asm.h). */
 
 static void rv_trap(NativeTarget* t) { rv64_emit32(t->mc, rv_ebreak()); }
 static void rv_set_loc(NativeTarget* t, SrcLoc loc) {
   rv_of(t)->loc = loc;
   if (t->mc->set_loc) t->mc->set_loc(t->mc, loc);
 }
 
 /* ============================ construction ============================ */
 
 NativeTarget* rv64_native_target_new(Compiler* c, ObjBuilder* obj,
                                      MCEmitter* mc) {
   RvNativeTarget* a = arena_znew(c->tu, RvNativeTarget);
   NativeTarget* t;
   if (!a) return NULL;
   t = &a->base;
   t->c = c;
   t->obj = obj;
   t->mc = mc;
   a->variant = riscv_variant_for_kind(c->target.arch);
   native_frame_init(&a->frame, c);
   t->regs = &rv_reg_info;
   t->class_for_type = native_class_for_type_fp_le8;
   t->imm_legal = rv_imm_legal;
   t->addr_legal = rv_addr_legal;
   t->func_begin = rv_func_begin;
   t->func_begin_known_frame = rv_func_begin_known_frame;
   t->note_frame_state = NULL;
   /* Non-NULL so the optimizer emit path (plan_frame) computes the callee-saved
    * set; rv_func_begin_known_frame derives the records from the masks. */
   t->reserve_callee_saves = rv_reserve_callee_saves;
   t->signature_stack_bytes = rv_signature_stack_bytes;
   t->call_stack_bytes = rv_call_stack_bytes;
   t->has_store_zero_reg = 1;
   t->store_zero_reg = RV_ZERO;
   t->func_end = rv_func_end;
   t->frame_slot = rv_frame_slot;
   t->frame_slot_debug_loc = rv_frame_slot_debug_loc;
   t->bind_param = rv_bind_native_param;
   t->label_new = rv_label_new;
   t->label_place = rv_label_place;
   t->jump = rv_jump;
   t->cmp_branch = rv_cmp_branch;
   t->indirect_branch = rv_indirect_branch;
   t->load_label_addr = rv_load_label_addr;
   t->move = rv_move;
   t->load_imm = rv_load_imm;
   t->load_const = rv_load_const;
   t->load_addr = rv_load_addr;
   t->load = rv_load;
   t->store = rv_store;
   t->tls_addr_of = rv_tls_addr_of;
   t->copy_bytes = rv_copy_bytes;
   t->set_bytes = rv_set_bytes;
   t->bitfield_load = rv_bitfield_load;
   t->bitfield_store = rv_bitfield_store;
   t->binop = rv_binop;
   t->unop = rv_unop;
   t->cmp = rv_cmp;
   t->convert = rv_convert;
   t->alloca_ = rv_alloca;
   t->spill = rv_spill;
   t->reload = rv_reload;
   t->plan_call = rv_plan_call;
   t->emit_call = rv_emit_call;
   t->plan_ret = rv_plan_ret;
   t->ret = rv_ret;
   t->atomic_load = rv_atomic_load;
   t->atomic_store = rv_atomic_store;
   t->atomic_rmw = rv_atomic_rmw;
   t->atomic_cas = rv_atomic_cas;
   t->fence = rv_fence;
   t->va_start_ = rv_va_start_native;
   t->va_arg_ = rv_va_arg_native;
   t->va_end_ = rv_va_end_native;
   t->va_copy_ = rv_va_copy_native;
   t->intrinsic = rv_intrinsic;
   t->asm_block = rv_asm_block_native;
   t->file_scope_asm = native_file_scope_asm;
   t->trap = rv_trap;
   t->set_loc = rv_set_loc;
   t->finalize = native_finalize;
   return t;
 }
 
 /* ============================ NativeOps (-O0) ============================ */
 
 static void rv_bind_param(NativeDirectTarget* d, const CGParamDesc* p,
                           CGLocal local, NativeDirectLocal* l) {
   NativeLoc dst;
   (void)local;
   memset(&dst, 0, sizeof dst);
   dst.kind = NATIVE_LOC_FRAME;
   dst.type = p->type;
   dst.v.frame = l->home;
   rv_bind_native_param(d->native, p, dst);
 }
 
 /* A sibling call is realizable when its outgoing stack-argument area fits the
  * window the caller itself received (so the args land in the caller's incoming
  * slots without overflowing into the caller's caller's frame). Register-only
  * calls (the common case) always qualify. Mirrors aa64's aa_no_tail. */
 static const char* rv_no_tail(NativeDirectTarget* d, const CGCallDesc* call) {
   RvNativeTarget* a = rv_of(d->native);
   NativeCallDesc nd;
   NativeLoc* args = NULL;
   NativeLoc* results = NULL;
   u32 i, stack;
   if (a->frame.ncallee_saves)
     return "rv64 tail call: callee-saved registers in use";
   memset(&nd, 0, sizeof nd);
   u32 nresults = call->result != CG_LOCAL_NONE ? 1u : 0u;
   if (call->nargs) args = arena_zarray(d->base.c->tu, NativeLoc, call->nargs);
   if (nresults) results = arena_zarray(d->base.c->tu, NativeLoc, nresults);
   for (i = 0; i < call->nargs; ++i) {
     args[i].kind = NATIVE_LOC_FRAME;
     args[i].type = d->locals[call->args[i] - 1u].type;
     args[i].cls = d->locals[call->args[i] - 1u].cls;
     args[i].v.frame = d->locals[call->args[i] - 1u].home;
   }
   if (nresults) {
     results[0].kind = NATIVE_LOC_FRAME;
     results[0].type = d->locals[call->result - 1u].type;
     results[0].cls = d->locals[call->result - 1u].cls;
     results[0].v.frame = d->locals[call->result - 1u].home;
   }
   nd.fn_type = call->fn_type;
   nd.args = args;
   nd.results = results;
   nd.nargs = call->nargs;
   nd.nresults = nresults;
   stack = rv_call_stack_size(d->native, &nd);
   if (stack > a->incoming_stack_size)
     return "rv64 tail call: stack argument area too small";
   return NULL;
 }
 
 /* Resolve a pointer-typed Operand (the address of a va_list object) into `reg`
  * and return a register-based NativeAddr. An OPK_LOCAL holds the va_list object
  * itself, so we take its frame address; an OPK_INDIRECT holds the pointer in
  * memory and must be loaded. The va cores use TMP1/TMP2 internally, so `reg`
  * must be distinct from those (callers pass TMP0 / TMP3). */
 /* ap_addr is the pointer value &ap (the va_list object's address). For an
  * OPK_LOCAL the local HOLDS that pointer, so load its home value; an
  * OPK_INDIRECT names *(base+ofs), whose address base+ofs is the pointer.
  * Mirrors aa64's aa_direct_pointer_addr. */
 static NativeAddr rv_direct_pointer_addr(NativeDirectTarget* d, Operand op) {
   RvNativeTarget* a = rv_of(d->native);
   NativeAddr addr;
   memset(&addr, 0, sizeof addr);
   if (op.kind == OPK_LOCAL) {
     NativeLoc base = native_loc_reg(op.type, NATIVE_REG_INT, RV_TMP1);
     NativeAddr load;
     memset(&load, 0, sizeof load);
     load.base_kind = NATIVE_ADDR_BASE_FRAME;
     load.base.frame = d->locals[op.v.local - 1u].home;
     load.base_type = op.type;
     rv_emit_mem(a, 1, base, load, native_mem_for_type(d->native, op.type, 8));
     addr.base_kind = NATIVE_ADDR_BASE_REG;
     addr.base.reg = RV_TMP1;
     addr.base_type = op.type;
     return addr;
   }
   return rv_direct_materialize_addr(d, op);
 }
 
 static NativeAddr rv_direct_va_base(NativeDirectTarget* d, Operand ap_addr,
                                     Reg reg) {
   NativeLoc dst =
       native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT, reg);
   NativeAddr addr;
   d->native->load_addr(d->native, dst, rv_direct_pointer_addr(d, ap_addr));
   memset(&addr, 0, sizeof addr);
   addr.base_kind = NATIVE_ADDR_BASE_REG;
   addr.cls = NATIVE_REG_INT;
   addr.base.reg = reg;
   addr.base_type = builtin_id(KIT_CG_BUILTIN_I64);
   return addr;
 }
 
 static void rv_va_start_(NativeDirectTarget* d, Operand ap_addr) {
   rv_va_start_core(rv_of(d->native), rv_direct_va_base(d, ap_addr, RV_TMP3));
 }
 static void rv_va_arg_(NativeDirectTarget* d, Operand dst, Operand ap_addr,
                        KitCgTypeId type) {
   RvNativeTarget* a = rv_of(d->native);
   NativeAllocClass cls;
   /* A value too wide for one GPR (8-byte i64 / soft-double on ilp32) is copied
    * straight from the save area into its destination memory. */
   if (rv_va_arg_is_wide(d->native, type)) {
     rv_va_arg_wide(a, rv_direct_addr(d, dst),
                    rv_direct_va_base(d, ap_addr, RV_TMP3),
                    native_type_size(d->native, type));
     return;
   }
   /* Float-ABI-aware class: a soft (or wider-than-flen) float is INT-class so
    * the va_arg fetch never lands a double in an FP register on rv32. */
   cls = native_class_for_type_fp_le8(d->native, type);
   NativeLoc res = native_loc_reg(type, cls,
                              cls == NATIVE_REG_FP ? RV_FTMP0 : RV_TMP0);
   NativeAddr dst_addr;
   rv_va_arg_core(a, res, rv_direct_va_base(d, ap_addr, RV_TMP3), type);
   /* Store the fetched value back into the semantic destination. */
   dst_addr = rv_direct_addr(d, dst);
   if (dst_addr.base_kind == NATIVE_ADDR_BASE_FRAME_VALUE) {
     NativeLoc base =
         native_loc_reg(dst_addr.base_type, NATIVE_REG_INT, RV_TMP1);
     NativeAddr load;
     memset(&load, 0, sizeof load);
     load.base_kind = NATIVE_ADDR_BASE_FRAME;
     load.base.frame = dst_addr.base.frame;
     load.base_type = dst_addr.base_type;
     rv_emit_mem(a, 1, base, load,
                 native_mem_for_type(d->native, dst_addr.base_type, 8));
     dst_addr.base_kind = NATIVE_ADDR_BASE_REG;
     dst_addr.base.reg = RV_TMP1;
   }
   rv_emit_mem(
       a, 0, res, dst_addr,
       native_mem_for_type(d->native, type, native_type_size(d->native, type)));
 }
 static void rv_va_end_(NativeDirectTarget* d, Operand ap_addr) {
   (void)d;
   (void)ap_addr;
 }
 static void rv_va_copy_(NativeDirectTarget* d, Operand dst, Operand src) {
   RvNativeTarget* a = rv_of(d->native);
   NativeAddr src_ap = rv_direct_va_base(d, src, RV_TMP0);
   NativeAddr dst_ap = rv_direct_va_base(d, dst, RV_TMP3);
   rv_va_copy_core(a, dst_ap, src_ap);
 }
 
 static void rv_direct_asm_block(NativeDirectTarget* d, const char* tmpl,
                                 const AsmConstraint* outs, u32 nout,
                                 Operand* out_ops, const AsmConstraint* ins,
                                 u32 nin, const Operand* in_ops,
                                 const Sym* clobbers, u32 nclob,
                                 u32 clobber_abi_sets) {
   RvNativeTarget* a = rv_of(d->native);
   Compiler* c = d->base.c;
   Operand* bound_outs = nout ? arena_zarray(c->tu, Operand, nout) : NULL;
   Operand* bound_ins = nin ? arena_zarray(c->tu, Operand, nin) : NULL;
   u32 clob_int, clob_fp, abi_int, abi_fp, used_int, used_fp;
   RvAsmSavedClobber* saved;
   u32 nsaved, i;
   Rv64Asm* asmh;
 
   rv_asm_clobber_masks(c, d->loc, clobbers, nclob, &clob_int, &clob_fp);
   native_asm_abi_clobber_masks(d->native, clobber_abi_sets, &abi_int, &abi_fp);
   clob_int |= abi_int;
   clob_fp |= abi_fp;
   /* Reserve emit scratch (t0/t1/t2/t3), sp/gp/tp/zero/ra and the frame pointer
    * so the operand allocator never hands them out. */
   used_int = clob_int | (1u << RV_ZERO) | (1u << RV_RA) | (1u << RV_SP) |
              (1u << RV_GP) | (1u << RV_TP) | (1u << RV_TMP0) | (1u << RV_TMP1) |
              (1u << RV_TMP2) | (1u << RV_TMP3) | (1u << RV_S0);
   used_fp =
       clob_fp | (1u << RV_FTMP0) | (1u << RV_FTMP1) | (1u << 2u) | (1u << 3u);
 
   for (i = 0; i < nout; ++i) {
     const char* body = native_asm_constraint_body(outs[i].str);
     KitCgTypeId type = outs[i].type ? outs[i].type : out_ops[i].type;
     NativeAsmRegPin pin;
     if (rv_asm_resolve_pin_or_panic(d, outs[i].reg, outs[i].str, &pin)) {
       /* GNU local register variable: pin to the named hard register. */
       if (pin.cls == NATIVE_REG_FP) {
         used_fp |= 1u << pin.reg;
         clob_fp |= 1u << pin.reg;
       } else {
         used_int |= 1u << pin.reg;
         clob_int |= 1u << pin.reg;
       }
       rv_asm_bound_reg(&bound_outs[i], type, pin.cls, pin.reg);
     } else {
       NativeAsmConstraintInfo info;
       if (native_asm_constraint_reg_info(d->native, outs[i].str, &info)) {
         Reg reg = info.fixed_reg != REG_NONE
                       ? info.fixed_reg
                       : rv_asm_alloc_reg(d, info.cls, info.allowed_mask,
                                          &used_int, &used_fp);
         if (info.cls == NATIVE_REG_FP) {
           used_fp |= 1u << reg;
           if (info.fixed_reg != REG_NONE) clob_fp |= 1u << reg;
         } else {
           used_int |= 1u << reg;
           if (info.fixed_reg != REG_NONE) clob_int |= 1u << reg;
         }
         rv_asm_bound_reg(&bound_outs[i], type, info.cls, reg);
       } else if (body[0] == 'm') {
         Reg reg = rv_asm_alloc_reg(d, NATIVE_REG_INT, 0, &used_int, &used_fp);
         rv_asm_bound_mem(&bound_outs[i], type, reg);
       } else {
         rv_asm_panic(d, "unsupported output constraint");
       }
     }
   }
 
   for (i = 0; i < nin; ++i) {
     const char* body = native_asm_constraint_body(ins[i].str);
     int matched = native_asm_match_index(body);
     KitCgTypeId type = ins[i].type ? ins[i].type : in_ops[i].type;
     if (matched >= 0) {
       if ((u32)matched >= nout)
         rv_asm_panic(d, "matching constraint out of range");
       if (native_asm_constraint_early(outs[matched].str))
         rv_asm_panic(d, "matching input names early-clobber output");
       if (bound_outs[matched].kind != RV64_INLINE_OPK_REG)
         rv_asm_panic(d, "matching constraint requires register output");
       bound_ins[i] = bound_outs[matched];
       continue;
     }
     NativeAsmRegPin pin;
     if (rv_asm_resolve_pin_or_panic(d, ins[i].reg, ins[i].str, &pin)) {
       /* GNU local register variable: pin to the named hard register. */
       if (pin.cls == NATIVE_REG_FP) {
         used_fp |= 1u << pin.reg;
         clob_fp |= 1u << pin.reg;
       } else {
         used_int |= 1u << pin.reg;
         clob_int |= 1u << pin.reg;
       }
       rv_asm_bound_reg(&bound_ins[i], type, pin.cls, pin.reg);
     } else {
       NativeAsmConstraintInfo info;
       if (native_asm_constraint_reg_info(d->native, ins[i].str, &info)) {
         Reg reg = info.fixed_reg != REG_NONE
                       ? info.fixed_reg
                       : rv_asm_alloc_reg(d, info.cls, info.allowed_mask,
                                          &used_int, &used_fp);
         if (info.cls == NATIVE_REG_FP) {
           used_fp |= 1u << reg;
           if (info.fixed_reg != REG_NONE) clob_fp |= 1u << reg;
         } else {
           used_int |= 1u << reg;
           if (info.fixed_reg != REG_NONE) clob_int |= 1u << reg;
         }
         rv_asm_bound_reg(&bound_ins[i], type, info.cls, reg);
       } else if (body[0] == 'i') {
         if (in_ops[i].kind != OPK_IMM)
           rv_asm_panic(d, "immediate constraint requires immediate operand");
         bound_ins[i] = in_ops[i];
       } else if (body[0] == 'm') {
         Reg reg = rv_asm_alloc_reg(d, NATIVE_REG_INT, 0, &used_int, &used_fp);
         rv_asm_bound_mem(&bound_ins[i], type, reg);
       } else {
         rv_asm_panic(d, "unsupported input constraint");
       }
     }
   }
 
   saved = rv_asm_save_callee_clobbers(a, clob_int, clob_fp, &nsaved);
   for (i = 0; i < nout; ++i) {
     if (bound_outs[i].kind == RV64_INLINE_OPK_REG) {
       NativeAllocClass cls = bound_outs[i].pad[0] == RV64_INLINE_OPCLS_FP
                                  ? NATIVE_REG_FP
                                  : NATIVE_REG_INT;
       if (outs[i].dir == KIT_CG_ASM_INOUT) {
         rv_direct_load_operand_to_reg(
             d, out_ops[i],
             native_loc_reg(bound_outs[i].type, cls,
                            (Reg)bound_outs[i].v.local));
       }
     } else if (bound_outs[i].kind == OPK_INDIRECT) {
       NativeLoc loc =
           native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT,
                          (Reg)bound_outs[i].v.ind.base);
       rv_direct_load_address_to_reg(d, out_ops[i], loc);
     }
   }
   for (i = 0; i < nin; ++i) {
     if (bound_ins[i].kind == RV64_INLINE_OPK_REG) {
       NativeAllocClass cls = bound_ins[i].pad[0] == RV64_INLINE_OPCLS_FP
                                  ? NATIVE_REG_FP
                                  : NATIVE_REG_INT;
       rv_direct_load_operand_to_reg(
           d, in_ops[i],
           native_loc_reg(bound_ins[i].type, cls, (Reg)bound_ins[i].v.local));
     } else if (bound_ins[i].kind == OPK_INDIRECT) {
       NativeLoc loc =
           native_loc_reg(builtin_id(KIT_CG_BUILTIN_I64), NATIVE_REG_INT,
                          (Reg)bound_ins[i].v.ind.base);
       rv_direct_load_address_to_reg(d, in_ops[i], loc);
     }
   }
   asmh = rv64_asm_open(c);
   rv64_inline_bind(asmh, outs, nout, bound_outs, ins, nin, bound_ins, clobbers,
                    nclob);
   rv64_asm_run_template(asmh, d->native->mc, tmpl);
   rv64_asm_close(asmh);
 
   for (i = 0; i < nout; ++i) {
     NativeAllocClass cls;
     NativeLoc src;
     if (bound_outs[i].kind != RV64_INLINE_OPK_REG) continue;
     cls = bound_outs[i].pad[0] == RV64_INLINE_OPCLS_FP ? NATIVE_REG_FP
                                                        : NATIVE_REG_INT;
     src = native_loc_reg(bound_outs[i].type, cls, (Reg)bound_outs[i].v.local);
     rv_direct_store_reg_to_operand(d, out_ops[i], src);
   }
   for (i = nsaved; i > 0; --i) rv_asm_restore_one(a, &saved[i - 1u]);
 }
 
 static const NativeOps rv_direct_ops = {
     .bind_param = rv_bind_param,
     .tail_call_unrealizable_reason = rv_no_tail,
     .va_start_ = rv_va_start_,
     .va_arg_ = rv_va_arg_,
     .va_end_ = rv_va_end_,
     .va_copy_ = rv_va_copy_,
     .asm_block = rv_direct_asm_block,
 };
 
 const NativeOps* rv64_native_direct_ops(void) { return &rv_direct_ops; }